import collections
import sys
from collections.abc import Set
from datetime import datetime, timedelta
import regex as re
from dateutil.relativedelta import relativedelta
from dateparser.date_parser import date_parser
from dateparser.freshness_date_parser import freshness_date_parser
from dateparser.languages.loader import LocaleDataLoader
from dateparser.conf import apply_settings, check_settings
from dateparser.parser import _parse_absolute, _parse_nospaces
from dateparser.timezone_parser import pop_tz_offset_from_string
from dateparser.utils import apply_timezone_from_settings, \
set_correct_day_from_settings
APOSTROPHE_LOOK_ALIKE_CHARS = [
'\N{RIGHT SINGLE QUOTATION MARK}', # '\u2019'
'\N{MODIFIER LETTER APOSTROPHE}', # '\u02bc'
'\N{MODIFIER LETTER TURNED COMMA}', # '\u02bb'
'\N{ARMENIAN APOSTROPHE}', # '\u055a'
'\N{LATIN SMALL LETTER SALTILLO}', # '\ua78c'
'\N{PRIME}', # '\u2032'
'\N{REVERSED PRIME}', # '\u2035'
'\N{MODIFIER LETTER PRIME}', # '\u02b9'
'\N{FULLWIDTH APOSTROPHE}', # '\uff07'
]
RE_NBSP = re.compile('\xa0', flags=re.UNICODE)
RE_SPACES = re.compile(r'\s+')
RE_TRIM_SPACES = re.compile(r'^\s+(\S.*?)\s+$')
RE_TRIM_COLONS = re.compile(r'(\S.*?):*$')
RE_SANITIZE_SKIP = re.compile(r'\t|\n|\r|\u00bb|,\s\u0432|\u200e|\xb7|\u200f|\u064e|\u064f', flags=re.M)
RE_SANITIZE_RUSSIAN = re.compile(r'([\W\d])\u0433\.', flags=re.I | re.U)
RE_SANITIZE_PERIOD = re.compile(r'(?<=\D+)\.', flags=re.U)
RE_SANITIZE_ON = re.compile(r'^.*?on:\s+(.*)')
RE_SANITIZE_APOSTROPHE = re.compile('|'.join(APOSTROPHE_LOOK_ALIKE_CHARS))
RE_SEARCH_TIMESTAMP = re.compile(r'^(\d{10})(\d{3})?(\d{3})?(?![^.])')
[docs]def sanitize_spaces(date_string):
date_string = RE_NBSP.sub(' ', date_string)
date_string = RE_SPACES.sub(' ', date_string)
date_string = RE_TRIM_SPACES.sub(r'\1', date_string)
return date_string
[docs]def date_range(begin, end, **kwargs):
dateutil_error_prone_args = ['year', 'month', 'week', 'day', 'hour',
'minute', 'second']
for arg in dateutil_error_prone_args:
if arg in kwargs:
raise ValueError("Invalid argument: %s" % arg)
step = relativedelta(**kwargs) if kwargs else relativedelta(days=1)
date = begin
while date < end:
yield date
date += step
# handles edge-case when iterating months and last interval is < 30 days
if kwargs.get('months', 0) > 0 and (date.year, date.month) == (end.year, end.month):
yield end
[docs]def get_intersecting_periods(low, high, period='day'):
if period not in ['year', 'month', 'week', 'day', 'hour', 'minute', 'second', 'microsecond']:
raise ValueError("Invalid period: {}".format(period))
if high <= low:
return
step = relativedelta(**{period + 's': 1})
current_period_start = low
if isinstance(current_period_start, datetime):
reset_arguments = {}
for test_period in ['microsecond', 'second', 'minute', 'hour']:
if test_period == period:
break
else:
reset_arguments[test_period] = 0
current_period_start = current_period_start.replace(**reset_arguments)
if period == 'week':
current_period_start \
= current_period_start - timedelta(days=current_period_start.weekday())
elif period == 'month':
current_period_start = current_period_start.replace(day=1)
elif period == 'year':
current_period_start = current_period_start.replace(month=1, day=1)
while current_period_start < high:
yield current_period_start
current_period_start += step
[docs]def sanitize_date(date_string):
date_string = RE_SANITIZE_SKIP.sub(' ', date_string)
date_string = RE_SANITIZE_RUSSIAN.sub(r'\1 ', date_string) # remove 'г.' (Russian for year) but not in words
date_string = sanitize_spaces(date_string)
date_string = RE_SANITIZE_PERIOD.sub('', date_string)
date_string = RE_SANITIZE_ON.sub(r'\1', date_string)
date_string = RE_TRIM_COLONS.sub(r'\1', date_string)
date_string = RE_SANITIZE_APOSTROPHE.sub("'", date_string)
return date_string
[docs]def get_date_from_timestamp(date_string, settings):
match = RE_SEARCH_TIMESTAMP.search(date_string)
if match:
seconds = int(match.group(1))
millis = int(match.group(2) or 0)
micros = int(match.group(3) or 0)
date_obj = datetime.fromtimestamp(seconds)
date_obj = date_obj.replace(microsecond=millis * 1000 + micros)
date_obj = apply_timezone_from_settings(date_obj, settings)
return date_obj
class _DateLocaleParser:
def __init__(self, locale, date_string, date_formats, settings=None):
self._settings = settings
if not (date_formats is None or isinstance(date_formats, (list, tuple, Set))):
raise TypeError("Date formats should be list, tuple or set of strings")
self.locale = locale
self.date_string = date_string
self.date_formats = date_formats
self._translated_date = None
self._translated_date_with_formatting = None
self._parsers = {
'timestamp': self._try_timestamp,
'relative-time': self._try_freshness_parser,
'custom-formats': self._try_given_formats,
'absolute-time': self._try_absolute_parser,
'no-spaces-time': self._try_nospaces_parser,
}
@classmethod
def parse(cls, locale, date_string, date_formats=None, settings=None):
instance = cls(locale, date_string, date_formats, settings)
return instance._parse()
def _parse(self):
for parser_name in self._settings.PARSERS:
date_data = self._parsers[parser_name]()
if self._is_valid_date_data(date_data):
return date_data
else:
return None
def _try_timestamp(self):
return DateData(
date_obj=get_date_from_timestamp(self.date_string, self._settings),
period='day',
)
def _try_freshness_parser(self):
try:
return freshness_date_parser.get_date_data(self._get_translated_date(), self._settings)
except (OverflowError, ValueError):
return None
def _try_absolute_parser(self):
return self._try_parser(parse_method=_parse_absolute)
def _try_nospaces_parser(self):
return self._try_parser(parse_method=_parse_nospaces)
def _try_parser(self, parse_method):
_order = self._settings.DATE_ORDER
try:
if self._settings.PREFER_LOCALE_DATE_ORDER:
if 'DATE_ORDER' not in self._settings._mod_settings:
self._settings.DATE_ORDER = self.locale.info.get('date_order', _order)
date_obj, period = date_parser.parse(
self._get_translated_date(), parse_method=parse_method, settings=self._settings)
self._settings.DATE_ORDER = _order
return DateData(
date_obj=date_obj,
period=period,
)
except ValueError:
self._settings.DATE_ORDER = _order
return None
def _try_given_formats(self):
if not self.date_formats:
return
return parse_with_formats(
self._get_translated_date_with_formatting(),
self.date_formats, settings=self._settings
)
def _get_translated_date(self):
if self._translated_date is None:
self._translated_date = self.locale.translate(
self.date_string, keep_formatting=False, settings=self._settings)
return self._translated_date
def _get_translated_date_with_formatting(self):
if self._translated_date_with_formatting is None:
self._translated_date_with_formatting = self.locale.translate(
self.date_string, keep_formatting=True, settings=self._settings)
return self._translated_date_with_formatting
def _is_valid_date_data(self, date_data):
if not isinstance(date_data, DateData):
return False
if not date_data['date_obj'] or not date_data['period']:
return False
if date_data['date_obj'] and not isinstance(date_data['date_obj'], datetime):
return False
if date_data['period'] not in ('time', 'day', 'week', 'month', 'year'):
return False
return True
[docs]class DateData:
"""
Class that represents the parsed data with useful information.
It can be accessed with square brackets like a dict object.
"""
def __init__(self, *, date_obj=None, period=None, locale=None):
self.date_obj = date_obj
self.period = period
self.locale = locale
def __getitem__(self, k):
if not hasattr(self, k):
raise KeyError(k)
return getattr(self, k)
def __setitem__(self, k, v):
if not hasattr(self, k):
raise KeyError(k)
setattr(self, k, v)
def __repr__(self):
if sys.version_info < (3, 6): # python 3.5 compatibility
properties_text = "date_obj={}, period={}, locale={}".format(
self.date_obj.__repr__(), self.period.__repr__(), self.locale.__repr__()
)
else:
properties_text = ', '.join('{}={}'.format(prop, val.__repr__()) for prop, val in self.__dict__.items())
return '{}({})'.format(
self.__class__.__name__, properties_text
)
[docs]class DateDataParser:
"""
Class which handles language detection, translation and subsequent generic parsing of
string representing date and/or time.
:param languages:
A list of language codes, e.g. ['en', 'es', 'zh-Hant'].
If locales are not given, languages and region are
used to construct locales for translation.
:type languages: list
:param locales:
A list of locale codes, e.g. ['fr-PF', 'qu-EC', 'af-NA'].
The parser uses only these locales to translate date string.
:type locales: list
:param region:
A region code, e.g. 'IN', '001', 'NE'.
If locales are not given, languages and region are
used to construct locales for translation.
:type region: str
:param try_previous_locales:
If True, locales previously used to translate date are tried first.
:type try_previous_locales: bool
:param use_given_order:
If True, locales are tried for translation of date string
in the order in which they are given.
:type use_given_order: bool
:param settings:
Configure customized behavior using settings defined in :mod:`dateparser.conf.Settings`.
:type settings: dict
:return: A parser instance
:raises:
``ValueError``: Unknown Language, ``TypeError``: Languages argument must be a list,
``SettingValidationError``: A provided setting is not valid.
"""
locale_loader = None
@apply_settings
def __init__(self, languages=None, locales=None, region=None, try_previous_locales=False,
use_given_order=False, settings=None):
if languages is not None and not isinstance(languages, (list, tuple, Set)):
raise TypeError("languages argument must be a list (%r given)" % type(languages))
if locales is not None and not isinstance(locales, (list, tuple, Set)):
raise TypeError("locales argument must be a list (%r given)" % type(locales))
if region is not None and not isinstance(region, str):
raise TypeError("region argument must be str (%r given)" % type(region))
if not isinstance(try_previous_locales, bool):
raise TypeError("try_previous_locales argument must be a boolean (%r given)"
% type(try_previous_locales))
if not isinstance(use_given_order, bool):
raise TypeError("use_given_order argument must be a boolean (%r given)"
% type(use_given_order))
if not locales and use_given_order:
raise ValueError("locales must be given if use_given_order is True")
check_settings(settings)
self._settings = settings
self.try_previous_locales = try_previous_locales
self.use_given_order = use_given_order
self.languages = languages
self.locales = locales
self.region = region
self.previous_locales = set()
[docs] def get_date_data(self, date_string, date_formats=None):
"""
Parse string representing date and/or time in recognizable localized formats.
Supports parsing multiple languages and timezones.
:param date_string:
A string representing date and/or time in a recognizably valid format.
:type date_string: str
:param date_formats:
A list of format strings using directives as given
`here <https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_.
The parser applies formats one by one, taking into account the detected languages.
:type date_formats: list
:return: a ``DateData`` object.
:raises: ValueError - Unknown Language
.. note:: *Period* values can be a 'day' (default), 'week', 'month', 'year', 'time'.
*Period* represents the granularity of date parsed from the given string.
In the example below, since no day information is present, the day is assumed to be current
day ``16`` from *current date* (which is June 16, 2015, at the moment of writing this).
Hence, the level of precision is ``month``:
>>> DateDataParser().get_date_data('March 2015')
DateData(date_obj=datetime.datetime(2015, 3, 16, 0, 0), period='month', locale='en')
Similarly, for date strings with no day and month information present, level of precision
is ``year`` and day ``16`` and month ``6`` are from *current_date*.
>>> DateDataParser().get_date_data('2014')
DateData(date_obj=datetime.datetime(2014, 6, 16, 0, 0), period='year', locale='en')
Dates with time zone indications or UTC offsets are returned in UTC time unless
specified using `Settings`_.
>>> DateDataParser().get_date_data('23 March 2000, 1:21 PM CET')
DateData(date_obj=datetime.datetime(2000, 3, 23, 13, 21, tzinfo=<StaticTzInfo 'CET'>),
period='day', locale='en')
"""
if not isinstance(date_string, str):
raise TypeError('Input type must be str')
res = parse_with_formats(date_string, date_formats or [], self._settings)
if res['date_obj']:
return res
date_string = sanitize_date(date_string)
for locale in self._get_applicable_locales(date_string):
parsed_date = _DateLocaleParser.parse(
locale, date_string, date_formats, settings=self._settings)
if parsed_date:
parsed_date['locale'] = locale.shortname
if self.try_previous_locales:
self.previous_locales.add(locale)
return parsed_date
else:
return DateData(date_obj=None, period='day', locale=None)
[docs] def get_date_tuple(self, *args, **kwargs):
date_data = self.get_date_data(*args, **kwargs)
if sys.version_info < (3, 6): # python 3.5 compatibility
fields = ['date_obj', 'period', 'locale']
else:
fields = date_data.__dict__.keys()
date_tuple = collections.namedtuple('DateData', fields)
return date_tuple(**date_data.__dict__)
def _get_applicable_locales(self, date_string):
pop_tz_cache = []
def date_strings():
""" A generator instead of a static list to avoid calling
pop_tz_offset_from_string if the first locale matches on unmodified
date_string.
"""
yield date_string
if not pop_tz_cache:
stripped_date_string, _ = pop_tz_offset_from_string(
date_string, as_offset=False)
if stripped_date_string == date_string:
stripped_date_string = None
pop_tz_cache[:] = [stripped_date_string]
stripped_date_string, = pop_tz_cache
if stripped_date_string is not None:
yield stripped_date_string
if self.try_previous_locales:
for locale in self.previous_locales:
for s in date_strings():
if self._is_applicable_locale(locale, s):
yield locale
for locale in self._get_locale_loader().get_locales(
languages=self.languages, locales=self.locales, region=self.region,
use_given_order=self.use_given_order):
for s in date_strings():
if self._is_applicable_locale(locale, s):
yield locale
def _is_applicable_locale(self, locale, date_string):
return locale.is_applicable(
date_string,
strip_timezone=False, # it is stripped outside
settings=self._settings)
@classmethod
def _get_locale_loader(cls):
if not cls.locale_loader:
cls.locale_loader = LocaleDataLoader()
return cls.locale_loader