CACHE_URL = os.environ.get('WIDUKIND_CACHE_URL', 'simple') #redis://localhost:6379/0
SCHEMAS_VALIDATION_DISABLE = os.environ.get('WIDUKIND_SCHEMAS_VALIDATION_DISABLE', 'false') def date_validator(value):
def typecheck(_type, msg=None):
    def validator(value): def last_error():
    f = StringIO()
    traceback.print_exc(file=f)
    return f.getvalue() def clean_datetime(dt=None,
                   rm_hour=False,
                   rm_minute=False,
                   rm_second=False,
                   rm_microsecond=True,
                   rm_tzinfo=True):

    now = dt or datetime.now()
    year = now.year
    month = now.month
    day = now.day
    hour = now.hour
    minute = now.minute
    second = now.second
    microsecond = now.microsecond
    tzinfo = now.tzinfo

    if rm_hour:
        hour = 0
    if rm_minute:
        minute = 0
    if rm_second:
        second = 0
    if rm_microsecond:
        microsecond = 0
    if rm_tzinfo:
        tzinfo = None
    return datetime(year, month, day, hour, minute, second, microsecond, tzinfo=tzinfo) def remove_file_and_dir(filepath, let_root=False):
    if not os.path.exists(filepath):
        #logger.warning("file not found [%s]" % filepath)
        return

    if not os.path.isfile(filepath):
        logger.warning("file is not file [%s]" % filepath)
        return

    dirname = os.path.dirname(filepath)

    if not os.path.isdir(dirname) or not os.path.exists(dirname):
        #logger.warning("dir is not dir or not found [%s]" % dirname)
        return
    if os.path.ismount(dirname):
        logger.critical("dir is protected [%s]" % dirname)
        return

    #if logger.isEnabledFor(logging.DEBUG):
    #    logger.debug("remove file[%s]" % filepath)
    #os.remove(filepath)

    #if logger.isEnabledFor(logging.DEBUG):
    #    logger.debug("remove dir[%s]" % dirname)

    for root, dirs, files in os.walk(dirname, topdown=False):
        for name in files:
            _filepath = os.path.join(root, name)
            try:
                os.remove(_filepath)
                logger.debug("remove file[%s]" % _filepath)
            except Exception as err:
                logger.error("not remove file[%s] - error[%s]" % (_filepath, str(err)))
        for name in dirs:
            _filepath = os.path.join(root, name)
            try:
                os.rmdir(_filepath)
                logger.debug("remove dir[%s]" % _filepath)
            except Exception as err:
                logger.error("not remove dir[%s] - error[%s]" % (_filepath, str(err)))

    if os.path.exists(dirname):
        if let_root:
            logger.debug("root dir not remove %s" % dirname)
        else:
            try:
                os.rmdir(dirname)
            except Exception as err:
                logger.error("root dir not remove %s" % dirname) def make_store_path(base_path=None, provider_name=None, dataset_code=None):
    store_filepath = None

    if not base_path:
        store_filepath = tempfile.mkdtemp()
    else:
        store_filepath = base_path

    if provider_name:
        store_filepath = os.path.abspath(os.path.join(store_filepath, provider_name))
        if not os.path.exists(store_filepath):
            os.makedirs(store_filepath)

    if dataset_code:
        store_filepath = os.path.abspath(os.path.join(store_filepath, dataset_code))
        if not os.path.exists(store_filepath):
            os.makedirs(store_filepath)

    return store_filepath def get_url_hash(url):
    return hashlib.sha224(url.encode("utf-8")).hexdigest() def json_dump_convert(obj):

    if isinstance(obj, ObjectId):
        return str(obj)

    elif isinstance(obj, datetime):
        return arrow.get(obj).for_json()

    return obj @timeit("utils.get_datetime_from_period", stats_only=True)
def get_datetime_from_period(date_str, freq=None):

    #TODO: cache

    year = None
    month = None
    day = None

    if freq == "A":
        year = int(get_year(date_str))
        month = 1
        day = 1
    elif freq == "M":
        year = int(get_year(date_str))
        month = int(get_month(date_str))
    elif freq == "D":
        year = int(get_year(date_str))
        month = int(get_month(date_str))
        day = int(get_day(date_str))
    elif freq == "Q":
        year = int(get_year(date_str))
        month_str = get_month(date_str)
        if month_str.startswith("Q"):
            if month_str == "Q1":
                month = 1
            elif month_str == "Q2":
                month = 4
            elif month_str == "Q3":
                month = 7
            elif month_str == "Q4":
                month = 10
            else:
                raise NotImplementedError("freq not implemented freq[%s] date[%s]" % (freq, date_str))
        #else:
        #    month = int(month_str)

    elif freq == "W":
        year = int(get_year(date_str))
        """
        W-WED
        W-MON
        W-FRI
        W-THU
        1998-W53
        2010-06-16
        >>> datetime(2016,12,31).strftime("%Y-%m-%d %W")
        '2016-12-31 52'

        >>> pd.Period("2016-01-27", freq="W-WED").to_timestamp()
        Timestamp('2016-01-21 00:00:00')
        >>> pd.Period("2016-01-27", freq="W-WED").ordinal
        2404

        >>> pd.Period("2016-01-27", freq="W-MON").to_timestamp()
        Timestamp('2016-01-26 00:00:00')
        >>> pd.Period("2016-01-27", freq="W-MON").ordinal
        2405
        """
        raise NotImplementedError("freq not implemented freq[%s] date[%s]" % (freq, date_str))
    elif freq == "S":
        """
        ECB:
            H: Half-yearly (2000-S2)
            S: Half Yearly
        """
        year = int(get_year(date_str))
        if date_str.endswith("S1"):
            month = 1
        elif date_str.endswith("S2"):
            month = 7
        else:
            raise NotImplementedError("freq not implemented freq[%s] date[%s]" % (freq, date_str))
    elif freq == "B":
        """
        Bimestre: période de 2 mois
        """
        raise NotImplementedError("freq not implemented freq[%s] date[%s]" % (freq, date_str))
    else:
        raise NotImplementedError("freq not implemented freq[%s] date[%s]" % (freq, date_str))


    dt = datetime(year, month or 1, day or 1)
    return clean_datetime(dt, rm_hour=True, rm_minute=True, rm_second=True, rm_microsecond=True, rm_tzinfo=True) @timeit("commons.slugify", stats_only=True)
def slugify(text, **kwargs):

    from dlstats.cache import cache

    key = "slugify.%s" % text

    if cache:
        slug_from_cache = cache.get(key)
        if slug_from_cache:
            return slug_from_cache

    slug = original_slugify(text, **kwargs)

    if cache:
        cache.set(key, slug)

    return slug
import time
import os
import tempfile
import logging
import pprint
import hashlib
import json
import pymongo
import pandas
from operator import itemgetter
from datetime import datetime, timedelta
from collections import OrderedDict, deque
from itertools import groupby
from pymongo import ReturnDocument
from bson.json_util import dumps as json_dumps
from widukind_common.utils import get_mongo_db, load_klass, series_archives_store
from widukind_common import errors
from widukind_common.tags import generate_tags_series
from widukind_common.debug import timeit, TRACE_ENABLE
from dlstats import constants
from dlstats.fetchers import schemas
from dlstats.utils import (last_error, 
                           clean_datetime, 
                           remove_file_and_dir, 
                           make_store_path,
                           get_url_hash,
                           json_dump_convert,
                           get_datetime_from_period,
                           slugify)

        categories = Categories.categories(self.provider_name, 
                                           db=self.db, **query)
        for category in categories.values():
            for d in category["datasets"]:
                if self.datasets_filter and not d['dataset_code'] in self.datasets_filter:
                    continue
                self.selected_datasets[d['dataset_code']] = d
        
        return self.selected_datasets

    def datasets_list(self):
        self.get_selected_datasets()
        datasets = self.selected_datasets.values()
        return sorted(datasets, key=itemgetter("dataset_code"))

    def provider_verify(self):
        
        if self.provider_verified:
            return
        
        if self.provider.from_db:
            return
        
        query = {"name": self.provider_name}
        
        provider = self.db[constants.COL_PROVIDERS].find_one(query)
        if not provider:
            self.provider.update_database()
        
        provider = self.load_provider_from_db()
        if provider:
            self.provider = provider
        
        if provider and self.provider.version != self.version:
            self.provider.update_database()

        self.provider_verified = True
            
    def load_provider_from_db(self):
        """Load and set provider fields from DB
        """
        query = {"name": self.provider_name}
        doc = self.db[constants.COL_PROVIDERS].find_one(query)
        if not doc:
            return

        doc.pop('_id')
        return Providers(fetcher=self, from_db=True, **doc)
    
    def upsert_all_datasets(self):
        start = time.time()        
        msg_op = "update"
        
        self.provider_verify()
        
        try:
            query = {"provider_name": self.provider_name}
            
            if self.db[constants.COL_DATASETS].count(query) == 0:
                msg_op = "load"
                msg = "fetcher load START: provider[%s] - bulk-size[%s]"
                logger.info(msg % (self.provider_name, self.bulk_size))
                return self.load_datasets_first()
            else:
                msg = "fetcher update START: provider[%s] - bulk-size[%s]"
                logger.info(msg % (self.provider_name, self.bulk_size))
                return self.load_datasets_update()

        except Exception:
            msg = "fetcher %s ERROR: provider[%s] - error[%s]"
            logger.critical(msg % (msg_op, self.provider_name, last_error()))

        finally:
            end = time.time() - start
            msg = "fetcher %s END: provider[%s] - time[%.3f seconds]"
            logger.info(msg % (msg_op, self.provider_name, end))
        
    def wrap_upsert_dataset(self, dataset_code):

        start = time.time()
        msg = " dataset upsert START: provider[%s] - dataset[%s] - bulk-size[%s] - dataset-only[%s]"
        logger.info(msg % (self.provider_name, dataset_code, self.bulk_size, self.dataset_only))

        self.provider_verify()

        try:
            query = {"provider_name": self.provider_name,
                     "dataset_code": dataset_code}
            projection = {"lock": True}
            dataset_doc = self.db[constants.COL_DATASETS].find_one(query, 
                                                                   projection)
            if dataset_doc and dataset_doc["lock"] is True:
                raise errors.LockedDataset("dataset is locked",
                                           provider_name=self.provider_name,
                                           dataset_code=dataset_code)

            return self.upsert_dataset(dataset_code)

        except errors.RejectUpdatedDataset as err:
            msg = "Reject dataset updated for provider[%s] - dataset[%s]"
            if err.comments:
                msg = "%s - %s" % (msg, err.comments)
            logger.info(msg % (self.provider_name, dataset_code))
        finally:
            end = time.time() - start
            msg = "dataset upsert END: provider[%s] - dataset[%s] - time[%.3f seconds]"
            logger.info(msg % (self.provider_name, dataset_code, end))
        
    def _hook_remove_temp_files(self, dataset):
        if dataset and dataset.for_delete and not self.not_remove_files:
            for filepath in dataset.for_delete:
                try:
                    remove_file_and_dir(filepath)
                except Exception:
                    logger.warning("not remove filepath[%s]" % filepath)

        if not self.not_remove_files:
            for filepath in self.for_delete:
                try:

                    remove_file_and_dir(filepath)