class Fetcher(object):
    """Abstract base class for all fetchers"""
    
    def __init__(self, 
                 provider_name=None, 
                 db=None, 
                 is_indexes=True,
                 version=0,
                 max_errors=10,
                 use_existing_file=False,
                 not_remove_files=False,
                 force_update=False,
                 dataset_only=False,
                 refresh_meta=False,
                 refresh_dsd=False,
                 async_mode=None,
                 bulk_size=500,
                 pool_size=20,
                 **kwargs):
        """
        :param str provider_name: Provider Name
        :param pymongo.database.Database db: MongoDB Database instance        
        :param bool is_indexes: Bypass create_or_update_indexes() if False 

        :raises ValueError: if provider_name is None
        """        
        if not provider_name:
            raise ValueError("provider_name is required")

        self.provider_name = provider_name
        self.db = db or get_mongo_db()
        self.version = version
        self.max_errors = max_errors
        self.use_existing_file = use_existing_file
        self.not_remove_files = not_remove_files
        self.dataset_only = dataset_only
        self.refresh_meta = refresh_meta
        self.refresh_dsd = refresh_dsd
        self.force_update = force_update
        
        self.async_mode = async_mode
        self.bulk_size = bulk_size
        self.pool_size = pool_size
        
        if self.async_mode:
            logger.info("ASYNC MODE [%s]" % self.async_mode)
        else:
            logger.info("ASYNC MODE DISABLE")
        
        self.provider = None
        
        self.errors = 0

        self.categories_filter = [] #[category_code]
        self.datasets_filter = []   #[dataset_code]
        
        self.selected_datasets = {}
        
        self.store_path = os.path.abspath(os.path.join(tempfile.gettempdir(), 
                                                       self.provider_name))
        self.for_delete = []
        
        self.provider_verified = False
        
        if IS_SCHEMAS_VALIDATION_DISABLE:
            logger.warning("schemas validation is disable")
    
    def upsert_calendar(self):
        try:
            for entry in self.get_calendar():
                entry_str = json.dumps(entry, default=json_dump_convert)
                key = hashlib.md5(entry_str.encode('utf_8')).hexdigest()
                entry["key"] = key
                self.db[constants.COL_CALENDARS].find_one_and_replace({"key": key}, 
                                                                      entry, 
                                                                      upsert=True)
                
        except NotImplementedError:
            pass
        except Exception as err:
            logger.critical('upsert_calendar failed for %s error[%s]' % (self.provider_name, last_error()))
    
    def upsert_data_tree(self, data_tree=None, force_update=False):
        #TODO: bulk

        if data_tree and not isinstance(data_tree, list):
            raise TypeError("data_tree is not instance of list")

        if data_tree is None or force_update is True:
            data_tree = self.build_data_tree()

        results = []
        if data_tree:
            Categories.remove_all(self.provider_name, db=self.db)
                        
            for data in data_tree:
                cat = Categories(fetcher=self, **data)
                results.append(cat.update_database())
            
        return results

    def get_selected_datasets(self, force=False):
        
        if self.selected_datasets and not force:
            return self.selected_datasets  

        if Categories.count(self.provider_name, db=self.db) == 0:
            self.upsert_data_tree()
        
        query = {
            "datasets.0": {"$exists": True}
        }
        
        #TODO: enable ?
        if self.categories_filter:
            query["$or"] = [
                {"category_code": {"$in": self.categories_filter}},
                {"all_parents": {"$in": self.categories_filter}},
            ]

        categories = Categories.categories(self.provider_name, 
                                           db=self.db, **query)
        for category in categories.values():
            for d in category["datasets"]:
                if self.datasets_filter and not d['dataset_code'] in self.datasets_filter:
                    continue
                self.selected_datasets[d['dataset_code']] = d
        
        return self.selected_datasets

    def datasets_list(self):
        self.get_selected_datasets()
        datasets = self.selected_datasets.values()
        return sorted(datasets, key=itemgetter("dataset_code"))

    def provider_verify(self):
        
        if self.provider_verified:
            return
        
        if self.provider.from_db:
            return
        
        query = {"name": self.provider_name}
        
        provider = self.db[constants.COL_PROVIDERS].find_one(query)
        if not provider:
            self.provider.update_database()
        
        provider = self.load_provider_from_db()
        if provider:
            self.provider = provider
        
        if provider and self.provider.version != self.version:
            self.provider.update_database()

        self.provider_verified = True
            
    def load_provider_from_db(self):
        """Load and set provider fields from DB
        """
        query = {"name": self.provider_name}
        doc = self.db[constants.COL_PROVIDERS].find_one(query)
        if not doc:
            return

        doc.pop('_id')
        return Providers(fetcher=self, from_db=True, **doc)
    
    def upsert_all_datasets(self):
        start = time.time()        
        msg_op = "update"
        
        self.provider_verify()
        
        try:
            query = {"provider_name": self.provider_name}
            
            if self.db[constants.COL_DATASETS].count(query) == 0:
                msg_op = "load"
                msg = "fetcher load START: provider[%s] - bulk-size[%s]"
                logger.info(msg % (self.provider_name, self.bulk_size))
                return self.load_datasets_first()
            else:
                msg = "fetcher update START: provider[%s] - bulk-size[%s]"
                logger.info(msg % (self.provider_name, self.bulk_size))
                return self.load_datasets_update()

        except Exception:
            msg = "fetcher %s ERROR: provider[%s] - error[%s]"
            logger.critical(msg % (msg_op, self.provider_name, last_error()))

        finally:
            end = time.time() - start
            msg = "fetcher %s END: provider[%s] - time[%.3f seconds]"
            logger.info(msg % (msg_op, self.provider_name, end))
        
    def wrap_upsert_dataset(self, dataset_code):

        start = time.time()
        msg = " dataset upsert START: provider[%s] - dataset[%s] - bulk-size[%s] - dataset-only[%s]"
        logger.info(msg % (self.provider_name, dataset_code, self.bulk_size, self.dataset_only))

        self.provider_verify()

        try:
            query = {"provider_name": self.provider_name,
                     "dataset_code": dataset_code}
            projection = {"lock": True}
            dataset_doc = self.db[constants.COL_DATASETS].find_one(query, 
                                                                   projection)
            if dataset_doc and dataset_doc["lock"] is True:
                raise errors.LockedDataset("dataset is locked",
                                           provider_name=self.provider_name,
                                           dataset_code=dataset_code)

            return self.upsert_dataset(dataset_code)

        except errors.RejectUpdatedDataset as err:
            msg = "Reject dataset updated for provider[%s] - dataset[%s]"
            if err.comments:
                msg = "%s - %s" % (msg, err.comments)
            logger.info(msg % (self.provider_name, dataset_code))
        finally:
            end = time.time() - start
            msg = "dataset upsert END: provider[%s] - dataset[%s] - time[%.3f seconds]"
            logger.info(msg % (self.provider_name, dataset_code, end))
        
    def _hook_remove_temp_files(self, dataset):
        if dataset and dataset.for_delete and not self.not_remove_files:
            for filepath in dataset.for_delete:
                try:
                    remove_file_and_dir(filepath)
                except Exception:
                    logger.warning("not remove filepath[%s]" % filepath)

        if not self.not_remove_files:
            for filepath in self.for_delete:
                try:
                    remove_file_and_dir(filepath)
                except Exception:
                    logger.warning("not remove filepath[%s]" % filepath)
    
    def hook_before_dataset(self, dataset):
        pass

    def hook_after_dataset(self, dataset):
        self._hook_remove_temp_files(dataset)

    def load_datasets_first(self):

        for dataset in self.datasets_list():
            dataset_code = dataset["dataset_code"]
            try:
                self.wrap_upsert_dataset(dataset_code)
            except Exception as err:
                if isinstance(err, errors.MaxErrors):
                    raise
                msg = "error for provider[%s] - dataset[%s]: %s"
                logger.critical(msg % (self.provider_name, 
                                       dataset_code, 
                                       str(err)))

    def load_datasets_update(self):
        #TODO: log and/or warning
        return self.load_datasets_first()

    def _add_to_metadata(self, key, bson):
        if not self.provider.metadata:
            self.provider.metadata = {}
        self.provider.metadata[key] = bson
        self.provider.update_database()
            
    def _structure_put(self, key, url, **values):
        #TODO: compress ?
        bson = {
            "url": url, #{"original": url, "hashed": get_url_hash(url)},
            "values": values
        }
        self._add_to_metadata(key, bson)

    def _structure_get(self, key):
        #TODO: uncompress ?
        if self.provider.metadata and key in self.provider.metadata:
            return self.provider.metadata[key]["values"]

    def build_data_tree(self):
        raise NotImplementedError()
    
    def get_calendar(self):
        raise NotImplementedError()

    def upsert_dataset(self, dataset_code):
        """Upsert a dataset in MongoDB
        
        :param str dataset_code: ID of :class:`Datasets`
        """        
        raise NotImplementedError("This method from the Fetcher class must"
                                  "be implemented.") class Datasets(DlstatsCollection):
    """Abstract base class for datasets
    
    Inherit from :class:`DlstatsCollection`
    """
    
    series_klass = "dlstats.fetchers._commons.Series"
    
    def __init__(self, 
                 provider_name=None,
                 dataset_code=None, 
                 name=None,
                 doc_href=None,
                 last_update=None,
                 metadata=None,
                 bulk_size=500,
                 fetcher=None, 
                 is_load_previous_version=True,
                 **kwargs):
        """
        :param str provider_name: Provider name
        :param str dataset_code: Dataset code
        :param str name: Dataset name
        :param str doc_href: Dataset link
        :param int bulk_size: Batch size for mongo bulk
        :param datetime.datetime last_update: Dataset Last updated 
        :param Fetcher fetcher: Fetcher instance
        :param bool is_load_previous_version: Bypass load previous version if False        
        """        
        super().__init__(fetcher=fetcher)
        self.provider_name = provider_name
        self.dataset_code = dataset_code
        self.name = name
        self.doc_href = doc_href
        self.last_update = last_update
        self.metadata = metadata or {}
        self.bulk_size = self.fetcher.bulk_size
        self.dimension_list = CodeDict()
        self.attribute_list = CodeDict()
        
        self.dimension_keys = []
        self.attribute_keys = []
        self.codelists = {}
        self.concepts = {}

        self.enable = False        
        self.lock = False
        
        self.tags = []

        self.download_first = None
        self.download_last = None

        self.for_delete = []

        self.from_db = False
        if is_load_previous_version:
            self.load_previous_version(provider_name, dataset_code)
            
        self.notes = None
        
        self.series = None
        
        self.set_series_class()
    
    def set_series_class(self):
        if self.fetcher.async_mode:
            if self.fetcher.async_mode == "future":
                self.series_klass = "dlstats.async._concurrent_futures.AsyncSeries"
        
        series_klass = load_klass(self.series_klass)
        self.series = series_klass(dataset=self,
                                   provider_name=self.provider_name, 
                                   dataset_code=self.dataset_code, 
                                   bulk_size=self.bulk_size, 
                                   fetcher=self.fetcher)

    def __repr__(self):
        return pprint.pformat([('provider_name', self.provider_name),
                               ('dataset_code', self.dataset_code)])
        
    def slug(self):
        txt = "-".join([self.provider_name, self.dataset_code])
        return slugify(txt, word_boundary=False, save_order=True)
        
    @property
    def bson(self):
        return {'provider_name': self.provider_name,
                'name': self.name,
                'dataset_code': self.dataset_code,
                'slug': self.slug(),
                'dimension_keys': self.dimension_keys,
                'attribute_keys': self.attribute_keys,
                'codelists': self.codelists,
                'concepts': self.concepts,
                'metadata': self.metadata,
                'doc_href': self.doc_href,
                'last_update': self.last_update,
                'download_first': self.download_first,
                'download_last': self.download_last,
                'notes': self.notes,
                "enable": self.enable,
                "lock": self.lock,
                "tags": self.tags}

    def load_previous_version(self, provider_name, dataset_code):
        dataset = self.fetcher.db[constants.COL_DATASETS].find_one(
                                            {'provider_name': provider_name,
                                             'dataset_code': dataset_code})
        if dataset:
            # convert to dict of dict
            self.dimension_keys = dataset.get("dimension_keys", [])
            self.attribute_keys = dataset.get("attribute_keys", [])
            self.codelists = dataset.get("codelists", {})
            self.concepts = dataset.get("concepts", {})
            self.metadata = dataset.get('metadata', {}) or {}
            self.doc_href = dataset.get('doc_href')
            self.last_update = dataset.get('last_update')
            self.download_last = dataset.get('download_last')
            self.download_first = dataset.get('download_first')
            self.notes = dataset.get('notes')
            self.enable = dataset.get('enable')
            self.lock = dataset.get('lock')
            self.tags = dataset.get('tags')
            
            dimension_list = {}
            attribute_list = {}
            
            if not self.codelists:
                msg = "load previous version fail. provider[%s] - dataset[%s]"
                raise Exception(msg % (self.provider_name, self.dataset_code))

            for key in self.dimension_keys:
                dimension_list[key] = OrderedDict([(k, v) for k, v in self.codelists.get(key).items() if key in self.codelists])

            for key in self.attribute_keys:
                attribute_list[key] = OrderedDict([(k, v) for k, v in self.codelists.get(key).items() if key in self.codelists])
            
            self.dimension_list.set_dict(dimension_list)
            self.attribute_list.set_dict(attribute_list)
            
            self.from_db = True
            
        else:
            msg = "dataset not found for previous loading. provider[%s] - dataset[%s]"
            logger.warning(msg % (provider_name, dataset_code))

    def set_dimension_frequency(self, dimension_name):
        '''Identify frequency field in dataset dimensions'''
        if not dimension_name:
            return
        if not self.metadata:
            self.metadata = {}
        self.metadata["dim_frequency"] = dimension_name

    def set_dimension_country(self, dimension_name):
        '''Identify country field in dataset dimensions'''
        if not dimension_name:
            return
        if not self.metadata:
            self.metadata = {}
        self.metadata["dim_country"] = dimension_name
    
    def add_frequency(self, frequency):
        '''Add used frequency for this dataset'''
        if not frequency:
            return
        if not self.metadata:
            self.metadata = {}
        if not "frequencies" in self.metadata:
            self.metadata["frequencies"] = []
        if not frequency in self.metadata["frequencies"]:
            self.metadata["frequencies"].append(frequency)
    
    def is_recordable(self):
        
        msg = None

        try:
            #if self.fetcher.max_errors and self.fetcher.errors >= self.fetcher.max_errors:
            #    msg = "fetcher max errors exceeded [%s]" % self.fetcher.errors
            #    return False
            
            if self.fetcher.db[constants.COL_PROVIDERS].count({"name": self.provider_name}) == 0:
                msg = "provider[%s] not found in DB" % self.provider_name
                logger.critical(msg)
                return False
            
            if not self.codelists or len(self.codelists) == 0:
                msg = "empty codelists for provider[%s] - dataset[%s]" % (self.provider_name, self.dataset_code)
                logger.critical(msg)                
                return False
            
            query = {"provider_name": self.provider_name,
                     "dataset_code": self.dataset_code}
            if self.fetcher.db[constants.COL_SERIES].count(query) == 0:
                msg = "not series for this dataset"
                return False
    
            return True
        finally:
            if msg:
                self.metadata["disable_reason"] = msg
            
    @timeit("commons.Datasets.update_database")
    def update_database(self, save_only=False):

        self.fetcher.hook_before_dataset(self)
        
        start = time.time()
        
        try:
            if not save_only and not self.fetcher.dataset_only:
                self.series.process_series_data()
        except Exception:
            self.fetcher.errors += 1
            logger.critical(last_error())
            if self.fetcher.max_errors and self.fetcher.errors >= self.fetcher.max_errors:
                msg = "The maximum number of errors is exceeded for provider[%s] - dataset[%s]. MAX[%s]"
                raise errors.MaxErrors(msg % (self.provider_name,
                                              self.dataset_code,
                                              self.fetcher.max_errors))
        finally:
            now = self.series.now
    
            if not self.download_first:
                self.download_first = now

            self.download_last = now
            
            if not self.is_recordable():
                self.enable = False
                msg = "disable dataset[%s] for provider[%s]"
                logger.warning(msg % (self.dataset_code, 
                                      self.provider_name))
            else:
                self.enable = True

            if self.fetcher.dataset_only:
                self.download_last = self.download_last - timedelta(seconds=1)
                self.last_update = self.last_update - timedelta(seconds=1)

            end = time.time()
            avg_all = 0
            avg_write = 0
            duration = end - start 
            try:
                avg_all = (self.series.count_accepts + self.series.count_rejects) / duration
                avg_write = (self.series.count_inserts + self.series.count_updates) / duration 
            except:
                pass
            
            _stats = {
                 "created": clean_datetime(),
                 "tags": ["fetcher", "Datasets.update_database", "commons"],
                 "provider_name": self.provider_name,
                 "dataset_code": self.dataset_code,
                 "fetcher_errors": self.fetcher.errors,
                 "fetcher_version": self.fetcher.version,
                 "count_accepts": self.series.count_accepts,
                 "count_rejects": self.series.count_rejects,
                 "count_inserts": self.series.count_inserts,
                 "count_updates": self.series.count_updates,
                 "count_errors": self.series.count_errors,
                 "avg_all": round(avg_all, 2),
                 "avg_write": round(avg_write, 2),
                 "duration": round(duration, 2),
                 "bulk_size": self.fetcher.bulk_size,
                 "pool_size": self.fetcher.pool_size,            
                 "dataset_only": self.fetcher.dataset_only,
                 "is_trace": TRACE_ENABLE,
                 "logger_level": logger.getEffectiveLevel(),
                 "async_mode": self.fetcher.async_mode,
                 "schema_validation_disable": IS_SCHEMAS_VALIDATION_DISABLE
            }
            
            try:
                self.fetcher.db[constants.COL_STATS_RUN].insert_one(_stats)
            except Exception as err:
                logger.error("record stats : %s" % str(err))
            
            if logger.isEnabledFor(logging.WARN):
                msg_stats = "STATS dataset-update: provider[%s] - dataset[%s] - accepts[%s] - rejects[%s] - inserts[%s] - updates[%s] - avg-all[%.3f] - avg-write[%.3f] - time[%.3f] - bulk-size[%s] - dataset-only[%s]"
                logger.warn(msg_stats % (self.provider_name,
                                         self.dataset_code,
                                         self.series.count_accepts,
                                         self.series.count_rejects,
                                         self.series.count_inserts,
                                         self.series.count_updates,
                                         avg_all,
                                         avg_write,
                                         duration,
                                         self.fetcher.bulk_size,
                                         self.fetcher.dataset_only))
            
            if self.series.count_inserts + self.series.count_updates > 0:
                schemas.dataset_schema(self.bson)
                result = self.update_mongo_collection(constants.COL_DATASETS,
                                                      ['slug'],
                                                      self.bson)
            else:
                result = self.minimal_update_database()
    
            self.fetcher.hook_after_dataset(self)
    
            return result
        
    def minimal_update_database(self):
        query = {"slug": self.slug()}
        query_update =  {"$set": {
                            "download_last": self.series.now,
                            "metadata": self.metadata
                        }}
        result = self.fetcher.db[constants.COL_DATASETS].update_one(query,
                                                                    query_update)
        result = result.upserted_id
        logger.warn("minimal update for dataset[%s]" % self.dataset_code) class Providers(DlstatsCollection):
    """Providers class
    
    Inherit from :class:`DlstatsCollection`
    """

    def __init__(self,
                 name=None,
                 long_name=None,
                 version=0,
                 region=None,
                 website=None,
                 terms_of_use=None,
                 metadata={},
                 enable=True,
                 lock=False,
                 from_db=False,
                 slug=None,
                 fetcher=None):
        """        
        :param str name: Provider Short Name
        :param str long_name: Provider Long Name
        :param str region: Region
        :param str website: Provider Web Site
        :param Fetcher fetcher: Fetcher instance
        """        
        super().__init__(fetcher=fetcher)
        self.name = name
        self.long_name = long_name
        self.version = version
        self.region = region
        self.website = website
        self.terms_of_use = terms_of_use
        self.metadata = metadata or {}
        self.enable = enable
        self.lock = lock
        self.from_db = from_db

        self.validate = schemas.provider_schema(self.bson)

    def __repr__(self):
        return pprint.pformat([(key, self.validate[key]) for key in sorted(self.validate.keys()) if not key in ["data_tree", "slug"]])

    def slug(self):
        if not self.name:
            return 
        return slugify(self.name, word_boundary=False, save_order=True)

    @property
    def bson(self):
        return {'name': self.name,
                'long_name': self.long_name,
                'version': self.version,
                'slug': self.slug(),
                'region': self.region,
                'website': self.website,
                'terms_of_use': self.terms_of_use,
                'metadata': self.metadata,
                "enable": self.enable,
                "lock": self.lock}

    def update_database(self):
        schemas.provider_schema(self.bson)
        return self.update_mongo_collection(constants.COL_PROVIDERS, 
                                            ['slug'], 
                                            self.bson) class SeriesIterator:
    """Base class for all Fetcher data class
    """
    
    def __init__(self, dataset):
        """
        :param Datasets dataset: Datasets instance
        """
        
        if not isinstance(dataset, Datasets):
            raise TypeError("dataset is not instance of Datasets")
        
        self.dataset = dataset
        self.fetcher = self.dataset.fetcher
        self.dataset_code = self.dataset.dataset_code
        self.provider_name = self.fetcher.provider_name
        self.dimension_list = dataset.dimension_list
        self.attribute_list = dataset.attribute_list
        
        self.rows = None
        
    def get_store_path(self):
        return make_store_path(base_path=self.fetcher.store_path,
                               dataset_code=self.dataset_code)

    def __next__(self):
        bson, err = next(self.rows)
        if err:
            return err
        
        if not bson:
            raise StopIteration()

        try:
            return self.clean_field(self.build_series(bson))
        except Exception as err:
            return err

    def _add_url_cache(self, url, status_code=0):
        key = get_url_hash(url)
        if not "cache_url" in self.dataset.metadata:
            self.dataset.metadata["cache_url"] = {}
        if not key in self.dataset.metadata["cache_url"]:
            self.dataset.metadata["cache_url"][key] = {"url": url, "status_code": status_code}
        
    def _is_good_url(self, url, good_codes=[200]):
        """
        FIXME: prendre en compte autre que 200
        """
        key = get_url_hash(url)
        if "cache_url" in self.dataset.metadata:
            if key in self.dataset.metadata["cache_url"]:
                return self.dataset.metadata["cache_url"][key]["status_code"] in good_codes
        return True
    
    def clean_field(self, bson):
        return series_clean_field(bson)

    def build_series(self, bson):
        raise NotImplementedError() CACHE_URL = os.environ.get('WIDUKIND_CACHE_URL', 'simple') #redis://localhost:6379/0
SCHEMAS_VALIDATION_DISABLE = os.environ.get('WIDUKIND_SCHEMAS_VALIDATION_DISABLE', 'false') class Downloader:

    DEFAULT_HEADERS = {
        'user-agent': 'dlstats - https://git.nomics.world/dbnomics/dlstats'
    }

    def __init__(self, url=None, filename=None, store_filepath=None,
                 timeout=None, max_retries=0,
                 replace=True, force_replace=True, use_existing_file=False,
                 headers={}, client=None):

        self.url = url
        self.filename = filename
        self.store_filepath = store_filepath
        self.timeout = timeout
        self.max_retries = max_retries
        self.force_replace = force_replace
        self.headers = headers
        self.client = client or requests
        self.use_existing_file = use_existing_file

        if not self.url:
            raise ValueError("url is required")

        if not self.filename:
            raise ValueError("filename is required")

        for item in self.DEFAULT_HEADERS.items():
            self.headers.setdefault(*item)

        if not self.store_filepath:
            self.store_filepath = tempfile.mkdtemp()
        else:
            if not os.path.exists(self.store_filepath):
                os.makedirs(self.store_filepath, exist_ok=True)

        self.filepath = os.path.abspath(os.path.join(self.store_filepath, self.filename))

        if os.path.exists(self.filepath) and not self.use_existing_file and not replace:
            raise Exception("filepath is already exist : %s" % self.filepath)

    def _download(self, raise_errors=True):

        #TODO: max_retries (self.max_retries)
        #TODO: analyse rate limit dans headers

        start = time.time()
        try:
            response = self.client.get(self.url,
                                    timeout=self.timeout,
                                    stream=True,
                                    allow_redirects=True,
                                    verify=True,
                                    headers=self.headers)

            code = int(response.status_code)

            if code == 304 or code >= 400:
                msg = "download url[%s] - status_code[%s] - reason[%s]" % (self.url,
                                                                           code,
                                                                           response.reason)
                if raise_errors:
                    logger.error(msg)
                    raise response.raise_for_status()
                else:
                    logger.warning(msg)
                    return response

            with open(self.filepath, mode='wb') as f:
                for chunk in response.iter_content(chunk_size=8192):
                    f.write(chunk)

            return response

        except Exception as err:
            logger.critical("Not captured exception : %s" % str(err))
            raise

        end = time.time() - start
        logger.info("download file[%s] - END - time[%.3f seconds]" % (self.url, end))

    def get_filepath(self):

        if os.path.exists(self.filepath) and not self.use_existing_file and self.force_replace:
            os.remove(self.filepath)

        if not os.path.exists(self.filepath):
            logger.warning("not found file[%s] - download dataset url[%s]" % (self.filepath, self.url))
            self._download()
        else:
            logger.warning("use local dataset file [%s]" % self.filepath)

        return self.filepath

    def get_filepath_and_response(self):

        response = None

        if os.path.exists(self.filepath) and not self.use_existing_file and self.force_replace:
            os.remove(self.filepath)

        if not os.path.exists(self.filepath):
            logger.warning("not found file[%s] - download dataset url[%s]" % (self.filepath, self.url))
            response = self._download(raise_errors=False)
        else:
            logger.warning("use local dataset file [%s]" % self.filepath)

        return self.filepath, response def clean_datetime(dt=None,
                   rm_hour=False,
                   rm_minute=False,
                   rm_second=False,
                   rm_microsecond=True,
                   rm_tzinfo=True):

    now = dt or datetime.now()
    year = now.year
    month = now.month
    day = now.day
    hour = now.hour
    minute = now.minute
    second = now.second
    microsecond = now.microsecond
    tzinfo = now.tzinfo

    if rm_hour:
        hour = 0
    if rm_minute:
        minute = 0
    if rm_second:
        second = 0
    if rm_microsecond:
        microsecond = 0
    if rm_tzinfo:
        tzinfo = None
    return datetime(year, month, day, hour, minute, second, microsecond, tzinfo=tzinfo) class XMLSDMX_2_1(XMLSDMX):
    pass class XMLStructure_2_1(XMLStructure_2_0):
    """Parsing SDMX 2.1 structure
    """

    TAGS_MAP = {
        'structure': 'str'
    }

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        if not self.sdmx_client:
            self.sdmx_client = XMLSDMX_2_1(sdmx_url=SDMX_PROVIDERS[self.provider_name]["url"],
                                           agencyID=self.provider_name)

    def get_codelist(self, cl_code):
        """If not cl_code self.codelists, load with remote SDMX
        """
        if not cl_code in self.codelists or not self.codelists.get(cl_code):
            logger.warning("codelist not found [%s] for provider[%s]" % (cl_code, self.provider_name))

            try:
                source = self.sdmx_client.codelist(cl_code=cl_code)
                tree = etree.parse(source)
                #TODO: namespace ?
                namespaces = tree.getroot().nsmap
                #TODO: attention str://
                codelists = tree.xpath('.//str:Codelist', namespaces=namespaces)
                for code_list in codelists:
                    self.process_codelist(code_list)
            except Exception as err:
                msg = "sdmx error for loading codelist[%s] - provider[%s] - error[%s]" % (cl_code, self.provider_name, str(err))
                logger.critical(msg)
                raise Exception(msg)

        return self.codelists[cl_code]

    def process_agency(self, element):
        """
        <str:Agency id="SDMX" urn="urn:sdmx:org.sdmx.infomodel.base.Agency=SDMX">
            <com:Name xml:lang="en">SDMX</com:Name>
        </str:Agency>
        """
        _id = element.attrib.get('id')
        self.agencies[_id] = {
            'id': _id,
            'name': xml_get_name(element),
            'attrs': dict(element.attrib)
        }
        element.clear()

    def process_dataflow(self, element):

        if element.getparent().tag != self.fixtag("structure", "Dataflows"):
            return

        _id = element.attrib.get('id')

        if not _id in self.dataflows:
            dataflow = element.xpath(".//Ref[@class='DataStructure']")[0]
            self.dataflows[_id] = {
                "id": _id,
                "name": xml_get_name(element),
                'attrs': dict(element.attrib),
                "dsd_id": dataflow.attrib.get('id'),
            }

        element.clear()

    def process_category(self, element):

        _id = element.attrib.get('id')

        if not _id in self.categories:
            self.categories[_id] = {
                'id': _id,
                'name': xml_get_name(element),
                'attrs': dict(element.attrib),
                'parent': None
            }
        for child in element.iterchildren(self.fixtag("str", "Category")):
            child_id = child.attrib.get('id')
            self.categories[child_id] = {
                'id': child_id,
                'name': xml_get_name(child),
                'attrs': dict(child.attrib),
                'parent': _id
            }
            child.clear()

    def process_categorisation(self, element):
        _id = element.attrib.get('id')

        dataflow = element.xpath(".//Ref[@class='Dataflow']")[0]
        category = element.xpath(".//Ref[@class='Category']")[0]

        dataflow = dict(dataflow.attrib)
        category = dict(category.attrib)

        self.categorisations[_id] = {
            "id": _id,
            "name": xml_get_name(element),
            "dataflow": dataflow,
            "category": category
        }

        dataflow_id = dataflow["id"]

        if not dataflow["id"] in self.categorisations_dataflows:
            self.categorisations_dataflows[dataflow_id] = []
        if not _id in self.categorisations_dataflows[dataflow_id]:
            self.categorisations_dataflows[dataflow_id].append(_id)

        category_id = category["id"]
        if not category["id"] in self.categorisations_categories:
            self.categorisations_categories[category_id] = []
        if not _id in self.categorisations_categories[category_id]:
            self.categorisations_categories[category_id].append(_id)

        element.clear()

    def process_codelist(self, element):

        _id = element.get('id')

        if not _id in self.codelists:
            self.codelists[_id] = {
                "id": _id,
                'name': self.get_name_element(element),
                "enum": OrderedDict(),
                "attrs": dict(element.attrib)
            }

            for child in element.getchildren():
                if child.tag == self.fixtag("structure", "Code"):
                    key = child.get("id")
                    self.codelists[_id]["enum"][key] = self.get_name_element(child)

        element.clear()

    def process_datastructure(self, element):
        """
        TODO:
        <structure:TimeDimension concept="TIME_PERIOD"/>
        <structure:PrimaryMeasure concept="OBS_VALUE"/>
        """

        _id = element.attrib.get('id')

        if not _id in self.datastructures:

            self.datastructures[_id] = {
                "id": _id,
                "name": xml_get_name(element),
                'attrs': dict(element.attrib),
            }

        if not _id in self.dataflows:

            self.dataflows[_id] = {
                "id": _id,
                "name": xml_get_name(element),
                'attrs': dict(element.attrib),
                "dsd_id": _id,
            }

        for child in element.xpath(".//*[local-name()='Dimension']"):
            self.process_dimension(child, _id)

        for child in element.xpath(".//*[local-name()='Attribute']"):
            self.process_attribute(child, _id)

        element.clear()

    def process_dimension(self, element, dsd_id):

        if element.getparent().tag != self.fixtag("structure", "DimensionList"):
            return

        if not dsd_id in self.dimensions_by_dsd:
            self.dimensions_by_dsd[dsd_id] = OrderedDict()

        if not dsd_id in self.dimension_keys_by_dsd:
            self.dimension_keys_by_dsd[dsd_id] = []

        concept = element.xpath(".//Ref[@class='Concept']")[0]
        _id = concept.attrib.get('id')

        codelist = element.xpath(".//Ref[@class='Codelist']")

        if codelist:
            codelist = codelist[0].attrib.get('id')
            codelist = self.get_codelist(codelist)
            values = codelist["enum"]
            name = codelist["name"]
        else:
            values = {}
            name = self.concepts[_id]

        self.dimensions_by_dsd[dsd_id][_id] = {
            "id": _id,
            "name": name,
            "enum": values,
            "attrs": dict(element.attrib)
        }
        self.dimension_keys_by_dsd[dsd_id].append(_id)

        element.clear()

    def process_attribute(self, element, dsd_id):

        if element.getparent().tag != self.fixtag("structure", "AttributeList"):
            return

        if not dsd_id in self.attributes_by_dsd:
            self.attributes_by_dsd[dsd_id] = OrderedDict()

        if not dsd_id in self.attribute_keys_by_dsd:
            self.attribute_keys_by_dsd[dsd_id] = []

        concept = element.xpath(".//Ref[@class='Concept']")[0]
        _id = concept.attrib.get('id')

        codelist = element.xpath(".//Ref[@class='Codelist']")
        if codelist:
            codelist = codelist[0].attrib.get('id')
            codelist = self.get_codelist(codelist)
            values = codelist["enum"]
            name = codelist["name"]
        else:
            values = {}
            name = self.concepts[_id]

        self.attributes_by_dsd[dsd_id][_id] = {
            "id": _id,
            "name": name,
            "enum": values,
            "attrs": dict(element.attrib)
        }
        self.attribute_keys_by_dsd[dsd_id].append(_id)

        element.clear()

    def process(self, filepath):

        tree_iterator = etree.iterparse(filepath, events=['end', 'start-ns'])

        self.nsmap = get_nsmap(tree_iterator)

        for event, element in tree_iterator:
            if event == 'end':

                #TODO: OrganisationSchemes

                if element.tag == self.fixtag("structure", "Agency"):
                    self.process_agency(element)

                elif element.tag == self.fixtag("structure", "Category"):
                    self.process_category(element)

                elif element.tag == self.fixtag("structure", "Categorisation"):
                    self.process_categorisation(element)

                elif element.tag == self.fixtag("structure", "Dataflow"):
                    self.process_dataflow(element)

                elif element.tag == self.fixtag("structure", "Codelist"):
                    self.process_codelist(element)

                elif element.tag == self.fixtag("structure", "Concept"):
                    self.process_concept(element)

                elif element.tag == self.fixtag("structure", "DataStructure"):
                    self.process_datastructure(element)

            #element.clear() class XMLSpecificData_2_1_INSEE(DataMixIn_INSEE, XMLSpecificData_2_1):

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        if not self.frequencies_supported:
            self.frequencies_supported = self._frequencies_supported def dataset_converter(xml, dataset_code, dsd_id=None):
    bson = {}

    if not dsd_id:
        dsd_id = xml.get_dsd_id(dataset_code)

    bson["provider_name"] = xml.provider_name
    bson["dataset_code"] = dataset_code
    bson["dsd_id"] = dsd_id
    bson["codelists"] = {}

    dimension_keys = xml.dimension_keys_by_dsd[dsd_id]
    attribute_keys = xml.attribute_keys_by_dsd[dsd_id]

    dimensions = xml.dimensions_by_dsd[dsd_id]
    attributes = xml.attributes_by_dsd[dsd_id]

    bson["dimension_keys"] = dimension_keys
    bson["attribute_keys"] = attribute_keys
    bson["dimensions"] = {}
    bson["attributes"] = {}

    bson["name"] = xml.get_dataset_name(dsd_id)
    #bson["name"] = xml.get_dataset_name(dataset_code)
    #TODO: si not ?
    #bson["name"] = xml.dataflows.get(dsd_id)

    for key in dimension_keys:
        if dimensions[key]["enum"]:
            bson["codelists"][key] = dict(dimensions[key]["enum"].items())
        else:
            bson["codelists"][key] = {}
        bson["dimensions"][key] = bson["codelists"][key]

    for key in attribute_keys:
        if attributes[key]["enum"]:
            bson["codelists"][key] = dict(attributes[key]["enum"].items())
        else:
            bson["codelists"][key] = {}
        bson["attributes"][key] = bson["codelists"][key]

    bson["concepts"] = {}
    for key in bson["codelists"].keys():
        if key in xml.concepts:
            bson["concepts"][key] = xml.concepts[key]["name"]

    return bson def select_dimension(dimension_keys, dimensions, choice="avg"):
    """Renvoie le nom de la dimension qui contient le nombre de valeurs demandées
    pour servir ensuite de filtre dans le chargement des données (..A.)
    en tenant compte du nombre de dimensions et de la position
    """
    _min = None
    _max = None
    _avg = None

    import statistics

    if not dimension_keys or not dimensions:
        return 0, None, []

    average = statistics.mean(
        len(values)
        for values in dimensions.values()
        )

    for key in dimension_keys:
        values = dimensions[key]
        count = len(values)
        if count == 0:
            continue

        average_distance = abs(count - average)

        if not _min:
            _min = (key, count)
            _max = (key, count)
            _avg = (key, average_distance)
            continue

        if count < _min[1]:
            _min = (key, count)
        if count > _max[1]:
            _max = (key, count)
        if average_distance < _avg[1]:
            _avg = (key, average_distance)

    if choice == "max":
        _key = _max[0]
    elif choice == "min":
        _key = _min[0]
    else:
        _key = _avg[0]

    position = dimension_keys.index(_key)
    dimension_values = list(dimensions[_key])
    return position, _key, dimension_values def get_key_for_dimension(count_dimensions, position, dimension_value):
    sdmx_key = []
    for i in range(count_dimensions):
        if i == position:
            sdmx_key.append(dimension_value)
        else:
            sdmx_key.append(".")
    return "".join(sdmx_key) def get_dimensions_from_dsd(xml_dsd=None, provider_name=None, dataset_code=None, dsd_id=None):

    dataset = dataset_converter(xml_dsd, dataset_code, dsd_id=dsd_id)

    dimension_keys = dataset["dimension_keys"]

    dimensions = {}
    for key in dimension_keys:
        if key in dataset["codelists"]:
            dimensions[key] = dataset["codelists"][key]
        else:
            msg = "dimension key[%s] is not in codelists - provider[%s] - dataset[%s]"
            logger.warning(msg % (key, provider_name, dataset_code))
            dimensions[key] = {}
    return dimension_keys, dimensions
import os
import logging
import re
import requests
from datetime import datetime
from collections import OrderedDict
from pyquery import PyQuery as pq
from widukind_common import errors
from dlstats.fetchers._commons import Fetcher, Datasets, Providers, SeriesIterator
from dlstats import constants
from dlstats.utils import Downloader, clean_datetime
from dlstats.xml_utils import (XMLSDMX_2_1 as XMLSDMX,
                               XMLStructure_2_1 as XMLStructure,
                               XMLSpecificData_2_1_INSEE as XMLData,
                               dataset_converter,
                               select_dimension,
                               get_key_for_dimension,
                               get_dimensions_from_dsd)
            self.for_delete.append(filepath)
            self.xml_dsd.process(filepath)
            self._categoryschemes = self.xml_dsd.categories
            #self._structure_put("categoryschemes", url, **self._categoryschemes)

        url = "http://www.bdm.insee.fr/series/sdmx/categorisation/%s" % self.provider_name
        """
        if self.refresh_meta is False:
            self._categorisations = self._structure_get("categorisation")
            if self._categorisations:
                self._categorisations_categories = self._structure_get("categorisations_categories")
                logger.info("load structure [categorisation] from metadata for url[%s]" % url)
        """

        if not self._categorisations:
            download = Downloader(url=url,
                                  filename="categorisation.xml",
                                  store_filepath=self.store_path,
                                  headers=SDMX_METADATA_HEADERS,
                                  use_existing_file=self.use_existing_file,
                                  client=self.requests_client)
            filepath = download.get_filepath()
            self.for_delete.append(filepath)
            self.xml_dsd.process(filepath)
            self._categorisations = self.xml_dsd.categorisations
            self._categorisations_categories = self.xml_dsd.categorisations_categories
            #self._structure_put("categorisation", url, **self._categorisations)
            #self._structure_put("categorisations_categories", url, **self._categorisations_categories)

    def _load_structure_concepts(self, force=False):

        if self._dataflows and self._concepts and not force:
            return

        self._load_structure_dataflows(force)

        url = "http://www.bdm.insee.fr/series/sdmx/conceptscheme/%s" % self.provider_name
        if self.refresh_meta is False:
            self._concepts = self._structure_get("concepts")
            if self._concepts:
                self.xml_dsd.concepts = self._concepts
                logger.info("load structure [concepts] from metadata for url[%s]" % url)

        if not self._concepts:
            download = Downloader(url=url,
                                  filename="conceptscheme.xml",
                                  store_filepath=self.store_path,
                                  headers=SDMX_METADATA_HEADERS,
                                  use_existing_file=self.use_existing_file,
                                  client=self.requests_client)
            filepath = download.get_filepath()
            self.for_delete.append(filepath)
            self.xml_dsd.process(filepath)
            self._concepts = self.xml_dsd.concepts
            self._structure_put("concepts", url, **self._concepts)

    def load_datasets_first(self):
        self._load_structure_datatree()
        return super().load_datasets_first()

    def build_data_tree(self):
        """Build data_tree from structure datas
        """
        self._load_structure_datatree()

        categories = []

        position = 0
        for category_code, category in self._categoryschemes.items():
            parent_ids = self.xml_dsd.iter_parent_category_id(category)

            parent = None
            all_parents = None
            if parent_ids:
                all_parents = parent_ids.copy()
                parent = parent_ids.pop()
            else:
                position += 1

            cat = {
                "provider_name": self.provider_name,
                "category_code": category_code,
                "name": category["name"],
                "position": position,
                "parent": parent,
                "all_parents": all_parents,
                "datasets": [],
                "doc_href": None,
                "metadata": {}
            }
            if category_code in self._categorisations_categories:
                categorisation_ids = self._categorisations_categories[category_code]

                for categorisation_id in categorisation_ids:
                    categorisation = self._categorisations[categorisation_id]
                    dataflow_id = categorisation["dataflow"]["id"]
                    #dataset = self.xml_dsd.dataflows[dataflow_id]
                    if not dataflow_id in self._dataflows:
                        logger.critical("dataflow not found [%s]" % dataflow_id)
                        continue
                    dataset = self._dataflows[dataflow_id]

                    cat["datasets"].append({
                        "dataset_code": dataset['id'],
                        "name":dataset["name"],
                        "last_update": None,
                        "metadata": {
                            "dsd_id": dataset["dsd_id"]
                        }
                    })

            categories.append(cat)

        return categories

    def upsert_dataset(self, dataset_code):

        self._load_structure_dataflows()
        self._load_structure_concepts()


        dataset = Datasets(provider_name=self.provider_name,