class Downloader(Thread):
    def __init__(self, start_id, conns=2, skip_timeout=120):
        Thread.__init__(self)
        self.m = pycurl.CurlMulti()

        self.handles = [self._create_handle() for i in range(conns)]
        self.free_handles = list(self.handles)
        self.last_request = None

        self.skip_ahead = False
        self.skip_timeout = skip_timeout
        self.dl_deltas = deque(maxlen=10)

        self.req_time = deque(maxlen=20)
        self.queue_time = deque(maxlen=20)
        self.peek_time = deque(maxlen=20)
        self.req_delay_time = deque(maxlen=20)

        self.stat_thread = Thread(target=self.calc_stat)
        self.dps = deque(maxlen=60)
        self.delta_lock = Lock()
        self.delta_count = 0

        # queue for requests so we can ensure a delay between each request
        # use a list instead of an actual queue to be able to choose min ID
        self.req_queue = []
        self.req_queue_lock = Lock()

        # requests by order
        self.requests = []
        self.requests_lock = Lock()

        self.res_queue = Queue(maxsize=RES_QUEUE_MAXSIZE)
        self.evt_stop = Event()

        self.add_request(start_id)

    def calc_stat(self):
        while not self.evt_stop.wait(1):
            if not self.is_alive():
                break

            with self.delta_lock:
                self.dps.append(self.delta_count)
                self.delta_count = 0

    def _create_handle(self):
        c = pycurl.Curl()

        # c.setopt(pycurl.VERBOSE, 1)
        c.setopt(pycurl.CONNECTTIMEOUT, CONNECT_TIMEOUT)
        c.setopt(pycurl.TIMEOUT, TIMEOUT)
        c.setopt(pycurl.ENCODING, 'gzip, deflate')

        return c

    def _prepare_handle(self, c, request):
        c.setopt(pycurl.URL, URL.format(request.req_id))
        c.setopt(pycurl.WRITEFUNCTION, functools.partial(request.write, self))
        c.req = request

    def run(self):
        try:
            msgr.send_msg('Downloader started', logging.INFO)
            if self.skip_ahead:
                msgr.send_msg('Skipping ahead.. please wait..')
            # self.skip_ahead = False

            self.stat_thread.start()

            start_time = time.time()

            while not self.evt_stop.is_set():
                while not self.evt_stop.is_set():
                    with self.req_queue_lock:
                        if self.req_queue and self.free_handles and \
                                (not self.last_request or time.time() - self.last_request >= config.request_delay):

                            # get unfinished request with minimal ID
                            # assumes any request in queue was properly registered with add_request first
                            req_sorted = sorted(self.req_queue, key=lambda r: self.requests.index(r))
                            req = req_sorted[0]
                            self.req_queue.remove(req)

                            # req = self.req_queue.pop(0)  # TODO: choose min ID (only needed if we resubmit failed attempts)
                            handle = self.free_handles.pop(0)
                            self._prepare_handle(handle, req)

                            if self.last_request:
                                delta = time.time() - self.last_request
                            else:
                                delta = 0

                            add_delay = time.time() - req.submit_time
                            self.queue_time.append(add_delay)

                            self.m.add_handle(handle)
                            self.last_request = time.time()
                            req.start_time = self.last_request

                            self.req_delay_time.append(delta)

                            # msgr.send_msg('Added: {}, delta: {:.3f}s, add delay: {:.3f}s'.format(req.req_id, delta, add_delay), logging.DEBUG)

                    ret, num_handles = self.m.perform()
                    if ret != pycurl.E_CALL_MULTI_PERFORM: break


                # Check for curl objects which have terminated, and add them to the freelist
                while not self.evt_stop.is_set():
                    num_q, ok_list, err_list = self.m.info_read()
                    for c in ok_list:
                        self.m.remove_handle(c)
                        cinfo = info(c)
                        # print("Success: {} - {}: total: {:.2f}s, speed: {} KB/s, size: {} KB, "
                        #       "start-transfer: {:.2f}s, pre-transfer: {:.2f}s"
                        #       .format(c.req.req_id, c.getinfo(pycurl.HTTP_CODE), cinfo['total-time'],
                        #               round(cinfo['speed-download']/1024), round(cinfo['size-download']/1024),
                        #               cinfo['starttransfer-time'], cinfo['pretransfer-time']))

                        if cinfo['http-code'] == 200:
                            if not c.req.submitted_next:
                                next_id = c.req.peek_id()
                                if next_id:
                                    msgr.send_msg('Full peek was required for ID: {}'.format(c.req.req_id), logging.INFO)
                                    self.add_request(next_id)
                                    c.req.submitted_next = next_id
                                    c.req.peek_time = time.time() - c.req.start_time

                            if c.req.submitted_next:
                                c.req.finished = True
                            else:
                                msgr.send_msg('Request successful for ID {}, but next ID was not found. Redownloading..'
                                              .format(c.req.req_id), logging.INFO)
                        else:
                            try:
                                err_data = json.loads(c.req.buffer.getvalue().decode())['error']
                                msgr.send_msg('Request for ID {} failed. {} - Code: {}, Message: {}'
                                              .format(c.req.req_id, cinfo['http-code'], err_data['code'], err_data['message']), logging.WARN)
                            except Exception:
                                msgr.send_msg('Request for ID {} failed. {} - {}'.format(c.req.req_id, cinfo['http-code'], c.req.buffer.getvalue().decode()), logging.WARN)

                        # if c.req.finished:
                        #     self.free_handles.append(c)
                        # else:
                        #     c.req.reset()
                        #     self.m.add_handle(c)
                        self.free_handles.append(c)

                        if not c.req.finished:
                            c.req.reset()
                            self._submit(c.req)
                        else:
                            self.req_time.append(cinfo['total-time'])
                            self.peek_time.append(c.req.peek_time)


                    for c, errno, errmsg in err_list:
                        self.m.remove_handle(c)

                        # if c.req.finished:  # for skips?
                        #     self.free_handles.append(c)
                        # else:
                        #     c.req.reset()
                        #     self.m.add_handle(c)
                        self.free_handles.append(c)

                        if not c.req.finished:
                            msgr.send_msg("Failed: {} - Code: {}, {}. Redownloading.."
                                          .format(c.req.req_id, errno, errmsg), logging.INFO)
                            c.req.reset()
                            self._submit(c.req)
                        else:
                            cinfo = info(c)
                            self.req_time.append(cinfo['total-time'])
                            self.peek_time.append(c.req.peek_time)

                    if num_q == 0:
                        break

                with self.requests_lock:
                    while self.requests and self.requests[0].finished:
                        req = self.requests.pop(0)

                        delta = get_delta(req.req_id, req.submitted_next)
                        self.dl_deltas.append(delta)
                        with self.delta_lock:
                            self.delta_count += delta

                        if not req.skip_data:
                            try:
                                self.res_queue.put((req.req_id, req.buffer), timeout=1)
                            except Full:
                                msgr.send_msg('Result queue is full.. waiting for consumer to free slots..', logging.WARN)
                                self.res_queue.put((req.req_id, req.buffer))

                if self.skip_ahead:
                    if self.dl_deltas and len(self.dl_deltas) == self.dl_deltas.maxlen:
                        avg_delta = sum(self.dl_deltas) / len(self.dl_deltas)
                        # msgr.send_msg("Delta avg: {}".format(avg_delta), logging.DEBUG)
                        if avg_delta <= SUFFICIENT_DELTA:
                            msgr.send_msg("Sufficient delta reached ({}) after {:.3f} seconds. Data processing started."
                                          .format(avg_delta, time.time() - start_time))
                            self.skip_ahead = False

                    if self.skip_ahead:
                        passed = time.time() - start_time
                        if passed > self.skip_timeout:
                            msgr.send_msg("Skip ahead timed out after {:.3f} seconds. Data processing started."
                                          .format(passed), logging.WARN)
                            self.skip_ahead = False

                self.m.select(1.0)

                if not len(self.requests):
                    # Should never happen, since there is always a next id.
                    # If it does, parent thread will end up restarting this from last saved point
                    msgr.send_msg('No requests left.. stopping..', logging.WARN)
                    self.stop()
        except Exception as e:
            msgr.send_msg("Unexpected error occurred while downloading: {}. Error details logged to file.".format(e),
                          logging.ERROR)
            logexception()
        finally:
            self._close()
            msgr.send_msg('Downloader stopped', logging.INFO)

    def stop(self):
        self.evt_stop.set()

    def _close(self):
        self.m.close()
        for handle in self.handles:
            handle.close()

    def add_request(self, next_id):
        req = Request(next_id, skip_data=self.skip_ahead)

        with self.requests_lock:
            req.add_time = time.time()
            self.requests.append(req)

        self._submit(req)

    def _submit(self, req):
        with self.req_queue_lock:
            req.submit_time = time.time()
            self.req_queue.append(req)

    def get(self, block=True, timeout=None):
        return self.res_queue.get(block, timeout)

    def get_stats(self):
        stats = {}

        stats['last-req'] = self.last_request
        stats['req-time'] = sum(self.req_time) / len(self.req_time) if len(self.req_time) else 0
        stats['peek-time'] = sum(self.peek_time) / len(self.peek_time) if len(self.peek_time) else 0
        stats['id-delta'] = sum(self.dl_deltas) / len(self.dl_deltas) if len(self.dl_deltas) else None
        stats['queue-time'] = sum(self.queue_time) / len(self.queue_time) if len(self.queue_time) else 0
        stats['req-delay-time'] = sum(self.req_delay_time) / len(self.req_delay_time) if len(self.req_delay_time) else 0
        stats['delta-rate'] = sum(self.dps) / len(self.dps) if len(self.dps) else 0

        return stats def get_delta(prev_id, curr_id):
    l1 = [int(n) for n in prev_id.split('-')]
    l2 = [int(n) for n in curr_id.split('-')]
    return sum(map(lambda x, y: int(y) - int(x), l1, l2)) class UpdateThread(Thread):
    def __init__(self, fm, interval):
        Thread.__init__(self)
        self.evt_stop = Event()
        self.fm = fm
        self.interval = interval

    def run(self):
        while not self.evt_stop.wait(self.interval):
            currency_updated = False
            filters_updated = False

            try:
                try:
                    cm.update()
                    # msgr.send_msg("Currency rates updated successfully.", logging.INFO)
                    currency_updated = True
                except AppException as e:
                    msgr.send_msg(e, logging.ERROR)

                try:
                    self.fm.fetchFromAPI()
                    # msgr.send_msg("Filters updated successfully.", logging.INFO)
                    filters_updated = True
                except AppException as e:
                    msgr.send_msg(e, logging.ERROR)

                if currency_updated or filters_updated:
                    self.fm.compileFilters()

                if currency_updated and filters_updated:
                    msgr.send_msg("Scheduled update completed successfully.", logging.INFO)
                elif not (currency_updated and filters_updated):
                    msgr.send_msg("Scheduled currency and filters update failed. Retrying in {} seconds.."
                                  .format(self.interval), logging.WARN)
                else:
                    msgr.send_msg("Scheduled update was partially successful.", logging.WARN)
            except Exception as e:
                msgr.send_msg("Unexpected error while updating: {}".format(e), logging.ERROR)
                logexception()

    def stop(self):
        self.evt_stop.set() class CompiledFilter:
    # __slots__ = _FILTER_PRIO.keys() | {'fltr', 'comp', 'enabled', 'crit_ordered'}
    __slots__ = ('rarity', 'armour_max', 'crafted', 'sockets_min', 'pdps_min', 'buyout', 'links_max', 'corrupted', 'block_min', 'stacksize_min', 'quality_max', 'crit_min', 'ilvl_min', 'es_max', 'base', 'aps_min', 'ilvl_max', 'crit_ordered', 'links_min', 'modifiable', 'armour_min', 'level_max', 'price_min', 'aps_max', 'modcount_min', 'edps_max', 'iclass', 'evasion_max', 'stacksize_max', 'enabled', 'identified', 'fltr', 'crit_max', 'sockets_max', 'quality_min', 'modcount_max', 'block_max', 'level_min', 'es_min', 'price_max', 'comp', 'dps_max', 'dps_min', 'enchanted', 'exp', 'pdps_max', 'name', 'fgs', 'evasion_min', 'edps_min')

    def __init__(self, fltr, comp):
        self.fltr = fltr
        self.comp = comp
        self.enabled = fltr.enabled
        self.crit_ordered = sorted(comp.keys(), key=lambda k: _FILTER_PRIO[k])

    def __str__(self):
        return self.getDisplayTitle()

    def finalize(self):
        for att in _FILTER_PRIO:
            setattr(self, att, self.comp.get(att, None))

    def getDisplayPrice(self):
        if 'price_max' not in self.comp:  # or self.comp['price'] <= 0:
            return ''

        return cm.toDisplayPrice(self.comp['price_max'])

    def getDisplayTitle(self, inc_price=False):
        title = self.fltr.title

        if inc_price:
            price = self.getDisplayPrice()

            if price:
                title = "{} ({})".format(self.fltr.title, price)

        return title


    def _checkNames(self, c_name, names):
        for name in names:
            if name:
                if name[0] == name[-1] == '"':
                # if name.startswith('"') and name.endswith('"'):
                    if name[1:-1] == c_name:
                        return True
                elif name in c_name:
                    return True

        return False

    def checkItem(self, item):
        if self.rarity is not None and item.rarity not in self.rarity:
            return False
        if self.price_min is not None and item.c_price is not None and item.c_price < self.price_min:
            return False
        if self.price_max is not None and item.c_price is not None and item.c_price > self.price_max:
            return False
        if self.buyout is not None and self.buyout != item.buyout:
            return False

        if self.name is not None and not self._checkNames(item.c_name, self.name):
            return False
        if self.links_min is not None and self.links_min > item.links_count:
            return False
        if self.links_max is not None and self.links_max < item.links_count:
            return False

        if self.base is not None and self.base not in item.c_base:
            return False
        if self.ilvl_min is not None and self.ilvl_min > item.ilvl:
            return False
        if self.ilvl_max is not None and self.ilvl_max < item.ilvl:
            return False
        if self.corrupted is not None and self.corrupted != item.corrupted:
            return False
        if self.modifiable is not None and self.modifiable != item.modifiable:
            return False
        if self.identified is not None and self.identified != item.identified:
            return False
        if self.crafted is not None and self.crafted != item.crafted:
            return False
        if self.enchanted is not None and self.enchanted != item.enchanted:
            return False
        if self.sockets_min is not None and self.sockets_min > item.sockets_count:
            return False
        if self.sockets_max is not None and self.sockets_max < item.sockets_count:
            return False
        if self.stacksize_min is not None and self.stacksize_min > item.stacksize:
            return False
        if self.stacksize_max is not None and self.stacksize_max < item.stacksize:
            return False

        if self.modcount_min is not None and self.modcount_min > item.modcount:
            return False
        if self.modcount_max is not None and self.modcount_max < item.modcount:
            return False

        if self.iclass is not None and self.iclass & item.iclass != item.iclass:
            return False

        if self.level_min is not None and self.level_min > max(item.level, item.tier):
            return False
        if self.level_max is not None and self.level_max < max(item.level, item.tier):
            return False

        if self.exp is not None and self.exp > item.exp:
            return False
        if self.quality_min is not None and self.quality_min > item.quality:
            return False
        if self.quality_max is not None and self.quality_max < item.quality:
            return False

        if self.es_min is not None and self.es_min > item.es:
            return False
        if self.es_max is not None and self.es_max < item.es:
            return False
        if self.armour_min is not None and self.armour_min > item.armour:
            return False
        if self.armour_max is not None and self.armour_max < item.armour:
            return False
        if self.evasion_min is not None and self.evasion_min > item.evasion:
            return False
        if self.evasion_max is not None and self.evasion_max < item.evasion:
            return False

        if self.dps_min is not None and self.dps_min > item.dps:
            return False
        if self.dps_max is not None and self.dps_max < item.dps:
            return False
        if self.edps_min is not None and self.edps_min > item.edps:
            return False
        if self.edps_max is not None and self.edps_max < item.edps:
            return False
        if self.pdps_min is not None and self.pdps_min > item.pdps:
            return False
        if self.pdps_max is not None and self.pdps_max < item.pdps:
            return False

        if self.aps_min is not None and self.aps_min > item.aps:
            return False
        if self.aps_max is not None and self.aps_max < item.aps:
            return False
        if self.crit_min is not None and self.crit_min > item.crit:
            return False
        if self.crit_max is not None and self.crit_max < item.crit:
            return False
        if self.block_min is not None and self.block_min > item.block:
            return False
        if self.block_max is not None and self.block_max < item.block:
            return False

        if self.fgs is not None:
            for fg in self.fgs:
                if not fg.checkMods(item):
                    return False

        return True

    # def checkItem_old(self, item):
    #     for key in self.crit_ordered:
    #         # if key == "type":
    #         #     if item.type not in self.comp[key]:
    #         #         return False
    #         if key == "rarity":
    #             if item.rarity not in self.comp[key]:
    #                 return False
    #         elif key == "price_min":
    #             if item.c_price is not None and item.c_price < self.comp[key]:
    #                     return False
    #         elif key == "price_max":
    #             if item.c_price is not None and item.c_price > self.comp[key]:
    #                     return False
    #         elif key == "name":
    #             # if not any(name in item.c_name for name in self.comp[key]):
    #             #     return False
    #             if not self._checkNames(item.c_name, self.comp[key]):
    #                 return False
    #         elif key == "iclass":
    #             if self.comp[key] & item.iclass != item.iclass:
    #                 return False
    #         elif key == "base":
    #             if self.comp[key] not in item.c_base:
    #                 return False
    #         elif key == "ilvl_min":
    #             if self.comp[key] > item.ilvl:
    #                 return False
    #         elif key == "ilvl_max":
    #             if self.comp[key] < item.ilvl:
    #                 return False
    #         elif key == "corrupted":
    #             if self.comp[key] != item.corrupted:
    #                 return False
    #         elif key == "modifiable":
    #             if self.comp[key] != item.modifiable:
    #                 return False
    #         elif key == "identified":
    #             if self.comp[key] != item.identified:
    #                 return False
    #         elif key == "crafted":
    #             if self.comp[key] != item.crafted:
    #                 return False
    #         elif key == "enchanted":
    #             if self.comp[key] != item.enchanted:
    #                 return False
    #         elif key == "sockets_min":
    #             if self.comp[key] > item.sockets_count:
    #                 return False
    #         elif key == "sockets_max":
    #             if self.comp[key] < item.sockets_count:
    #                 return False
    #         elif key == "links_min":
    #             if self.comp[key] > item.links_count:
    #                 return False
    #         elif key == "links_max":
    #             if self.comp[key] < item.links_count:
    #                 return False
    #         elif key == "stacksize_min":
    #             if self.comp[key] > item.stacksize:
    #                 return False
    #         elif key == "stacksize_max":
    #             if self.comp[key] < item.stacksize:
    #                 return False
    #         elif key == "modcount_min":
    #             if self.comp[key] > item.modcount:
    #                 return False
    #         elif key == "modcount_max":
    #             if self.comp[key] < item.modcount:
    #                 return False
    #         elif key == "buyout":
    #             if self.comp[key] != item.buyout:
    #                 return False
    #         elif key == "fgs":
    #             for fg in self.comp[key]:
    #                 if not fg.checkMods(item):
    #                     return False
    #         elif key == "level_min":
    #             if self.comp[key] > item.level:
    #                 return False
    #         elif key == "level_max":
    #             if self.comp[key] < item.level:
    #                 return False
    #         elif key == "exp":
    #             if self.comp[key] > item.exp:
    #                 return False
    #         elif key == "quality_min":
    #             if self.comp[key] > item.quality:
    #                 return False
    #         elif key == "quality_max":
    #             if self.comp[key] < item.quality:
    #                 return False
    #
    #         elif key == "es_min":
    #             if self.comp[key] > item.es:
    #                 return False
    #         elif key == "es_max":
    #             if self.comp[key] < item.es:
    #                 return False
    #         elif key == "armour_min":
    #             if self.comp[key] > item.armour:
    #                 return False
    #         elif key == "armour_max":
    #             if self.comp[key] < item.armour:
    #                 return False
    #         elif key == "evasion_min":
    #             if self.comp[key] > item.evasion:
    #                 return False
    #         elif key == "evasion_max":
    #             if self.comp[key] < item.evasion:
    #                 return False
    #
    #         elif key == "edps_min":
    #             if self.comp[key] > item.edps:
    #                 return False
    #         elif key == "edps_max":
    #             if self.comp[key] < item.edps:
    #                 return False
    #         elif key == "pdps_min":
    #             if self.comp[key] > item.pdps:
    #                 return False
    #         elif key == "pdps_max":
    #             if self.comp[key] < item.pdps:
    #                 return False
    #         elif key == "dps_min":
    #             if self.comp[key] > item.dps:
    #                 return False
    #         elif key == "dps_max":
    #             if self.comp[key] < item.dps:
    #                 return False
    #
    #         elif key == 'aps_min':
    #             if self.comp[key] > item.aps:
    #                 return False
    #         elif key == 'aps_max':
    #             if self.comp[key] < item.aps:
    #                 return False
    #         elif key == 'crit_min':
    #             if self.comp[key] > item.crit:
    #                 return False
    #         elif key == 'crit_max':
    #             if self.comp[key] < item.crit:
    #                 return False
    #         elif key == 'block_min':
    #             if self.comp[key] > item.block:
    #                 return False
    #         elif key == 'block_max':
    #             if self.comp[key] < item.block:
    #                 return False
    #
    #     return True

    def getDisplayTotals(self, item):
        totals = []

        for fg in self.comp.get('fgs', []):
            for mf in fg.mfs:
                if mf.type in (ModFilterType.Pseudo, ModFilterType.Total):
                    val = get_mod_val(item, mf)
                    if val:
                        if isinstance(mf.expr, RE_COMPILED_TYPE):
                            expr = mf.expr.pattern
                        else:
                            expr = mf.expr
                        totals.append((mf.type, expr, val))
        return set(totals) _PRICE_REGEX = re.compile('\s*([0-9]+|[0-9]+\.[0-9]+)\s+([a-z\-]+)')
_OVERRIDE_REGEX = re.compile('\s*([+\-*/]?)\s*(.+)')
_NUMBER_REGEX = re.compile('[0-9]+(?:\.[0-9]+)?$')
INVALID_OVERRIDE = "Invalid override price \'{}\' for {}"
INVALID_OVERRIDE_RATE = "Invalid override rate \'{}\' = {} for {}. Rate must be a positive number"
INVALID_PRICE = "Invalid price {}"
    CURRENCY_FNAME = "cfg\\currency.json"
    CURRENCY_BASE_FNAME = "res\\currency_base.json"
    CURRENCY_API = [urljoin(POE_NINJA_API, "GetCurrencyOverview?league={}"),
                    urljoin(POE_NINJA_API, "GetFragmentOverview?league={}")]
    UPDATE_INTERVAL = 10  # minutes
    DEFAULT_CONFIDENCE_LEVEL = ConfidenceLevel.Medium.value
    CHAOS_NAME = 'Chaos Orb'
class CurrencyManager:
class CompiledCurrencyManager:
class CurrencyInfo:
class CurrencyEncoder(NoIndentEncoder):
    def __init__(self):
    def init(self):
    def load_base(self):
    def load(self):
    def save(self):
    def compile(self, shorts=None, rates=None, overrides=None, last_update=None):
    def needUpdate(self):
    def update(self, force_update=False, accept_empty=False):
            def get_count(currency):
    def convert(self, amount, short):
    def isPriceValid(self, price_str):
    def toDisplayPrice(self, rate):
    def toWhisper(self, short):
    def toFull(self, short):
    def _apply_override(self, key, val, overrides, path=None):
    def _compile(self, rates, shorts, overrides):
    def priceFromString(price):
    def overridePriceFromString(override_price):
    def isOverridePriceValid(self, override_price):
    def isPriceRelative(override_price):
    def compilePrice(self, override_price, base_price=None):
    def clearCache(cls):
    def fromCCM(self, ccm):
    def toCCM(self):
    def __init__(self, crates, cshorts):
    def __init__(self):
    def default(self, o): FILTER_FILE_MISSING = "Missing file: {}"
FILTER_INVALID_JSON = "Error decoding JSON: {} in file {}"
FILTERS_FILE_VALIDATION_ERROR = "Error validating filters file: {}"
FILTERS_FILE_SCHEMA_ERROR = "Filters file schema is invalid: {}"
_AUTO_FILTERS_FNAME = "tmp\\filters.auto.json"
_USER_FILTERS_FNAME = "cfg\\filters.json"
_USER_DEFAULT_FILTERS_FNAME = "cfg\\filters.example.json"
FILTERS_CFG_FNAME = "cfg\\filters.config.json"
FILTERS_FILE_SCHEMA_FNAME = "res\\filters_file.schema.json"
_URLS = [
    urljoin(POE_NINJA_API, "GetDivinationCardsOverview?league={}"),
    urljoin(POE_NINJA_API, "GetEssenceOverview?league={}"),
    urljoin(POE_NINJA_API, "GetUniqueJewelOverview?league={}"),
    urljoin(POE_NINJA_API, "GetUniqueMapOverview?league={}"),
    urljoin(POE_NINJA_API, "GetUniqueFlaskOverview?league={}"),
    urljoin(POE_NINJA_API, "GetUniqueWeaponOverview?league={}"),
    urljoin(POE_NINJA_API, "GetUniqueArmourOverview?league={}"),
    urljoin(POE_NINJA_API, "GetUniqueAccessoryOverview?league={}"),
    urljoin(POE_NINJA_API, "GetProphecyOverview?league={}"),
    # urljoin(POE_NINJA_API, "GetMapOverview?league={}"
]
_VARIANTS = {
    # Doryani's Invitation
    "Physical":			("([0-9]+)% increased Physical Damage$", ),
    "Cold":				("([0-9]+)% increased Cold Damage$", ),
    "Fire":				("([0-9]+)% increased Fire Damage$", ),
    "Lightning":		("([0-9]+)% increased Lightning Damage$", ),

    # Atziri's Splendour
    "Armour": ("([0-9]+)% increased Armour$", "\\+([0-9]+) to maximum Life$"),
    "Evasion": ("([0-9]+)% increased Evasion Rating$", "\\+([0-9]+) to maximum Life$"),
    "ES": ("([0-9]+)% increased Energy Shield$", "\\+([0-9]+) to maximum Energy Shield$"),

    "Armour/ES": ("([0-9]+)% increased Armour and Energy Shield$", "\\+([0-9]+) to maximum Energy Shield$"),
    "Evasion/ES": ("([0-9]+)% increased Evasion and Energy Shield$", "\\+([0-9]+) to maximum Energy Shield$"),

    "Armour/Evasion": ("([0-9]+)% increased Armour and Evasion$", "\\+([0-9]+) to maximum Life$"),
    "Evasion/ES/Life": ("([0-9]+)% increased Evasion and Energy Shield$", "\\+([0-9]+) to maximum Life$"),
    "Armour/ES/Life": ("([0-9]+)% increased Armour and Energy Shield$", "\\+([0-9]+) to maximum Life$"),

    "Armour/Evasion/ES": ("([0-9]+)% increased Armour, Evasion and Energy Shield$", ),

    # Vessel of Vinktar
    "Added Attacks":	("Adds ([0-9]+) to ([0-9]+) Lightning Damage to Attacks during Flask effect$", ),
    "Added Spells":		("Adds ([0-9]+) to ([0-9]+) Lightning Damage to Spells during Flask effect$", ),
    "Penetration":		("Damage Penetrates ([0-9]+)% Lightning Resistance during Flask effect$", ),
    "Conversion": 		("([0-9]+)% of Physical Damage Converted to Lightning during Flask effect$", ),

    # Beachhead
    "T5": (),
    "T10": (),
    "T15": (),
}
    V1 = 1
    V2 = 2
    UPDATE_INTERVAL = 10  # minutes
    DEFAULT_BUDGET = '200 chaos'
    DEFAULT_MIN_PRICE = ''
    DEFAULT_PRICE_THRESHOLD = '10 chaos'
    DEFAULT_PRICE_OVERRIDE = '* 1'
    DEFAULT_FPRICE_OVERRIDE = '-15 chaos'
    DEFAULT_CONFIDENCE_LEVEL = ConfidenceLevel.Medium.value
    DEFAULT_ENABLE_5L_FILTERS = True
class FilterVersion(IntEnum):
class FilterManager:
class FiltersInfo:
    def __init__(self):
    def init(self):
    def loadConfig(self):
    def saveConfig(self):
    def needUpdate(self):
    def filter_prices(self):
    def fetchFromAPI(self, force_update=False, accept_empty=False):
            def name_to_id(name):
            def get_unique_id(title, name, category, links):
    def _loadDefaultFilters(self, validate=True):
    def loadUserFilters(self, validate=True):
    def loadAutoFilters(self, validate=True):
    def saveUserFilters(self):
    def saveAutoFilters(self):
    def validateFilters(self):
    def updateConfig(self, default_price_override, default_fprice_override, price_threshold, budget, min_price,
                     price_overrides, filter_price_overrides, filter_state_overrides, confidence_level, enable_5l_filters):
    def compileFilters(self, force_validation=False):
    def _compileFilters(self):
    def applyItemPriceOverrides(self, filters):
    def applyOverrides(self, filters):
    def validateConfig(self):
    def validateOverrides(self):
    def getRawFilters(self):
    def getFilterIds(self):
    def getFilters(self):
    def getActiveFilters(self):
    def compileFilter(self, fltr, path=None):
    def getCategories(self):
    def getPrices(filters):
    def getCompiledPrices(filters):
    def getFilterById(id, filters, match=lambda x, y: x.id == y):
    def getFiltersById(id, filters, match=lambda x, y: x.id == y):
    def saveFiltersToFile(cls, fname, filters):
    def loadFiltersFromFile(cls, fname, validate_data):
    def clearCache(cls):
    def convert(ver, filters):
            def is_type_rarity(item_type):
            def type_to_item_class(item_type):
def lower_json(x):
    def __init__(self): class Item:
    __slots__ = ('_item', 'c_name', 'c_base', 'ilvl', 'links_count', 'corrupted', 'mirrored', 'identified', 'stacksize',
                 'implicit', 'explicit', 'enchant', 'craft', '_mods', 'sockets_count', 'buyout', 'type',
                 'crafted', 'enchanted', 'modcount',
                 '_quality', '_level', '_exp', '_tier',

                 'price',  # price before conversion
                 'c_price',
                 '_iclass', 'rarity',

                 '_armour', '_evasion', '_es', '_life',
                 '_fres', '_cres', '_lres', '_chres', '_ele_res',
                 '_aps', '_crit', '_block',
                 '_dps', '_pdps', '_edps',
                 '_formatted_properties',
                 '_strength_bonus', '_dex_bonus', '_int_bonus', '_attributes_bonus')

    def __init__(self, item, stash_price):
        self._item = item

        self.links_count = self._get_item_links_count()
        self.sockets_count = len(self.sockets)
        self.ilvl = item['ilvl']
        self.corrupted = item['corrupted']
        self.mirrored = item.get('duplicated', False)
        self.identified = item['identified']

        self.c_base = self.base.lower()
        self.c_name = '{} {}'.format(self._get_name().lower(), self.c_base).strip()

        # self.type = _ITEM_TYPE[item['frameType']]
        self.type = item['frameType']
        self.rarity = self.get_rarity()
        self.stacksize = item.get("stackSize", 1)

        self.price = self.get_price(stash_price)
        self.c_price = cm.convert(*self.price) if self.price is not None else None
        self.buyout = self.c_price is not None and self.c_price > 0

        self.implicit = self._item.get('implicitMods', [])
        self.explicit = self._item.get('explicitMods', [])
        self.enchant = self._item.get('enchantMods', [])
        self.craft = self._item.get('craftedMods', [])
        self._mods = None

        self.crafted = len(self.craft) > 0
        self.enchanted = len(self.enchant) > 0
        self.modcount = len(self.implicit) + len(self.explicit) + len(self.enchant) + len(self.craft)

        # Properties and on-demand computed fields
        self._iclass = None
        self._quality = None
        self._level = None
        self._exp = None
        self._tier = None

        self._es = None
        self._armour = None
        self._evasion = None

        self._aps = None
        self._crit = None
        self._block = None

        self._edps = None
        self._pdps = None
        self._dps = None

        self._attributes_bonus = None
        self._strength_bonus = None
        self._dex_bonus = None
        self._int_bonus = None
        self._life = None

        self._fres = None
        self._cres = None
        self._lres = None
        self._chres = None
        self._ele_res = None

        self._formatted_properties = None

    @property
    def mods(self):
        if self._mods is None:
            self._mods = list(itertools.chain(self.explicit, self.implicit, self.enchant, self.craft))
        return self._mods

    @property
    def modifiable(self):
        return not (self.corrupted or self.mirrored)

    @property
    def iclass(self):
        if self._iclass is None:
            self._iclass = self.get_item_class()
        return self._iclass

    @property
    def quality(self):
        if self._quality is None:
            quality = self.get_prop_value('Quality')
            self._quality = int(quality[0][0].strip('+%')) if quality else 0
        return self._quality

    @property
    def level(self):
        if self._level is None:
            level = self.get_prop_value('Level')
            self._level = float(level[0][0].split()[0]) if level else 0
        return self._level

    @property
    def tier(self):
        if self._tier is None:
            tier = self.get_prop_value('Map Tier')
            self._tier = int(tier[0][0]) if tier else 0
        return self._tier

    @property
    def exp(self):
        if self._exp is None:
            exp = self.get_item_prop('Experience')
            self._exp = float(exp['progress']) * 100 if exp else 0
        return self._exp

    @property
    def es(self):
        if self._es is None:
            val = self.get_prop_value('Energy Shield')
            self._es = self.get_item_es(self.quality, self.modifiable,
                                   self.mods, float(val[0][0])) if val else 0
        return self._es

    @property
    def armour(self):
        if self._armour is None:
            armour = self.get_prop_value('Armour')
            self._armour = self.get_item_armour(self.quality, self.modifiable,
                                           self.mods, float(armour[0][0])) if armour else 0
        return self._armour

    @property
    def evasion(self):
        if self._evasion is None:
            val = self.get_prop_value('Evasion Rating')
            self._evasion = self.get_item_evasion(self.quality, self.modifiable,
                                             self.mods, float(val[0][0])) if val else 0
        return self._evasion

    @property
    def edps(self):
        if self._edps is None:
            self._fill_dps()
        return self._edps

    @property
    def pdps(self):
        if self._pdps is None:
            self._fill_dps()
        return self._pdps

    @property
    def dps(self):
        if self._dps is None:
            self._fill_dps()
        return self._dps

    @property
    def aps(self):
        if self._aps is None:
            aps = self.get_prop_value('Attacks per Second')
            self._aps = float(aps[0][0]) if aps else 0
        return self._aps

    @property
    def crit(self):
        if self._crit is None:
            crit = self.get_prop_value('Critical Strike Chance')
            self._crit = float(crit[0][0].strip('%')) if crit else 0
        return self._crit

    @property
    def block(self):
        if self._block is None:
            block = self.get_prop_value('Chance to Block')
            self._block = float(block[0][0].strip('%')) if block else 0
        return self._block

    def _fill_dps(self):
        if self.aps:
            pavg, eavg, cavg = self.get_prop_value('Physical Damage'), \
                               self.get_prop_value('Elemental Damage'), self.get_prop_value('Chaos Damage')

            if pavg:
                pavg = sum((float(i) for i in pavg[0][0].split('-'))) / 2
                self._pdps = self.get_item_pdps(self.quality, self.modifiable, self.mods, pavg, self.aps)
            else:
                self._pdps = 0

            self._edps = sum((float(i) for i in eavg[0][0].split('-'))) / 2 * self.aps if eavg else 0
            cavg = sum((float(i) for i in cavg[0][0].split('-')))/2 if cavg else 0

            self._dps = self._pdps + self._edps + cavg * self.aps
        else:
            self._dps = 0
            self._pdps = 0
            self._edps = 0

    @property
    def fres(self):
        if self._fres is None:
            self._fres = Item.get_mod_total(fire_res_expr, self.mods)
        return self._fres

    @property
    def cres(self):
        if self._cres is None:
            self._cres = Item.get_mod_total(cold_res_expr, self.mods)
        return self._cres

    @property
    def lres(self):
        if self._lres is None:
            self._lres = Item.get_mod_total(lightning_res_expr, self.mods)
        return self._lres

    @property
    def chres(self):
        if self._chres is None:
            self._chres = Item.get_mod_total(chaos_res_expr, self.mods)
        return self._chres

    @property
    def ele_res(self):
        if self._ele_res is None:
            self._ele_res = Item.get_mod_total(ele_res_expr, self.mods)
        return self._ele_res

    @property
    def strength_bonus(self):
        if self._strength_bonus is None:
            self._strength_bonus = Item.get_mod_total(strength_expr, self.mods)
        return self._strength_bonus

    @property
    def dex_bonus(self):
        if self._dex_bonus is None:
            self._dex_bonus = Item.get_mod_total(dex_expr, self.mods)
        return self._dex_bonus

    @property
    def int_bonus(self):
        if self._int_bonus is None:
            self._int_bonus = Item.get_mod_total(int_expr, self.mods)
        return self._int_bonus

    @property
    def attributes_bonus(self):
        if self._attributes_bonus is None:
            self._attributes_bonus = Item.get_mod_total(attributes_expr, self.mods)
        return self._attributes_bonus

    @property
    def life(self):
        if self._life is None:
            self._life = Item.get_mod_total(life_expr, self.mods)
        return self._life

    @property
    def formatted_properties(self):
        if self._formatted_properties is None:
            self._formatted_properties = \
                [ItemProperty.format_property(prop['name'], prop['values'])
                 for prop in self.properties
                 if prop['displayMode'] == PropDisplayMode.Format]
        return self._formatted_properties

    @property
    def prophecy(self):
        return self._item.get('prophecyText', '')

    @property
    def w(self):
        return self._item['w']

    @property
    def h(self):
        return self._item['h']

    @property
    def x(self):
        return self._item['x']

    @property
    def y(self):
        return self._item['y']

    @property
    def league(self):
        return self._item['league']

    @property
    def utility(self):
        return self._item.get('utilityMods', [])

    @property
    def icon(self):
        return self._item['icon']

    @property
    def requirements(self):
        return self._item.get('requirements', [])

    @property
    def properties(self):
        return self._item.get('properties', [])

    @property
    def additional_properties(self):
        return self._item.get('additionalProperties', [])

    @property
    def note(self):
        return self._item.get('note', '')

    @property
    def base(self):
        return _LOCALIZATION_REGEX.sub('', self._item['typeLine'])

    @property
    def name(self):
        return '{} {}'.format(self._get_name(), self.base).strip()

    @property
    def sockets(self):
        return self._item['sockets']

    @property
    def id(self):
        return self._item['id']

    def _get_name(self):
        return _LOCALIZATION_REGEX.sub('', self._item['name'])

    def _get_item_links_count(self):
        groups = array('I', [0]) * 6
        for socket in self.sockets:
            groups[socket['group']] += 1

        return max(groups)

    def get_item_prop(self, name):
        for prop in itertools.chain(self.properties, self.additional_properties):
            if prop['name'] == name:
                return prop
        return None

    def get_prop_value(self, name):
        prop = self.get_item_prop(name)
        if prop:
            return prop['values']
        return None

    # def get_property_value(self, name):
    #     vals = get_prop_value(self._item, name)
    #     if vals:
    #         vals = [val[0] for val in vals]
    #     return vals

    def get_rarity(self):
        try:
            return ItemRarity(self.type)
        except ValueError:
            return ItemRarity.Normal



    @staticmethod
    def get_mod_total(expr, mods, skip_vals=False):
        total = 0
        matched = False

        if not expr.groups:
            skip_vals = True

        for mod in mods:
            match = expr.match(mod)
            if match:
                if skip_vals:
                    return 1

                matched = True
                for val in match.groups():
                    total += float(val)
                    # return total / expr.groups

        if matched:
            return total / expr.groups
        return 0
        # return None maybe do this to allow differentiation between unmatched and a total of 0

    def get_item_links_string(self):
        links = ''
        link_group = None
        for socket in self.sockets:
            if link_group is not None:
                links += '-' if link_group == socket['group'] else ' '
            links += socket['attr']
            link_group = socket['group']

        return links

    @staticmethod
    def get_item_pdps(quality, modifiable, mods, pavg, aps):
        if not modifiable or quality == 20:
            return pavg * aps

        total = 0
        for mod in mods:
            match = phys_expr.match(mod)
            if match:
                total += float(match.group(1))
        return pavg * (120 + total) / (quality + 100 + total) * aps

    @staticmethod
    def get_item_es(quality, modifiable, mods, es):
        if not modifiable or quality == 20:
            return es

        total = 0
        for mod in mods:
            match = es_expr.match(mod)
            if match:
                total += float(match.group(1))
        return es * (120 + total) / (quality + 100 + total)

    @staticmethod
    def get_item_armour(quality, modifiable, mods, armour):
        if not modifiable or quality == 20:
            return armour

        total = 0
        for mod in mods:
            match = armour_expr.match(mod)
            if match:
                total += float(match.group(1))
        return armour * (120 + total) / (quality + 100 + total)

    @staticmethod
    def get_item_evasion(quality, modifiable, mods, evasion):
        if not modifiable or quality == 20:
            return evasion

        total = 0
        for mod in mods:
            match = evasion_expr.match(mod)
            if match:
                total += float(match.group(1))
        return evasion * (120 + total) / (quality + 100 + total)

    def get_item_class(self):
        global superior_expr
        base_line = superior_expr.sub('', self.base, 1)
        item_class = ItemClass(0)
        try:
            # this will fail for magic items with affixes since we dont strip those
            item_class = ItemClass[ItemCollection.base_type_to_id[base_line]]
        except KeyError:
            match = dir_expr.match(self.icon)
            # seems to be accurate for the remaining cases
            if match:
                item_dirs = re.split(r'[/\\]', match.group(1))[:-1]
                for item_dir in item_dirs:
                    class_id = dir_to_id.get(item_dir)
                    if class_id:
                        item_class = ItemClass[class_id]
                        break
            # not all flasks have a traditional link
            elif 'Flask' in base_line:
                item_class = ItemClass.Flask

        if not item_class:
            logger.warn('Failed determining item class. item: {}, base_line: {}, link {}'.format(self.name, base_line, self.icon))

        return item_class

    def get_item_base(self):
        if self.iclass:
            bases = ItemCollection.get_base_types_by_class(self.iclass)
            typeLine = self._item['typeLine']
            for base in bases:
                if re.search(r'\b{}\b'.format(base), typeLine):
                    return base
        return None

    def get_max_sockets(self):
        """ ignores item type, only considers ilvl """
        if self.ilvl >= 50:
            return 6
        if self.ilvl >= 35:
            return 5
        if self.ilvl >= 25:
            return 4
        if self.ilvl >= 2:
            return 3
        return 2

    def get_type_max_sockets(self):
        iclass = self.iclass
        # if self.name in ItemCollection.SIX_LINK_EXCEPTIONS:
        #     return 6
        if (ItemClass.OneHandWeapon | ItemClass.Shield) & iclass == iclass:
            return 3
        if (ItemClass.BodyArmour | ItemClass.TwoHandWeapon) & iclass == iclass:
            return 6
        if (ItemClass.Helmet | ItemClass.Boots | ItemClass.Gloves) & iclass == iclass:
            return 4
        # if iclass & (ItemClass.Ring | ItemClass.Amulet) != 0:
        if (ItemClass.Ring | ItemClass.Amulet) & iclass == iclass:
            return 1  # Unset Ring, and Black Maw Talisman
        return 0

    def get_item_price_raw(self):
        if get_price(self.note):
            return self.note
        return None

    def get_price(self, stash_price):
        price = get_price(self.note)
        return price if price is not None else stash_price

    def get_price_raw(self, stash_raw_price):
        raw_price = self.get_item_price_raw()
        if raw_price is not None:
            return raw_price
        return stash_raw_price

    # TODO MOVE?
    def get_item_price_whisper(self):
        # Returns format of {amount} {currency}

        price = self.price
        if price is not None:
            amount, currency = price
            return dround(amount), cm.toWhisper(currency)

        return None

    # TODO MOVE?
    def get_item_price_display(self):
        # Returns format of {amount} {currency}

        price = self.price
        if price is not None:
            amount, currency = price
            return dround(amount), cm.toFull(currency)

        return None

    # TODO MOVE?
    def get_whisper_msg(self, stash):
        template = '@{} Hi, I would like to buy your {}{} listed{} in {} (stash tab \"{}\"; position: left {}, top {})'

        price = self.get_item_price_whisper()
        price_str = ' for {} {}'.format(*price) if price is not None else ''

        # stack_size_str = ''
        # stack_size_str = '' if self.stacksize == 1 else str(self.stacksize) + ' '
        if self.iclass and ItemClass.Gem & self.iclass == self.iclass:
            gem_text = 'level {} {}% '.format(int(self.level), self.quality)
        else:
            gem_text = ''

        return template.format(stash['lastCharacterName'], gem_text, self.name,
                               price_str, self.league, stash['stash'],
                               self.x + 1, self.y + 1) class ItemType(IntEnum):
    Normal = 0
    Magic = 1
    Rare = 2
    Unique = 3
    Gem = 4
    Currency = 5
    DivinationCard = 6
    QuestItem = 7
    Prophecy = 8
    Relic = 9 class ItemSocket:
    def __init__(self, socket):
        self.type = ItemSocketType(socket['attr'])
        self.group = socket['group'] class ItemProperty:
    class PropertyValue:
        def __init__(self, val):
            self.val = val[0]
            # try:
            self.type = PropValueType(val[1])
            # except ValueError:
            #     self.type = PropValueType.WhiteOrPhysical

    def __init__(self, prop):
        self.values = [ItemProperty.PropertyValue(val) for val in prop.get('values', [])]

        # try:
        self.display_mode = PropDisplayMode(prop['displayMode'])
        # except ValueError:
        #     self.display_mode = PropDisplayMode.Normal

        self.name = prop['name']
        self.progress = prop.get('progress')

    def format(self):
        format_string = re.sub('%[0-9]+', '{}', self.name)
        return format_string.format(*[pv.val for pv in self.values])

    @staticmethod
    def format_property(name, values):
        format_string = re.sub('%[0-9]+', '{}', name)
        return format_string.format(*[val[0] for val in values]) class NotifyThread(threading.Thread):
    def __init__(self,):
        threading.Thread.__init__(self)
        self.ntfy_queue = Queue()
        self._finished = False
        self.daemon = True
        self.registered = False
        self.growl = gntp.notifier.GrowlNotifier(
            applicationName=_APP_NAME,
            notifications=["Item Alert"],
            defaultNotifications=["Item Alert"])

    def send(self, item):
        self.ntfy_queue.put(item)

    def stop(self):
        self._finished = True
        self.ntfy_queue.put(None)

    def run(self):
        while not self._finished:
            item = self.ntfy_queue.get()
            if item is None:
                break
            title, msg, whisperMsg = item

            if config.notify:
                if config.notify_copy_msg:
                    pyperclip.copy(whisperMsg)

                delay = float(config.notification_duration)

                if delay > 0 and self.ntfy_queue.qsize():
                    title = "{} ({} more)".format(title, self.ntfy_queue.qsize())

                if config.notify_play_sound:
                    try:
                        winsound.PlaySound(ALERT_FNAME, winsound.SND_ASYNC | winsound.SND_FILENAME)
                    except RuntimeError as e:
                        pass  # failed to play sound (probably because of excessive notifications)
                    except Exception as e:
                        msgr.send_msg("Error playing sound: {}".format(e), logging.ERROR)

                try:
                    if not self.registered:
                        self.registered = self.growl.register()

                    if self.registered:
                        self.growl.notify(noteType="Item Alert",
                                          title=title,
                                          description=msg)
                    else:
                        msgr.send_msg('Failed to register with Growl, Notifications will not work, '
                                      'please check your settings.', logging.WARN)
                except gntp.errors.NetworkError as e:
                    msgr.send_msg('Failed to send notification. Make sure Growl is running.', logging.WARN)

                time.sleep(float(config.notification_duration))

            self.ntfy_queue.task_done()
        self.ntfy_queue.task_done() class ParserThread(Thread):
    def __init__(self, num_workers, league, stateMgr, resultHandler):
        Thread.__init__(self)
        self.queue = Queue(maxsize=25)
        self.num_workers = num_workers
        self.league = league
        self.stateMgr = stateMgr
        self.resultHandler = resultHandler
        self._finished = False
        self.signal_stop = False

        self.parse_times = deque(maxlen=20)
        self.parse_speed = deque(maxlen=20)

    def stop(self):
        self._finished = True
        self.queue.put(None)

    def put(self, request_id, data, timeout=None):
        self.queue.put((request_id, data), timeout=timeout)

    def run(self):
        # pr = cProfile.Profile()
        # pr.enable()

        with Pool(processes=self.num_workers) as pool:

            data = None
            request_id = None

            msgr.send_msg('Parser started..', logging.INFO)
            while not self._finished:
                try:
                    item = self.queue.get()
                    if item is None:
                        break

                    request_id, b = item
                    msgr.send_update_id(request_id)

                    last_parse = time.time()
                    data = json.loads(b.getvalue().decode())

                    # snapshot filters and currency information
                    with cm.compile_lock:
                        filters = fm.getActiveFilters()
                        c_budget = cm.compilePrice(fm.budget) if fm.budget else None
                        ccm = cm.toCCM()

                    if not len(filters):
                        msgr.send_msg("No filters are active. Stopping..")
                        self.signal_stop = True
                        break

                    # pr.enable()
                    tabs, league_tabs, items = parse_stashes_parallel(data, filters, ccm, self.league, c_budget,
                                                            self.stateMgr, self.resultHandler, self.num_workers, pool)

                    # pr.disable()

                    # parse_next_id(data, self.stateMgr)

                    parse_time = time.time() - last_parse
                    speed = items / max(parse_time, 0.001)
                    self.parse_speed.append(speed)
                    self.parse_times.append(parse_time)

                    msgr.send_msg("Parse: {:.3f}s, Tabs: {}, League tabs: {}, Items: {}"
                                  .format(parse_time, tabs, league_tabs, items), logging.DEBUG)
                # except Empty:
                #     pass
                except Exception as e:
                    msgr.send_msg("Unexpected error occurred while parsing: {}. Error details logged to file. ID: {}".format(e, request_id),
                                  logging.ERROR)
                    logexception()
                    if data:
                        fname = os.path.join(JSON_ERROR_DIR, JSON_ERROR_FNAME.format(request_id))
                        with open(fname, "w") as f:
                            json.dump(data, f, indent=4, separators=(',', ': '))

            msgr.send_msg('Parser stopped', logging.INFO)

    def get_stats(self):
        stats = {}

        stats['parse-time'] = sum(self.parse_times) / len(self.parse_times) if len(self.parse_times) else 0
        stats['parse-speed'] = sum(self.parse_speed) / len(self.parse_speed) if len(self.parse_speed) else 0

        return stats class SearchParams:
    _atts = ['league','type','base','name','dmg_min','dmg_max','aps_min','aps_max','crit_min','crit_max','dps_min','dps_max','edps_min','edps_max','pdps_min','pdps_max','armour_min','armour_max','evasion_min','evasion_max','shield_min','shield_max','block_min','block_max','sockets_min','sockets_max','link_min','link_max','sockets_r','sockets_g','sockets_b','sockets_w','linked_r','linked_g','linked_b','linked_w','rlevel_min','rlevel_max','rstr_min','rstr_max','rdex_min','rdex_max','rint_min','rint_max','q_min','q_max','level_min','level_max','ilvl_min','ilvl_max','rarity','seller','thread','identified','corrupted','online','has_buyout','altart','capquality','buyout_min','buyout_max','buyout_currency','crafted','enchanted']
    # _defaults = {'online': 'x', 'capquality': 'x'}
    _cb_atts = ['altart', 'online', 'capquality']
    _bool_atts = ['has_buyout', 'identified', 'corrupted', 'crafted', 'enchanted']
    _fixed_types = {ItemClass.OneHandWeapon: '1h', ItemClass.TwoHandWeapon: '2h', ItemClass.MapFragment: 'Vaal Fragments', ItemClass.FishingRod: 'Fishing Rods'}
    _valid_types = {'Staff', 'Leaguestone', 'Body Armour', 'Dagger', 'Prophecy', '2h', 'Belt', 'Ring', 'Two Hand Axe', 'Gem', 'One Hand Sword', 'Currency', 'Claw', 'Boots', 'One Hand Axe', 'Divination Card', 'Quiver', '1h', 'Vaal Fragments', 'Two Hand Sword', 'Map', 'Shield', 'Two Hand Mace', 'Fishing Rods', 'One Hand Mace', 'Gloves', 'Breach', 'Helmet', 'Amulet', 'Sceptre', 'Essence', 'Jewel', 'Wand', 'Bow', 'Flask'}
    __slots__ = tuple(_atts + ['fgs'])

    def __init__(self):
        for att in self.__slots__:
            setattr(self, att, None)

    @classmethod
    def genFilterSearch(cls, item, cf):
        if not isinstance(item, Item):
            raise TypeError('item is expected to be of type Item')
        if not isinstance(cf, CompiledFilter):
            raise TypeError('cf is expected to be of type CompiledFilter')

        sp = cls()
        sp.league = config.league
        # sp.dmg_min = cf.dmg_min
        # sp.dmg_max = cf.dmg_max
        sp.aps_min = cf.aps_min
        sp.aps_max = cf.aps_max
        sp.crit_min = cf.crit_min
        sp.crit_max = cf.crit_max
        sp.dps_min = cf.dps_min
        sp.dps_max = cf.dps_max
        sp.edps_min = cf.edps_min
        sp.edps_max = cf.edps_max
        sp.pdps_min = cf.pdps_min
        sp.pdps_max = cf.pdps_max
        sp.armour_min = cf.armour_min
        sp.armour_max = cf.armour_max
        sp.evasion_min = cf.evasion_min
        sp.evasion_max = cf.evasion_max
        sp.shield_min = cf.es_min
        sp.shield_max = cf.es_max
        sp.block_min = cf.block_min
        sp.block_max = cf.block_max
        sp.sockets_min = cf.sockets_min
        sp.sockets_max = cf.sockets_max
        sp.link_min = cf.links_min
        sp.link_max = cf.links_max

        sp.q_min = cf.quality_min
        sp.q_max = cf.quality_max
        sp.level_min = cf.level_min
        sp.level_max = cf.level_max
        sp.ilvl_min = cf.ilvl_min
        sp.ilvl_max = cf.ilvl_max

        # sp.sockets_r = cf.sockets_r
        # sp.sockets_g = cf.sockets_g
        # sp.sockets_b = cf.sockets_b
        # sp.sockets_w = cf.sockets_w
        # sp.linked_r = cf.linked_r
        # sp.linked_g = cf.linked_g
        # sp.linked_b = cf.linked_b
        # sp.linked_w = cf.linked_w
        # sp.rlevel_min = cf.rlevel_min
        # sp.rlevel_max = cf.rlevel_max
        # sp.rstr_min = cf.rstr_min
        # sp.rstr_max = cf.rstr_max
        # sp.rdex_min = cf.rdex_min
        # sp.rdex_max = cf.rdex_max
        # sp.rint_min = cf.rint_min
        # sp.rint_max = cf.rint_max

        # sp.seller = None
        # sp.thread = None
        # sp.altart = None

        sp.identified = cf.identified

        if cf.corrupted is not None:
            sp.corrupted = cf.corrupted
        elif cf.modifiable is not None:
            sp.corrupted = not cf.modifiable

        sp.crafted = cf.crafted
        sp.enchanted = cf.enchanted

        sp.online = True
        sp.capquality = True

        sp.has_buyout = cf.buyout

        # # TODO: limit prices?
        # sp.buyout_min = cf.price_min
        # sp.buyout_max = cf.price_max
        # sp.buyout_currency = 'chaos'

        sp.base = item.get_item_base() if cf.base else None
        sp.type = cf.iclass

        if cf.rarity:
            if len(cf.rarity) == 1:
                rarity = cf.rarity[0]
            else:  # show all rarities instead of choosing first one or item's rarity
                rarity = None
            sp.rarity = rarity

        # the search engine only accepts full words in name field, according to the tutorial
        # any partial names from filters which arent considered words won't work properly
        sp.name = cls._find_matched_name(item, cf)
        sp.fgs = cf.fgs

        return sp

    @classmethod
    def genItemSearch(cls, item, cf):
        if not isinstance(item, Item):
            raise TypeError('item is expected to be of type Item')
        if not isinstance(cf, CompiledFilter):
            raise TypeError('cf is expected to be of type CompiledFilter')

        sp = cls()
        sp.league = config.league

        sp.identified = cf.identified
        sp.rarity = item.rarity

        sp.online = True
        sp.capquality = True
        sp.has_buyout = True

        if item.links_count == 6:
            sp.link_min = item.links_count

        # if not item.modifiable:
        #     sp.corrupted = True
        sp.corrupted = not item.modifiable

        if item.rarity in (ItemRarity.Unique, ItemRarity.Relic):
            sp.name = item.name
            if item.links_count >= 5:
                sp.link_min = item.links_count
        else:
            sp.name = item.get_item_base()

        if item.iclass:
            if ItemClass.Gem & item.iclass == item.iclass:
                if item.level < 18:
                    sp.level_min = item.level - 2
                else:
                    sp.level_min = item.level

                if item.quality > 20:
                    sp.q_min = item.quality
                else:
                    sp.q_min = max(item.quality - 3, 0)
            elif ItemClass.Map & item.iclass == item.iclass:
                sp.level_min = item.tier
                sp.level_max = item.tier



        # # TODO: limit prices?
        # sp.buyout_min = cf.price_min
        # sp.buyout_max = cf.price_max
        # sp.buyout_currency = 'chaos'

        sp.fgs = cf.fgs

        return sp

    def convert(self):
        base_params = {}
        for att in self._atts:
            val = getattr(self, att, None)
            if att == 'type':
                itype = self._convert_type(val)
                val = itype if itype in self._valid_types else ''
            elif att == 'rarity':
                try:
                    val = ItemRarity(val).name.lower()
                except ValueError:
                    val = ''
            elif val is not None:
                if att in self._bool_atts:
                    val = '1' if val else '0'
                elif att in self._cb_atts:
                    val = 'x' if val else ''

            base_params[att] = val or ''

        fgs = self.fgs or []
        fgs_params = list(chain.from_iterable((self._fgToParam(fg) for fg in fgs)))

        return list(base_params.items()) + fgs_params

    @staticmethod
    def _find_matched_name(item, cf):
        if cf.name:
            for name in cf.name:  # cf names are lower case, so is item.c_name
                if name and name[0] == name[-1] == '"':
                    if name[1:-1] == item.c_name:
                        return item.name
                elif name in item.c_name:
                    return name
        return None

    @staticmethod
    def _convert_type(iclass):
        if not iclass:
            return None

        if ItemClass.OneHandSword & iclass == iclass:
            iclass = ItemClass.OneHandSword
        elif iclass == ItemClass.MiscOneHandMace:
            iclass = ItemClass.OneHandMace
        # elif iclass == ItemClass.MiscMapItem:
        #     iclass = ItemClass.MapFragment  # not perfect

        if iclass in SearchParams._fixed_types:
            return SearchParams._fixed_types[iclass]

        return namify(iclass.name)

    # TODO: MOVE TO CLASSES?
    @staticmethod
    def _mfToParam(mf):
        # if compiled:
        # pseudo = text
        # others = re.compiled
        if isinstance(mf.expr, RE_COMPILED_TYPE):
            expr = mf.expr.pattern
        else:
            expr = mf.expr

        return [
            ('mod_name', mod_helper.modToParam(mf.type, expr)),
            ('mod_min', '' if mf.min is None else mf.min),
            ('mod_max', '' if mf.max is None else mf.max),
        ]

    @staticmethod
    def _fgToParam(fg):
        searchable_mfs = []
        for mf in fg.mfs:
            try:
                searchable_mfs.append(SearchParams._mfToParam(mf))
            except ValueError:
                pass

        g_min = ''
        g_max = ''

        if fg.getType() == FilterGroupType.Count:
            g_type = 'Count'
        elif fg.getType() == FilterGroupType.Nothing:
            g_type = 'None'
        else:
            g_type = 'And'

        if fg.getType() == FilterGroupType.Count:
            if fg.match_min is not None:
                g_min = fg.match_min
            if fg.match_max is not None:
                g_max = fg.match_max

        if not searchable_mfs:
            return []

        l = list(chain.from_iterable(searchable_mfs))
        l.extend([
            ('group_type', g_type),
            ('group_min', g_min),
            ('group_max', g_max),
            ('group_count', len(searchable_mfs))
        ])

        return l def parse_next_id(data, stateMgr):
    stateMgr.saveState(data["next_change_id"]) def parse_stashes_parallel(data, filters, ccm, league, c_budget, stateMgr, resultHandler, numCores, pool):
    item_count = 0
    league_stashes = []
    for stash in data["stashes"]:
        if stash["public"] and stash["items"] and stash["items"][0]["league"] == league:
            item_count += len(stash["items"])
            league_stashes.append(stash)

    # results = Parallel(n_jobs=numCores, verbose=10)(delayed(parse_stash)(stash, filters, c_budget) for stash in league_stashes)
    # results = pool(delayed(parse_stash)(stash, filters, c_budget) for stash in league_stashes)
    # pool = None
    # if pool:
    #TODO: send multiple stashes to reduce overhead
    # results = pool.starmap(parse_stash, ((stash, filters, c_budget, ccm) for stash in league_stashes), round_up(len(league_stashes)/numCores))

    grouped_stashes = []
    step, extra = divmod(len(league_stashes), numCores)
    for i in range(0, numCores):
        if i == 0:
            grouped_stashes.append(league_stashes[:step + extra])
        else:
            sx = i * step + extra
            grouped_stashes.append(league_stashes[sx:sx + step])

    results = pool.starmap(parse_stashes, ((stashes, filters, c_budget, ccm) for stashes in grouped_stashes), 1)

    # else:
    #     results = (parse_stash(stash, filters, c_budget) for stash in league_stashes)

    for item, stash, fltr in chain.from_iterable(results):
        if stateMgr.addItem(item.id, item.get_price_raw(get_stash_price_raw(stash)), stash["accountName"]):
            resultHandler(copy.deepcopy(item), copy.copy(stash), copy.deepcopy(fltr))

    parse_next_id(data, stateMgr)
    return len(data["stashes"]), len(league_stashes), item_count def get_stash_price_raw(self):
    if self.price is not None:
        return self.name
    return None class StateManager:
    STATE_ID_FNAME = "tmp\\stateid.dat"
    STATE_FNAME = "tmp\\state.dat"

    def __init__(self):
        self.items = {}
        self.delta = {}
        self.changeid = ""
        self.fState = None
        self.fStateId = None

        # self.loadState()

    def itemExists(self, id):
        item = self.items.get(id)
        return item and \
               ((config.history_retention > 0 and datetime.utcnow() - item[0] <= timedelta(days=config.history_retention))
                or not config.history_retention)

    def addItem(self, id, price, acc):
        if id not in self.items:
            b_update = True
        else:
            timestamp, curprice, curacc = self.items[id]

            b_update = curprice != price or curacc != acc or \
                       (config.history_retention > 0 and datetime.utcnow() - timestamp > timedelta(days=config.history_retention))

        if b_update:
            self.items[id] = (datetime.utcnow(), price, acc)
            self.delta[id] = self.items[id]

        return b_update

    def __saveStateDelta(self):
        if self.fState is None or self.fState.closed:
            self.fState = open(StateManager.STATE_FNAME, mode="a+", encoding="utf-8", errors="replace")
        else:
            self.fState.seek(0, 2)

        for id in self.delta:
            self.fState.write(itemToString(id, self.delta[id]) + "\n")

        self.delta.clear()

    def saveStateItems(self):
        if self.fState is not None:
            self.fState.close()

        self.clearOldEntries()

        self.fState = open(StateManager.STATE_FNAME, mode="w", encoding="utf-8", errors="replace")

        for id in self.items:
            self.fState.write(itemToString(id, self.items[id]) + "\n")

        self.fState.flush()

    def saveState(self, id):
        if self.fStateId is None or self.fStateId.closed:
            self.fStateId = open(StateManager.STATE_ID_FNAME, "w")
        else:
            self.fStateId.seek(0)

        self.__saveStateDelta()
        self.fStateId.write(id)
        self.changeid = id

    def __loadStateItems(self):
        try:
            if self.fState is None or self.fState.closed:
                self.fState = open(StateManager.STATE_FNAME, encoding="utf-8", errors="replace")

            line = self.fState.readline()
            while line:
                fields = line.strip('\n').split(sep=";")
                try:
                    item_id, timestamp, price, acc = \
                        fields[0], getDateTimeFromString(fields[1]), None if fields[2] == "" else fields[2], fields[3]

                    self.items[item_id] = [timestamp, price, acc]
                except IndexError:
                    # 'ignoring entry {}'.format(line)
                    pass  # silently ignore invalid entries
                line = self.fState.readline()
        except FileNotFoundError:
            pass

        # This will remove unnecessary entries
        self.saveStateItems()

    def loadState(self):
        try:
            with open(StateManager.STATE_ID_FNAME, "r") as f:
                self.changeid = f.readline().strip('\n')
        except FileNotFoundError:
            self.changeid = ""

        self.__loadStateItems()

    def clearOldEntries(self):
        if config.history_retention > 0:
            last_date = datetime.utcnow() - timedelta(days=config.history_retention)
            for item_id, item in dict(self.items).items():
                timestamp = item[0]
                if timestamp < last_date:
                    self.items.pop(item_id, None)

    def getChangeId(self):
        return self.changeid

    def close(self):
        if self.fState:
            self.fState.close()
        if self.fStateId:
            self.fStateId.close()

    @classmethod
    def clearCache(cls):
        files = (cls.STATE_ID_FNAME, cls.STATE_FNAME)
        for fname in files:
            try:
                os.remove(fname)
            except FileNotFoundError:
                pass RE_COMPILED_TYPE = type(re.compile(''))
POE_NINJA_API = 'http://api.poe.ninja/api/Data/'
    CONFIG_FNAME = "cfg\\app.ini"
class MsgType(IntEnum):
class Messenger:
class AppException(Exception):
class CompileException(AppException):
class AppConfiguration:
class NoIndent(object):
class NoIndentEncoder(json.JSONEncoder):
class ConfidenceLevel(IntEnum):
    def __init__(self):
    def send_tmsg(self, msg, log_level=logging.NOTSET, tag=None):
    def send_msg(self, msg, log_level=logging.NOTSET, timed=False, tag=None):
    def send_stopped(self):
    def send_update_id(self, id):
    def send_object(self, obj):
    def __init__(self):
    def load(self):
    def load_default(self):
    def _load_settings(self, settings=None):
    def update(self, cfg):
    def save(self):
def getBytesFromURL(url, handle=None, max_attempts=1, connect_timeout=None, timeout=None, gzip=True):
def getJsonFromURL(url, handle=None, max_attempts=1, gzip=True):
def getDataFromUrl(url, callback, max_attempts=1):
def str2bool(val):
def round_up(num):
def round_down(num):
def dround(num, ndigits=2):
def tmsg(msg):
def getBaseUrl(url):
def isAbsoluteUrl(url):
def logexception():
def get_verror_msg(verror, data=None):
    def __init__(self, value):
    def __init__(self, *args, **kwargs):
    def default(self, o):
    def encode(self, o):
def utc_to_local(utc_dt):
def namify(text):
def normalize_id(text):
import itertools
import json
import logging
import os
import pycurl
from datetime import datetime
from queue import Empty, Full
from threading import Event
from urllib.parse import urljoin
from lib.Downloader import Downloader, get_delta
from lib.UpdateThread import UpdateThread
from lib.CompiledFilter import CompiledFilter
from lib.CurrencyManager import cm
from lib.FilterManager import fm
from lib.ItemHelper import *
from lib.ItemHelper import Item, ItemType, ItemSocket, ItemProperty
from lib.NotifyThread import NotifyThread
from lib.ParserThread import ParserThread
from lib.SearchParams import SearchParams
from lib.StashHelper import parse_next_id, parse_stashes_parallel, get_stash_price_raw
from lib.StateManager import StateManager
from lib.Utility import config, AppException, getJsonFromURL, msgr, logexception, getBytesFromURL, isAbsoluteUrl, \
    dround, getBaseUrl, POE_NINJA_API

# import time
# from contextlib import closing
# from multiprocessing.pool import Pool




# import pstats
# from io import StringIO

# API URLS
NINJA_API = urljoin(POE_NINJA_API, "GetStats")
POE_API = "http://api.pathofexile.com/public-stash-tabs?id={}"
POE_BETA_API = "http://betaapi.pathofexile.com/api/public-stash-tabs?id={}"
# STASH https://www.pathofexile.com/character-window/get-stash-items?league=Legacy&tabs=1&tabIndex=1&accountName=##

JSON_ERROR_FNAME = "log\\error.json"
ITEM_ERROR_FNAME = 'log\\item_error.json'


class StashScanner:
    def __init__(self):
        self.stateMgr = StateManager()

        self.notifier = NotifyThread()

        self.updater = UpdateThread(fm, 10 * 60)