Create a fragment generator that executes the search plan.
        :param on_load: Function to be called just after a new URI is dereferenced
        :param on_seeds: Function to be called just after a seed of a tree is identified
        :param on_plink: Function to be called when a pattern link is reached
        :param on_link: Function to be called when following a property that is not of a pattern
        :param on_type: Function to be called when search for a type triple
        :param on_type_validation: Function to be called just after a type is validated
        :param on_tree: Function to be called just before a tree is going to be explored
        :param provider:
        :param queue_size:
        :param workers:
        :param stop_event:
        :param queue_wait:
        :param lazy:
        :return:

def get_fragment_generator(self, on_load=None, on_seeds=None, on_plink=None, on_link=None, on_type=None,
                               on_type_validation=None, on_tree=None, workers=None, stop_event=None, queue_wait=None,
                               queue_size=100, provider=None, lazy=True):
        """
        Create a fragment generator that executes the search plan.
        :param on_load: Function to be called just after a new URI is dereferenced
        :param on_seeds: Function to be called just after a seed of a tree is identified
        :param on_plink: Function to be called when a pattern link is reached
        :param on_link: Function to be called when following a property that is not of a pattern
        :param on_type: Function to be called when search for a type triple
        :param on_type_validation: Function to be called just after a type is validated
        :param on_tree: Function to be called just before a tree is going to be explored
        :param provider:
        :param queue_size:
        :param workers:
        :param stop_event:
        :param queue_wait:
        :param lazy:
        :return:
        """

        if workers is None:
            workers = multiprocessing.cpu_count()

        fragment_queue = Queue.Queue(maxsize=queue_size)
        workers_queue = Queue.Queue(maxsize=workers)

        if stop_event is None:
            stop_event = Event()

        def __create_graph():
            if provider is None:
                return ConjunctiveGraph()
            else:
                return provider.create(conjunctive=True)

        def __release_graph(g):
            if provider is not None:
                provider.release(g)
            else:
                g.remove((None, None, None))
                g.close()

        def __open_graph(gid, loader, format):
            if provider is None:
                content, headers = loader(gid, format)
                if not isinstance(content, bool):
                    g = ConjunctiveGraph()
                    g.parse(source=content, format=format)
                    return g
                return content
            else:
                return provider.create(gid=gid, loader=loader, format=format)

        def __get_content(uri, format):
            try:
                # log.debug('[Dereference][START] {}'.format(uri))
                response = requests.get(uri, headers={'Accept': _accept_mimes[format]}, timeout=30)
            except requests.Timeout:
                log.debug('[Dereference][TIMEOUT][GET] {}'.format(uri))
                return True
            except UnicodeEncodeError:
                log.debug('[Dereference][ERROR][ENCODE] {}'.format(uri))
                return True
            except Exception:
                log.debug('[Dereference][ERROR][GET] {}'.format(uri))
                return True

            if response.status_code == 200:
                try:
                    return StringIO.StringIO(response.content), response.headers
                except SyntaxError:
                    traceback.print_exc()
                    log.error('[Dereference][ERROR][PARSE] {}'.format(uri))
                    return False
                except ValueError:
                    traceback.print_exc()
                    log.debug('[Dereference][ERROR][VAL] {}'.format(uri))
                    return False
                except DBNotFoundError:
                    # Ignore this exception... it is raised due to a stupid problem with prefixes
                    return True
                except SAXParseException:
                    traceback.print_exc()
                    log.error('[Dereference][ERROR][SAX] {}'.format(uri))
                    return False
                except Exception:
                    traceback.print_exc()
                    log.error('[Dereference][ERROR] {}'.format(uri))
                    return True

        def __dereference_uri(tg, uri):

            if not isinstance(uri, URIRef):
                return

            uri = uri.encode('utf-8')

            def treat_resource_content(parse_format):
                g = __open_graph(uri, loader=__get_content, format=parse_format)
                if isinstance(g, bool):
                    return g

                try:
                    tg.get_context(uri).__iadd__(g)
                    return True
                finally:
                    if g is not None:
                        __release_graph(g)

            """
            Load in a tree graph the set of triples contained in uri, trying to not deference the same uri
            more than once in the context of a search plan execution
            :param tg: The graph to be loaded with all the triples obtained from uri
            :param uri: A resource uri to be dereferenced
            :return:
            """
            loaded = False
            for fmt in sorted(_accept_mimes.keys(), key=lambda x: x != self.__last_success_format):
                loaded = treat_resource_content(fmt)
                if loaded:
                    self.__last_success_format = fmt
                    break

            if loaded and on_load is not None:
                triples = list(tg.get_context(uri).triples((None, None, None)))
                on_load(uri, triples)

        def __process_link_seed(seed, tree_graph, link, next_seeds):
            __check_stop()
            try:
                __dereference_uri(tree_graph, seed)
                seed_pattern_objects = tree_graph.objects(subject=seed, predicate=link)
                next_seeds.update(seed_pattern_objects)
            except Exception as e:
                traceback.print_exc()
                log.warning(e.message)

        def __process_pattern_link_seed(seed, tree_graph, pattern_link):
            __check_stop()
            try:
                __dereference_uri(tree_graph, seed)
            except:
                pass
            seed_pattern_objects = tree_graph.objects(subject=seed, predicate=pattern_link)
            return seed_pattern_objects

        def __check_stop():
            if stop_event.isSet():
                with self.__resource_lock:
                    self.__fragment.clear()
                    for tg in self.__resource_queue.keys():
                        try:
                            tg.remove((None, None, None))
                            tg.store.close()
                        except KeyError:
                            pass
                        tg.close()
                        __release_graph(tg)
                    self.__plan_graph = None
                    self.__uri_cache = None
                    self.__node_spaces = None
                    self.__node_patterns = None
                    self.__spaces = None
                    self.__patterns = None
                    self.__subjects_to_ignore.clear()
                    self.__resource_queue.clear()
                gc.collect()
                raise StopException()

        def __put_triple_in_queue(quad):
            if (dt.now() - self.__last_iteration_ts).total_seconds() > 100:
                log.info('Aborted fragment collection!')
                stop_event.set()
            fragment_queue.put(quad, timeout=queue_wait)

        def __follow_node(node, tree_graph, seed_space, seed):
            """
            Recursively search for relevant triples following the current node and all its successors
            :param node: Tree node to be followed
            :param tree_graph:
            :param seed_space:
            :param seed: Collected seed for the current node
            :return:
            """

            def node_has_filter(x):
                """
                Check if a node is a pattern node and has an object filter
                """
                p_node = list(self.__plan_graph.objects(subject=x, predicate=AGORA.byPattern))
                try:
                    p_node = p_node.pop()
                    return 'filter_object' in self.__patterns[p_node] or 'filter_subject' in self.__patterns[p_node]
                except IndexError:
                    return False

            try:
                # Get the sorted list of current node's successors
                nxt = sorted(list(self.__plan_graph.objects(node, AGORA.next)),
                             key=lambda x: node_has_filter(x), reverse=True)

                # Per each successor...
                for n in nxt:
                    if seed_space in self.__node_spaces[n]:
                        node_patterns = self.__node_patterns.get(n, [])

                        # In case the node is not a leaf, 'onProperty' tells which is the next link to follow
                        try:
                            link = list(self.__plan_graph.objects(subject=n, predicate=AGORA.onProperty)).pop()
                        except IndexError:
                            link = None

                        filter_next_seeds = set([])
                        next_seeds = set([])
                        # If the current node is a pattern node, it must search for triples to yield
                        for pattern in node_patterns:
                            pattern_space = self.__patterns[pattern].get('space', None)
                            if pattern_space != seed_space or seed in self.__subjects_to_ignore[pattern_space]:
                                continue

                            subject_filter = self.__patterns[pattern].get('filter_subject', None)
                            if subject_filter is not None and seed != subject_filter:
                                self.__subjects_to_ignore[pattern_space].add(seed)
                                continue

                            pattern_link = self.__patterns[pattern].get('property', None)

                            # If pattern is of type '?s prop O'...
                            if pattern_link is not None:
                                if (seed, pattern_link) not in self.__fragment:
                                    obj_filter = self.__patterns[pattern].get('filter_object', None)
                                    if on_plink is not None:
                                        on_plink(pattern_link, [seed], pattern_space)

                                    seed_was_filtered = True
                                    try:
                                        for seed_object in list(
                                                __process_pattern_link_seed(seed, tree_graph, pattern_link)):
                                            __check_stop()
                                            quad = (pattern, seed, pattern_link, seed_object)
                                            if obj_filter is None or u''.join(seed_object).encode(
                                                    'utf-8') == u''.join(obj_filter.toPython()).encode('utf-8'):
                                                self.__fragment.add((seed, pattern_link))
                                                __put_triple_in_queue(quad)
                                                seed_was_filtered = False
                                                if isinstance(obj_filter, URIRef):
                                                    filter_next_seeds.add(obj_filter)
                                        if obj_filter is not None and seed_was_filtered:
                                            self.__subjects_to_ignore[pattern_space].add(seed)
                                    except AttributeError as e:
                                        log.warning('Trying to find {} objects of {}: {}'.format(link, seed, e.message))

                            # If pattern is of type '?s a Concept'...
                            obj_type = self.__patterns[pattern].get('type', None)
                            if obj_type is not None:
                                check_type = self.__patterns[pattern].get('check', False)
                                if on_type is not None:
                                    on_type(obj_type, [seed], pattern_space)

                                __dereference_uri(tree_graph, seed)
                                try:
                                    seed_objects = list(tree_graph.objects(subject=seed, predicate=link))
                                    for seed_object in seed_objects:
                                        type_triple = (pattern, seed_object, RDF.type, obj_type)
                                        # In some cases, it is necessary to verify the type of the seed
                                        if (seed_object, obj_type) not in self.__fragment:
                                            if check_type:
                                                __dereference_uri(tree_graph, seed_object)
                                                types = list(
                                                    tree_graph.objects(subject=seed_object, predicate=RDF.type))
                                                if obj_type in types:
                                                    self.__fragment.add((seed_object, obj_type))
                                                    __put_triple_in_queue(type_triple)
                                                else:
                                                    self.__subjects_to_ignore[pattern_space].add(seed_object)
                                            else:
                                                self.__fragment.add((seed_object, obj_type))
                                                __put_triple_in_queue(type_triple)
                                except AttributeError as e:
                                    log.warning('Trying to find {} objects of {}: {}'.format(link, seed, e.message))

                        # If the current node is not a leaf... go on finding seeds for the successors
                        if link is not None and seed not in self.__subjects_to_ignore[seed_space]:
                            if on_link is not None:
                                on_link(link, [seed], seed_space)
                            __process_link_seed(seed, tree_graph, link, next_seeds)

                        if filter_next_seeds:
                            next_seeds = set.intersection(next_seeds, filter_next_seeds)

                        chs = list(chunks(list(next_seeds), min(len(next_seeds), max(1, workers / 2))))
                        next_seeds.clear()
                        try:
                            while True:
                                __check_stop()
                                chunk = chs.pop()
                                threads = []
                                for s in chunk:
                                    try:
                                        workers_queue.put_nowait(s)
                                        future = pool.submit(__follow_node, n, tree_graph, seed_space, s)
                                        threads.append(future)
                                    except Queue.Full:
                                        # If all threads are busy...I'll do it myself
                                        __follow_node(n, tree_graph, seed_space, s)
                                    except Queue.Empty:
                                        pass

                                wait(threads)
                                [(workers_queue.get_nowait(), workers_queue.task_done()) for _ in threads]
                        except (IndexError, KeyError):
                            pass
            except Queue.Full:
                stop_event.set()
            except Exception as e:
                traceback.print_exc()
                log.error(e.message)
                return

        def get_fragment_triples():
            """
            Iterate over all search trees and yield relevant triples
            :return:
            """

            def execute_plan():
                for tree in trees:
                    if on_tree is not None:
                        on_tree(tree)

                    # Prepare an dedicated graph for the current tree and a set of type triples (?s a Concept)
                    # to be evaluated retrospectively
                    tree_graph = __create_graph()

                    try:
                        self.__resource_queue[tree_graph] = []

                        # Get all seeds of the current tree
                        seeds = list(self.__plan_graph.objects(tree, AGORA.hasSeed))
                        if on_seeds is not None:
                            on_seeds(seeds)

                        # Check if the tree root is a pattern node and in that case, adds a type triple to the
                        # respective set
                        root_pattern = list(self.__plan_graph.objects(tree, AGORA.byPattern))
                        if len(root_pattern):
                            pattern_node = list(
                                self.__plan_graph.objects(subject=tree, predicate=AGORA.byPattern)).pop()
                            seed_type = self.__patterns[pattern_node].get('type', None)
                            [type_triples.add((pattern_node, sd, seed_type)) for sd in seeds]

                        # Get the children of the root node and follow them recursively
                        nxt = list(self.__plan_graph.objects(tree, AGORA.next))
                        if len(nxt):
                            # Prepare the list of seeds to start the exploration with, taking into account all
                            # search spaces that were defined
                            s_seeds = set(seeds)
                            for sp in self.__spaces:
                                for seed in s_seeds:
                                    __follow_node(tree, tree_graph, sp, seed)
                    finally:
                        __release_graph(tree_graph)

                        if lazy and found_data and len(self.__spaces) == 1:
                            break

                self.__completed = True

            def get_tree_length(x):
                """
                Return the value of the Agora length property in the given tree node
                :param x:
                :return:
                """
                length = list(self.__plan_graph.objects(subject=x, predicate=AGORA.length)).pop()
                return length

            # Get all search trees contained in the search plan and sort them by length. A shorter tree is going
            # to be explored first.
            trees = self.__plan_graph.subjects(RDF.type, AGORA.SearchTree)
            trees = sorted(trees, key=lambda x: get_tree_length(x))

            type_triples = set([])

            thread = Thread(target=execute_plan)
            thread.daemon = True
            thread.start()

            found_data = False
            while not self.__completed or fragment_queue.not_empty:
                try:
                    (t, s, p, o) = fragment_queue.get(timeout=1)
                    found_data = True
                    fragment_queue.task_done()
                    if p == RDF.type:
                        type_triples.add((t, s, o))
                    else:
                        yield (t, s, p, o)
                except Queue.Empty:
                    if self.__completed:
                        break
                self.__last_iteration_ts = dt.now()
            thread.join()

            # All type triples that are of subjects to ignore won't be returned (this has to be done this way
            # because of generators nature)
            all_ignores = {}
            if self.__subjects_to_ignore.values():
                all_ignores = set.intersection(*self.__subjects_to_ignore.values())
            valid_type_triples = [(t, s, o) for (t, s, o) in type_triples if s not in all_ignores]
            for (t, s, o) in valid_type_triples:
                if on_type_validation is not None:
                    on_type_validation((t, s, RDF.type, o))
                yield (t, s, RDF.type, o)

        return get_fragment_triples(), self.__plan_graph.namespaces(), self.__plan_graph