def find_file(filename, include_paths):
def __parse_include(include):
def __get_content(fpath, line_ranges, symbol):
def resolve(uri, include_paths): class Extension(Configurable):
    """
    All extensions should inherit from this base class

    Attributes:
        extension_name: str, the unique name of this extension, should
            be overriden and namespaced appropriately.
        project: project.Project, the Project instance which documentation
            hotdoc is working on.
        formatter: formatter.Formatter, may be subclassed.
        argument_prefix (str): Short name of this extension, used as a prefix
            to add to automatically generated command-line arguments.
    """
    # pylint: disable=unused-argument
    extension_name = "base-extension"
    argument_prefix = ''
    paths_arguments = {}
    path_arguments = {}
    written_out_sitemaps = set()

    def __init__(self, app, project):
        """Constructor for `Extension`.

        This should never get called directly.

        Args:
            project: The `project.Project` instance which documentation
                is being generated.
        """
        self.project = project
        self.app = app
        self.sources = set()
        self.smart_sources = []
        self.index = None
        self.source_roots = OrderedSet()
        self._created_symbols = DefaultOrderedDict(OrderedSet)
        self.__package_root = None
        self.__toplevel_comments = OrderedSet()

        self.formatter = self._make_formatter()

    # pylint: disable=no-self-use
    def warn(self, code, message):
        """
        Shortcut function for `loggable.warn`

        Args:
            code: see `utils.loggable.warn`
            message: see `utils.loggable.warn`
        """
        warn(code, message)

    # pylint: disable=no-self-use
    def error(self, code, message):
        """
        Shortcut function for `utils.loggable.error`

        Args:
            code: see `utils.loggable.error`
            message: see `utils.loggable.error`
        """
        error(code, message)

    def debug(self, message, domain=None):
        """
        Shortcut function for `utils.loggable.debug`

        Args:
            message: see `utils.loggable.debug`
            domain: see `utils.loggable.debug`
        """
        if domain is None:
            domain = self.extension_name
        debug(message, domain)

    def info(self, message, domain=None):
        """
        Shortcut function for `utils.loggable.info`

        Args:
            message: see `utils.loggable.info`
            domain: see `utils.loggable.info`
        """
        if domain is None:
            domain = self.extension_name
        info(message, domain)

    def reset(self):
        """
        This function is only useful for testing purposes, at least
        for now.
        """
        self._created_symbols = DefaultOrderedDict(OrderedSet)
        self.__package_root = None

    def get_pagename(self, name):
        self.__find_package_root()
        # Find the longest prefix
        longest = None
        for path in OrderedSet([self.__package_root]) | self.source_roots:
            commonprefix = os.path.commonprefix([path, name])
            if commonprefix == path and (longest is None or len(path) > len(longest)):
                longest = path

        if longest is not None:
            return os.path.relpath(name, longest)

        return name

    def get_symbol_page(self, symbol_name, symbol_pages, smart_pages, section_links):
        if symbol_name in symbol_pages:
            return symbol_pages[symbol_name]

        symbol = self.app.database.get_symbol(symbol_name)
        assert symbol is not None

        if symbol.parent_name and symbol.parent_name != symbol_name:
            page = self.get_symbol_page(
                symbol.parent_name, symbol_pages, smart_pages, section_links)
        else:
            smart_key = self._get_smart_key(symbol)
            if smart_key is None:
                return None

            if smart_key in smart_pages:
                page = smart_pages[smart_key]
            else:
                pagename = self.get_pagename(smart_key)
                page = Page(smart_key, True, self.project.sanitized_name, self.extension_name,
                            output_path=os.path.dirname(pagename))
                if page.link.ref in section_links:
                    self.warn('output-page-conflict',
                              'Creating a page for symbol %s would overwrite the page '
                              'declared in a toplevel comment (%s)' % (symbol_name, page.link.ref))
                    page = None
                else:
                    smart_pages[smart_key] = page

        if page is not None:
            symbol_pages[symbol_name] = page

        return page

    def _get_toplevel_comments(self):
        return self.__toplevel_comments

    # pylint: disable=too-many-locals
    # pylint: disable=too-many-branches
    # pylint: disable=too-many-statements
    def make_pages(self):
        # All symbol names that no longer need to be assigned to a page
        dispatched_symbol_names = set()

        # Map symbol names with pages
        # This is used for assigning symbols with a parent to the page
        # where their parent will be rendered, unless the symbol has been
        # explicitly assigned or ignored
        symbol_pages = {}

        smart_pages = OrderedDict()

        # Map pages with the sources they explicitly import
        imported_sources = {}

        # This is simply used as a conflict detection mechanism, see
        # hotdoc.core.tests.test_doc_tree.TestTree.test_section_and_path_conflict
        section_links = set()

        # These are used as a duplicate detection mechanism, map
        # relocated or ignored symbols to the source files where they were initially
        # listed
        relocated_symbols = {}
        private_symbols = {}

        # First we make one page per toplevel comment (eg. SECTION comment)
        # This is the highest priority mechanism for sorting symbols
        for comment in self._get_toplevel_comments():
            # Programming error from extension author
            assert comment.name
            symbol_names = comment.meta.pop('symbols', [])
            private_symbol_names = comment.meta.pop('private-symbols', [])
            sources = comment.meta.pop('sources', None)

            page = Page(comment.name, True, self.project.sanitized_name, self.extension_name,
                        comment=comment)

            for symbol_name in symbol_names:
                if symbol_name in relocated_symbols:
                    self.warn('symbol-listed-twice',
                              'Symbol %s listed in %s was already listed in %s' %
                              (symbol_name, comment.filename, relocated_symbols[symbol_name]))
                    continue
                elif symbol_name in private_symbols:
                    self.warn('symbol-listed-twice',
                              'Symbol %s listed in %s was marked as private in %s' %
                              (symbol_name, comment.filename, private_symbols[symbol_name]))
                    continue
                else:
                    page.symbol_names.add(symbol_name)
                    symbol_pages[symbol_name] = page
                    relocated_symbols[symbol_name] = comment.filename
                    dispatched_symbol_names.add(symbol_name)

            for symbol_name in private_symbol_names:
                if symbol_name in relocated_symbols:
                    self.warn('symbol-listed-twice',
                              'Symbol %s marked private in %s was already listed in %s' %
                              (symbol_name, comment.filename, relocated_symbols[symbol_name]))
                    continue
                elif symbol_name in private_symbols:
                    self.warn('symbol-listed-twice',
                              'Symbol %s marked as private in %s was '
                              'already marked as private in %s' %
                              (symbol_name, comment.filename, private_symbols[symbol_name]))
                    continue
                private_symbols[symbol_name] = comment.filename
                symbol_pages[symbol_name] = None
                dispatched_symbol_names.add(symbol_name)

            section_links.add(page.link.ref)
            smart_key = self._get_comment_smart_key(comment)
            if smart_key in smart_pages:
                smart_pages[comment.name] = page
            else:
                smart_pages[smart_key] = page

            if sources is not None:
                abs_sources = []
                for source in sources:
                    if os.path.isabs(source):
                        abs_sources.append(source)
                    else:
                        abs_sources.append(os.path.abspath(os.path.join(
                            os.path.dirname(comment.filename), source)))
                imported_sources[page] = abs_sources

        # Used as a duplicate detection mechanism
        relocated_sources = {}

        # We now browse all the pages with explicitly imported sources
        # Importing sources has a lower level of priority than importing
        # symbols, which is why we do that in a separate loop
        for page, sources in imported_sources.items():
            for source in sources:
                if source not in self._get_all_sources():
                    self.warn('invalid-relocated-source',
                              'Source %s does not exist but is relocated in %s' %
                              (source, page.name))
                    continue

                if source in relocated_sources:
                    self.warn('invalid-relocated-source',
                              'Source %s relocated in %s was already relocated in %s' %
                              (source, page.name, relocated_sources[source]))
                    continue

                if source in self._created_symbols:
                    symbol_names = OrderedSet(
                        self._created_symbols[source]) - dispatched_symbol_names
                    page.symbol_names |= symbol_names
                    dispatched_symbol_names |= symbol_names

                relocated_sources[source] = page.name

        # We now browse all the symbols we have created
        for _, symbol_names in self._created_symbols.items():
            for symbol_name in symbol_names:
                if symbol_name in dispatched_symbol_names:
                    continue

                page = self.get_symbol_page(
                    symbol_name, symbol_pages, smart_pages, section_links)

                # Can be None if creating a page to hold the symbol conflicts with
                # a page explicitly declared in a toplevel comment or a parent has been
                # marked as private
                if page is None:
                    continue

                page.symbol_names.add(symbol_name)
                dispatched_symbol_names.add(symbol_name)

        # Finally we make our index page
        if self.index:
            index_page = self.project.tree.parse_page(
                self.index, next(iter(self.project.include_paths)), self.extension_name)
        else:
            index_page = Page('%s-index' % self.argument_prefix, True, self.project.sanitized_name,
                              self.extension_name)

        if not index_page.title:
            index_page.title = self._get_smart_index_title()

        smart_pages['%s-index' % self.argument_prefix] = index_page

        return smart_pages

    def setup(self):
        """
        Extension subclasses should implement this to scan whatever
        source files they have to scan, and connect to the various
        signals they have to connect to.

        Note that this will be called *after* the `tree.Tree`
        of this instance's `Extension.project` has been fully
        constructed, but before its `tree.Tree.resolve_symbols`
        method has been called.
        """
        pass

    @staticmethod
    def get_dependencies():
        """
        Override this to return the list of extensions this extension
        depends on if needed.

        Returns:
            list: A list of `ExtDependency` instances.
        """
        return []

    def parse_toplevel_config(self, config):
        """Parses and make use of the toplevel configuration."""
        self.formatter.parse_toplevel_config(config)

    def parse_config(self, config):
        """
        Override this, making sure to chain up first, if your extension adds
        its own custom command line arguments, or you want to do any further
        processing on the automatically added arguments.

        The default implementation will set attributes on the extension:
        - 'sources': a set of absolute paths to source files for this extension
        - 'index': absolute path to the index for this extension

        Additionally, it will set an attribute for each argument added with
        `Extension.add_path_argument` or `Extension.add_paths_argument`, with
        the extension's `Extension.argument_prefix` stripped, and dashes
        changed to underscores.

        Args:
            config: a `config.Config` instance
        """
        prefix = self.argument_prefix
        self.sources = config.get_sources(prefix)
        self.smart_sources = [
            self._get_smart_filename(s) for s in self.sources]
        self.index = config.get_index(prefix)
        self.source_roots = OrderedSet(
            config.get_paths('%s_source_roots' % prefix))

        for arg, dest in list(self.paths_arguments.items()):
            val = config.get_paths(arg)
            setattr(self, dest, val)

        for arg, dest in list(self.path_arguments.items()):
            val = config.get_path(arg)
            setattr(self, dest, val)

        self.formatter.parse_config(config)

    def add_attrs(self, symbol, **kwargs):
        """
        Helper for setting symbol extension attributes
        """
        for key, val in kwargs.items():
            symbol.add_extension_attribute(self.extension_name, key, val)

    def get_attr(self, symbol, attrname):
        """
        Helper for getting symbol extension attributes
        """
        return symbol.extension_attributes.get(self.extension_name, {}).get(
            attrname, None)

    @staticmethod
    def add_arguments(parser):
        """
        Subclasses may implement this method to add their own arguments to
        the hotdoc binary.

        In this function, you should add an argument group to the passed-in
        parser, corresponding to your extension.
        You can then add arguments to that argument group.

        Example::

            @staticmethod
            def add_arguments(parser):
                group = parser.add_argument_group('Chilidoc',
                    'Delicious Hotdoc extension')
                Chilidoc.add_sources_argument(group)
                group.add_argument('--chili-spicy', action='store_true',
                    help='Whether to add extra pepper')

        Args:
            parser (argparse.ArgumentParser): Main hotdoc argument parser
        """
        pass

    @classmethod
    def get_assets_licensing(cls):
        return {}

    @classmethod
    def add_index_argument(cls, group):
        """
        Subclasses may call this to add an index argument.

        Args:
            group: arparse.ArgumentGroup, the extension argument group
            prefix: str, arguments have to be namespaced
        """
        prefix = cls.argument_prefix

        group.add_argument(
            '--%s-index' % prefix, action="store",
            dest="%s_index" % prefix,
            help=("Name of the %s root markdown file, can be None" % (
                cls.extension_name)))

    @classmethod
    def add_sources_argument(cls, group, allow_filters=True, prefix=None, add_root_paths=False):
        """
        Subclasses may call this to add sources and source_filters arguments.

        Args:
            group: arparse.ArgumentGroup, the extension argument group
            allow_filters: bool,  Whether the extension wishes to expose a
                source_filters argument.
            prefix: str, arguments have to be namespaced.
        """
        prefix = prefix or cls.argument_prefix

        group.add_argument("--%s-sources" % prefix,
                           action="store", nargs="+",
                           dest="%s_sources" % prefix.replace('-', '_'),
                           help="%s source files to parse" % prefix)

        if allow_filters:
            group.add_argument("--%s-source-filters" % prefix,
                               action="store", nargs="+",
                               dest="%s_source_filters" % prefix.replace(
                                   '-', '_'),
                               help="%s source files to ignore" % prefix)

        if add_root_paths:
            group.add_argument("--%s-source-roots" % prefix,
                               action="store", nargs="+",
                               dest="%s_source_roots" % prefix.replace(
                                   '-', '_'),
                               help="%s source root directories allowing files "
                                    "to be referenced relatively to those" % prefix)

    @classmethod
    def add_path_argument(cls, group, argname, dest=None, help_=None):
        """
        Subclasses may call this to expose a path argument.

        Args:
            group: arparse.ArgumentGroup, the extension argument group
            argname: str, the name of the argument, will be namespaced.
            dest: str, similar to the `dest` argument of
                `argparse.ArgumentParser.add_argument`, will be namespaced.
            help_: str, similar to the `help` argument of
                `argparse.ArgumentParser.add_argument`.
        """
        prefixed = '%s-%s' % (cls.argument_prefix, argname)
        if dest is None:
            dest = prefixed.replace('-', '_')
            final_dest = dest[len(cls.argument_prefix) + 1:]
        else:
            final_dest = dest
            dest = '%s_%s' % (cls.argument_prefix, dest)

        group.add_argument('--%s' % prefixed, action='store',
                           dest=dest, help=help_)
        cls.path_arguments[dest] = final_dest

    @classmethod
    def add_paths_argument(cls, group, argname, dest=None, help_=None):
        """
        Subclasses may call this to expose a paths argument.

        Args:
            group: arparse.ArgumentGroup, the extension argument group
            argname: str, the name of the argument, will be namespaced.
            dest: str, similar to the `dest` argument of
                `argparse.ArgumentParser.add_argument`, will be namespaced.
            help_: str, similar to the `help` argument of
                `argparse.ArgumentParser.add_argument`.
        """
        prefixed = '%s-%s' % (cls.argument_prefix, argname)
        if dest is None:
            dest = prefixed.replace('-', '_')
            final_dest = dest[len(cls.argument_prefix) + 1:]
        else:
            final_dest = dest
            dest = '%s_%s' % (cls.argument_prefix, dest)

        group.add_argument('--%s' % prefixed, action='store', nargs='+',
                           dest=dest, help=help_)
        cls.paths_arguments[dest] = final_dest

    def add_comment(self, comment):
        if comment.toplevel:
            self.__toplevel_comments.add(comment)
        else:
            self.app.database.add_comment(comment)

    def create_symbol(self, *args, **kwargs):
        """
        Extensions that discover and create instances of `symbols.Symbol`
        should do this through this method, as it will keep an index
        of these which can be used when generating a "naive index".

        See `database.Database.create_symbol` for more
        information.

        Args:
            args: see `database.Database.create_symbol`
            kwargs: see `database.Database.create_symbol`

        Returns:
            symbols.Symbol: the created symbol, or `None`.
        """
        if not kwargs.get('project_name'):
            kwargs['project_name'] = self.project.project_name

        sym = self.app.database.create_symbol(*args, **kwargs)
        if sym:
            # pylint: disable=unidiomatic-typecheck
            if type(sym) != Symbol:
                self._created_symbols[sym.filename].add(sym.unique_name)

        return sym

    def rename_symbol(self, unique_name, target):
        sym = self.app.database.rename_symbol(unique_name, target)
        # pylint: disable=unidiomatic-typecheck
        if sym and type(sym) != Symbol:
            self._created_symbols[sym.filename].remove(target)
            self._created_symbols[sym.filename].add(sym.unique_name)

    def _make_formatter(self):
        return Formatter(self)

    def get_possible_path(self, name):
        self.__find_package_root()

        for path in OrderedSet([self.__package_root]) | self.source_roots:
            possible_path = os.path.join(path, name)
            if possible_path in self._get_all_sources():
                return self._get_smart_filename(possible_path)
        return None

    def __find_package_root(self):
        if self.__package_root:
            return

        commonprefix = os.path.commonprefix(
            list(self._get_all_sources()) + list(self.source_roots))
        self.__package_root = os.path.dirname(commonprefix)

    def _get_smart_index_title(self):
        return 'Reference Manual'

    def _get_all_sources(self):
        return self.sources

    def _get_smart_key(self, symbol):
        return symbol.filename

    def _get_smart_filename(self, filename):
        return filename

    def _get_comment_smart_key(self, comment):
        return comment.filename

    def format_page(self, page, link_resolver, output):
        """
        Called by `project.Project.format_page`, to leave full control
        to extensions over the formatting of the pages they are
        responsible of.

        Args:
            page: tree.Page, the page to format.
            link_resolver: links.LinkResolver, object responsible
                for resolving links potentially mentioned in `page`
            output: str, path to the output directory.
        """
        debug('Formatting page %s' % page.link.ref, 'formatting')

        if output:
            actual_output = os.path.join(output,
                                         'html')
            if not os.path.exists(actual_output):
                os.makedirs(actual_output)
        else:
            actual_output = None

        page.format(self.formatter, link_resolver, actual_output)

    def write_out_sitemap(self, opath):
        """
        Banana banana
        """
        if opath not in self.written_out_sitemaps:
            Extension.formatted_sitemap = self.formatter.format_navigation(
                self.app.project)
            if Extension.formatted_sitemap:
                escaped_sitemap = Extension.formatted_sitemap.replace(
                    '\\', '\\\\').replace('"', '\\"').replace('\n', '')
                js_wrapper = 'sitemap_downloaded_cb("%s");' % escaped_sitemap
                with open(opath, 'w') as _:
                    _.write(js_wrapper)

        self.written_out_sitemaps.add(opath)

    # pylint: disable=too-many-locals
    def write_out_page(self, output, page):
        """
        Banana banana
        """
        subpages = OrderedDict({})
        all_pages = self.project.tree.get_pages()
        subpage_names = self.get_subpages_sorted(all_pages, page)
        for pagename in subpage_names:
            proj = self.project.subprojects.get(pagename)

            if not proj:
                cpage = all_pages[pagename]
                sub_formatter = self.project.extensions[
                    cpage.extension_name].formatter
            else:
                cpage = proj.tree.root
                sub_formatter = proj.extensions[cpage.extension_name].formatter

            subpage_link, _ = cpage.link.get_link(self.app.link_resolver)
            prefix = sub_formatter.get_output_folder(cpage)
            if prefix:
                subpage_link = '%s/%s' % (prefix, subpage_link)
            subpages[subpage_link] = cpage

        html_subpages = self.formatter.format_subpages(page, subpages)

        js_dir = os.path.join(output, 'html', 'assets', 'js')
        if not os.path.exists(js_dir):
            os.makedirs(js_dir)
        sm_path = os.path.join(js_dir, 'sitemap.js')
        self.write_out_sitemap(sm_path)

        self.formatter.write_out(page, html_subpages, output)

    def get_subpages_sorted(self, pages, page):
        """Get @page subpages sorted appropriately."""

        sorted_pages = []
        to_sort = []
        for subpage in page.subpages:
            # Do not resort subprojects even if they are
            # 'generated'.
            if pages[subpage].pre_sorted:
                sorted_pages.append(subpage)
            else:
                to_sort.append(subpage)

        return sorted_pages + sorted(
            to_sort, key=lambda p: pages[p].get_title().lower()) class ParsingException(HotdocException):
    """Banana banana"""
    pass class BadInclusionException(HotdocException):
    """Banana banana"""
    pass class HotdocException(Exception):
    """Base Hotdoc exception"""

    def __init__(self, message):
        self.message = message
        super(HotdocException, self).__init__(message) class Link:
    """
    Banana banana
    """
    resolving_title_signal = Signal()

    def __init__(self, ref, title, id_, mandatory=False):
        self.ref = None
        self._title = None
        # Whether the link should be always resolved when true and a warning
        # otherwise
        self.__mandatory = mandatory
        # Ensure we warn only once for a unresolved mandatory link
        self.__warned = False

        if title:
            self._title = str(title)
        if ref:
            self.ref = str(ref)

        self.id_ = id_

    @property
    def title(self):
        """
        Banana banana
        """
        resolved_title = Link.resolving_title_signal(self)
        resolved_title = [elem for elem in resolved_title if elem is not
                          None]
        if resolved_title:
            return str(resolved_title[0])
        return self._title

    @title.setter
    def title(self, value):
        self._title = str(value)

    def get_title(self):
        """
        Convenience wrapper for the `title` property.

        Exists mainly to facilitate the task of the cmark C extension.
        """
        return self.title

    def get_link(self, link_resolver):
        """
        Banana banana
        """
        res = link_resolver.resolving_link_signal(self)

        if not res:
            ref = self.ref
        else:
            ref = res[0] or self.ref

        if self.__mandatory and not self.__warned:
            if not ref:
                warn('mandatory-link-not-found',
                     'Mandatory link %s could not be resolved' % self)
            self.__warned = True

        if not res:
            return ref, None

        if res[1]:
            return ref, dict_to_html_attrs(res[1])
        return ref, None

    def __repr__(self):
        return "Link %s -> %s (%s)" % (self.id_, self.ref, self._title) class GtkDocParser:
    """
    Banana banana
    """

    def __init__(self, project, section_file_matching=True):
        """
        Lifted from
        http://stackoverflow.com/questions/5323703/regex-how-to-match-sequence-of-key-value-pairs-at-end-of-string
        """
        self.kv_regex = re.compile(r'''
                                   [\S]+=
                                   (?:
                                   \s*
                                   (?!\S+=)\S+
                                   )+
                                   ''', re.VERBOSE)
        self.project = project

        tag_validation_regex = r'((?:^|\n)[ \t]*('
        tag_validation_regex += 'returns|Returns|since|Since|deprecated'\
            '|Deprecated|stability|Stability|Return value|topic|Topic'
        for validator in list(project.tag_validators.values()):
            tag_validation_regex += '|%s|%s' % (validator.name,
                                                validator.name.lower())
        tag_validation_regex += '):)'

        self.tag_validation_regex = re.compile(tag_validation_regex)
        self.__section_file_matching = section_file_matching

    def __parse_title(self, source_filename, raw_title):
        if raw_title.startswith('SECTION'):
            section_name = raw_title.split('SECTION:')[1].strip()
            return section_name, [], True

        split = raw_title.split(': ', 1)

        title = split[0].rstrip(':')
        if not len(split) > 1 and not title.endswith(':') and " " in title:
            raise HotdocSourceException(
                message='Unexpected title in gtk-doc comment')

        annotations = []

        if len(split) > 1:
            annotations = self.__parse_annotations(split[1])
        return title, annotations, False

    def __parse_key_value_annotation(self, name, string):
        kvs = self.kv_regex.findall(string)
        kvs = {kv.split('=', 1)[0]: kv.split('=', 1)[1] for kv in kvs}
        return Annotation(name=name, argument=kvs)

    def __parse_annotation(self, string):
        split = string.split()
        name = split[0].strip()
        if len(split) == 1:
            return Annotation(name=name)
        if '=' in split[1]:
            return self.__parse_key_value_annotation(name, split[1])
        return Annotation(name=name, argument=[split[1]])

    def __parse_annotations(self, string):
        parsed_annotations = []
        par_level = 0
        current_annotation = ""
        for _ in string:
            if _ == '(':
                par_level += 1
            elif _ == ')':
                par_level -= 1

            if par_level > 1:
                return []
            if par_level == 1 and _ not in '()':
                current_annotation += _
            elif par_level == 0:
                if _ not in ' \t\n\r():':
                    return []
                if current_annotation:
                    ann = self.__parse_annotation(current_annotation)
                    if ann:
                        parsed_annotations.append(ann)
                    current_annotation = ""
            elif par_level < 0:
                return []

        if par_level != 0:
            return []

        return parsed_annotations

    def __extract_annotations(self, desc):
        split = desc.split(': ', 1)

        if len(split) == 1:
            return desc, []

        annotations = self.__parse_annotations(split[0])
        if not annotations:
            return desc, []

        return split[1].strip(), annotations

    def __parse_parameter(self, name, desc):
        name = name.strip()[1:-1].strip()
        raw_comment = '%s:%s' % (name, desc)
        desc = desc.strip()
        desc, annotations = self.__extract_annotations(desc)
        annotations = {annotation.name: annotation for annotation in
                       annotations}
        return Comment(name=name, annotations=annotations,
                       meta={'description': desc}, raw_comment=raw_comment)

    def __parse_title_and_parameters(self, filename, title_and_params):
        tps = re.split(r'(\n[ \t]*@[\S]+[ \t]*:)', title_and_params)
        title, annotations, is_section = self.__parse_title(filename, tps[0])
        parameters = []
        for name, desc in _grouper(tps[1:], 2):
            n_chars = len(name.strip()) + len(desc)
            param = self.__parse_parameter(name, desc)
            param.line_offset = len(desc.split('\n'))
            parameters.append(param)
            param.initial_col_offset = n_chars - len(param.description)
        return title, parameters, annotations, is_section

    # pylint: disable=no-self-use
    def __parse_since_tag(self, name, desc):
        return Tag(name, desc, value=desc)

    def __parse_topic_tag(self, name, desc):
        return Tag(name, None, value=desc)

    # pylint: disable=no-self-use
    def __parse_deprecated_tag(self, name, desc):
        split = desc.split(':', 1)
        if len(split) == 2 and len(split[0]) > 1:
            value = split[0]
            if ' ' in value:
                value = None
        else:
            value = None

        return Tag(name, desc, value=value)

    # pylint: disable=no-self-use
    def __parse_stability_tag(self, name, desc):
        value = desc.strip().lower()
        if value not in ('private', 'stable', 'unstable'):
            # FIXME warn
            return None
        return Tag(name, desc, value=value)

    # pylint: disable=no-self-use
    def __parse_returns_tag(self, name, desc):
        desc, annotations = self.__extract_annotations(desc)
        annotations = {annotation.name: annotation for annotation in
                       annotations}
        return Tag(name, desc, annotations=annotations)

    # pylint: disable=too-many-return-statements
    def __parse_tag(self, name, desc):
        if name.lower() == "since":
            return self.__parse_since_tag(name, desc)
        if name.lower() == "returns":
            return self.__parse_returns_tag(name, desc)
        if name.lower() == "return value":
            return self.__parse_returns_tag("returns", desc)
        if name.lower() == "stability":
            return self.__parse_stability_tag("stability", desc)
        if name.lower() == "deprecated":
            return self.__parse_deprecated_tag("deprecated", desc)
        if name.lower() == "topic":
            return self.__parse_topic_tag("topic", desc)

        validator = self.project.tag_validators.get(name)
        if not validator:
            warn('gtk-doc', "FIXME no tag validator")
            return None
        if not validator.validate(desc):
            warn('gtk-doc', "invalid value for tag %s : %s" % (name, desc))
            return None
        return Tag(name=name, description=desc, value=desc)

    def __parse_description_and_tags(self, desc_and_tags):
        dts = self.tag_validation_regex.split(desc_and_tags)
        tags = []

        desc = dts[0]
        if len(dts) == 1:
            return desc, tags

        # pylint: disable=unused-variable
        for raw, name, tag_desc in _grouper(dts[1:], 3):
            tag = self.__parse_tag(name.strip(), tag_desc.strip())
            if tag:
                tags.append(tag)
            else:
                desc += '\n%s: %s' % (name, tag_desc)

        return desc, tags

    def __strip_comment(self, comment):
        n_lines = len(comment.split('\n'))
        comment = re.sub(r'^[\W]*\/[\*]+[\W]*', '', comment)
        title_offset = n_lines - len(comment.split('\n'))
        comment = re.sub(r'\*\/[\W]*$', '', comment)
        comment = re.sub(r'\n[ \t]*\*', '\n', comment)
        return comment.strip(), title_offset

    def __validate_c_comment(self, comment):
        return re.match(r'(/\*\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)$',
                        comment) is not None

    def __parse_yaml_comment(self, comment, filename):
        res = {}
        try:
            blocks = yaml.load_all(comment.raw_comment, Loader=yaml.SafeLoader)
            for block in blocks:
                if block:
                    res.update(block)
        except (ConstructorError, yaml.parser.ParserError) as exception:
            warn('invalid-page-metadata',
                 '%s: Invalid metadata: \n%s' % (filename, str(exception)))
        return res

    def __extract_titles_params_and_description(self, comment):
        titleandparams_description = re.split(r'\n[\s]*\n', comment, maxsplit=1)
        title_and_params = titleandparams_description[0]

        title_and_params_lines = title_and_params.split('\n')
        if len(title_and_params_lines) > 1 and not title_and_params_lines[1].strip().startswith('@'):
            return title_and_params_lines[0].rstrip(':'), '\n'.join([l.strip() for l in title_and_params_lines[1:]])

        if len(titleandparams_description) > 1:
            description = titleandparams_description[1]
        else:
            description = None

        return title_and_params, description

    # pylint: disable=too-many-arguments
    # pylint: disable=too-many-locals
    # pylint: disable=unused-argument
    # pylint: disable=too-many-branches
    # pylint: disable=too-many-statements
    def parse_comment(self, comment, filename, lineno, endlineno,
                      include_paths=None, stripped=False):
        """
        Returns a Comment given a string
        """
        if not stripped and not self.__validate_c_comment(comment.strip()):
            return None

        title_offset = 0

        column_offset = 0

        raw_comment = comment
        if not stripped:
            try:
                while comment[column_offset * -1 - 1] != '\n':
                    column_offset += 1
            except IndexError:
                column_offset = 0
            comment, title_offset = self.__strip_comment(comment)

        title_and_params, description = self.__extract_titles_params_and_description(comment)
        try:
            block_name, parameters, annotations, is_section = \
                self.__parse_title_and_parameters(filename, title_and_params)
        except HotdocSourceException as _:
            warn('gtk-doc-bad-syntax',
                 message=_.message,
                 filename=filename,
                 lineno=lineno + title_offset)
            return None

        params_offset = 0
        for param in parameters:
            param.filename = filename
            param.lineno = lineno
            param_offset = param.line_offset
            param.line_offset = title_offset + params_offset + 1
            params_offset += param_offset
            param.col_offset = column_offset

        if not block_name:
            return None

        description_offset = 0
        meta = {}
        tags = []
        if description is not None:
            n_lines = len(comment.split('\n'))
            description_offset = (title_offset + n_lines -
                                  len(description.split('\n')))
            meta['description'], tags = self.__parse_description_and_tags(description)

        actual_parameters = OrderedDict({})
        for param in parameters:
            if is_section:
                cleaned_up_name = param.name.lower().replace('_', '-')
                if cleaned_up_name in ['symbols', 'private-symbols', 'auto-sort', 'sources']:
                    meta.update(self.__parse_yaml_comment(param, filename))
                    if cleaned_up_name == 'sources':
                        sources_paths = [os.path.abspath(os.path.join(os.path.dirname(filename), path)) for path in meta[cleaned_up_name]]
                        meta[cleaned_up_name] = sources_paths
                else:
                    meta[param.name] = param.description
            else:
                actual_parameters[param.name] = param

        annotations = {annotation.name: annotation for annotation in
                       annotations}
        tags = {tag.name.lower(): tag for tag in tags}

        block = Comment(name=block_name, filename=filename, lineno=lineno,
                        endlineno=endlineno,
                        annotations=annotations, params=actual_parameters,
                        tags=tags, raw_comment=raw_comment,
                        meta=meta, toplevel=is_section)
        block.line_offset = description_offset
        block.col_offset = column_offset

        return block def gather_links():
    global GATHERED_GTKDOC_LINKS

    if GATHERED_GTKDOC_LINKS:
        return

    GATHERED_GTKDOC_LINKS = True

    # XDG_DATA_DIRS is preference-ordered, we reverse so that preferred
    # links override less-preferred ones
    for datadir in reversed([XDG_DATA_HOME] + XDG_DATA_DIRS):
        for path in (os.path.join(datadir, 'devhelp', 'books'), os.path.join(datadir, 'gtk-doc', 'html')):
            if not os.path.exists(path):
                info("no gtk doc to gather links from in %s" % path)
                continue

            for node in os.listdir(path):
                dir_ = os.path.join(path, node)
                if os.path.isdir(dir_):
                    if not parse_devhelp_index(dir_):
                        try:
                            parse_sgml_index(dir_)
                        except IOError:
                            pass def search_online_links(resolver, name):
    href = GTKDOC_HREFS.get(name)
    if href:
        return Link(href, name, name)
    return None class GIExtension(Extension):
    extension_name = "gi-extension"
    argument_prefix = "gi"

    def __init__(self, app, project):
        Extension.__init__(self, app, project)

        self.languages = None

        self.__all_sources = None
        self.__current_output_filename = None
        self.__class_gtype_structs = {}
        self.__default_page = DEFAULT_PAGE
        self.__relocated_symbols = set()
        self.created_symbols = set()
        self.__raw_comment_parser = GtkDocParser(self.project)
        self.__c_comment_extractor = CCommentExtractor(
            self, self.__raw_comment_parser)

    # Static vmethod implementations

    @staticmethod
    def add_arguments(parser):
        group = parser.add_argument_group('GObject-introspection extension',
                                          DESCRIPTION)
        GIExtension.add_index_argument(group)
        GIExtension.add_sources_argument(group, allow_filters=False)
        GIExtension.add_sources_argument(
            group, prefix='gi-c', add_root_paths=True)
        group.add_argument("--languages", action="store",
                           nargs='*',
                           help="Languages to translate documentation in %s"
                           ", default is to make all languages" % str(
                               OUTPUT_LANGUAGES))

    @staticmethod
    def get_dependencies():
        dependencies = [ExtDependency('c-extension', is_upstream=True, optional=True)]
        for lang_type in get_language_classes():
            dependencies += lang_type.get_dependencies ()
        return dependencies

    # Chained-up vmethod overrides

    def parse_config(self, config):
        super(GIExtension, self).parse_config(config)
        ALL_GIRS.update({os.path.basename(s): s for s in self.sources})
        self.c_sources = config.get_sources('gi-c')
        self.source_roots = OrderedSet(config.get_paths('gi_c_source_roots'))

        chosen_languages = [l.lower() for l in config.get('languages', [])]
        languages = []
        for lang_type in get_language_classes():
            if lang_type.language_name in chosen_languages:
                languages.append(lang_type())

        if languages:
            self.languages = languages
        else:
            self.languages = [lang_type() for lang_type in get_language_classes()]

        # Make sure C always gets formatted first
        c_language = self.get_language('c')
        if c_language:
            self.languages.remove(c_language)
            self.languages.insert(0, c_language)

        for gir_file in self.sources:
            gir_root = etree.parse(gir_file).getroot()
            cache_nodes(gir_root, ALL_GIRS, self.languages)

    def __formatting_page(self, formatter, page):
        if ALL_GIRS:
            page.meta['extra']['gi-languages'] = [lang.language_name for lang in self.languages]

    def __list_relocated_symbols(self):
        for comment in self._get_toplevel_comments():
            self.__relocated_symbols |= set(comment.meta.get('symbols', []))
            self.__relocated_symbols |= set(comment.meta.get('private-symbols', []))

    def setup(self):
        for ext in self.project.extensions.values():
            ext.formatter.formatting_page_signal.connect(
                self.__formatting_page)
        commonprefix = os.path.commonprefix(list(self._get_all_sources()))
        self.__default_page = os.path.join(os.path.dirname(commonprefix),
                                           DEFAULT_PAGE)

        super(GIExtension, self).setup()

        gather_links ()

        self.app.link_resolver.resolving_link_signal.connect_after(
            self.__translate_link_ref, None)
        if not self.sources:
            return

        self.__scan_comments()
        self.__list_relocated_symbols()
        self.__scan_sources()
        self.__create_macro_symbols()

    def format_page(self, page, link_resolver, output):
        link_resolver.get_link_signal.connect(search_online_links)

        prev_l = None
        page.meta['extra']['gi-languages'] = [lang.language_name for lang in self.languages]
        page.meta['extra']['gi-language'] = self.languages[0].language_name
        Extension.format_page(self, page, link_resolver, output)

        link_resolver.get_link_signal.disconnect(search_online_links)

    def write_out_page(self, output, page):
        prev_l = None
        page.meta['extra']['gi-language'] = 'c'
        Extension.write_out_page(self, output, page)

    def __symbol_is_relocated(self, unique_name, parent_name):
        if unique_name in self.__relocated_symbols:
            return True

        if parent_name in self.__relocated_symbols:
            return True

        return False

    def __check_symbol_is_skipped(self, *args, **kwargs):
        for comment_name in [kwargs.get('unique_name', kwargs.get('display_name')), kwargs.get('parent_name')]:
            comment = self.app.database.get_comment(comment_name)
            if not comment:
                continue

            if 'attributes' in comment.annotations:
                if comment.annotations['attributes'].argument.get('doc.skip') is not None:
                    return True
        return False

    def create_symbol(self, *args, **kwargs):
        args = list(args)
        node = None
        if len(args) > 1:
            node = args.pop(1)
        aliases = kwargs.get('aliases', [])

        unique_name = kwargs.get('unique_name', kwargs.get('display_name'))
        if self.__check_symbol_is_skipped(*args, **kwargs):
            return None

        filename = kwargs.get('filename', self.__default_page)
        if filename == self.__default_page:
            parent_name = kwargs.get('parent_name')
            if not self.__symbol_is_relocated(unique_name, parent_name):
                name = kwargs['display_name']
                kwargs['filename'] = self.__get_symbol_filename(unique_name, node)
                if kwargs.get('filename', self.__default_page) == self.__default_page:
                    self.warn("no-location-indication",
                            "No way to determine where %s should land"
                            " putting it to %s."
                            " Document the symbol for smart indexing to work" % (
                                name, os.path.basename(self.__default_page)))

        res = super(GIExtension, self).create_symbol(*args, **kwargs)

        if res:
            self.created_symbols.add(res.unique_name)

            if node is not None:
                for language in self.languages:
                    language.make_translations(res.unique_name, node)
                    for alias in aliases:
                        language.make_translations(alias, node)

        return res

    # VMethod implementations

    def _make_formatter(self):
        return GIFormatter(self)

    def _get_smart_index_title(self):
        return 'GObject API Reference'

    def _get_smart_filename(self, filename):
        if filename == self.__default_page:
            return filename
        return os.path.splitext(filename)[0]

    def _get_smart_key(self, symbol):
        if symbol.filename == self.__default_page:
            return symbol.filename
        return os.path.splitext(symbol.filename)[0]

    def _get_comment_smart_key(self, comment):
        return os.path.splitext(comment.filename)[0]

    def _get_all_sources(self):
        if not self.__all_sources:
            self.__all_sources = list({
                s for s in self.c_sources if s.endswith('.h') or
                s.replace('.c', '.h') not in self.c_sources})

        return self.__all_sources

    # setup-time private methods
    def __get_filename_from_gir(self, node):
        source_position = node.find('{http://www.gtk.org/introspection/core/1.0}source-position')
        if source_position is not None:
            filename = source_position.attrib['filename']
            for common_path in self.source_roots:
                path = os.path.realpath(os.path.join(common_path, filename))
                if path in self.c_sources:
                    return path
        return None

    def __get_symbol_filename(self, unique_name, node=None):
        if self.__current_output_filename:
            return self.__current_output_filename

        comment = self.app.database.get_comment(unique_name)
        if comment and comment.filename:
            return '%s.h' % os.path.splitext(comment.filename)[0]

        if node is not None:
            filename = self.__get_filename_from_gir(node)
            if filename:
                return filename

        return self.__default_page

    def __get_structure_members(self, node, filename, struct_name, parent_name,
                                in_union=False):
        members = []
        for field in node.getchildren():
            if field.tag in [core_ns('record'), core_ns('union')]:
                if 'name' in field.attrib:
                    struct_name = '%s.%s' % (struct_name, field.attrib['name'])

                new_union = field.tag == core_ns('union')
                union_members = self.__get_structure_members(
                    node=field,
                    filename=filename,
                    struct_name=struct_name,
                    parent_name=parent_name,
                    in_union=in_union or new_union)
                members += union_members
                continue
            elif field.tag != core_ns('field'):
                continue

            children = field.getchildren()
            if not children:
                continue

            if field.attrib.get('private', False):
                continue

            if children[0].tag == core_ns('callback'):
                continue

            type_desc = type_description_from_node(field)

            field_name = field.attrib['name']
            name = "%s.%s" % (struct_name, field_name)

            qtype = QualifiedSymbol(type_tokens=type_desc.type_tokens)
            self.add_attrs(qtype, type_desc=type_desc)

            member = self.create_symbol(
                FieldSymbol,
                member_name=field_name, qtype=qtype,
                filename=filename, display_name=name,
                unique_name=name, parent_name=parent_name)

            if member:
                self.add_attrs(member, type_desc=type_desc, in_union=in_union)
                members.append(member)

        return members

    def __find_structure_pagename(self, node, unique_name, is_class):
        filename = self.__get_symbol_filename(unique_name, node)
        if filename != self.__default_page:
            return filename

        if self.__symbol_is_relocated(unique_name, None):
            return self.__default_page

        if not is_class:
            sym = self.__class_gtype_structs.get(node.attrib['name'])
            if sym and sym.filename:
                return sym.filename

        filenames = []
        for cnode in node:
            cunique_name = get_symbol_names(cnode)[0]
            if not cunique_name:
                continue
            fname = self.__get_symbol_filename(cunique_name, cnode)
            if fname != self.__default_page:
                if cnode.tag == core_ns('constructor'):
                    filenames.insert(0, fname)
                else:
                    filenames.append(fname)

        unique_filenames = list(OrderedSet(filenames))
        if not filenames:
            # Did not find any symbols, trying to can get information
            # about the class structure linked to that object class.
            nextnode = node.getnext()
            name = node.attrib['name']
            if nextnode is not None and nextnode.tag == core_ns('record'):
                nextnode_classfor = nextnode.attrib.get(glib_ns(
                    'is-gtype-struct-for'))
                if nextnode_classfor == name:
                    nunique_name = get_symbol_names(nextnode)[0]
                    filename = self.__get_symbol_filename(nunique_name)

            if filename == self.__default_page:
                self.warn("no-location-indication",
                          "No way to determine where %s should land"
                          " putting it to %s."
                          " Document the symbol for smart indexing to work" % (
                              unique_name, os.path.basename(filename)))
        else:
            filename = unique_filenames[0]
            if len(unique_filenames) > 1:
                self.warn("no-location-indication",
                          " Going wild here to determine where %s needs to land"
                          " as we could detect the following possibilities: %s."
                          % (unique_name, unique_filenames))
            else:
                self.debug(" No class comment for %s determined that it should"
                           " land into %s with all other class related documentation."
                           % (unique_name, os.path.basename(filename)))

        return filename

    def __sort_parameters(self, symbol, retval, parameters):
        in_parameters = []
        out_parameters = []

        for i, param in enumerate(parameters):
            if isinstance(symbol, MethodSymbol) and i == 0:
                continue

            direction = self.get_attr(param, 'direction')

            if direction == 'in' or direction == 'inout':
                in_parameters.append(param)
            if direction == 'out' or direction == 'inout':
                out_parameters.append(param)

        self.add_attrs(symbol, parameters=in_parameters)

    def __create_parameter_symbol(self, gi_parameter):
        param_name = gi_parameter.attrib['name']

        type_desc = type_description_from_node(gi_parameter)
        direction = gi_parameter.attrib.get('direction')
        if direction is None:
            direction = 'in'

        res = ParameterSymbol(argname=param_name,
                              type_tokens=type_desc.type_tokens)
        self.add_attrs(res, type_desc=type_desc, direction=direction)

        return res, direction

    def __create_return_value_symbol(self, gi_retval, out_parameters):
        type_desc = type_description_from_node(gi_retval)

        if type_desc.gi_name == 'none':
            ret_item = None
        else:
            ret_item = ReturnItemSymbol(type_tokens=type_desc.type_tokens)
            self.add_attrs(ret_item, type_desc=type_desc)

        res = [ret_item]

        for out_param in out_parameters:
            ret_item = ReturnItemSymbol(type_tokens=out_param.input_tokens,
                                        name=out_param.argname)
            self.add_attrs(ret_item, type_desc=self.get_attr(
                out_param, 'type_desc'))

            res.append(ret_item)

        return res

    def __create_parameters_and_retval(self, node):
        gi_parameters = node.find(
            '{http://www.gtk.org/introspection/core/1.0}parameters')

        if gi_parameters is None:
            instance_param = None
            gi_parameters = []
        else:
            instance_param = \
                gi_parameters.find(
                    '{http://www.gtk.org/introspection/core/1.0}instance-parameter')
            gi_parameters = gi_parameters.findall(
                '{http://www.gtk.org/introspection/core/1.0}parameter')

        parameters = []
        closure_indices = set()
        destroy_indices = set()

        out_parameters = []
        for gi_parameter in gi_parameters:
            param, direction = self.__create_parameter_symbol(gi_parameter)
            parameters.append(param)
            if 'destroy' in gi_parameter.attrib:
                destroy_indices.add(int(gi_parameter.attrib['destroy']))
            if 'closure' in gi_parameter.attrib:
                closure_indices.add(int(gi_parameter.attrib['closure']))

            if direction != 'in':
                out_parameters.append(param)

        for idx in destroy_indices:
            param = parameters[idx]
            self.add_attrs(param, is_destroy=True)

        for idx in closure_indices:
            param = parameters[idx]
            self.add_attrs(param, is_closure=True)

        # We add our instance parameter last in order not to mess with
        # the destroy indices
        if instance_param is not None:
            param, direction = self.__create_parameter_symbol(instance_param)
            parameters.insert(0, param)

        if node.attrib.get('throws') == '1':
            type_desc = SymbolTypeDesc([Link(None, 'GError', 'GError'), '*', '*'], 'GLib.Error', 'GError**', 0)
            param = ParameterSymbol(argname='error', type_tokens=type_desc.type_tokens)
            parameters.append(param)
            self.add_attrs(param, type_desc=type_desc, direction='out')

        retval = node.find(
            '{http://www.gtk.org/introspection/core/1.0}return-value')
        retval = self.__create_return_value_symbol(retval, out_parameters)

        return (parameters, retval)

    def __create_callback_symbol(self, node, parent_name):
        name = node.attrib[c_ns('type')]
        parameters, retval = self.__create_parameters_and_retval(node)

        filename = self.__get_symbol_filename(name, node)
        sym = self.create_symbol(
            CallbackSymbol, node, parameters=parameters,
            return_value=retval, display_name=name,
            filename=filename, parent_name=parent_name)

        if sym:
            self.__sort_parameters(sym, retval, parameters)

        return sym

    def __create_enum_symbol(self, node, spelling=None):
        name = node.attrib[c_ns('type')]

        filename = self.__get_symbol_filename(name, node)
        psymbol = self.app.database.get_symbol(name)
        is_genum = node.attrib.get(glib_ns('type'), False)
        if psymbol:
            if not isinstance(psymbol, EnumSymbol):
                self.warn('symbol-redefined', "EnumSymbol(unique_name=%s, filename=%s, project=%s)"
                    " has already been defined: %s" % (name, filename, self.project.project_name, psymbol))
                return None

            psymbol_is_genum = self.get_attr(psymbol, "is_genum")
            if psymbol_is_genum is None or psymbol_is_genum == is_genum:
                self.warn('symbol-redefined', "EnumSymbol(unique_name=%s, filename=%s, project=%s)"
                     " has already been defined: %s" % (name, filename, self.project.project_name, psymbol))
                return None

            # FIXME Handle nicks in GEnums
            return None

        # FIXME Working around https://gitlab.gnome.org/GNOME/glib/issues/1756
        members = OrderedDict()
        for field in node.findall(core_ns('member')):
            unique_name = field.attrib[c_ns('identifier')]
            if unique_name in members:
                continue
            member = self.create_symbol(
                EnumMemberSymbol, field, display_name=unique_name,
                filename=filename, parent_name=name)

            if member:
                member.enum_value = field.attrib['value']
                members[unique_name] = member

        res = self.create_symbol(
            EnumSymbol, node, members=list(members.values()),
            anonymous=False, display_name=name,
            filename=filename, raw_text=None)

        if res:
            self.add_attrs(res, is_genum=is_genum)

            for cnode in node:
                parent_name = res.unique_name
                if cnode.tag == core_ns('function'):
                    parent_name = None
                self.__scan_node(cnode, parent_name=parent_name)

        return res

    def __create_signal_symbol(self, node, parent_name):
        unique_name, name, klass_name = get_symbol_names(node)

        parameters, retval = self.__create_parameters_and_retval(node)

        parent_node = node.getparent()
        parent_gi_name = get_gi_name(parent_node)
        parent_link = Link(None, parent_name, parent_name)

        instance_param = ParameterSymbol(
            argname='self', type_tokens=[parent_link, '*'])
        type_desc = SymbolTypeDesc([], parent_gi_name, None, 0)
        self.add_attrs(instance_param, type_desc=type_desc, direction='in')
        parameters.insert(0, instance_param)

        udata_link = Link(None, 'gpointer', 'gpointer')
        udata_param = ParameterSymbol(
            argname='user_data', type_tokens=[udata_link])
        type_desc = SymbolTypeDesc([], 'gpointer', None, 0)
        self.add_attrs(udata_param, type_desc=type_desc, direction='in', is_closure=True)
        parameters.append(udata_param)

        if node.attrib.get('action') == '1':
            typ = ActionSignalSymbol
            for sym in parameters + retval:
                if sym is not None:
                    self.add_attrs(sym, action=True)
        else:
            typ = SignalSymbol

        res = self.create_symbol(typ, node,
            parameters=parameters, return_value=retval,
            display_name=name, unique_name=unique_name,
            filename=self.__get_symbol_filename(klass_name, node),
            parent_name=parent_name)

        if res:
            klass = self.app.database.get_symbol(klass_name)
            assert(klass)
            klass.signals.append(res)

            flags = []

            when = node.attrib.get('when')
            if when == "first":
                flags.append(RunFirstFlag())
            elif when == "last":
                flags.append(RunLastFlag())
            elif when == "cleanup":
                flags.append(RunCleanupFlag())

            no_hooks = node.attrib.get('no-hooks')
            if no_hooks == '1':
                flags.append(NoHooksFlag())

            action = node.attrib.get('action')
            if action == '1':
                flags.append(ActionFlag())

            # This is incorrect, it's not yet format time
            extra_content = self.formatter._format_flags(flags)
            res.extension_contents['Flags'] = extra_content

            self.__sort_parameters(res, retval, parameters)

        return res

    def __create_property_symbol(self, node, parent_name):
        unique_name, name, klass_name = get_symbol_names(node)

        type_desc = type_description_from_node(node)
        type_ = QualifiedSymbol(type_tokens=type_desc.type_tokens)
        self.add_attrs(type_, type_desc=type_desc)

        flags = []
        writable = node.attrib.get('writable')
        construct = node.attrib.get('construct')
        construct_only = node.attrib.get('construct-only')

        flags.append(ReadableFlag())
        if writable == '1':
            flags.append(WritableFlag())
        if construct_only == '1':
            flags.append(ConstructOnlyFlag())
        elif construct == '1':
            flags.append(ConstructFlag())

        res = self.create_symbol(PropertySymbol, node,
                                        prop_type=type_,
                                        display_name=name,
                                        unique_name=unique_name,
                                        filename=self.__get_symbol_filename(
                                            klass_name, node),
                                        parent_name=parent_name)

        if res:
            klass = self.app.database.get_symbol(klass_name)
            assert(klass)
            klass.properties.append(res)

            extra_content = self.formatter._format_flags(flags)
            res.extension_contents['Flags'] = extra_content

        return res

    def __create_vfunc_symbol(self, node, parent_name):
        klass_node = node.getparent()
        ns = klass_node.getparent()
        gtype_struct = klass_node.attrib.get(glib_ns('type-struct'))

        klass_comment = self.app.database.get_comment('%s%s' %
                                                      (ns.attrib[c_ns('identifier-prefixes')], gtype_struct))

        unique_name, name, klass_name = get_symbol_names(node)

        # Virtual methods are documented in the class comment
        if klass_comment:
            param_comment = klass_comment.params.get(name)
            if (param_comment):
                self.add_comment(
                    Comment(name=unique_name,
                            meta={'description': param_comment.description},
                            annotations=param_comment.annotations))

        parameters, retval = self.__create_parameters_and_retval(node)
        symbol = self.create_symbol(VFunctionSymbol, node,
                                           parameters=parameters,
                                           return_value=retval, display_name=name,
                                           unique_name=unique_name,
                                           filename=self.__get_symbol_filename(
                                               klass_name, node),
                                           parent_name=parent_name,
                                           aliases=[unique_name.replace('::', '.')])

        if symbol:
            klass = self.app.database.get_symbol(klass_name)
            assert(klass)
            klass.vfuncs.append(symbol)

            self.__sort_parameters(symbol, retval, parameters)

        return symbol

    def __create_alias_symbol(self, node, gi_name, parent_name):
        name = get_symbol_names(node)[0]

        type_desc = type_description_from_node(node)
        aliased_type = QualifiedSymbol(type_tokens=type_desc.type_tokens)
        self.add_attrs(aliased_type, type_desc=type_desc)
        filename = self.__get_symbol_filename(name, node)

        alias_link = [l for l in type_desc.type_tokens if isinstance(l, Link)]
        for lang in self.languages:
            if lang.language_name == 'c':
                continue
            fund_type = lang.get_fundamental(type_desc.c_name)
            if fund_type:
                # The alias name is now considered as a FUNDAMENTAL type.
                lang.add_fundamental (name, fund_type)
            else:
                if alias_link:
                    lang.add_alias_type(name, aliased_type)

        return self.create_symbol(AliasSymbol, node,
                                         aliased_type=aliased_type,
                                         display_name=name,
                                         filename=filename,
                                         parent_name=parent_name)

    def __create_structure(self, symbol_type, node, gi_name):
        unique_name, unused_name, klass_name = get_symbol_names(node)
        # Hidding class private structures
        if node.attrib.get('disguised') == '1' and \
                unique_name.endswith(('Priv', 'Private')):
            self.debug('%s seems to be a GObject class private structure, hiding it.'
                       % (unique_name))
            return

        filename = self.__find_structure_pagename(node, unique_name,
                                                  symbol_type == GIClassSymbol)

        self.__current_output_filename = filename
        parent_name = unique_name
        if symbol_type == GIClassSymbol:
            res = self.__create_class_symbol(node, gi_name,
                                             klass_name,
                                             unique_name,
                                             filename)
            class_struct = node.attrib.get(glib_ns('type-struct'))
            if class_struct:
                self.__class_gtype_structs[class_struct] = res
        elif symbol_type == GIStructSymbol:
            # If we are working with a Class structure,
            class_symbol = self.__class_gtype_structs.get(node.attrib['name'])
            if class_symbol:
                parent_name = class_symbol.unique_name

                # Class struct should never be renderer on their own,
                # smart_key will lookup the value in that dict
                self.__class_gtype_structs[unique_name] = class_symbol
            res = self.__create_struct_symbol(node, unique_name, filename,
                                              class_symbol.unique_name if class_symbol else None)

            if class_symbol:
                class_symbol.class_struct_symbol = res
        else:  # Interface
            res = self.__create_interface_symbol(node, unique_name, filename)
            class_struct = node.attrib.get(glib_ns('type-struct'))
            if class_struct:
                self.__class_gtype_structs[class_struct] = res

        for cnode in node:
            if cnode.tag in [core_ns('record'), core_ns('union')]:
                continue
            self.__scan_node(cnode, parent_name=parent_name)

        self.__current_output_filename = None

        return res

    def __create_class_symbol(self, node, gi_name, klass_name,
                              unique_name, filename):
        hierarchy = get_klass_parents(gi_name)
        children = get_klass_children(gi_name)

        members = self.__get_structure_members(node, filename,
                                               klass_name,
                                               unique_name)

        return self.create_symbol(GIClassSymbol, node,
                                        hierarchy=hierarchy,
                                        children=children,
                                        display_name=klass_name,
                                        unique_name=unique_name,
                                        filename=filename,
                                        members=members,
                                        parent_name=unique_name)

    def __create_struct_symbol(self, node, struct_name, filename,
                               parent_name):


        members = self.__get_structure_members(
            node, filename, struct_name,
            parent_name=parent_name or struct_name)

        if not parent_name:
            return self.create_symbol(GIStructSymbol, node,
                                             display_name=struct_name,
                                             unique_name=struct_name,
                                             anonymous=False,
                                             filename=filename,
                                             members=members,
                                             parent_name=struct_name)
        else:
            res = StructSymbol()
            res.display_name = struct_name
            res.unique_name = struct_name
            res.filename = filename
            res.members = members
            return res

    def __create_interface_symbol(self, node, unique_name, filename):
        return self.create_symbol(GIInterfaceSymbol, node,
                                         display_name=unique_name,
                                         unique_name=unique_name,
                                         parent_name=unique_name,
                                         filename=filename)

    def __create_function_symbol(self, node, parent_name):
        unique_name, name, klass_name = get_symbol_names(node)

        gi_params, retval = self.__create_parameters_and_retval(node)

        if node.tag.endswith('method'):
            if node.getparent().attrib.get(glib_ns('is-gtype-struct-for')):
                type_ = ClassMethodSymbol
            else:
                type_ = MethodSymbol
        elif node.tag == core_ns('constructor'):
            type_ = ConstructorSymbol
        else:
            type_ = FunctionSymbol

        func = self.create_symbol(type_, node,
                                         parameters=gi_params,
                                         return_value=retval,
                                         display_name=name,
                                         unique_name=unique_name,
                                         throws='throws' in node.attrib,
                                         filename=self.__get_symbol_filename(
                                             name, node),
                                         parent_name=parent_name)

        if func:
            if klass_name is not None:
                klass = self.app.database.get_symbol(klass_name)
                assert(klass)
                klass.methods.append(func)

            self.__sort_parameters(func, func.return_value, func.parameters)

        return func

    def __scan_comments(self):
        comment_parser = GtkDocParser(self.project)
        block = self.__raw_comment_parser.parse_comment(DEFAULT_PAGE_COMMENT,
                                                        DEFAULT_PAGE, 0, 0)
        self.add_comment(block)

        self.__c_comment_extractor.parse_comments(self.c_sources)

    def __create_macro_symbols(self):
        self.__c_comment_extractor.create_macro_symbols(SMART_FILTERS, self.c_sources)

    def __scan_node(self, node, parent_name=None):
        gi_name = get_gi_name(node)

        if 'moved-to' in node.attrib:
            return False

        if node.tag == core_ns('class'):
            self.__create_structure(GIClassSymbol, node, gi_name)
        elif node.tag in (core_ns('function'), core_ns('method'), core_ns('constructor')):
            self.__create_function_symbol(node, parent_name)
        elif node.tag == core_ns('virtual-method'):
            self.__create_vfunc_symbol(node, parent_name)
        elif node.tag == core_ns('property'):
            self.__create_property_symbol(node, parent_name)
        elif node.tag == glib_ns('signal'):
            self.__create_signal_symbol(node, parent_name)
        elif node.tag == core_ns('alias'):
            self.__create_alias_symbol(node, gi_name, parent_name)
        elif node.tag == core_ns('record'):
            self.__create_structure(GIStructSymbol, node, gi_name)
        elif node.tag == core_ns('interface'):
            self.__create_structure(GIInterfaceSymbol, node, gi_name)
        elif node.tag == core_ns('enumeration'):
            self.__create_enum_symbol(node)
        elif node.tag == core_ns('bitfield'):
            self.__create_enum_symbol(node)
        elif node.tag == core_ns('callback'):
            self.__create_callback_symbol(node, parent_name)
        elif node.tag == core_ns('field'):
            pass
        else:
            for cnode in node:
                self.__scan_node(cnode)

    def __scan_sources(self):
        for gir_file in self.sources:
            root = etree.parse(gir_file).getroot()
            self.__scan_node(root)

    # Format-time private methods
    def __translate_ref(self, link, language):
        fund = language.get_fundamental(link.id_)
        if fund:
            return fund.ref

        aliased_link = language.get_alias_link(link.id_)
        if aliased_link:
            return self.__translate_ref(aliased_link, language)

        page = self.project.get_page_for_symbol(link.id_)
        if page:
            if page.extension_name != self.extension_name:
                return None
            return link.ref

        return GTKDOC_HREFS.get(link.id_)

    def __translate_title(self, link, language):
        fund = language.get_fundamental(link.id_)
        if fund:
            return fund._title

        if language.language_name != 'c' and not is_introspectable(link.id_, language):
            return link._title + ' (not introspectable)'

        aliased_link = language.get_alias_link(link.id_)
        if aliased_link:
            return self.__translate_link_title(aliased_link, language)

        translated = language.get_translation(link.id_)
        if translated:
            return translated

        if language.language_name == 'c' and link.id_ in GTKDOC_HREFS:
            return link.id_
        return None

    def __translate_link_ref(self, link, language):
        if not language:
            actual_language = self.get_language('c')
        else:
            actual_language = language

        ref = self.__translate_ref(link, actual_language)
        if ref is None:
            return None

        extra_attrs = {}
        if not language:
            for lang in self.languages:
                if lang.language_name == 'c':
                    continue

                extra_attrs['data-gi-href-' + lang.language_name] = self.__translate_ref(
                    link, lang) or ref
                extra_attrs['data-gi-title-' + lang.language_name] = self.__translate_title(
                    link, lang)
        return ref, extra_attrs

    def __translate_link_title(self, link, language):
        return self.__translate_title(link, language)

    def get_language(self, language):
        for lang in self.languages:
            if lang.language_name == language:
                return lang

        return None

    def get_languages(self):
        return self.languages

    def setup_language(self, language, prev_l):
        if prev_l:
            Link.resolving_title_signal.disconnect(self.__translate_link_title,
                                                   prev_l)
            self.app.link_resolver.resolving_link_signal.disconnect(
                self.__translate_link_ref, prev_l)
        else:
            self.app.link_resolver.resolving_link_signal.disconnect(
                self.__translate_link_ref, None)

        if language:
            Link.resolving_title_signal.connect(self.__translate_link_title,
                                                language)
            self.app.link_resolver.resolving_link_signal.connect(
                self.__translate_link_ref, language)
        else:
            self.app.link_resolver.resolving_link_signal.connect_after(
                self.__translate_link_ref, None) @staticmethod
def info(message, domain):
    """Log simple info"""
    if domain in Logger._ignored_domains:
        return

    Logger._log(None, message, INFO, domain) @staticmethod
def warn(code, message, **kwargs):
    """
    Call this to store a warning in the journal.

    Will raise if `Logger.fatal_warnings` is set to True and
    `Logger.raise_on_fatal_warnings` as well.
    """

    if code in Logger._ignored_codes:
        return

    assert code in Logger._warning_code_to_exception
    exc_type, domain = Logger._warning_code_to_exception[code]

    if domain in Logger._ignored_domains:
        return

    level = WARNING
    if Logger.fatal_warnings:
        level = ERROR

    exc = exc_type(message, **kwargs)

    Logger._log(code, exc.message, level, domain)

    if Logger.fatal_warnings:
        if Logger.raise_on_fatal_warnings:
            raise exc
        else:
            Logger.n_fatal_warnings += 1 class Logger(Configurable):

    """Subclasses can inherit from this class to report recoverable errors."""

    _error_code_to_exception = defaultdict()
    _domain_codes = defaultdict(set)
    _warning_code_to_exception = defaultdict()
    journal = []
    fatal_warnings = False
    raise_on_fatal_warnings = False
    _ignored_codes = set()
    _ignored_domains = set()
    _last_checkpoint = 0
    _verbosity = WARNING
    silent = False
    n_fatal_warnings = 0

    @staticmethod
    def register_error_code(code, exception_type, domain='core'):
        """Register a new error code"""
        Logger._error_code_to_exception[code] = (exception_type, domain)
        Logger._domain_codes[domain].add(code)

    @staticmethod
    def register_warning_code(code, exception_type, domain='core'):
        """Register a new warning code"""
        Logger._warning_code_to_exception[code] = (exception_type, domain)
        Logger._domain_codes[domain].add(code)

    @staticmethod
    def _log(code, message, level, domain):
        """Call this to add an entry in the journal"""
        entry = LogEntry(level, domain, code, message)
        Logger.journal.append(entry)

        if Logger.silent:
            return

        if level >= Logger._verbosity:
            _print_entry(entry)

    @staticmethod
    def error(code, message, **kwargs):
        """Call this to raise an exception and have it stored in the journal"""
        assert code in Logger._error_code_to_exception
        exc_type, domain = Logger._error_code_to_exception[code]
        exc = exc_type(message, **kwargs)
        Logger._log(code, exc.message, ERROR, domain)
        raise exc

    @staticmethod
    def warn(code, message, **kwargs):
        """
        Call this to store a warning in the journal.

        Will raise if `Logger.fatal_warnings` is set to True and
        `Logger.raise_on_fatal_warnings` as well.
        """

        if code in Logger._ignored_codes:
            return

        assert code in Logger._warning_code_to_exception
        exc_type, domain = Logger._warning_code_to_exception[code]

        if domain in Logger._ignored_domains:
            return

        level = WARNING
        if Logger.fatal_warnings:
            level = ERROR

        exc = exc_type(message, **kwargs)

        Logger._log(code, exc.message, level, domain)

        if Logger.fatal_warnings:
            if Logger.raise_on_fatal_warnings:
                raise exc
            else:
                Logger.n_fatal_warnings += 1

    @staticmethod
    def debug(message, domain):
        """Log debugging information"""
        if domain in Logger._ignored_domains:
            return

        Logger._log(None, message, DEBUG, domain)

    @staticmethod
    def info(message, domain):
        """Log simple info"""
        if domain in Logger._ignored_domains:
            return

        Logger._log(None, message, INFO, domain)

    @staticmethod
    def add_ignored_code(code):
        """Add a code to ignore. Errors cannot be ignored."""
        Logger._ignored_codes.add(code)

    @staticmethod
    def add_ignored_domain(code):
        """Add a domain to ignore. Errors cannot be ignored."""
        Logger._ignored_domains.add(code)

    @staticmethod
    def checkpoint():
        """Add a checkpoint"""
        Logger._last_checkpoint = len(Logger.journal)

    @staticmethod
    def since_checkpoint():
        """Get journal since last checkpoint"""
        return Logger.journal[Logger._last_checkpoint:]

    @staticmethod
    def get_issues():
        """Get actual issues in the journal."""
        issues = []
        for entry in Logger.journal:
            if entry.level >= WARNING:
                issues.append(entry)
        return issues

    @staticmethod
    def reset():
        """Resets Logger to its initial state"""
        Logger.journal = []
        Logger.fatal_warnings = False
        Logger.raise_on_fatal_warnings = False
        Logger._ignored_codes = set()
        Logger._ignored_domains = set()
        Logger._verbosity = 2
        Logger._last_checkpoint = 0
        Logger.n_fatal_warnings = 0

    @staticmethod
    def add_arguments(parser):
        """Banana banana
        """
        group = parser.add_argument_group(
            'Logger', 'logging options')
        group.add_argument("--verbose", '-v', action="count",
                           dest="verbose",
                           help="Turn on verbosity, -vv for debug")
        group.add_argument("--fatal-warnings", action="store_true",
                           dest="fatal_warnings", help="Make warnings fatal")

    @staticmethod
    def set_verbosity(verbosity):
        """Banana banana
        """
        Logger._verbosity = min(max(0, WARNING - verbosity), 2)
        debug("Verbosity set to %d" % (WARNING - Logger._verbosity), 'logging')

    @staticmethod
    def parse_config(config):
        Logger._verbosity = max(0, Logger._verbosity - (
            config.get('verbose') or 0))

        Logger.fatal_warnings = bool(config.get("fatal_warnings")) @staticmethod
def debug(message, domain):
    """Log debugging information"""
    if domain in Logger._ignored_domains:
        return

    Logger._log(None, message, DEBUG, domain)
import os, sys, linecache, pkgconfig, glob, subprocess, shutil
from hotdoc.extensions.c.clang import cindex
from ctypes import *
from fnmatch import fnmatch
from hotdoc.core import inclusions
from hotdoc.core.extension import Extension
from hotdoc.core.exceptions import ParsingException, BadInclusionException, HotdocException
from hotdoc.core.symbols import *
from hotdoc.core.comment import comment_from_tag
from hotdoc.core.links import Link
from hotdoc.parsers.gtk_doc import GtkDocParser, gather_links, search_online_links
from hotdoc.extensions.gi.gi_extension import GIExtension
from hotdoc.extensions.c.utils import CCommentExtractor
from hotdoc.utils.loggable import (info as core_info, warn, Logger,
    debug as core_debug)

    def __parse_file (self, filename, tu, full_scan):
        if filename in self.parsed:
            return

        self.parsed.add (filename)

        if filename not in self.filenames:
            return

        debug('scanning %s' % filename)

        start = tu.get_location (filename, 0)
        end = tu.get_location (filename, int(os.path.getsize(filename)))
        extent = cindex.SourceRange.from_locations (start, end)
        cursors = self.__get_cursors(tu, extent)

        # Happens with empty source files
        if cursors is None:
            return

        if filename in self.filenames:
            self.__create_symbols (cursors, tu)

    # That's the fastest way of obtaining our ast nodes for a given filename
    def __get_cursors (self, tu, extent):
        tokens_memory = POINTER(cindex.Token)()
        tokens_count = c_uint()

        cindex.conf.lib.clang_tokenize(tu, extent, byref(tokens_memory),
                byref(tokens_count))

        count = int(tokens_count.value)

        if count < 1:
            return

        cursors = (cindex.Cursor * count)()
        cindex.conf.lib.clang_annotateTokens (tu, tokens_memory, tokens_count,
                cursors)

        return cursors

    def __create_symbols(self, nodes, tu):
        for node in nodes:
            node._tu = tu

            # This is dubious, needed to parse G_DECLARE_FINAL_TYPE
            # investigate further (fortunately this doesn't seem to
            # significantly impact performance ( ~ 5% )
            if node.kind == cindex.CursorKind.TYPE_REF:
                node = node.get_definition()
                if not node:
                    continue

                if not str(node.location.file) in self.filenames:
                    continue

            if node.spelling in self.symbols or node.spelling in self.__renamed_symbols:
                continue

            sym = None
            func_dec = self.__getFunctionDeclNode(node)
            if func_dec and func_dec.spelling not in self.symbols:
                sym = self.__create_function_symbol(func_dec)
            elif node.kind == cindex.CursorKind.VAR_DECL:
                sym = self.__create_exported_variable_symbol (node)
            elif node.kind == cindex.CursorKind.TYPEDEF_DECL:
                sym = self.__create_typedef_symbol (node)
            elif node.kind == cindex.CursorKind.STRUCT_DECL and node.spelling:
                sym = self.__create_struct_symbol(node)
            elif node.kind == cindex.CursorKind.ENUM_DECL and node.spelling:
                sym = self.__create_enum_symbol(node)

            if sym is not None:
                self.symbols[sym.unique_name] = sym

            self.__create_symbols(node.get_children(), tu)

    def __getFunctionDeclNode(self, node):
        if not node.location.file:
            return None
        elif node.location.file.name.endswith(".h"):
            if node.kind == cindex.CursorKind.FUNCTION_DECL:
                return node
            else:
                return None

        if node.kind != cindex.CursorKind.COMPOUND_STMT:
            return None

        if node.semantic_parent and node.semantic_parent.kind == cindex.CursorKind.FUNCTION_DECL:
            return node.semantic_parent

        return None

    def __apply_qualifiers (self, type_, tokens):
        if type_.is_const_qualified():
            tokens.append ('const ')
        if type_.is_restrict_qualified():
            tokens.append ('restrict ')
        if type_.is_volatile_qualified():
            tokens.append ('volatile ')

    def make_c_style_type_name (self, type_):
        tokens = []
        while (type_.kind == cindex.TypeKind.POINTER):
            self.__apply_qualifiers(type_, tokens)
            tokens.append ('*')
            type_ = type_.get_pointee()

        if type_.kind == cindex.TypeKind.TYPEDEF:
            d = type_.get_declaration ()
            link = Link (None, d.displayname, d.displayname)

            tokens.append (link)
            self.__apply_qualifiers(type_, tokens)
        elif type_.kind in (cindex.TypeKind.UNEXPOSED, cindex.TypeKind.ELABORATED):
            d = type_.get_declaration()
            if d.spelling:

                tokens.append(Link(None, d.displayname, d.displayname))