class AbstractCommon(object):
    """Class grouping methods common to AbstractSearch and
    AbstractTypeCommon."""

    @classmethod
    def _typeconvert(cls, text, returntype):
        """Parse the text to the given returntype.

        Parameters
        ----------
        text : str
           Text to convert
        returntype : str
            Parse the text to this output datatype. One of
            `string`, `float`, `integer`, `date`, `datetime`, `boolean`.

        Returns
        -------
        str or float or int or bool or datetime.date or datetime.datetime
            Returns the parsed text converted to the type described by
            `returntype`.

        """
        if returntype == 'string':
            def typeconvert(x):
                return u'' + (x.strip())
        elif returntype == 'integer':
            def typeconvert(x):
                return int(x)
        elif returntype == 'float':
            def typeconvert(x):
                return float(x)
        elif returntype == 'date':
            def typeconvert(x):
                # Patch for Zulu-time issue of geoserver for WFS 1.1.0
                if x.endswith('Z'):
                    return datetime.datetime.strptime(x, '%Y-%m-%dZ').date() \
                        + datetime.timedelta(days=1)
                else:
                    return datetime.datetime.strptime(x, '%Y-%m-%d').date()
        elif returntype == 'datetime':
            def typeconvert(x):
                if x.endswith('Z'):
                    return datetime.datetime.strptime(
                        x, '%Y-%m-%dT%H:%M:%SZ').date() \
                        + datetime.timedelta(days=1)
                else:
                    return datetime.datetime.strptime(
                        x.split('.')[0], '%Y-%m-%dT%H:%M:%S')
        elif returntype == 'boolean':
            def typeconvert(x):
                return strtobool(x) == 1
        else:
            def typeconvert(x):
                return x

        return typeconvert(text) class AbstractField(dict):
    """Abstract base class for pydov field definitions. Not to be
    instantiated directly."""

    def __init__(self, name, source, datatype, **kwargs):
        """Initialise a field.

        Parameters
        ----------
        name : str
            Name of this field in the return dataframe.
        source : one of 'wfs', 'xml', 'custom'
            Source of this field.
        datatype : one of 'string', 'integer', 'float', 'date', 'datetime' \
                   or 'boolean'
            Datatype of the values of this field in the return dataframe.

        """
        super(AbstractField, self).__init__(**kwargs)
        self.__setitem__('name', name)
        self.__setitem__('source', source)
        self.__setitem__('type', datatype) def get_dov_xml(url, session=None):
    """Request the XML from the remote DOV webservices and return it.

    Parameters
    ----------
    url : str
        URL of the DOV object to download.
    session : requests.Session
        Session to use to perform HTTP requests for data. Defaults to None,
        which means a new session will be created for each request.

    Returns
    -------
    xml : bytes
        The raw XML data of this DOV object as bytes.

    """
    response = HookRunner.execute_inject_xml_response(url)

    if response is None:
        response = get_remote_url(url, session)

    HookRunner.execute_xml_received(url, response)

    return response def parse_dov_xml(xml_data):
    """Parse the given XML data into an ElementTree.

    Parameters
    ----------
    xml_data : bytes
        The raw XML data of a DOV object as bytes.

    Returns
    -------
    tree : etree.ElementTree
        Parsed XML tree of the DOV object.

    """
    try:
        parser = etree.XMLParser(ns_clean=True, recover=True)
    except TypeError:
        parser = etree.XMLParser()

    try:
        tree = etree.fromstring(xml_data, parser=parser)
        return tree
    except Exception:
        raise XmlParseError("Failed to parse XML record.") class RemoteFetchError(DOVError):
    """General error while fetching remote data from DOV webservices."""
    pass class XmlFetchWarning(DOVWarning):
    """Emitted when an XML document fails to be fetched from the DOV
    webservice, resulting in an incomplete dataframe.""" class LocalSessionThreadPool:
    """Thread pool of LocalSessionThreads used to perform XML I/O operations
    in parallel.
    """

    def __init__(self, workers=4):
        """Initialisation.

        Set up the pool and start all workers.

        Parameters
        ----------
        workers : int, optional
            Number of worker threads to use, defaults to 4.
        """
        self.workers = []
        self.input_queue = Queue(maxsize=100)
        self.result_queue = Queue()

        for i in range(workers):
            self.workers.append(LocalSessionThread(self.input_queue))

        self._start()

    def _start(self):
        """Start all worker threads. """
        for w in self.workers:
            w.start()

    def stop(self):
        """Stop all worker threads. """
        for w in self.workers:
            w.stop()

    def execute(self, fn, args):
        """Execute the given function with its arguments in a worker thread.

        This will add the job to the queue and will not wait for the result.
        Use join() to retrieve the result.

        Parameters
        ----------
        fn : function
            Function to execute.
        args : tuple
            Arguments that will be passed to the function.
        """
        r = WorkerResult()
        self.input_queue.put((fn, args, r))
        self.result_queue.put(r)

    def join(self):
        """Wait for all the jobs to be executed and return the results of all
        jobs in a list.

        Yields
        ------
        WorkerResult
            Results of the executed functions in the order they were
            submitted.
        """
        self.input_queue.join()
        self.stop()

        while not self.result_queue.empty():
            yield self.result_queue.get() class InvalidFieldError(DOVError):
    """Error that occurs when using a field outside its scope."""
    pass class XmlParseError(DOVError):
    """Error that occurs when the parsing of a DOV XML document failed."""
    pass class XmlParseWarning(DOVWarning):
    """Emitted when the failure to parse an XML document results in
    an incomplete dataframe.""" class HookRunner(object):
    """Class for executing registered hooks."""
    @staticmethod
    def __execute_read(hook_name, args):
        """Execute the read hook with the given name for all registered hooks.

        Parameters
        ----------
        hook_name : str
            Name of the hook function to execute.
        args : list
            List of arguments to pass to the hook function call.

        """
        for h in pydov.hooks.get_read_hooks():
            getattr(h, hook_name)(*args)

    @staticmethod
    def __execute_inject(hook_name, args):
        """Execute the inject hook with given name for all registered hooks
        and return the last non-null result.

        Parameters
        ----------
        hook_name : str
            Name of the hook function to execute.
        args : list
            List of arguments to pass to the hook function call.

        Returns
        -------
        object or bytes or None
            Returns the last non-null result of the execution of the inject
            hook. If all inject hooks return None, return None as well.

        """
        result = None
        for h in pydov.hooks.get_inject_hooks():
            r = getattr(h, hook_name)(*args)
            if r is not None:
                result = r
        return result

    @staticmethod
    def execute_meta_received(url, response):
        """Execute the meta_received method for all registered hooks.

        Parameters
        ----------
        url : str
            URL of the metadata request.
        response : bytes
            The raw response as received from resolving the URL.

        """
        HookRunner.__execute_read('meta_received', [url, response])

    @staticmethod
    def execute_wfs_search_init(typename):
        """Execute the wfs_search_init method for all registered hooks.

        Parameters
        ----------
        typename : str
            The typename (layername) of the WFS service used for searching.

        """
        HookRunner.__execute_read('wfs_search_init', [typename])

    @staticmethod
    def execute_wfs_search_result(number_of_results):
        """Execute the wfs_search_result method for all registered hooks.

        Parameters
        ----------
        number_of_results : int
            The number of features returned by the WFS search.

        """
        HookRunner.__execute_read('wfs_search_result', [number_of_results])

    @staticmethod
    def execute_wfs_search_result_received(query, features):
        """Execute the wfs_search_result_received method for all registered
        hooks.

        Parameters
        ----------
        query : etree.ElementTree
            The WFS GetFeature request sent to the WFS server.
        features : etree.ElementTree
            The WFS GetFeature response containings the features.

        """
        HookRunner.__execute_read('wfs_search_result_received', [
            query, features])

    @staticmethod
    def execute_xml_received(pkey_object, xml):
        """Execute the xml_received method for all registered hooks.

        Parameters
        ----------
        pkey_object : str
            Permanent key of the retrieved object.
        xml : bytes
            The raw XML data of this DOV object as bytes.

        """
        HookRunner.__execute_read('xml_received', [pkey_object, xml])

    @staticmethod
    def execute_xml_cache_hit(pkey_object):
        """Execute the xml_cache_hit method for all registered hooks.

        Parameters
        ----------
        pkey_object : str
            Permanent key of the requested object.
        """
        HookRunner.__execute_read('xml_cache_hit', [pkey_object])

    @staticmethod
    def execute_xml_stale_hit(pkey_object):
        """Execute the xml_stale_hit method for all registered hooks.

        Parameters
        ----------
        pkey_object : str
            Permanent key of the requested object.

        """
        HookRunner.__execute_read('xml_stale_hit', [pkey_object])

    @staticmethod
    def execute_xml_fetch_error(pkey_object):
        """Execute the xml_fetch_error method for all registered hooks.

        Parameters
        ----------
        pkey_object : str
            Permanent key of the requested object.

        """
        HookRunner.__execute_read('xml_fetch_error', [pkey_object])

    @staticmethod
    def execute_xml_downloaded(pkey_object):
        """Execute the xml_downloaded method for all registered hooks.

        Parameters
        ----------
        pkey_object : str
            Permanent key of the requested object.

        """
        HookRunner.__execute_read('xml_downloaded', [pkey_object])

    @staticmethod
    def execute_inject_meta_response(url):
        """Execute the inject_meta_response method for all registered hooks.

        Parameters
        ----------
        url : str
            URL of the metadata request.

        Returns
        -------
        bytes, optional
            The response to use in favor of resolving the URL. Returns None if
            this inject hook is unused.

        """
        return HookRunner.__execute_inject(
            'inject_meta_response', [url])

    @staticmethod
    def execute_inject_wfs_getfeature_response(query):
        """Execute the inject_wfs_getfeature_response method for all
        registered hooks.

        Parameters
        ----------
        query : etree.ElementTree
            The WFS GetFeature request sent to the WFS server.

        Returns
        -------
        xml: bytes, optional
            The GetFeature response to use in favor of resolving the URL.
            Returns None if this inject hook is unused.

        """
        return HookRunner.__execute_inject(
            'inject_wfs_getfeature_response', [query])

    @staticmethod
    def execute_inject_xml_response(pkey_object):
        """Execute the inject_xml_response method for all registered hooks.

        Parameters
        ----------
        query : etree.ElementTree
            The WFS GetFeature request sent to the WFS server.

        Returns
        -------
        xml : bytes, optional
            The XML response to use in favor of resolving the URL. Returns
            None if this inject hook is unused.

        """
        return HookRunner.__execute_inject(
            'inject_xml_response', [pkey_object])
import types
import warnings
import numpy as np
import pydov
from collections import OrderedDict
from owslib.etree import etree
from pydov.search.abstract import AbstractCommon
from pydov.types.fields import AbstractField
from pydov.util.dovutil import get_dov_xml, parse_dov_xml
from pydov.util.errors import RemoteFetchError, XmlFetchWarning
from pydov.util.net import LocalSessionThreadPool
from ..util.errors import InvalidFieldError, XmlParseError, XmlParseWarning
from ..util.hooks import HookRunner
# -*- coding: utf-8 -*-
"""Module containing the base DOV data types."""






class AbstractTypeCommon(AbstractCommon):
    """Class grouping methods common to AbstractDovType and
    AbstractDovSubType.

    Attributes
    ----------
    fields : list of pydov.types.fields.AbstractField
        List of fields of this type.

    """

    fields = []

    @classmethod
    def _parse(cls, func, xpath, namespace, returntype):
        """Parse the result of an XML path function, stripping the namespace
        and adding type conversion.

        Parameters
        ----------
        func : reference to function
           Function to call.
        xpath : str
            XML path of the element, used as the argument of `func`.
        namespace : str or None
            Namespace to be added to each item in the `xpath`. None to use
            the xpath as is.
        returntype : str
            Parse the text found with `func` to this output datatype. One of
            `string`, `float`, `integer`, `date`, `datetime`, `boolean`.

        Returns
        -------
        str or float or int or bool or datetime.date or datetime.datetime
            Returns the parsed value of the output from calling `func` on
            `xpath`, converted to the type described by `returntype`.

        """
        if namespace is not None:
            ns = '{{{}}}'.format(namespace)
            text = func('./' + ns + ('/' + ns).join(xpath.split('/')))
        else:
            text = func('./' + xpath.lstrip('/'))

        if text is None:
            return np.nan

        return cls._typeconvert(text, returntype)

    @classmethod
    def extend_fields(cls, extra_fields):
        """Extend the fields of this type with given extra fields and return
        the new fieldset.

        Parameters
        ----------
        extra_fields : list of pydov.types.fields.AbstractField
            Extra fields to be appended to the existing fields of this type.

        Returns
        -------
        list of pydov.types.fields.AbstractField
            List of the existing fields of this type, extended with the
            extra fields supplied in extra_fields.

        """
        fields = list(cls.fields)
        fields.extend(extra_fields)
        return fields


class AbstractDovSubType(AbstractTypeCommon):
    """Abstract DOV type grouping fields and methods common to all DOV
    subtypes. Not to be instantiated or used directly.

    Attributes
    ----------
    rootpath : str
        XPath expression of the root element of this subtype. Should return
        all elements of this subtype.

    Raises
    ------
    RuntimeError
        When the defined fields of this type are invalid.

    """

    rootpath = None

    _UNRESOLVED = "{UNRESOLVED}"

    def __init__(self):
        """Initialisation.

        Parameters
        ----------
        name : str
            The name associated with this subtype.

        """
        for f in self.fields:

            if not isinstance(f, AbstractField):