#!/usr/bin/env python
# encoding: utf-8

# Copyright (c) 2016, <NAME> (www.karlsruhe.de)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

'''
OParl client library.

This package provides tools for easily retrieving information from an
OParl_ server. OParl is a standard interface for publishing information
about parliaments and their work.

.. _OParl: https://oparl.org

The main tool of the library is the ``Object`` class and its subclasses.
They provide wrappers for the various OParl classes (``Body``,
``Person``, ...) and support lazy loading and automatic conversion of
OParl data types (e.g. ``date-time``) to Python types
(``datetime.datetime``).

You will typically start by loading a single object from an OParl server
using the ``from_id`` function::

    import oparl
    system = oparl.from_id('https://politik-bei-uns.de/oparl')

If you've already got OParl JSON data you can also use ``from_json``.

Instances of ``Object`` and its subclasses support a read-only dict-
interface::

    for body in system['body']:
        print(body['name'])

This library tries to be as compatible with OParl 1.0 as possible but
does not enforce strict compliance. In some cases non-compliant server
behavior that has been seen "in the wild" is supported. These cases
trigger a ``SpecificationWarning``. If invalid values are encountered
during auto-conversion (e.g. illegal date strings) then a
``ContentWarning`` is issued and conversion is skipped.

By default, HTTPS certificates are verified. You can disable that
verification by setting ``VERIFY_HTTPS`` to ``False``.

The libraries logger (``log``) doesn't have a handler attached to it by
default, but may come in handy during development.
'''

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import collections
import json
import logging
import sys
from warnings import warn

import dateutil.parser
import requests
import six
from unidecode import unidecode


log = logging.getLogger(__name__)


__version__ = '0.1.1'


# Official OParl 1.0 schema URI
SCHEMA_URI = 'https://schema.oparl.org/1.0'

# Should HTTPS certificates be verified?
VERIFY_HTTPS = True


class Warning(UserWarning):
    '''
    Base class for OParl warnings.
    '''
    pass


class SpecificationWarning(Warning):
    '''
    Warning that something did not comply with the OParl specification.
    '''
    pass


class ContentWarning(Warning):
    '''
    Warning that some content was malformed.

    This occurs, for example, if a date string does not contain a valid
    date.
    '''
    pass


def _class_from_type_uri(uri):
    '''
    Convert a type URI to a class.
    '''
    parts = uri.rsplit('/', 1)
    if len(parts) != 2:
        raise ValueError('Invalid type URI "{uri}".'.format(uri=uri))
    if parts[0] != SCHEMA_URI:
        warn(('Invalid schema URI "{schema_uri}" in type URI "{type_uri}" '
             + '(should be "{oparl_uri}").').format(schema_uri=parts[0],
             type_uri=uri, oparl_uri=SCHEMA_URI), SpecificationWarning)
    import oparl.objects
    try:
        return getattr(sys.modules['oparl.objects'], parts[1])
    except AttributeError:
        raise ValueError('Unknown type "{name}" in type URI "{uri}".'.format(
                         name=parts[1], uri=uri))


def _get_json(url):
    '''
    Download JSON from an URL and parse it.
    '''
    log.debug('Downloading {url}'.format(url=url))
    r = requests.get(url, verify=VERIFY_HTTPS)
    r.raise_for_status()
    return r.json()


def from_json(data):
    '''
    Initialize an OParl object from JSON.

    ``data`` is raw OParl JSON data (either as a Python data
    structure or as a string).

    Returns an appropriate subclass of ``Object`` initialized using
    the given data.
    '''
    if isinstance(data, six.string_types):
        data = json.loads(data)
    if not 'id' in data:
        raise ValueError('JSON data does not have an `id` field.')
    if not 'type' in data:
        raise ValueError('JSON data does not have a `type` field.')
    cls = _class_from_type_uri(data['type'])
    obj = cls(data['id'], data['type'])
    obj._init_from_json(data)
    return obj


def from_id(id):
    '''
    Initialize an OParl object from its ID (URL).

    The object's data is downloaded and parsed. The resulting object is
    returned.
    '''
    return from_json(_get_json(id))


def _lazy(id, type):
    '''
    Create a lazy OParl object.

    The returned object doesn't contain any data (aside from the ID and
    the type). The data is downloaded once it is required
    '''
    cls = _class_from_type_uri(type)
    return cls(id, type)


def _is_url(value):
    '''
    Check if a value looks like an URL.
    '''
    return isinstance(value, six.string_types) and value.startswith('http')


class ExternalObjectList(collections.abc.Sequence):
    '''
    (Lazy) list of OParl objects.

    OParl has "external object lists". These lists contain OParl objects
    and are accessible via URL but are not true OParl objects themselves
    (they don't have their own fields, IDs, etc.). They exist to allow
    a pagination of large lists.

    This class lazily wraps such a list. Pages are retrieved only once
    further items are requested by indexing (``my_list[34]``) or by
    iterating over the list.

    To prevent storing large lists completely in memory only the
    currently requested page is stored. Random access to the list may
    therefore lead to repeated downloads of the same page. To download
    the complete list use ``my_list = list(my_list)``.

    Using ``len`` on an instance of this class returns the currently
    known number of items in the list. This number may increase once
    more items are requested. In OParl, the only portable way to get the
    total number of available items is to download the complete list.
    This can be done by using ``my_list = list(my_list)``.

    Due to its dynamic nature, instances of this class only support non-
    negative integer indices. In particular, slicing and negative
    indices are not supported
    '''
    # The only mandatory link between sub-pages of a paginated list in
    # OParl is ``next``. While OParl offers several other such links
    # (e.g. ``last``) these are optional. Similarly, OParl doesn't
    # require the server to mention the total number of items.

    def __init__(self, url):
        self.url = url
        self._data = []
        self._current_page_index = None
        self._page_urls = [(0, url)]
        self._offset = 0
        self._len = 0

    def __len__(self):
        return self._len

    def _load_page_for_index(self, i):
        '''
        Loads the sub-page which contains an index.

        The page which contains the index ``i`` is loaded. If ``i`` is
        larger than the number of items in the list then an
        ``IndexError`` is raised.
        '''
        j = 0
        while True:
            if j == len(self._page_urls) - 1:
                if self._page_urls[j][1] is None:
                    raise IndexError()
                self._load_page(j)
            if self._page_urls[j + 1][0] > i:
                self._load_page(j)
                return
            j += 1

    def _load_page(self, page_index):
        '''
        Load a sub-page.

        Sub-pages must be loaded incrementally, i.e. page ``i`` must be
        loaded before page ``i + 1``.
        '''
        if page_index == self._current_page_index:
            return
        log.debug('Getting page {index} for list {url}'.format(
                  index=page_index, url=self.url))
        self._offset, url = self._page_urls[page_index]
        if url is None:
            raise IndexError()
        data = _get_json(url)
        self._data = [from_json(obj) for obj in data['data']]
        next_offset = self._offset + len(self._data)
        self._len = max(self._len, next_offset)
        if page_index == len(self._page_urls) - 1:
            next_url = data['links'].get('next')
            self._page_urls.append((next_offset, next_url))
        self._current_page_index = page_index

    def __getitem__(self, i):
        if not isinstance(i, int) or i < 0:
            raise IndexError('Only non-negative integer indices are '
                             + 'supported.')
        self._load_page_for_index(i)
        return self._data[i - self._offset]

    def __repr__(self):
        return unidecode('<OParl ExternalObjectList {url}>'.format(
                         url=self.url))


class Object(collections.abc.Mapping):
    '''
    Base class for all OParl objects.

    The subclasses of this class (e.g. ``Person``) represent the various
    OParl objects. The classes are dict-like read-only wrappers around
    the objects' JSON data.

    An object can be initialized using the ``from_id`` and ``from_json``
    functions. The classes' constructors are not intended for public use.

    Non-trivial fields defined by the OParl standard (e.g. fields of
    type ``date-time``) are automatically converted to an appropriate
    Python object. Nested objects referenced via an URL are loaded
    lazily, i.e. their full data is only downloaded once it is required.
    You can check whether that has happened using the ``loaded``
    attribute and force it via the ``load`` method.
    '''
    # Fields that have type 'Date'. Their values are automatically
    # parsed from the string representation.
    _DATE_FIELDS = []

    # Fields that have type 'DateTime'. Their values are automatically
    # parsed from the string representation.
    _DATETIME_FIELDS = ['created', 'modified']

    # Fields that contain another OParl objects. The object is
    # automatically parsed from its JSON representation.
    _OBJECT_FIELDS = []

    # Fields that contain a list of other OParl objects. The objects are
    # automatically parsed from their JSON representation.
    _OBJECT_LIST_FIELDS = []

    # Fields that contain a reference (URL) to another OParl object.
    # Their value is automatically replaced by a corresponding lazy
    # OParl object. This dict maps field names to the OParl type URIs of
    # the referenced object.
    _REFERENCE_FIELDS = {}

    # Fields that contain a list of references (URLs) to other OParl
    # objects. The values in the lists are automatically replaced by
    # corresponding lazy OParl objects. This dict maps field names to
    # the OParl type URIs of the referenced objects.
    _REFERENCE_LIST_FIELDS = {}

    # Fields that contain an URL to a a list of OParl objects. Their
    # values are automatically wrapped in ``ExternalObjectList``
    # instances.
    _EXTERNAL_LIST_FIELDS = []

    def __init__(self, id, type):
        '''
        Private constructor.

        Use ``from_id`` or ``from_json`` instead.
        '''
        self._data = {'id': id, 'type': type}
        self.loaded = False

    def load(self, force=False):
        '''
        Load the object's data if it hasn't been loaded, yet.

        If ``force`` is true then the data is always downloaded.
        '''
        if self.loaded and not force:
            return
        self._init_from_json(_get_json(self._data['id']))

    def __getitem__(self, key):
        try:
            return self._data[key]
        except KeyError:
            if not self.loaded:
                self.load()
                return self._data[key]
            raise

    def __iter__(self):
        self.load()
        return self._data.__iter__()

    def __len__(self):
        self.load()
        return len(self._data)

    def _convert_value(self, field, value):
        '''
        Convert a JSON value to its proper type.

        If the field has a special type (as defined by the class'
        ``_DATE_FIELDS``, ``_DATETIME_FIELDS``, etc.) then the value is
        converted accordingly. Otherwise the value is returned
        unchanged.
        '''
        if field in self._DATE_FIELDS:
            return self._parse_date(value, field)
        if field in self._DATETIME_FIELDS:
            return self._parse_datetime(value, field)
        if field in self._OBJECT_FIELDS:
            return self._parse_object(value, field)
        if field in self._OBJECT_LIST_FIELDS:
            return self._parse_object_list(value, field)
        if field in self._REFERENCE_FIELDS:
            return self._parse_reference(value, field)
        if field in self._REFERENCE_LIST_FIELDS:
            return self._parse_reference_list(value, field)
        if field in self._EXTERNAL_LIST_FIELDS:
            return ExternalObjectList(value)
        return value

    def _ensure_list(self, value, field):
        if (not isinstance(value, collections.Sequence)
                or isinstance(value, six.string_types)):
            warn(('In object "{id}": Field "{field}" of type "{type}" must '
                 + 'contain a list, but a non-list value was found '
                 + 'instead.').format(id=self._data['id'], field=field,
                 type=self._data['type']), SpecificationWarning)
            value = [value]
        return value

    def _parse_date(self, value, field):
        try:
            return dateutil.parser.parse(value).date()
        except ValueError as e:
            warn(('In object "{id}": Field "{field}" contains an invalid '
                 + 'date string ("{value}"): {error}').format(
                 id=self._data['id'], field=field, value=value, error=e),
                 ContentWarning)
            return value

    def _parse_datetime(self, value, field):
        try:
            return dateutil.parser.parse(value)
        except ValueError as e:
            warn(('In object "{id}": Field "{field}" contains an invalid '
                 + 'date-time string ("{value}"): {error}').format(
                 id=self._data['id'], field=field, value=value, error=e),
                 ContentWarning)
            return value

    def _parse_object(self, value, field):
        if _is_url(value):
            warn(('In object "{id}": Field "{field}" of type "{type}" '
                 + 'must contain an object, but a URL ("{url}") was found '
                 + 'instead.').format(id=self._data['id'], field=field,
                 type=self._data['type'], url=value), SpecificationWarning)
            return from_id(value)
        else:
            return from_json(value)

    def _parse_object_list(self, value, field):
        values = []
        for v in self._ensure_list(value, field):
            if _is_url(v):
                warn(('In object "{id}": The list in field "{field}" of '
                     + 'type "{type}" must contain objects, but an URL '
                     + '("{url}") was found instead.').format(
                     id=self._data['id'], field=field, type=self._data['type'],
                     url=v), SpecificationWarning)
                values.append(from_id(v))
            else:
                values.append(from_json(v))
        return values

    def _parse_reference(self, value, field):
        if isinstance(value, dict):
            warn(('In object "{id}": Field "{field}" of type "{type}" '
                 + 'must contain an object reference (URL), but an object '
                 + 'was found instead.').format(id=self._data['id'],
                 field=field, type=self._data['type']), SpecificationWarning)
            return from_json(value)
        else:
            return _lazy(value, self._REFERENCE_FIELDS[field])

    def _parse_reference_list(self, value, field):
        obj_type = self._REFERENCE_LIST_FIELDS[field]
        values = []
        for v in self._ensure_list(value, field):
            if isinstance(v, dict):
                warn(('In object "{id}": The list in field "{field}" of '
                     + 'type "{type}" must contain references (URLs), but '
                     + 'an object was found instead.').format(
                     id=self._data['id'], field=field,
                     type=self._data['type']), SpecificationWarning)
                values.append(from_json(v))
            else:
                values.append(_lazy(v, obj_type))
        return values

    def _init_from_json(self, data):
        '''
        Init the object from (parsed) JSON data.
        '''
        if not 'id' in data:
            raise ValueError('JSON data does not have an `id` field.')
        if data['id'] != self._data['id']:
            warn(('Initializing object "{id}" from JSON data which contains a '
                 + 'different ID ("{json_id}").').format(id=self._data['id'],
                 json_id=data['id']), ContentWarning)
        try:
            type =  data['type']
        except KeyError:
            raise ValueError('JSON data does not have a `type` field.')
        cls = _class_from_type_uri(type)
        if cls != self.__class__:
            raise ValueError(('Type from JSON data ({type}) does not match '
                             + 'instance type.').format(type=type))
        for key, value in six.iteritems(data):
            self._data[key] = self._convert_value(key, value)
        self.loaded = True

    def __repr__(self):
        s = '<oparl:{cls}'.format(cls=self.__class__.__name__)
        if not self.loaded:
            s += '?'
        s += ' {id}'.format(id=self._data['id'])
        name = self._data.get('shortname', self._data.get('name'))
        if name:
            s += ' ({name})'.format(name=name)
        s += '>'
        return unidecode(s)


#!/usr/bin/env python
# encoding: utf-8

# Copyright (c) 2016, <NAME> (www.karlsruhe.de)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

'''
OParl client library.

This package provides tools for easily retrieving information from an
OParl_ server. OParl is a standard interface for publishing information
about parliaments and their work.

.. _OParl: https://oparl.org

The main tool of the library is the ``Object`` class and its subclasses.
They provide wrappers for the various OParl classes (``Body``,
``Person``, ...) and support lazy loading and automatic conversion of
OParl data types (e.g. ``date-time``) to Python types
(``datetime.datetime``).

You will typically start by loading a single object from an OParl server
using the ``from_id`` function::

    import oparl
    system = oparl.from_id('https://politik-bei-uns.de/oparl')

If you've already got OParl JSON data you can also use ``from_json``.

Instances of ``Object`` and its subclasses support a read-only dict-
interface::

    for body in system['body']:
        print(body['name'])

This library tries to be as compatible with OParl 1.0 as possible but
does not enforce strict compliance. In some cases non-compliant server
behavior that has been seen "in the wild" is supported. These cases
trigger a ``SpecificationWarning``. If invalid values are encountered
during auto-conversion (e.g. illegal date strings) then a
``ContentWarning`` is issued and conversion is skipped.

By default, HTTPS certificates are verified. You can disable that
verification by setting ``VERIFY_HTTPS`` to ``False``.

The libraries logger (``log``) doesn't have a handler attached to it by
default, but may come in handy during development.
'''

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import collections
import json
import logging
import sys
from warnings import warn

import dateutil.parser
import requests
import six
from unidecode import unidecode


log = logging.getLogger(__name__)


__version__ = '0.1.1'


# Official OParl 1.0 schema URI
SCHEMA_URI = 'https://schema.oparl.org/1.0'

# Should HTTPS certificates be verified?
VERIFY_HTTPS = True


class Warning(UserWarning):
    '''
    Base class for OParl warnings.
    '''
    pass


class SpecificationWarning(Warning):
    '''
    Warning that something did not comply with the OParl specification.
    '''
    pass


class ContentWarning(Warning):
    '''
    Warning that some content was malformed.

    This occurs, for example, if a date string does not contain a valid
    date.
    '''
    pass


def _class_from_type_uri(uri):
    '''
    Convert a type URI to a class.
    '''
    parts = uri.rsplit('/', 1)
    if len(parts) != 2:
        raise ValueError('Invalid type URI "{uri}".'.format(uri=uri))
    if parts[0] != SCHEMA_URI:
        warn(('Invalid schema URI "{schema_uri}" in type URI "{type_uri}" '
             + '(should be "{oparl_uri}").').format(schema_uri=parts[0],
             type_uri=uri, oparl_uri=SCHEMA_URI), SpecificationWarning)
    import oparl.objects
    try:
        return getattr(sys.modules['oparl.objects'], parts[1])
    except AttributeError:
        raise ValueError('Unknown type "{name}" in type URI "{uri}".'.format(
                         name=parts[1], uri=uri))


def _get_json(url):
    '''
    Download JSON from an URL and parse it.
    '''
    log.debug('Downloading {url}'.format(url=url))
    r = requests.get(url, verify=VERIFY_HTTPS)
    r.raise_for_status()
    return r.json()


def from_json(data):
    '''
    Initialize an OParl object from JSON.

    ``data`` is raw OParl JSON data (either as a Python data
    structure or as a string).

    Returns an appropriate subclass of ``Object`` initialized using
    the given data.
    '''
    if isinstance(data, six.string_types):
        data = json.loads(data)
    if not 'id' in data:
        raise ValueError('JSON data does not have an `id` field.')
    if not 'type' in data:
        raise ValueError('JSON data does not have a `type` field.')
    cls = _class_from_type_uri(data['type'])
    obj = cls(data['id'], data['type'])
    obj._init_from_json(data)
    return obj


def from_id(id):
    '''
    Initialize an OParl object from its ID (URL).

    The object's data is downloaded and parsed. The resulting object is
    returned.
    '''
    return from_json(_get_json(id))


def _lazy(id, type):
    '''
    Create a lazy OParl object.

    The returned object doesn't contain any data (aside from the ID and
    the type). The data is downloaded once it is required
    '''
    cls = _class_from_type_uri(type)
    return cls(id, type)


def _is_url(value):
    '''
    Check if a value looks like an URL.
    '''
    return isinstance(value, six.string_types) and value.startswith('http')


class ExternalObjectList(collections.abc.Sequence):
    '''
    (Lazy) list of OParl objects.

    OParl has "external object lists". These lists contain OParl objects
    and are accessible via URL but are not true OParl objects themselves
    (they don't have their own fields, IDs, etc.). They exist to allow
    a pagination of large lists.

    This class lazily wraps such a list. Pages are retrieved only once
    further items are requested by indexing (``my_list[34]``) or by
    iterating over the list.

    To prevent storing large lists completely in memory only the
    currently requested page is stored. Random access to the list may
    therefore lead to repeated downloads of the same page. To download
    the complete list use ``my_list = list(my_list)``.

    Using ``len`` on an instance of this class returns the currently
    known number of items in the list. This number may increase once
    more items are requested. In OParl, the only portable way to get the
    total number of available items is to download the complete list.
    This can be done by using ``my_list = list(my_list)``.

    Due to its dynamic nature, instances of this class only support non-
    negative integer indices. In particular, slicing and negative
    indices are not supported
    '''
    # The only mandatory link between sub-pages of a paginated list in
    # OParl is ``next``. While OParl offers several other such links
    # (e.g. ``last``) these are optional. Similarly, OParl doesn't
    # require the server to mention the total number of items.

    def __init__(self, url):
        self.url = url
        self._data = []
        self._current_page_index = None
        self._page_urls = [(0, url)]
        self._offset = 0
        self._len = 0

    def __len__(self):
        return self._len

    def _load_page_for_index(self, i):
        '''
        Loads the sub-page which contains an index.

        The page which contains the index ``i`` is loaded. If ``i`` is
        larger than the number of items in the list then an
        ``IndexError`` is raised.
        '''
        j = 0
        while True:
            if j == len(self._page_urls) - 1:
                if self._page_urls[j][1] is None:
                    raise IndexError()
                self._load_page(j)
            if self._page_urls[j + 1][0] > i:
                self._load_page(j)
                return
            j += 1

    def _load_page(self, page_index):
        '''
        Load a sub-page.

        Sub-pages must be loaded incrementally, i.e. page ``i`` must be
        loaded before page ``i + 1``.
        '''
        if page_index == self._current_page_index:
            return
        log.debug('Getting page {index} for list {url}'.format(
                  index=page_index, url=self.url))
        self._offset, url = self._page_urls[page_index]
        if url is None:
            raise IndexError()
        data = _get_json(url)
        self._data = [from_json(obj) for obj in data['data']]
        next_offset = self._offset + len(self._data)
        self._len = max(self._len, next_offset)
        if page_index == len(self._page_urls) - 1:
            next_url = data['links'].get('next')
            self._page_urls.append((next_offset, next_url))
        self._current_page_index = page_index

    def __getitem__(self, i):
        if not isinstance(i, int) or i < 0:
            raise IndexError('Only non-negative integer indices are '
                             + 'supported.')
        self._load_page_for_index(i)
        return self._data[i - self._offset]

    def __repr__(self):
        return unidecode('<OParl ExternalObjectList {url}>'.format(
                         url=self.url))


class Object(collections.abc.Mapping):
    '''
    Base class for all OParl objects.

    The subclasses of this class (e.g. ``Person``) represent the various
    OParl objects. The classes are dict-like read-only wrappers around
    the objects' JSON data.

    An object can be initialized using the ``from_id`` and ``from_json``
    functions. The classes' constructors are not intended for public use.

    Non-trivial fields defined by the OParl standard (e.g. fields of
    type ``date-time``) are automatically converted to an appropriate
    Python object. Nested objects referenced via an URL are loaded
    lazily, i.e. their full data is only downloaded once it is required.
    You can check whether that has happened using the ``loaded``
    attribute and force it via the ``load`` method.
    '''
    # Fields that have type 'Date'. Their values are automatically
    # parsed from the string representation.
    _DATE_FIELDS = []

    # Fields that have type 'DateTime'. Their values are automatically
    # parsed from the string representation.
    _DATETIME_FIELDS = ['created', 'modified']

    # Fields that contain another OParl objects. The object is
    # automatically parsed from its JSON representation.
    _OBJECT_FIELDS = []

    # Fields that contain a list of other OParl objects. The objects are
    # automatically parsed from their JSON representation.
    _OBJECT_LIST_FIELDS = []

    # Fields that contain a reference (URL) to another OParl object.
    # Their value is automatically replaced by a corresponding lazy
    # OParl object. This dict maps field names to the OParl type URIs of
    # the referenced object.
    _REFERENCE_FIELDS = {}

    # Fields that contain a list of references (URLs) to other OParl
    # objects. The values in the lists are automatically replaced by
    # corresponding lazy OParl objects. This dict maps field names to
    # the OParl type URIs of the referenced objects.
    _REFERENCE_LIST_FIELDS = {}

    # Fields that contain an URL to a a list of OParl objects. Their
    # values are automatically wrapped in ``ExternalObjectList``
    # instances.
    _EXTERNAL_LIST_FIELDS = []

    def __init__(self, id, type):
        '''
        Private constructor.

        Use ``from_id`` or ``from_json`` instead.
        '''
        self._data = {'id': id, 'type': type}
        self.loaded = False

    def load(self, force=False):
        '''
        Load the object's data if it hasn't been loaded, yet.

        If ``force`` is true then the data is always downloaded.
        '''
        if self.loaded and not force:
            return
        self._init_from_json(_get_json(self._data['id']))

    def __getitem__(self, key):
        try:
            return self._data[key]
        except KeyError:
            if not self.loaded:
                self.load()
                return self._data[key]
            raise

    def __iter__(self):
        self.load()
        return self._data.__iter__()

    def __len__(self):
        self.load()
        return len(self._data)

    def _convert_value(self, field, value):
        '''
        Convert a JSON value to its proper type.

        If the field has a special type (as defined by the class'
        ``_DATE_FIELDS``, ``_DATETIME_FIELDS``, etc.) then the value is
        converted accordingly. Otherwise the value is returned
        unchanged.
        '''
        if field in self._DATE_FIELDS:
            return self._parse_date(value, field)
        if field in self._DATETIME_FIELDS:
            return self._parse_datetime(value, field)
        if field in self._OBJECT_FIELDS:
            return self._parse_object(value, field)
        if field in self._OBJECT_LIST_FIELDS:
            return self._parse_object_list(value, field)
        if field in self._REFERENCE_FIELDS:
            return self._parse_reference(value, field)
        if field in self._REFERENCE_LIST_FIELDS:
            return self._parse_reference_list(value, field)
        if field in self._EXTERNAL_LIST_FIELDS:
            return ExternalObjectList(value)
        return value

    def _ensure_list(self, value, field):
        if (not isinstance(value, collections.Sequence)
                or isinstance(value, six.string_types)):
            warn(('In object "{id}": Field "{field}" of type "{type}" must '
                 + 'contain a list, but a non-list value was found '
                 + 'instead.').format(id=self._data['id'], field=field,
                 type=self._data['type']), SpecificationWarning)
            value = [value]
        return value

    def _parse_date(self, value, field):
        try:
            return dateutil.parser.parse(value).date()
        except ValueError as e:
            warn(('In object "{id}": Field "{field}" contains an invalid '
                 + 'date string ("{value}"): {error}').format(
                 id=self._data['id'], field=field, value=value, error=e),
                 ContentWarning)
            return value

    def _parse_datetime(self, value, field):
        try:
            return dateutil.parser.parse(value)
        except ValueError as e:
            warn(('In object "{id}": Field "{field}" contains an invalid '
                 + 'date-time string ("{value}"): {error}').format(
                 id=self._data['id'], field=field, value=value, error=e),
                 ContentWarning)
            return value

    def _parse_object(self, value, field):
        if _is_url(value):
            warn(('In object "{id}": Field "{field}" of type "{type}" '
                 + 'must contain an object, but a URL ("{url}") was found '
                 + 'instead.').format(id=self._data['id'], field=field,
                 type=self._data['type'], url=value), SpecificationWarning)
            return from_id(value)
        else:
            return from_json(value)

    def _parse_object_list(self, value, field):
        values = []
        for v in self._ensure_list(value, field):
            if _is_url(v):
                warn(('In object "{id}": The list in field "{field}" of '
                     + 'type "{type}" must contain objects, but an URL '
                     + '("{url}") was found instead.').format(
                     id=self._data['id'], field=field, type=self._data['type'],
                     url=v), SpecificationWarning)
                values.append(from_id(v))
            else:
                values.append(from_json(v))
        return values

    def _parse_reference(self, value, field):
        if isinstance(value, dict):
            warn(('In object "{id}": Field "{field}" of type "{type}" '
                 + 'must contain an object reference (URL), but an object '
                 + 'was found instead.').format(id=self._data['id'],
                 field=field, type=self._data['type']), SpecificationWarning)
            return from_json(value)
        else:
            return _lazy(value, self._REFERENCE_FIELDS[field])

    def _parse_reference_list(self, value, field):
        obj_type = self._REFERENCE_LIST_FIELDS[field]
        values = []
        for v in self._ensure_list(value, field):
            if isinstance(v, dict):
                warn(('In object "{id}": The list in field "{field}" of '
                     + 'type "{type}" must contain references (URLs), but '
                     + 'an object was found instead.').format(
                     id=self._data['id'], field=field,
                     type=self._data['type']), SpecificationWarning)
                values.append(from_json(v))
            else:
                values.append(_lazy(v, obj_type))
        return values

    def _init_from_json(self, data):
        '''
        Init the object from (parsed) JSON data.
        '''
        if not 'id' in data:
            raise ValueError('JSON data does not have an `id` field.')
        if data['id'] != self._data['id']:
            warn(('Initializing object "{id}" from JSON data which contains a '
                 + 'different ID ("{json_id}").').format(id=self._data['id'],
                 json_id=data['id']), ContentWarning)
        try:
            type =  data['type']
        except KeyError:
            raise ValueError('JSON data does not have a `type` field.')
        cls = _class_from_type_uri(type)
        if cls != self.__class__:
            raise ValueError(('Type from JSON data ({type}) does not match '
                             + 'instance type.').format(type=type))
        for key, value in six.iteritems(data):
            self._data[key] = self._convert_value(key, value)
        self.loaded = True

    def __repr__(self):
        s = '<oparl:{cls}'.format(cls=self.__class__.__name__)
        if not self.loaded:
            s += '?'
        s += ' {id}'.format(id=self._data['id'])
        name = self._data.get('shortname', self._data.get('name'))
        if name:
            s += ' ({name})'.format(name=name)
        s += '>'
        return unidecode(s)


#!/usr/bin/env python # encoding: utf-8 # Copyright (c) 2016, <NAME> (www.karlsruhe.de) # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. OParl client library.  This package provides tools for easily retrieving information from an OParl_ server. OParl is a standard interface for publishing information about parliaments and their work.  .. _OParl: https://oparl.org  The main tool of the library is the ``Object`` class and its subclasses. They provide wrappers for the various OParl classes (``Body``, ``Person``, ...) and support lazy loading and automatic conversion of OParl data types (e.g. ``date-time``) to Python types (``datetime.datetime``).  You will typically start by loading a single object from an OParl server using the ``from_id`` function::      import oparl     system = oparl.from_id('https://politik-bei-uns.de/oparl')  If you've already got OParl JSON data you can also use ``from_json``.  Instances of ``Object`` and its subclasses support a read-only dict- interface::      for body in system['body']:         print(body['name'])  This library tries to be as compatible with OParl 1.0 as possible but does not enforce strict compliance. In some cases non-compliant server behavior that has been seen "in the wild" is supported. These cases trigger a ``SpecificationWarning``. If invalid values are encountered during auto-conversion (e.g. illegal date strings) then a ``ContentWarning`` is issued and conversion is skipped.  By default, HTTPS certificates are verified. You can disable that verification by setting ``VERIFY_HTTPS`` to ``False``.  The libraries logger (``log``) doesn't have a handler attached to it by default, but may come in handy during development. # Official OParl 1.0 schema URI # Should HTTPS certificates be verified? Base class for OParl warnings. Warning that something did not comply with the OParl specification. Warning that some content was malformed.      This occurs, for example, if a date string does not contain a valid     date. Convert a type URI to a class. Download JSON from an URL and parse it. Initialize an OParl object from JSON.      ``data`` is raw OParl JSON data (either as a Python data     structure or as a string).      Returns an appropriate subclass of ``Object`` initialized using     the given data. Initialize an OParl object from its ID (URL).      The object's data is downloaded and parsed. The resulting object is     returned. Create a lazy OParl object.      The returned object doesn't contain any data (aside from the ID and     the type). The data is downloaded once it is required Check if a value looks like an URL. (Lazy) list of OParl objects.      OParl has "external object lists". These lists contain OParl objects     and are accessible via URL but are not true OParl objects themselves     (they don't have their own fields, IDs, etc.). They exist to allow     a pagination of large lists.      This class lazily wraps such a list. Pages are retrieved only once     further items are requested by indexing (``my_list[34]``) or by     iterating over the list.      To prevent storing large lists completely in memory only the     currently requested page is stored. Random access to the list may     therefore lead to repeated downloads of the same page. To download     the complete list use ``my_list = list(my_list)``.      Using ``len`` on an instance of this class returns the currently     known number of items in the list. This number may increase once     more items are requested. In OParl, the only portable way to get the     total number of available items is to download the complete list.     This can be done by using ``my_list = list(my_list)``.      Due to its dynamic nature, instances of this class only support non-     negative integer indices. In particular, slicing and negative     indices are not supported # The only mandatory link between sub-pages of a paginated list in # OParl is ``next``. While OParl offers several other such links # (e.g. ``last``) these are optional. Similarly, OParl doesn't # require the server to mention the total number of items. Loads the sub-page which contains an index.          The page which contains the index ``i`` is loaded. If ``i`` is         larger than the number of items in the list then an         ``IndexError`` is raised. Load a sub-page.          Sub-pages must be loaded incrementally, i.e. page ``i`` must be         loaded before page ``i + 1``. Base class for all OParl objects.      The subclasses of this class (e.g. ``Person``) represent the various     OParl objects. The classes are dict-like read-only wrappers around     the objects' JSON data.      An object can be initialized using the ``from_id`` and ``from_json``     functions. The classes' constructors are not intended for public use.      Non-trivial fields defined by the OParl standard (e.g. fields of     type ``date-time``) are automatically converted to an appropriate     Python object. Nested objects referenced via an URL are loaded     lazily, i.e. their full data is only downloaded once it is required.     You can check whether that has happened using the ``loaded``     attribute and force it via the ``load`` method. # Fields that have type 'Date'. Their values are automatically # parsed from the string representation. # Fields that have type 'DateTime'. Their values are automatically # parsed from the string representation. # Fields that contain another OParl objects. The object is # automatically parsed from its JSON representation. # Fields that contain a list of other OParl objects. The objects are # automatically parsed from their JSON representation. # Fields that contain a reference (URL) to another OParl object. # Their value is automatically replaced by a corresponding lazy # OParl object. This dict maps field names to the OParl type URIs of # the referenced object. # Fields that contain a list of references (URLs) to other OParl # objects. The values in the lists are automatically replaced by # corresponding lazy OParl objects. This dict maps field names to # the OParl type URIs of the referenced objects. # Fields that contain an URL to a a list of OParl objects. Their # values are automatically wrapped in ``ExternalObjectList`` # instances. Private constructor.          Use ``from_id`` or ``from_json`` instead. Load the object's data if it hasn't been loaded, yet.          If ``force`` is true then the data is always downloaded. Convert a JSON value to its proper type.          If the field has a special type (as defined by the class'         ``_DATE_FIELDS``, ``_DATETIME_FIELDS``, etc.) then the value is         converted accordingly. Otherwise the value is returned         unchanged. Init the object from (parsed) JSON data.