def add_model(self, model, force=False):
        """
        Add a model.

        The model will be asssigned to a class attribute with the YANG name of the model.

        Args:
            model (PybindBase): Model to add.
            force (bool): If not set, verify the model is in SUPPORTED_MODELS

        Examples:

            >>> import napalm_yang
            >>> config = napalm_yang.base.Root()
            >>> config.add_model(napalm_yang.models.openconfig_interfaces)
            >>> config.interfaces
            <pyangbind.lib.yangtypes.YANGBaseClass object at 0x10bef6680>
        """
        if isinstance(model, str):
            self._load_model(model)
            return

        try:
            model = model()
        except Exception:
            pass

        if model._yang_name not in [a[0] for a in SUPPORTED_MODELS] and not force:
            raise ValueError(
                "Only models in SUPPORTED_MODELS can be added without `force=True`"
            )

        for k, v in model:
            self._elements[k] = v
            setattr(self, k, v)


def get(self, filter=False):
        """
        Returns a dictionary with the values of the model. Note that the values
        of the leafs are YANG classes.

        Args:
            filter (bool): If set to ``True``, show only values that have been set.

        Returns:
            dict: A dictionary with the values of the model.

        Example:

            >>> pretty_print(config.get(filter=True))
            >>> {
            >>>     "interfaces": {
            >>>         "interface": {
            >>>             "et1": {
            >>>                 "config": {
            >>>                     "description": "My description",
            >>>                     "mtu": 1500
            >>>                 },
            >>>                 "name": "et1"
            >>>             },
            >>>             "et2": {
            >>>                 "config": {
            >>>                     "description": "Another description",
            >>>                     "mtu": 9000
            >>>                 },
            >>>                 "name": "et2"
            >>>             }
            >>>         }
            >>>     }
            >>> }

        """
        result = {}

        for k, v in self.elements().items():
            intermediate = v.get(filter=filter)
            if intermediate:
                result[k] = intermediate

        return result

def get ( self , filter = False ) : result = { } for k , v in self . elements ( ) . items ( ) : intermediate = v . get ( filter = filter ) if intermediate : result [ k ] = intermediate return result

Returns a dictionary with the values of the model . Note that the values of the leafs are YANG classes .


def load_dict(self, data, overwrite=False, auto_load_model=True):
        """
        Load a dictionary into the model.

        Args:
            data(dict): Dictionary to load
            overwrite(bool): Whether the data present in the model should be overwritten by the
                data in the dict or not.
            auto_load_model(bool): If set to true models will be loaded as they are needed

        Examples:

            >>> vlans_dict = {
            >>>     "vlans": { "vlan": { 100: {
            >>>                             "config": {
            >>>                                 "vlan_id": 100, "name": "production"}},
            >>>                          200: {
            >>>                             "config": {
            >>>                                 "vlan_id": 200, "name": "dev"}}}}}
            >>> config.load_dict(vlans_dict)
            >>> print(config.vlans.vlan.keys())
            ... [200, 100]
            >>> print(100, config.vlans.vlan[100].config.name)
            ... (100, u'production')
            >>> print(200, config.vlans.vlan[200].config.name)
            ... (200, u'dev')
        """
        for k, v in data.items():
            if k not in self._elements.keys() and not auto_load_model:
                raise AttributeError("Model {} is not loaded".format(k))

            elif k not in self._elements.keys() and auto_load_model:
                self._load_model(k)

            attr = getattr(self, k)
            _load_dict(attr, v)

def
load_dict
(
self
,
data
,
overwrite
=
False
,
auto_load_model
=
True
)
:
for
k
,
v
in
data
.
items
(
)
:
if
k
not
in
self
.
_elements
.
keys
(
)
and
not
auto_load_model
:
raise
AttributeError
(
"Model {} is not loaded"
.
format
(
k
)
)
elif
k
not
in
self
.
_elements
.
keys
(
)
and
auto_load_model
:
self
.
_load_model
(
k
)
attr
=
getattr
(
self
,
k
)
_load_dict
(
attr
,
v
)


def to_dict(self, filter=True):
        """
        Returns a dictionary with the values of the model. Note that the values
        of the leafs are evaluated to python types.

        Args:
            filter (bool): If set to ``True``, show only values that have been set.

        Returns:
            dict: A dictionary with the values of the model.

        Example:

            >>> pretty_print(config.to_dict(filter=True))
            >>> {
            >>>     "interfaces": {
            >>>         "interface": {
            >>>             "et1": {
            >>>                 "config": {
            >>>                     "description": "My description",
            >>>                     "mtu": 1500
            >>>                 },
            >>>                 "name": "et1"
            >>>             },
            >>>             "et2": {
            >>>                 "config": {
            >>>                     "description": "Another description",
            >>>                     "mtu": 9000
            >>>                 },
            >>>                 "name": "et2"
            >>>             }
            >>>         }
            >>>     }
            >>> }

        """
        result = {}
        for k, v in self:
            r = _to_dict(v, filter)
            if r:
                result[k] = r
        return result

def to_dict ( self , filter = True ) : result = { } for k , v in self : r = _to_dict ( v , filter ) if r : result [ k ] = r return result

Returns a dictionary with the values of the model . Note that the values of the leafs are evaluated to python types .


def parse_config(self, device=None, profile=None, native=None, attrs=None):
        """
        Parse native configuration and load it into the corresponding models. Only models
        that have been added to the root object will be parsed.

        If ``native`` is passed to the method that's what we will parse, otherwise, we will use the
        ``device`` to retrieve it.

        Args:
            device (NetworkDriver): Device to load the configuration from.
            profile (list): Profiles that the device supports. If no ``profile`` is passed it will
              be read from ``device``.
            native (list of strings): Native configuration to parse.

        Examples:

            >>> # Load from device
            >>> running_config = napalm_yang.base.Root()
            >>> running_config.add_model(napalm_yang.models.openconfig_interfaces)
            >>> running_config.parse_config(device=d)

            >>> # Load from file
            >>> with open("junos.config", "r") as f:
            >>>     config = f.read()
            >>>
            >>> running_config = napalm_yang.base.Root()
            >>> running_config.add_model(napalm_yang.models.openconfig_interfaces)
            >>> running_config.parse_config(native=[config], profile="junos")
        """
        if attrs is None:
            attrs = self.elements().values()

        for v in attrs:
            parser = Parser(
                v, device=device, profile=profile, native=native, is_config=True
            )
            parser.parse()

def parse_config ( self , device = None , profile = None , native = None , attrs = None ) : if attrs is None : attrs = self . elements ( ) . values ( ) for v in attrs : parser = Parser ( v , device = device , profile = profile , native = native , is_config = True ) parser . parse ( )

Parse native configuration and load it into the corresponding models . Only models that have been added to the root object will be parsed .


def parse_state(self, device=None, profile=None, native=None, attrs=None):
        """
        Parse native state and load it into the corresponding models. Only models
        that have been added to the root object will be parsed.

        If ``native`` is passed to the method that's what we will parse, otherwise, we will use the
        ``device`` to retrieve it.

        Args:
            device (NetworkDriver): Device to load the configuration from.
            profile (list): Profiles that the device supports. If no ``profile`` is passed it will
              be read from ``device``.
            native (list string): Native output to parse.

        Examples:

            >>> # Load from device
            >>> state = napalm_yang.base.Root()
            >>> state.add_model(napalm_yang.models.openconfig_interfaces)
            >>> state.parse_config(device=d)

            >>> # Load from file
            >>> with open("junos.state", "r") as f:
            >>>     state_data = f.read()
            >>>
            >>> state = napalm_yang.base.Root()
            >>> state.add_model(napalm_yang.models.openconfig_interfaces)
            >>> state.parse_config(native=[state_data], profile="junos")
        """
        if attrs is None:
            attrs = self.elements().values()

        for v in attrs:
            parser = Parser(
                v, device=device, profile=profile, native=native, is_config=False
            )
            parser.parse()

def parse_state ( self , device = None , profile = None , native = None , attrs = None ) : if attrs is None : attrs = self . elements ( ) . values ( ) for v in attrs : parser = Parser ( v , device = device , profile = profile , native = native , is_config = False ) parser . parse ( )

Parse native state and load it into the corresponding models . Only models that have been added to the root object will be parsed .


def translate_config(self, profile, merge=None, replace=None):
        """
        Translate the object to native configuration.

        In this context, merge and replace means the following:

        * **Merge** - Elements that exist in both ``self`` and ``merge`` will use by default the
          values in ``merge`` unless ``self`` specifies a new one. Elements that exist only
          in ``self`` will be translated as they are and elements present only in ``merge``
          will be removed.
        * **Replace** - All the elements in ``replace`` will either be removed or replaced by
          elements in ``self``.

        You can specify one of ``merge``, ``replace`` or none of them. If none of them are set we
        will just translate configuration.

        Args:
            profile (list): Which profiles to use.
            merge (Root): Object we want to merge with.
            replace (Root): Object we want to replace.
        """
        result = []
        for k, v in self:
            other_merge = getattr(merge, k) if merge else None
            other_replace = getattr(replace, k) if replace else None
            translator = Translator(
                v, profile, merge=other_merge, replace=other_replace
            )
            result.append(translator.translate())

        return "\n".join(result)

def translate_config ( self , profile , merge = None , replace = None ) : result = [ ] for k , v in self : other_merge = getattr ( merge , k ) if merge else None other_replace = getattr ( replace , k ) if replace else None translator = Translator ( v , profile , merge = other_merge , replace = other_replace ) result . append ( translator . translate ( ) ) return "\n" . join ( result )

Translate the object to native configuration .


def load_filters():
    """
    Loads and returns all filters.
    """
    all_filters = {}
    for m in JINJA_FILTERS:
        if hasattr(m, "filters"):
            all_filters.update(m.filters())
    return all_filters

def load_filters ( ) : all_filters = { } for m in JINJA_FILTERS : if hasattr ( m , "filters" ) : all_filters . update ( m . filters ( ) ) return all_filters


def find_yang_file(profile, filename, path):
    """
    Find the necessary file for the given test case.

    Args:
        device(napalm device connection): for which device
        filename(str): file to find
        path(str): where to find it relative to where the module is installed
    """
    # Find base_dir of submodule
    module_dir = os.path.dirname(__file__)
    full_path = os.path.join(module_dir, "mappings", profile, path, filename)

    if os.path.exists(full_path):
        return full_path
    else:
        msg = "Couldn't find parsing file: {}".format(full_path)
        logger.error(msg)
        raise IOError(msg)

def
find_yang_file
(
profile
,
filename
,
path
)
:
# Find base_dir of submodule
module_dir
=
os
.
path
.
dirname
(
__file__
)
full_path
=
os
.
path
.
join
(
module_dir
,
"mappings"
,
profile
,
path
,
filename
)
if
os
.
path
.
exists
(
full_path
)
:
return
full_path
else
:
msg
=
"Couldn't find parsing file: {}"
.
format
(
full_path
)
logger
.
error
(
msg
)
raise
IOError
(
msg
)

Find the necessary file for the given test case .


def model_to_dict(model, mode="", show_defaults=False):
    """
    Given a model, return a representation of the model in a dict.

    This is mostly useful to have a quick visual represenation of the model.

    Args:

        model (PybindBase): Model to transform.
        mode (string): Whether to print config, state or all elements ("" for all)

    Returns:

        dict: A dictionary representing the model.

    Examples:

        >>> config = napalm_yang.base.Root()
        >>>
        >>> # Adding models to the object
        >>> config.add_model(napalm_yang.models.openconfig_interfaces())
        >>> config.add_model(napalm_yang.models.openconfig_vlan())
        >>> # Printing the model in a human readable format
        >>> pretty_print(napalm_yang.utils.model_to_dict(config))
        >>> {
        >>>     "openconfig-interfaces:interfaces [rw]": {
        >>>         "interface [rw]": {
        >>>             "config [rw]": {
        >>>                 "description [rw]": "string",
        >>>                 "enabled [rw]": "boolean",
        >>>                 "mtu [rw]": "uint16",
        >>>                 "name [rw]": "string",
        >>>                 "type [rw]": "identityref"
        >>>             },
        >>>             "hold_time [rw]": {
        >>>                 "config [rw]": {
        >>>                     "down [rw]": "uint32",
        >>>                     "up [rw]": "uint32"
            (trimmed for clarity)
    """

    def is_mode(obj, mode):
        if mode == "":
            return True
        elif mode == "config":
            return obj._yang_name == "config" or obj._is_config
        elif mode == "state":
            return obj._yang_name == "state" or not obj._is_config
        else:
            raise ValueError(
                "mode can only be config, state or ''. Passed: {}".format(mode)
            )

    def get_key(key, model, parent_defining_module, show_defaults):
        if not show_defaults:
            # No need to display rw/ro when showing the defaults.
            key = "{} {}".format(key, "[rw]" if model._is_config else "[ro]")
        if parent_defining_module != model._defining_module:
            key = "{}:{}".format(model._defining_module, key)
        return key

    if model._yang_type in ("container", "list"):
        cls = model if model._yang_type in ("container",) else model._contained_class()
        result = {}
        for k, v in cls:
            r = model_to_dict(v, mode=mode, show_defaults=show_defaults)
            if r:
                result[get_key(k, v, model._defining_module, show_defaults)] = r
        return result
    else:
        if show_defaults:
            if model._default is False:
                if model._yang_type != "boolean":
                    # Unless the datatype is bool, when the _default attribute
                    # is False, it means there is not default value defined in
                    # the YANG model.
                    return None
            return model._default
        return model._yang_type if is_mode(model, mode) else None

Given a model return a representation of the model in a dict .


def diff(f, s):
    """
    Given two models, return the difference between them.

    Args:

        f (Pybindbase): First element.
        s (Pybindbase): Second element.

    Returns:

        dict: A dictionary highlighting the differences.

    Examples:

        >>> diff = napalm_yang.utils.diff(candidate, running)
        >>> pretty_print(diff)
        >>> {
        >>>     "interfaces": {
        >>>         "interface": {
        >>>             "both": {
        >>>                 "Port-Channel1": {
        >>>                     "config": {
        >>>                         "mtu": {
        >>>                             "first": "0",
        >>>                             "second": "9000"
        >>>                         }
        >>>                     }
        >>>                 }
        >>>             },
        >>>             "first_only": [
        >>>                 "Loopback0"
        >>>             ],
        >>>             "second_only": [
        >>>                 "Loopback1"
        >>>             ]
        >>>         }
        >>>     }
        >>> }
    """
    if isinstance(f, base.Root) or f._yang_type in ("container", None):
        result = _diff_root(f, s)
    elif f._yang_type in ("list",):
        result = _diff_list(f, s)
    else:
        result = {}
        first = "{}".format(f)
        second = "{}".format(s)
        if first != second:
            result = {"first": first, "second": second}

    return result

def diff ( f , s ) : if isinstance ( f , base . Root ) or f . _yang_type in ( "container" , None ) : result = _diff_root ( f , s ) elif f . _yang_type in ( "list" , ) : result = _diff_list ( f , s ) else : result = { } first = "{}" . format ( f ) second = "{}" . format ( s ) if first != second : result = { "first" : first , "second" : second } return result

Given two models return the difference between them .


def http_post(self, url, data=None):
        """POST to URL and get result as a response object.

        :param url: URL to POST.
        :type url: str
        :param data: Data to send in the form body.
        :type data: str
        :rtype: requests.Response
        """
        if not url.startswith('https://'):
            raise ValueError('Protocol must be HTTPS, invalid URL: %s' % url)
        return requests.post(url, data, verify=True)

POST to URL and get result as a response object .


def get_authorization_code_uri(self, **params):
        """Construct a full URL that can be used to obtain an authorization
        code from the provider authorization_uri. Use this URI in a client
        frame to cause the provider to generate an authorization code.

        :rtype: str
        """
        if 'response_type' not in params:
            params['response_type'] = self.default_response_type
        params.update({'client_id': self.client_id,
                       'redirect_uri': self.redirect_uri})
        return utils.build_url(self.authorization_uri, params)

def get_authorization_code_uri ( self , * * params ) : if 'response_type' not in params : params [ 'response_type' ] = self . default_response_type params . update ( { 'client_id' : self . client_id , 'redirect_uri' : self . redirect_uri } ) return utils . build_url ( self . authorization_uri , params )

Construct a full URL that can be used to obtain an authorization code from the provider authorization_uri . Use this URI in a client frame to cause the provider to generate an authorization code .


def get_token(self, code, **params):
        """Get an access token from the provider token URI.

        :param code: Authorization code.
        :type code: str
        :return: Dict containing access token, refresh token, etc.
        :rtype: dict
        """
        params['code'] = code
        if 'grant_type' not in params:
            params['grant_type'] = self.default_grant_type
        params.update({'client_id': self.client_id,
                       'client_secret': self.client_secret,
                       'redirect_uri': self.redirect_uri})
        response = self.http_post(self.token_uri, params)
        try:
            return response.json()
        except TypeError:
            return response.json

def get_token ( self , code , * * params ) : params [ 'code' ] = code if 'grant_type' not in params : params [ 'grant_type' ] = self . default_grant_type params . update ( { 'client_id' : self . client_id , 'client_secret' : self . client_secret , 'redirect_uri' : self . redirect_uri } ) response = self . http_post ( self . token_uri , params ) try : return response . json ( ) except TypeError : return response . json

Get an access token from the provider token URI .


def url_query_params(url):
    """Return query parameters as a dict from the specified URL.

    :param url: URL.
    :type url: str
    :rtype: dict
    """
    return dict(urlparse.parse_qsl(urlparse.urlparse(url).query, True))

def url_query_params ( url ) : return dict ( urlparse . parse_qsl ( urlparse . urlparse ( url ) . query , True ) )

Return query parameters as a dict from the specified URL .


def url_dequery(url):
    """Return a URL with the query component removed.

    :param url: URL to dequery.
    :type url: str
    :rtype: str
    """
    url = urlparse.urlparse(url)
    return urlparse.urlunparse((url.scheme,
                                url.netloc,
                                url.path,
                                url.params,
                                '',
                                url.fragment))

def url_dequery ( url ) : url = urlparse . urlparse ( url ) return urlparse . urlunparse ( ( url . scheme , url . netloc , url . path , url . params , '' , url . fragment ) )

Return a URL with the query component removed .


def build_url(base, additional_params=None):
    """Construct a URL based off of base containing all parameters in
    the query portion of base plus any additional parameters.

    :param base: Base URL
    :type base: str
    ::param additional_params: Additional query parameters to include.
    :type additional_params: dict
    :rtype: str
    """
    url = urlparse.urlparse(base)
    query_params = {}
    query_params.update(urlparse.parse_qsl(url.query, True))
    if additional_params is not None:
        query_params.update(additional_params)
        for k, v in additional_params.iteritems():
            if v is None:
                query_params.pop(k)

    return urlparse.urlunparse((url.scheme,
                                url.netloc,
                                url.path,
                                url.params,
                                urllib.urlencode(query_params),
                                url.fragment))

def build_url ( base , additional_params = None ) : url = urlparse . urlparse ( base ) query_params = { } query_params . update ( urlparse . parse_qsl ( url . query , True ) ) if additional_params is not None : query_params . update ( additional_params ) for k , v in additional_params . iteritems ( ) : if v is None : query_params . pop ( k ) return urlparse . urlunparse ( ( url . scheme , url . netloc , url . path , url . params , urllib . urlencode ( query_params ) , url . fragment ) )

Construct a URL based off of base containing all parameters in the query portion of base plus any additional parameters .


def _handle_exception(self, exc):
        """Handle an internal exception that was caught and suppressed.

        :param exc: Exception to process.
        :type exc: Exception
        """
        logger = logging.getLogger(__name__)
        logger.exception(exc)

def _handle_exception ( self , exc ) : logger = logging . getLogger ( __name__ ) logger . exception ( exc )

Handle an internal exception that was caught and suppressed .


def _make_response(self, body='', headers=None, status_code=200):
        """Return a response object from the given parameters.

        :param body: Buffer/string containing the response body.
        :type body: str
        :param headers: Dict of headers to include in the requests.
        :type headers: dict
        :param status_code: HTTP status code.
        :type status_code: int
        :rtype: requests.Response
        """
        res = Response()
        res.status_code = status_code
        if headers is not None:
            res.headers.update(headers)
        res.raw = StringIO(body)
        return res

def _make_response ( self , body = '' , headers = None , status_code = 200 ) : res = Response ( ) res . status_code = status_code if headers is not None : res . headers . update ( headers ) res . raw = StringIO ( body ) return res

Return a response object from the given parameters .


def _make_redirect_error_response(self, redirect_uri, err):
        """Return a HTTP 302 redirect response object containing the error.

        :param redirect_uri: Client redirect URI.
        :type redirect_uri: str
        :param err: OAuth error message.
        :type err: str
        :rtype: requests.Response
        """
        params = {
            'error': err,
            'response_type': None,
            'client_id': None,
            'redirect_uri': None
        }
        redirect = utils.build_url(redirect_uri, params)
        return self._make_response(headers={'Location': redirect},
                                   status_code=302)

def _make_redirect_error_response ( self , redirect_uri , err ) : params = { 'error' : err , 'response_type' : None , 'client_id' : None , 'redirect_uri' : None } redirect = utils . build_url ( redirect_uri , params ) return self . _make_response ( headers = { 'Location' : redirect } , status_code = 302 )

Return a HTTP 302 redirect response object containing the error .


def _make_json_response(self, data, headers=None, status_code=200):
        """Return a response object from the given JSON data.

        :param data: Data to JSON-encode.
        :type data: mixed
        :param headers: Dict of headers to include in the requests.
        :type headers: dict
        :param status_code: HTTP status code.
        :type status_code: int
        :rtype: requests.Response
        """
        response_headers = {}
        if headers is not None:
            response_headers.update(headers)
        response_headers['Content-Type'] = 'application/json;charset=UTF-8'
        response_headers['Cache-Control'] = 'no-store'
        response_headers['Pragma'] = 'no-cache'
        return self._make_response(json.dumps(data),
                                   response_headers,
                                   status_code)

def _make_json_response ( self , data , headers = None , status_code = 200 ) : response_headers = { } if headers is not None : response_headers . update ( headers ) response_headers [ 'Content-Type' ] = 'application/json;charset=UTF-8' response_headers [ 'Cache-Control' ] = 'no-store' response_headers [ 'Pragma' ] = 'no-cache' return self . _make_response ( json . dumps ( data ) , response_headers , status_code )

Return a response object from the given JSON data .


def get_authorization_code(self,
                               response_type,
                               client_id,
                               redirect_uri,
                               **params):
        """Generate authorization code HTTP response.

        :param response_type: Desired response type. Must be exactly "code".
        :type response_type: str
        :param client_id: Client ID.
        :type client_id: str
        :param redirect_uri: Client redirect URI.
        :type redirect_uri: str
        :rtype: requests.Response
        """

        # Ensure proper response_type
        if response_type != 'code':
            err = 'unsupported_response_type'
            return self._make_redirect_error_response(redirect_uri, err)

        # Check redirect URI
        is_valid_redirect_uri = self.validate_redirect_uri(client_id,
                                                           redirect_uri)
        if not is_valid_redirect_uri:
            return self._invalid_redirect_uri_response()

        # Check conditions
        is_valid_client_id = self.validate_client_id(client_id)
        is_valid_access = self.validate_access()
        scope = params.get('scope', '')
        is_valid_scope = self.validate_scope(client_id, scope)

        # Return proper error responses on invalid conditions
        if not is_valid_client_id:
            err = 'unauthorized_client'
            return self._make_redirect_error_response(redirect_uri, err)

        if not is_valid_access:
            err = 'access_denied'
            return self._make_redirect_error_response(redirect_uri, err)

        if not is_valid_scope:
            err = 'invalid_scope'
            return self._make_redirect_error_response(redirect_uri, err)

        # Generate authorization code
        code = self.generate_authorization_code()

        # Save information to be used to validate later requests
        self.persist_authorization_code(client_id=client_id,
                                        code=code,
                                        scope=scope)

        # Return redirection response
        params.update({
            'code': code,
            'response_type': None,
            'client_id': None,
            'redirect_uri': None
        })
        redirect = utils.build_url(redirect_uri, params)
        return self._make_response(headers={'Location': redirect},
                                   status_code=302)

Generate authorization code HTTP response .


def refresh_token(self,
                      grant_type,
                      client_id,
                      client_secret,
                      refresh_token,
                      **params):
        """Generate access token HTTP response from a refresh token.

        :param grant_type: Desired grant type. Must be "refresh_token".
        :type grant_type: str
        :param client_id: Client ID.
        :type client_id: str
        :param client_secret: Client secret.
        :type client_secret: str
        :param refresh_token: Refresh token.
        :type refresh_token: str
        :rtype: requests.Response
        """

        # Ensure proper grant_type
        if grant_type != 'refresh_token':
            return self._make_json_error_response('unsupported_grant_type')

        # Check conditions
        is_valid_client_id = self.validate_client_id(client_id)
        is_valid_client_secret = self.validate_client_secret(client_id,
                                                             client_secret)
        scope = params.get('scope', '')
        is_valid_scope = self.validate_scope(client_id, scope)
        data = self.from_refresh_token(client_id, refresh_token, scope)
        is_valid_refresh_token = data is not None

        # Return proper error responses on invalid conditions
        if not (is_valid_client_id and is_valid_client_secret):
            return self._make_json_error_response('invalid_client')

        if not is_valid_scope:
            return self._make_json_error_response('invalid_scope')

        if not is_valid_refresh_token:
            return self._make_json_error_response('invalid_grant')

        # Discard original refresh token
        self.discard_refresh_token(client_id, refresh_token)

        # Generate access tokens once all conditions have been met
        access_token = self.generate_access_token()
        token_type = self.token_type
        expires_in = self.token_expires_in
        refresh_token = self.generate_refresh_token()

        # Save information to be used to validate later requests
        self.persist_token_information(client_id=client_id,
                                       scope=scope,
                                       access_token=access_token,
                                       token_type=token_type,
                                       expires_in=expires_in,
                                       refresh_token=refresh_token,
                                       data=data)

        # Return json response
        return self._make_json_response({
            'access_token': access_token,
            'token_type': token_type,
            'expires_in': expires_in,
            'refresh_token': refresh_token
        })

Generate access token HTTP response from a refresh token .


def get_token(self,
                  grant_type,
                  client_id,
                  client_secret,
                  redirect_uri,
                  code,
                  **params):
        """Generate access token HTTP response.

        :param grant_type: Desired grant type. Must be "authorization_code".
        :type grant_type: str
        :param client_id: Client ID.
        :type client_id: str
        :param client_secret: Client secret.
        :type client_secret: str
        :param redirect_uri: Client redirect URI.
        :type redirect_uri: str
        :param code: Authorization code.
        :type code: str
        :rtype: requests.Response
        """

        # Ensure proper grant_type
        if grant_type != 'authorization_code':
            return self._make_json_error_response('unsupported_grant_type')

        # Check conditions
        is_valid_client_id = self.validate_client_id(client_id)
        is_valid_client_secret = self.validate_client_secret(client_id,
                                                             client_secret)
        is_valid_redirect_uri = self.validate_redirect_uri(client_id,
                                                           redirect_uri)

        scope = params.get('scope', '')
        is_valid_scope = self.validate_scope(client_id, scope)
        data = self.from_authorization_code(client_id, code, scope)
        is_valid_grant = data is not None

        # Return proper error responses on invalid conditions
        if not (is_valid_client_id and is_valid_client_secret):
            return self._make_json_error_response('invalid_client')

        if not is_valid_grant or not is_valid_redirect_uri:
            return self._make_json_error_response('invalid_grant')

        if not is_valid_scope:
            return self._make_json_error_response('invalid_scope')

        # Discard original authorization code
        self.discard_authorization_code(client_id, code)

        # Generate access tokens once all conditions have been met
        access_token = self.generate_access_token()
        token_type = self.token_type
        expires_in = self.token_expires_in
        refresh_token = self.generate_refresh_token()

        # Save information to be used to validate later requests
        self.persist_token_information(client_id=client_id,
                                       scope=scope,
                                       access_token=access_token,
                                       token_type=token_type,
                                       expires_in=expires_in,
                                       refresh_token=refresh_token,
                                       data=data)

        # Return json response
        return self._make_json_response({
            'access_token': access_token,
            'token_type': token_type,
            'expires_in': expires_in,
            'refresh_token': refresh_token
        })


def get_authorization_code_from_uri(self, uri):
        """Get authorization code response from a URI. This method will
        ignore the domain and path of the request, instead
        automatically parsing the query string parameters.

        :param uri: URI to parse for authorization information.
        :type uri: str
        :rtype: requests.Response
        """
        params = utils.url_query_params(uri)
        try:
            if 'response_type' not in params:
                raise TypeError('Missing parameter response_type in URL query')

            if 'client_id' not in params:
                raise TypeError('Missing parameter client_id in URL query')

            if 'redirect_uri' not in params:
                raise TypeError('Missing parameter redirect_uri in URL query')

            return self.get_authorization_code(**params)
        except TypeError as exc:
            self._handle_exception(exc)

            # Catch missing parameters in request
            err = 'invalid_request'
            if 'redirect_uri' in params:
                u = params['redirect_uri']
                return self._make_redirect_error_response(u, err)
            else:
                return self._invalid_redirect_uri_response()
        except StandardError as exc:
            self._handle_exception(exc)

            # Catch all other server errors
            err = 'server_error'
            u = params['redirect_uri']
            return self._make_redirect_error_response(u, err)

Get authorization code response from a URI . This method will ignore the domain and path of the request instead automatically parsing the query string parameters .


def get_token_from_post_data(self, data):
        """Get a token response from POST data.

        :param data: POST data containing authorization information.
        :type data: dict
        :rtype: requests.Response
        """
        try:
            # Verify OAuth 2.0 Parameters
            for x in ['grant_type', 'client_id', 'client_secret']:
                if not data.get(x):
                    raise TypeError("Missing required OAuth 2.0 POST param: {0}".format(x))

            # Handle get token from refresh_token
            if 'refresh_token' in data:
                return self.refresh_token(**data)

            # Handle get token from authorization code
            for x in ['redirect_uri', 'code']:
                if not data.get(x):
                    raise TypeError("Missing required OAuth 2.0 POST param: {0}".format(x))
            return self.get_token(**data)
        except TypeError as exc:
            self._handle_exception(exc)

            # Catch missing parameters in request
            return self._make_json_error_response('invalid_request')
        except StandardError as exc:
            self._handle_exception(exc)

            # Catch all other server errors
            return self._make_json_error_response('server_error')


def get_authorization(self):
        """Get authorization object representing status of authentication."""
        auth = self.authorization_class()
        header = self.get_authorization_header()
        if not header or not header.split:
            return auth
        header = header.split()
        if len(header) > 1 and header[0] == 'Bearer':
            auth.is_oauth = True
            access_token = header[1]
            self.validate_access_token(access_token, auth)
            if not auth.is_valid:
                auth.error = 'access_denied'
        return auth

def get_authorization ( self ) : auth = self . authorization_class ( ) header = self . get_authorization_header ( ) if not header or not header . split : return auth header = header . split ( ) if len ( header ) > 1 and header [ 0 ] == 'Bearer' : auth . is_oauth = True access_token = header [ 1 ] self . validate_access_token ( access_token , auth ) if not auth . is_valid : auth . error = 'access_denied' return auth

Get authorization object representing status of authentication .


def open(self, bus):
        """Open the smbus interface on the specified bus."""
        # Close the device if it's already open.
        if self._device is not None:
            self.close()
        # Try to open the file for the specified bus.  Must turn off buffering
        # or else Python 3 fails (see: https://bugs.python.org/issue20074)
        self._device = open('/dev/i2c-{0}'.format(bus), 'r+b', buffering=0)

Open the smbus interface on the specified bus .


def read_byte(self, addr):
        """Read a single byte from the specified device."""
        assert self._device is not None, 'Bus must be opened before operations are made against it!'
        self._select_device(addr)
        return ord(self._device.read(1))

Read a single byte from the specified device .


def read_bytes(self, addr, number):
        """Read many bytes from the specified device."""
        assert self._device is not None, 'Bus must be opened before operations are made against it!'
        self._select_device(addr)
        return self._device.read(number)

Read many bytes from the specified device .


def read_byte_data(self, addr, cmd):
        """Read a single byte from the specified cmd register of the device."""
        assert self._device is not None, 'Bus must be opened before operations are made against it!'
        # Build ctypes values to marshall between ioctl and Python.
        reg = c_uint8(cmd)
        result = c_uint8()
        # Build ioctl request.
        request = make_i2c_rdwr_data([
            (addr, 0, 1, pointer(reg)),             # Write cmd register.
            (addr, I2C_M_RD, 1, pointer(result))    # Read 1 byte as result.
        ])
        # Make ioctl call and return result data.
        ioctl(self._device.fileno(), I2C_RDWR, request)
        return result.value

Read a single byte from the specified cmd register of the device .


def write_bytes(self, addr, buf):
        """Write many bytes to the specified device. buf is a bytearray"""
        assert self._device is not None, 'Bus must be opened before operations are made against it!'
        self._select_device(addr)
        self._device.write(buf)

Write many bytes to the specified device . buf is a bytearray


def write_byte_data(self, addr, cmd, val):
        """Write a byte of data to the specified cmd register of the device.
        """
        assert self._device is not None, 'Bus must be opened before operations are made against it!'
        # Construct a string of data to send with the command register and byte value.
        data = bytearray(2)
        data[0] = cmd & 0xFF
        data[1] = val & 0xFF
        # Send the data to the device.
        self._select_device(addr)
        self._device.write(data)

Write a byte of data to the specified cmd register of the device .


def write_i2c_block_data(self, addr, cmd, vals):
        """Write a buffer of data to the specified cmd register of the device.
        """
        assert self._device is not None, 'Bus must be opened before operations are made against it!'
        # Construct a string of data to send, including room for the command register.
        data = bytearray(len(vals)+1)
        data[0] = cmd & 0xFF  # Command register at the start.
        data[1:] = vals[0:]   # Copy in the block data (ugly but necessary to ensure
                              # the entire write happens in one transaction).
        # Send the data to the device.
        self._select_device(addr)
        self._device.write(data)

Write a buffer of data to the specified cmd register of the device .


def cdn_url(self):
        """Returns file's CDN url.

        Usage example::

            >>> file_ = File('a771f854-c2cb-408a-8c36-71af77811f3b')
            >>> file_.cdn_url
            https://ucarecdn.com/a771f854-c2cb-408a-8c36-71af77811f3b/

        You can set default effects::

            >>> file_.default_effects = 'effect/flip/-/effect/mirror/'
            >>> file_.cdn_url
            https://ucarecdn.com/a771f854-c2cb-408a-8c36-71af77811f3b/-/effect/flip/-/effect/mirror/

        """
        return '{cdn_base}{path}'.format(cdn_base=conf.cdn_base,
                                         path=self.cdn_path(self.default_effects))

def cdn_url ( self ) : return '{cdn_base}{path}' . format ( cdn_base = conf . cdn_base , path = self . cdn_path ( self . default_effects ) )


def copy(self, effects=None, target=None):
        """Creates a File Copy on Uploadcare or Custom Storage.

        File.copy method is deprecated and will be removed in 4.0.0.
        Please use `create_local_copy` and `create_remote_copy` instead.

        Args:
            - effects:
                Adds CDN image effects. If ``self.default_effects`` property
                is set effects will be combined with default effects.
            - target:
                Name of a custom storage connected to your project.
                Uploadcare storage is used if target is absent.

        """
        warning = """File.copy method is deprecated and will be
            removed in 4.0.0.
            Please use `create_local_copy`
            and `create_remote_copy` instead.
        """
        logger.warn('API Warning: {0}'.format(warning))

        if target is not None:
            return self.create_remote_copy(target, effects)
        else:
            return self.create_local_copy(effects)

Creates a File Copy on Uploadcare or Custom Storage . File . copy method is deprecated and will be removed in 4 . 0 . 0 . Please use create_local_copy and create_remote_copy instead .


def create_local_copy(self, effects=None, store=None):
        """Creates a Local File Copy on Uploadcare Storage.

        Args:
            - effects:
                Adds CDN image effects. If ``self.default_effects`` property
                is set effects will be combined with default effects.
            - store:
                If ``store`` option is set to False the copy of your file will
                be deleted in 24 hour period after the upload.
                Works only if `autostore` is enabled in the project.

        """
        effects = self._build_effects(effects)
        store = store or ''
        data = {
            'source': self.cdn_path(effects)
        }
        if store:
            data['store'] = store
        return rest_request('POST', 'files/', data=data)

def create_local_copy ( self , effects = None , store = None ) : effects = self . _build_effects ( effects ) store = store or '' data = { 'source' : self . cdn_path ( effects ) } if store : data [ 'store' ] = store return rest_request ( 'POST' , 'files/' , data = data )

Creates a Local File Copy on Uploadcare Storage .


def create_remote_copy(self, target, effects=None, make_public=None,
                           pattern=None):
        """Creates file copy in remote storage.

        Args:
            - target:
                Name of a custom storage connected to the project.
            - effects:
                Adds CDN image effects to ``self.default_effects`` if any.
            - make_public:
                To forbid public from accessing your files on the storage set
                ``make_public`` option to be False.
                Default value is None. Files have public access by default.
            - pattern:
                Specify ``pattern`` option to set S3 object key name.
                Takes precedence over pattern set in project settings.
                If neither is specified defaults to
                `${uuid}/${filename}${effects}${ext}`.

        For more information on each of the options above please refer to
        REST API docs https://uploadcare.com/docs/api_reference/rest/accessing_files/.

        Following example copies a file to custom storage named ``samplefs``:

             >>> file = File('e8ebfe20-8c11-4a94-9b40-52ecad7d8d1a')
             >>> file.create_remote_copy(target='samplefs',
             >>>                         make_public=True,
             >>>                         pattern='${uuid}/${filename}${ext}')

        Now custom storage ``samplefs`` contains publicly available file
        with original filename billmurray.jpg in
        in the directory named ``e8ebfe20-8c11-4a94-9b40-52ecad7d8d1a``.

        """
        effects = self._build_effects(effects)
        data = {
            'source': self.cdn_path(effects),
            'target': target
        }

        if make_public is not None:
            data['make_public'] = make_public
        if pattern is not None:
            data['pattern'] = pattern
        return rest_request('POST', 'files/', data=data)

def create_remote_copy ( self , target , effects = None , make_public = None , pattern = None ) : effects = self . _build_effects ( effects ) data = { 'source' : self . cdn_path ( effects ) , 'target' : target } if make_public is not None : data [ 'make_public' ] = make_public if pattern is not None : data [ 'pattern' ] = pattern return rest_request ( 'POST' , 'files/' , data = data )


def construct_from(cls, file_info):
        """Constructs ``File`` instance from file information.

        For example you have result of
        ``/files/1921953c-5d94-4e47-ba36-c2e1dd165e1a/`` API request::

            >>> file_info = {
                    # ...
                    'uuid': '1921953c-5d94-4e47-ba36-c2e1dd165e1a',
                    # ...
                }
            >>> File.construct_from(file_info)
            <uploadcare.File 1921953c-5d94-4e47-ba36-c2e1dd165e1a>

        """
        file_ = cls(file_info['uuid'])
        file_.default_effects = file_info.get('default_effects')
        file_._info_cache = file_info
        return file_

Constructs File instance from file information .


def upload(cls, file_obj, store=None):
        """Uploads a file and returns ``File`` instance.

        Args:
            - file_obj: file object to upload to
            - store (Optional[bool]): Should the file be automatically stored
                upon upload. Defaults to None.
                - False - do not store file
                - True - store file (can result in error if autostore
                               is disabled for project)
                - None - use project settings

        Returns:
            ``File`` instance

        """
        if store is None:
            store = 'auto'
        elif store:
            store = '1'
        else:
            store = '0'

        data = {
            'UPLOADCARE_STORE': store,
        }

        files = uploading_request('POST', 'base/', data=data,
                                  files={'file': file_obj})
        file_ = cls(files['file'])
        return file_

def upload ( cls , file_obj , store = None ) : if store is None : store = 'auto' elif store : store = '1' else : store = '0' data = { 'UPLOADCARE_STORE' : store , } files = uploading_request ( 'POST' , 'base/' , data = data , files = { 'file' : file_obj } ) file_ = cls ( files [ 'file' ] ) return file_

Uploads a file and returns File instance .


def upload_from_url(cls, url, store=None, filename=None):
        """Uploads file from given url and returns ``FileFromUrl`` instance.

        Args:
            - url (str): URL of file to upload to
            - store (Optional[bool]): Should the file be automatically stored
                upon upload. Defaults to None.
                - False - do not store file
                - True - store file (can result in error if autostore
                               is disabled for project)
                - None - use project settings
            - filename (Optional[str]): Name of the uploaded file. If this not
                specified the filename will be obtained from response headers
                or source URL. Defaults to None.

        Returns:
            ``FileFromUrl`` instance

        """
        if store is None:
            store = 'auto'
        elif store:
            store = '1'
        else:
            store = '0'

        data = {
            'source_url': url,
            'store': store,
        }
        if filename:
            data['filename'] = filename

        result = uploading_request('POST', 'from_url/',
                                   data=data)
        if 'token' not in result:
            raise APIError(
                'could not find token in result: {0}'.format(result)
            )
        file_from_url = cls.FileFromUrl(result['token'])
        return file_from_url

def
upload_from_url
(
cls
,
url
,
store
=
,
filename
=
)
:
if
store
is
:
store
=
'auto'
elif
store
:
store
=
'1'
else
:
store
=
'0'
data
=
{
'source_url'
:
url
,
'store'
:
store
,
}
if
filename
:
data
[
'filename'
]
=
filename
result
=
uploading_request
(
'POST'
,
'from_url/'
,
data
=
data
)
if
'token'
not
in
result
:
raise
APIError
(
'could not find token in result: {0}'
.
format
(
result
)
)
file_from_url
=
cls
.
FileFromUrl
(
result
[
'token'
]
)
return
file_from_url

Uploads file from given url and returns FileFromUrl instance .


def upload_from_url_sync(cls, url, timeout=30, interval=0.3,
                             until_ready=False, store=None, filename=None):
        """Uploads file from given url and returns ``File`` instance.

        Args:
            - url (str): URL of file to upload to
            - store (Optional[bool]): Should the file be automatically stored
                upon upload. Defaults to None.
                - False - do not store file
                - True - store file (can result in error if autostore
                               is disabled for project)
                - None - use project settings
            - filename (Optional[str]): Name of the uploaded file. If this not
                specified the filename will be obtained from response headers
                or source URL. Defaults to None.
            - timeout (Optional[int]): seconds to wait for successful upload.
                Defaults to 30.
            - interval (Optional[float]): interval between upload status checks.
                Defaults to 0.3.
            - until_ready (Optional[bool]): should we wait until file is
                available via CDN. Defaults to False.

        Returns:
            ``File`` instance

        Raises:
            ``TimeoutError`` if file wasn't uploaded in time

        """
        ffu = cls.upload_from_url(url, store, filename)
        return ffu.wait(timeout=timeout, interval=interval,
                        until_ready=until_ready)

def upload_from_url_sync ( cls , url , timeout = 30 , interval = 0.3 , until_ready = False , store = None , filename = None ) : ffu = cls . upload_from_url ( url , store , filename ) return ffu . wait ( timeout = timeout , interval = interval , until_ready = until_ready )

Uploads file from given url and returns File instance .


def file_cdn_urls(self):
        """Returns CDN urls of all files from group without API requesting.

        Usage example::

            >>> file_group = FileGroup('0513dda0-582f-447d-846f-096e5df9e2bb~2')
            >>> file_group.file_cdn_urls[0]
            'https://ucarecdn.com/0513dda0-582f-447d-846f-096e5df9e2bb~2/nth/0/'

        """
        file_cdn_urls = []
        for file_index in six.moves.xrange(len(self)):
            file_cdn_url = '{group_cdn_url}nth/{file_index}/'.format(
                group_cdn_url=self.cdn_url,
                file_index=file_index
            )
            file_cdn_urls.append(file_cdn_url)
        return file_cdn_urls

def file_cdn_urls ( self ) : file_cdn_urls = [ ] for file_index in six . moves . xrange ( len ( self ) ) : file_cdn_url = '{group_cdn_url}nth/{file_index}/' . format ( group_cdn_url = self . cdn_url , file_index = file_index ) file_cdn_urls . append ( file_cdn_url ) return file_cdn_urls

Returns CDN urls of all files from group without API requesting .


def construct_from(cls, group_info):
        """Constructs ``FileGroup`` instance from group information."""
        group = cls(group_info['id'])
        group._info_cache = group_info
        return group

def construct_from ( cls , group_info ) : group = cls ( group_info [ 'id' ] ) group . _info_cache = group_info return group

Constructs FileGroup instance from group information .


def create(cls, files):
        """Creates file group and returns ``FileGroup`` instance.

        It expects iterable object that contains ``File`` instances, e.g.::

            >>> file_1 = File('6c5e9526-b0fe-4739-8975-72e8d5ee6342')
            >>> file_2 = File('a771f854-c2cb-408a-8c36-71af77811f3b')
            >>> FileGroup.create((file_1, file_2))
            <uploadcare.FileGroup 0513dda0-6666-447d-846f-096e5df9e2bb~2>

        """
        data = {}
        for index, file_ in enumerate(files):
            if isinstance(file_, File):
                file_index = 'files[{index}]'.format(index=index)
                data[file_index] = six.text_type(file_)
            else:
                raise InvalidParamError(
                    'all items have to be ``File`` instance'
                )
        if not data:
            raise InvalidParamError('set of files is empty')

        group_info = uploading_request('POST', 'group/', data=data)

        group = cls.construct_from(group_info)
        return group

Creates file group and returns FileGroup instance .


def _base_opration(self, method):
        """ Base method for storage operations.
        """
        uuids = self.uuids()

        while True:
            chunk = list(islice(uuids, 0, self.chunk_size))

            if not chunk:
                return

            rest_request(method, self.storage_url, chunk)

def _base_opration ( self , method ) : uuids = self . uuids ( ) while True : chunk = list ( islice ( uuids , 0 , self . chunk_size ) ) if not chunk : return rest_request ( method , self . storage_url , chunk )


def uuids(self):
        """ Extract uuid from each item of specified ``seq``.
        """
        for f in self._seq:
            if isinstance(f, File):
                yield f.uuid
            elif isinstance(f, six.string_types):
                yield f
            else:
                raise ValueError(
                    'Invalid type for sequence item: {0}'.format(type(f)))

def
uuids
(
self
)
:
for
f
in
self
.
_seq
:
if
isinstance
(
f
,
File
)
:
yield
f
.
uuid
elif
isinstance
(
f
,
six
.
string_types
)
:
yield
f
else
:
raise
ValueError
(
'Invalid type for sequence item: {0}'
.
format
(
type
(
f
)
)
)

Extract uuid from each item of specified seq .


def _list(api_list_class, arg_namespace, **extra):
    """ A common function for building methods of the "list showing".
    """
    if arg_namespace.starting_point:
        ordering_field = (arg_namespace.ordering or '').lstrip('-')
        if ordering_field in ('', 'datetime_uploaded', 'datetime_created'):
            arg_namespace.starting_point = parser.parse(
                arg_namespace.starting_point)

    items = api_list_class(
        starting_point=arg_namespace.starting_point,
        ordering=arg_namespace.ordering,
        limit=arg_namespace.limit,
        request_limit=arg_namespace.request_limit,
        **extra
    )
    items.constructor = lambda x: x

    try:
        pprint(list(items))
    except ValueError as e:
        print(e)

def _list ( api_list_class , arg_namespace , * * extra ) : if arg_namespace . starting_point : ordering_field = ( arg_namespace . ordering or '' ) . lstrip ( '-' ) if ordering_field in ( '' , 'datetime_uploaded' , 'datetime_created' ) : arg_namespace . starting_point = parser . parse ( arg_namespace . starting_point ) items = api_list_class ( starting_point = arg_namespace . starting_point , ordering = arg_namespace . ordering , limit = arg_namespace . limit , request_limit = arg_namespace . request_limit , * * extra ) items . constructor = lambda x : x try : pprint ( list ( items ) ) except ValueError as e : print ( e )

A common function for building methods of the list showing .


def bar(iter_content, parts, title=''):
    """ Iterates over the "iter_content" and draws a progress bar to stdout.
    """
    parts = max(float(parts), 1.0)
    cells = 10
    progress = 0
    step = cells / parts

    draw = lambda progress: sys.stdout.write(
        '\r[{0:10}] {1:.2f}% {2}'.format(
            '#'*int(progress), progress * cells, title))

    for chunk in iter_content:
        yield chunk

        progress += step
        draw(progress)
        sys.stdout.flush()

    draw(cells)
    print('')

def
bar
(
iter_content
,
parts
,
title
=
''
)
:
parts
=
max
(
float
(
parts
)
,
1.0
)
cells
=
10
progress
=
0
step
=
cells
/
parts
draw
=
lambda
progress
:
sys
.
stdout
.
write
(
'\r[{0:10}] {1:.2f}% {2}'
.
format
(
'#'
*
int
(
progress
)
,
progress
*
cells
,
title
)
)
for
chunk
in
iter_content
:
yield
chunk
progress
+=
step
draw
(
progress
)
sys
.
stdout
.
flush
(
)
draw
(
cells
)
print
(
''
)

Iterates over the iter_content and draws a progress bar to stdout .


def uploading_request(verb, path, data=None, files=None, timeout=conf.DEFAULT):
    """Makes Uploading API request and returns response as ``dict``.

    It takes settings from ``conf`` module.

    Make sure that given ``path`` does not contain leading slash.

    Usage example::

        >>> file_obj = open('photo.jpg', 'rb')
        >>> uploading_request('POST', 'base/', files={'file': file_obj})
        {
            'file': '9b9f4483-77b8-40ae-a198-272ba6280004'
        }
        >>> File('9b9f4483-77b8-40ae-a198-272ba6280004')

    """
    path = path.lstrip('/')
    url = urljoin(conf.upload_base, path)

    if data is None:
        data = {}
    data['pub_key'] = conf.pub_key
    data['UPLOADCARE_PUB_KEY'] = conf.pub_key

    headers = {
        'User-Agent': _build_user_agent(),
    }

    try:
        response = session.request(
            str(verb), url, allow_redirects=True,
            verify=conf.verify_upload_ssl, data=data, files=files,
            headers=headers, timeout=_get_timeout(timeout),
        )
    except requests.RequestException as exc:
        raise APIConnectionError(exc.args[0])

    # No content.
    if response.status_code == 204:
        return {}

    if 200 <= response.status_code < 300:
        if _content_type_from_response(response).endswith(('/json', '+json')):
            try:
                return response.json()
            except ValueError as exc:
                raise APIError(exc.args[0])

    if response.status_code in (400, 404):
        raise InvalidRequestError(response.content)

    # Not json or unknown status code.
    raise APIError(response.content)

def
uploading_request
(
verb
,
path
,
data
=
,
files
=
,
timeout
=
conf
.
DEFAULT
)
:
path
=
path
.
lstrip
(
'/'
)
url
=
urljoin
(
conf
.
upload_base
,
path
)
if
data
is
:
data
=
{
}
data
[
'pub_key'
]
=
conf
.
pub_key
data
[
'UPLOADCARE_PUB_KEY'
]
=
conf
.
pub_key
headers
=
{
'User-Agent'
:
_build_user_agent
(
)
,
}
try
:
response
=
session
.
request
(
str
(
verb
)
,
url
,
allow_redirects
=
True
,
verify
=
conf
.
verify_upload_ssl
,
data
=
data
,
files
=
files
,
headers
=
headers
,
timeout
=
_get_timeout
(
timeout
)
,
)
except
requests
.
RequestException
as
exc
:
raise
APIConnectionError
(
exc
.
args
[
0
]
)
# No content.
if
response
.
status_code
==
204
:
return
{
}
if
200
<=
response
.
status_code
<
300
:
if
_content_type_from_response
(
response
)
.
endswith
(
(
'/json'
,
'+json'
)
)
:
try
:
return
response
.
json
(
)
except
ValueError
as
exc
:
raise
APIError
(
exc
.
args
[
0
]
)
if
response
.
status_code
in
(
400
,
404
)
:
raise
InvalidRequestError
(
response
.
content
)
# Not json or unknown status code.
raise
APIError
(
response
.
content
)

Makes Uploading API request and returns response as dict .


def home_mode_status(self, **kwargs):
        """Returns the status of Home Mode"""
        api = self._api_info['home_mode']
        payload = dict({
            'api': api['name'],
            'method': 'GetInfo',
            'version': api['version'],
            '_sid': self._sid
        }, **kwargs)
        response = self._get_json_with_retry(api['url'], payload)

        return response['data']['on']

def home_mode_status ( self , * * kwargs ) : api = self . _api_info [ 'home_mode' ] payload = dict ( { 'api' : api [ 'name' ] , 'method' : 'GetInfo' , 'version' : api [ 'version' ] , '_sid' : self . _sid } , * * kwargs ) response = self . _get_json_with_retry ( api [ 'url' ] , payload ) return response [ 'data' ] [ 'on' ]


def camera_list(self, **kwargs):
        """Return a list of cameras."""
        api = self._api_info['camera']
        payload = dict({
            '_sid': self._sid,
            'api': api['name'],
            'method': 'List',
            'version': api['version'],
        }, **kwargs)
        response = self._get_json_with_retry(api['url'], payload)

        cameras = []

        for data in response['data']['cameras']:
            cameras.append(Camera(data, self._video_stream_url))

        return cameras

def camera_list ( self , * * kwargs ) : api = self . _api_info [ 'camera' ] payload = dict ( { '_sid' : self . _sid , 'api' : api [ 'name' ] , 'method' : 'List' , 'version' : api [ 'version' ] , } , * * kwargs ) response = self . _get_json_with_retry ( api [ 'url' ] , payload ) cameras = [ ] for data in response [ 'data' ] [ 'cameras' ] : cameras . append ( Camera ( data , self . _video_stream_url ) ) return cameras


def camera_info(self, camera_ids, **kwargs):
        """Return a list of cameras matching camera_ids."""
        api = self._api_info['camera']
        payload = dict({
            '_sid': self._sid,
            'api': api['name'],
            'method': 'GetInfo',
            'version': api['version'],
            'cameraIds': ', '.join(str(id) for id in camera_ids),
        }, **kwargs)
        response = self._get_json_with_retry(api['url'], payload)

        cameras = []

        for data in response['data']['cameras']:
            cameras.append(Camera(data, self._video_stream_url))

        return cameras

def
camera_info
(
self
,
camera_ids
,
*
*
kwargs
)
:
api
=
self
.
_api_info
[
'camera'
]
payload
=
dict
(
{
'_sid'
:
self
.
_sid
,
'api'
:
api
[
'name'
]
,
'method'
:
'GetInfo'
,
'version'
:
api
[
'version'
]
,
'cameraIds'
:
', '
.
join
(
str
(
id
)
for
id
in
camera_ids
)
,
}
,
*
*
kwargs
)
response
=
self
.
_get_json_with_retry
(
api
[
'url'
]
,
payload
)
cameras
=
[
]
for
data
in
response
[
'data'
]
[
'cameras'
]
:
cameras
.
append
(
Camera
(
data
,
self
.
_video_stream_url
)
)
return
cameras

Return a list of cameras matching camera_ids .


def camera_snapshot(self, camera_id, **kwargs):
        """Return bytes of camera image."""
        api = self._api_info['camera']
        payload = dict({
            '_sid': self._sid,
            'api': api['name'],
            'method': 'GetSnapshot',
            'version': api['version'],
            'cameraId': camera_id,
        }, **kwargs)
        response = self._get(api['url'], payload)

        return response.content

def camera_snapshot ( self , camera_id , * * kwargs ) : api = self . _api_info [ 'camera' ] payload = dict ( { '_sid' : self . _sid , 'api' : api [ 'name' ] , 'method' : 'GetSnapshot' , 'version' : api [ 'version' ] , 'cameraId' : camera_id , } , * * kwargs ) response = self . _get ( api [ 'url' ] , payload ) return response . content


def camera_disable(self, camera_id, **kwargs):
        """Disable camera."""
        api = self._api_info['camera']
        payload = dict({
            '_sid': self._sid,
            'api': api['name'],
            'method': 'Disable',
            'version': 9,
            'idList': camera_id,
        }, **kwargs)
        print(api['url'])
        print(payload)
        response = self._get(api['url'], payload)

        return response['success']

def camera_disable ( self , camera_id , * * kwargs ) : api = self . _api_info [ 'camera' ] payload = dict ( { '_sid' : self . _sid , 'api' : api [ 'name' ] , 'method' : 'Disable' , 'version' : 9 , 'idList' : camera_id , } , * * kwargs ) print ( api [ 'url' ] ) print ( payload ) response = self . _get ( api [ 'url' ] , payload ) return response [ 'success' ]


def camera_event_motion_enum(self, camera_id, **kwargs):
        """Return motion settings matching camera_id."""
        api = self._api_info['camera_event']
        payload = dict({
            '_sid': self._sid,
            'api': api['name'],
            'method': 'MotionEnum',
            'version': api['version'],
            'camId': camera_id,
        }, **kwargs)
        response = self._get_json_with_retry(api['url'], payload)

        return MotionSetting(camera_id, response['data']['MDParam'])

def camera_event_motion_enum ( self , camera_id , * * kwargs ) : api = self . _api_info [ 'camera_event' ] payload = dict ( { '_sid' : self . _sid , 'api' : api [ 'name' ] , 'method' : 'MotionEnum' , 'version' : api [ 'version' ] , 'camId' : camera_id , } , * * kwargs ) response = self . _get_json_with_retry ( api [ 'url' ] , payload ) return MotionSetting ( camera_id , response [ 'data' ] [ 'MDParam' ] )

Return motion settings matching camera_id .


def camera_event_md_param_save(self, camera_id, **kwargs):
        """Update motion settings matching camera_id with keyword args."""
        api = self._api_info['camera_event']
        payload = dict({
            '_sid': self._sid,
            'api': api['name'],
            'method': 'MDParamSave',
            'version': api['version'],
            'camId': camera_id,
        }, **kwargs)
        response = self._get_json_with_retry(api['url'], payload)

        return response['data']['camId']

def camera_event_md_param_save ( self , camera_id , * * kwargs ) : api = self . _api_info [ 'camera_event' ] payload = dict ( { '_sid' : self . _sid , 'api' : api [ 'name' ] , 'method' : 'MDParamSave' , 'version' : api [ 'version' ] , 'camId' : camera_id , } , * * kwargs ) response = self . _get_json_with_retry ( api [ 'url' ] , payload ) return response [ 'data' ] [ 'camId' ]

Update motion settings matching camera_id with keyword args .


def update(self):
        """Update cameras and motion settings with latest from API."""
        cameras = self._api.camera_list()
        self._cameras_by_id = {v.camera_id: v for i, v in enumerate(cameras)}

        motion_settings = []
        for camera_id in self._cameras_by_id.keys():
            motion_setting = self._api.camera_event_motion_enum(camera_id)
            motion_settings.append(motion_setting)

        self._motion_settings_by_id = {
            v.camera_id: v for i, v in enumerate(motion_settings)}

def update ( self ) : cameras = self . _api . camera_list ( ) self . _cameras_by_id = { v . camera_id : v for i , v in enumerate ( cameras ) } motion_settings = [ ] for camera_id in self . _cameras_by_id . keys ( ) : motion_setting = self . _api . camera_event_motion_enum ( camera_id ) motion_settings . append ( motion_setting ) self . _motion_settings_by_id = { v . camera_id : v for i , v in enumerate ( motion_settings ) }

Update cameras and motion settings with latest from API .


def is_last_li(li, meta_data, current_numId):
    """
    Determine if ``li`` is the last list item for a given list
    """
    if not is_li(li, meta_data):
        return False
    w_namespace = get_namespace(li, 'w')
    next_el = li
    while True:
        # If we run out of element this must be the last list item
        if next_el is None:
            return True

        next_el = next_el.getnext()
        # Ignore elements that are not a list item
        if not is_li(next_el, meta_data):
            continue

        new_numId = get_numId(next_el, w_namespace)
        if current_numId != new_numId:
            return True
        # If we have gotten here then we have found another list item in the
        # current list, so ``li`` is not the last li in the list.
        return False

Determine if li is the last list item for a given list


def get_single_list_nodes_data(li, meta_data):
    """
    Find consecutive li tags that have content that have the same list id.
    """
    yield li
    w_namespace = get_namespace(li, 'w')
    current_numId = get_numId(li, w_namespace)
    starting_ilvl = get_ilvl(li, w_namespace)
    el = li
    while True:
        el = el.getnext()
        if el is None:
            break
        # If the tag has no content ignore it.
        if not has_text(el):
            continue

        # Stop the lists if you come across a list item that should be a
        # heading.
        if _is_top_level_upper_roman(el, meta_data):
            break

        if (
                is_li(el, meta_data) and
                (starting_ilvl > get_ilvl(el, w_namespace))):
            break

        new_numId = get_numId(el, w_namespace)
        if new_numId is None or new_numId == -1:
            # Not a p tag or a list item
            yield el
            continue
        # If the list id of the next tag is different that the previous that
        # means a new list being made (not nested)
        if current_numId != new_numId:
            # Not a subsequent list.
            break
        if is_last_li(el, meta_data, current_numId):
            yield el
            break
        yield el

Find consecutive li tags that have content that have the same list id .


def get_ilvl(li, w_namespace):
    """
    The ilvl on an li tag tells the li tag at what level of indentation this
    tag is at. This is used to determine if the li tag needs to be nested or
    not.
    """
    ilvls = li.xpath('.//w:ilvl', namespaces=li.nsmap)
    if len(ilvls) == 0:
        return -1
    return int(ilvls[0].get('%sval' % w_namespace))

def get_ilvl ( li , w_namespace ) : ilvls = li . xpath ( './/w:ilvl' , namespaces = li . nsmap ) if len ( ilvls ) == 0 : return - 1 return int ( ilvls [ 0 ] . get ( '%sval' % w_namespace ) )

The ilvl on an li tag tells the li tag at what level of indentation this tag is at . This is used to determine if the li tag needs to be nested or not .


def get_v_merge(tc):
    """
    vMerge is what docx uses to denote that a table cell is part of a rowspan.
    The first cell to have a vMerge is the start of the rowspan, and the vMerge
    will be denoted with 'restart'. If it is anything other than restart then
    it is a continuation of another rowspan.
    """
    if tc is None:
        return None
    v_merges = tc.xpath('.//w:vMerge', namespaces=tc.nsmap)
    if len(v_merges) != 1:
        return None
    v_merge = v_merges[0]
    return v_merge

def get_v_merge ( tc ) : if tc is None : return None v_merges = tc . xpath ( './/w:vMerge' , namespaces = tc . nsmap ) if len ( v_merges ) != 1 : return None v_merge = v_merges [ 0 ] return v_merge

vMerge is what docx uses to denote that a table cell is part of a rowspan . The first cell to have a vMerge is the start of the rowspan and the vMerge will be denoted with restart . If it is anything other than restart then it is a continuation of another rowspan .


def get_grid_span(tc):
    """
    gridSpan is what docx uses to denote that a table cell has a colspan. This
    is much more simple than rowspans in that there is a one-to-one mapping
    from gridSpan to colspan.
    """
    w_namespace = get_namespace(tc, 'w')
    grid_spans = tc.xpath('.//w:gridSpan', namespaces=tc.nsmap)
    if len(grid_spans) != 1:
        return 1
    grid_span = grid_spans[0]
    return int(grid_span.get('%sval' % w_namespace))

def get_grid_span ( tc ) : w_namespace = get_namespace ( tc , 'w' ) grid_spans = tc . xpath ( './/w:gridSpan' , namespaces = tc . nsmap ) if len ( grid_spans ) != 1 : return 1 grid_span = grid_spans [ 0 ] return int ( grid_span . get ( '%sval' % w_namespace ) )

gridSpan is what docx uses to denote that a table cell has a colspan . This is much more simple than rowspans in that there is a one - to - one mapping from gridSpan to colspan .


def get_td_at_index(tr, index):
    """
    When calculating the rowspan for a given cell it is required to find all
    table cells 'below' the initial cell with a v_merge. This function will
    return the td element at the passed in index, taking into account colspans.
    """
    current = 0
    for td in tr.xpath('.//w:tc', namespaces=tr.nsmap):
        if index == current:
            return td
        current += get_grid_span(td)

def get_td_at_index ( tr , index ) : current = 0 for td in tr . xpath ( './/w:tc' , namespaces = tr . nsmap ) : if index == current : return td current += get_grid_span ( td )

When calculating the rowspan for a given cell it is required to find all table cells below the initial cell with a v_merge . This function will return the td element at the passed in index taking into account colspans .


def style_is_false(style):
    """
    For bold, italics and underline. Simply checking to see if the various tags
    are present will not suffice. If the tag is present and set to False then
    the style should not be present.
    """
    if style is None:
        return False
    w_namespace = get_namespace(style, 'w')
    return style.get('%sval' % w_namespace) != 'false'

def style_is_false ( style ) : if style is None : return False w_namespace = get_namespace ( style , 'w' ) return style . get ( '%sval' % w_namespace ) != 'false'

For bold italics and underline . Simply checking to see if the various tags are present will not suffice . If the tag is present and set to False then the style should not be present .


def is_bold(r):
    """
    The function will return True if the r tag passed in is considered bold.
    """
    w_namespace = get_namespace(r, 'w')
    rpr = r.find('%srPr' % w_namespace)
    if rpr is None:
        return False
    bold = rpr.find('%sb' % w_namespace)
    return style_is_false(bold)

def is_bold ( r ) : w_namespace = get_namespace ( r , 'w' ) rpr = r . find ( '%srPr' % w_namespace ) if rpr is None : return False bold = rpr . find ( '%sb' % w_namespace ) return style_is_false ( bold )

The function will return True if the r tag passed in is considered bold .


def is_italics(r):
    """
    The function will return True if the r tag passed in is considered
    italicized.
    """
    w_namespace = get_namespace(r, 'w')
    rpr = r.find('%srPr' % w_namespace)
    if rpr is None:
        return False
    italics = rpr.find('%si' % w_namespace)
    return style_is_false(italics)

def is_italics ( r ) : w_namespace = get_namespace ( r , 'w' ) rpr = r . find ( '%srPr' % w_namespace ) if rpr is None : return False italics = rpr . find ( '%si' % w_namespace ) return style_is_false ( italics )

The function will return True if the r tag passed in is considered italicized .


def is_underlined(r):
    """
    The function will return True if the r tag passed in is considered
    underlined.
    """
    w_namespace = get_namespace(r, 'w')
    rpr = r.find('%srPr' % w_namespace)
    if rpr is None:
        return False
    underline = rpr.find('%su' % w_namespace)
    return style_is_false(underline)

def is_underlined ( r ) : w_namespace = get_namespace ( r , 'w' ) rpr = r . find ( '%srPr' % w_namespace ) if rpr is None : return False underline = rpr . find ( '%su' % w_namespace ) return style_is_false ( underline )

The function will return True if the r tag passed in is considered underlined .


def is_title(p):
    """
    Certain p tags are denoted as ``Title`` tags. This function will return
    True if the passed in p tag is considered a title.
    """
    w_namespace = get_namespace(p, 'w')
    styles = p.xpath('.//w:pStyle', namespaces=p.nsmap)
    if len(styles) == 0:
        return False
    style = styles[0]
    return style.get('%sval' % w_namespace) == 'Title'

def is_title ( p ) : w_namespace = get_namespace ( p , 'w' ) styles = p . xpath ( './/w:pStyle' , namespaces = p . nsmap ) if len ( styles ) == 0 : return False style = styles [ 0 ] return style . get ( '%sval' % w_namespace ) == 'Title'

Certain p tags are denoted as Title tags . This function will return True if the passed in p tag is considered a title .


def get_text_run_content_data(r):
    """
    It turns out that r tags can contain both t tags and drawing tags. Since we
    need both, this function will return them in the order in which they are
    found.
    """
    w_namespace = get_namespace(r, 'w')
    valid_elements = (
        '%st' % w_namespace,
        '%sdrawing' % w_namespace,
        '%spict' % w_namespace,
        '%sbr' % w_namespace,
    )
    for el in r:
        if el.tag in valid_elements:
            yield el

def get_text_run_content_data ( r ) : w_namespace = get_namespace ( r , 'w' ) valid_elements = ( '%st' % w_namespace , '%sdrawing' % w_namespace , '%spict' % w_namespace , '%sbr' % w_namespace , ) for el in r : if el . tag in valid_elements : yield el

It turns out that r tags can contain both t tags and drawing tags . Since we need both this function will return them in the order in which they are found .


def get_relationship_info(tree, media, image_sizes):
    """
    There is a separate file holds the targets to links as well as the targets
    for images. Return a dictionary based on the relationship id and the
    target.
    """
    if tree is None:
        return {}
    result = {}
    # Loop through each relationship.
    for el in tree.iter():
        el_id = el.get('Id')
        if el_id is None:
            continue
        # Store the target in the result dict.
        target = el.get('Target')
        if any(
                target.lower().endswith(ext) for
                ext in IMAGE_EXTENSIONS_TO_SKIP):
            continue
        if target in media:
            image_size = image_sizes.get(el_id)
            target = convert_image(media[target], image_size)
        # cgi will replace things like & < > with & < >
        result[el_id] = cgi.escape(target)

    return result

There is a separate file holds the targets to links as well as the targets for images . Return a dictionary based on the relationship id and the target .


def _get_document_data(f, image_handler=None):
    '''
    ``f`` is a ``ZipFile`` that is open
    Extract out the document data, numbering data and the relationship data.
    '''
    if image_handler is None:
        def image_handler(image_id, relationship_dict):
            return relationship_dict.get(image_id)

    document_xml = None
    numbering_xml = None
    relationship_xml = None
    styles_xml = None
    parser = etree.XMLParser(strip_cdata=False)
    path, _ = os.path.split(f.filename)
    media = {}
    image_sizes = {}
    # Loop through the files in the zip file.
    for item in f.infolist():
        # This file holds all the content of the document.
        if item.filename == 'word/document.xml':
            xml = f.read(item.filename)
            document_xml = etree.fromstring(xml, parser)
        # This file tells document.xml how lists should look.
        elif item.filename == 'word/numbering.xml':
            xml = f.read(item.filename)
            numbering_xml = etree.fromstring(xml, parser)
        elif item.filename == 'word/styles.xml':
            xml = f.read(item.filename)
            styles_xml = etree.fromstring(xml, parser)
        # This file holds the targets for hyperlinks and images.
        elif item.filename == 'word/_rels/document.xml.rels':
            xml = f.read(item.filename)
            try:
                relationship_xml = etree.fromstring(xml, parser)
            except XMLSyntaxError:
                relationship_xml = etree.fromstring('<xml></xml>', parser)
        if item.filename.startswith('word/media/'):
            # Strip off the leading word/
            media[item.filename[len('word/'):]] = f.extract(
                item.filename,
                path,
            )
    # Close the file pointer.
    f.close()

    # Get dictionaries for the numbering and the relationships.
    numbering_dict = get_numbering_info(numbering_xml)
    image_sizes = get_image_sizes(document_xml)
    relationship_dict = get_relationship_info(
        relationship_xml,
        media,
        image_sizes
    )
    styles_dict = get_style_dict(styles_xml)
    font_sizes_dict = defaultdict(int)
    if DETECT_FONT_SIZE:
        font_sizes_dict = get_font_sizes_dict(document_xml, styles_dict)
    meta_data = MetaData(
        numbering_dict=numbering_dict,
        relationship_dict=relationship_dict,
        styles_dict=styles_dict,
        font_sizes_dict=font_sizes_dict,
        image_handler=image_handler,
        image_sizes=image_sizes,
    )
    return document_xml, meta_data

f is a ZipFile that is open Extract out the document data numbering data and the relationship data .


def get_ordered_list_type(meta_data, numId, ilvl):
    """
    Return the list type. If numId or ilvl not in the numbering dict then
    default to returning decimal.

    This function only cares about ordered lists, unordered lists get dealt
    with elsewhere.
    """

    # Early return if numId or ilvl are not valid
    numbering_dict = meta_data.numbering_dict
    if numId not in numbering_dict:
        return DEFAULT_LIST_NUMBERING_STYLE
    if ilvl not in numbering_dict[numId]:
        return DEFAULT_LIST_NUMBERING_STYLE
    return meta_data.numbering_dict[numId][ilvl]

Return the list type . If numId or ilvl not in the numbering dict then default to returning decimal .


def build_list(li_nodes, meta_data):
    """
    Build the list structure and return the root list
    """
    # Need to keep track of all incomplete nested lists.
    ol_dict = {}

    # Need to keep track of the current indentation level.
    current_ilvl = -1

    # Need to keep track of the current list id.
    current_numId = -1

    # Need to keep track of list that new li tags should be added too.
    current_ol = None

    # Store the first list created (the root list) for the return value.
    root_ol = None
    visited_nodes = []
    list_contents = []

    def _build_li(list_contents):
        data = '
'.join(t for t in list_contents if t is not None)
        return etree.XML('
%s
' % data)

    def _build_non_li_content(el, meta_data):
        w_namespace = get_namespace(el, 'w')
        if el.tag == '%stbl' % w_namespace:
            new_el, visited_nodes = build_table(el, meta_data)
            return etree.tostring(new_el), visited_nodes
        elif el.tag == '%sp' % w_namespace:
            return get_element_content(el, meta_data), [el]
        if has_text(el):
            raise UnintendedTag('Did not expect %s' % el.tag)

    def _merge_lists(ilvl, current_ilvl, ol_dict, current_ol):
        for i in reversed(range(ilvl, current_ilvl)):
            # Any list that is more indented that ilvl needs to
            # be merged to the list before it.
            if i not in ol_dict:
                continue
            if ol_dict[i] is not current_ol:
                if ol_dict[i] is current_ol:
                    continue
                ol_dict[i][-1].append(current_ol)
                current_ol = ol_dict[i]

        # Clean up finished nested lists.
        for key in list(ol_dict):
            if key > ilvl:
                del ol_dict[key]
        return current_ol

    for li_node in li_nodes:
        w_namespace = get_namespace(li_node, 'w')
        if not is_li(li_node, meta_data):
            # Get the content and visited nodes
            new_el, el_visited_nodes = _build_non_li_content(
                li_node,
                meta_data,
            )
            list_contents.append(new_el)
            visited_nodes.extend(el_visited_nodes)
            continue
        if list_contents:
            li_el = _build_li(list_contents)
            list_contents = []
            current_ol.append(li_el)
        # Get the data needed to build the current list item
        list_contents.append(get_element_content(
            li_node,
            meta_data,
        ))
        ilvl = get_ilvl(li_node, w_namespace)
        numId = get_numId(li_node, w_namespace)
        list_type = get_ordered_list_type(meta_data, numId, ilvl)

        # If the ilvl is greater than the current_ilvl or the list id is
        # changing then we have the first li tag in a nested list. We need to
        # create a new list object and update all of our variables for keeping
        # track.
        if (ilvl > current_ilvl) or (numId != current_numId):
            # Only create a new list
            ol_dict[ilvl] = create_list(list_type)
            current_ol = ol_dict[ilvl]
            current_ilvl = ilvl
            current_numId = numId
        # Both cases above are not True then we need to close all lists greater
        # than ilvl and then remove them from the ol_dict
        else:
            # Merge any nested lists that need to be merged.
            current_ol = _merge_lists(
                ilvl=ilvl,
                current_ilvl=current_ilvl,
                ol_dict=ol_dict,
                current_ol=current_ol,
            )

        # Set the root list after the first list is created.
        if root_ol is None:
            root_ol = current_ol

        # Set the current list.
        if ilvl in ol_dict:
            current_ol = ol_dict[ilvl]
        else:
            # In some instances the ilvl is not in the ol_dict, if that is the
            # case, create it here (not sure how this happens but it has
            # before.) Only do this if the current_ol is not the root_ol,
            # otherwise etree will crash.

            if current_ol is not root_ol:

                # Merge the current_ol into the root_ol. _merge_lists is not
                # equipped to handle this situation since the only way to get
                # into this block of code is to have mangled ilvls.
                root_ol[-1].append(current_ol)

                # Reset the current_ol
                current_ol = create_list(list_type)

        # Create the li element.
        visited_nodes.extend(list(li_node.iter()))

    # If a list item is the last thing in a document, then you will need to add
    # it here. Should probably figure out how to get the above logic to deal
    # with it.
    if list_contents:
        li_el = _build_li(list_contents)
        list_contents = []
        current_ol.append(li_el)

    # Merge up any nested lists that have not been merged.
    current_ol = _merge_lists(
        ilvl=0,
        current_ilvl=current_ilvl,
        ol_dict=ol_dict,
        current_ol=current_ol,
    )

    return root_ol, visited_nodes


def build_tr(tr, meta_data, row_spans):
    """
    This will return a single tr element, with all tds already populated.
    """

    # Create a blank tr element.
    tr_el = etree.Element('tr')
    w_namespace = get_namespace(tr, 'w')
    visited_nodes = []
    for el in tr:
        if el in visited_nodes:
            continue
        visited_nodes.append(el)
        # Find the table cells.
        if el.tag == '%stc' % w_namespace:
            v_merge = get_v_merge(el)
            # If there is a v_merge and it is not restart then this cell can be
            # ignored.
            if (
                    v_merge is not None and
                    v_merge.get('%sval' % w_namespace) != 'restart'):
                continue

            # Loop through each and build a list of all the content.
            texts = []
            for td_content in el:
                # Since we are doing look-a-heads in this loop we need to check
                # again to see if we have already visited the node.
                if td_content in visited_nodes:
                    continue

                # Check to see if it is a list or a regular paragraph.
                if is_li(td_content, meta_data):
                    # If it is a list, create the list and update
                    # visited_nodes.
                    li_nodes = get_single_list_nodes_data(
                        td_content,
                        meta_data,
                    )
                    list_el, list_visited_nodes = build_list(
                        li_nodes,
                        meta_data,
                    )
                    visited_nodes.extend(list_visited_nodes)
                    texts.append(etree.tostring(list_el))
                elif td_content.tag == '%stbl' % w_namespace:
                    table_el, table_visited_nodes = build_table(
                        td_content,
                        meta_data,
                    )
                    visited_nodes.extend(table_visited_nodes)
                    texts.append(etree.tostring(table_el))
                elif td_content.tag == '%stcPr' % w_namespace:
                    # Do nothing
                    visited_nodes.append(td_content)
                    continue
                else:
                    text = get_element_content(
                        td_content,
                        meta_data,
                        is_td=True,
                    )
                    texts.append(text)

            data = '
'.join(t for t in texts if t is not None)
            td_el = etree.XML('%s	' % data)
            # if there is a colspan then set it here.
            colspan = get_grid_span(el)
            if colspan > 1:
                td_el.set('colspan', '%d' % colspan)
            v_merge = get_v_merge(el)

            # If this td has a v_merge and it is restart then set the rowspan
            # here.
            if (
                    v_merge is not None and
                    v_merge.get('%sval' % w_namespace) == 'restart'):
                rowspan = next(row_spans)
                td_el.set('rowspan', '%d' % rowspan)

            tr_el.append(td_el)
    return tr_el

This will return a single tr element with all tds already populated .


def build_table(table, meta_data):
    """
    This returns a table object with all rows and cells correctly populated.
    """

    # Create a blank table element.
    table_el = etree.Element('table')
    w_namespace = get_namespace(table, 'w')

    # Get the rowspan values for cells that have a rowspan.
    row_spans = get_rowspan_data(table)
    for el in table:
        if el.tag == '%str' % w_namespace:
            # Create the tr element.
            tr_el = build_tr(
                el,
                meta_data,
                row_spans,
            )
            # And append it to the table.
            table_el.append(tr_el)

    visited_nodes = list(table.iter())
    return table_el, visited_nodes

This returns a table object with all rows and cells correctly populated .


def get_t_tag_content(
        t, parent, remove_bold, remove_italics, meta_data):
    """
    Generate the string data that for this particular t tag.
    """
    if t is None or t.text is None:
        return ''

    # Need to escape the text so that we do not accidentally put in text
    # that is not valid XML.
    # cgi will replace things like & < > with & < >
    text = cgi.escape(t.text)

    # Wrap the text with any modifiers it might have (bold, italics or
    # underline)
    el_is_bold = not remove_bold and (
        is_bold(parent) or
        is_underlined(parent)
    )
    el_is_italics = not remove_italics and is_italics(parent)
    if el_is_bold:
        text = '%s' % text
    if el_is_italics:
        text = '%s' % text
    return text

Generate the string data that for this particular t tag .


def _strip_tag(tree, tag):
    """
    Remove all tags that have the tag name ``tag``
    """
    for el in tree.iter():
        if el.tag == tag:
            el.getparent().remove(el)

def _strip_tag ( tree , tag ) : for el in tree . iter ( ) : if el . tag == tag : el . getparent ( ) . remove ( el )

Remove all tags that have the tag name tag


def find(dataset, url):
    '''Find the location of a dataset on disk, downloading if needed.'''
    fn = os.path.join(DATASETS, dataset)
    dn = os.path.dirname(fn)
    if not os.path.exists(dn):
        print('creating dataset directory: %s', dn)
        os.makedirs(dn)
    if not os.path.exists(fn):
        if sys.version_info < (3, ):
            urllib.urlretrieve(url, fn)
        else:
            urllib.request.urlretrieve(url, fn)
    return fn

def
find
(
dataset
,
url
)
:
fn
=
os
.
path
.
join
(
DATASETS
,
dataset
)
dn
=
os
.
path
.
dirname
(
fn
)
if
not
os
.
path
.
exists
(
dn
)
:
print
(
'creating dataset directory: %s'
,
dn
)
os
.
makedirs
(
dn
)
if
not
os
.
path
.
exists
(
fn
)
:
if
sys
.
version_info
<
(
3
,
)
:
urllib
.
urlretrieve
(
url
,
fn
)
else
:
urllib
.
request
.
urlretrieve
(
url
,
fn
)
return
fn

Find the location of a dataset on disk downloading if needed .


def load_mnist(flatten=True, labels=False):
    '''Load the MNIST digits dataset.'''
    fn = find('mnist.pkl.gz', 'http://deeplearning.net/data/mnist/mnist.pkl.gz')
    h = gzip.open(fn, 'rb')
    if sys.version_info < (3, ):
        (timg, tlab), (vimg, vlab), (simg, slab) = pickle.load(h)
    else:
        (timg, tlab), (vimg, vlab), (simg, slab) = pickle.load(h, encoding='bytes')
    h.close()
    if not flatten:
        timg = timg.reshape((-1, 28, 28, 1))
        vimg = vimg.reshape((-1, 28, 28, 1))
        simg = simg.reshape((-1, 28, 28, 1))
    if labels:
        return ((timg, tlab.astype('i')),
                (vimg, vlab.astype('i')),
                (simg, slab.astype('i')))
    return (timg, ), (vimg, ), (simg, )


def load_cifar(flatten=True, labels=False):
    '''Load the CIFAR10 image dataset.'''
    def extract(name):
        print('extracting data from {}'.format(name))
        h = tar.extractfile(name)
        if sys.version_info < (3, ):
            d = pickle.load(h)
        else:
            d = pickle.load(h, encoding='bytes')
            for k in list(d):
                d[k.decode('utf8')] = d[k]
        h.close()
        img = d['data'].reshape(
            (-1, 3, 32, 32)).transpose((0, 2, 3, 1)).astype('f') / 128 - 1
        if flatten:
            img = img.reshape((-1, 32 * 32 * 3))
        d['data'] = img
        return d

    fn = find('cifar10.tar.gz', 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz')
    tar = tarfile.open(fn)

    imgs = []
    labs = []
    for i in range(1, 6):
        d = extract('cifar-10-batches-py/data_batch_{}'.format(i))
        imgs.extend(d['data'])
        labs.extend(d['labels'])
    timg = np.asarray(imgs[:40000])
    tlab = np.asarray(labs[:40000], 'i')
    vimg = np.asarray(imgs[40000:])
    vlab = np.asarray(labs[40000:], 'i')

    d = extract('cifar-10-batches-py/test_batch')
    simg = d['data']
    slab = d['labels']

    tar.close()

    if labels:
        return (timg, tlab), (vimg, vlab), (simg, slab)
    return (timg, ), (vimg, ), (simg, )


def plot_images(imgs, loc, title=None, channels=1):
    '''Plot an array of images.

    We assume that we are given a matrix of data whose shape is (n*n, s*s*c) --
    that is, there are n^2 images along the first axis of the array, and each
    image is c squares measuring s pixels on a side. Each row of the input will
    be plotted as a sub-region within a single image array containing an n x n
    grid of images.
    '''
    n = int(np.sqrt(len(imgs)))
    assert n * n == len(imgs), 'images array must contain a square number of rows!'
    s = int(np.sqrt(len(imgs[0]) / channels))
    assert s * s == len(imgs[0]) / channels, 'images must be square!'

    img = np.zeros(((s+1) * n - 1, (s+1) * n - 1, channels), dtype=imgs[0].dtype)
    for i, pix in enumerate(imgs):
        r, c = divmod(i, n)
        img[r * (s+1):(r+1) * (s+1) - 1,
            c * (s+1):(c+1) * (s+1) - 1] = pix.reshape((s, s, channels))

    img -= img.min()
    img /= img.max()

    ax = plt.gcf().add_subplot(loc)
    ax.xaxis.set_visible(False)
    ax.yaxis.set_visible(False)
    ax.set_frame_on(False)
    ax.imshow(img.squeeze(), cmap=plt.cm.gray)
    if title:
        ax.set_title(title)