"""Create an example gallery out of a bunch of ipython notebooks

This sphinx extension extracts the ipython notebooks in a given folder to
create an example gallery. It provides the follwing configuration values
for you sphinx configuration file ``'conf.py'``:

.. autosummary::

    process_examples
    gallery_config

Notes
-----
This module was motivated by the
`sphinx-gallery <http://sphinx-gallery.readthedocs.org/en/latest/>`__ module
by Oscar Najera and in fact uses parts of it's html template for including the
thumbnails and the download containers"""
from __future__ import division
import datetime as dt
import os
import os.path as osp
import re
import six
from itertools import chain
import nbconvert
import nbformat
from shutil import copyfile
from copy import deepcopy
try:
    from sphinx.util import logging
    logger = logging.getLogger(__name__)
except (ImportError, AttributeError):
    import logging
    logger = logging.getLogger(__name__)

try:
    warn = logger.warn
except AttributeError:  # necessary for python 2.7
    warn = logger.warning

import subprocess as spr
from docutils.parsers.rst import Directive
from docutils.parsers.rst import directives
from docutils.statemachine import ViewList
from docutils import nodes

if six.PY2:
    from itertools import imap as map

try:
    from cyordereddict import OrderedDict
except ImportError:
    try:
        from collections import OrderedDict
    except ImportError:
        from ordereddict import OrderedDict

__version__ = '0.3.1'

__author__ = "Philipp Sommer"

if nbconvert.__version__ < '5.0':
    code_blocks = re.compile(r'\.\. code:: python\n(?s)(.+?)(?=\n\S+|$)')
    inner_code_blocks = re.compile(
        r'(?<=.. code:: python\n)(?s)(.+?)(?=\n\S+|$)')
else:
    code_blocks = re.compile(r'\.\. code:: ipython\d\n(?s)(.+?)(?=\n\S+|$)')
    inner_code_blocks = re.compile(
        r'(?<=.. code:: ipython\d\n)(?s)(.+?)(?=\n\S+|$)')
magic_patt = re.compile(r'(?m)^(\s+)(%.*\n)')

def isstring(s):
    return isinstance(s, six.string_types)

def create_dirs(*dirs):
    for d in dirs:
        if os.path.exists(d) and not os.path.isdir(d):
            raise IOError("Could not create directory %s because an "
                          "ordinary file with that name exists already!")
        elif not os.path.exists(d):
            os.makedirs(d)

def nbviewer_link(url):
    """Return the link to the Jupyter nbviewer for the given notebook url"""
    if six.PY2:
        from urlparse import urlparse as urlsplit
    else:
        from urllib.parse import urlsplit
    info = urlsplit(url)
    domain = info.netloc
    url_type = 'github' if domain == 'github.com' else 'url'
    return 'https://nbviewer.jupyter.org/%s%s' % (url_type, info.path)

NOIMAGE = os.path.join(os.path.dirname(__file__), '_static', 'no_image.png')

class NotebookProcessor(object):
    """Class to run process one ipython notebook and create the necessary files
    """

    #: base string for downloading the python file and ipython notebook
    CODE_DOWNLOAD = """

.. only:: html

    .. container:: sphx-glr-download

        **Download python file:** :download:`{pyfile}`

        **Download IPython notebook:** :download:`{nbfile}`
"""

    #: base string for viewing the notebook in the jupyter nbviewer
    CODE_DOWNLOAD_NBVIEWER = CODE_DOWNLOAD + """
        **View the notebook in the** `Jupyter nbviewer <{url}>`__
"""

    #: base string for downloading supplementary data
    DATA_DOWNLOAD = """

.. only:: html

    .. container:: sphx-glr-download

        **Download supplementary data:** %s
"""

    #: base string for creating the thumbnail
    THUMBNAIL_TEMPLATE = """
.. raw:: html

.. only:: html

    .. figure:: /{thumbnail}

        :ref:`{ref_name}`

.. raw:: html

"""

    CODE_TEMPLATE = """
.. raw:: html

.. only:: html

    .. code:: python

        {code}

    :ref:`{ref_name}`

.. raw:: html

"""

    BOKEH_STYLE_SHEET = (
        "http://cdn.pydata.org/bokeh/release/bokeh-{version}.min.css")

    BOKEH_JS = (
        "http://cdn.pydata.org/bokeh/release/bokeh-{version}.min.js")

    _BOKEH_TEMPLATE = """
.. raw:: html
    <link
        href="%s"
        rel="stylesheet" type="text/css">

"""
    BOKEH_TEMPLATE = _BOKEH_TEMPLATE % (BOKEH_STYLE_SHEET, BOKEH_JS)

    BOKEH_WIDGETS_STYLE_SHEET = (
        "http://cdn.pydata.org/bokeh/release/bokeh-{version}.min.css")

    BOKEH_WIDGETS_JS = (
        "http://cdn.pydata.org/bokeh/release/bokeh-{version}.min.js")

    BOKEH_WIDGETS_TEMPLATE = _BOKEH_TEMPLATE % (BOKEH_WIDGETS_STYLE_SHEET,
                                                BOKEH_WIDGETS_JS)

    #: Path to the thumbnail image
    thumb_file = NOIMAGE

    #: Paths to the pictures of this notebook
    pictures = []

    @property
    def thumbnail_div(self):
        """The string for creating the thumbnail of this example"""
        return self.THUMBNAIL_TEMPLATE.format(
            snippet=self.get_description()[1], thumbnail=self.thumb_file,
            ref_name=self.reference)

    @property
    def code_div(self):
        """The string for creating a code example for the gallery"""
        code_example = self.code_example
        if code_example is None:
            return None
        return self.CODE_TEMPLATE.format(
            snippet=self.get_description()[1], code=code_example,
            ref_name=self.reference)

    @property
    def code_example(self):
        """The code example out of the notebook metadata"""
        if self._code_example is not None:
            return self._code_example
        return getattr(self.nb.metadata, 'code_example', None)

    @property
    def supplementary_files(self):
        """The supplementary files of this notebook"""
        if self._supplementary_files is not None:
            return self._supplementary_files
        return getattr(self.nb.metadata, 'supplementary_files', None)

    @property
    def other_supplementary_files(self):
        """The supplementary files of this notebook"""
        if self._other_supplementary_files is not None:
            return self._other_supplementary_files
        return getattr(self.nb.metadata, 'other_supplementary_files', None)

    @property
    def reference(self):
        """The rst label of this example"""
        return 'gallery_' + self.outfile.replace(os.path.sep, '_').lower()

    @property
    def url(self):
        """The url on jupyter nbviewer for this notebook or None if unknown"""
        if self._url is not None:
            url = self._url
        else:
            url = getattr(self.nb.metadata, 'url', None)
        if url is not None:
            return nbviewer_link(url)

    @property
    def remove_tags(self):
        return any(self.tag_options.values())

    def __init__(self, infile, outfile, disable_warnings=True,
                 preprocess=True, clear=True, code_example=None,
                 supplementary_files=None, other_supplementary_files=None,
                 thumbnail_figure=None, url=None, insert_bokeh=False,
                 insert_bokeh_widgets=False, tag_options={}):
        """
        Parameters
        ----------
        infile: str
            path to the existing notebook
        outfile: str
            path to the new notebook
        disable_warnings: bool
            Boolean to control whether warnings shall be included in the rst
            file or not
        preprocess: bool
            If True, the notebook is processed when generating the rst file
        clear: bool
            If True, the output in the download notebook is cleared
        code_example: str
            A python code sample that shall be used instead of a thumbnail
            figure in the gallery. Note that you can also include a
            ``'code_example'`` key in the metadata of the notebook
        supplementary_files: list of str
            Supplementary data files that shall be copied to the output
            directory and inserted in the rst file for download
        other_supplementary_files: list of str
            Other supplementary data files that shall be copied but not
            inserted for download
        thumbnail_figure: int
            The number of the figure that shall be used for download or a path
            to a file
        url: str
            The url where to download the notebook
        insert_bokeh: False or str
            The version string for bokeh to use for the style sheet
        insert_bokeh_widgets: bool or str
            The version string for bokeh to use for the widgets style sheet
        tag_options: dict
            A dictionary with traitlets for the
            :class:`nbconvert.preprocessors.TagRemovePreprocessor`"""
        self.infile = infile
        self.outfile = outfile
        self.preprocess = preprocess
        self.clear = clear
        self._code_example = code_example
        self._supplementary_files = supplementary_files
        self._other_supplementary_files = other_supplementary_files
        self._thumbnail_figure = thumbnail_figure
        self._url = url
        self.insert_bokeh = insert_bokeh
        self.insert_bokeh_widgets = insert_bokeh_widgets
        self.tag_options = tag_options
        self.process_notebook(disable_warnings)
        self.create_thumb()

    def get_out_file(self, ending='rst'):
        """get the output file with the specified `ending`"""
        return os.path.splitext(self.outfile)[0] + os.path.extsep + ending

    def process_notebook(self, disable_warnings=True):
        """Process the notebook and create all the pictures and files

        This method runs the notebook using the :mod:`nbconvert` and
        :mod:`nbformat` modules. It creates the :attr:`outfile` notebook,
        a python and a rst file"""
        infile = self.infile
        outfile = self.outfile
        in_dir = os.path.dirname(infile) + os.path.sep
        odir = os.path.dirname(outfile) + os.path.sep
        create_dirs(os.path.join(odir, 'images'))
        ep = nbconvert.preprocessors.ExecutePreprocessor(
            timeout=300)
        cp = nbconvert.preprocessors.ClearOutputPreprocessor(
            timeout=300)

        self.nb = nb = nbformat.read(infile, nbformat.current_nbformat)
        # disable warnings in the rst file
        if disable_warnings:
            for i, cell in enumerate(nb.cells):
                if cell['cell_type'] == 'code':
                    cell = cell.copy()
                    break
            cell = cell.copy()
            cell.source = """
import logging
logging.captureWarnings(True)
logging.getLogger('py.warnings').setLevel(logging.ERROR)
"""
            nb.cells.insert(i, cell)
        # write and process rst_file
        if self.preprocess:
            t = dt.datetime.now()
            logger.info('Processing %s', self.infile)
            try:
                ep.preprocess(nb, {'metadata': {'path': in_dir}})
            except nbconvert.preprocessors.execute.CellExecutionError:
                logger.critical(
                    'Error while processing %s!', self.infile, exc_info=True)
            else:
                logger.info('Done. Seconds needed: %i',
                            (dt.datetime.now() - t).seconds)
            if disable_warnings:
                nb.cells.pop(i)

        self.py_file = self.get_out_file('py')

        if self.remove_tags:
            tp = nbconvert.preprocessors.TagRemovePreprocessor(timeout=300)
            for key, val in self.tag_options.items():
                setattr(tp, key, set(val))
            nb4rst = deepcopy(nb)
            tp.preprocess(nb4rst, {'metadata': {'path': in_dir}})
        else:
            nb4rst = nb

        self.create_rst(nb4rst, in_dir, odir)

        if self.clear:
            cp.preprocess(nb, {'metadata': {'path': in_dir}})
        # write notebook file
        nbformat.write(nb, outfile)
        self.create_py(nb)

    def create_rst(self, nb, in_dir, odir):
        """Create the rst file from the notebook node"""
        raw_rst, resources = nbconvert.export_by_name('rst', nb)
        # remove ipython magics
        rst_content = ''
        i0 = 0
        m = None
        # HACK: we insert the bokeh style sheets here as well, since for some
        # themes (e.g. the sphinx_rtd_theme) it is not sufficient to include
        # the style sheets only via app.add_stylesheet
        bokeh_str = ''
        if 'bokeh' in raw_rst and self.insert_bokeh:
            bokeh_str += self.BOKEH_TEMPLATE.format(
                version=self.insert_bokeh)
        if 'bokeh' in raw_rst and self.insert_bokeh_widgets:
            bokeh_str += self.BOKEH_WIDGETS_TEMPLATE.format(
                version=self.insert_bokeh_widgets)
        for m in code_blocks.finditer(raw_rst):
            lines = m.group().splitlines(True)
            header, content = lines[0], ''.join(lines[1:])
            no_magics = magic_patt.sub('\g<1>', content)
            # if the code cell only contained magic commands, we skip it
            if no_magics.strip():
                rst_content += (
                    raw_rst[i0:m.start()] + bokeh_str + header + no_magics)
                bokeh_str = ''
                i0 = m.end()
            else:
                rst_content += raw_rst[i0:m.start()]
                i0 = m.end()
        if m is not None:
            rst_content += bokeh_str + raw_rst[m.end():]
        else:
            rst_content = raw_rst
        rst_content = '.. _%s:\n\n' % self.reference + \
            rst_content
        url = self.url
        if url is not None:
            rst_content += self.CODE_DOWNLOAD_NBVIEWER.format(
                pyfile=os.path.basename(self.py_file),
                nbfile=os.path.basename(self.outfile),
                url=url)
        else:
            rst_content += self.CODE_DOWNLOAD.format(
                pyfile=os.path.basename(self.py_file),
                nbfile=os.path.basename(self.outfile))
        supplementary_files = self.supplementary_files
        other_supplementary_files = self.other_supplementary_files
        if supplementary_files or other_supplementary_files:
            for f in (supplementary_files or []) + (
                    other_supplementary_files or []):
                if not os.path.exists(os.path.join(odir, f)):
                    copyfile(os.path.join(in_dir, f), os.path.join(odir, f))
        if supplementary_files:
            rst_content += self.data_download(supplementary_files)

        rst_file = self.get_out_file()
        outputs = sorted(resources['outputs'], key=rst_content.find)
        base = os.path.join('images', os.path.splitext(
            os.path.basename(self.infile))[0] + '_%i.png')
        out_map = {os.path.basename(original): base % i
                   for i, original in enumerate(outputs)}
        for original, final in six.iteritems(out_map):
            rst_content = rst_content.replace(original, final)
        with open(rst_file, 'w') \
                as f:
            f.write(rst_content.rstrip() + '\n')
        pictures = []
        for original in outputs:
            fname = os.path.join(odir, out_map[os.path.basename(original)])
            pictures.append(fname)
            if six.PY3:
                f = open(fname, 'w+b')
            else:
                f = open(fname, 'w')
            f.write(resources['outputs'][original])
            f.close()
        self.pictures = pictures

    def create_py(self, nb, force=False):
        """Create the python script from the notebook node"""
        # Although we would love to simply use ``nbconvert.export_python(nb)``
        # this causes troubles in other cells processed by the ipython
        # directive. Instead of getting something like ``Out [5]:``, we get
        # some weird like '[0;31mOut[5]: ' which look like
        # color information if we allow the call of nbconvert.export_python
        if list(map(int, re.findall('\d+', nbconvert.__version__))) >= [4, 2]:
            py_file = os.path.basename(self.py_file)
        else:
            py_file = self.py_file
        try:
            level = logger.logger.level
        except AttributeError:
            level = logger.level
        spr.call(['jupyter', 'nbconvert', '--to=python',
                  '--output=' + py_file, '--log-level=%s' % level,
                  self.outfile])
        with open(self.py_file) as f:
            py_content = f.read()
        # comment out ipython magics
        py_content = re.sub('^\s*get_ipython\(\).magic.*', '# \g<0>',
                            py_content, flags=re.MULTILINE)
        with open(self.py_file, 'w') as f:
            f.write(py_content)

    def data_download(self, files):
        """Create the rst string to download supplementary data"""
        if len(files) > 1:
            return self.DATA_DOWNLOAD % (
                ('\n\n' + ' '*8) + ('\n' + ' '*8).join(
                    '* :download:`%s`' % f for f in files))
        return self.DATA_DOWNLOAD % ':download:`%s`' % files[0]

    def create_thumb(self):
        """Create the thumbnail for html output"""
        thumbnail_figure = self.copy_thumbnail_figure()
        if thumbnail_figure is not None:
            if isinstance(thumbnail_figure, six.string_types):
                pic = thumbnail_figure
            else:
                pic = self.pictures[thumbnail_figure]
            self.save_thumbnail(pic)
        else:
            for pic in self.pictures[::-1]:
                if pic.endswith('png'):
                    self.save_thumbnail(pic)
                    return

    def get_description(self):
        """Get summary and description of this notebook"""
        def split_header(s, get_header=True):
            s = s.lstrip().rstrip()
            parts = s.splitlines()
            if parts[0].startswith('#'):
                if get_header:
                    header = re.sub('#+\s*', '', parts.pop(0))
                    if not parts:
                        return header, ''
                else:
                    header = ''
                rest = '\n'.join(parts).lstrip().split('\n\n')
                desc = rest[0].replace('\n', ' ')
                return header, desc
            else:
                if get_header:
                    if parts[0].startswith(('=', '-')):
                        parts = parts[1:]
                    header = parts.pop(0)
                    if parts and parts[0].startswith(('=', '-')):
                        parts.pop(0)
                    if not parts:
                        return header, ''
                else:
                    header = ''
                rest = '\n'.join(parts).lstrip().split('\n\n')
                desc = rest[0].replace('\n', ' ')
                return header, desc

        first_cell = self.nb['cells'][0]

        if not first_cell['cell_type'] == 'markdown':
            return '', ''
        header, desc = split_header(first_cell['source'])
        if not desc and len(self.nb['cells']) > 1:
            second_cell = self.nb['cells'][1]
            if second_cell['cell_type'] == 'markdown':
                _, desc = split_header(second_cell['source'], False)
        return header, desc

    def scale_image(self, in_fname, out_fname, max_width, max_height):
        """Scales an image with the same aspect ratio centered in an
           image with a given max_width and max_height
           if in_fname == out_fname the image can only be scaled down
        """
        # local import to avoid testing dependency on PIL:
        try:
            from PIL import Image
        except ImportError:
            import Image
        img = Image.open(in_fname)
        width_in, height_in = img.size
        scale_w = max_width / float(width_in)
        scale_h = max_height / float(height_in)

        if height_in * scale_w <= max_height:
            scale = scale_w
        else:
            scale = scale_h

        if scale >= 1.0 and in_fname == out_fname:
            return

        width_sc = int(round(scale * width_in))
        height_sc = int(round(scale * height_in))

        # resize the image
        img.thumbnail((width_sc, height_sc), Image.ANTIALIAS)

        # insert centered
        thumb = Image.new('RGB', (max_width, max_height), (255, 255, 255))
        pos_insert = (
            (max_width - width_sc) // 2, (max_height - height_sc) // 2)
        thumb.paste(img, pos_insert)

        thumb.save(out_fname)

    def save_thumbnail(self, image_path):
        """Save the thumbnail image"""
        thumb_dir = os.path.join(os.path.dirname(image_path), 'thumb')
        create_dirs(thumb_dir)

        thumb_file = os.path.join(thumb_dir,
                                  '%s_thumb.png' % self.reference)
        if os.path.exists(image_path):
            logger.info('Scaling %s to thumbnail %s', image_path, thumb_file)
            self.scale_image(image_path, thumb_file, 400, 280)
        self.thumb_file = thumb_file

    def get_thumb_path(self, base_dir):
        """Get the relative path to the thumb nail of this notebook"""
        return os.path.relpath(self.thumb_file, base_dir)

    def copy_thumbnail_figure(self):
        """The integer of the thumbnail figure"""
        ret = None
        if self._thumbnail_figure is not None:
            if not isstring(self._thumbnail_figure):
                ret = self._thumbnail_figure
            else:
                ret = osp.join(osp.dirname(self.outfile),
                               osp.basename(self._thumbnail_figure))
                copyfile(self._thumbnail_figure, ret)
                return ret
        elif hasattr(self.nb.metadata, 'thumbnail_figure'):
            if not isstring(self.nb.metadata.thumbnail_figure):
                ret = self.nb.metadata.thumbnail_figure
            else:
                ret = osp.join(osp.dirname(self.outfile), 'images',
                               osp.basename(self.nb.metadata.thumbnail_figure))
                copyfile(osp.join(osp.dirname(self.infile),
                                  self.nb.metadata.thumbnail_figure),
                         ret)
        return ret

class Gallery(object):
    """Class to create one or more example gallerys"""

    #: The input directories
    in_dir = []

    #: The output directories
    out_dir = []

    @property
    def urls(self):
        return self._all_urls[self._in_dir_count]

    def __init__(self, examples_dirs=['../examples'], gallery_dirs=None,
                 pattern='example_.+.ipynb', disable_warnings=True,
                 dont_preprocess=[], preprocess=True, clear=True,
                 dont_clear=[], code_examples={}, supplementary_files={},
                 other_supplementary_files={}, thumbnail_figures={},
                 urls=None, insert_bokeh=False, insert_bokeh_widgets=False,
                 remove_all_outputs_tags=set(), remove_cell_tags=set(),
                 remove_input_tags=set(), remove_single_output_tags=set()):
        """
        Parameters
        ----------
        examples_dirs: list of str
            list containing the directories to loop through. Default:
            ``['../examples']``
        gallerys_dirs: list of str
            None or list of directories where the rst files shall be created.
            If None, the current working directory and the name of the
            corresponding directory in the `examples_dirs` is used. Default:
            ``None``
        pattern: list of str
            str. The pattern to use to find the ipython  notebooks.
            Default: ``'example_.+.ipynb'``
        disable_warnings: bool
            Boolean controlling whether warnings shall be disabled when
            processing the examples. Defaultt: True
        preprocess: bool or list of str
            If True, all examples (except those specified in the
            `dont_preprocess` item) will be preprocessed when creating the rst
            files. Otherwise it might be a list of files that shall be
            preprocessed.
        dont_preprocess: bool or list of str
            If True, no example will be preprocessed when creating the rst
            files. Otherwise it might be a list of files that shall not be
            preprocessed
        clear: bool or list of str
            If True, the output in all notebooks to download will be cleared.
            Otherwise it might be a list of notebook files of whom to clear the
            output
        dont_clear: bool or list of str
            If True, the output in all notebooks to download will not be
            cleared. Otherwise it might be a list of notebook files  of whom
            not to clear the output
        code_examples: dict
            A mapping from filename to code samples that shall be used instead
            of a thumbnail figure in the gallery. Note that you can also
            include a  ``'code_example'`` key in the metadata of the notebook
        supplementary_files: dict
            A mapping from filename to a list of supplementary data files that
            shall copied to the documentation directory and can be downloaded.
            Note that you can also include a  ``'supplementary_files'`` key in
            the metadata of the notebook
        other_supplementary_files: dict
            A mapping from filename to a list of other supplementary data files
            that shall copied to the documentation directory but can not be
            downloaded (e.g. pictures).
            Note that you can also include a  ``'other_supplementary_files'``
            key in the metadata of the notebook
        thumbnail_figures: dict
            A mapping from filename to an integer or the path of a file to use
            for the thumbnail
        urls: str or dict
            The urls where to download the notebook. Necessary to provide a
            link to the jupyter nbviewer. If string, the path to the notebook
            will be appended for each example notebook. Otherwise it should be
            a dictionary with links for the given notebook
        insert_bokeh: bool or str
            If True, the bokeh js [1]_ and the stylesheet [2]_ are inserted in
            the notebooks that have bokeh loaded (using the installed or
            specified bokeh version)
        insert_bokeh_widgets: bool or str
            If True, the bokeh widget js [3]_ is inserted in the notebooks that
            have bokeh loaded (using the installed or specified bokeh version)
        remove_all_outputs_tags: set
            Tags indicating cells for which the outputs are to be removed,
            matches tags in cell.metadata.tags.
        remove_cell_tags: set
            Tags indicating which cells are to be removed, matches tags in
            cell.metadata.tags.
        remove_input_tags: set
            Tags indicating cells for which input is to be removed,
            matches tags in cell.metadata.tags.
        remove_single_output_tags: set
            Tags indicating which individual outputs are to be removed, matches
            output i tags in cell.outputs[i].metadata.tags.

        References
        ----------
        .. [1] http://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.js
        .. [2] http://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.css
        .. [3] http://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.js
        """
        if isinstance(examples_dirs, six.string_types):
            examples_dirs = [examples_dirs]
        if gallery_dirs is None:
            gallery_dirs = list(map(os.path.basename, examples_dirs))
        if isinstance(gallery_dirs, six.string_types):
            gallery_dirs = [gallery_dirs]

        for i, s in enumerate(examples_dirs):
            if not s.endswith(os.path.sep):
                examples_dirs[i] += os.path.sep

        for i, s in enumerate(gallery_dirs):
            if not s.endswith(os.path.sep):
                gallery_dirs[i] += os.path.sep

        self.in_dir = examples_dirs
        self.out_dir = gallery_dirs

        if isinstance(pattern, six.string_types):
            pattern = re.compile(pattern)
        self.pattern = pattern
        self.disable_warnings = disable_warnings
        self.dont_preprocess = dont_preprocess
        self.preprocess = preprocess
        self.clear = clear
        self.dont_clear = dont_clear
        self.code_examples = code_examples
        self.supplementary_files = supplementary_files
        self.osf = other_supplementary_files
        self.thumbnail_figures = thumbnail_figures
        if urls is None or isinstance(urls, (dict, six.string_types)):
            urls = [urls] * len(self.in_dir)
        self._all_urls = urls
        if insert_bokeh and not isstring(insert_bokeh):
            import bokeh
            insert_bokeh = bokeh.__version__
        if insert_bokeh_widgets and not isstring(insert_bokeh_widgets):
            import bokeh
            insert_bokeh_widgets = bokeh.__version__
        tag_options = {
            'remove_all_outputs_tags': remove_all_outputs_tags,
            'remove_cell_tags': remove_cell_tags,
            'remove_input_tags': remove_input_tags,
            'remove_single_output_tags': remove_single_output_tags}
        self._nbp_kws = {'insert_bokeh': insert_bokeh,
                         'insert_bokeh_widgets': insert_bokeh_widgets,
                         'tag_options': tag_options}

    def process_directories(self):
        """Create the rst files from the input directories in the
        :attr:`in_dir` attribute"""
        for i, (base_dir, target_dir, paths) in enumerate(zip(
                self.in_dir, self.out_dir, map(os.walk, self.in_dir))):
            self._in_dir_count = i
            self.recursive_processing(base_dir, target_dir, paths)

    def recursive_processing(self, base_dir, target_dir, it):
        """Method to recursivly process the notebooks in the `base_dir`

        Parameters
        ----------
        base_dir: str
            Path to the base example directory (see the `examples_dir`
            parameter for the :class:`Gallery` class)
        target_dir: str
            Path to the output directory for the rst files (see the
            `gallery_dirs` parameter for the :class:`Gallery` class)
        it: iterable
            The iterator over the subdirectories and files in `base_dir`
            generated by the :func:`os.walk` function"""
        try:
            file_dir, dirs, files = next(it)
        except StopIteration:
            return '', []
        readme_files = {'README.md', 'README.rst', 'README.txt'}
        if readme_files.intersection(files):
            foutdir = file_dir.replace(base_dir, target_dir)
            create_dirs(foutdir)
            this_nbps = [
                NotebookProcessor(
                    infile=f,
                    outfile=os.path.join(foutdir, os.path.basename(f)),
                    disable_warnings=self.disable_warnings,
                    preprocess=(
                        (self.preprocess is True or f in self.preprocess) and
                        not (self.dont_preprocess is True or
                             f in self.dont_preprocess)),
                    clear=((self.clear is True or f in self.clear) and not
                           (self.dont_clear is True or f in self.dont_clear)),
                    code_example=self.code_examples.get(f),
                    supplementary_files=self.supplementary_files.get(f),
                    other_supplementary_files=self.osf.get(f),
                    thumbnail_figure=self.thumbnail_figures.get(f),
                    url=self.get_url(f.replace(base_dir, '')),
                    **self._nbp_kws)
                for f in map(lambda f: os.path.join(file_dir, f),
                             filter(self.pattern.match, files))]
            readme_file = next(iter(readme_files.intersection(files)))
        else:
            return '', []
        labels = OrderedDict()
        this_label = 'gallery_' + foutdir.replace(os.path.sep, '_')
        if this_label.endswith('_'):
            this_label = this_label[:-1]
        for d in dirs:
            label, nbps = self.recursive_processing(
                base_dir, target_dir, it)
            if label:
                labels[label] = nbps
        s = ".. _%s:\n\n" % this_label
        with open(os.path.join(file_dir, readme_file)) as f:
            s += f.read().rstrip() + '\n\n'

        s += "\n\n.. toctree::\n\n"
        s += ''.join('    %s\n' % os.path.splitext(os.path.basename(
            nbp.get_out_file()))[0] for nbp in this_nbps)
        for d in dirs:
            findex = os.path.join(d, 'index.rst')
            if os.path.exists(os.path.join(foutdir, findex)):
                s += '    %s\n' % os.path.splitext(findex)[0]

        s += '\n'

        for nbp in this_nbps:
            code_div = nbp.code_div
            if code_div is not None:
                s += code_div + '\n'
            else:
                s += nbp.thumbnail_div + '\n'
        s += "\n.. raw:: html\n\n

\n"
        for label, nbps in labels.items():
            s += '\n.. only:: html\n\n    .. rubric:: :ref:`%s`\n\n' % (
                label)
            for nbp in nbps:
                code_div = nbp.code_div
                if code_div is not None:
                    s += code_div + '\n'
                else:
                    s += nbp.thumbnail_div + '\n'
            s += "\n.. raw:: html\n\n

\n"

        s += '\n'

        with open(os.path.join(foutdir, 'index.rst'), 'w') as f:
            f.write(s)
        return this_label, list(chain(this_nbps, *labels.values()))

    @classmethod
    def from_sphinx(cls, app):
        """Class method to create a :class:`Gallery` instance from the
        configuration of a sphinx application"""
        app.config.html_static_path.append(os.path.join(
            os.path.dirname(__file__), '_static'))
        config = app.config.example_gallery_config

        insert_bokeh = config.get('insert_bokeh')
        if insert_bokeh:
            if not isstring(insert_bokeh):
                import bokeh
                insert_bokeh = bokeh.__version__
            app.add_stylesheet(
                NotebookProcessor.BOKEH_STYLE_SHEET.format(
                    version=insert_bokeh))
            app.add_javascript(
                NotebookProcessor.BOKEH_JS.format(version=insert_bokeh))

        insert_bokeh_widgets = config.get('insert_bokeh_widgets')
        if insert_bokeh_widgets:
            if not isstring(insert_bokeh_widgets):
                import bokeh
                insert_bokeh_widgets = bokeh.__version__
            app.add_stylesheet(
                NotebookProcessor.BOKEH_WIDGETS_STYLE_SHEET.format(
                    version=insert_bokeh_widgets))
            app.add_javascript(
                NotebookProcessor.BOKEH_WIDGETS_JS.format(
                    version=insert_bokeh_widgets))

        if not app.config.process_examples:
            return
        cls(**app.config.example_gallery_config).process_directories()

    def get_url(self, nbfile):
        """Return the url corresponding to the given notebook file

        Parameters
        ----------
        nbfile: str
            The path of the notebook relative to the corresponding
            :attr:``in_dir``

        Returns
        -------
        str or None
            The url or None if no url has been specified
        """
        urls = self.urls
        if isinstance(urls, dict):
            return urls.get(nbfile)
        elif isstring(urls):
            if not urls.endswith('/'):
                urls += '/'
            return urls + nbfile

def align(argument):
    """Conversion function for the "align" option."""
    return directives.choice(argument, ('left', 'center', 'right'))

class LinkGalleriesDirective(Directive):

    has_content = True

    if hasattr(directives, 'images'):
        option_spec = directives.images.Figure.option_spec
    else:
        option_spec = {'alt': directives.unchanged,
                       'height': directives.nonnegative_int,
                       'width': directives.nonnegative_int,
                       'scale': directives.nonnegative_int,
                       'align': align
                       }

    def create_image_nodes(self, header, thumb_url, key, link_url=None):
        options = {'target': link_url} if link_url else {}
        options.update(self.options)
        d1 = directives.misc.Raw(
            'raw', ['html'], {}, ViewList([
                '
']
                ),
            self.lineno, self.content_offset, self.block_text, self.state,
            self.state_machine)
        d = directives.images.Figure(
            'image', [thumb_url], options, ViewList([':ref:`%s`' % key]),
            self.lineno, self.content_offset, self.block_text, self.state,
            self.state_machine)
        d2 = directives.misc.Raw(
            'raw', ['html'], {}, ViewList(['
']),
            self.lineno, self.content_offset, self.block_text, self.state,
            self.state_machine)
        return list(chain(d1.run(), d.run(), d2.run()))

    def get_outdirs(self):
        conf = self.env.config.example_gallery_config
        gallery_dirs = conf.get('gallery_dirs')
        if not gallery_dirs:
            examples_dirs = conf.get('example_dirs', ['../examples'])
            if isinstance(examples_dirs, six.string_types):
                examples_dirs = [examples_dirs]
            gallery_dirs = list(map(osp.basename, examples_dirs))
        if isinstance(gallery_dirs, six.string_types):
            gallery_dirs = [gallery_dirs]
        for i, s in enumerate(gallery_dirs):
            if not s.endswith(os.path.sep):
                gallery_dirs[i] += os.path.sep
        return gallery_dirs

    def run(self):
        self.env = self.state.document.settings.env
        conf = self.env.config
        ret = nodes.paragraph()
        try:
            inventory = self.env.intersphinx_named_inventory
        except AttributeError:
            warn('The %s directive requires the sphinx.ext.intersphinx '
                 'extension!', self.name)
            return [ret]
        for pkg_str in self.content:
            try:
                pkg, directory = pkg_str.split()
            except ValueError:
                pkg, directory = pkg_str, ''
            directory_ = directory.replace('/', '_').lower()
            if pkg == conf.project:
                pass
                if not directory:
                    directories = self.get_outdirs()
                else:
                    directories = [directory]
                for directory in directories:
                    for file_dir, dirs, files in os.walk(directory):
                        if not file_dir.endswith(osp.sep):
                            file_dir += osp.sep
                        file_dir_ = file_dir.replace(osp.sep, '_').lower()
                        if 'index.rst' in files:
                            for f in files:
                                if f.endswith('.ipynb'):
                                    ref = 'gallery_' + file_dir_ + f
                                    thumb = osp.join(
                                        file_dir, 'images', 'thumb',
                                        ref + '_thumb.png')
                                    if osp.isabs(thumb):
                                        thumb = osp.relpath(thumb,
                                                            self.env.srcdir)
                                    header = ':ref:`%s`' % (ref, )
                                    ret.extend(self.create_image_nodes(
                                        header, thumb, ref))
            else:
                try:
                    refs = inventory[pkg]['std:label']
                except KeyError:
                    warn('Could not load the inventory of %s!', pkg)
                    continue
                base_url = self.env.config.intersphinx_mapping[pkg][0]
                if not base_url.endswith('/'):
                    base_url += '/'
                for key, val in refs.items():
                    if (key.startswith('gallery_' + directory_) and
                            key.endswith('.ipynb')):
                        link_url = val[2]
                        header = val[3]
                        thumb_url = base_url + '_images/%s_thumb.png' % key
                        ret.extend(self.create_image_nodes(
                            header, thumb_url, '%s:%s' % (pkg, key),
                            link_url))
        ret.extend(directives.misc.Raw(
            'raw', ['html'], {}, ViewList(["

"]),
            self.lineno, self.content_offset, self.block_text, self.state,
            self.state_machine).run())
        return [ret]

#: dictionary containing the configuration of the example gallery.
#:
#: Possible keys for the dictionary are the initialization keys of the
#: :class:`Gallery` class
gallery_config = {
    'examples_dirs': ['../examples'],
    'gallery_dirs': None,
    'pattern': 'example_.+.ipynb',
    'disable_warnings': True,
    'preprocess': True,
    'dont_preprocess': [],
    'clear': True,
    'dont_clear': [],
    'code_examples': {},
    'supplementary_files': {},
    'insert_bokeh': False,
    'insert_bokeh_widgets': False}

#: Boolean controlling whether the rst files shall created and examples
#: processed
process_examples = True

def setup(app):
    app.add_config_value('process_examples', process_examples, 'html')

    app.add_config_value('example_gallery_config', gallery_config, 'html')

    app.add_stylesheet('example_gallery_styles.css')

    app.add_directive('linkgalleries', LinkGalleriesDirective)

    app.connect('builder-inited', Gallery.from_sphinx)


"""
 Detect new translatable fields in all models and sync database structure.

 You will need to execute this command in two cases:

   1. When you add new languages to settings.LANGUAGES.
   2. When you new translatable fields to your models.

"""
import re

from optparse import make_option

from django.conf import settings
from django.core.management.base import BaseCommand
from django.core.management.color import no_style
from django.db import connection, transaction
from django.db import backend
from django.db.models import get_models
from django.db.models.fields import FieldDoesNotExist

from transmeta import (mandatory_language, get_real_fieldname,
                       get_languages, get_all_translatable_fields)

VALUE_DEFAULT = 'WITHOUT VALUE'

def ask_for_confirmation(sql_sentences, model_full_name, assume_yes):
    print ('\nSQL to synchronize "%s" schema:' % model_full_name)
    for sentence in sql_sentences:
        print ('   %s' % sentence)
    if assume_yes:
        print ('\nAre you sure that you want to execute the previous SQL: (y/n) [n]: YES')
        return True
    while True:
        prompt = '\nAre you sure that you want to execute the previous SQL: (y/n) [n]: '
        answer = raw_input(prompt).strip()
        if answer == '':
            return False
        elif answer not in ('y', 'n', 'yes', 'no'):
            print ('Please answer yes or no')
        elif answer == 'y' or answer == 'yes':
            return True
        else:
            return False

def print_db_change_langs(db_change_langs, field_name, model_name):
    print ('\nThis languages can change in "%s" field from "%s" model: %s' % \
        (field_name, model_name, ", ".join(db_change_langs)))

class Command(BaseCommand):
    help = "Detect new translatable fields or new available languages and sync database structure"

    option_list = BaseCommand.option_list + (
        make_option('-y', '--yes', action='store_true', dest='assume_yes',
                    help="Assume YES on all queries"),
        make_option('-d', '--default', dest='default_language',
                    help="Language code of your default language"),
        )

    def handle(self, *args, **options):
        """ command execution """
        assume_yes = options.get('assume_yes', False)
        default_language = options.get('default_language', None)

        # set manual transaction management
        transaction.commit_unless_managed()
        transaction.enter_transaction_management()
        transaction.managed(True)

        self.cursor = connection.cursor()
        self.introspection = connection.introspection

        self.default_lang = default_language or mandatory_language()

        all_models = get_models()
        found_db_change_fields = False
        for model in all_models:
            if hasattr(model._meta, 'translatable_fields'):
                model_full_name = '%s.%s' % (model._meta.app_label, model._meta.module_name)
                translatable_fields = get_all_translatable_fields(model, column_in_current_table=True)
                db_table = model._meta.db_table
                for field_name in translatable_fields:
                    db_table_fields = self.get_table_fields(db_table)
                    db_change_langs = list(set(list(self.get_db_change_languages(field_name, db_table_fields)) + [self.default_lang]))
                    if db_change_langs:
                        sql_sentences = self.get_sync_sql(field_name, db_change_langs, model, db_table_fields)
                        if sql_sentences:
                            found_db_change_fields = True
                            print_db_change_langs(db_change_langs, field_name, model_full_name)
                            execute_sql = ask_for_confirmation(sql_sentences, model_full_name, assume_yes)
                            if execute_sql:
                                print ('Executing SQL...')
                                for sentence in sql_sentences:
                                    self.cursor.execute(sentence)
                                    # commit
                                    transaction.commit()
                                print ('Done')
                            else:
                                print ('SQL not executed')

        if transaction.is_dirty():
            transaction.commit()
        transaction.leave_transaction_management()

        if not found_db_change_fields:
            print ('\nNo new translatable fields detected')
        if default_language:
            variable = 'TRANSMETA_DEFAULT_LANGUAGE'
            has_transmeta_default_language = getattr(settings, variable, False)
            if not has_transmeta_default_language:
                variable = 'LANGUAGE_CODE'
            if getattr(settings, variable) != default_language:
                print (('\n\nYou should change in your settings '
                       'the %s variable to "%s"' % (variable, default_language)))

    def get_table_fields(self, db_table):
        """ get table fields from schema """
        db_table_desc = self.introspection.get_table_description(self.cursor, db_table)
        return [t[0] for t in db_table_desc]

    def get_field_required_in_db(self, db_table, field_name, value_not_implemented=False):
        table_fields = self.introspection.get_table_description(self.cursor, db_table)
        for f in table_fields:
            if f[0] == field_name:
                is_null = f[-1]
                if is_null is None:  # Not Implemented
                    return value_not_implemented
                return not is_null
        return False

    def get_db_change_languages(self, field_name, db_table_fields):
        """ get only db changes fields """
        for lang_code, lang_name in get_languages():
            if get_real_fieldname(field_name, lang_code) not in db_table_fields:
                yield lang_code
        for db_table_field in db_table_fields:
            pattern = re.compile('^%s_(?P<lang>\w{2})$' % field_name)
            m = pattern.match(db_table_field)
            if not m:
                continue
            lang = m.group('lang')
            yield lang

    def was_translatable_before(self, field_name, db_table_fields):
        """ check if field_name was translatable before syncing schema """
        if field_name in db_table_fields:
            # this implies field was never translatable before, data is in this field
            return False
        else:
            return True

    def get_default_field(self, field_name, model):
        for lang_code, lang_name in get_languages():
            field_name_i18n = get_real_fieldname(field_name, lang_code)
            f = model._meta.get_field(field_name_i18n)
            if not f.null:
                return f
        try:
            return model._meta.get_field(field_name)
        except FieldDoesNotExist:
            return None

    def get_value_default(self):
        return getattr(settings, 'TRANSMETA_VALUE_DEFAULT', VALUE_DEFAULT)

    def get_type_of_db_field(self, field_name, model):
        field = self.get_default_field(field_name, model)
        if not field:
            field = model._meta.get_field(get_real_fieldname(field_name))
        try:
            col_type = field.db_type(connection)
        except TypeError:  # old django
            col_type = field.db_type()
        return col_type

    def get_sync_sql(self, field_name, db_change_langs, model, db_table_fields):
        """ returns SQL needed for sync schema for a new translatable field """
        qn = connection.ops.quote_name
        style = no_style()
        sql_output = []
        db_table = model._meta.db_table
        was_translatable_before = self.was_translatable_before(field_name, db_table_fields)
        default_f = self.get_default_field(field_name, model)
        default_f_required = default_f and self.get_field_required_in_db(db_table,
                                                    default_f.name,
                                                    value_not_implemented=False)
        for lang in db_change_langs:
            new_field = get_real_fieldname(field_name, lang)
            try:
                f = model._meta.get_field(new_field)
                col_type = self.get_type_of_db_field(field_name, model)
                field_column = f.column
            except FieldDoesNotExist:  # columns in db, removed the settings.LANGUGES
                field_column = new_field
                col_type = self.get_type_of_db_field(field_name, model)
            field_sql = [style.SQL_FIELD(qn(field_column)), style.SQL_COLTYPE(col_type)]

            alter_colum_set = 'ALTER COLUMN %s SET' % qn(field_column)
            if default_f:
                alter_colum_drop = 'ALTER COLUMN %s DROP' % qn(field_column)
            not_null = style.SQL_KEYWORD('NOT NULL')

            if 'mysql' in backend.__name__:
                alter_colum_set = 'MODIFY %s %s' % (qn(field_column), col_type)
                not_null = style.SQL_KEYWORD('NULL')
                if default_f:
                    alter_colum_drop = 'MODIFY %s %s' % (qn(field_column), col_type)

            # column creation
            if not new_field in db_table_fields:
                sql_output.append("ALTER TABLE %s ADD COLUMN %s" % (qn(db_table), ' '.join(field_sql)))

            if lang == self.default_lang and not was_translatable_before:
                # data copy from old field (only for default language)
                sql_output.append("UPDATE %s SET %s = %s" % (qn(db_table), \
                                    qn(field_column), qn(field_name)))
                if not f.null:
                    # changing to NOT NULL after having data copied
                    sql_output.append("ALTER TABLE %s %s %s" % \
                                    (qn(db_table), alter_colum_set, \
                                    style.SQL_KEYWORD('NOT NULL')))
            elif default_f and not default_f.null:
                if lang == self.default_lang:
                    f_required = self.get_field_required_in_db(db_table,
                                                           field_column,
                                                           value_not_implemented=False)
                    if default_f.name == new_field and default_f_required:
                        continue
                    if not f_required:
                        # data copy from old field (only for default language)
                        sql_output.append(("UPDATE %(db_table)s SET %(f_colum)s = '%(value_default)s' "
                                    "WHERE %(f_colum)s is %(null)s or %(f_colum)s = '' " %
                                        {'db_table': qn(db_table),
                                        'f_colum': qn(field_column),
                                        'value_default': self.get_value_default(),
                                        'null': style.SQL_KEYWORD('NULL'),
                                        }))
                        # changing to NOT NULL after having data copied
                        sql_output.append("ALTER TABLE %s %s %s" % \
                                        (qn(db_table), alter_colum_set, \
                                        style.SQL_KEYWORD('NOT NULL')))
                else:
                    f_required = self.get_field_required_in_db(db_table,
                                                           field_column,
                                                           value_not_implemented=True)
                    if f_required:
                        sql_output.append(("ALTER TABLE %s %s %s" %
                                        (qn(db_table), alter_colum_drop, not_null)))

        if not was_translatable_before:
            # we drop field only if field was no translatable before
            sql_output.append("ALTER TABLE %s DROP COLUMN %s" % (qn(db_table), qn(field_name)))
        return sql_output


import copy

from django.db import models
from django.db.models.fields import NOT_PROVIDED
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.utils.datastructures import SortedDict
from django.utils.translation import get_language, ugettext_lazy as _

LANGUAGE_CODE = 0
LANGUAGE_NAME = 1

def get_languages():
    return getattr(settings, 'TRANSMETA_LANGUAGES', settings.LANGUAGES)

def get_real_fieldname(field, lang=None):
    if lang is None:
        lang = get_language().split('-')[0]  # both 'en-US' and 'en' -> 'en'
    return str('%s_%s' % (field, lang))

def get_field_language(real_field):
    """ return language for a field. i.e. returns "en" for "name_en" """
    return real_field.split('_')[-1]

def get_fallback_fieldname(field, lang=None):
    return get_real_fieldname(field, lang=fallback_language())

def get_mandatory_fieldname(field, lang=None):
    return get_real_fieldname(field, lang=mandatory_language())

def get_real_fieldname_in_each_language(field):
    return [get_real_fieldname(field, lang[LANGUAGE_CODE])
            for lang in get_languages()]

def canonical_fieldname(db_field):
    """ all "description_en", "description_fr", etc. field names will return "description" """
    return getattr(db_field, 'original_fieldname', db_field.name)  # original_fieldname is set by transmeta

def mandatory_language():
    """ returns mandatory language. the language which is required to fill """
    return getattr(settings, 'TRANSMETA_MANDATORY_LANGUAGE', fallback_language())

def fallback_language():
    """ returns fallback language """
    return getattr(settings, 'TRANSMETA_DEFAULT_LANGUAGE',
                   settings.LANGUAGE_CODE)

def get_all_translatable_fields(model, model_trans_fields=None, column_in_current_table=False):
    """ returns all translatable fields in a model (including superclasses ones) """
    if model_trans_fields is None:
        model_trans_fields = set()
    model_trans_fields.update(set(getattr(model._meta, 'translatable_fields', [])))
    for parent in model.__bases__:
        if getattr(parent, '_meta', None) and (not column_in_current_table or parent._meta.abstract):
            get_all_translatable_fields(parent, model_trans_fields, column_in_current_table)
    return tuple(model_trans_fields)

def default_value(field):
    '''
    When accessing to the name of the field itself, the value
    in the current language will be returned. Unless it's set,
    the value in the default language will be returned.
    '''

    def default_value_func(self):
        attname = lambda x: get_real_fieldname(field, x)

        if getattr(self, attname(get_language()), None):
            result = getattr(self, attname(get_language()))
        elif getattr(self, attname(get_language()[:2]), None):
            result = getattr(self, attname(get_language()[:2]))
        else:
            default_language = fallback_language()
            if getattr(self, attname(default_language), None):
                result = getattr(self, attname(default_language), None)
            else:
                result = getattr(self, attname(settings.LANGUAGE_CODE), None)
        return result

    return default_value_func

class TransMeta(models.base.ModelBase):
    '''
    Metaclass that allow a django field, to store a value for
    every language. The syntax to us it is next:

        class MyClass(models.Model):
            __metaclass__ = transmeta.TransMeta

            my_field = models.CharField(max_length=20)
            my_i18n_field = models.CharField(max_length=30)

            class Meta:
                translate = ('my_i18n_field',)

    Then we'll be able to access a specific language by
    <field_name>_<language_code>. If just <field_name> is
    accessed, we'll get the value of the current language,
    or if null, the value in the default language.
    '''

    def __new__(cls, name, bases, attrs):
        attrs = SortedDict(attrs)
        if 'Meta' in attrs and hasattr(attrs['Meta'], 'translate'):
            fields = attrs['Meta'].translate
            delattr(attrs['Meta'], 'translate')
        else:
            new_class = super(TransMeta, cls).__new__(cls, name, bases, attrs)
            # we inherits possible translatable_fields from superclasses
            abstract_model_bases = [base for base in bases if hasattr(base, '_meta') \
                                    and base._meta.abstract]
            translatable_fields = []
            for base in abstract_model_bases:
                if hasattr(base._meta, 'translatable_fields'):
                    translatable_fields.extend(list(base._meta.translatable_fields))
            new_class._meta.translatable_fields = tuple(translatable_fields)
            return new_class

        if not isinstance(fields, tuple):
            raise ImproperlyConfigured("Meta's translate attribute must be a tuple")

        for field in fields:
            if not field in attrs or \
               not isinstance(attrs[field], models.fields.Field):
                    raise ImproperlyConfigured(
                        "There is no field %(field)s in model %(name)s, "\
                        "as specified in Meta's translate attribute" % \
                        dict(field=field, name=name))
            original_attr = attrs[field]
            for lang in get_languages():
                lang_code = lang[LANGUAGE_CODE]
                lang_name = lang[LANGUAGE_NAME]
                lang_attr = copy.copy(original_attr)
                lang_attr.original_fieldname = field
                lang_attr_name = get_real_fieldname(field, lang_code)
                if lang_code != mandatory_language():
                    # only will be required for mandatory language
                    if not lang_attr.null and lang_attr.default is NOT_PROVIDED:
                        lang_attr.null = True
                    if not lang_attr.blank:
                        lang_attr.blank = True
                if hasattr(lang_attr, 'verbose_name'):
                    lang_attr.verbose_name = LazyString(lang_attr.verbose_name, _(lang_name))
                attrs[lang_attr_name] = lang_attr
            del attrs[field]
            attrs[field] = property(default_value(field))

        new_class = super(TransMeta, cls).__new__(cls, name, bases, attrs)
        if hasattr(new_class, '_meta'):
            new_class._meta.translatable_fields = fields
        return new_class

class LazyString(object):

    def __init__(self, proxy, lang):
        self.proxy = proxy
        self.lang = lang

    def __unicode__(self):
        return u'%s (%s)' % (self.proxy, self.lang)


import imghdr
import os
from subprocess import call
import tempfile
import shortuuid

from django.core.files import File

def get_or_create_temp_dir():
    temp_dir = os.path.join(tempfile.gettempdir(), 'thumbnails')
    if not os.path.exists(temp_dir):
        os.mkdir(temp_dir)
    return temp_dir

def process(thumbnail_file, size, **kwargs):
    """
    Post processors are functions that receive file objects,
    performs necessary operations and return the results as file objects.
    """
    from . import conf

    size_dict = conf.SIZES[size]
    for processor in size_dict['POST_PROCESSORS']:
        processor['processor'](thumbnail_file, **processor['kwargs'])

    return thumbnail_file

def optimize(thumbnail_file, jpg_command=None, png_command=None,
             gif_command=None):
    """
    A post processing function to optimize file size. Accepts commands
    to optimize JPG, PNG and GIF images as arguments. Example:

    THUMBNAILS = {
        # Other options...
        'POST_PROCESSORS': [
            {
                'processor': 'thumbnails.post_processors.optimize',
                'png_command': 'optipng -force -o3 "%(filename)s"',
                'jpg_command': 'jpegoptim -f --strip-all "%(filename)s"',
            },
        ],
    }

    Note: using output redirection in commands may cause unpredictable results.
    For example 'optipng -force -o3 "%(filename)s" &> /dev/null' may cause
    optimize command to fail on some systems.
    """
    temp_dir = get_or_create_temp_dir()
    thumbnail_filename = os.path.join(temp_dir, "%s" % shortuuid.uuid())

    f = open(thumbnail_filename, 'wb')
    f.write(thumbnail_file.read())
    f.close()

    # Detect filetype
    filetype = imghdr.what(thumbnail_filename)

    # Construct command to optimize image based on filetype
    command = None
    if filetype == "jpg" or filetype == "jpeg":
        command = jpg_command
    elif filetype == "png":
        command = png_command
    elif filetype == "gif":
        command = gif_command

    # Run Command
    if command:
        command = command % {'filename': thumbnail_filename}
        call(command, shell=True)

    optimized_file = File(open(thumbnail_filename, 'rb'))
    os.remove(thumbnail_filename)

    return optimized_file


from copy import deepcopy

try:
    import importlib
except ImportError:
    from django.utils import importlib

def import_attribute(name):
    """
    Return an attribute from a dotted path name (e.g. "path.to.func").
    Copied from nvie's rq https://github.com/nvie/rq/blob/master/rq/utils.py
    """
    if hasattr(name, '__call__'):
        return name
    module_name, attribute = name.rsplit('.', 1)
    module = importlib.import_module(module_name)
    return getattr(module, attribute)

def parse_processors(processor_definition):
    """
    Returns a dictionary that contains the imported processors and
    kwargs. For example, passing in:

    processors = [
        {'processor': 'thumbnails.processors.resize', 'width': 10, 'height': 10},
        {'processor': 'thumbnails.processors.crop', 'width': 10, 'height': 10},
    ]

    Would return:

    [
        {'processor': resize_function, kwargs: {'width': 10, 'height': 10}}
        {'processor': crop_function, kwargs: {'width': 10, 'height': 10}}
    ]
    """
    parsed_processors = []
    for processor in processor_definition:
        processor_function = import_attribute(processor['PATH'])
        kwargs = deepcopy(processor)
        kwargs.pop('PATH')
        parsed_processors.append({
            'processor': processor_function,
            'kwargs': kwargs
        })

    return parsed_processors


import io

from django.core.files.base import ContentFile

from da_vinci import images

def resize(image, **kwargs):
    """
    Resized an image based on given width and height
    """
    image.resize(**kwargs)
    return image

def rotate(image, **kwargs):
    """
    Rotates an argument by X degrees.
    """
    image.rotate(**kwargs)
    return image

def flip(image, **kwargs):
    """
    Flips an image. Expects "horizontal" or "vertical" as argument.
    """
    image.flip(**kwargs)
    return image

def crop(image, **kwargs):
    """
    Crops an image based on given width or height
    """
    image.crop(**kwargs)
    return image

def set_quality(image, **kwargs):
    """
    Sets JPG images' quality.
    """
    image.quality = kwargs['quality']
    return image

def process(file, size):
    """
    Process an image through its defined processors
    params :file: filename or file-like object
    params :size: string for size defined in settings
    return a ContentFile
    """
    from . import conf
    # open image in piccaso
    raw_image = images.from_file(file)

    # run through all processors, if defined
    size_dict = conf.SIZES[size]
    for processor in size_dict['PROCESSORS']:
        raw_image = processor['processor'](raw_image, **processor['kwargs'])

    # write to Content File
    image_io = io.BytesIO()
    raw_image.save(file=image_io)
    image_file = ContentFile(image_io.getvalue())
    #print dir(image_file)

    return image_file


import shortuuid
import os

from django.db.models import ImageField as DjangoImageField

from .backends import metadata, storage
from .files import ThumbnailedImageFile
from . import processors, post_processors

class ImageField(DjangoImageField):
    attr_class = ThumbnailedImageFile

    def __init__(self, *args, **kwargs):
        self.resize_source_to = kwargs.pop('resize_source_to', None)
        if kwargs.get('storage'):
            raise ValueError('Please define storage backend in settings.py instead on the field itself')
        kwargs['storage'] = storage.get_backend()
        self.metadata_backend = metadata.get_backend()
        super(ImageField, self).__init__(*args, **kwargs)

    def deconstruct(self):
        name, path, args, kwargs = super(ImageField, self).deconstruct()
        del kwargs['storage']
        return name, path, args, kwargs

    def __unicode__(self):
        return self.attname

    def pre_save(self, model_instance, add):
        """
        Process the source image through the defined processors.
        """
        file = getattr(model_instance, self.attname)

        if file and not file._committed:
            image_file = file
            if self.resize_source_to:
                file.seek(0)
                image_file = processors.process(file, self.resize_source_to)
                image_file = post_processors.process(image_file, self.resize_source_to)
            filename = str(shortuuid.uuid()) + os.path.splitext(file.name)[1]
            file.save(filename, image_file, save=False)
        return file

    def south_field_triple(self):
        """
        Return a suitable description of this field for South.
        Taken from smiley chris' easy_thumbnails
        """
        from south.modelsinspector import introspector
        field_class = 'django.db.models.fields.files.ImageField'
        args, kwargs = introspector(self)
        return (field_class, args, kwargs)


from django.db.models.fields.files import ImageFieldFile

from . import conf, images
from .backends.storage import get_backend
from .images import Thumbnail, FallbackImage
from .metadata import get_path

class SourceImage(ImageFieldFile):

    def __init__(self, name):
        self.name = name

class ThumbnailedImageFile(ImageFieldFile):

    def __init__(self, instance, field, name, **kwargs):
        super(ThumbnailedImageFile, self).__init__(instance, field, name, **kwargs)
        self.metadata_backend = field.metadata_backend
        self.thumbnails = ThumbnailManager(
            metadata_backend=self.metadata_backend,
            storage=self.storage,
            source_image=self
        )

    def save(self, name, content, save=True):
        thumbnail = super(ThumbnailedImageFile, self).save(name, content, save)
        self.metadata_backend.add_source(self.name)
        return thumbnail

class ThumbnailManager(object):
    """A class that manages creation and retrieval of thumbnails."""

    def __init__(self, metadata_backend, storage, source_image):
        self.metadata_backend = metadata_backend
        self.storage = storage
        self.source_image = source_image
        self._thumbnails = None

    def __getattr__(self, name):
        if name in conf.SIZES.keys():
            if not self.source_image:
                fallback_image_url = conf.SIZES[name].get('FALLBACK_IMAGE_URL')
                if fallback_image_url:
                    return FallbackImage(fallback_image_url)
                else:
                    return Thumbnail(metadata=None, storage=self.storage)
            return self.get(name)
        else:
            raise AttributeError("'%s' has no attribute '%s'" % (self, name))

    def _refresh_cache(self):
        """Populate self._thumbnails."""
        self._thumbnails = {}
        metadatas = self.metadata_backend.get_thumbnails(self.source_image.name)
        for metadata in metadatas:
            self._thumbnails[metadata.size] = Thumbnail(metadata=metadata, storage=self.storage)

    def all(self):
        """
        Return all thumbnails in a dict format.
        """
        if self._thumbnails is not None:
            return self._thumbnails
        self._refresh_cache()
        return self._thumbnails

    def get(self, size, create=True):
        """
        Returns a Thumbnail instance.
        First check whether thumbnail is already cached. If it doesn't:
        1. Try to fetch the thumbnail
        2. Create thumbnail if it's not present
        3. Cache the thumbnail for future use
        """
        if self._thumbnails is None:
            self._refresh_cache()

        thumbnail = self._thumbnails.get(size)

        if thumbnail is None:
            thumbnail = images.get(self.source_image.name, size,
                                   self.metadata_backend, self.storage)

            if thumbnail is None:
                thumbnail = self.create(size)

            self._thumbnails[size] = thumbnail

        return thumbnail

    def create(self, size):
        """
        Creates and return a thumbnail of a given size.
        """
        thumbnail = images.create(self.source_image.name, size,
                                  self.metadata_backend, self.storage)
        return thumbnail

    def delete(self, size):
        """
        Deletes a thumbnail of a given size
        """
        images.delete(self.source_image.name, size,
                      self.metadata_backend, self.storage)
        del(self._thumbnails[size])

def exists(source_name, size=None):
    path = get_path(source_name, size)
    if path is not None:
        return get_backend().exists(path)
    else:
        return False

def delete(source_name, size=None):
    path = get_path(source_name, size)
    return get_backend().delete(path)


import os

from django.utils.encoding import smart_text, python_2_unicode_compatible

from . import conf
from . import backends
from . import post_processors
from . import processors

@python_2_unicode_compatible
class Thumbnail(object):
    """
    An object that contains relevant information about a thumbnailed image.
    """

    def __init__(self, metadata, storage):
        self.metadata = metadata
        self.storage = storage
        self.name = getattr(metadata, 'name', None)

    def __str__(self):
        return smart_text(self.name or '')

    def __repr__(self):
        return "<%s: %s>" % (self.__class__.__name__, self.name or "None")

    def __eq__(self, other):
        try:
            return self.__dict__ == other.__dict__
        except AttributeError:
            return False

    def __bool__(self):
        return bool(self.name)

    def check_metadata(self):
        if self.metadata is None:
            raise ValueError('Thumbnail has no source file')

    @property
    def size(self):
        self.check_metadata()
        return self.metadata.size

    @property
    def url(self):
        self.check_metadata()
        return self.storage.url(self.name)

class FallbackImage(object):

    def __init__(self, image_url):
        self.image_url = image_url

    @property
    def url(self):
        return self.image_url

def get_thumbnail_name(source_name, size):
    name, extension = os.path.splitext(source_name)
    filename = "%s_%s%s" % (name, size, extension)
    return os.path.join(conf.BASE_DIR, filename)

def create(source_name, size, metadata_backend=None, storage_backend=None):
    """
    Creates a thumbnail file and its relevant metadata. Returns a
    Thumbnail instance.
    """

    if storage_backend is None:
        storage_backend = backends.storage.get_backend()
    if metadata_backend is None:
        metadata_backend = backends.metadata.get_backend()

    thumbnail_file = processors.process(storage_backend.open(source_name), size)
    thumbnail_file = post_processors.process(thumbnail_file, size)
    name = get_thumbnail_name(source_name, size)
    name = storage_backend.save(name, thumbnail_file)

    metadata = metadata_backend.add_thumbnail(source_name, size, name)
    return Thumbnail(metadata=metadata, storage=storage_backend)

def get(source_name, size, metadata_backend=None, storage_backend=None):
    """
    Returns a Thumbnail instance, or None if thumbnail does not yet exist.
    """
    if storage_backend is None:
        storage_backend = backends.storage.get_backend()
    if metadata_backend is None:
        metadata_backend = backends.metadata.get_backend()

    metadata = metadata_backend.get_thumbnail(source_name, size)
    if metadata is None:
        return None
    else:
        return Thumbnail(metadata=metadata, storage=storage_backend)

def delete(source_name, size, metadata_backend=None, storage_backend=None):
    """
    Deletes a thumbnail file and its relevant metadata.
    """
    if storage_backend is None:
        storage_backend = backends.storage.get_backend()
    if metadata_backend is None:
        metadata_backend = backends.metadata.get_backend()
    storage_backend.delete(get_thumbnail_name(source_name, size))
    metadata_backend.delete_thumbnail(source_name, size)


from .null import NullProvider
from ..lib import digits_only
from ..data import IncomingMessage, MessageAccepted, MessageDelivered

class LoopbackProvider(NullProvider):
    """ Loopback Provider

        Sends messages to registered subscriber callbacks.

        Configuration: none

        Sending: sends message to a registered subscriber (see: :meth:`LoopbackProvider.subscribe`),
            silently ignores other messages

        Receipt: simulation with a method

        Status: always reports success (if was requested for the message)
    """

    def __init__(self, gateway, name):
        super(LoopbackProvider, self).__init__(gateway, name)

        #: Virtual subscribers
        #: { Phone number : callable(message) }
        self._subscribers = {}

        #: Message traffic
        #: list<IncomingMessage|OutgoingMessage>
        self._traffic = []

    #region Public API

    def get_traffic(self):
        """ Fetch the accumulated messages and reset

            :rtype: list
            :returns: List of both IncomingMessage & OutgoingMessage objects
        """
        try:
            return self._traffic
        finally:
            self._traffic = []

    def received(self, src, body):
        """ Simulate an incoming message

            :type src: str
            :param src: Message source
            :type boby: str | unicode
            :param body: Message body
            :rtype: IncomingMessage
        """
        # Create the message
        self._msgid += 1
        message = IncomingMessage(src, body, self._msgid)

        # Log traffic
        self._traffic.append(message)

        # Handle it
        self._receive_message(message)

        # Finish
        return message

    def subscribe(self, number, callback):
        """ Register a virtual subscriber which receives messages to the matching number.

            :type number: str
            :param number: Subscriber phone number
            :type callback: callable
            :param callback: A callback(OutgoingMessage) which handles the messages directed to the subscriber.
                The message object is augmented with the .reply(str) method which allows to send a reply easily!
            :rtype: LoopbackProvider
        """
        self._subscribers[digits_only(number)] = callback
        return self

    #endregion

    def send(self, message):
        message = super(LoopbackProvider, self).send(message)

        # Log traffic
        self._traffic.append(message)

        # Deliver to the subscriber
        subscriber_found = message.dst in self._subscribers
        if subscriber_found:
            # Augment the message with .reply(str)
            def reply(body):
                if message.provider_options.allow_reply:
                    self.received(message.dst, body)
            message.reply = reply

            # Deliver
            self._subscribers[message.dst](message)

        # Delivery notification
        if message.provider_options.status_report:
            # Decide on the MessageStatus class to use
            StatusCls = MessageDelivered if subscriber_found else MessageAccepted

            # Handle status
            status = StatusCls(message.msgid)
            status.status = 'OK'
            self._receive_status(status)

        return message


from datetime import datetime

class MessageStatus(object):
    """ Sent Message Status

        Represent network's response to a sent message.
    """

    #: Provider name
    provider = None

    #: Was the message accepted by the network?
    #: True | False | None (unknown)
    accepted = None

    #: Was the message delivered to the recipient?
    #: True | False | None (unknown)
    delivered = None

    #: Has the mesage expired?
    #: True | False
    expired = False

    #: Has an error occurred? See status then
    #: True | False
    error = False

    #: Status code from the provider, if any
    status_code = None

    #: Status text from the provider, if any
    status = None

    #: Provider-dependent info dict, if any
    meta = None

    def __init__(self, msgid, rtime=None, meta=None):
        """ Create the message status struct.

            Most fields are defined with direct property access, or by using subclasses

            :type msgid: str
            :param msgid: Unique message id
            :type rtime: datetime | None
            :param rtime: Status timestamp, naive, UTC
            :type meta: dict | None
            :param meta: Provider-dependent info
        """
        self.msgid = msgid
        self.rtime = rtime or datetime.utcnow()
        self.meta = meta or {}

    @property
    def states(self):
        """ Get the set of states. Mostly used for pretty printing

            :rtype: set
            :returns: Set of 'accepted', 'delivered', 'expired', 'error'
        """
        ret = set()
        if self.accepted:
            ret.add('accepted')
        if self.delivered:
            ret.add('delivered')
        if self.expired:
            ret.add('expired')
        if self.error:
            ret.add('error')
        return ret

    def __repr__(self):
        return '{cls}({provider!r}, {msgid!r}, state={state!r}, status={status!r})'.format(
            cls=self.__class__.__name__,
            provider=self.provider,
            msgid=self.msgid,
            state=self.states,
            status=self.status
        )

class MessageAccepted(MessageStatus):
    """ Accepted for processing

        The message contained no errors and was accepted, but not yet delivered.
        Not all providers report this status.
    """
    accepted = True
    delivered = False
    expired = False

class MessageDelivered(MessageAccepted):
    """ Delivered successfully

        The message was accepted and finally delivered
    """
    accepted = True
    delivered = True
    expired = False

class MessageExpired(MessageAccepted):
    """ Message has expired

        The message was accepted, has stayed idle for some time, and finally expired
    """
    accepted = True
    delivered = False
    expired = True

class MessageError(MessageStatus):
    """ Late Error

        The message was accepted (as no exception was raised by the Gateway.send() method), but later failed
    """
    accepted = True
    delivered = False
    expired = False
    error = True


from .IProvider import IProvider
from .lib.events import EventHook

class Gateway(object):
    """ SMS Gateway

        The primary object to send & receive messages.

        Sending Steps:

        1. Instantiate the gateway:

            from smsframework import Gateway
            gw = Gateway()

        1. Register providers with add_provider():

            from smsframework.providers import ClickatellProvider
            gw.add_provider('main', ClickatellProvider, ...)

        2. You already can send messages with send()!

            from smsframework import OutgoingMessage
            gw.send(OutgoingMessage('+123456789', 'hi there!'))

        3. Replace the router() function to get customized routing that depends on message fields:

            gw.router = lambda message: 'main'

        4. Use get_provider() to access provider-specific APIs:

            print gw.get_provider('main').get_balance()

        Receiving steps:

        1. Register providers
        2. Subscribe to onReceive and onStatus events:

            gw.onReceive += my_receive_handler
            gw.onStatus += my_status_handler

        3. Initialize message receiver URLs with ....
        4. Visit your provider's control panel and set up the receiver URLs
        5. Enjoy!
    """

    def __init__(self):
        #: Registered providers
        self._providers = {}

        # Events
        self.onSend = EventHook()
        self.onReceive = EventHook()
        self.onStatus = EventHook()

    #region Providers

    def add_provider(self, name, Provider, **config):
        """ Register a provider on the gateway

            The first provider defined becomes the default one: used in case the routing function has no better idea.

            :type name: str
            :param name: Provider name that will be used to uniquely identify it
            :type Provider: type
            :param Provider: Provider class that inherits from `smsframework.IProvider`
            :param config: Provider configuration. Please refer to the Provider documentation.
            :rtype: IProvider
            :returns: The created provider
        """
        assert issubclass(Provider, IProvider), 'Provider does not implement IProvider'
        assert isinstance(name, str), 'Provider name must be a string'

        # Configure
        provider = Provider(self, name, **config)

        # Register
        assert name not in self._providers, 'Provider is already registered'
        self._providers[name] = provider

        # If first - set default
        if self.default_provider is None:
            self.default_provider = name

        # Finish
        return provider

    _default_provider = None

    @property
    def default_provider(self):
        """ Default provider name

            :rtype: str | None
        """
        return self._default_provider

    @default_provider.setter
    def default_provider(self, name):
        assert name in self._providers, 'Provider "{}" is not registered'.format(name)
        self._default_provider = name

    def get_provider(self, name):
        """ Get a provider by name

            You don't normally need this, unless the provider has some public API:
            refer to the provider documentation for the details.

            :type name: str
            :param name: Provider name
            :rtype: IProvider
            :raises KeyError: provider not found
        """
        return self._providers[name]

    #endregion

    #region Sending

    def router(self, message, *args):
        """ Router function that decides which provider to use for the given message for sending.

            Replace it with a function of your choice for custom routing.

            :type message: data.OutgoingMessage
            :param message: The message being sent
            :type args: Message routing values, as specified with the OutgoingMessage.route
            :rtype: str | None
            :returns: Provider name to use, or None to use the default one
        """
        # By default, this always uses the default provider
        return None

    def send(self, message):
        """ Send a message object

            :type message: data.OutgoingMessage
            :param message: The message to send
            :rtype: data.OutgoingMessage
            :returns: The sent message with populated fields
            :raises AssertionError: wrong provider name encountered (returned by the router, or provided to OutgoingMessage)
            :raises MessageSendError: generic errors
            :raises AuthError: provider authentication failed
            :raises LimitsError: sending limits exceeded
            :raises CreditError: not enough money on the account
        """
        # Which provider to use?
        provider_name = self._default_provider  # default
        if message.provider is not None:
            assert message.provider in self._providers, \
                'Unknown provider specified in OutgoingMessage.provideer: {}'.format(provider_name)
            provider = self.get_provider(message.provider)
        else:
            # Apply routing
            if message.routing_values is not None: # Use the default provider when no routing values are given
                # Routing values are present
                provider_name = self.router(message, *message.routing_values) or self._default_provider
                assert provider_name in self._providers, \
                    'Routing function returned an unknown provider name: {}'.format(provider_name)
            provider = self.get_provider(provider_name)

        # Set message provider name
        message.provider = provider.name

        # Send the message using the provider
        message = provider.send(message)

        # Emit the send event
        self.onSend(message)

        # Finish
        return message

    #region

    #region Receipt

    def receiver_blueprint_for(self, name):
        """ Get a Flask blueprint for the named provider that handles incoming messages & status reports

            Note: this requires Flask microframework.

            :rtype: flask.blueprints.Blueprint
            :returns: Flask Blueprint, fully functional
            :raises KeyError: provider not found
            :raises NotImplementedError: Provider does not implement a receiver
        """
        # Get the provider & blueprint
        provider = self.get_provider(name)
        bp = provider.make_receiver_blueprint()

        # Register a Flask handler that initializes `g.provider`
        # This is the only way for the blueprint to get the current IProvider instance
        from flask.globals import g  # local import as the user is not required to use receivers at all

        @bp.before_request
        def init_g():
            g.provider = provider

        # Finish
        return bp

    def receiver_blueprints(self):
        """ Get Flask blueprints for every provider that supports it

            Note: this requires Flask microframework.

            :rtype: dict
            :returns: A dict { provider-name: Blueprint }
        """
        blueprints = {}
        for name in self._providers:
            try:
                blueprints[name] = self.receiver_blueprint_for(name)
            except NotImplementedError:
                pass  # Ignore providers that does not support receivers
        return blueprints

    def receiver_blueprints_register(self, app, prefix='/'):
        """ Register all provider receivers on the provided Flask application under '/{prefix}/provider-name'

            Note: this requires Flask microframework.

            :type app: flask.Flask
            :param app: Flask app to register the blueprints on
            :type prefix: str
            :param prefix: URL prefix to hide the receivers under.
                You likely want some random stuff here so no stranger can simulate incoming messages.
            :rtype: flask.Flask
        """
        # Register
        for name, bp in self.receiver_blueprints().items():
            app.register_blueprint(
                bp,
                url_prefix='{prefix}{name}'.format(
                    prefix='/'+prefix.strip('/')+'/' if prefix else '/',
                    name=name
                )
            )

        # Finish
        return app

    #endregion


from abc import abstractmethod

class IProvider(object):
    """ SmsFramework provider interface

        Implements methods to interact with the :class:`smsframework.Gateway`
    """

    def __init__(self, gateway, name, **config):
        """ Initialize the provider

            :type gateway: Gateway
            :param gateway: Parent Gateway
            :type name: str
            :param name: Provider name. Used to uniquely identify the provider
            :param config: Provider-dependent configuration
        """
        self.gateway = gateway
        self.name = name

    def send(self, message):
        """ Send a message

            Providers are required to:
            * Populate `message.msgid` and `message.meta` on completion
            * Expect that `message.src` can be empty
            * Support both ASCII and Unicode messages
            * Use `message.params` for provider-dependent configuration
            * Raise exceptions from `exc.py` for errors

            :type message: data.OutgoingMessage
            :param message: The message to send
            :rtype: OutgoingMessage
            :returns: The sent message with populated fields
            :raises MessageSendError: sending errors
        """
        raise NotImplementedError('Provider.send not implemented')

    def make_receiver_blueprint(self):
        """ Get a Blueprint for the HTTP receiver

            :rtype: flask.Blueprint
            :returns: configured Flask Blueprint receiver
            :raises NotImplementedError: Provider does not support message reception
        """
        raise NotImplementedError('Provider does not support message reception')

    #region Receiver callbacks

    def _receive_message(self, message):
        """ Incoming message callback

            Calls Gateway.onReceive event hook

            Providers are required to:
            * Cast phone numbers to digits-only
            * Support both ASCII and Unicode messages
            * Populate `message.msgid` and `message.meta` fields
            * If this method fails with an exception, the provider is required to respond with an error to the service

            :type message: IncomingMessage
            :param message: The received message
            :rtype: IncomingMessage
        """
        # Populate fields
        message.provider = self.name

        # Fire the event hook
        self.gateway.onReceive(message)

        # Finish
        return message

    def _receive_status(self, status):
        """ Incoming status callback

            Calls Gateway.onStatus event hook

            Providers are required to:
            * Cast phone numbers to digits-only
            * Use proper MessageStatus subclasses
            * Populate `status.msgid` and `status.meta` fields
            * If this method fails with an exception, the provider is required to respond with an error to the service

            :type status: MessageStatus
            :param status: The received status
            :rtype: MessageStatus
        """
        # Populate fields
        status.provider = self.name

        # Fire the event hook
        self.gateway.onStatus(status)

        # Finish
        return status

    #endregion


from flask import Blueprint
from flask.globals import request, g

from .provider import jsonex_loads, jsonex_api

bp = Blueprint('smsframework-forward-client', __name__, url_prefix='/')

@bp.route('/im', methods=['POST'])
@jsonex_api
def im():
    """ Incoming message handler: forwarded by ForwardServerProvider """
    req = jsonex_loads(request.get_data())
    message = g.provider._receive_message(req['message'])
    return {'message': message}

@bp.route('/status', methods=['POST'])
@jsonex_api
def status():
    """ Incoming status handler: forwarded by ForwardServerProvider """
    req = jsonex_loads(request.get_data())
    status = g.provider._receive_status(req['status'])
    return {'status': status}