import json
import copy
import requests
import re

from sdcclient._monitor import SdMonitorClient

try:
    basestring
except NameError:
    basestring = str

class SdMonitorClientV1(SdMonitorClient):
    '''**Description**
        Handles dashboards version 1 (ie. up to February 2019). For later Sysdig Monitor versions, please use :class:`~SdMonitorClient` instead.
    '''

    def __init__(self, token="", sdc_url='https://app.sysdigcloud.com', ssl_verify=True):
        super(SdMonitorClientV1, self).__init__(token, sdc_url, ssl_verify)
        self._dashboards_api_version = 'v1'
        self._dashboards_api_endpoint = '/ui/dashboards'
        self._default_dashboards_api_endpoint = '/api/defaultDashboards'

    def create_dashboard_from_template(self, dashboard_name, template, scope, shared=False, public=False, annotations={}):
        if scope is not None:
            if isinstance(scope, basestring) == False:
                return [False, 'Invalid scope format: Expected a string']

        #
        # Clean up the dashboard we retireved so it's ready to be pushed
        #
        template['id'] = None
        template['version'] = None
        template['schema'] = 1
        template['name'] = dashboard_name
        template['isShared'] = shared
        template['isPublic'] = public
        template['publicToken'] = None

        # set dashboard scope to the specific parameter
        scopeExpression = self.convert_scope_string_to_expression(scope)
        if scopeExpression[0] == False:
            return scopeExpression
        template['filterExpression'] = scope
        template['scopeExpressionList'] = map(lambda ex: {'operand':ex['operand'], 'operator':ex['operator'],'value':ex['value'],'displayName':'', 'isVariable':False}, scopeExpression[1])

        if 'widgets' in template and template['widgets'] is not None:
            # Default dashboards (aka Explore views) specify panels with the property `widgets`,
            # while custom dashboards use `items`
            template['items'] = list(template['widgets'])
            del template['widgets']

        # NOTE: Individual panels might override the dashboard scope, the override will NOT be reset
        if 'items' in template and template['items'] is not None:
            for chart in template['items']:
                if 'overrideFilter' not in chart:
                    chart['overrideFilter'] = False

                if chart['overrideFilter'] == False:
                    # patch frontend bug to hide scope override warning even when it's not really overridden
                    chart['scope'] = scope

                # if chart scope is equal to dashboard scope, set it as non override
                chart_scope = chart['scope'] if 'scope' in chart else None
                chart['overrideFilter'] = chart_scope != scope

        if 'annotations' in template:
            template['annotations'].update(annotations)
        else:
            template['annotations'] = annotations

        template['annotations']['createdByEngine'] = True

        #
        # Create the new dashboard
        #
        res = requests.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': template}), verify=self.ssl_verify)
        return self._request_result(res)

    def create_dashboard(self, name):
        '''
        **Description**
            Creates an empty dashboard. You can then add panels by using ``add_dashboard_panel``.

        **Arguments**
            - **name**: the name of the dashboard that will be created.

        **Success Return Value**
            A dictionary showing the details of the new dashboard.

        **Example**
            `examples/dashboard.py <https://github.com/draios/python-sdc-client/blob/master/examples/dashboard.py>`_
        '''
        dashboard_configuration = {
            'name': name,
            'schema': 2,
            'items': []
        }

        #
        # Create the new dashboard
        #
        res = requests.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}),
                            verify=self.ssl_verify)
        return self._request_result(res)

    def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, sort_by=None, limit=None, layout=None):
        """**Description**
            Adds a panel to the dashboard. A panel can be a time series, or a top chart (i.e. bar chart), or a number panel.

        **Arguments**
            - **dashboard**: dashboard to edit
            - **name**: name of the new panel
            - **panel_type**: type of the new panel. Valid values are: ``timeSeries``, ``top``, ``number``
            - **metrics**:  a list of dictionaries, specifying the metrics to show in the panel, and optionally, if there is only one metric, a grouping key to segment that metric by. A metric is any of the entries that can be found in the *Metrics* section of the Explore page in Sysdig Monitor. Metric entries require an *aggregations* section specifying how to aggregate the metric across time and groups of containers/hosts. A grouping key is any of the entries that can be found in the *Show* or *Segment By* sections of the Explore page in Sysdig Monitor. Refer to the examples section below for ready to use code snippets. Note, certain panels allow certain combinations of metrics and grouping keys:
                - ``timeSeries``: 1 or more metrics OR 1 metric + 1 grouping key
                - ``top``: 1 or more metrics OR 1 metric + 1 grouping key
                - ``number``: 1 metric only
            - **scope**: filter to apply to the panel; must be based on metadata available in Sysdig Monitor; Example: *kubernetes.namespace.name='production' and container.image='nginx'*.
            - **sort_by**: Data sorting; The parameter is optional and it's a dictionary of ``metric`` and ``mode`` (it can be ``desc`` or ``asc``)
            - **limit**: This parameter sets the limit on the number of lines/bars shown in a ``timeSeries`` or ``top`` panel. In the case of more entities being available than the limit, the top entities according to the sort will be shown. The default value is 10 for ``top`` panels (for ``timeSeries`` the default is defined by Sysdig Monitor itself). Note that increasing the limit above 10 is not officially supported and may cause performance and rendering issues
            - **layout**: Size and position of the panel. The dashboard layout is defined by a grid of 12 columns, each row height is equal to the column height. For example, say you want to show 2 panels at the top: one panel might be 6 x 3 (half the width, 3 rows height) located in row 1 and column 1 (top-left corner of the viewport), the second panel might be 6 x 3 located in row 1 and position 7. The location is specified by a dictionary of ``row`` (row position), ``col`` (column position), ``size_x`` (width), ``size_y`` (height).

        **Success Return Value**
            A dictionary showing the details of the edited dashboard.

        **Example**
            `examples/dashboard.py <https://github.com/draios/python-sdc-client/blob/master/examples/dashboard.py>`_
        """
        panel_configuration = {
            'name': name,
            'showAs': None,
            'showAsType': None,
            'metrics': [],
            'gridConfiguration': {
                'col': 1,
                'row': 1,
                'size_x': 12,
                'size_y': 6
            }
        }

        if panel_type == 'timeSeries':
            #
            # In case of a time series, the current dashboard implementation
            # requires the timestamp to be explicitly specified as "key".
            # However, this function uses the same abstraction of the data API
            # that doesn't require to specify a timestamp key (you only need to
            # specify time window and sampling)
            #
            metrics = copy.copy(metrics)
            metrics.insert(0, {'id': 'timestamp'})

        #
        # Convert list of metrics to format used by Sysdig Monitor
        #
        property_names = {}
        k_count = 0
        v_count = 0
        for i, metric in enumerate(metrics):
            property_name = 'v' if 'aggregations' in metric else 'k'

            if property_name == 'k':
                i = k_count
                k_count += 1
            else:
                i = v_count
                v_count += 1
            property_names[metric['id']] = property_name + str(i)

            panel_configuration['metrics'].append({
                'metricId': metric['id'],
                'aggregation': metric['aggregations']['time'] if 'aggregations' in metric else None,
                'groupAggregation': metric['aggregations']['group'] if 'aggregations' in metric else None,
                'propertyName': property_name + str(i)
            })

        panel_configuration['scope'] = scope
        # if chart scope is equal to dashboard scope, set it as non override
        panel_configuration['overrideFilter'] = ('scope' in dashboard and dashboard['scope'] != scope) or ('scope' not in dashboard and scope != None)

        #
        # Configure panel type
        #
        if panel_type == 'timeSeries':
            panel_configuration['showAs'] = 'timeSeries'
            panel_configuration['showAsType'] = 'line'

            if limit != None:
                panel_configuration['paging'] = {
                    'from': 0,
                    'to': limit - 1
                }

        elif panel_type == 'number':
            panel_configuration['showAs'] = 'summary'
            panel_configuration['showAsType'] = 'summary'
        elif panel_type == 'top':
            panel_configuration['showAs'] = 'top'
            panel_configuration['showAsType'] = 'bars'

            if sort_by is None:
                panel_configuration['sorting'] = [{
                    'id': 'v0',
                    'mode': 'desc'
                }]
            else:
                panel_configuration['sorting'] = [{
                    'id': property_names[sort_by['metric']],
                    'mode': sort_by['mode']
                }]

            if limit is None:
                panel_configuration['paging'] = {
                    'from': 0,
                    'to': 10
                }
            else:
                panel_configuration['paging'] = {
                    'from': 0,
                    'to': limit - 1
                }

        #
        # Configure layout
        #
        if layout != None:
            panel_configuration['gridConfiguration'] = layout

        #
        # Clone existing dashboard...
        #
        dashboard_configuration = copy.deepcopy(dashboard)
        dashboard_configuration['id'] = None

        #
        # ... and add the new panel
        #
        dashboard_configuration['items'].append(panel_configuration)

        #
        # Update dashboard
        #
        res = requests.put(self.url + self._dashboards_api_endpoint + '/' + str(dashboard['id']), headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}),
                           verify=self.ssl_verify)
        return self._request_result(res)

    def remove_dashboard_panel(self, dashboard, panel_name):
        '''**Description**
            Removes a panel from the dashboard. The panel to remove is identified by the specified ``name``.

        **Arguments**
            - **name**: name of the panel to find and remove

        **Success Return Value**
            A dictionary showing the details of the edited dashboard.

        **Example**
            `examples/dashboard.py <https://github.com/draios/python-sdc-client/blob/master/examples/dashboard.py>`_
        '''
        #
        # Clone existing dashboard...
        #
        dashboard_configuration = copy.deepcopy(dashboard)
        dashboard_configuration['id'] = None

        #
        # ... find the panel
        #
        def filter_fn(panel):
            return panel['name'] == panel_name

        panels = list(filter(filter_fn, dashboard_configuration['items']))

        if len(panels) > 0:
            #
            # ... and remove it
            #
            for panel in panels:
                dashboard_configuration['items'].remove(panel)

            #
            # Update dashboard
            #
            res = requests.put(self.url + self._dashboards_api_endpoint + '/' + str(dashboard['id']), headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}),
                               verify=self.ssl_verify)
            return self._request_result(res)
        else:
            return [False, 'Not found']

    def _get_dashboard_converters(self):
        '''**Description**
            Internal function to return dashboard converters from one version to another one.
        '''
        # There's not really a previous version...
        return {}


import base64
import hashlib
import json
import re
import requests
import time

try:
    from urllib.parse import quote_plus, unquote_plus
except ImportError:
    from urllib import quote_plus, unquote_plus

from sdcclient._common import _SdcCommon

class SdScanningClient(_SdcCommon):

    def __init__(self, token="", sdc_url='https://secure.sysdig.com', ssl_verify=True):
        super(SdScanningClient, self).__init__(token, sdc_url, ssl_verify)
        self.product = "SDS"

    def add_image(self, image, force=False, dockerfile=None, annotations={}, autosubscribe=True):
        '''**Description**
            Add an image to the scanner

        **Arguments**
            - image: Input image can be in the following formats: registry/repo:tag
            - dockerfile: The contents of the dockerfile as a str.
            - annotations: A dictionary of annotations {str: str}.
            - autosubscribe: Should active the subscription to this image?

        **Success Return Value**
            A JSON object representing the image that was added.
        '''
        itype = self._discover_inputimage_format(image)
        if itype != 'tag':
            return [False, "can only add a tag"]

        payload = {}
        if dockerfile:
            payload['dockerfile'] = base64.b64encode(dockerfile.encode()).decode("utf-8")
        payload['tag'] = image
        if annotations:
            payload['annotations'] = annotations

        url = "{base_url}/api/scanning/v1/anchore/images?autosubscribe={autosubscribe}{force}".format(
            base_url=self.url,
            autosubscribe=str(autosubscribe),
            force="&force=true" if force else "")

        res = requests.post(url, data=json.dumps(payload), headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def import_image(self, image_data):
        '''**Description**
            Import an image from the scanner export

        **Arguments**
            - image_data: A JSON with the image information.

        **Success Return Value**
            A JSON object representing the image that was imported.
        '''
        url = self.url + "/api/scanning/v1/anchore/imageimport"
        res = requests.post(url, data=json.dumps(image_data), headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def get_image(self, image, show_history=False):
        '''**Description**
            Find the image with the tag <image> and return its json description

        **Arguments**
            - image: Input image can be in the following formats: registry/repo:tag

        **Success Return Value**
            A JSON object representing the image.
        '''
        itype = self._discover_inputimage_format(image)
        if itype not in ['tag', 'imageid', 'imageDigest']:
            return [False, "cannot use input image string: no discovered imageDigest"]

        params = {}
        params['history'] = str(show_history and itype not in ['imageid', 'imageDigest']).lower()
        if itype == 'tag':
            params['fulltag'] = image

        url = self.url + "/api/scanning/v1/anchore/images"
        url += {
            'imageid': '/by_id/{}'.format(image),
            'imageDigest': '/{}'.format(image)
        }.get(itype, '')

        res = requests.get(url, params=params, headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def list_images(self):
        '''**Description**
            List the current set of images in the scanner.

        **Arguments**
            - None

        **Success Return Value**
            A JSON object containing all the images.
        '''
        url = self.url + "/api/scanning/v1/anchore/images"
        res = requests.get(url, headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def query_image_content(self, image, content_type=""):
        '''**Description**
            Find the image with the tag <image> and return its content.

        **Arguments**
            - image: Input image can be in the following formats: registry/repo:tag
            - content_type: The content type can be one of the following types:
                - os: Operating System Packages
                - npm: Node.JS NPM Module
                - gem: Ruby GEM
                - files: Files

        **Success Return Value**
            A JSON object representing the image content.
        '''
        return self._query_image(image, query_group='content', query_type=content_type)

    def query_image_metadata(self, image, metadata_type=""):
        '''**Description**
            Find the image with the tag <image> and return its metadata.

        **Arguments**
            - image: Input image can be in the following formats: registry/repo:tag
            - metadata_type: The metadata type can be one of the types returned by running without a type specified

        **Success Return Value**
            A JSON object representing the image metadata.
        '''
        return self._query_image(image, query_group='metadata', query_type=metadata_type)

    def query_image_vuln(self, image, vuln_type="", vendor_only=True):
        '''**Description**
            Find the image with the tag <image> and return its vulnerabilities.

        **Arguments**
            - image: Input image can be in the following formats: registry/repo:tag
            - vuln_type: Vulnerability type can be one of the following types:
                - os: CVE/distro vulnerabilities against operating system packages

        **Success Return Value**
            A JSON object representing the image vulnerabilities.
        '''
        return self._query_image(image, query_group='vuln', query_type=vuln_type, vendor_only=vendor_only)

    def _query_image(self, image, query_group="", query_type="", vendor_only=True):
        if not query_group:
            raise Exception("need to specify a query group")

        _, _, image_digest = self._discover_inputimage(image)
        if not image_digest:
            return [False, "cannot use input image string (no discovered imageDigest)"]

        url = "{base_url}/api/scanning/v1/anchore/images/{image_digest}/{query_group}/{query_type}{vendor_only}".format(
            base_url=self.url,
            image_digest=image_digest,
            query_group=query_group,
            query_type=query_type if query_type else '',
            vendor_only="?vendor_only={}".format(vendor_only) if query_group == 'vuln' else '')

        res = requests.get(url, headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def delete_image(self, image, force=False):
        '''**Description**
            Delete image from the scanner.

        **Arguments**
            - None
        '''
        _, _, image_digest = self._discover_inputimage(image)
        if not image_digest:
            return [False, "cannot use input image string: no discovered imageDigest"]

        url = self.url + "/api/scanning/v1/anchore/images/" + image_digest
        res = requests.delete(url, headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def check_image_evaluation(self, image, show_history=False, detail=False, tag=None, policy=None):
        '''**Description**
            Check the latest policy evaluation for an image

        **Arguments**
            - image: Input image can be in the following formats: registry/repo:tag
            - show_history: Show all previous policy evaluations
            - detail: Show detailed policy evaluation report
            - tag: Specify which TAG is evaluated for a given image ID or Image Digest
            - policy: Specify which POLICY to use for evaluate (defaults currently active policy)

        **Success Return Value**
            A JSON object representing the evaluation status.
        '''
        itype, _, image_digest = self._discover_inputimage(image)
        if not image_digest:
            return [False, "could not get image record from anchore"]
        if not tag and itype != 'tag':
            return [False, "input image name is not a tag, and no --tag is specified"]

        thetag = tag if tag else image

        url = "{base_url}/api/scanning/v1/anchore/images/{image_digest}/check?history={history}&detail={detail}&tag={tag}{policy_id}"
        url = url.format(
            base_url=self.url,
            image_digest=image_digest,
            history=str(show_history).lower(),
            detail=str(detail).lower(),
            tag=thetag,
            policy_id=("&policyId=%s" % policy) if policy else "")

        res = requests.get(url, headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def add_registry(self, registry, registry_user, registry_pass, insecure=False, registry_type="docker_v2", validate=True):
        '''**Description**
            Add image registry

        **Arguments**
            - registry: Full hostname/port of registry. Eg. myrepo.example.com:5000
            - registry_user: Username
            - registry_pass: Password
            - insecure: Allow connection to registry without SSL cert checks (ex: if registry uses a self-signed SSL certificate)
            - registry_type: Specify the registry type. 'docker_v2' and 'awsecr' are supported (default='docker_v2')
            - validate: If set to 'False' will not attempt to validate registry/creds on registry add

        **Success Return Value**
            A JSON object representing the registry.
        '''
        registry_types = ['docker_v2', 'awsecr']
        if registry_type and registry_type not in registry_types:
            return [False, "input registry type not supported (supported registry_types: " + str(registry_types)]
        if self._registry_string_is_valid(registry):
            return [False, "input registry name cannot contain '/' characters - valid registry names are of the form <host>:<port> where :<port> is optional"]

        if not registry_type:
            registry_type = self._get_registry_type(registry)

        payload = {
            'registry': registry,
            'registry_user': registry_user,
            'registry_pass': registry_pass,
            'registry_type': registry_type,
            'registry_verify': not insecure}
        url = "{base_url}/api/scanning/v1/anchore/registries?validate={validate}".format(
            base_url=self.url,
            validate=validate)

        res = requests.post(url, data=json.dumps(payload), headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def update_registry(self, registry, registry_user, registry_pass, insecure=False, registry_type="docker_v2", validate=True):
        '''**Description**
            Update an existing image registry.

        **Arguments**
            - registry: Full hostname/port of registry. Eg. myrepo.example.com:5000
            - registry_user: Username
            - registry_pass: Password
            - insecure: Allow connection to registry without SSL cert checks (ex: if registry uses a self-signed SSL certificate)
            - registry_type: Specify the registry type. 'docker_v2' and 'awsecr' are supported (default='docker_v2')
            - validate: If set to 'False' will not attempt to validate registry/creds on registry add

        **Success Return Value**
            A JSON object representing the registry.
        '''
        if self._registry_string_is_valid(registry):
            return [False, "input registry name cannot contain '/' characters - valid registry names are of the form <host>:<port> where :<port> is optional"]

        payload = {
            'registry': registry,
            'registry_user': registry_user,
            'registry_pass': registry_pass,
            'registry_type': registry_type,
            'registry_verify': not insecure}
        url = "{base_url}/api/scanning/v1/anchore/registries/{registry}?validate={validate}".format(
            base_url=self.url,
            registry=registry,
            validate=validate)

        res = requests.put(url, data=json.dumps(payload), headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def delete_registry(self, registry):
        '''**Description**
            Delete an existing image registry

        **Arguments**
            - registry: Full hostname/port of registry. Eg. myrepo.example.com:5000
        '''
        # do some input string checking
        if re.match(".*\\/.*", registry):
            return [False, "input registry name cannot contain '/' characters - valid registry names are of the form <host>:<port> where :<port> is optional"]

        url = self.url + "/api/scanning/v1/anchore/registries/" + registry
        res = requests.delete(url, headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def list_registry(self):
        '''**Description**
            List all current image registries

        **Arguments**
            - None

        **Success Return Value**
            A JSON object representing the list of registries.
        '''
        url = self.url + "/api/scanning/v1/anchore/registries"
        res = requests.get(url, headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def get_registry(self, registry):
        '''**Description**
            Find the registry and return its json description

        **Arguments**
            - registry: Full hostname/port of registry. Eg. myrepo.example.com:5000

        **Success Return Value**
            A JSON object representing the registry.
        '''
        if self._registry_string_is_valid(registry):
            return [False, "input registry name cannot contain '/' characters - valid registry names are of the form <host>:<port> where :<port> is optional"]

        url = self.url + "/api/scanning/v1/anchore/registries/" + registry
        res = requests.get(url, headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def _get_registry_type(self, registry):
        if re.match("[0-9]+\\.dkr\\.ecr\\..*\\.amazonaws\\.com", registry):
            return "awsecr"
        return "docker_v2"

    def _registry_string_is_valid(self, registry):
        return re.match(".*\\/.*", registry)

    def add_policy(self, name, rules, comment="", bundleid=None):
        '''**Description**
            Create a new policy

        **Arguments**
            - name: The name of the policy.
            - rules: A list of Anchore PolicyRule elements (while creating/updating a policy, new rule IDs will be created backend side)
            - comment: A human-readable description.
            - bundleid: Target bundle. If not specified, the currently active bundle will be used.

        **Success Return Value**
            A JSON object containing the policy description.
        '''
        policy = {
            'name': name,
            'comment': comment,
            'rules': rules,
            'version': '1_0'
        }
        if bundleid:
            policy['policyBundleId'] = bundleid

        url = self.url + '/api/scanning/v1/policies'
        data = json.dumps(policy)
        res = requests.post(url, headers=self.hdrs, data=data, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def list_policy_bundles(self, detail=False):
        url = "{base_url}/api/scanning/v1/anchore/policies?detail={detail}".format(
            base_url=self.url,
            detail=str(detail))
        res = requests.get(url, headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def list_policies(self, bundleid=None):
        '''**Description**
            List the current set of scanning policies.

        **Arguments**
            - bundleid: Target bundle. If not specified, the currently active bundle will be used.

        **Success Return Value**
            A JSON object containing the list of policies.
        '''
        url = self.url + '/api/scanning/v1/policies'
        if bundleid:
            url += '?bundleId=' + bundleid

        res = requests.get(url, headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def get_policy(self, policyid, bundleid=None):
        '''**Description**
            Retrieve the policy with the given id in the targeted policy bundle

        **Arguments**
            - policyid: Unique identifier associated with this policy.
            - bundleid: Target bundle. If not specified, the currently active bundle will be used.

        **Success Return Value**
            A JSON object containing the policy description.
        '''
        url = self.url + '/api/scanning/v1/policies/' + policyid
        if bundleid:
            url += '?bundleId=' + bundleid

    def update_policy(self, policyid, policy_description):
        '''**Description**
            Update the policy with the given id

        **Arguments**
            - policyid: Unique identifier associated with this policy.
            - policy_description: A dictionary with the policy description.

        **Success Return Value**
            A JSON object containing the policy description.
        '''
        url = self.url + '/api/scanning/v1/policies/' + policyid
        data = json.dumps(policy_description)
        res = requests.put(url, headers=self.hdrs, data=data, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def delete_policy(self, policyid, bundleid=None):
        '''**Description**
            Delete the policy with the given id in the targeted policy Bundle

        **Arguments**
            - policyid: Unique identifier associated with this policy.
            - policy_description: A dictionary with the policy description.
        '''
        url = self.url + '/api/scanning/v1/policies/' + policyid
        if bundleid:
            url += '?bundleId=' + bundleid

        res = requests.delete(url, headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.text]

    def add_alert(self, name, description=None, scope="", triggers={'failed': True, 'unscanned': True},
                  enabled=False, notification_channels=[]):
        '''**Description**
            Create a new alert

        **Arguments**
            - name: The name of the alert.
            - description: The descprition of the alert.
            - scope: An AND-composed string of predicates that selects the scope in which the alert will be applied. (like: 'host.domain = "example.com" and container.image != "alpine:latest"')
            - tiggers: A dict {str: bool} indicating wich triggers should be enabled/disabled. (default: {'failed': True, 'unscanned': True})
            - enabled: Whether this alert should actually be applied.
            - notification_channels: A list of notification channel ids.

        **Success Return Value**
            A JSON object containing the alert description.
        '''
        alert = {
            'name': name,
            'description': description,
            'triggers': triggers,
            'scope': scope,
            'enabled': enabled,
            'autoscan': True,
            'notificationChannelIds': notification_channels,
        }

        url = self.url + '/api/scanning/v1/alerts'
        data = json.dumps(alert)
        res = requests.post(url, headers=self.hdrs, data=data, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def list_alerts(self, limit=None, cursor=None):
        '''**Description**
            List the current set of scanning alerts.

        **Arguments**
            - limit: Maximum number of alerts in the response.
            - cursor: An opaque string representing the current position in the list of alerts. It's provided in the 'responseMetadata' of the list_alerts response.

        **Success Return Value**
            A JSON object containing the list of alerts.
        '''
        url = self.url + '/api/scanning/v1/alerts'
        if limit:
            url += '?limit=' + str(limit)
            if cursor:
                url += '&cursor=' + cursor

        res = requests.get(url, headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def get_alert(self, alertid):
        '''**Description**
            Retrieve the scanning alert with the given id

        **Arguments**
            - alertid: Unique identifier associated with this alert.

        **Success Return Value**
            A JSON object containing the alert description.
        '''
        url = self.url + '/api/scanning/v1/alerts/' + alertid
        res = requests.get(url, headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def update_alert(self, alertid, alert_description):
        '''**Description**
            Update the alert with the given id

        **Arguments**
            - alertid: Unique identifier associated with this alert.
            - alert_description: A dictionary with the alert description.

        **Success Return Value**
            A JSON object containing the alert description.
        '''
        url = self.url + '/api/scanning/v1/alerts/' + alertid
        data = json.dumps(alert_description)
        res = requests.put(url, headers=self.hdrs, data=data, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def delete_alert(self, policyid):
        '''**Description**
            Delete the alert with the given id

        **Arguments**
            - alertid: Unique identifier associated with this alert.
        '''
        url = self.url + '/api/scanning/v1/alerts/' + policyid
        res = requests.delete(url, headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.text]

    def activate_subscription(self, subscription_type, subscription_key):
        '''**Description**
            Activate a subscription

        **Arguments**
            - subscription_type: Type of subscription. Valid options:
                - 'tag_update': Receive notification when new image is pushed
                - 'policy_eval': Receive notification when image policy status changes
                - 'vuln_update': Receive notification when vulnerabilities are added, removed or modified
            - subscription_key: Fully qualified name of tag to subscribe to. Eg. docker.io/library/alpine:latest
        '''
        return self._update_subscription(subscription_type, subscription_key, True)

    def deactivate_subscription(self, subscription_type, subscription_key):
        '''**Description**
            Deactivate a subscription

        **Arguments**
            - subscription_type: Type of subscription. Valid options:
                - 'tag_update': Receive notification when new image is pushed
                - 'policy_eval': Receive notification when image policy status changes
                - 'vuln_update': Receive notification when vulnerabilities are added, removed or modified
            - subscription_key: Fully qualified name of tag to subscribe to. Eg. docker.io/library/alpine:latest
        '''
        return self._update_subscription(subscription_type, subscription_key, False)

    def _update_subscription(self, subscription_type, subscription_key, activate):
        hashstr = '+'.join([self.token, subscription_key, subscription_type]).encode('utf-8')
        subscription_id = hashlib.md5(hashstr).hexdigest()
        url = self.url + "/api/scanning/v1/anchore/subscriptions/" + subscription_id
        payload = {'active': activate, 'subscription_key': subscription_key, 'subscription_type': subscription_type}

        res = requests.put(url, data=json.dumps(payload), headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def list_subscription(self):
        '''**Description**
            List all subscriptions

        **Arguments**
            - None

        **Success Return Value**
            A JSON object representing the list of subscriptions.
        '''
        url = self.url + "/api/scanning/v1/anchore/subscriptions"
        res = requests.get(url, headers=self.hdrs, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def list_runtime(self, scope="", skip_policy_evaluation=True, start_time=None, end_time=None):
        '''**Description**
            List runtime containers

        **Arguments**
            - scope: An AND-composed string of predicates that selects the scope in which the alert will be applied. (like: 'host.domain = "example.com" and container.image != "alpine:latest"')
            - skip_policy_evaluation: If true, no policy evaluations will be triggered for the images.
            - start_time: Start of the time range (integer of unix time).
            - end_time: End of the time range (integer of unix time).

        **Success Return Value**
            A JSON object representing the list of runtime containers.
        '''
        containers = {
            'scope': scope,
            'skipPolicyEvaluation': skip_policy_evaluation
        }
        if start_time or end_time:
            containers['time'] = {}
            containers['time']['from'] = int(start_time * 100000) if start_time else 0
            end_time = end_time if end_time else time.time()
            containers['time']['to'] = int(end_time * 1000000)

        url = self.url + '/api/scanning/v1/query/containers'
        data = json.dumps(containers)
        res = requests.post(url, headers=self.hdrs, data=data, verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]

        return [True, res.json()]

    def _discover_inputimage_format(self, input_string):
        itype = None

        if re.match("^sha256:[0-9a-fA-F]{64}", input_string):
            itype = 'imageDigest'
        elif re.match("[0-9a-fA-F]{64}", input_string):
            itype = 'imageid'
        else:
            itype = 'tag'

        return itype

    def _discover_inputimage(self, input_string):
        patt = re.match(".*(sha256:.*)", input_string)
        if patt:
            urldigest = quote_plus(patt.group(1))
            return "digest", input_string, urldigest

        try:
            digest = unquote_plus(str(input_string))
            for tpe in ["sha256", "local"]:
                patt = re.match(".*({}:.*)".format(tpe), digest)
                if patt:
                    return "imageDigest", input_string, input_string
        except Exception:
            pass

        urldigest = None
        ret_type = "tag"
        ok, ret = self.get_image(input_string)
        if ok:
            image_record = ret[0]
            urldigest = image_record.get('imageDigest', None)
            for image_detail in image_record.get('image_detail', []):
                if input_string == image_detail.get('imageId', ''):
                    ret_type = "imageid"
                    break

        return ret_type, input_string, urldigest


"""
A threaded discovery daemon implementation.
"""

# Python2 compatibility
from __future__ import print_function
from __future__ import unicode_literals

import logging
import random
import time
import uuid
import socket
import struct
import threading
import select

from .udp import UDPMessage
from .envelope import SoapEnvelope
from .actions import *
from .uri import URI
from .util import _getNetworkAddrs, matchScope
from .util import _generateInstanceId, extractSoapUdpAddressFromURI
from .message import createSOAPMessage, parseSOAPMessage
from .service import Service
from .namespaces import NS_D

logger = logging.getLogger("ws-discovery")

BUFFER_SIZE = 0xffff
_NETWORK_ADDRESSES_CHECK_TIMEOUT = 5
MULTICAST_PORT = 3702
MULTICAST_IPV4_ADDRESS = "239.255.255.250"

APP_MAX_DELAY = 500 # miliseconds

ADDRESS_ALL = "urn:schemas-xmlsoap-org:ws:2005:04:discovery"
ADDRESS_UNKNOWN = "http://schemas.xmlsoap.org/ws/2004/08/addressing/role/anonymous"

class _StoppableDaemonThread(threading.Thread):
    """Stoppable daemon thread.

    run() method shall exit, when self._quitEvent.wait() returned True
    """
    def __init__(self):
        self._quitEvent = threading.Event()
        super(_StoppableDaemonThread, self).__init__()
        self.daemon = True

    def schedule_stop(self):
        """Schedule stopping the thread.
        Use join() to wait, until thread really has been stopped
        """
        self._quitEvent.set()

class AddressMonitorThread(_StoppableDaemonThread):
    def __init__(self, wsd):
        self._addrs = set()
        self._wsd = wsd
        super(AddressMonitorThread, self).__init__()
        self._updateAddrs()

    def _updateAddrs(self):
        addrs = set(_getNetworkAddrs())

        disappeared = self._addrs.difference(addrs)
        new = addrs.difference(self._addrs)

        for addr in disappeared:
            self._wsd._networkAddressRemoved(addr)

        for addr in new:
            self._wsd._networkAddressAdded(addr)

        self._addrs = addrs

    def run(self):
        while not self._quitEvent.wait(_NETWORK_ADDRESSES_CHECK_TIMEOUT):
            self._updateAddrs()

class NetworkingThread(_StoppableDaemonThread):
    def __init__(self, observer, capture=None):
        super(NetworkingThread, self).__init__()

        self.setDaemon(True)
        self._queue = []    # FIXME synchronisation

        self._knownMessageIds = set()
        self._iidMap = {}
        self._observer = observer
        self._capture = observer._capture
        self._seqnum = 1 # capture sequence number
        self._poll = select.poll()

    @staticmethod
    def _makeMreq(addr):
        return struct.pack("4s4s", socket.inet_aton(MULTICAST_IPV4_ADDRESS), socket.inet_aton(addr))

    @staticmethod
    def _createMulticastOutSocket(addr, ttl):
        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        sock.setblocking(0)
        sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl)
        if addr is None:
            sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, socket.INADDR_ANY)
        else:
            sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, socket.inet_aton(addr))

        return sock

    @staticmethod
    def _createMulticastInSocket():
        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

        sock.bind(('', MULTICAST_PORT))

        sock.setblocking(0)

        return sock

    def addSourceAddr(self, addr):
        """None means 'system default'"""
        try:
            self._multiInSocket.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, self._makeMreq(addr))
        except socket.error:  # if 1 interface has more than 1 address, exception is raised for the second
            pass

        sock = self._createMulticastOutSocket(addr, self._observer.ttl)
        self._multiOutUniInSockets[addr] = sock
        self._poll.register(sock, select.POLLIN)

    def removeSourceAddr(self, addr):
        try:
            self._multiInSocket.setsockopt(socket.IPPROTO_IP, socket.IP_DROP_MEMBERSHIP, self._makeMreq(addr))
        except socket.error:  # see comments for setsockopt(.., socket.IP_ADD_MEMBERSHIP..
            pass

        sock = self._multiOutUniInSockets[addr]
        self._poll.unregister(sock)
        sock.close()
        del self._multiOutUniInSockets[addr]

    def addUnicastMessage(self, env, addr, port, initialDelay=0):
        msg = UDPMessage(env, addr, port, UDPMessage.UNICAST, initialDelay)

        self._queue.append(msg)
        self._knownMessageIds.add(env.getMessageId())

    def addMulticastMessage(self, env, addr, port, initialDelay=0):
        msg = UDPMessage(env, addr, port, UDPMessage.MULTICAST, initialDelay)

        self._queue.append(msg)
        self._knownMessageIds.add(env.getMessageId())

    def run(self):
        while not self._quitEvent.is_set() or self._queue:
            self._sendPendingMessages()
            self._recvMessages()

    def _recvMessages(self):
        for fd, event in self._poll.poll(0):
            sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_DGRAM)
            try:
                data, addr = sock.recvfrom(BUFFER_SIZE)
            except socket.error as e:
                time.sleep(0.01)
                continue

            env = parseSOAPMessage(data, addr[0])

            if env is None: # fault or failed to parse
                continue

            _own_addrs = self._observer._addrsMonitorThread._addrs
            if addr[0] not in _own_addrs:
                if env.getAction() == ACTION_PROBE_MATCH:
                    prms = "\n ".join((str(prm) for prm in env.getProbeResolveMatches()))
                    msg = "probe response from %s:\n --- begin ---\n%s\n--- end ---\n"
                    logger.debug(msg, addr[0], prms)

                if self._capture:
                    self._capture.write("%i RECV %s:%s\n" % (self._seqnum, addr[0], addr[1]))
                    self._capture.write(data.decode("utf-8") + "\n")
                    self._seqnum += 1

            mid = env.getMessageId()
            if mid in self._knownMessageIds:
                continue
            else:
                self._knownMessageIds.add(mid)

            iid = env.getInstanceId()
            mid = env.getMessageId()
            if len(iid) > 0 and int(iid) > 0:
                mnum = env.getMessageNumber()
                key = addr[0] + ":" + str(addr[1]) + ":" + str(iid)
                if mid is not None and len(mid) > 0:
                    key = key + ":" + mid
                if key not in self._iidMap:
                    self._iidMap[key] = iid
                else:
                    tmnum = self._iidMap[key]
                    if mnum > tmnum:
                        self._iidMap[key] = mnum
                    else:
                        continue

            self._observer.envReceived(env, addr)

    def _sendMsg(self, msg):
        data = createSOAPMessage(msg.getEnv()).encode("UTF-8")

        if msg.msgType() == UDPMessage.UNICAST:
            self._uniOutSocket.sendto(data, (msg.getAddr(), msg.getPort()))
            if self._capture:
                self._capture.write("%i SEND %s:%s\n" % (self._seqnum, msg.getAddr(), msg.getPort()))
                self._capture.write(data.decode("utf-8") + "\n")
                self._seqnum += 1
        else:
            for sock in list(self._multiOutUniInSockets.values()):
                sock.sendto(data, (msg.getAddr(), msg.getPort()))
                if self._capture:
                    self._capture.write("%i SEND %s:%s\n" % (self._seqnum, msg.getAddr(), msg.getPort()))
                    self._capture.write(data.decode("utf-8") + "\n")
                    self._seqnum += 1

    def _sendPendingMessages(self):
        """Method sleeps, if nothing to do"""
        if len(self._queue) == 0:
            time.sleep(0.1)
            return
        msg = self._queue.pop(0)
        if msg.canSend():
            self._sendMsg(msg)
            msg.refresh()
            if not (msg.isFinished()):
                self._queue.append(msg)
        else:
            self._queue.append(msg)
            time.sleep(0.01)

    def start(self):
        super(NetworkingThread, self).start()

        self._uniOutSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

        self._multiInSocket = self._createMulticastInSocket()
        self._poll.register(self._multiInSocket)

        self._multiOutUniInSockets = {}  # FIXME synchronisation

    def join(self):
        super(NetworkingThread, self).join()
        self._uniOutSocket.close()

        self._poll.unregister(self._multiInSocket)
        self._multiInSocket.close()

class WSDiscovery:

    def __init__(self, uuid_=None, capture=None, ttl=1):

        self._networkingThread = None
        self._serverStarted = False
        self._remoteServices = {}
        self._localServices = {}

        self._dpActive = False
        self._dpAddr = None
        self._dpEPR = None

        self._remoteServiceHelloCallback = None
        self._remoteServiceHelloCallbackTypesFilter = None
        self._remoteServiceHelloCallbackScopesFilter = None
        self._remoteServiceByeCallback = None
        self._capture = capture

        if uuid_ is not None:
            self.uuid = uuid_
        else:
            self.uuid = uuid.uuid4().urn

        self.ttl = ttl

    def setRemoteServiceHelloCallback(self, cb, types=None, scopes=None):
        """Set callback, which will be called when new service appeared online
        and sent Hi message

        typesFilter and scopesFilter might be list of types and scopes.
        If filter is set, callback is called only for Hello messages,
        which match filter

        Set None to disable callback
        """
        self._remoteServiceHelloCallback = cb
        self._remoteServiceHelloCallbackTypesFilter = types
        self._remoteServiceHelloCallbackScopesFilter = scopes

    def setRemoteServiceByeCallback(self, cb):
        """Set callback, which will be called when new service appeared online
        and sent Hi message
        Service is passed as a parameter to the callback
        Set None to disable callback
        """
        self._remoteServiceByeCallback = cb

    def setRemoveServiceDisappearedCallback(self, cb):
        """Set callback, which will be called when new service disappears
        Service uuid is passed as a parameter to the callback
        Set None to disable callback
        """
        self._remoteServiceDisppearedCallback = cb

    def _addRemoteService(self, service):
        self._remoteServices[service.getEPR()] = service

    def _removeRemoteService(self, epr):
        if epr in self._remoteServices:
            del self._remoteServices[epr]

    def handleEnv(self, env, addr):
        if (env.getAction() == ACTION_PROBE_MATCH):
            for match in env.getProbeResolveMatches():
                self._addRemoteService(Service(match.getTypes(), match.getScopes(), match.getXAddrs(), match.getEPR(), 0))
                if match.getXAddrs() is None or len(match.getXAddrs()) == 0:
                    self._sendResolve(match.getEPR())

        elif env.getAction() == ACTION_RESOLVE_MATCH:
            for match in env.getProbeResolveMatches():
                self._addRemoteService(Service(match.getTypes(), match.getScopes(), match.getXAddrs(), match.getEPR(), 0))

        elif env.getAction() == ACTION_PROBE:
            services = self._filterServices(list(self._localServices.values()), env.getTypes(), env.getScopes())
            self._sendProbeMatch(services, env.getMessageId(), addr)

        elif env.getAction() == ACTION_RESOLVE:
            if env.getEPR() in self._localServices:
                service = self._localServices[env.getEPR()]
                self._sendResolveMatch(service, env.getMessageId(), addr)

        elif env.getAction() == ACTION_HELLO:
            #check if it is from a discovery proxy
            rt = env.getRelationshipType()
            if rt is not None and rt.getLocalname() == "Suppression" and rt.getNamespace() == NS_D:
                xAddr = env.getXAddrs()[0]
                #only support 'soap.udp'
                if xAddr.startswith("soap.udp:"):
                    self._dpActive = True
                    self._dpAddr = extractSoapUdpAddressFromURI(URI(xAddr))
                    self._dpEPR = env.getEPR()

            service = Service(env.getTypes(), env.getScopes(), env.getXAddrs(), env.getEPR(), 0)
            self._addRemoteService(service)
            if self._remoteServiceHelloCallback is not None:
                if self._matchesFilter(service,
                                        self._remoteServiceHelloCallbackTypesFilter,
                                        self._remoteServiceHelloCallbackScopesFilter):
                    self._remoteServiceHelloCallback(service)

        elif env.getAction() == ACTION_BYE:
            #if the bye is from discovery proxy... revert back to multicasting
            if self._dpActive and self._dpEPR == env.getEPR():
                self._dpActive = False
                self._dpAddr = None
                self._dpEPR = None

            self._removeRemoteService(env.getEPR())
            if self._remoteServiceByeCallback is not None:
                self._remoteServiceByeCallback(env.getEPR())

    def envReceived(self, env, addr):
        self.handleEnv(env, addr)

    def _sendResolveMatch(self, service, relatesTo, addr):
        service.incrementMessageNumber()

        env = SoapEnvelope()
        env.setAction(ACTION_RESOLVE_MATCH)
        env.setTo(ADDRESS_UNKNOWN)
        env.setMessageId(uuid.uuid4().urn)
        env.setInstanceId(str(service.getInstanceId()))
        env.setMessageNumber(str(service.getMessageNumber()))
        env.setRelatesTo(relatesTo)

        env.getProbeResolveMatches().append(ProbeResolveMatch(service.getEPR(), \
                                                              service.getTypes(), service.getScopes(), \
                                                              service.getXAddrs(), str(service.getMetadataVersion())))
        self._networkingThread.addUnicastMessage(env, addr[0], addr[1])

    def _sendProbeMatch(self, services, relatesTo, addr):
        env = SoapEnvelope()
        env.setAction(ACTION_PROBE_MATCH)
        env.setTo(ADDRESS_UNKNOWN)
        env.setMessageId(uuid.uuid4().urn)
        random.seed((int)(time.time() * 1000000))
        env.setInstanceId(_generateInstanceId())
        env.setMessageNumber("1")
        env.setRelatesTo(relatesTo)

        for service in services:
            env.getProbeResolveMatches().append(ProbeResolveMatch(service.getEPR(), \
                                                                  service.getTypes(), service.getScopes(), \
                                                                  service.getXAddrs(), str(service.getMetadataVersion())))

        self._networkingThread.addUnicastMessage(env, addr[0], addr[1], random.randint(0, APP_MAX_DELAY))

    def _sendProbe(self, types=None, scopes=None):
        env = SoapEnvelope()
        env.setAction(ACTION_PROBE)
        env.setTo(ADDRESS_ALL)
        env.setMessageId(uuid.uuid4().urn)
        env.setTypes(types)
        env.setScopes(scopes)

        if self._dpActive:
            self._networkingThread.addUnicastMessage(env, self._dpAddr[0], self._dpAddr[1])
        else:
            self._networkingThread.addMulticastMessage(env, MULTICAST_IPV4_ADDRESS, MULTICAST_PORT)

    def _sendResolve(self, epr):
        env = SoapEnvelope()
        env.setAction(ACTION_RESOLVE)
        env.setTo(ADDRESS_ALL)
        env.setMessageId(uuid.uuid4().urn)
        env.setEPR(epr)

        if self._dpActive:
            self._networkingThread.addUnicastMessage(env, self._dpAddr[0], self._dpAddr[1])
        else:
            self._networkingThread.addMulticastMessage(env, MULTICAST_IPV4_ADDRESS, MULTICAST_PORT)

    def _sendHello(self, service):
        service.incrementMessageNumber()

        env = SoapEnvelope()
        env.setAction(ACTION_HELLO)
        env.setTo(ADDRESS_ALL)
        env.setMessageId(uuid.uuid4().urn)
        env.setInstanceId(str(service.getInstanceId()))
        env.setMessageNumber(str(service.getMessageNumber()))
        env.setTypes(service.getTypes())
        env.setScopes(service.getScopes())
        env.setXAddrs(service.getXAddrs())
        env.setEPR(service.getEPR())

        random.seed((int)(time.time() * 1000000))

        self._networkingThread.addMulticastMessage(env, MULTICAST_IPV4_ADDRESS, MULTICAST_PORT, random.randint(0, APP_MAX_DELAY))

    def _sendBye(self, service):
        env = SoapEnvelope()
        env.setAction(ACTION_BYE)
        env.setTo(ADDRESS_ALL)
        env.setMessageId(uuid.uuid4().urn)
        env.setInstanceId(str(service.getInstanceId()))
        env.setMessageNumber(str(service.getMessageNumber()))
        env.setEPR(service.getEPR())

        service.incrementMessageNumber()
        self._networkingThread.addMulticastMessage(env, MULTICAST_IPV4_ADDRESS, MULTICAST_PORT)

    def start(self):
        'start the discovery server - should be called before using other functions'
        self._startThreads()
        self._serverStarted = True

    def stop(self):
        'cleans up and stops the discovery server'

        self.clearRemoteServices()
        self.clearLocalServices()

        self._stopThreads()
        self._serverStarted = False

    def  _networkAddressAdded(self, addr):
        self._networkingThread.addSourceAddr(addr)
        for service in list(self._localServices.values()):
            self._sendHello(service)

    def _networkAddressRemoved(self, addr):
        self._networkingThread.removeSourceAddr(addr)

    def _startThreads(self):
        if self._networkingThread is not None:
            return

        self._networkingThread = NetworkingThread(self)
        self._networkingThread.start()

        self._addrsMonitorThread = AddressMonitorThread(self)
        self._addrsMonitorThread.start()

    def _stopThreads(self):
        if self._networkingThread is None:
            return

        self._networkingThread.schedule_stop()
        self._addrsMonitorThread.schedule_stop()

        self._networkingThread.join()
        self._addrsMonitorThread.join()

        self._networkingThread = None

    def _isTypeInList(self, ttype, types):
        for entry in types:
            if ttype.getFullname() == entry.getFullname():
                return True

        return False

    def _isScopeInList(self, scope, scopes):
        for entry in scopes:
            if matchScope(scope.getValue(), entry.getValue(), scope.getMatchBy()):
                return True

        return False

    def _matchesFilter(self, service, types, scopes):
        if types is not None:
            for ttype in types:
                if not self._isTypeInList(ttype, service.getTypes()):
                    return False
        if scopes is not None:
            for scope in scopes:
                if not self._isScopeInList(scope, service.getScopes()):
                    return False
        return True

    def _filterServices(self, services, types, scopes):
        return [service for service in services \
                    if self._matchesFilter(service, types, scopes)]

    def clearRemoteServices(self):
        'clears remotely discovered services'

        self._remoteServices.clear()

    def clearLocalServices(self):
        'send Bye messages for the services and remove them'

        for service in list(self._localServices.values()):
            self._sendBye(service)

        self._localServices.clear()

    def searchServices(self, types=None, scopes=None, timeout=3):
        'search for services given the TYPES and SCOPES in a given TIMEOUT'

        if not self._serverStarted:
            raise Exception("Server not started")

        self._sendProbe(types, scopes)

        time.sleep(timeout)

        return self._filterServices(list(self._remoteServices.values()), types, scopes)

    def publishService(self, types, scopes, xAddrs):
        """Publish a service with the given TYPES, SCOPES and XAddrs (service addresses)

        if xAddrs contains item, which includes {ip} pattern, one item per IP addres will be sent
        """

        if not self._serverStarted:
            raise Exception("Server not started")

        instanceId = _generateInstanceId()

        service = Service(types, scopes, xAddrs, self.uuid, instanceId)
        self._localServices[self.uuid] = service
        self._sendHello(service)

        time.sleep(0.001)


"""
Top-level SOAP message creation and parsing
"""

import io
from .namespaces import NS_A, NS_S
from .actions import *
from xml.dom import minidom

def createSOAPMessage(env):
    "construct a a raw SOAP XML string, given a prepared SoapEnvelope object"
    if env.getAction() == ACTION_PROBE:
        return createProbeMessage(env)
    if env.getAction() == ACTION_PROBE_MATCH:
        return createProbeMatchMessage(env)
    if env.getAction() == ACTION_RESOLVE:
        return createResolveMessage(env)
    if env.getAction() == ACTION_RESOLVE_MATCH:
        return createResolveMatchMessage(env)
    if env.getAction() == ACTION_HELLO:
        return createHelloMessage(env)
    if env.getAction() == ACTION_BYE:
        return createByeMessage(env)

def parseSOAPMessage(data, ipAddr):
    "parse raw XML data string, return a (minidom) xml document"

    try:
        dom = minidom.parseString(data)
    except Exception:
        #print('Failed to parse message from %s\n"%s": %s' % (ipAddr, data, ex), file=sys.stderr)
        return None

    if dom.getElementsByTagNameNS(NS_S, "Fault"):
        #print('Fault received from %s:' % (ipAddr, data), file=sys.stderr)
        return None

    soapAction = dom.getElementsByTagNameNS(NS_A, "Action")[0].firstChild.data.strip()
    if soapAction == ACTION_PROBE:
        return parseProbeMessage(dom)
    elif soapAction == ACTION_PROBE_MATCH:
        return parseProbeMatchMessage(dom)
    elif soapAction == ACTION_RESOLVE:
        return parseResolveMessage(dom)
    elif soapAction == ACTION_RESOLVE_MATCH:
        return parseResolveMatchMessage(dom)
    elif soapAction == ACTION_BYE:
        return parseByeMessage(dom)
    elif soapAction == ACTION_HELLO:
        return parseHelloMessage(dom)


import os
import sys
import logging
import click
import netifaces
from wsdiscovery.daemon import WSDiscovery
from wsdiscovery.scope import Scope

try:
    from urlparse import urlparse
except:
    from urllib.parse import urlparse

logging.basicConfig()
logger = logging.getLogger("ws-discovery")
logger.setLevel(logging.INFO)

# an external interface is needed
# darwin has en0, linux has eth0 by default
external_ifaces = ("en0", "eth0")

def get_valid_interface():
    ifaces = netifaces.interfaces()
    valid = list(set(external_ifaces).intersection(ifaces))
    return valid[0] if len(valid) == 1 else None

def get_ip_address(ifname):
    ipv4_addresses = netifaces.ifaddresses(ifname)[netifaces.AF_INET]
    return ipv4_addresses[0]["addr"]

def run(scope=None, capture=None):

    ifname = get_valid_interface()
    if not ifname:
        logger.error("no interface found; use one of: %s", ", ".join(external_ifaces))
        return

    ipaddr = get_ip_address(ifname)
    logger.debug("using %s interface with IPv4 address %s", ifname, ipaddr)

    wsd = WSDiscovery(capture=capture)
    wsd.start()

    if not scope:
        svcs = wsd.searchServices()
    else:
        # we support giving just one scope, for now
        svcs = wsd.searchServices(scopes=[Scope(scope)])

    print("\nDiscovered:\n")
    for service in svcs:
        url = urlparse(service.getXAddrs()[0])
        print(" address: %s" % url.netloc)
        print("  - %s\n" % "\n  - ".join([str(s) for s in service.getScopes()]))
    wsd.stop()

@click.command()
@click.option('--scope', '-s', help='Full scope URI, eg. onvif://www.onvif.org/Model/')
@click.option('--loglevel', '-l',  help='Log level; one of INFO, DEBUG, WARNING, ERROR')
@click.option('--capture', '-c', nargs=1, type=click.File('w'), help='Capture messages to a file')
def discover(scope, loglevel, capture):
    "Discover systems using WS-Discovery"

    if loglevel:
        level = getattr(logging, loglevel, None)
        if not level:
           print("Invalid log level '%s'" % loglevel)
           return
        logger.setLevel(level)

    run(scope=scope, capture=capture)

if __name__ == '__main__':
    discover()


from __future__ import unicode_literals
from __future__ import absolute_import

from taggit import VERSION as TAGGIT_VERSION
from taggit.managers import TaggableManager, _TaggableManager
from taggit.utils import require_instance_manager

from modelcluster.queryset import FakeQuerySet

if TAGGIT_VERSION < (0, 20, 0):
    raise Exception("modelcluster.contrib.taggit requires django-taggit version 0.20 or above")

class _ClusterTaggableManager(_TaggableManager):
    @require_instance_manager
    def get_tagged_item_manager(self):
        """Return the manager that handles the relation from this instance to the tagged_item class.
        If content_object on the tagged_item class is defined as a ParentalKey, this will be a
        DeferringRelatedManager which allows writing related objects without committing them
        to the database.
        """
        rel_name = self.through._meta.get_field('content_object').remote_field.get_accessor_name()
        return getattr(self.instance, rel_name)

    def get_queryset(self, extra_filters=None):
        if self.instance is None:
            # this manager is not associated with a specific model instance
            # (which probably means it's being invoked within a prefetch_related operation);
            # this means that we don't have to deal with uncommitted models/tags, and can just
            # use the standard taggit handler
            return super(_ClusterTaggableManager, self).get_queryset(extra_filters)
        else:
            # FIXME: we ought to have some way of querying the tagged item manager about whether
            # it has uncommitted changes, and return a real queryset (using the original taggit logic)
            # if not
            return FakeQuerySet(
                self.through.tag_model(),
                [tagged_item.tag for tagged_item in self.get_tagged_item_manager().all()]
            )

    @require_instance_manager
    def add(self, *tags):
        tag_objs = self._to_tag_model_instances(tags)

        # Now write these to the relation
        tagged_item_manager = self.get_tagged_item_manager()
        for tag in tag_objs:
            if not tagged_item_manager.filter(tag=tag):
                # make an instance of the self.through model and add it to the relation
                tagged_item = self.through(tag=tag)
                tagged_item_manager.add(tagged_item)

    @require_instance_manager
    def remove(self, *tags):
        tagged_item_manager = self.get_tagged_item_manager()
        tagged_items = [
            tagged_item for tagged_item in tagged_item_manager.all()
            if tagged_item.tag.name in tags
        ]
        tagged_item_manager.remove(*tagged_items)

    @require_instance_manager
    def set(self, *tags, **kwargs):
        # Ignore the 'clear' kwarg (which defaults to False) and override it to be always true;
        # this means that set is implemented as a clear then an add, which was the standard behaviour
        # prior to django-taggit 0.19 (https://github.com/alex/django-taggit/commit/6542a702b590a5cfb91ea0de218b7f71ffd07c33).
        #
        # In this way, we avoid a live database lookup that occurs in the clear=False branch.
        #
        # The clear=True behaviour is fine for our purposes; the distinction only exists in django-taggit
        # to ensure that the correct set of m2m_changed signals is fired, and our reimplementation here
        # doesn't fire them at all (which makes logical sense, because the whole point of this module is
        # that the add/remove/set/clear operations don't write to the database).
        return super(_ClusterTaggableManager, self).set(*tags, clear=True)

    @require_instance_manager
    def clear(self):
        self.get_tagged_item_manager().clear()

class ClusterTaggableManager(TaggableManager):
    _need_commit_after_assignment = True

    def __get__(self, instance, model):
        # override TaggableManager's requirement for instance to have a primary key
        # before we can access its tags
        manager = _ClusterTaggableManager(
            through=self.through, model=model, instance=instance, prefetch_cache_name=self.name
        )

        return manager

    def value_from_object(self, instance):
        # retrieve the queryset via the related manager on the content object,
        # to accommodate the possibility of this having uncommitted changes relative to
        # the live database
        rel_name = self.through._meta.get_field('content_object').remote_field.get_accessor_name()
        return getattr(instance, rel_name).all()


from __future__ import unicode_literals

import json
import datetime

from django.db import models
from django.db.models.fields.related import ForeignObjectRel
from django.db.models.fields import FieldDoesNotExist
from django.utils.encoding import is_protected_type
from django.core.serializers.json import DjangoJSONEncoder
from django.conf import settings
from django.utils import timezone

from modelcluster.fields import ParentalKey, ParentalManyToManyField

def get_field_value(field, model):
    if field.remote_field is None:
        value = field.pre_save(model, add=model.pk is None)

        # Make datetimes timezone aware
        # https://github.com/django/django/blob/master/django/db/models/fields/__init__.py#L1394-L1403
        if isinstance(value, datetime.datetime) and settings.USE_TZ:
            if timezone.is_naive(value):
                default_timezone = timezone.get_default_timezone()
                value = timezone.make_aware(value, default_timezone).astimezone(timezone.utc)
            # convert to UTC
            value = timezone.localtime(value, timezone.utc)

        if is_protected_type(value):
            return value
        else:
            return field.value_to_string(model)
    else:
        return getattr(model, field.get_attname())

def get_serializable_data_for_fields(model):
    """
    Return a serialised version of the model's fields which exist as local database
    columns (i.e. excluding m2m and incoming foreign key relations)
    """
    pk_field = model._meta.pk
    # If model is a child via multitable inheritance, use parent's pk
    while pk_field.remote_field and pk_field.remote_field.parent_link:
        pk_field = pk_field.remote_field.model._meta.pk

    obj = {'pk': get_field_value(pk_field, model)}

    for field in model._meta.fields:
        if field.serialize:
            obj[field.name] = get_field_value(field, model)

    return obj

def model_from_serializable_data(model, data, check_fks=True, strict_fks=False):
    pk_field = model._meta.pk
    # If model is a child via multitable inheritance, use parent's pk
    while pk_field.remote_field and pk_field.remote_field.parent_link:
        pk_field = pk_field.remote_field.model._meta.pk

    kwargs = {pk_field.attname: data['pk']}
    for field_name, field_value in data.items():
        try:
            field = model._meta.get_field(field_name)
        except FieldDoesNotExist:
            continue

        # Filter out reverse relations
        if isinstance(field, ForeignObjectRel):
            continue

        if field.remote_field and isinstance(field.remote_field, models.ManyToManyRel):
            related_objects = field.remote_field.model._default_manager.filter(pk__in=field_value)
            kwargs[field.attname] = list(related_objects)

        elif field.remote_field and isinstance(field.remote_field, models.ManyToOneRel):
            if field_value is None:
                kwargs[field.attname] = None
            else:
                clean_value = field.remote_field.model._meta.get_field(field.remote_field.field_name).to_python(field_value)
                kwargs[field.attname] = clean_value
                if check_fks:
                    try:
                        field.remote_field.model._default_manager.get(**{field.remote_field.field_name: clean_value})
                    except field.remote_field.model.DoesNotExist:
                        if field.remote_field.on_delete == models.DO_NOTHING:
                            pass
                        elif field.remote_field.on_delete == models.CASCADE:
                            if strict_fks:
                                return None
                            else:
                                kwargs[field.attname] = None

                        elif field.remote_field.on_delete == models.SET_NULL:
                            kwargs[field.attname] = None

                        else:
                            raise Exception("can't currently handle on_delete types other than CASCADE, SET_NULL and DO_NOTHING")
        else:
            value = field.to_python(field_value)

            # Make sure datetimes are converted to localtime
            if isinstance(field, models.DateTimeField) and settings.USE_TZ and value is not None:
                default_timezone = timezone.get_default_timezone()
                if timezone.is_aware(value):
                    value = timezone.localtime(value, default_timezone)
                else:
                    value = timezone.make_aware(value, default_timezone)

            kwargs[field.name] = value

    obj = model(**kwargs)

    if data['pk'] is not None:
        # Set state to indicate that this object has come from the database, so that
        # ModelForm validation doesn't try to enforce a uniqueness check on the primary key
        obj._state.adding = False

    return obj

def get_all_child_relations(model):
    """
    Return a list of RelatedObject records for child relations of the given model,
    including ones attached to ancestors of the model
    """
    return [
        field for field in model._meta.get_fields()
        if isinstance(field.remote_field, ParentalKey)
    ]

def get_all_child_m2m_relations(model):
    """
    Return a list of ParentalManyToManyFields on the given model,
    including ones attached to ancestors of the model
    """
    return [
        field for field in model._meta.get_fields()
        if isinstance(field, ParentalManyToManyField)
    ]

class ClusterableModel(models.Model):
    def __init__(self, *args, **kwargs):
        """
        Extend the standard model constructor to allow child object lists to be passed in
        via kwargs
        """
        child_relation_names = (
            [rel.get_accessor_name() for rel in get_all_child_relations(self)] +
            [field.name for field in get_all_child_m2m_relations(self)]
        )

        if any(name in kwargs for name in child_relation_names):
            # One or more child relation values is being passed in the constructor; need to
            # separate these from the standard field kwargs to be passed to 'super'
            kwargs_for_super = kwargs.copy()
            relation_assignments = {}
            for rel_name in child_relation_names:
                if rel_name in kwargs:
                    relation_assignments[rel_name] = kwargs_for_super.pop(rel_name)

            super(ClusterableModel, self).__init__(*args, **kwargs_for_super)
            for (field_name, related_instances) in relation_assignments.items():
                setattr(self, field_name, related_instances)
        else:
            super(ClusterableModel, self).__init__(*args, **kwargs)

    def save(self, **kwargs):
        """
        Save the model and commit all child relations.
        """
        child_relation_names = [rel.get_accessor_name() for rel in get_all_child_relations(self)]
        child_m2m_field_names = [field.name for field in get_all_child_m2m_relations(self)]

        update_fields = kwargs.pop('update_fields', None)
        if update_fields is None:
            real_update_fields = None
            relations_to_commit = child_relation_names
            m2m_fields_to_commit = child_m2m_field_names
        else:
            real_update_fields = []
            relations_to_commit = []
            m2m_fields_to_commit = []
            for field in update_fields:
                if field in child_relation_names:
                    relations_to_commit.append(field)
                elif field in child_m2m_field_names:
                    m2m_fields_to_commit.append(field)
                else:
                    real_update_fields.append(field)

        super(ClusterableModel, self).save(update_fields=real_update_fields, **kwargs)

        for relation in relations_to_commit:
            getattr(self, relation).commit()

        for field in m2m_fields_to_commit:
            getattr(self, field).commit()

    def serializable_data(self):
        obj = get_serializable_data_for_fields(self)

        for rel in get_all_child_relations(self):
            rel_name = rel.get_accessor_name()
            children = getattr(self, rel_name).all()

            if hasattr(rel.related_model, 'serializable_data'):
                obj[rel_name] = [child.serializable_data() for child in children]
            else:
                obj[rel_name] = [get_serializable_data_for_fields(child) for child in children]

        for field in get_all_child_m2m_relations(self):
            if field.serialize:
                children = getattr(self, field.name).all()
                obj[field.name] = [child.pk for child in children]

        return obj

    def to_json(self):
        return json.dumps(self.serializable_data(), cls=DjangoJSONEncoder)

    @classmethod
    def from_serializable_data(cls, data, check_fks=True, strict_fks=False):
        """
        Build an instance of this model from the JSON-like structure passed in,
        recursing into related objects as required.
        If check_fks is true, it will check whether referenced foreign keys still
        exist in the database.
        - dangling foreign keys on related objects are dealt with by either nullifying the key or
        dropping the related object, according to the 'on_delete' setting.
        - dangling foreign keys on the base object will be nullified, unless strict_fks is true,
        in which case any dangling foreign keys with on_delete=CASCADE will cause None to be
        returned for the entire object.
        """
        obj = model_from_serializable_data(cls, data, check_fks=check_fks, strict_fks=strict_fks)
        if obj is None:
            return None

        child_relations = get_all_child_relations(cls)

        for rel in child_relations:
            rel_name = rel.get_accessor_name()
            try:
                child_data_list = data[rel_name]
            except KeyError:
                continue

            related_model = rel.related_model
            if hasattr(related_model, 'from_serializable_data'):
                children = [
                    related_model.from_serializable_data(child_data, check_fks=check_fks, strict_fks=True)
                    for child_data in child_data_list
                ]
            else:
                children = [
                    model_from_serializable_data(related_model, child_data, check_fks=check_fks, strict_fks=True)
                    for child_data in child_data_list
                ]

            children = filter(lambda child: child is not None, children)

            setattr(obj, rel_name, children)

        return obj

    @classmethod
    def from_json(cls, json_data, check_fks=True, strict_fks=False):
        return cls.from_serializable_data(json.loads(json_data), check_fks=check_fks, strict_fks=strict_fks)

    class Meta:
        abstract = True


from __future__ import unicode_literals

from django.forms import ValidationError
from django.core.exceptions import NON_FIELD_ERRORS
from django.forms.models import (
    BaseModelFormSet, modelformset_factory,
    ModelForm, _get_foreign_key, ModelFormMetaclass, ModelFormOptions
)
from django.db.models.fields.related import ForeignObjectRel

from modelcluster.models import get_all_child_relations

class BaseTransientModelFormSet(BaseModelFormSet):
    """ A ModelFormSet that doesn't assume that all its initial data instances exist in the db """
    def _construct_form(self, i, **kwargs):
        # Need to override _construct_form to avoid calling to_python on an empty string PK value

        if self.is_bound and i < self.initial_form_count():
            pk_key = "%s-%s" % (self.add_prefix(i), self.model._meta.pk.name)
            pk = self.data[pk_key]
            if pk == '':
                kwargs['instance'] = self.model()
            else:
                pk_field = self.model._meta.pk
                to_python = self._get_to_python(pk_field)
                pk = to_python(pk)
                kwargs['instance'] = self._existing_object(pk)
        if i < self.initial_form_count() and 'instance' not in kwargs:
            kwargs['instance'] = self.get_queryset()[i]
        if i >= self.initial_form_count() and self.initial_extra:
            # Set initial values for extra forms
            try:
                kwargs['initial'] = self.initial_extra[i - self.initial_form_count()]
            except IndexError:
                pass

        # bypass BaseModelFormSet's own _construct_form
        return super(BaseModelFormSet, self)._construct_form(i, **kwargs)

    def save_existing_objects(self, commit=True):
        # Need to override _construct_form so that it doesn't skip over initial forms whose instance
        # has a blank PK (which is taken as an indication that the form was constructed with an
        # instance not present in our queryset)

        self.changed_objects = []
        self.deleted_objects = []
        if not self.initial_forms:
            return []

        saved_instances = []
        forms_to_delete = self.deleted_forms
        for form in self.initial_forms:
            obj = form.instance
            if form in forms_to_delete:
                if obj.pk is None:
                    # no action to be taken to delete an object which isn't in the database
                    continue
                self.deleted_objects.append(obj)
                self.delete_existing(obj, commit=commit)
            elif form.has_changed():
                self.changed_objects.append((obj, form.changed_data))
                saved_instances.append(self.save_existing(form, obj, commit=commit))
                if not commit:
                    self.saved_forms.append(form)
        return saved_instances

def transientmodelformset_factory(model, formset=BaseTransientModelFormSet, **kwargs):
    return modelformset_factory(model, formset=formset, **kwargs)

class BaseChildFormSet(BaseTransientModelFormSet):
    def __init__(self, data=None, files=None, instance=None, queryset=None, **kwargs):
        if instance is None:
            self.instance = self.fk.remote_field.model()
        else:
            self.instance = instance

        self.rel_name = ForeignObjectRel(self.fk, self.fk.remote_field.model, related_name=self.fk.remote_field.related_name).get_accessor_name()

        if queryset is None:
            queryset = getattr(self.instance, self.rel_name).all()

        super(BaseChildFormSet, self).__init__(data, files, queryset=queryset, **kwargs)

    def save(self, commit=True):
        # The base ModelFormSet's save(commit=False) will populate the lists
        # self.changed_objects, self.deleted_objects and self.new_objects;
        # use these to perform the appropriate updates on the relation's manager.
        saved_instances = super(BaseChildFormSet, self).save(commit=False)

        manager = getattr(self.instance, self.rel_name)

        # if model has a sort_order_field defined, assign order indexes to the attribute
        # named in it
        if self.can_order and hasattr(self.model, 'sort_order_field'):
            sort_order_field = getattr(self.model, 'sort_order_field')
            for i, form in enumerate(self.ordered_forms):
                setattr(form.instance, sort_order_field, i)

        # If the manager has existing instances with a blank ID, we have no way of knowing
        # whether these correspond to items in the submitted data. We'll assume that they do,
        # as that's the most common case (i.e. the formset contains the full set of child objects,
        # not just a selection of additions / updates) and so we delete all ID-less objects here
        # on the basis that they will be re-added by the formset saving mechanism.
        no_id_instances = [obj for obj in manager.all() if obj.pk is None]
        if no_id_instances:
            manager.remove(*no_id_instances)

        manager.add(*saved_instances)
        manager.remove(*self.deleted_objects)

        self.save_m2m()  # ensures any parental-m2m fields are saved.
        if commit:
            manager.commit()

        return saved_instances

    def clean(self, *args, **kwargs):
        self.validate_unique()
        return super(BaseChildFormSet, self).clean(*args, **kwargs)

    def validate_unique(self):
        '''This clean method will check for unique_together condition'''
        # Collect unique_checks and to run from all the forms.
        all_unique_checks = set()
        all_date_checks = set()
        forms_to_delete = self.deleted_forms
        valid_forms = [form for form in self.forms if form.is_valid() and form not in forms_to_delete]
        for form in valid_forms:
            unique_checks, date_checks = form.instance._get_unique_checks()
            all_unique_checks.update(unique_checks)
            all_date_checks.update(date_checks)

        errors = []
        # Do each of the unique checks (unique and unique_together)
        for uclass, unique_check in all_unique_checks:
            seen_data = set()
            for form in valid_forms:
                # Get the data for the set of fields that must be unique among the forms.
                row_data = (
                    field if field in self.unique_fields else form.cleaned_data[field]
                    for field in unique_check if field in form.cleaned_data
                )
                # Reduce Model instances to their primary key values
                row_data = tuple(d._get_pk_val() if hasattr(d, '_get_pk_val') else d
                                 for d in row_data)
                if row_data and None not in row_data:
                    # if we've already seen it then we have a uniqueness failure
                    if row_data in seen_data:
                        # poke error messages into the right places and mark
                        # the form as invalid
                        errors.append(self.get_unique_error_message(unique_check))
                        form._errors[NON_FIELD_ERRORS] = self.error_class([self.get_form_error()])
                        # remove the data from the cleaned_data dict since it was invalid
                        for field in unique_check:
                            if field in form.cleaned_data:
                                del form.cleaned_data[field]
                    # mark the data as seen
                    seen_data.add(row_data)

        if errors:
            raise ValidationError(errors)

def childformset_factory(
    parent_model, model, form=ModelForm,
    formset=BaseChildFormSet, fk_name=None, fields=None, exclude=None,
    extra=3, can_order=False, can_delete=True, max_num=None, validate_max=False,
    formfield_callback=None, widgets=None, min_num=None, validate_min=False
):

    fk = _get_foreign_key(parent_model, model, fk_name=fk_name)
    # enforce a max_num=1 when the foreign key to the parent model is unique.
    if fk.unique:
        max_num = 1
        validate_max = True

    if exclude is None:
        exclude = []
    exclude += [fk.name]

    kwargs = {
        'form': form,
        'formfield_callback': formfield_callback,
        'formset': formset,
        'extra': extra,
        'can_delete': can_delete,
        # if the model supplies a sort_order_field, enable ordering regardless of
        # the current setting of can_order
        'can_order': (can_order or hasattr(model, 'sort_order_field')),
        'fields': fields,
        'exclude': exclude,
        'max_num': max_num,
        'validate_max': validate_max,
        'widgets': widgets,
        'min_num': min_num,
        'validate_min': validate_min,
    }
    FormSet = transientmodelformset_factory(model, **kwargs)
    FormSet.fk = fk
    return FormSet

class ClusterFormOptions(ModelFormOptions):
    def __init__(self, options=None):
        super(ClusterFormOptions, self).__init__(options=options)
        self.formsets = getattr(options, 'formsets', None)
        self.exclude_formsets = getattr(options, 'exclude_formsets', None)

class ClusterFormMetaclass(ModelFormMetaclass):
    extra_form_count = 3

    @classmethod
    def child_form(cls):
        return ClusterForm

    def __new__(cls, name, bases, attrs):
        try:
            parents = [b for b in bases if issubclass(b, ClusterForm)]
        except NameError:
            # We are defining ClusterForm itself.
            parents = None

        # grab any formfield_callback that happens to be defined in attrs -
        # so that we can pass it on to child formsets - before ModelFormMetaclass deletes it.
        # BAD METACLASS NO BISCUIT.
        formfield_callback = attrs.get('formfield_callback')

        new_class = super(ClusterFormMetaclass, cls).__new__(cls, name, bases, attrs)
        if not parents:
            return new_class

        # ModelFormMetaclass will have set up new_class._meta as a ModelFormOptions instance;
        # replace that with ClusterFormOptions so that we can access _meta.formsets
        opts = new_class._meta = ClusterFormOptions(getattr(new_class, 'Meta', None))
        if opts.model:
            formsets = {}
            for rel in get_all_child_relations(opts.model):
                # to build a childformset class from this relation, we need to specify:
                # - the base model (opts.model)
                # - the child model (rel.field.model)
                # - the fk_name from the child model to the base (rel.field.name)

                rel_name = rel.get_accessor_name()

                # apply 'formsets' and 'exclude_formsets' rules from meta
                if opts.formsets is not None and rel_name not in opts.formsets:
                    continue
                if opts.exclude_formsets and rel_name in opts.exclude_formsets:
                    continue

                try:
                    widgets = opts.widgets.get(rel_name)
                except AttributeError:  # thrown if opts.widgets is None
                    widgets = None

                kwargs = {
                    'extra': cls.extra_form_count,
                    'formfield_callback': formfield_callback,
                    'fk_name': rel.field.name,
                    'widgets': widgets
                }

                # see if opts.formsets looks like a dict; if so, allow the value
                # to override kwargs
                try:
                    kwargs.update(opts.formsets.get(rel_name))
                except AttributeError:
                    pass

                formset = childformset_factory(opts.model, rel.field.model, form=cls.child_form(), **kwargs)
                formsets[rel_name] = formset

            new_class.formsets = formsets

        return new_class

class ClusterForm(ModelForm, metaclass=ClusterFormMetaclass):
    def __init__(self, data=None, files=None, instance=None, prefix=None, **kwargs):
        super(ClusterForm, self).__init__(data, files, instance=instance, prefix=prefix, **kwargs)

        self.formsets = {}
        for rel_name, formset_class in self.__class__.formsets.items():
            if prefix:
                formset_prefix = "%s-%s" % (prefix, rel_name)
            else:
                formset_prefix = rel_name
            self.formsets[rel_name] = formset_class(data, files, instance=instance, prefix=formset_prefix)

    def as_p(self):
        form_as_p = super(ClusterForm, self).as_p()
        return form_as_p + ''.join([formset.as_p() for formset in self.formsets.values()])

    def is_valid(self):
        form_is_valid = super(ClusterForm, self).is_valid()
        formsets_are_valid = all([formset.is_valid() for formset in self.formsets.values()])
        return form_is_valid and formsets_are_valid

    def is_multipart(self):
        return (
            super(ClusterForm, self).is_multipart()
            or any(formset.is_multipart() for formset in self.formsets.values())
        )

    @property
    def media(self):
        media = super(ClusterForm, self).media
        for formset in self.formsets.values():
            media = media + formset.media
        return media

    def save(self, commit=True):
        # do we have any fields that expect us to call save_m2m immediately?
        save_m2m_now = False
        exclude = self._meta.exclude
        fields = self._meta.fields

        for f in self.instance._meta.get_fields():
            if fields and f.name not in fields:
                continue
            if exclude and f.name in exclude:
                continue
            if getattr(f, '_need_commit_after_assignment', False):
                save_m2m_now = True
                break

        instance = super(ClusterForm, self).save(commit=(commit and not save_m2m_now))

        # The M2M-like fields designed for use with ClusterForm (currently
        # ParentalManyToManyField and ClusterTaggableManager) will manage their own in-memory
        # relations, and not immediately write to the database when we assign to them.
        # For these fields (identified by the _need_commit_after_assignment
        # flag), save_m2m() is a safe operation that does not affect the database and is thus
        # valid for commit=False. In the commit=True case, committing to the database happens
        # in the subsequent instance.save (so this needs to happen after save_m2m to ensure
        # we have the updated relation data in place).

        # For annoying legacy reasons we sometimes need to accommodate 'classic' M2M fields
        # (particularly taggit.TaggableManager) within ClusterForm. These fields
        # generally do require our instance to exist in the database at the point we call
        # save_m2m() - for this reason, we only proceed with the customisation described above
        # (i.e. postpone the instance.save() operation until after save_m2m) if there's a
        # _need_commit_after_assignment field on the form that demands it.

        if save_m2m_now:
            self.save_m2m()

            if commit:
                instance.save()

        for formset in self.formsets.values():
            formset.instance = instance
            formset.save(commit=commit)
        return instance

    def has_changed(self):
        """Return True if data differs from initial."""

        # Need to recurse over nested formsets so that the form is saved if there are changes
        # to child forms but not the parent
        if self.formsets:
            for formset in self.formsets.values():
                for form in formset.forms:
                    if form.has_changed():
                        return True
        return bool(self.changed_data)