<reponame>chrisinmtown/PyMISP
#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
<NAME>

Cytomic Automation
Put this script in crontab to run every /15 or /60
    */15 *    * * *   mispuser   /usr/bin/python3 /home/mispuser/PyMISP/examples/cytomic_orion.py


Fetches the configuration set in the Cytomic Orion enrichment module
- events : upload events tagged with the 'upload' tag, all the attributes supported by Cytomic Orion
- upload : upload attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion)
- delete : delete attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion)

'''

from pymisp import ExpandedPyMISP
from keys import misp_url, misp_key, misp_verifycert
import argparse
import os
import re
import sys
import requests
import json
import urllib3


def get_token(token_url, clientid, clientsecret, scope, grant_type, username, password):
    '''
    Get oAuth2 token
    Configuration settings are fetched first from the MISP module configu
    '''

    try:
        if scope and grant_type and username and password:
            data = {'scope': scope, 'grant_type': grant_type, 'username': username, 'password': password}

            if token_url and clientid and clientsecret:
                access_token_response = requests.post(token_url, data=data, verify=False, allow_redirects=False, auth=(clientid, clientsecret))
                tokens = json.loads(access_token_response.text)
                if 'access_token' in tokens:
                    access_token = tokens['access_token']
                    return access_token
                else:
                    sys.exit('No token received')
            else:
                sys.exit('No token_url, clientid or clientsecret supplied')
        else:
            sys.exit('No scope, grant_type, username or password supplied')
    except Exception:
        sys.exit('Unable to connect to token_url')


def get_config(url, key, misp_verifycert):
    '''
    Get the module config and the settings needed to access the API
    Also contains the settings to do the query
    '''
    try:
        misp_headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': key}
        req = requests.get(url + 'servers/serverSettings.json', verify=misp_verifycert, headers=misp_headers)
        if req.status_code == 200:
            req_json = req.json()
            if 'finalSettings' in req_json:
                finalSettings = req_json['finalSettings']

                clientid = clientsecret = scope = username = password = grant_type = api_url = token_url = ''
                module_enabled = False
                scope = 'orion.api'
                grant_type = 'password'
                limit_upload_events = 50
                limit_upload_attributes = 50
                ttlDays = "1"
                last_attributes = '5d'
                post_threat_level_id = 2
                for el in finalSettings:
                    # Is the module enabled?
                    if el['setting'] == 'Plugin.Enrichment_cytomic_orion_enabled':
                        module_enabled = el['value']
                        if module_enabled is False:
                            break
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_clientid':
                        clientid = el['value']
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_clientsecret':
                        clientsecret = el['value']
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_username':
                        username = el['value']
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_password':
                        password = el['value']
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_api_url':
                        api_url = el['value'].replace('\\/', '/')
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_token_url':
                        token_url = el['value'].replace('\\/', '/')
                    elif el['setting'] == 'MISP.baseurl':
                        misp_baseurl = el['value']
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_threat_level_id':
                        if el['value']:
                            try:
                                post_threat_level_id = int(el['value'])
                            except:
                                continue
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_ttlDays':
                        if el['value']:
                            try:
                                ttlDays = "{last_days}".format(last_days=int(el['value']))
                            except:
                                continue
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_timeframe':
                        if el['value']:
                            try:
                                last_attributes = "{last_days}d".format(last_days=int(el['value']))
                            except:
                                continue
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_tag':
                        upload_tag = el['value']
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_delete_tag':
                        delete_tag = el['value']
                    elif el['setting'] == 'Plugin.Enrichment_limit_upload_events':
                        if el['value']:
                            try:
                                limit_upload_events = "{limit_upload_events}".format(limit_upload_events=int(el['value']))
                            except:
                                continue
                    elif el['setting'] == 'Plugin.Enrichment_limit_upload_attributes':
                        if el['value']:
                            try:
                                limit_upload_attributes = "{limit_upload_attributes}".format(limit_upload_attributes=int(el['value']))
                            except:
                                continue
        else:
            sys.exit('Did not receive a 200 code from MISP')

        if module_enabled and api_url and token_url and clientid and clientsecret and username and password and grant_type:

            return {'cytomic_policy': 'Detect',
                    'upload_timeframe': last_attributes,
                    'upload_tag': upload_tag,
                    'delete_tag': delete_tag,
                    'upload_ttlDays': ttlDays,
                    'post_threat_level_id': post_threat_level_id,
                    'clientid': clientid,
                    'clientsecret': clientsecret,
                    'scope': scope,
                    'username': username,
                    'password': password,
                    'grant_type': grant_type,
                    'api_url': api_url,
                    'token_url': token_url,
                    'misp_baseurl': misp_baseurl,
                    'limit_upload_events': limit_upload_events,
                    'limit_upload_attributes': limit_upload_attributes}
        else:
            sys.exit('Did not receive all the necessary configuration information from MISP')

    except Exception as e:
        sys.exit('Unable to get module config from MISP')


class cytomicobject:
    misp = None
    lst_evtid = None
    lst_attuuid = None
    lst_attuuid_error = None
    endpoint_ioc = None
    api_call_headers = None
    post_data = None
    args = None
    tag = None
    limit_events = None
    limit_attributes = None
    atttype_misp = None
    atttype_cytomic = None
    attlabel_cytomic = None
    att_types = {
       "ip-dst": {"ip": "ipioc"},
       "ip-src": {"ip": "ipioc"},
       "url": {"url": "urlioc"},
       "md5": {"hash": "filehashioc"},
       "domain": {"domain": "domainioc"},
       "hostname": {"domain": "domainioc"},
       "domain|ip": {"domain": "domainioc"},
       "hostname|port": {"domain": "domainioc"}
    }
    debug = True
    error = False
    res = False
    res_msg = None


def collect_events_ids(cytomicobj, moduleconfig):
    # Get events that contain Cytomic tag.
    try:
        evt_result = cytomicobj.misp.search(controller='events', limit=cytomicobj.limit_events, tags=cytomicobj.tag, last=moduleconfig['upload_timeframe'], published=True, deleted=False, pythonify=True)
        cytomicobj.lst_evtid = ['x', 'y']
        for evt in evt_result:
            evt = cytomicobj.misp.get_event(event=evt['id'], pythonify=True)
            if len(evt.tags) > 0:
                for tg in evt.tags:
                    if tg.name == cytomicobj.tag:
                        if not cytomicobj.lst_evtid:
                            cytomicobj.lst_evtid = str(evt['id'])
                        else:
                            if not evt['id'] in cytomicobj.lst_evtid:
                                cytomicobj.lst_evtid.append(str(evt['id']))
                        break
        cytomicobj.lst_evtid.remove('x')
        cytomicobj.lst_evtid.remove('y')
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to collect events ids')


def find_eventid(cytomicobj, evtid):
    # Get events that contain Cytomic tag.
    try:
        cytomicobj.res = False
        for id in cytomicobj.lst_evtid:
            if id == evtid:
                cytomicobj.res = True
                break
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to collect events ids')


def print_result_events(cytomicobj):
    try:
        if cytomicobj.res_msg is not None:
            for key, msg in cytomicobj.res_msg.items():
                if msg is not None:
                    print(key, msg)
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to print result')


def set_postdata(cytomicobj, moduleconfig, attribute):
    # Set JSON to send to the API.
    try:

        if cytomicobj.args.upload or cytomicobj.args.events:
            event = attribute['Event']
            event_title = event['info']
            event_id = event['id']
            threat_level_id = int(event['threat_level_id'])
            if moduleconfig['post_threat_level_id'] <= threat_level_id:

                if cytomicobj.atttype_misp == 'domain|ip' or cytomicobj.atttype_misp == 'hostname|port':
                    post_value = attribute['value'].split('|')[0]
                else:
                    post_value = attribute['value']

                if cytomicobj.atttype_misp == 'url' and 'http' not in post_value:
                    pass
                else:
                    if cytomicobj.post_data is None:
                        cytomicobj.post_data = [{cytomicobj.attlabel_cytomic: post_value, 'AdditionalData': '{} {}'.format(cytomicobj.atttype_misp, attribute['comment']).strip(), 'Source': 'Uploaded from MISP', 'Policy': moduleconfig['cytomic_policy'], 'Description': '{} - {}'.format(event_id, event_title).strip()}]
                    else:
                        if post_value not in str(cytomicobj.post_data):
                            cytomicobj.post_data.append({cytomicobj.attlabel_cytomic: post_value, 'AdditionalData': '{} {}'.format(cytomicobj.atttype_misp, attribute['comment']).strip(), 'Source': 'Uploaded from MISP', 'Policy': moduleconfig['cytomic_policy'], 'Description': '{} - {}'.format(event_id, event_title).strip()})
            else:
                if cytomicobject.debug:
                    print('Event %s skipped because of lower threat level' % event_id)
        else:
            event = attribute['Event']
            threat_level_id = int(event['threat_level_id'])
            if moduleconfig['post_threat_level_id'] <= threat_level_id:
                if cytomicobj.atttype_misp == 'domain|ip' or cytomicobj.atttype_misp == 'hostname|port':
                    post_value = attribute['value'].split('|')[0]
                else:
                    post_value = attribute['value']

                if cytomicobj.atttype_misp == 'url' and 'http' not in post_value:
                    pass
                else:
                    if cytomicobj.post_data is None:
                        cytomicobj.post_data = [{cytomicobj.attlabel_cytomic: post_value}]
                    else:
                        cytomicobj.post_data.append({cytomicobj.attlabel_cytomic: post_value})
            else:
                if cytomicobject.debug:
                    print('Event %s skipped because of lower threat level' % event_id)
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to process post-data')


def send_postdata(cytomicobj, evtid=None):
    # Batch post to upload event attributes.
    try:
        if cytomicobj.post_data is not None:
            if cytomicobj.debug:
                print('POST: {} {}'.format(cytomicobj.endpoint_ioc, cytomicobj.post_data))
            result_post_endpoint_ioc = requests.post(cytomicobj.endpoint_ioc, headers=cytomicobj.api_call_headers, json=cytomicobj.post_data, verify=False)
            json_result_post_endpoint_ioc = json.loads(result_post_endpoint_ioc.text)
            print(result_post_endpoint_ioc)
            if 'true' not in (result_post_endpoint_ioc.text):
                cytomicobj.error = True
                if evtid is not None:
                    if cytomicobj.res_msg['Event: ' + str(evtid)] is None:
                        cytomicobj.res_msg['Event: ' + str(evtid)] = '(Send POST data: errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
                    else:
                        cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (Send POST data -else: errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
            if cytomicobj.debug:
                print('RESULT: {}'.format(json_result_post_endpoint_ioc))
        else:
            if evtid is None:
                cytomicobj.error = True
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to post attributes')


def process_attributes(cytomicobj, moduleconfig, evtid=None):
    # Get attributes to process.
    try:
        for misptype, cytomictypes in cytomicobject.att_types.items():
            cytomicobj.atttype_misp = misptype
            for cytomiclabel, cytomictype in cytomictypes.items():
                cytomicobj.attlabel_cytomic = cytomiclabel
                cytomicobj.atttype_cytomic = cytomictype
                cytomicobj.post_data = None
                icont = 0
                if cytomicobj.args.upload or cytomicobj.args.events:
                    cytomicobj.endpoint_ioc = moduleconfig['api_url'] + '/iocs/' + cytomicobj.atttype_cytomic + '?ttlDays=' + str(moduleconfig['upload_ttlDays'])
                else:
                    cytomicobj.endpoint_ioc = moduleconfig['api_url'] + '/iocs/eraser/' + cytomicobj.atttype_cytomic

                # Get attributes to upload/delete and prepare JSON
                # If evtid is set; we're called from --events
                if cytomicobject.debug:
                    print("\nSearching for attributes of type %s" % cytomicobj.atttype_misp)

                if evtid is None:
                    cytomicobj.error = False
                    attr_result = cytomicobj.misp.search(controller='attributes', last=moduleconfig['upload_timeframe'], limit=cytomicobj.limit_attributes, type_attribute=cytomicobj.atttype_misp, tag=cytomicobj.tag, published=True, deleted=False, includeProposals=False, include_context=True, to_ids=True)
                else:
                    if cytomicobj.error:
                        break
                    # We don't search with tags; we have an event for which we want to upload all events
                    attr_result = cytomicobj.misp.search(controller='attributes', eventid=evtid, last=moduleconfig['upload_timeframe'], limit=cytomicobj.limit_attributes, type_attribute=cytomicobj.atttype_misp, published=True, deleted=False, includeProposals=False, include_context=True, to_ids=True)

                cytomicobj.lst_attuuid = ['x', 'y']

                if len(attr_result['Attribute']) > 0:
                    for attribute in attr_result['Attribute']:
                        if evtid is not None:
                            if cytomicobj.error:
                                cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
                                break
                        if icont >= cytomicobj.limit_attributes:
                            if not cytomicobj.error and cytomicobj.post_data is not None:
                                # Send data to Cytomic
                                send_postdata(cytomicobj, evtid)
                            if not cytomicobj.error:
                                if 'Event: ' + str(evtid) in cytomicobj.res_msg:
                                    if cytomicobj.res_msg['Event: ' + str(evtid)] is None:
                                        cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.attlabel_cytomic + 's: ' + str(icont)
                                    else:
                                        cytomicobj.res_msg['Event: ' + str(evtid)] += ' | ' + cytomicobj.attlabel_cytomic + 's: ' + str(icont)
                                else:
                                    if cytomicobject.debug:
                                        print('Data sent (' + cytomicobj.attlabel_cytomic + '): ' + str(icont))

                                cytomicobj.post_data = None
                            if cytomicobj.error:
                                if evtid is not None:
                                    cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
                                break
                            icont = 0

                        if evtid is None:
                            event = attribute['Event']
                            event_id = event['id']
                            find_eventid(cytomicobj, str(event_id))
                            if not cytomicobj.res:
                                if not cytomicobj.lst_attuuid:
                                    cytomicobj.lst_attuuid = attribute['uuid']
                                else:
                                    if not attribute['uuid'] in cytomicobj.lst_attuuid:
                                        cytomicobj.lst_attuuid.append(attribute['uuid'])
                                icont += 1
                                # Prepare data to send
                                set_postdata(cytomicobj, moduleconfig, attribute)
                        else:
                            icont += 1
                            # Prepare data to send
                            set_postdata(cytomicobj, moduleconfig, attribute)

                    if not cytomicobj.error:
                        # Send data to Cytomic
                        send_postdata(cytomicobj, evtid)

                    if not cytomicobj.error and cytomicobj.post_data is not None and icont > 0:
                        # Data sent; process response
                        if cytomicobj.res_msg is not None and 'Event: ' + str(evtid) in cytomicobj.res_msg:
                            if cytomicobj.res_msg['Event: ' + str(evtid)] is None:
                                cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.attlabel_cytomic + 's: ' + str(icont)
                            else:
                                cytomicobj.res_msg['Event: ' + str(evtid)] += ' | ' + cytomicobj.attlabel_cytomic + 's: ' + str(icont)
                        else:
                            if cytomicobject.debug:
                                print('Data sent (' + cytomicobj.attlabel_cytomic + '): ' + str(icont))

                    if not cytomicobj.error:
                        cytomicobj.lst_attuuid.remove('x')
                        cytomicobj.lst_attuuid.remove('y')
                        # Untag attributes
                        untag_attributes(cytomicobj)
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to get attributes')


def untag_event(evtid):
    # Remove tag of the event being processed.
    try:
        cytomicobj.records = 0
        evt = cytomicobj.misp.get_event(event=evtid, pythonify=True)
        if len(evt.tags) > 0:
            for tg in evt.tags:
                if tg.name == cytomicobj.tag:
                    cytomicobj.misp.untag(evt['uuid'], cytomicobj.tag)
                    cytomicobj.records += 1
                    cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (event untagged)'
                    break
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to untag events')


def process_events(cytomicobj, moduleconfig):
    # Get events that contain Cytomic tag.
    try:
        collect_events_ids(cytomicobj, moduleconfig)
        total_attributes_sent = 0
        for evtid in cytomicobj.lst_evtid:
            cytomicobj.error = False
            if cytomicobj.res_msg is None:
                cytomicobj.res_msg = {'Event: ' + str(evtid): None}
            else:
                cytomicobj.res_msg['Event: ' + str(evtid)] = None
            if cytomicobject.debug:
                print('Event id: ' + str(evtid))

            # get attributes of each known type of the event / prepare data to send / send data to Cytomic
            process_attributes(cytomicobj, moduleconfig, evtid)
            if not cytomicobj.error:
                untag_event(evtid)
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to process events ids')


def untag_attributes(cytomicobj):
    # Remove tag of attributes sent.
    try:
        icont = 0
        if len(cytomicobj.lst_attuuid) > 0:
            for uuid in cytomicobj.lst_attuuid:
                attr = cytomicobj.misp.get_attribute(attribute=uuid, pythonify=True)
                if len(attr.tags) > 0:
                    for tg in attr.tags:
                        if tg.name == cytomicobj.tag:
                            cytomicobj.misp.untag(uuid, cytomicobj.tag)
                            icont += 1
                            break
            print('Attributes untagged (' + str(icont) + ')')
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to untag attributes')


def process_attributes_upload(cytomicobj, moduleconfig):
    # get attributes of each known type / prepare data to send / send data to Cytomic
    try:
        collect_events_ids(cytomicobj, moduleconfig)
        process_attributes(cytomicobj, moduleconfig)
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to upload attributes to Cytomic')


def process_attributes_delete(cytomicobj, moduleconfig):
    # get attributes of each known type / prepare data to send / send data to Cytomic
    try:
        collect_events_ids(cytomicobj, moduleconfig)
        process_attributes(cytomicobj, moduleconfig)
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to delete attributes in Cytomic')


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Upload or delete indicators to Cytomic API')
    group = parser.add_mutually_exclusive_group()
    group.add_argument('--events', action='store_true', help='Upload events indicators')
    group.add_argument('--upload', action='store_true', help='Upload indicators')
    group.add_argument('--delete', action='store_true', help='Delete indicators')
    args = parser.parse_args()
    if not args.upload and not args.delete and not args.events:
        sys.exit("No valid action for the API")

    if misp_verifycert is False:
        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

    module_config = get_config(misp_url, misp_key, misp_verifycert)
    cytomicobj = cytomicobject
    misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=cytomicobject.debug)

    cytomicobj.misp = misp
    cytomicobj.args = args

    access_token = get_token(module_config['token_url'], module_config['clientid'], module_config['clientsecret'], module_config['scope'], module_config['grant_type'], module_config['username'], module_config['password'])
    cytomicobj.api_call_headers = {'Authorization': 'Bearer ' + access_token}
    if cytomicobj.debug:
        print('Received access token')

    if cytomicobj.args.events:
        cytomicobj.tag = module_config['upload_tag']
        cytomicobj.limit_events = module_config['limit_upload_events']
        cytomicobj.limit_attributes = module_config['limit_upload_attributes']
        process_events(cytomicobj, module_config)
        print_result_events(cytomicobj)

    elif cytomicobj.args.upload:
        cytomicobj.tag = module_config['upload_tag']
        cytomicobj.limit_events = 0
        cytomicobj.limit_attributes = module_config['limit_upload_attributes']
        process_attributes_upload(cytomicobj, module_config)

    else:
        cytomicobj.tag = module_config['delete_tag']
        cytomicobj.limit_events = 0
        cytomicobj.limit_attributes = module_config['limit_upload_attributes']
        process_attributes_delete(cytomicobj, module_config)

#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
<NAME>

Cytomic Automation
Put this script in crontab to run every /15 or /60
    */15 *    * * *   mispuser   /usr/bin/python3 /home/mispuser/PyMISP/examples/cytomic_orion.py


Fetches the configuration set in the Cytomic Orion enrichment module
- events : upload events tagged with the 'upload' tag, all the attributes supported by Cytomic Orion
- upload : upload attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion)
- delete : delete attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion)

'''

from pymisp import ExpandedPyMISP
from keys import misp_url, misp_key, misp_verifycert
import argparse
import os
import re
import sys
import requests
import json
import urllib3


def get_token(token_url, clientid, clientsecret, scope, grant_type, username, password):
    '''
    Get oAuth2 token
    Configuration settings are fetched first from the MISP module configu
    '''

    try:
        if scope and grant_type and username and password:
            data = {'scope': scope, 'grant_type': grant_type, 'username': username, 'password': password}

            if token_url and clientid and clientsecret:
                access_token_response = requests.post(token_url, data=data, verify=False, allow_redirects=False, auth=(clientid, clientsecret))
                tokens = json.loads(access_token_response.text)
                if 'access_token' in tokens:
                    access_token = tokens['access_token']
                    return access_token
                else:
                    sys.exit('No token received')
            else:
                sys.exit('No token_url, clientid or clientsecret supplied')
        else:
            sys.exit('No scope, grant_type, username or password supplied')
    except Exception:
        sys.exit('Unable to connect to token_url')


def get_config(url, key, misp_verifycert):
    '''
    Get the module config and the settings needed to access the API
    Also contains the settings to do the query
    '''
    try:
        misp_headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': key}
        req = requests.get(url + 'servers/serverSettings.json', verify=misp_verifycert, headers=misp_headers)
        if req.status_code == 200:
            req_json = req.json()
            if 'finalSettings' in req_json:
                finalSettings = req_json['finalSettings']

                clientid = clientsecret = scope = username = password = grant_type = api_url = token_url = ''
                module_enabled = False
                scope = 'orion.api'
                grant_type = 'password'
                limit_upload_events = 50
                limit_upload_attributes = 50
                ttlDays = "1"
                last_attributes = '5d'
                post_threat_level_id = 2
                for el in finalSettings:
                    # Is the module enabled?
                    if el['setting'] == 'Plugin.Enrichment_cytomic_orion_enabled':
                        module_enabled = el['value']
                        if module_enabled is False:
                            break
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_clientid':
                        clientid = el['value']
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_clientsecret':
                        clientsecret = el['value']
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_username':
                        username = el['value']
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_password':
                        password = el['value']
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_api_url':
                        api_url = el['value'].replace('\\/', '/')
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_token_url':
                        token_url = el['value'].replace('\\/', '/')
                    elif el['setting'] == 'MISP.baseurl':
                        misp_baseurl = el['value']
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_threat_level_id':
                        if el['value']:
                            try:
                                post_threat_level_id = int(el['value'])
                            except:
                                continue
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_ttlDays':
                        if el['value']:
                            try:
                                ttlDays = "{last_days}".format(last_days=int(el['value']))
                            except:
                                continue
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_timeframe':
                        if el['value']:
                            try:
                                last_attributes = "{last_days}d".format(last_days=int(el['value']))
                            except:
                                continue
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_tag':
                        upload_tag = el['value']
                    elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_delete_tag':
                        delete_tag = el['value']
                    elif el['setting'] == 'Plugin.Enrichment_limit_upload_events':
                        if el['value']:
                            try:
                                limit_upload_events = "{limit_upload_events}".format(limit_upload_events=int(el['value']))
                            except:
                                continue
                    elif el['setting'] == 'Plugin.Enrichment_limit_upload_attributes':
                        if el['value']:
                            try:
                                limit_upload_attributes = "{limit_upload_attributes}".format(limit_upload_attributes=int(el['value']))
                            except:
                                continue
        else:
            sys.exit('Did not receive a 200 code from MISP')

        if module_enabled and api_url and token_url and clientid and clientsecret and username and password and grant_type:

            return {'cytomic_policy': 'Detect',
                    'upload_timeframe': last_attributes,
                    'upload_tag': upload_tag,
                    'delete_tag': delete_tag,
                    'upload_ttlDays': ttlDays,
                    'post_threat_level_id': post_threat_level_id,
                    'clientid': clientid,
                    'clientsecret': clientsecret,
                    'scope': scope,
                    'username': username,
                    'password': password,
                    'grant_type': grant_type,
                    'api_url': api_url,
                    'token_url': token_url,
                    'misp_baseurl': misp_baseurl,
                    'limit_upload_events': limit_upload_events,
                    'limit_upload_attributes': limit_upload_attributes}
        else:
            sys.exit('Did not receive all the necessary configuration information from MISP')

    except Exception as e:
        sys.exit('Unable to get module config from MISP')


class cytomicobject:
    misp = None
    lst_evtid = None
    lst_attuuid = None
    lst_attuuid_error = None
    endpoint_ioc = None
    api_call_headers = None
    post_data = None
    args = None
    tag = None
    limit_events = None
    limit_attributes = None
    atttype_misp = None
    atttype_cytomic = None
    attlabel_cytomic = None
    att_types = {
       "ip-dst": {"ip": "ipioc"},
       "ip-src": {"ip": "ipioc"},
       "url": {"url": "urlioc"},
       "md5": {"hash": "filehashioc"},
       "domain": {"domain": "domainioc"},
       "hostname": {"domain": "domainioc"},
       "domain|ip": {"domain": "domainioc"},
       "hostname|port": {"domain": "domainioc"}
    }
    debug = True
    error = False
    res = False
    res_msg = None


def collect_events_ids(cytomicobj, moduleconfig):
    # Get events that contain Cytomic tag.
    try:
        evt_result = cytomicobj.misp.search(controller='events', limit=cytomicobj.limit_events, tags=cytomicobj.tag, last=moduleconfig['upload_timeframe'], published=True, deleted=False, pythonify=True)
        cytomicobj.lst_evtid = ['x', 'y']
        for evt in evt_result:
            evt = cytomicobj.misp.get_event(event=evt['id'], pythonify=True)
            if len(evt.tags) > 0:
                for tg in evt.tags:
                    if tg.name == cytomicobj.tag:
                        if not cytomicobj.lst_evtid:
                            cytomicobj.lst_evtid = str(evt['id'])
                        else:
                            if not evt['id'] in cytomicobj.lst_evtid:
                                cytomicobj.lst_evtid.append(str(evt['id']))
                        break
        cytomicobj.lst_evtid.remove('x')
        cytomicobj.lst_evtid.remove('y')
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to collect events ids')


def find_eventid(cytomicobj, evtid):
    # Get events that contain Cytomic tag.
    try:
        cytomicobj.res = False
        for id in cytomicobj.lst_evtid:
            if id == evtid:
                cytomicobj.res = True
                break
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to collect events ids')


def print_result_events(cytomicobj):
    try:
        if cytomicobj.res_msg is not None:
            for key, msg in cytomicobj.res_msg.items():
                if msg is not None:
                    print(key, msg)
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to print result')


def set_postdata(cytomicobj, moduleconfig, attribute):
    # Set JSON to send to the API.
    try:

        if cytomicobj.args.upload or cytomicobj.args.events:
            event = attribute['Event']
            event_title = event['info']
            event_id = event['id']
            threat_level_id = int(event['threat_level_id'])
            if moduleconfig['post_threat_level_id'] <= threat_level_id:

                if cytomicobj.atttype_misp == 'domain|ip' or cytomicobj.atttype_misp == 'hostname|port':
                    post_value = attribute['value'].split('|')[0]
                else:
                    post_value = attribute['value']

                if cytomicobj.atttype_misp == 'url' and 'http' not in post_value:
                    pass
                else:
                    if cytomicobj.post_data is None:
                        cytomicobj.post_data = [{cytomicobj.attlabel_cytomic: post_value, 'AdditionalData': '{} {}'.format(cytomicobj.atttype_misp, attribute['comment']).strip(), 'Source': 'Uploaded from MISP', 'Policy': moduleconfig['cytomic_policy'], 'Description': '{} - {}'.format(event_id, event_title).strip()}]
                    else:
                        if post_value not in str(cytomicobj.post_data):
                            cytomicobj.post_data.append({cytomicobj.attlabel_cytomic: post_value, 'AdditionalData': '{} {}'.format(cytomicobj.atttype_misp, attribute['comment']).strip(), 'Source': 'Uploaded from MISP', 'Policy': moduleconfig['cytomic_policy'], 'Description': '{} - {}'.format(event_id, event_title).strip()})
            else:
                if cytomicobject.debug:
                    print('Event %s skipped because of lower threat level' % event_id)
        else:
            event = attribute['Event']
            threat_level_id = int(event['threat_level_id'])
            if moduleconfig['post_threat_level_id'] <= threat_level_id:
                if cytomicobj.atttype_misp == 'domain|ip' or cytomicobj.atttype_misp == 'hostname|port':
                    post_value = attribute['value'].split('|')[0]
                else:
                    post_value = attribute['value']

                if cytomicobj.atttype_misp == 'url' and 'http' not in post_value:
                    pass
                else:
                    if cytomicobj.post_data is None:
                        cytomicobj.post_data = [{cytomicobj.attlabel_cytomic: post_value}]
                    else:
                        cytomicobj.post_data.append({cytomicobj.attlabel_cytomic: post_value})
            else:
                if cytomicobject.debug:
                    print('Event %s skipped because of lower threat level' % event_id)
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to process post-data')


def send_postdata(cytomicobj, evtid=None):
    # Batch post to upload event attributes.
    try:
        if cytomicobj.post_data is not None:
            if cytomicobj.debug:
                print('POST: {} {}'.format(cytomicobj.endpoint_ioc, cytomicobj.post_data))
            result_post_endpoint_ioc = requests.post(cytomicobj.endpoint_ioc, headers=cytomicobj.api_call_headers, json=cytomicobj.post_data, verify=False)
            json_result_post_endpoint_ioc = json.loads(result_post_endpoint_ioc.text)
            print(result_post_endpoint_ioc)
            if 'true' not in (result_post_endpoint_ioc.text):
                cytomicobj.error = True
                if evtid is not None:
                    if cytomicobj.res_msg['Event: ' + str(evtid)] is None:
                        cytomicobj.res_msg['Event: ' + str(evtid)] = '(Send POST data: errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
                    else:
                        cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (Send POST data -else: errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
            if cytomicobj.debug:
                print('RESULT: {}'.format(json_result_post_endpoint_ioc))
        else:
            if evtid is None:
                cytomicobj.error = True
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to post attributes')


def process_attributes(cytomicobj, moduleconfig, evtid=None):
    # Get attributes to process.
    try:
        for misptype, cytomictypes in cytomicobject.att_types.items():
            cytomicobj.atttype_misp = misptype
            for cytomiclabel, cytomictype in cytomictypes.items():
                cytomicobj.attlabel_cytomic = cytomiclabel
                cytomicobj.atttype_cytomic = cytomictype
                cytomicobj.post_data = None
                icont = 0
                if cytomicobj.args.upload or cytomicobj.args.events:
                    cytomicobj.endpoint_ioc = moduleconfig['api_url'] + '/iocs/' + cytomicobj.atttype_cytomic + '?ttlDays=' + str(moduleconfig['upload_ttlDays'])
                else:
                    cytomicobj.endpoint_ioc = moduleconfig['api_url'] + '/iocs/eraser/' + cytomicobj.atttype_cytomic

                # Get attributes to upload/delete and prepare JSON
                # If evtid is set; we're called from --events
                if cytomicobject.debug:
                    print("\nSearching for attributes of type %s" % cytomicobj.atttype_misp)

                if evtid is None:
                    cytomicobj.error = False
                    attr_result = cytomicobj.misp.search(controller='attributes', last=moduleconfig['upload_timeframe'], limit=cytomicobj.limit_attributes, type_attribute=cytomicobj.atttype_misp, tag=cytomicobj.tag, published=True, deleted=False, includeProposals=False, include_context=True, to_ids=True)
                else:
                    if cytomicobj.error:
                        break
                    # We don't search with tags; we have an event for which we want to upload all events
                    attr_result = cytomicobj.misp.search(controller='attributes', eventid=evtid, last=moduleconfig['upload_timeframe'], limit=cytomicobj.limit_attributes, type_attribute=cytomicobj.atttype_misp, published=True, deleted=False, includeProposals=False, include_context=True, to_ids=True)

                cytomicobj.lst_attuuid = ['x', 'y']

                if len(attr_result['Attribute']) > 0:
                    for attribute in attr_result['Attribute']:
                        if evtid is not None:
                            if cytomicobj.error:
                                cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
                                break
                        if icont >= cytomicobj.limit_attributes:
                            if not cytomicobj.error and cytomicobj.post_data is not None:
                                # Send data to Cytomic
                                send_postdata(cytomicobj, evtid)
                            if not cytomicobj.error:
                                if 'Event: ' + str(evtid) in cytomicobj.res_msg:
                                    if cytomicobj.res_msg['Event: ' + str(evtid)] is None:
                                        cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.attlabel_cytomic + 's: ' + str(icont)
                                    else:
                                        cytomicobj.res_msg['Event: ' + str(evtid)] += ' | ' + cytomicobj.attlabel_cytomic + 's: ' + str(icont)
                                else:
                                    if cytomicobject.debug:
                                        print('Data sent (' + cytomicobj.attlabel_cytomic + '): ' + str(icont))

                                cytomicobj.post_data = None
                            if cytomicobj.error:
                                if evtid is not None:
                                    cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
                                break
                            icont = 0

                        if evtid is None:
                            event = attribute['Event']
                            event_id = event['id']
                            find_eventid(cytomicobj, str(event_id))
                            if not cytomicobj.res:
                                if not cytomicobj.lst_attuuid:
                                    cytomicobj.lst_attuuid = attribute['uuid']
                                else:
                                    if not attribute['uuid'] in cytomicobj.lst_attuuid:
                                        cytomicobj.lst_attuuid.append(attribute['uuid'])
                                icont += 1
                                # Prepare data to send
                                set_postdata(cytomicobj, moduleconfig, attribute)
                        else:
                            icont += 1
                            # Prepare data to send
                            set_postdata(cytomicobj, moduleconfig, attribute)

                    if not cytomicobj.error:
                        # Send data to Cytomic
                        send_postdata(cytomicobj, evtid)

                    if not cytomicobj.error and cytomicobj.post_data is not None and icont > 0:
                        # Data sent; process response
                        if cytomicobj.res_msg is not None and 'Event: ' + str(evtid) in cytomicobj.res_msg:
                            if cytomicobj.res_msg['Event: ' + str(evtid)] is None:
                                cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.attlabel_cytomic + 's: ' + str(icont)
                            else:
                                cytomicobj.res_msg['Event: ' + str(evtid)] += ' | ' + cytomicobj.attlabel_cytomic + 's: ' + str(icont)
                        else:
                            if cytomicobject.debug:
                                print('Data sent (' + cytomicobj.attlabel_cytomic + '): ' + str(icont))

                    if not cytomicobj.error:
                        cytomicobj.lst_attuuid.remove('x')
                        cytomicobj.lst_attuuid.remove('y')
                        # Untag attributes
                        untag_attributes(cytomicobj)
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to get attributes')


def untag_event(evtid):
    # Remove tag of the event being processed.
    try:
        cytomicobj.records = 0
        evt = cytomicobj.misp.get_event(event=evtid, pythonify=True)
        if len(evt.tags) > 0:
            for tg in evt.tags:
                if tg.name == cytomicobj.tag:
                    cytomicobj.misp.untag(evt['uuid'], cytomicobj.tag)
                    cytomicobj.records += 1
                    cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (event untagged)'
                    break
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to untag events')


def process_events(cytomicobj, moduleconfig):
    # Get events that contain Cytomic tag.
    try:
        collect_events_ids(cytomicobj, moduleconfig)
        total_attributes_sent = 0
        for evtid in cytomicobj.lst_evtid:
            cytomicobj.error = False
            if cytomicobj.res_msg is None:
                cytomicobj.res_msg = {'Event: ' + str(evtid): None}
            else:
                cytomicobj.res_msg['Event: ' + str(evtid)] = None
            if cytomicobject.debug:
                print('Event id: ' + str(evtid))

            # get attributes of each known type of the event / prepare data to send / send data to Cytomic
            process_attributes(cytomicobj, moduleconfig, evtid)
            if not cytomicobj.error:
                untag_event(evtid)
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to process events ids')


def untag_attributes(cytomicobj):
    # Remove tag of attributes sent.
    try:
        icont = 0
        if len(cytomicobj.lst_attuuid) > 0:
            for uuid in cytomicobj.lst_attuuid:
                attr = cytomicobj.misp.get_attribute(attribute=uuid, pythonify=True)
                if len(attr.tags) > 0:
                    for tg in attr.tags:
                        if tg.name == cytomicobj.tag:
                            cytomicobj.misp.untag(uuid, cytomicobj.tag)
                            icont += 1
                            break
            print('Attributes untagged (' + str(icont) + ')')
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to untag attributes')


def process_attributes_upload(cytomicobj, moduleconfig):
    # get attributes of each known type / prepare data to send / send data to Cytomic
    try:
        collect_events_ids(cytomicobj, moduleconfig)
        process_attributes(cytomicobj, moduleconfig)
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to upload attributes to Cytomic')


def process_attributes_delete(cytomicobj, moduleconfig):
    # get attributes of each known type / prepare data to send / send data to Cytomic
    try:
        collect_events_ids(cytomicobj, moduleconfig)
        process_attributes(cytomicobj, moduleconfig)
    except Exception:
        cytomicobj.error = True
        if cytomicobj.debug:
            sys.exit('Unable to delete attributes in Cytomic')


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Upload or delete indicators to Cytomic API')
    group = parser.add_mutually_exclusive_group()
    group.add_argument('--events', action='store_true', help='Upload events indicators')
    group.add_argument('--upload', action='store_true', help='Upload indicators')
    group.add_argument('--delete', action='store_true', help='Delete indicators')
    args = parser.parse_args()
    if not args.upload and not args.delete and not args.events:
        sys.exit("No valid action for the API")

    if misp_verifycert is False:
        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

    module_config = get_config(misp_url, misp_key, misp_verifycert)
    cytomicobj = cytomicobject
    misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=cytomicobject.debug)

    cytomicobj.misp = misp
    cytomicobj.args = args

    access_token = get_token(module_config['token_url'], module_config['clientid'], module_config['clientsecret'], module_config['scope'], module_config['grant_type'], module_config['username'], module_config['password'])
    cytomicobj.api_call_headers = {'Authorization': 'Bearer ' + access_token}
    if cytomicobj.debug:
        print('Received access token')

    if cytomicobj.args.events:
        cytomicobj.tag = module_config['upload_tag']
        cytomicobj.limit_events = module_config['limit_upload_events']
        cytomicobj.limit_attributes = module_config['limit_upload_attributes']
        process_events(cytomicobj, module_config)
        print_result_events(cytomicobj)

    elif cytomicobj.args.upload:
        cytomicobj.tag = module_config['upload_tag']
        cytomicobj.limit_events = 0
        cytomicobj.limit_attributes = module_config['limit_upload_attributes']
        process_attributes_upload(cytomicobj, module_config)

    else:
        cytomicobj.tag = module_config['delete_tag']
        cytomicobj.limit_events = 0
        cytomicobj.limit_attributes = module_config['limit_upload_attributes']
        process_attributes_delete(cytomicobj, module_config)
#!/usr/bin/env python # -*- coding: utf-8 -*- <NAME>  Cytomic Automation Put this script in crontab to run every /15 or /60     */15 *    * * *   mispuser   /usr/bin/python3 /home/mispuser/PyMISP/examples/cytomic_orion.py   Fetches the configuration set in the Cytomic Orion enrichment module - events : upload events tagged with the 'upload' tag, all the attributes supported by Cytomic Orion - upload : upload attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion) - delete : delete attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion) Get oAuth2 token     Configuration settings are fetched first from the MISP module configu Get the module config and the settings needed to access the API     Also contains the settings to do the query # Is the module enabled? # Get events that contain Cytomic tag. # Get events that contain Cytomic tag. # Set JSON to send to the API. # Batch post to upload event attributes. # Get attributes to process. # Get attributes to upload/delete and prepare JSON # If evtid is set; we're called from --events # We don't search with tags; we have an event for which we want to upload all events # Send data to Cytomic # Prepare data to send # Prepare data to send # Send data to Cytomic # Data sent; process response # Untag attributes # Remove tag of the event being processed. # Get events that contain Cytomic tag. # get attributes of each known type of the event / prepare data to send / send data to Cytomic # Remove tag of attributes sent. # get attributes of each known type / prepare data to send / send data to Cytomic # get attributes of each known type / prepare data to send / send data to Cytomic