{"action":"created","comment":{"url":"https://api.github.com/repos/vertexproject/synapse/pulls/comments/197254515","pull_request_review_id":130964583,"id":197254515,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDE5NzI1NDUxNQ==","diff_hunk":"@@ -0,0 +1,714 @@\n+import time\n+import logging\n+import pathlib\n+import argparse\n+import tempfile\n+from binascii import unhexlify, hexlify\n+\n+import lmdb  # type: ignore\n+\n+\n+import synapse.cortex as s_cortex\n+import synapse.common as s_common\n+import synapse.lib.lmdb as s_lmdb\n+import synapse.lib.const as s_const\n+import synapse.lib.msgpack as s_msgpack\n+import synapse.models.inet as s_inet\n+\n+logger = logging.getLogger(__name__)\n+\n+# Topologically sorted comp and sepr types that are form types that have other comp types as elements.  The beginning\n+# of the list has more dependencies than the end.\n+_comp_and_sepr_forms = [\n+    'tel:mob:imsiphone', 'tel:mob:imid', 'syn:tagform', 'syn:fifo', 'syn:auth:userrole', 'seen', 'rsa:key', 'recref',\n+    'ps:image', 'ou:user', 'ou:suborg', 'ou:member', 'ou:meet:attendee', 'ou:hasalias', 'ou:conference:attendee',\n+    'mat:specimage', 'mat:itemimage', 'it:hostsoft', 'it:dev:regval', 'it:av:filehit', 'it:av:sig',\n+    'inet:wifi:ap', 'inet:whois:regmail', 'inet:whois:recns', 'inet:whois:contact', 'inet:whois:rec',\n+    'inet:web:post', 'inet:web:mesg', 'inet:web:memb', 'inet:web:group', 'inet:web:follows', 'inet:web:file',\n+    'inet:web:acct', 'inet:urlredir', 'inet:urlfile', 'inet:ssl:tcp4cert', 'inet:servfile', 'inet:http:resphead',\n+    'inet:http:reqparam', 'inet:http:reqhead', 'inet:http:param', 'inet:http:header', 'inet:dns:txt',\n+    'inet:dns:soa', 'inet:dns:rev6', 'inet:dns:rev', 'inet:dns:req', 'inet:dns:ns', 'inet:dns:mx',\n+    'inet:dns:cname', 'inet:dns:aaaa', 'inet:dns:a', 'inet:asnet4', 'geo:nloc', 'file:subfile']\n+\n+def _enc_iden(iden):\n+    return unhexlify(iden)\n+\n+class ConsistencyError(Exception):\n+    pass\n+\n+def _special_sort_key(t):\n+    '''\n+    For processing secondary properties in order, always processing tcp after udp, and v4 after v6\n+    '''\n+    k, v = t\n+    k = k.replace('tcp', chr(127) + 'tcp', 1)\n+    return (k.replace('v4', chr(127) + 'v4', 1), v)\n+\n+class Migrator:\n+    '''\n+    Sucks all rows out of a < .0.1.0 cortex, into a temporary LMDB database, migrates the schema, then dumps to new\n+    file suitable for ingesting into a >= .0.1.0 cortex.\n+    '''\n+    def __init__(self, core, outfh, tmpdir=None, stage1_fn=None, rejects_fh=None):\n+        '''\n+        Create a migrator.\n+\n+        Args:\n+            core (synapse.cores.common.Cortex): 0.0.x *local* cortex to export from\n+            outfh (IO['bin']): file handle opened for binary to push messagepacked data into\n+            tmpdir (Optional[str]):  location to write stage 1 LMDB database.  Please note that /tmp on Linux might\n+                *not* a good location since it is usually mounted tmpfs with not enough space.  This parameter is not\n+                used if stage1_fn parameter is specified.\n+            stage1_fn (Optional[str]):   Skips stage1 altogether and starts with existing stage 1 DB.\n+            rejects_fh (Optional[IO['bin']]):  file handle in which to place the nodes that couldn't be migrated.  If\n+                not provided, no rejects file will be used.\n+        '''\n+\n+        self.core = core\n+        self.dbenv = None\n+        self.skip_stage1 = bool(stage1_fn)\n+        assert tmpdir or stage1_fn\n+        self.next_val = 0\n+        self.rejects_fh = rejects_fh\n+\n+        if stage1_fn is None:\n+            with tempfile.NamedTemporaryFile(prefix='stage1_', suffix='.lmdb', delete=True, dir=str(tmpdir)) as fh:\n+                stage1_fn = fh.name\n+            logger.info('Creating stage 1 file at %s.  Delete when migration deemed successful.', stage1_fn)\n+\n+        map_size = s_lmdb.DEFAULT_MAP_SIZE\n+        self.dbenv = lmdb.Environment(stage1_fn,\n+                                      map_size=map_size,\n+                                      subdir=False,\n+                                      metasync=False,\n+                                      writemap=True,\n+                                      max_readers=1,\n+                                      max_dbs=4,\n+                                      sync=False,\n+                                      lock=False)\n+        self.iden_tbl = self.dbenv.open_db(key=b'idens', dupsort=True)  # iden -> row\n+        self.form_tbl = self.dbenv.open_db(key=b'forms', dupsort=True)  # formname -> iden\n+        self.comp_tbl = self.dbenv.open_db(key=b'comp')  # guid -> comp tuple\n+        self.valu_tbl = self.dbenv.open_db(key=b'vals', integerkey=True)\n+        self.outfh = outfh\n+        self._precalc_types()\n+        self.first_forms = ['file:bytes'] + [f for f in reversed(_comp_and_sepr_forms) if self.is_comp(f)]\n+\n+    def _precalc_types(self):\n+        '''\n+        Precalculate which types are sepr and which are comps, and which are subs of comps\n+        '''\n+        seprs = []\n+        comps = []\n+        subs = []\n+        filebytes = []\n+        xrefs = []\n+        forms = self.core.getTufoForms()\n+        for formname in forms:\n+            parents = self.core.getTypeOfs(formname)\n+            if 'sepr' in parents:\n+                seprs.append(formname)\n+            if 'comp' in parents:\n+                comps.append(formname)\n+            if 'file:bytes' in parents:\n+                filebytes.append(formname)\n+            if 'xref' in parents:\n+                xrefs.append(formname)\n+            subpropnameprops = self.core.getSubProps(formname)\n+            subpropnames = [x[0] for x in subpropnameprops]\n+            for subpropname, subpropprops in subpropnameprops:\n+                parents = self.core.getTypeOfs(subpropprops['ptype'])\n+                if 'sepr' in parents:\n+                    seprs.append(subpropname)\n+                if 'comp' in parents:\n+                    comps.append(subpropname)\n+                if ('comp' in parents) or ('sepr' in parents) or subpropprops['ptype'] in forms:\n+                    subs.extend(x for x in subpropnames if x.startswith(subpropname + ':') and ':seen:' not in x)\n+\n+        self.filebytes = set(filebytes)\n+        self.seprs = set(seprs)\n+        self.comps = set(comps)\n+        self.subs = set(subs)\n+        self.xrefs = set(xrefs)\n+\n+    def migrate(self):\n+        '''\n+        Convenience function to do both stages\n+        '''\n+        if not self.skip_stage1:\n+            self.do_stage1()\n+        self.do_stage2a()\n+        self.do_stage2b()\n+\n+    def _write_props(self, txn, rows):\n+        '''\n+        Emit propbags to the migration DB\n+        '''\n+        MAX_VAL_LEN = 511\n+\n+        idens = []\n+        bigvals = []\n+        for i, p, v, _ in rows:\n+            enci = _enc_iden(i)\n+            val = s_msgpack.en((p, v))\n+            if len(val) > MAX_VAL_LEN:\n+                next_val_enc = self.next_val.to_bytes(8, 'big')\n+                bigvals.append((next_val_enc, val))\n+                val = next_val_enc\n+                self.next_val += 1\n+            idens.append((enci, val))\n+\n+        with txn.cursor(self.iden_tbl) as icurs, txn.cursor(self.valu_tbl) as vcurs:\n+            consumed, added = icurs.putmulti(idens)\n+            if consumed != added or added != len(rows):\n+                raise ConsistencyError('Failure writing to Db.  consumed %d != added %d != len(rows) %d',\n+                                       consumed, added, len(rows))\n+            consumed, added = vcurs.putmulti(bigvals)\n+            if consumed != added:\n+                raise ConsistencyError('Failure writing to Db.  consumed %d != added %d',\n+                                       consumed, added)\n+\n+    def do_stage1(self):\n+        '''\n+        Do the first stage: suck all the data out of a cortex into an intermediate LMDB DB\n+        '''\n+        start_time = time.time()\n+        last_update = start_time\n+        logger.debug('Getting row count')\n+        totalrows = self.core.store.getSize()\n+        logger.debug('Total row count is %d', totalrows)\n+        rowcount = 0\n+        last_rowcount = 0\n+\n+        for rows in self.core.store.genStoreRows(gsize=10000):\n+            lenrows = len(rows)\n+            rowcount += lenrows\n+            now = time.time()\n+            if last_update + 60 < now:  # pragma: no cover\n+                percent_complete = rowcount / totalrows * 100\n+                timeperrow = (now - last_update) / (rowcount - last_rowcount)\n+                estimate = int((totalrows - rowcount) * timeperrow)\n+                last_update = now\n+                last_rowcount = rowcount\n+                logger.info('%02.2f%% complete.  Estimated completion in %ds', percent_complete, estimate)\n+\n+            with self.dbenv.begin(write=True) as txn:\n+                self._write_props(txn, rows)\n+\n+                # if prop is tufo:form, and form is a comp, add to form->iden table\n+                compi = ((v.encode('utf8'), _enc_iden(i)) for i, p, v, _ in rows\n+                         if p == 'tufo:form' and (self.is_comp(v) or v == 'file:bytes'))\n+                with txn.cursor(self.form_tbl) as curs:\n+                        consumed, added = curs.putmulti(compi)\n+                        if consumed != added:\n+                            raise ConsistencyError('Failure writing to Db.  consumed %d != added %d',\n+                                                   consumed, added)\n+\n+        logger.info('Stage 1 complete in %.1fs.', time.time() - start_time)\n+\n+    def write_node_to_file(self, node):\n+        self.outfh.write(s_msgpack.en(node))\n+\n+    def _get_props_from_cursor(self, txn, curs):\n+        props = {}\n+        for pv_enc in curs.iternext_dup():\n+            if len(pv_enc) == 8:\n+                pv_enc = txn.get(pv_enc, db=self.valu_tbl)\n+                if pv_enc is None:\n+                    raise ConsistencyError('Missing big val')\n+            p, v = s_msgpack.un(pv_enc)\n+            props[p] = v\n+        return props\n+\n+    def do_stage2a(self):\n+        '''\n+        Do the first part of the second stage:  take all the data out of the intermediate DB and emit to a flat file,\n+        just for the \"first forms\", all the seprs and comps and those that are migrated to comps.\n+        '''\n+        start_time = time.time()\n+\n+        # Do the comp forms\n+        for i, formname in enumerate(self.first_forms):\n+            if formname in self.forms_to_drop:\n+                continue\n+            with self.dbenv.begin(write=True, db=self.form_tbl) as txn:\n+                curs = txn.cursor(self.form_tbl)\n+                if not curs.set_key(formname.encode('utf8')):\n+                    continue\n+                logger.info('Stage 2a: (%2d/%2d) processing nodes from %s', i + 1, len(self.first_forms), formname)\n+                for enc_iden in curs.iternext_dup():\n+                    props = None\n+                    try:\n+                        with txn.cursor(self.iden_tbl) as curs:\n+                            if not curs.set_key(enc_iden):\n+                                raise ConsistencyError('missing iden', hexlify(s_msgpack.un(enc_iden)))\n+                            props = self._get_props_from_cursor(txn, curs)\n+                        node = self.convert_props(props)\n+                        if node is None:\n+                            continue\n+                        if formname == 'file:bytes':\n+                            if not txn.put(props['node:ndef'].encode('utf8'), s_msgpack.en(node[0]), db=self.comp_tbl):\n+                                raise ConsistencyError('put failure')\n+                            if not txn.put(props[formname].encode('utf8'), s_msgpack.en(node[0]), db=self.comp_tbl):\n+                                raise ConsistencyError('put failure')\n+                        else:\n+                            txn.put(props[formname].encode('utf8'), s_msgpack.en(node[0]), db=self.comp_tbl)\n+                        self.write_node_to_file(node)\n+                    except Exception:\n+                        logger.debug('Failed on processing node of form %s', formname, exc_info=True)\n+                        if props is not None and self.rejects_fh is not None:\n+                            self.rejects_fh.write(s_msgpack.en(props))\n+\n+        comp_node_time = time.time()\n+        logger.debug('Stage 2a complete in %.1fs', comp_node_time - start_time)\n+\n+    def do_stage2b(self):\n+        '''\n+        Do the rest (the non-comp) forms\n+        '''\n+        # Do all the non-comp forms\n+        start_time = time.time()\n+        iden_first = 0\n+        iden_last = 2 ** 128 - 1\n+        prev_update = time.time()\n+        prev_iden = iden_first\n+        with self.dbenv.begin(db=self.iden_tbl) as txn:\n+            curs = txn.cursor(self.iden_tbl)\n+            if not curs.set_range(iden_first.to_bytes(16, 'big')):\n+                raise Exception('no data!')\n+            while True:\n+                iden_int = int.from_bytes(curs.key(), 'big')\n+                if iden_int > iden_last:\n+                    break\n+\n+                now = time.time()\n+                if prev_update + 60 < now:  # pragma: no cover\n+                    percent_complete = (iden_int - iden_first) / (iden_last - iden_first) * 100.0\n+                    try:\n+                        timeperiden = (now - prev_update) / (iden_int - prev_iden)\n+                    except ZeroDivisionError:\n+                        timeperiden = 0.0\n+                    estimate = int((iden_last - iden_int) * timeperiden)\n+                    logger.info('Stage 2 %02.2f%% complete.  Estimated completion in %ds', percent_complete, estimate)\n+\n+                    prev_update = now\n+                    prev_iden = iden_int\n+\n+                props = self._get_props_from_cursor(txn, curs)\n+                rv = curs.next()\n+                formname = props.get('tufo:form')\n+                if not (formname is None or formname in self.first_forms):\n+                    try:\n+                        node = self.convert_props(props)\n+                        if node is not None:\n+                            self.write_node_to_file(node)\n+                    except Exception:\n+                        logger.debug('Failed on processing node with props: %s', props, exc_info=True)\n+                        if self.rejects_fh is not None:\n+                            self.rejects_fh.write(s_msgpack.en(props))\n+                if not rv:\n+                    break  # end of data\n+\n+        logger.info('Stage 2b complete in %.1fs.', time.time() - start_time)\n+\n+    def convert_ps_name(self, formname, propname, typename, val, props):\n+        names = val.split(',', 1)\n+        return propname, ' '.join(reversed(names))\n+\n+    def just_guid(self, formname, props):\n+        return props[formname]\n+\n+    def is_sepr(self, formname):\n+        return formname in self.seprs\n+\n+    def is_comp(self, formname):\n+        return formname in self.comps\n+\n+    def convert_sepr(self, formname, propname, propval, props):\n+        t = self.core.getPropType(propname)\n+        retn = []\n+        valdict = t.norm(propval)[1]\n+        for field in (x[0] for x in t._get_fields()):\n+            oldval = valdict[field]\n+            full_member = '%s:%s' % (propname, field)\n+            _, val = self.convert_subprop(formname, full_member, oldval, props)\n+            retn.append(val)\n+\n+        return tuple(retn)\n+\n+    def convert_file_bytes(self, formname, props):\n+        sha256 = props.get(formname + ':sha256')\n+        if sha256 is None:\n+            return 'guid:' + props[formname]\n+        return 'sha256:' + sha256\n+\n+    def convert_xref(self, propname, propval, props):\n+        t = self.core.getPropType(propname)\n+        sourceval = props[propname + ':' + t._sorc_name]\n+        destprop = props[propname + ':xref:prop']\n+        destval = props.get(propname + ':xref:intval', props.get(propname + ':xref:strval'))\n+        if destval is None:\n+            destval = props[propname + ':xref'].split('=', 1)[1]\n+        source_final = self.convert_foreign_key(propname, t._sorc_type, sourceval)\n+        dest_final = self.convert_foreign_key(propname, destprop, destval)\n+        return (source_final, (destprop, dest_final))\n+\n+    def convert_primary(self, props):\n+        formname = props['tufo:form']\n+        pkval = props[formname]\n+        if formname in self.primary_prop_special:\n+            return self.primary_prop_special[formname](self, formname, props)\n+        if self.is_comp(formname):\n+            return self.convert_comp_primary(props)\n+        if self.is_sepr(formname):\n+            return self.convert_sepr(formname, formname, pkval, props)\n+        if formname in self.xrefs:\n+            return self.convert_xref(formname, pkval, props)\n+        _, val = self.convert_subprop(formname, formname, pkval, props)\n+        return val\n+\n+    def convert_comp_secondary(self, formname, propval):\n+        '''\n+        Convert secondary prop that is a comp type\n+        '''\n+        with self.dbenv.begin(db=self.comp_tbl) as txn:\n+            comp_enc = txn.get(propval.encode('utf8'), db=self.comp_tbl)\n+            if comp_enc is None:\n+                raise ConsistencyError('guid accessed before determined')\n+            return s_msgpack.un(comp_enc)[1]\n+\n+    def convert_filebytes_secondary(self, formname, propval):\n+        '''\n+        Convert secondary prop that is a filebytes type\n+        '''\n+        with self.dbenv.begin(db=self.comp_tbl) as txn:\n+            comp_enc = txn.get(propval.encode('utf8'), db=self.comp_tbl)\n+            if comp_enc is None:\n+                raise ConsistencyError('ndef accessed before determined')\n+            return s_msgpack.un(comp_enc)[1]\n+\n+    def convert_foreign_key(self, formname, pivot_formname, pivot_fk):\n+        '''\n+        Convert secondary prop that is a pivot to another node\n+        '''\n+        if pivot_formname in self.filebytes:\n+            return self.convert_filebytes_secondary(pivot_formname, pivot_fk)\n+        if self.is_comp(pivot_formname):\n+            return self.convert_comp_secondary(pivot_formname, pivot_fk)\n+        if self.is_sepr(pivot_formname):\n+            return self.convert_sepr(formname, pivot_formname, pivot_fk, None)\n+\n+        return pivot_fk\n+\n+    def convert_comp_primary(self, props):\n+        formname = props['tufo:form']\n+        # logger.debug('convert_comp_primary_property: %s, %s', formname, compspec)\n+        t = self.core.getPropType(formname)\n+        members = [x[0] for x in t.fields]\n+        if formname == 'inet:dns:soa':\n+            members = ['ns', 'email']\n+        retn = []\n+        for member in members:\n+            full_member = '%s:%s' % (formname, member)\n+            _, val = self.convert_subprop(formname, full_member, props[full_member], props)\n+            retn.append(val)\n+        return tuple(retn)\n+\n+    def default_subprop_convert(self, formname, subpropname, subproptype, val):\n+        # logger.debug('default_subprop_convert : %s(type=%s)=%s', subpropname, subproptype, val)\n+        if subproptype != subpropname and subproptype in self.core.getTufoForms():\n+            return self.convert_foreign_key(formname, subproptype, val)\n+        return val\n+\n+    def ipv4_to_client(self, formname, propname, typename, val, props):\n+        return formname + ':client', 'tcp://%s' % s_inet.ipv4str(val)\n+\n+    def ipv6_to_client(self, formname, propname, typename, val, props):\n+        return formname + ':client', 'tcp://[%s]' % val\n+\n+    def convert_inet_xxp_primary(self, formname, props):\n+        oldpkval = props[formname]\n+        _, newpkval = self.xxp_to_server(formname, formname, formname, oldpkval, props)\n+        return newpkval\n+\n+    def xxp_to_server(self, formname, propname, typename, val, props):\n+        if typename[-1] == '4':\n+            addrport = self.core.getTypeRepr('inet:srv4', val)\n+        else:\n+            addrport = val\n+        return None, '%s://%s' % (typename[5:8], addrport)\n+\n+    def check_file_base_primary(self, formname, props):\n+        pk = props[formname]\n+        if '\\\\' not in pk:\n+            return props[formname]\n+        _, new_pk = self.check_file_base(formname, formname, formname, pk, props)\n+\n+        # Make a file:path node with the same tags\n+        ndef = ('file:path', self.core.getTypeRepr('file:path', pk.replace('\\\\', '/')))\n+        propscopy = props.copy()\n+        propscopy[formname] = new_pk\n+        template = self.convert_props(propscopy)\n+        new_node = (ndef, {'props': {'.created': template[1]['props']['.created']},\n+                           'tags': template[1].get('tags', {})})\n+\n+        self.write_node_to_file(new_node)\n+\n+        return new_pk\n+\n+    def check_file_base(self, formname, propname, typename, val, props):\n+        return None, val.rsplit('\\\\', 1)[-1]\n+\n+    type_special = {\n+        'inet:tcp4': xxp_to_server,\n+        'inet:tcp6': xxp_to_server,\n+        'inet:udp4': xxp_to_server,\n+        'inet:udp6': xxp_to_server,\n+        'ps:name': convert_ps_name,\n+        'file:base': check_file_base\n+    }\n+\n+    form_renames = {\n+        'file:txtref': 'file:ref',\n+        'file:imgof': 'file:ref',\n+        'ps:image': 'file:ref',\n+        'it:exec:bind:tcp': 'it:exec:bind',\n+        'it:exec:bind:udp': 'it:exec:bind',\n+        'inet:tcp4': 'inet:server',\n+        'inet:tcp6': 'inet:server',\n+        'inet:udp4': 'inet:server',\n+        'inet:udp4': 'inet:server',\n+        'inet:ssl:tcp4cert': 'inet:ssl:cert',\n+    }\n+\n+    prop_renames = {\n+        'inet:fqdn:zone': 'inet:fqdn:iszone',\n+        'inet:fqdn:sfx': 'inet:fqdn:issuffix',\n+        'inet:ipv4:cc': 'inet:ipv4:loc',\n+        'tel:phone:cc': 'tel:phone:loc',\n+        'ou:org:cc': 'ou:org:loc',\n+        'inet:web:acct:client': 'inet:web:acct:signup:client',\n+        'inet:dns:look:ipv4': 'inet:dns:look:client',\n+        'inet:dns:look:tcp4': 'inet:dns:look:server',\n+        'inet:dns:look:udp4': 'inet:dns:look:server',\n+        'inet:flow:dst:udp4': 'inet:flow:dst',\n+        'inet:flow:dst:udp6': 'inet:flow:dst',\n+        'inet:flow:dst:tcp4': 'inet:flow:dst',\n+        'inet:flow:dst:tcp6': 'inet:flow:dst',\n+        'inet:flow:src:udp4': 'inet:flow:src',\n+        'inet:flow:src:udp6': 'inet:flow:src',\n+        'inet:flow:src:tcp4': 'inet:flow:src',\n+        'inet:flow:src:tcp6': 'inet:flow:src',\n+        'file:bytes:mime:pe:timestamp': 'file:bytes:mime:pe:compiled',\n+        'it:exec:bind:tcp:ipv4': 'it:exec:bind:tcp:server',\n+        'it:exec:bind:tcp:ipv6': 'it:exec:bind:tcp:server',\n+        'it:exec:bind:udp:ipv4': 'it:exec:bind:tcp:server',\n+        'it:exec:bind:udp:ipv6': 'it:exec:bind:tcp:server',\n+    }\n+\n+    forms_to_drop = set((\n+        'syn:tagform',\n+    ))\n+\n+    secondary_props_to_drop = set((\n+        'syn:tag:depth',\n+        'ps:name:middle',\n+        'ps:name:sur',\n+        'ps:name:given',\n+        'inet:tcp4:ipv4',\n+        'inet:tcp4:port',\n+        'inet:udp4:ipv4',\n+        'inet:udp4:port',\n+        'inet:tcp6:ipv6',\n+        'inet:tcp6:port',\n+        'inet:udp6:ipv6',\n+        'inet:udp6:port',\n+        'inet:cidr4:ipv4',\n+        'file:path:ext',\n+        'ps:person:guidname',\n+        'ps:persona:guidname',\n+        'inet:web:action:info',\n+        'inet:web:action:ipv4',\n+        'inet:ssl:tcp4cert:file',\n+        'inet:ssl:tcp4cert:cert',\n+        'inet:ssl:tcp4cert:tcp4',\n+        'ps:image:person',\n+        'ps:image:file',\n+        'it:exec:bind:tcp:port',\n+        'it:exec:bind:udp:port'\n+    ))\n+\n+    def convert_subprop(self, formname, propname, val, props):\n+        typename = self.core.getPropTypeName(propname)\n+        converted = False\n+\n+        if propname in self.subprop_special:\n+            propname, val = self.subprop_special[propname](self, formname, propname, typename, val, props)\n+            converted = True\n+        elif typename in self.type_special:\n+            newpropname, val = self.type_special[typename](self, formname, propname, typename, val, props)\n+            if newpropname is not None:\n+                propname = newpropname\n+            converted = True\n+\n+        newpropname = self.prop_renames.get(propname, propname)\n+        newpropname = newpropname[len(formname) + 1:]\n+        if converted:\n+            return newpropname, val\n+\n+        return newpropname, self.default_subprop_convert(formname, propname, typename, val)\n+\n+    def convert_props(self, oldprops):\n+        formname = oldprops['tufo:form']\n+        if formname in self.forms_to_drop:\n+            return None\n+        props = {}\n+        tags = {}\n+        propsmeta = self.core.getSubProps(formname)\n+        pk = None\n+        for oldk, oldv in sorted(oldprops.items(), key=_special_sort_key):\n+            propmeta = next((x[1] for x in propsmeta if x[0] == oldk), {})\n+            if oldk in self.secondary_props_to_drop:\n+                continue\n+            if oldk[0] == '#':\n+                tags[oldk[1:]] = (None, None)\n+            elif oldk[0] == '>':\n+                start, end = tags.get(oldk[2:], (None, None))\n+                start = oldv\n+                tags[oldk[2:]] = (start, end)\n+            elif oldk[0] == '<':\n+                start, end = tags.get(oldk[2:], (None, None))\n+                end = oldv\n+                tags[oldk[2:]] = (start, end)\n+            elif oldk == formname:\n+                pk = self.convert_primary(oldprops)\n+            elif oldk in self.subs:\n+                # Skip if a sub to a comp or sepr that's another secondary property\n+                continue\n+            elif 'defval' in propmeta and oldv == propmeta['defval']:\n+                # logger.debug('Skipping field %s with default value', oldk)\n+                continue\n+            elif oldk.startswith(formname + ':'):\n+                if oldk[len(formname) + 1:] in ('seen:min', 'seen:max'):\n+                    new_pname, newv = self.handle_seen(oldk, oldv, props)\n+                elif formname in self.xrefs:\n+                    # skip all non-seed secondary props in xrefs\n+                    continue\n+                else:\n+                    new_pname, newv = self.convert_subprop(formname, oldk, oldv, oldprops)\n+                props[new_pname] = newv\n+            elif oldk == 'node:created':\n+                props['.created'] = oldv\n+            else:\n+                # logger.debug('Skipping propname %s', oldk)\n+                pass\n+        if pk is None:\n+            raise ConsistencyError('Missing pk', oldprops)\n+\n+        if formname == 'file:txtref':\n+            props['type'] = 'text'\n+        elif formname == 'file:imgof':\n+            props['type'] = 'image'\n+        elif formname == 'ps:image':\n+            props['type'] = 'image'\n+\n+        if formname in self.form_renames:\n+            formname = self.form_renames[formname]\n+\n+        assert '' not in props\n+\n+        retn = ((formname, pk), {'props': props})\n+        if tags:\n+            retn[1]['tags'] = tags\n+\n+        return retn\n+\n+    def convert_ps_image(self, formname, props):\n+        '''\n+        Transform ps:image into a file:ref\n+        '''\n+        person, file = self.convert_sepr(formname, formname, props[formname], props)\n+        return (file, ('ps:person', person))\n+\n+    primary_prop_special = {\n+        'inet:web:post': just_guid,\n+        'it:dev:regval': just_guid,\n+        'file:bytes': convert_file_bytes,\n+        'inet:udp4': convert_inet_xxp_primary,\n+        'inet:udp6': convert_inet_xxp_primary,\n+        'inet:tcp4': convert_inet_xxp_primary,\n+        'inet:tcp6': convert_inet_xxp_primary,\n+        'ps:image': convert_ps_image,\n+        'file:base': check_file_base_primary,\n+    }\n+\n+    def handle_seen(self, propname, propval, newprops):\n+        '''\n+        :seen:min, :seen:max -> .seen = (min, max)\n+        '''\n+        seenmin, seenmax = newprops.get('.seen', (None, None))\n+        if propname.endswith(':min'):\n+            seenmin = propval\n+        if propname.endswith(':max'):\n+            seenmax = propval\n+        if seenmax is None:","path":"synapse/tools/migrate_010.py","position":718,"original_position":654,"commit_id":"adc5ee8fe000761c8539f64bbaead2cb0475e523","original_commit_id":"5c410a10a5e6b0fdd6f4bfdd10ad3766e1cc036f","user":{"login":"jnwatson","id":3623079,"node_id":"MDQ6VXNlcjM2MjMwNzk=","avatar_url":"https://avatars1.githubusercontent.com/u/3623079?v=4","gravatar_id":"","url":"https://api.github.com/users/jnwatson","html_url":"https://github.com/jnwatson","followers_url":"https://api.github.com/users/jnwatson/followers","following_url":"https://api.github.com/users/jnwatson/following{/other_user}","gists_url":"https://api.github.com/users/jnwatson/gists{/gist_id}","starred_url":"https://api.github.com/users/jnwatson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jnwatson/subscriptions","organizations_url":"https://api.github.com/users/jnwatson/orgs","repos_url":"https://api.github.com/users/jnwatson/repos","events_url":"https://api.github.com/users/jnwatson/events{/privacy}","received_events_url":"https://api.github.com/users/jnwatson/received_events","type":"User","site_admin":false},"body":"The only way that is possible is if propval is None, which is impossible I think.\r\n\r\nEither seenmin or seenmax are set. This sets the other.","created_at":"2018-06-21T19:40:39Z","updated_at":"2018-06-21T19:40:39Z","html_url":"https://github.com/vertexproject/synapse/pull/826#discussion_r197254515","pull_request_url":"https://api.github.com/repos/vertexproject/synapse/pulls/826","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/vertexproject/synapse/pulls/comments/197254515"},"html":{"href":"https://github.com/vertexproject/synapse/pull/826#discussion_r197254515"},"pull_request":{"href":"https://api.github.com/repos/vertexproject/synapse/pulls/826"}},"in_reply_to_id":194415641},"pull_request":{"url":"https://api.github.com/repos/vertexproject/synapse/pulls/826","id":192486050,"node_id":"MDExOlB1bGxSZXF1ZXN0MTkyNDg2MDUw","html_url":"https://github.com/vertexproject/synapse/pull/826","diff_url":"https://github.com/vertexproject/synapse/pull/826.diff","patch_url":"https://github.com/vertexproject/synapse/pull/826.patch","issue_url":"https://api.github.com/repos/vertexproject/synapse/issues/826","number":826,"state":"open","locked":false,"title":"Add tool to migrate cortex to 0.1.0","user":{"login":"jnwatson","id":3623079,"node_id":"MDQ6VXNlcjM2MjMwNzk=","avatar_url":"https://avatars1.githubusercontent.com/u/3623079?v=4","gravatar_id":"","url":"https://api.github.com/users/jnwatson","html_url":"https://github.com/jnwatson","followers_url":"https://api.github.com/users/jnwatson/followers","following_url":"https://api.github.com/users/jnwatson/following{/other_user}","gists_url":"https://api.github.com/users/jnwatson/gists{/gist_id}","starred_url":"https://api.github.com/users/jnwatson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jnwatson/subscriptions","organizations_url":"https://api.github.com/users/jnwatson/orgs","repos_url":"https://api.github.com/users/jnwatson/repos","events_url":"https://api.github.com/users/jnwatson/events{/privacy}","received_events_url":"https://api.github.com/users/jnwatson/received_events","type":"User","site_admin":false},"body":"","created_at":"2018-06-04T18:06:58Z","updated_at":"2018-06-21T19:40:39Z","closed_at":null,"merged_at":null,"merge_commit_sha":"f8c17632ab5a2c21c2de167eeff0491fed59b047","assignee":{"login":"jnwatson","id":3623079,"node_id":"MDQ6VXNlcjM2MjMwNzk=","avatar_url":"https://avatars1.githubusercontent.com/u/3623079?v=4","gravatar_id":"","url":"https://api.github.com/users/jnwatson","html_url":"https://github.com/jnwatson","followers_url":"https://api.github.com/users/jnwatson/followers","following_url":"https://api.github.com/users/jnwatson/following{/other_user}","gists_url":"https://api.github.com/users/jnwatson/gists{/gist_id}","starred_url":"https://api.github.com/users/jnwatson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jnwatson/subscriptions","organizations_url":"https://api.github.com/users/jnwatson/orgs","repos_url":"https://api.github.com/users/jnwatson/repos","events_url":"https://api.github.com/users/jnwatson/events{/privacy}","received_events_url":"https://api.github.com/users/jnwatson/received_events","type":"User","site_admin":false},"assignees":[{"login":"jnwatson","id":3623079,"node_id":"MDQ6VXNlcjM2MjMwNzk=","avatar_url":"https://avatars1.githubusercontent.com/u/3623079?v=4","gravatar_id":"","url":"https://api.github.com/users/jnwatson","html_url":"https://github.com/jnwatson","followers_url":"https://api.github.com/users/jnwatson/followers","following_url":"https://api.github.com/users/jnwatson/following{/other_user}","gists_url":"https://api.github.com/users/jnwatson/gists{/gist_id}","starred_url":"https://api.github.com/users/jnwatson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jnwatson/subscriptions","organizations_url":"https://api.github.com/users/jnwatson/orgs","repos_url":"https://api.github.com/users/jnwatson/repos","events_url":"https://api.github.com/users/jnwatson/events{/privacy}","received_events_url":"https://api.github.com/users/jnwatson/received_events","type":"User","site_admin":false}],"requested_reviewers":[{"login":"invisig0th","id":2624245,"node_id":"MDQ6VXNlcjI2MjQyNDU=","avatar_url":"https://avatars2.githubusercontent.com/u/2624245?v=4","gravatar_id":"","url":"https://api.github.com/users/invisig0th","html_url":"https://github.com/invisig0th","followers_url":"https://api.github.com/users/invisig0th/followers","following_url":"https://api.github.com/users/invisig0th/following{/other_user}","gists_url":"https://api.github.com/users/invisig0th/gists{/gist_id}","starred_url":"https://api.github.com/users/invisig0th/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/invisig0th/subscriptions","organizations_url":"https://api.github.com/users/invisig0th/orgs","repos_url":"https://api.github.com/users/invisig0th/repos","events_url":"https://api.github.com/users/invisig0th/events{/privacy}","received_events_url":"https://api.github.com/users/invisig0th/received_events","type":"User","site_admin":false}],"requested_teams":[],"labels":[{"id":222989314,"node_id":"MDU6TGFiZWwyMjI5ODkzMTQ=","url":"https://api.github.com/repos/vertexproject/synapse/labels/enhancement","name":"enhancement","color":"0052cc","default":true},{"id":641431006,"node_id":"MDU6TGFiZWw2NDE0MzEwMDY=","url":"https://api.github.com/repos/vertexproject/synapse/labels/triage:%20high","name":"triage: high","color":"b60205","default":false}],"milestone":{"url":"https://api.github.com/repos/vertexproject/synapse/milestones/48","html_url":"https://github.com/vertexproject/synapse/milestone/48","labels_url":"https://api.github.com/repos/vertexproject/synapse/milestones/48/labels","id":3313065,"node_id":"MDk6TWlsZXN0b25lMzMxMzA2NQ==","number":48,"title":"v0.0.56","description":"Bugfix release as needed","creator":{"login":"vEpiphyte","id":26100278,"node_id":"MDQ6VXNlcjI2MTAwMjc4","avatar_url":"https://avatars2.githubusercontent.com/u/26100278?v=4","gravatar_id":"","url":"https://api.github.com/users/vEpiphyte","html_url":"https://github.com/vEpiphyte","followers_url":"https://api.github.com/users/vEpiphyte/followers","following_url":"https://api.github.com/users/vEpiphyte/following{/other_user}","gists_url":"https://api.github.com/users/vEpiphyte/gists{/gist_id}","starred_url":"https://api.github.com/users/vEpiphyte/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vEpiphyte/subscriptions","organizations_url":"https://api.github.com/users/vEpiphyte/orgs","repos_url":"https://api.github.com/users/vEpiphyte/repos","events_url":"https://api.github.com/users/vEpiphyte/events{/privacy}","received_events_url":"https://api.github.com/users/vEpiphyte/received_events","type":"User","site_admin":false},"open_issues":1,"closed_issues":0,"state":"open","created_at":"2018-05-02T00:09:39Z","updated_at":"2018-06-04T20:37:33Z","due_on":"2018-05-15T07:00:00Z","closed_at":null},"commits_url":"https://api.github.com/repos/vertexproject/synapse/pulls/826/commits","review_comments_url":"https://api.github.com/repos/vertexproject/synapse/pulls/826/comments","review_comment_url":"https://api.github.com/repos/vertexproject/synapse/pulls/comments{/number}","comments_url":"https://api.github.com/repos/vertexproject/synapse/issues/826/comments","statuses_url":"https://api.github.com/repos/vertexproject/synapse/statuses/adc5ee8fe000761c8539f64bbaead2cb0475e523","head":{"label":"vertexproject:migrate-tool","ref":"migrate-tool","sha":"adc5ee8fe000761c8539f64bbaead2cb0475e523","user":{"login":"vertexproject","id":21201579,"node_id":"MDEyOk9yZ2FuaXphdGlvbjIxMjAxNTc5","avatar_url":"https://avatars2.githubusercontent.com/u/21201579?v=4","gravatar_id":"","url":"https://api.github.com/users/vertexproject","html_url":"https://github.com/vertexproject","followers_url":"https://api.github.com/users/vertexproject/followers","following_url":"https://api.github.com/users/vertexproject/following{/other_user}","gists_url":"https://api.github.com/users/vertexproject/gists{/gist_id}","starred_url":"https://api.github.com/users/vertexproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vertexproject/subscriptions","organizations_url":"https://api.github.com/users/vertexproject/orgs","repos_url":"https://api.github.com/users/vertexproject/repos","events_url":"https://api.github.com/users/vertexproject/events{/privacy}","received_events_url":"https://api.github.com/users/vertexproject/received_events","type":"Organization","site_admin":false},"repo":{"id":37228107,"node_id":"MDEwOlJlcG9zaXRvcnkzNzIyODEwNw==","name":"synapse","full_name":"vertexproject/synapse","owner":{"login":"vertexproject","id":21201579,"node_id":"MDEyOk9yZ2FuaXphdGlvbjIxMjAxNTc5","avatar_url":"https://avatars2.githubusercontent.com/u/21201579?v=4","gravatar_id":"","url":"https://api.github.com/users/vertexproject","html_url":"https://github.com/vertexproject","followers_url":"https://api.github.com/users/vertexproject/followers","following_url":"https://api.github.com/users/vertexproject/following{/other_user}","gists_url":"https://api.github.com/users/vertexproject/gists{/gist_id}","starred_url":"https://api.github.com/users/vertexproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vertexproject/subscriptions","organizations_url":"https://api.github.com/users/vertexproject/orgs","repos_url":"https://api.github.com/users/vertexproject/repos","events_url":"https://api.github.com/users/vertexproject/events{/privacy}","received_events_url":"https://api.github.com/users/vertexproject/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/vertexproject/synapse","description":"Distributed Key-Value Hypergraph Analysis Framework","fork":false,"url":"https://api.github.com/repos/vertexproject/synapse","forks_url":"https://api.github.com/repos/vertexproject/synapse/forks","keys_url":"https://api.github.com/repos/vertexproject/synapse/keys{/key_id}","collaborators_url":"https://api.github.com/repos/vertexproject/synapse/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/vertexproject/synapse/teams","hooks_url":"https://api.github.com/repos/vertexproject/synapse/hooks","issue_events_url":"https://api.github.com/repos/vertexproject/synapse/issues/events{/number}","events_url":"https://api.github.com/repos/vertexproject/synapse/events","assignees_url":"https://api.github.com/repos/vertexproject/synapse/assignees{/user}","branches_url":"https://api.github.com/repos/vertexproject/synapse/branches{/branch}","tags_url":"https://api.github.com/repos/vertexproject/synapse/tags","blobs_url":"https://api.github.com/repos/vertexproject/synapse/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/vertexproject/synapse/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/vertexproject/synapse/git/refs{/sha}","trees_url":"https://api.github.com/repos/vertexproject/synapse/git/trees{/sha}","statuses_url":"https://api.github.com/repos/vertexproject/synapse/statuses/{sha}","languages_url":"https://api.github.com/repos/vertexproject/synapse/languages","stargazers_url":"https://api.github.com/repos/vertexproject/synapse/stargazers","contributors_url":"https://api.github.com/repos/vertexproject/synapse/contributors","subscribers_url":"https://api.github.com/repos/vertexproject/synapse/subscribers","subscription_url":"https://api.github.com/repos/vertexproject/synapse/subscription","commits_url":"https://api.github.com/repos/vertexproject/synapse/commits{/sha}","git_commits_url":"https://api.github.com/repos/vertexproject/synapse/git/commits{/sha}","comments_url":"https://api.github.com/repos/vertexproject/synapse/comments{/number}","issue_comment_url":"https://api.github.com/repos/vertexproject/synapse/issues/comments{/number}","contents_url":"https://api.github.com/repos/vertexproject/synapse/contents/{+path}","compare_url":"https://api.github.com/repos/vertexproject/synapse/compare/{base}...{head}","merges_url":"https://api.github.com/repos/vertexproject/synapse/merges","archive_url":"https://api.github.com/repos/vertexproject/synapse/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/vertexproject/synapse/downloads","issues_url":"https://api.github.com/repos/vertexproject/synapse/issues{/number}","pulls_url":"https://api.github.com/repos/vertexproject/synapse/pulls{/number}","milestones_url":"https://api.github.com/repos/vertexproject/synapse/milestones{/number}","notifications_url":"https://api.github.com/repos/vertexproject/synapse/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/vertexproject/synapse/labels{/name}","releases_url":"https://api.github.com/repos/vertexproject/synapse/releases{/id}","deployments_url":"https://api.github.com/repos/vertexproject/synapse/deployments","created_at":"2015-06-10T23:29:41Z","updated_at":"2018-06-06T13:33:11Z","pushed_at":"2018-06-21T19:37:56Z","git_url":"git://github.com/vertexproject/synapse.git","ssh_url":"git@github.com:vertexproject/synapse.git","clone_url":"https://github.com/vertexproject/synapse.git","svn_url":"https://github.com/vertexproject/synapse","homepage":"","size":8606,"stargazers_count":44,"watchers_count":44,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":28,"mirror_url":null,"archived":false,"open_issues_count":28,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":28,"open_issues":28,"watchers":44,"default_branch":"master"}},"base":{"label":"vertexproject:master","ref":"master","sha":"9627cffd3134bc89c30df653f1a09de6d9cb39bd","user":{"login":"vertexproject","id":21201579,"node_id":"MDEyOk9yZ2FuaXphdGlvbjIxMjAxNTc5","avatar_url":"https://avatars2.githubusercontent.com/u/21201579?v=4","gravatar_id":"","url":"https://api.github.com/users/vertexproject","html_url":"https://github.com/vertexproject","followers_url":"https://api.github.com/users/vertexproject/followers","following_url":"https://api.github.com/users/vertexproject/following{/other_user}","gists_url":"https://api.github.com/users/vertexproject/gists{/gist_id}","starred_url":"https://api.github.com/users/vertexproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vertexproject/subscriptions","organizations_url":"https://api.github.com/users/vertexproject/orgs","repos_url":"https://api.github.com/users/vertexproject/repos","events_url":"https://api.github.com/users/vertexproject/events{/privacy}","received_events_url":"https://api.github.com/users/vertexproject/received_events","type":"Organization","site_admin":false},"repo":{"id":37228107,"node_id":"MDEwOlJlcG9zaXRvcnkzNzIyODEwNw==","name":"synapse","full_name":"vertexproject/synapse","owner":{"login":"vertexproject","id":21201579,"node_id":"MDEyOk9yZ2FuaXphdGlvbjIxMjAxNTc5","avatar_url":"https://avatars2.githubusercontent.com/u/21201579?v=4","gravatar_id":"","url":"https://api.github.com/users/vertexproject","html_url":"https://github.com/vertexproject","followers_url":"https://api.github.com/users/vertexproject/followers","following_url":"https://api.github.com/users/vertexproject/following{/other_user}","gists_url":"https://api.github.com/users/vertexproject/gists{/gist_id}","starred_url":"https://api.github.com/users/vertexproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vertexproject/subscriptions","organizations_url":"https://api.github.com/users/vertexproject/orgs","repos_url":"https://api.github.com/users/vertexproject/repos","events_url":"https://api.github.com/users/vertexproject/events{/privacy}","received_events_url":"https://api.github.com/users/vertexproject/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/vertexproject/synapse","description":"Distributed Key-Value Hypergraph Analysis Framework","fork":false,"url":"https://api.github.com/repos/vertexproject/synapse","forks_url":"https://api.github.com/repos/vertexproject/synapse/forks","keys_url":"https://api.github.com/repos/vertexproject/synapse/keys{/key_id}","collaborators_url":"https://api.github.com/repos/vertexproject/synapse/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/vertexproject/synapse/teams","hooks_url":"https://api.github.com/repos/vertexproject/synapse/hooks","issue_events_url":"https://api.github.com/repos/vertexproject/synapse/issues/events{/number}","events_url":"https://api.github.com/repos/vertexproject/synapse/events","assignees_url":"https://api.github.com/repos/vertexproject/synapse/assignees{/user}","branches_url":"https://api.github.com/repos/vertexproject/synapse/branches{/branch}","tags_url":"https://api.github.com/repos/vertexproject/synapse/tags","blobs_url":"https://api.github.com/repos/vertexproject/synapse/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/vertexproject/synapse/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/vertexproject/synapse/git/refs{/sha}","trees_url":"https://api.github.com/repos/vertexproject/synapse/git/trees{/sha}","statuses_url":"https://api.github.com/repos/vertexproject/synapse/statuses/{sha}","languages_url":"https://api.github.com/repos/vertexproject/synapse/languages","stargazers_url":"https://api.github.com/repos/vertexproject/synapse/stargazers","contributors_url":"https://api.github.com/repos/vertexproject/synapse/contributors","subscribers_url":"https://api.github.com/repos/vertexproject/synapse/subscribers","subscription_url":"https://api.github.com/repos/vertexproject/synapse/subscription","commits_url":"https://api.github.com/repos/vertexproject/synapse/commits{/sha}","git_commits_url":"https://api.github.com/repos/vertexproject/synapse/git/commits{/sha}","comments_url":"https://api.github.com/repos/vertexproject/synapse/comments{/number}","issue_comment_url":"https://api.github.com/repos/vertexproject/synapse/issues/comments{/number}","contents_url":"https://api.github.com/repos/vertexproject/synapse/contents/{+path}","compare_url":"https://api.github.com/repos/vertexproject/synapse/compare/{base}...{head}","merges_url":"https://api.github.com/repos/vertexproject/synapse/merges","archive_url":"https://api.github.com/repos/vertexproject/synapse/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/vertexproject/synapse/downloads","issues_url":"https://api.github.com/repos/vertexproject/synapse/issues{/number}","pulls_url":"https://api.github.com/repos/vertexproject/synapse/pulls{/number}","milestones_url":"https://api.github.com/repos/vertexproject/synapse/milestones{/number}","notifications_url":"https://api.github.com/repos/vertexproject/synapse/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/vertexproject/synapse/labels{/name}","releases_url":"https://api.github.com/repos/vertexproject/synapse/releases{/id}","deployments_url":"https://api.github.com/repos/vertexproject/synapse/deployments","created_at":"2015-06-10T23:29:41Z","updated_at":"2018-06-06T13:33:11Z","pushed_at":"2018-06-21T19:37:56Z","git_url":"git://github.com/vertexproject/synapse.git","ssh_url":"git@github.com:vertexproject/synapse.git","clone_url":"https://github.com/vertexproject/synapse.git","svn_url":"https://github.com/vertexproject/synapse","homepage":"","size":8606,"stargazers_count":44,"watchers_count":44,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":28,"mirror_url":null,"archived":false,"open_issues_count":28,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":28,"open_issues":28,"watchers":44,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/vertexproject/synapse/pulls/826"},"html":{"href":"https://github.com/vertexproject/synapse/pull/826"},"issue":{"href":"https://api.github.com/repos/vertexproject/synapse/issues/826"},"comments":{"href":"https://api.github.com/repos/vertexproject/synapse/issues/826/comments"},"review_comments":{"href":"https://api.github.com/repos/vertexproject/synapse/pulls/826/comments"},"review_comment":{"href":"https://api.github.com/repos/vertexproject/synapse/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/vertexproject/synapse/pulls/826/commits"},"statuses":{"href":"https://api.github.com/repos/vertexproject/synapse/statuses/adc5ee8fe000761c8539f64bbaead2cb0475e523"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"jnwatson"}}