{"action":"created","comment":{"url":"https://api.github.com/repos/oduwsdl/ipwb/pulls/comments/210127288","pull_request_review_id":146276533,"id":210127288,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIxMDEyNzI4OA==","diff_hunk":"@@ -170,6 +170,83 @@ def indexFileAt(warcPaths, encryptionKey=None,\n         print('\\n'.join(cdxjLines))\n \n \n+def generateCDXJMetadata(cdxjLines=None):\n+    metadata = ['!context [\"http://tools.ietf.org/html/rfc7089\"]']\n+    metaVals = {\n+        'generator': \"InterPlanetary Wayback v.{0}\".format(ipwbVersion),\n+        'created_at': '{0}'.format(datetime.datetime.now().isoformat())\n+    }\n+    metaVals = '!meta {0}'.format(json.dumps(metaVals))\n+    metadata.append(metaVals)\n+\n+    return metadata\n+\n+\n+def joinCDXJFiles(cdxjPath1, cdxjPath2, outputFilePath):\n+    # CDXJ2 takes precedence in surt uri and datetimes identity\n+\n+    # Join two files quickly\n+    with open(outputFilePath, 'wb') as wfd:\n+        for f in [cdxjPath1, cdxjPath2]:\n+            with open(f, 'rb') as fd:\n+                shutil.copyfileobj(fd, wfd, 1024 * 1024 * 10)\n+\n+    cdxjLines = ''\n+    with open(outputFilePath, 'r') as wfd:\n+        cdxjLines = wfd.read().split('\\n')\n+\n+        # De-dupe and sort, needed for CDXJ adherence (pulled from indexer.py)\n+        cdxjLines = list(set(cdxjLines))\n+        cdxjLines.sort()\n+\n+        cdxjLines[:] = [line for line in cdxjLines\n+                        if len(line) > 0 and line[0] != '!']\n+\n+        # Prepend metadata\n+        cdxjMetadataLines = generateCDXJMetadata(cdxjLines)\n+        cdxjLines = cdxjMetadataLines + cdxjLines\n+\n+        cdxjLines = '\\n'.join(cdxjLines)\n+\n+    with open(outputFilePath, 'w') as wfd:\n+        wfd.write(cdxjLines)\n+\n+\n+def isCDXJMetadataRecord(cdxjLine):","path":"ipwb/indexer.py","position":54,"original_position":46,"commit_id":"4ef1c420c402489ad201262d55258ee1c66780bb","original_commit_id":"5a2ac5520834fecddebeccf35f3e57e4b1ffec03","user":{"login":"ibnesayeed","id":65147,"node_id":"MDQ6VXNlcjY1MTQ3","avatar_url":"https://avatars2.githubusercontent.com/u/65147?v=4","gravatar_id":"","url":"https://api.github.com/users/ibnesayeed","html_url":"https://github.com/ibnesayeed","followers_url":"https://api.github.com/users/ibnesayeed/followers","following_url":"https://api.github.com/users/ibnesayeed/following{/other_user}","gists_url":"https://api.github.com/users/ibnesayeed/gists{/gist_id}","starred_url":"https://api.github.com/users/ibnesayeed/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ibnesayeed/subscriptions","organizations_url":"https://api.github.com/users/ibnesayeed/orgs","repos_url":"https://api.github.com/users/ibnesayeed/repos","events_url":"https://api.github.com/users/ibnesayeed/events{/privacy}","received_events_url":"https://api.github.com/users/ibnesayeed/received_events","type":"User","site_admin":false},"body":"This function can be as simple as:\r\n\r\n```py\r\n    return bool(cdxjLine.strip()) and cdxjLine.strip().startswith('!')\r\n```","created_at":"2018-08-14T22:44:37Z","updated_at":"2018-08-14T22:47:22Z","html_url":"https://github.com/oduwsdl/ipwb/pull/494#discussion_r210127288","pull_request_url":"https://api.github.com/repos/oduwsdl/ipwb/pulls/494","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/oduwsdl/ipwb/pulls/comments/210127288"},"html":{"href":"https://github.com/oduwsdl/ipwb/pull/494#discussion_r210127288"},"pull_request":{"href":"https://api.github.com/repos/oduwsdl/ipwb/pulls/494"}}},"pull_request":{"url":"https://api.github.com/repos/oduwsdl/ipwb/pulls/494","id":207740793,"node_id":"MDExOlB1bGxSZXF1ZXN0MjA3NzQwNzkz","html_url":"https://github.com/oduwsdl/ipwb/pull/494","diff_url":"https://github.com/oduwsdl/ipwb/pull/494.diff","patch_url":"https://github.com/oduwsdl/ipwb/pull/494.patch","issue_url":"https://api.github.com/repos/oduwsdl/ipwb/issues/494","number":494,"state":"open","locked":false,"title":"[WIP] WARC Upload from replay UI","user":{"login":"machawk1","id":2514780,"node_id":"MDQ6VXNlcjI1MTQ3ODA=","avatar_url":"https://avatars0.githubusercontent.com/u/2514780?v=4","gravatar_id":"","url":"https://api.github.com/users/machawk1","html_url":"https://github.com/machawk1","followers_url":"https://api.github.com/users/machawk1/followers","following_url":"https://api.github.com/users/machawk1/following{/other_user}","gists_url":"https://api.github.com/users/machawk1/gists{/gist_id}","starred_url":"https://api.github.com/users/machawk1/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/machawk1/subscriptions","organizations_url":"https://api.github.com/users/machawk1/orgs","repos_url":"https://api.github.com/users/machawk1/repos","events_url":"https://api.github.com/users/machawk1/events{/privacy}","received_events_url":"https://api.github.com/users/machawk1/received_events","type":"User","site_admin":false},"body":"","created_at":"2018-08-10T23:07:57Z","updated_at":"2018-08-14T22:47:22Z","closed_at":null,"merged_at":null,"merge_commit_sha":"3088e607251625231815d496535bf90b39e4c368","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/oduwsdl/ipwb/pulls/494/commits","review_comments_url":"https://api.github.com/repos/oduwsdl/ipwb/pulls/494/comments","review_comment_url":"https://api.github.com/repos/oduwsdl/ipwb/pulls/comments{/number}","comments_url":"https://api.github.com/repos/oduwsdl/ipwb/issues/494/comments","statuses_url":"https://api.github.com/repos/oduwsdl/ipwb/statuses/4ef1c420c402489ad201262d55258ee1c66780bb","head":{"label":"oduwsdl:issue-436","ref":"issue-436","sha":"4ef1c420c402489ad201262d55258ee1c66780bb","user":{"login":"oduwsdl","id":4416806,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ0MTY4MDY=","avatar_url":"https://avatars0.githubusercontent.com/u/4416806?v=4","gravatar_id":"","url":"https://api.github.com/users/oduwsdl","html_url":"https://github.com/oduwsdl","followers_url":"https://api.github.com/users/oduwsdl/followers","following_url":"https://api.github.com/users/oduwsdl/following{/other_user}","gists_url":"https://api.github.com/users/oduwsdl/gists{/gist_id}","starred_url":"https://api.github.com/users/oduwsdl/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/oduwsdl/subscriptions","organizations_url":"https://api.github.com/users/oduwsdl/orgs","repos_url":"https://api.github.com/users/oduwsdl/repos","events_url":"https://api.github.com/users/oduwsdl/events{/privacy}","received_events_url":"https://api.github.com/users/oduwsdl/received_events","type":"Organization","site_admin":false},"repo":{"id":53143022,"node_id":"MDEwOlJlcG9zaXRvcnk1MzE0MzAyMg==","name":"ipwb","full_name":"oduwsdl/ipwb","owner":{"login":"oduwsdl","id":4416806,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ0MTY4MDY=","avatar_url":"https://avatars0.githubusercontent.com/u/4416806?v=4","gravatar_id":"","url":"https://api.github.com/users/oduwsdl","html_url":"https://github.com/oduwsdl","followers_url":"https://api.github.com/users/oduwsdl/followers","following_url":"https://api.github.com/users/oduwsdl/following{/other_user}","gists_url":"https://api.github.com/users/oduwsdl/gists{/gist_id}","starred_url":"https://api.github.com/users/oduwsdl/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/oduwsdl/subscriptions","organizations_url":"https://api.github.com/users/oduwsdl/orgs","repos_url":"https://api.github.com/users/oduwsdl/repos","events_url":"https://api.github.com/users/oduwsdl/events{/privacy}","received_events_url":"https://api.github.com/users/oduwsdl/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/oduwsdl/ipwb","description":"InterPlanetary Wayback: A distributed and persistent archive replay system using IPFS","fork":false,"url":"https://api.github.com/repos/oduwsdl/ipwb","forks_url":"https://api.github.com/repos/oduwsdl/ipwb/forks","keys_url":"https://api.github.com/repos/oduwsdl/ipwb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/oduwsdl/ipwb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/oduwsdl/ipwb/teams","hooks_url":"https://api.github.com/repos/oduwsdl/ipwb/hooks","issue_events_url":"https://api.github.com/repos/oduwsdl/ipwb/issues/events{/number}","events_url":"https://api.github.com/repos/oduwsdl/ipwb/events","assignees_url":"https://api.github.com/repos/oduwsdl/ipwb/assignees{/user}","branches_url":"https://api.github.com/repos/oduwsdl/ipwb/branches{/branch}","tags_url":"https://api.github.com/repos/oduwsdl/ipwb/tags","blobs_url":"https://api.github.com/repos/oduwsdl/ipwb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/oduwsdl/ipwb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/oduwsdl/ipwb/git/refs{/sha}","trees_url":"https://api.github.com/repos/oduwsdl/ipwb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/oduwsdl/ipwb/statuses/{sha}","languages_url":"https://api.github.com/repos/oduwsdl/ipwb/languages","stargazers_url":"https://api.github.com/repos/oduwsdl/ipwb/stargazers","contributors_url":"https://api.github.com/repos/oduwsdl/ipwb/contributors","subscribers_url":"https://api.github.com/repos/oduwsdl/ipwb/subscribers","subscription_url":"https://api.github.com/repos/oduwsdl/ipwb/subscription","commits_url":"https://api.github.com/repos/oduwsdl/ipwb/commits{/sha}","git_commits_url":"https://api.github.com/repos/oduwsdl/ipwb/git/commits{/sha}","comments_url":"https://api.github.com/repos/oduwsdl/ipwb/comments{/number}","issue_comment_url":"https://api.github.com/repos/oduwsdl/ipwb/issues/comments{/number}","contents_url":"https://api.github.com/repos/oduwsdl/ipwb/contents/{+path}","compare_url":"https://api.github.com/repos/oduwsdl/ipwb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/oduwsdl/ipwb/merges","archive_url":"https://api.github.com/repos/oduwsdl/ipwb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/oduwsdl/ipwb/downloads","issues_url":"https://api.github.com/repos/oduwsdl/ipwb/issues{/number}","pulls_url":"https://api.github.com/repos/oduwsdl/ipwb/pulls{/number}","milestones_url":"https://api.github.com/repos/oduwsdl/ipwb/milestones{/number}","notifications_url":"https://api.github.com/repos/oduwsdl/ipwb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/oduwsdl/ipwb/labels{/name}","releases_url":"https://api.github.com/repos/oduwsdl/ipwb/releases{/id}","deployments_url":"https://api.github.com/repos/oduwsdl/ipwb/deployments","created_at":"2016-03-04T15:01:33Z","updated_at":"2018-08-14T17:18:02Z","pushed_at":"2018-08-14T22:28:02Z","git_url":"git://github.com/oduwsdl/ipwb.git","ssh_url":"git@github.com:oduwsdl/ipwb.git","clone_url":"https://github.com/oduwsdl/ipwb.git","svn_url":"https://github.com/oduwsdl/ipwb","homepage":"","size":5338,"stargazers_count":152,"watchers_count":152,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":13,"mirror_url":null,"archived":false,"open_issues_count":124,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":13,"open_issues":124,"watchers":152,"default_branch":"master"}},"base":{"label":"oduwsdl:master","ref":"master","sha":"eacaba45cb943438ad6781ee6cdd8ee1183e30d7","user":{"login":"oduwsdl","id":4416806,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ0MTY4MDY=","avatar_url":"https://avatars0.githubusercontent.com/u/4416806?v=4","gravatar_id":"","url":"https://api.github.com/users/oduwsdl","html_url":"https://github.com/oduwsdl","followers_url":"https://api.github.com/users/oduwsdl/followers","following_url":"https://api.github.com/users/oduwsdl/following{/other_user}","gists_url":"https://api.github.com/users/oduwsdl/gists{/gist_id}","starred_url":"https://api.github.com/users/oduwsdl/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/oduwsdl/subscriptions","organizations_url":"https://api.github.com/users/oduwsdl/orgs","repos_url":"https://api.github.com/users/oduwsdl/repos","events_url":"https://api.github.com/users/oduwsdl/events{/privacy}","received_events_url":"https://api.github.com/users/oduwsdl/received_events","type":"Organization","site_admin":false},"repo":{"id":53143022,"node_id":"MDEwOlJlcG9zaXRvcnk1MzE0MzAyMg==","name":"ipwb","full_name":"oduwsdl/ipwb","owner":{"login":"oduwsdl","id":4416806,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ0MTY4MDY=","avatar_url":"https://avatars0.githubusercontent.com/u/4416806?v=4","gravatar_id":"","url":"https://api.github.com/users/oduwsdl","html_url":"https://github.com/oduwsdl","followers_url":"https://api.github.com/users/oduwsdl/followers","following_url":"https://api.github.com/users/oduwsdl/following{/other_user}","gists_url":"https://api.github.com/users/oduwsdl/gists{/gist_id}","starred_url":"https://api.github.com/users/oduwsdl/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/oduwsdl/subscriptions","organizations_url":"https://api.github.com/users/oduwsdl/orgs","repos_url":"https://api.github.com/users/oduwsdl/repos","events_url":"https://api.github.com/users/oduwsdl/events{/privacy}","received_events_url":"https://api.github.com/users/oduwsdl/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/oduwsdl/ipwb","description":"InterPlanetary Wayback: A distributed and persistent archive replay system using IPFS","fork":false,"url":"https://api.github.com/repos/oduwsdl/ipwb","forks_url":"https://api.github.com/repos/oduwsdl/ipwb/forks","keys_url":"https://api.github.com/repos/oduwsdl/ipwb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/oduwsdl/ipwb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/oduwsdl/ipwb/teams","hooks_url":"https://api.github.com/repos/oduwsdl/ipwb/hooks","issue_events_url":"https://api.github.com/repos/oduwsdl/ipwb/issues/events{/number}","events_url":"https://api.github.com/repos/oduwsdl/ipwb/events","assignees_url":"https://api.github.com/repos/oduwsdl/ipwb/assignees{/user}","branches_url":"https://api.github.com/repos/oduwsdl/ipwb/branches{/branch}","tags_url":"https://api.github.com/repos/oduwsdl/ipwb/tags","blobs_url":"https://api.github.com/repos/oduwsdl/ipwb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/oduwsdl/ipwb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/oduwsdl/ipwb/git/refs{/sha}","trees_url":"https://api.github.com/repos/oduwsdl/ipwb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/oduwsdl/ipwb/statuses/{sha}","languages_url":"https://api.github.com/repos/oduwsdl/ipwb/languages","stargazers_url":"https://api.github.com/repos/oduwsdl/ipwb/stargazers","contributors_url":"https://api.github.com/repos/oduwsdl/ipwb/contributors","subscribers_url":"https://api.github.com/repos/oduwsdl/ipwb/subscribers","subscription_url":"https://api.github.com/repos/oduwsdl/ipwb/subscription","commits_url":"https://api.github.com/repos/oduwsdl/ipwb/commits{/sha}","git_commits_url":"https://api.github.com/repos/oduwsdl/ipwb/git/commits{/sha}","comments_url":"https://api.github.com/repos/oduwsdl/ipwb/comments{/number}","issue_comment_url":"https://api.github.com/repos/oduwsdl/ipwb/issues/comments{/number}","contents_url":"https://api.github.com/repos/oduwsdl/ipwb/contents/{+path}","compare_url":"https://api.github.com/repos/oduwsdl/ipwb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/oduwsdl/ipwb/merges","archive_url":"https://api.github.com/repos/oduwsdl/ipwb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/oduwsdl/ipwb/downloads","issues_url":"https://api.github.com/repos/oduwsdl/ipwb/issues{/number}","pulls_url":"https://api.github.com/repos/oduwsdl/ipwb/pulls{/number}","milestones_url":"https://api.github.com/repos/oduwsdl/ipwb/milestones{/number}","notifications_url":"https://api.github.com/repos/oduwsdl/ipwb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/oduwsdl/ipwb/labels{/name}","releases_url":"https://api.github.com/repos/oduwsdl/ipwb/releases{/id}","deployments_url":"https://api.github.com/repos/oduwsdl/ipwb/deployments","created_at":"2016-03-04T15:01:33Z","updated_at":"2018-08-14T17:18:02Z","pushed_at":"2018-08-14T22:28:02Z","git_url":"git://github.com/oduwsdl/ipwb.git","ssh_url":"git@github.com:oduwsdl/ipwb.git","clone_url":"https://github.com/oduwsdl/ipwb.git","svn_url":"https://github.com/oduwsdl/ipwb","homepage":"","size":5338,"stargazers_count":152,"watchers_count":152,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":13,"mirror_url":null,"archived":false,"open_issues_count":124,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":13,"open_issues":124,"watchers":152,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/oduwsdl/ipwb/pulls/494"},"html":{"href":"https://github.com/oduwsdl/ipwb/pull/494"},"issue":{"href":"https://api.github.com/repos/oduwsdl/ipwb/issues/494"},"comments":{"href":"https://api.github.com/repos/oduwsdl/ipwb/issues/494/comments"},"review_comments":{"href":"https://api.github.com/repos/oduwsdl/ipwb/pulls/494/comments"},"review_comment":{"href":"https://api.github.com/repos/oduwsdl/ipwb/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/oduwsdl/ipwb/pulls/494/commits"},"statuses":{"href":"https://api.github.com/repos/oduwsdl/ipwb/statuses/4ef1c420c402489ad201262d55258ee1c66780bb"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"ibnesayeed"}}