{"action":"created","comment":{"url":"https://api.github.com/repos/wellcometrust/platform/pulls/comments/205049869","pull_request_review_id":140241336,"id":205049869,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIwNTA0OTg2OQ==","diff_hunk":"@@ -72,4 +72,96 @@ class MiroTransformerFeatureTest\n       }\n     }\n   }\n+\n+  // This is based on a specific bug that we found where different records\n+  // were written to the same s3 key because of the hashing algorithm clashing\n+  it(\"sends different messages for different miro records\") {\n+    withLocalSnsTopic { topic =>\n+      withLocalSqsQueue { queue =>\n+        withLocalS3Bucket { storageBucket =>\n+          withLocalS3Bucket { messageBucket =>\n+            val flags: Map[String, String] = Map(\n+              \"aws.metrics.namespace\" -> \"sierra-transformer\"\n+            ) ++ s3LocalFlags(storageBucket) ++\n+              sqsLocalFlags(queue) ++ messageWriterLocalFlags(\n+              messageBucket,\n+              topic)\n+\n+            withServer(flags) { _ =>\n+              val miroHybridRecordMessage1 =\n+                hybridRecordNotificationMessage(\n+                  message = createValidMiroTransformableJson(\n+                    MiroID = \"L0011975\",\n+                    MiroCollection = \"images-L\",\n+                    data = \"\"\"\n+                      {\n+                          \"image_cleared\": \"Y\",\n+                          \"image_copyright_cleared\": \"Y\",\n+                          \"image_credit_line\": \"Wellcome Library, London\",\n+                          \"image_image_desc\": \"Antonio Dionisi\",\n+                          \"image_innopac_id\": \"12917175\",\n+                          \"image_library_dept\": \"General Collections\",\n+                          \"image_no_calc\": \"L0011975\",\n+                          \"image_phys_format\": \"Book\",\n+                          \"image_tech_file_size\": [\n+                              \"5247788\"\n+                          ],\n+                          \"image_title\": \"Antonio Dionisi\",\n+                          \"image_use_restrictions\": \"CC-BY\"\n+                      }\n+                    \"\"\"\n+                  ),\n+                  sourceName = \"miro\",\n+                  s3Client = s3Client,\n+                  bucket = storageBucket\n+                )\n+              val miroHybridRecordMessage2 =\n+                hybridRecordNotificationMessage(\n+                  message = createValidMiroTransformableJson(\n+                    MiroID = \"L0023034\",\n+                    MiroCollection = \"images-L\",\n+                    data =\n+                      \"\"\"\n+                      {\n+                          \"image_cleared\": \"Y\",\n+                          \"image_copyright_cleared\": \"Y\",\n+                          \"image_image_desc\": \"Use of the guillotine\",\n+                          \"image_innopac_id\": \"12074536\",\n+                          \"image_keywords\": [\n+                              \"Surgery\"\n+                          ],\n+                          \"image_library_dept\": \"General Collections\",\n+                          \"image_no_calc\": \"L0023034\",\n+                          \"image_tech_file_size\": [\n+                              \"5710662\"\n+                          ],\n+                          \"image_title\": \"Greenfield Sluder, Tonsillectomy..., use of guillotine\",\n+                          \"image_use_restrictions\": \"CC-BY\"\n+                      }\n+                    \"\"\"\n+                  ),\n+                  sourceName = \"miro\",\n+                  s3Client = s3Client,\n+                  bucket = storageBucket\n+                )\n+              sqsClient.sendMessage(\n+                queue.url,\n+                JsonUtil.toJson(miroHybridRecordMessage1).get\n+              )\n+\n+              sqsClient.sendMessage(\n+                queue.url,\n+                JsonUtil.toJson(miroHybridRecordMessage2).get\n+              )\n+              eventually {\n+                val works = getMessages[UnidentifiedWork](topic)\n+                works.distinct.length shouldBe >=(2)\n+","path":"catalogue_pipeline/transformer/src/test/scala/uk/ac/wellcome/platform/transformer/MiroTransformerFeatureTest.scala","position":88,"original_position":88,"commit_id":"df12dbbceeb6e86339c5428d5eaee6c400d37f0a","original_commit_id":"df12dbbceeb6e86339c5428d5eaee6c400d37f0a","user":{"login":"alicefuzier","id":26736746,"node_id":"MDQ6VXNlcjI2NzM2NzQ2","avatar_url":"https://avatars3.githubusercontent.com/u/26736746?v=4","gravatar_id":"","url":"https://api.github.com/users/alicefuzier","html_url":"https://github.com/alicefuzier","followers_url":"https://api.github.com/users/alicefuzier/followers","following_url":"https://api.github.com/users/alicefuzier/following{/other_user}","gists_url":"https://api.github.com/users/alicefuzier/gists{/gist_id}","starred_url":"https://api.github.com/users/alicefuzier/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alicefuzier/subscriptions","organizations_url":"https://api.github.com/users/alicefuzier/orgs","repos_url":"https://api.github.com/users/alicefuzier/repos","events_url":"https://api.github.com/users/alicefuzier/events{/privacy}","received_events_url":"https://api.github.com/users/alicefuzier/received_events","type":"User","site_admin":false},"body":"yeah you're right. It's a copy pasta error from test where we have to account for at least once delivery","created_at":"2018-07-25T09:56:27Z","updated_at":"2018-07-25T09:56:27Z","html_url":"https://github.com/wellcometrust/platform/pull/2454#discussion_r205049869","pull_request_url":"https://api.github.com/repos/wellcometrust/platform/pulls/2454","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/wellcometrust/platform/pulls/comments/205049869"},"html":{"href":"https://github.com/wellcometrust/platform/pull/2454#discussion_r205049869"},"pull_request":{"href":"https://api.github.com/repos/wellcometrust/platform/pulls/2454"}},"in_reply_to_id":205040964},"pull_request":{"url":"https://api.github.com/repos/wellcometrust/platform/pulls/2454","id":203569706,"node_id":"MDExOlB1bGxSZXF1ZXN0MjAzNTY5NzA2","html_url":"https://github.com/wellcometrust/platform/pull/2454","diff_url":"https://github.com/wellcometrust/platform/pull/2454.diff","patch_url":"https://github.com/wellcometrust/platform/pull/2454.patch","issue_url":"https://api.github.com/repos/wellcometrust/platform/issues/2454","number":2454,"state":"open","locked":false,"title":"Reduce likelihood of clashes in messages sent by MessageWriter (and the transformer)","user":{"login":"alicefuzier","id":26736746,"node_id":"MDQ6VXNlcjI2NzM2NzQ2","avatar_url":"https://avatars3.githubusercontent.com/u/26736746?v=4","gravatar_id":"","url":"https://api.github.com/users/alicefuzier","html_url":"https://github.com/alicefuzier","followers_url":"https://api.github.com/users/alicefuzier/followers","following_url":"https://api.github.com/users/alicefuzier/following{/other_user}","gists_url":"https://api.github.com/users/alicefuzier/gists{/gist_id}","starred_url":"https://api.github.com/users/alicefuzier/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alicefuzier/subscriptions","organizations_url":"https://api.github.com/users/alicefuzier/orgs","repos_url":"https://api.github.com/users/alicefuzier/repos","events_url":"https://api.github.com/users/alicefuzier/events{/privacy}","received_events_url":"https://api.github.com/users/alicefuzier/received_events","type":"User","site_admin":false},"body":"For #2361 ","created_at":"2018-07-24T15:36:17Z","updated_at":"2018-07-25T09:56:27Z","closed_at":null,"merged_at":null,"merge_commit_sha":"d2bad4f61840489d50152af3632c1c84b13ec787","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/wellcometrust/platform/pulls/2454/commits","review_comments_url":"https://api.github.com/repos/wellcometrust/platform/pulls/2454/comments","review_comment_url":"https://api.github.com/repos/wellcometrust/platform/pulls/comments{/number}","comments_url":"https://api.github.com/repos/wellcometrust/platform/issues/2454/comments","statuses_url":"https://api.github.com/repos/wellcometrust/platform/statuses/df12dbbceeb6e86339c5428d5eaee6c400d37f0a","head":{"label":"wellcometrust:transformer-s3-key-hash","ref":"transformer-s3-key-hash","sha":"df12dbbceeb6e86339c5428d5eaee6c400d37f0a","user":{"login":"wellcometrust","id":2814213,"node_id":"MDEyOk9yZ2FuaXphdGlvbjI4MTQyMTM=","avatar_url":"https://avatars3.githubusercontent.com/u/2814213?v=4","gravatar_id":"","url":"https://api.github.com/users/wellcometrust","html_url":"https://github.com/wellcometrust","followers_url":"https://api.github.com/users/wellcometrust/followers","following_url":"https://api.github.com/users/wellcometrust/following{/other_user}","gists_url":"https://api.github.com/users/wellcometrust/gists{/gist_id}","starred_url":"https://api.github.com/users/wellcometrust/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wellcometrust/subscriptions","organizations_url":"https://api.github.com/users/wellcometrust/orgs","repos_url":"https://api.github.com/users/wellcometrust/repos","events_url":"https://api.github.com/users/wellcometrust/events{/privacy}","received_events_url":"https://api.github.com/users/wellcometrust/received_events","type":"Organization","site_admin":false},"repo":{"id":80218943,"node_id":"MDEwOlJlcG9zaXRvcnk4MDIxODk0Mw==","name":"platform","full_name":"wellcometrust/platform","owner":{"login":"wellcometrust","id":2814213,"node_id":"MDEyOk9yZ2FuaXphdGlvbjI4MTQyMTM=","avatar_url":"https://avatars3.githubusercontent.com/u/2814213?v=4","gravatar_id":"","url":"https://api.github.com/users/wellcometrust","html_url":"https://github.com/wellcometrust","followers_url":"https://api.github.com/users/wellcometrust/followers","following_url":"https://api.github.com/users/wellcometrust/following{/other_user}","gists_url":"https://api.github.com/users/wellcometrust/gists{/gist_id}","starred_url":"https://api.github.com/users/wellcometrust/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wellcometrust/subscriptions","organizations_url":"https://api.github.com/users/wellcometrust/orgs","repos_url":"https://api.github.com/users/wellcometrust/repos","events_url":"https://api.github.com/users/wellcometrust/events{/privacy}","received_events_url":"https://api.github.com/users/wellcometrust/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/wellcometrust/platform","description":"Wellcome Collection digital platform","fork":false,"url":"https://api.github.com/repos/wellcometrust/platform","forks_url":"https://api.github.com/repos/wellcometrust/platform/forks","keys_url":"https://api.github.com/repos/wellcometrust/platform/keys{/key_id}","collaborators_url":"https://api.github.com/repos/wellcometrust/platform/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/wellcometrust/platform/teams","hooks_url":"https://api.github.com/repos/wellcometrust/platform/hooks","issue_events_url":"https://api.github.com/repos/wellcometrust/platform/issues/events{/number}","events_url":"https://api.github.com/repos/wellcometrust/platform/events","assignees_url":"https://api.github.com/repos/wellcometrust/platform/assignees{/user}","branches_url":"https://api.github.com/repos/wellcometrust/platform/branches{/branch}","tags_url":"https://api.github.com/repos/wellcometrust/platform/tags","blobs_url":"https://api.github.com/repos/wellcometrust/platform/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/wellcometrust/platform/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/wellcometrust/platform/git/refs{/sha}","trees_url":"https://api.github.com/repos/wellcometrust/platform/git/trees{/sha}","statuses_url":"https://api.github.com/repos/wellcometrust/platform/statuses/{sha}","languages_url":"https://api.github.com/repos/wellcometrust/platform/languages","stargazers_url":"https://api.github.com/repos/wellcometrust/platform/stargazers","contributors_url":"https://api.github.com/repos/wellcometrust/platform/contributors","subscribers_url":"https://api.github.com/repos/wellcometrust/platform/subscribers","subscription_url":"https://api.github.com/repos/wellcometrust/platform/subscription","commits_url":"https://api.github.com/repos/wellcometrust/platform/commits{/sha}","git_commits_url":"https://api.github.com/repos/wellcometrust/platform/git/commits{/sha}","comments_url":"https://api.github.com/repos/wellcometrust/platform/comments{/number}","issue_comment_url":"https://api.github.com/repos/wellcometrust/platform/issues/comments{/number}","contents_url":"https://api.github.com/repos/wellcometrust/platform/contents/{+path}","compare_url":"https://api.github.com/repos/wellcometrust/platform/compare/{base}...{head}","merges_url":"https://api.github.com/repos/wellcometrust/platform/merges","archive_url":"https://api.github.com/repos/wellcometrust/platform/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/wellcometrust/platform/downloads","issues_url":"https://api.github.com/repos/wellcometrust/platform/issues{/number}","pulls_url":"https://api.github.com/repos/wellcometrust/platform/pulls{/number}","milestones_url":"https://api.github.com/repos/wellcometrust/platform/milestones{/number}","notifications_url":"https://api.github.com/repos/wellcometrust/platform/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/wellcometrust/platform/labels{/name}","releases_url":"https://api.github.com/repos/wellcometrust/platform/releases{/id}","deployments_url":"https://api.github.com/repos/wellcometrust/platform/deployments","created_at":"2017-01-27T15:27:42Z","updated_at":"2018-07-25T09:55:28Z","pushed_at":"2018-07-25T09:55:30Z","git_url":"git://github.com/wellcometrust/platform.git","ssh_url":"git@github.com:wellcometrust/platform.git","clone_url":"https://github.com/wellcometrust/platform.git","svn_url":"https://github.com/wellcometrust/platform","homepage":"https://developers.wellcomecollection.org/","size":25563,"stargazers_count":13,"watchers_count":13,"language":"Scala","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":6,"mirror_url":null,"archived":false,"open_issues_count":152,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":6,"open_issues":152,"watchers":13,"default_branch":"master"}},"base":{"label":"wellcometrust:master","ref":"master","sha":"12605813583097c4438f4c924eca272695427919","user":{"login":"wellcometrust","id":2814213,"node_id":"MDEyOk9yZ2FuaXphdGlvbjI4MTQyMTM=","avatar_url":"https://avatars3.githubusercontent.com/u/2814213?v=4","gravatar_id":"","url":"https://api.github.com/users/wellcometrust","html_url":"https://github.com/wellcometrust","followers_url":"https://api.github.com/users/wellcometrust/followers","following_url":"https://api.github.com/users/wellcometrust/following{/other_user}","gists_url":"https://api.github.com/users/wellcometrust/gists{/gist_id}","starred_url":"https://api.github.com/users/wellcometrust/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wellcometrust/subscriptions","organizations_url":"https://api.github.com/users/wellcometrust/orgs","repos_url":"https://api.github.com/users/wellcometrust/repos","events_url":"https://api.github.com/users/wellcometrust/events{/privacy}","received_events_url":"https://api.github.com/users/wellcometrust/received_events","type":"Organization","site_admin":false},"repo":{"id":80218943,"node_id":"MDEwOlJlcG9zaXRvcnk4MDIxODk0Mw==","name":"platform","full_name":"wellcometrust/platform","owner":{"login":"wellcometrust","id":2814213,"node_id":"MDEyOk9yZ2FuaXphdGlvbjI4MTQyMTM=","avatar_url":"https://avatars3.githubusercontent.com/u/2814213?v=4","gravatar_id":"","url":"https://api.github.com/users/wellcometrust","html_url":"https://github.com/wellcometrust","followers_url":"https://api.github.com/users/wellcometrust/followers","following_url":"https://api.github.com/users/wellcometrust/following{/other_user}","gists_url":"https://api.github.com/users/wellcometrust/gists{/gist_id}","starred_url":"https://api.github.com/users/wellcometrust/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wellcometrust/subscriptions","organizations_url":"https://api.github.com/users/wellcometrust/orgs","repos_url":"https://api.github.com/users/wellcometrust/repos","events_url":"https://api.github.com/users/wellcometrust/events{/privacy}","received_events_url":"https://api.github.com/users/wellcometrust/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/wellcometrust/platform","description":"Wellcome Collection digital platform","fork":false,"url":"https://api.github.com/repos/wellcometrust/platform","forks_url":"https://api.github.com/repos/wellcometrust/platform/forks","keys_url":"https://api.github.com/repos/wellcometrust/platform/keys{/key_id}","collaborators_url":"https://api.github.com/repos/wellcometrust/platform/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/wellcometrust/platform/teams","hooks_url":"https://api.github.com/repos/wellcometrust/platform/hooks","issue_events_url":"https://api.github.com/repos/wellcometrust/platform/issues/events{/number}","events_url":"https://api.github.com/repos/wellcometrust/platform/events","assignees_url":"https://api.github.com/repos/wellcometrust/platform/assignees{/user}","branches_url":"https://api.github.com/repos/wellcometrust/platform/branches{/branch}","tags_url":"https://api.github.com/repos/wellcometrust/platform/tags","blobs_url":"https://api.github.com/repos/wellcometrust/platform/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/wellcometrust/platform/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/wellcometrust/platform/git/refs{/sha}","trees_url":"https://api.github.com/repos/wellcometrust/platform/git/trees{/sha}","statuses_url":"https://api.github.com/repos/wellcometrust/platform/statuses/{sha}","languages_url":"https://api.github.com/repos/wellcometrust/platform/languages","stargazers_url":"https://api.github.com/repos/wellcometrust/platform/stargazers","contributors_url":"https://api.github.com/repos/wellcometrust/platform/contributors","subscribers_url":"https://api.github.com/repos/wellcometrust/platform/subscribers","subscription_url":"https://api.github.com/repos/wellcometrust/platform/subscription","commits_url":"https://api.github.com/repos/wellcometrust/platform/commits{/sha}","git_commits_url":"https://api.github.com/repos/wellcometrust/platform/git/commits{/sha}","comments_url":"https://api.github.com/repos/wellcometrust/platform/comments{/number}","issue_comment_url":"https://api.github.com/repos/wellcometrust/platform/issues/comments{/number}","contents_url":"https://api.github.com/repos/wellcometrust/platform/contents/{+path}","compare_url":"https://api.github.com/repos/wellcometrust/platform/compare/{base}...{head}","merges_url":"https://api.github.com/repos/wellcometrust/platform/merges","archive_url":"https://api.github.com/repos/wellcometrust/platform/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/wellcometrust/platform/downloads","issues_url":"https://api.github.com/repos/wellcometrust/platform/issues{/number}","pulls_url":"https://api.github.com/repos/wellcometrust/platform/pulls{/number}","milestones_url":"https://api.github.com/repos/wellcometrust/platform/milestones{/number}","notifications_url":"https://api.github.com/repos/wellcometrust/platform/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/wellcometrust/platform/labels{/name}","releases_url":"https://api.github.com/repos/wellcometrust/platform/releases{/id}","deployments_url":"https://api.github.com/repos/wellcometrust/platform/deployments","created_at":"2017-01-27T15:27:42Z","updated_at":"2018-07-25T09:55:28Z","pushed_at":"2018-07-25T09:55:30Z","git_url":"git://github.com/wellcometrust/platform.git","ssh_url":"git@github.com:wellcometrust/platform.git","clone_url":"https://github.com/wellcometrust/platform.git","svn_url":"https://github.com/wellcometrust/platform","homepage":"https://developers.wellcomecollection.org/","size":25563,"stargazers_count":13,"watchers_count":13,"language":"Scala","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":6,"mirror_url":null,"archived":false,"open_issues_count":152,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":6,"open_issues":152,"watchers":13,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/wellcometrust/platform/pulls/2454"},"html":{"href":"https://github.com/wellcometrust/platform/pull/2454"},"issue":{"href":"https://api.github.com/repos/wellcometrust/platform/issues/2454"},"comments":{"href":"https://api.github.com/repos/wellcometrust/platform/issues/2454/comments"},"review_comments":{"href":"https://api.github.com/repos/wellcometrust/platform/pulls/2454/comments"},"review_comment":{"href":"https://api.github.com/repos/wellcometrust/platform/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/wellcometrust/platform/pulls/2454/commits"},"statuses":{"href":"https://api.github.com/repos/wellcometrust/platform/statuses/df12dbbceeb6e86339c5428d5eaee6c400d37f0a"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"alicefuzier"}}