{"action":"created","comment":{"url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/pulls/comments/81833228","pull_request_review_id":2776050,"id":81833228,"diff_hunk":"@@ -31,13 +25,68 @@ def test_generate_f_file(monkeypatch, mock_broker_config_paths):\n \n     fileF_mock.mappings = OrderedDict(\n         [('key4', 'mapping4'), ('key11', 'mapping11')])\n-    file_path = str(mock_broker_config_paths['broker_files'].join('unique1'))\n+    file_path = str(mock_broker_config_paths['broker_files'].join('uniq1'))\n     expected = [['key4', 'key11'], ['a', 'b'], ['c', 'd']]\n-    assert read_f_file_rows('unique1', file_path) == expected\n+    jobQueue.generate_f_file(1, 1, Mock(), 'uniq1', 'uniq1', is_local=True)\n+    assert read_file_rows(file_path) == expected\n \n     # re-order\n     fileF_mock.mappings = OrderedDict(\n         [('key11', 'mapping11'), ('key4', 'mapping4')])\n-    file_path = str(mock_broker_config_paths['broker_files'].join('unique2'))\n+    file_path = str(mock_broker_config_paths['broker_files'].join('uniq2'))\n     expected = [['key11', 'key4'], ['b', 'a'], ['d', 'c']]\n-    assert read_f_file_rows('unique2', file_path) == expected\n+    jobQueue.generate_f_file(1, 1, Mock(), 'uniq2', 'uniq2', is_local=True)\n+    assert read_file_rows(file_path) == expected\n+\n+\n+def test_generate_e_file(monkeypatch, mock_broker_config_paths, database):\n+    \"\"\"A CSV with fields in the right order should be written to the file\n+    system\"\"\"\n+    # Generate several file D1 entries, largely with the same submission_id,\n+    # and with two overlapping DUNS\n+    model = AwardProcurementFactory()\n+    aps = [AwardProcurementFactory(submission_id=model.submission_id)\n+           for i in range(4)]\n+    same_duns = AwardProcurementFactory(\n+        submission_id=model.submission_id,\n+        awardee_or_recipient_uniqu=model.awardee_or_recipient_uniqu)\n+    unrelated = AwardProcurementFactory(submission_id=model.submission_id + 1)\n+    database.session.add_all(aps + [model, same_duns, unrelated])\n+\n+    monkeypatch.setattr(jobQueue.fileE, 'retrieveRows', Mock())\n+    jobQueue.fileE.retrieveRows.return_value = [\n+        fileE.Row('a', 'b', 'c', '1a', '1b', '2a', '2b', '3a', '3b',\n+                  '4a', '4b', '5a', '5b'),\n+        fileE.Row('A', 'B', 'C', '1A', '1B', '2A', '2B', '3A', '3B',\n+                  '4A', '4B', '5A', '5B')\n+    ]\n+\n+    # Mock out the interface holder class; rather nasty, as we want to _keep_\n+    # the database session handler\n+    interface_class = Mock()\n+    interface_class.return_value.jobDb.session = database.session\n+    jobQueue.generate_e_file(\n+        model.submission_id, 1, interface_class, 'uniq', 'uniq',\n+        is_local=True)\n+    file_path = str(mock_broker_config_paths['broker_files'].join('uniq'))\n+    expected = [\n+        ['AwardeeOrRecipientUniqueIdentifier',\n+         'UltimateParentUniqueIdentifier',\n+         'UltimateParentLegalEntityName',\n+         'HighCompOfficer1Name', 'HighCompOfficer1Amount',\n+         'HighCompOfficer2Name', 'HighCompOfficer2Amount',\n+         'HighCompOfficer3Name', 'HighCompOfficer3Amount',\n+         'HighCompOfficer4Name', 'HighCompOfficer4Amount',\n+         'HighCompOfficer5Name', 'HighCompOfficer5Amount'],\n+        ['a', 'b', 'c', '1a', '1b', '2a', '2b', '3a', '3b', '4a', '4b',\n+         '5a', '5b'],\n+        ['A', 'B', 'C', '1A', '1B', '2A', '2B', '3A', '3B', '4A', '4B',\n+         '5A', '5B']\n+    ]\n+    assert read_file_rows(file_path) == expected","path":"tests/unit/dataactcore/test_job_queue.py","position":87,"original_position":87,"commit_id":"550951e3d817a0db63cc64558e5e107556f8604a","original_commit_id":"550951e3d817a0db63cc64558e5e107556f8604a","user":{"login":"jworcestBAH","id":11858333,"avatar_url":"https://avatars.githubusercontent.com/u/11858333?v=3","gravatar_id":"","url":"https://api.github.com/users/jworcestBAH","html_url":"https://github.com/jworcestBAH","followers_url":"https://api.github.com/users/jworcestBAH/followers","following_url":"https://api.github.com/users/jworcestBAH/following{/other_user}","gists_url":"https://api.github.com/users/jworcestBAH/gists{/gist_id}","starred_url":"https://api.github.com/users/jworcestBAH/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jworcestBAH/subscriptions","organizations_url":"https://api.github.com/users/jworcestBAH/orgs","repos_url":"https://api.github.com/users/jworcestBAH/repos","events_url":"https://api.github.com/users/jworcestBAH/events{/privacy}","received_events_url":"https://api.github.com/users/jworcestBAH/received_events","type":"User","site_admin":false},"body":"I'm a bit confused about what's being tested on this line.  It looks like we're mocking the result of the entire file E code, and then checking that the mock result is written to the csv, so the file writing and the Row tuple are the parts that are tested here?  I get that we don't want to test by calling the soap API, but perhaps we could mock something lower like getEntities so that the retrieveRows functionality gets tested?  Alternatively we could add a retrieveRows test to the test_utils_fileE below with the other fileE.py tests.","created_at":"2016-10-04T19:22:43Z","updated_at":"2016-10-04T19:42:29Z","html_url":"https://github.com/fedspendingtransparency/data-act-broker-backend/pull/331#discussion_r81833228","pull_request_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/pulls/331","_links":{"self":{"href":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/pulls/comments/81833228"},"html":{"href":"https://github.com/fedspendingtransparency/data-act-broker-backend/pull/331#discussion_r81833228"},"pull_request":{"href":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/pulls/331"}}},"pull_request":{"url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/pulls/331","id":87803435,"html_url":"https://github.com/fedspendingtransparency/data-act-broker-backend/pull/331","diff_url":"https://github.com/fedspendingtransparency/data-act-broker-backend/pull/331.diff","patch_url":"https://github.com/fedspendingtransparency/data-act-broker-backend/pull/331.patch","issue_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/issues/331","number":331,"state":"open","locked":false,"title":"File E generation","user":{"login":"cmc333333","id":326918,"avatar_url":"https://avatars.githubusercontent.com/u/326918?v=3","gravatar_id":"","url":"https://api.github.com/users/cmc333333","html_url":"https://github.com/cmc333333","followers_url":"https://api.github.com/users/cmc333333/followers","following_url":"https://api.github.com/users/cmc333333/following{/other_user}","gists_url":"https://api.github.com/users/cmc333333/gists{/gist_id}","starred_url":"https://api.github.com/users/cmc333333/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cmc333333/subscriptions","organizations_url":"https://api.github.com/users/cmc333333/orgs","repos_url":"https://api.github.com/users/cmc333333/repos","events_url":"https://api.github.com/users/cmc333333/events{/privacy}","received_events_url":"https://api.github.com/users/cmc333333/received_events","type":"User","site_admin":false},"body":"Functions to grab file E data from the SAM SOAP endpoints, convert that into the fields we care about, add an asynchronous task for dealing with this processing in the job queue, and connecting all of that to the HTTP `fileHandler` endpoint.\r\n\r\nThis'll probably be review changeset by changeset.","created_at":"2016-10-03T22:26:43Z","updated_at":"2016-10-04T19:42:29Z","closed_at":null,"merged_at":null,"merge_commit_sha":"bf5fba3910f03b5953d7281ebc7819cd8501b18d","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/pulls/331/commits","review_comments_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/pulls/331/comments","review_comment_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/pulls/comments{/number}","comments_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/issues/331/comments","statuses_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/statuses/550951e3d817a0db63cc64558e5e107556f8604a","head":{"label":"fedspendingtransparency:db-1012-gen-file-e","ref":"db-1012-gen-file-e","sha":"550951e3d817a0db63cc64558e5e107556f8604a","user":{"login":"fedspendingtransparency","id":8397762,"avatar_url":"https://avatars.githubusercontent.com/u/8397762?v=3","gravatar_id":"","url":"https://api.github.com/users/fedspendingtransparency","html_url":"https://github.com/fedspendingtransparency","followers_url":"https://api.github.com/users/fedspendingtransparency/followers","following_url":"https://api.github.com/users/fedspendingtransparency/following{/other_user}","gists_url":"https://api.github.com/users/fedspendingtransparency/gists{/gist_id}","starred_url":"https://api.github.com/users/fedspendingtransparency/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fedspendingtransparency/subscriptions","organizations_url":"https://api.github.com/users/fedspendingtransparency/orgs","repos_url":"https://api.github.com/users/fedspendingtransparency/repos","events_url":"https://api.github.com/users/fedspendingtransparency/events{/privacy}","received_events_url":"https://api.github.com/users/fedspendingtransparency/received_events","type":"Organization","site_admin":false},"repo":{"id":57313310,"name":"data-act-broker-backend","full_name":"fedspendingtransparency/data-act-broker-backend","owner":{"login":"fedspendingtransparency","id":8397762,"avatar_url":"https://avatars.githubusercontent.com/u/8397762?v=3","gravatar_id":"","url":"https://api.github.com/users/fedspendingtransparency","html_url":"https://github.com/fedspendingtransparency","followers_url":"https://api.github.com/users/fedspendingtransparency/followers","following_url":"https://api.github.com/users/fedspendingtransparency/following{/other_user}","gists_url":"https://api.github.com/users/fedspendingtransparency/gists{/gist_id}","starred_url":"https://api.github.com/users/fedspendingtransparency/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fedspendingtransparency/subscriptions","organizations_url":"https://api.github.com/users/fedspendingtransparency/orgs","repos_url":"https://api.github.com/users/fedspendingtransparency/repos","events_url":"https://api.github.com/users/fedspendingtransparency/events{/privacy}","received_events_url":"https://api.github.com/users/fedspendingtransparency/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/fedspendingtransparency/data-act-broker-backend","description":"","fork":false,"url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend","forks_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/forks","keys_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/keys{/key_id}","collaborators_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/teams","hooks_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/hooks","issue_events_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/issues/events{/number}","events_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/events","assignees_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/assignees{/user}","branches_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/branches{/branch}","tags_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/tags","blobs_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/git/refs{/sha}","trees_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/git/trees{/sha}","statuses_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/statuses/{sha}","languages_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/languages","stargazers_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/stargazers","contributors_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/contributors","subscribers_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/subscribers","subscription_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/subscription","commits_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/commits{/sha}","git_commits_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/git/commits{/sha}","comments_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/comments{/number}","issue_comment_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/issues/comments{/number}","contents_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/contents/{+path}","compare_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/compare/{base}...{head}","merges_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/merges","archive_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/downloads","issues_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/issues{/number}","pulls_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/pulls{/number}","milestones_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/milestones{/number}","notifications_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/labels{/name}","releases_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/releases{/id}","deployments_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/deployments","created_at":"2016-04-28T15:39:36Z","updated_at":"2016-08-30T17:03:39Z","pushed_at":"2016-10-04T18:26:37Z","git_url":"git://github.com/fedspendingtransparency/data-act-broker-backend.git","ssh_url":"git@github.com:fedspendingtransparency/data-act-broker-backend.git","clone_url":"https://github.com/fedspendingtransparency/data-act-broker-backend.git","svn_url":"https://github.com/fedspendingtransparency/data-act-broker-backend","homepage":null,"size":54273,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":3,"forks":2,"open_issues":3,"watchers":1,"default_branch":"master"}},"base":{"label":"fedspendingtransparency:development","ref":"development","sha":"1419e8e48609ca2b28a8e0a5541c8f2b50646d89","user":{"login":"fedspendingtransparency","id":8397762,"avatar_url":"https://avatars.githubusercontent.com/u/8397762?v=3","gravatar_id":"","url":"https://api.github.com/users/fedspendingtransparency","html_url":"https://github.com/fedspendingtransparency","followers_url":"https://api.github.com/users/fedspendingtransparency/followers","following_url":"https://api.github.com/users/fedspendingtransparency/following{/other_user}","gists_url":"https://api.github.com/users/fedspendingtransparency/gists{/gist_id}","starred_url":"https://api.github.com/users/fedspendingtransparency/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fedspendingtransparency/subscriptions","organizations_url":"https://api.github.com/users/fedspendingtransparency/orgs","repos_url":"https://api.github.com/users/fedspendingtransparency/repos","events_url":"https://api.github.com/users/fedspendingtransparency/events{/privacy}","received_events_url":"https://api.github.com/users/fedspendingtransparency/received_events","type":"Organization","site_admin":false},"repo":{"id":57313310,"name":"data-act-broker-backend","full_name":"fedspendingtransparency/data-act-broker-backend","owner":{"login":"fedspendingtransparency","id":8397762,"avatar_url":"https://avatars.githubusercontent.com/u/8397762?v=3","gravatar_id":"","url":"https://api.github.com/users/fedspendingtransparency","html_url":"https://github.com/fedspendingtransparency","followers_url":"https://api.github.com/users/fedspendingtransparency/followers","following_url":"https://api.github.com/users/fedspendingtransparency/following{/other_user}","gists_url":"https://api.github.com/users/fedspendingtransparency/gists{/gist_id}","starred_url":"https://api.github.com/users/fedspendingtransparency/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fedspendingtransparency/subscriptions","organizations_url":"https://api.github.com/users/fedspendingtransparency/orgs","repos_url":"https://api.github.com/users/fedspendingtransparency/repos","events_url":"https://api.github.com/users/fedspendingtransparency/events{/privacy}","received_events_url":"https://api.github.com/users/fedspendingtransparency/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/fedspendingtransparency/data-act-broker-backend","description":"","fork":false,"url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend","forks_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/forks","keys_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/keys{/key_id}","collaborators_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/teams","hooks_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/hooks","issue_events_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/issues/events{/number}","events_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/events","assignees_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/assignees{/user}","branches_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/branches{/branch}","tags_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/tags","blobs_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/git/refs{/sha}","trees_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/git/trees{/sha}","statuses_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/statuses/{sha}","languages_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/languages","stargazers_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/stargazers","contributors_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/contributors","subscribers_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/subscribers","subscription_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/subscription","commits_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/commits{/sha}","git_commits_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/git/commits{/sha}","comments_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/comments{/number}","issue_comment_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/issues/comments{/number}","contents_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/contents/{+path}","compare_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/compare/{base}...{head}","merges_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/merges","archive_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/downloads","issues_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/issues{/number}","pulls_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/pulls{/number}","milestones_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/milestones{/number}","notifications_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/labels{/name}","releases_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/releases{/id}","deployments_url":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/deployments","created_at":"2016-04-28T15:39:36Z","updated_at":"2016-08-30T17:03:39Z","pushed_at":"2016-10-04T18:26:37Z","git_url":"git://github.com/fedspendingtransparency/data-act-broker-backend.git","ssh_url":"git@github.com:fedspendingtransparency/data-act-broker-backend.git","clone_url":"https://github.com/fedspendingtransparency/data-act-broker-backend.git","svn_url":"https://github.com/fedspendingtransparency/data-act-broker-backend","homepage":null,"size":54273,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":3,"forks":2,"open_issues":3,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/pulls/331"},"html":{"href":"https://github.com/fedspendingtransparency/data-act-broker-backend/pull/331"},"issue":{"href":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/issues/331"},"comments":{"href":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/issues/331/comments"},"review_comments":{"href":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/pulls/331/comments"},"review_comment":{"href":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/pulls/331/commits"},"statuses":{"href":"https://api.github.com/repos/fedspendingtransparency/data-act-broker-backend/statuses/550951e3d817a0db63cc64558e5e107556f8604a"}}}}
{"actor":{"display_login":"jworcestBAH"}}