{"action":"created","comment":{"url":"https://api.github.com/repos/dpla/ingestion3/pulls/comments/213817017","pull_request_review_id":150727166,"id":213817017,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIxMzgxNzAxNw==","diff_hunk":"@@ -0,0 +1,121 @@\n+package dpla.ingestion3.harvesters.api\n+\n+import java.net.URL\n+\n+import dpla.ingestion3.confs.i3Conf\n+import dpla.ingestion3.utils.HttpUtils\n+import org.apache.http.client.utils.URIBuilder\n+import org.apache.log4j.Logger\n+import org.apache.spark.sql.SparkSession\n+import org.json4s.DefaultFormats\n+import org.json4s.jackson.JsonMethods.{compact, parse, render}\n+\n+import scala.util.{Failure, Success}\n+\n+class IaHarvester (spark: SparkSession,\n+                   shortName: String,\n+                   conf: i3Conf,\n+                   outputDir: String,\n+                   harvestLogger: Logger)\n+  extends ApiHarvester(spark, shortName, conf, outputDir, harvestLogger) {\n+\n+  def mimeType: String = \"application_json\"\n+\n+  override protected val queryParams: Map[String, String] = Map(\n+    \"q\" -> conf.harvest.query\n+  ).collect{ case (key, Some(value)) => key -> value } // remove None values\n+\n+  override def localHarvest: Unit = {\n+    implicit val formats: DefaultFormats.type = DefaultFormats\n+\n+    val iaCollections = conf.harvest.setlist.getOrElse(\"\").split(\",\")\n+\n+    iaCollections.foreach(collection => {\n+      // Mutable vars for controlling harvest loop\n+      var continueHarvest = true\n+      var cursor = \"\"\n+\n+      while(continueHarvest) getSinglePage(cursor, collection) match {\n+        // Handle errors\n+        case error: ApiError with ApiResponse =>\n+          harvestLogger.error(\"Error returned by request %s\\n%s\\n%s\".format(\n+            error.errorSource.url.getOrElse(\"Undefined url\"),\n+            error.errorSource.queryParams,\n+            error.message\n+          ))\n+          continueHarvest = false\n+        // Handle a successful response\n+        case src: ApiSource with ApiResponse =>\n+          src.text match {\n+            case Some(docs) =>\n+              val json = parse(docs)\n+              val iaRecords = (json \\\\ \"items\").children.map(doc => {\n+                ApiRecord((doc \\\\ \"identifier\").toString, compact(render(doc)))\n+              })\n+\n+              // @see ApiHarvester\n+              saveOutRecords(iaRecords)\n+\n+              // Loop control\n+              cursor = (json \\\\ \"cursor\").extractOrElse[String](\"\")\n+\n+              if (cursor.isEmpty)\n+                continueHarvest = false\n+            // Handle unknown case\n+            case _ =>\n+              harvestLogger.error(s\"Response body is empty.\\n\" +\n+                s\"URL: ${src.url.getOrElse(\"!!! URL not set !!!\")}\\n\" +","path":"src/main/scala/dpla/ingestion3/harvesters/api/IaHarvester.scala","position":67,"original_position":67,"commit_id":"a6d7ffcb4d2e4597c0d96003aa529eb47292c20e","original_commit_id":"a6d7ffcb4d2e4597c0d96003aa529eb47292c20e","user":{"login":"AudreyAltman","id":3615206,"node_id":"MDQ6VXNlcjM2MTUyMDY=","avatar_url":"https://avatars3.githubusercontent.com/u/3615206?v=4","gravatar_id":"","url":"https://api.github.com/users/AudreyAltman","html_url":"https://github.com/AudreyAltman","followers_url":"https://api.github.com/users/AudreyAltman/followers","following_url":"https://api.github.com/users/AudreyAltman/following{/other_user}","gists_url":"https://api.github.com/users/AudreyAltman/gists{/gist_id}","starred_url":"https://api.github.com/users/AudreyAltman/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AudreyAltman/subscriptions","organizations_url":"https://api.github.com/users/AudreyAltman/orgs","repos_url":"https://api.github.com/users/AudreyAltman/repos","events_url":"https://api.github.com/users/AudreyAltman/events{/privacy}","received_events_url":"https://api.github.com/users/AudreyAltman/received_events","type":"User","site_admin":false},"body":"I like the exclamation marks :)","created_at":"2018-08-29T20:09:11Z","updated_at":"2018-08-29T20:11:17Z","html_url":"https://github.com/dpla/ingestion3/pull/199#discussion_r213817017","pull_request_url":"https://api.github.com/repos/dpla/ingestion3/pulls/199","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/dpla/ingestion3/pulls/comments/213817017"},"html":{"href":"https://github.com/dpla/ingestion3/pull/199#discussion_r213817017"},"pull_request":{"href":"https://api.github.com/repos/dpla/ingestion3/pulls/199"}}},"pull_request":{"url":"https://api.github.com/repos/dpla/ingestion3/pulls/199","id":211474553,"node_id":"MDExOlB1bGxSZXF1ZXN0MjExNDc0NTUz","html_url":"https://github.com/dpla/ingestion3/pull/199","diff_url":"https://github.com/dpla/ingestion3/pull/199.diff","patch_url":"https://github.com/dpla/ingestion3/pull/199.patch","issue_url":"https://api.github.com/repos/dpla/ingestion3/issues/199","number":199,"state":"open","locked":false,"title":"Initial IA harvester","user":{"login":"moltude","id":1419766,"node_id":"MDQ6VXNlcjE0MTk3NjY=","avatar_url":"https://avatars2.githubusercontent.com/u/1419766?v=4","gravatar_id":"","url":"https://api.github.com/users/moltude","html_url":"https://github.com/moltude","followers_url":"https://api.github.com/users/moltude/followers","following_url":"https://api.github.com/users/moltude/following{/other_user}","gists_url":"https://api.github.com/users/moltude/gists{/gist_id}","starred_url":"https://api.github.com/users/moltude/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/moltude/subscriptions","organizations_url":"https://api.github.com/users/moltude/orgs","repos_url":"https://api.github.com/users/moltude/repos","events_url":"https://api.github.com/users/moltude/events{/privacy}","received_events_url":"https://api.github.com/users/moltude/received_events","type":"User","site_admin":false},"body":"Internet Archive harvester. Pulls from the Search API","created_at":"2018-08-28T15:58:41Z","updated_at":"2018-08-29T20:11:17Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7aac4610a50a7e05d32648127d8d8ddd6359b44b","assignee":null,"assignees":[],"requested_reviewers":[{"login":"mdellabitta","id":925372,"node_id":"MDQ6VXNlcjkyNTM3Mg==","avatar_url":"https://avatars2.githubusercontent.com/u/925372?v=4","gravatar_id":"","url":"https://api.github.com/users/mdellabitta","html_url":"https://github.com/mdellabitta","followers_url":"https://api.github.com/users/mdellabitta/followers","following_url":"https://api.github.com/users/mdellabitta/following{/other_user}","gists_url":"https://api.github.com/users/mdellabitta/gists{/gist_id}","starred_url":"https://api.github.com/users/mdellabitta/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mdellabitta/subscriptions","organizations_url":"https://api.github.com/users/mdellabitta/orgs","repos_url":"https://api.github.com/users/mdellabitta/repos","events_url":"https://api.github.com/users/mdellabitta/events{/privacy}","received_events_url":"https://api.github.com/users/mdellabitta/received_events","type":"User","site_admin":false}],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/dpla/ingestion3/pulls/199/commits","review_comments_url":"https://api.github.com/repos/dpla/ingestion3/pulls/199/comments","review_comment_url":"https://api.github.com/repos/dpla/ingestion3/pulls/comments{/number}","comments_url":"https://api.github.com/repos/dpla/ingestion3/issues/199/comments","statuses_url":"https://api.github.com/repos/dpla/ingestion3/statuses/a6d7ffcb4d2e4597c0d96003aa529eb47292c20e","head":{"label":"dpla:feature/ia-harvester","ref":"feature/ia-harvester","sha":"a6d7ffcb4d2e4597c0d96003aa529eb47292c20e","user":{"login":"dpla","id":1252916,"node_id":"MDEyOk9yZ2FuaXphdGlvbjEyNTI5MTY=","avatar_url":"https://avatars2.githubusercontent.com/u/1252916?v=4","gravatar_id":"","url":"https://api.github.com/users/dpla","html_url":"https://github.com/dpla","followers_url":"https://api.github.com/users/dpla/followers","following_url":"https://api.github.com/users/dpla/following{/other_user}","gists_url":"https://api.github.com/users/dpla/gists{/gist_id}","starred_url":"https://api.github.com/users/dpla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dpla/subscriptions","organizations_url":"https://api.github.com/users/dpla/orgs","repos_url":"https://api.github.com/users/dpla/repos","events_url":"https://api.github.com/users/dpla/events{/privacy}","received_events_url":"https://api.github.com/users/dpla/received_events","type":"Organization","site_admin":false},"repo":{"id":78565889,"node_id":"MDEwOlJlcG9zaXRvcnk3ODU2NTg4OQ==","name":"ingestion3","full_name":"dpla/ingestion3","owner":{"login":"dpla","id":1252916,"node_id":"MDEyOk9yZ2FuaXphdGlvbjEyNTI5MTY=","avatar_url":"https://avatars2.githubusercontent.com/u/1252916?v=4","gravatar_id":"","url":"https://api.github.com/users/dpla","html_url":"https://github.com/dpla","followers_url":"https://api.github.com/users/dpla/followers","following_url":"https://api.github.com/users/dpla/following{/other_user}","gists_url":"https://api.github.com/users/dpla/gists{/gist_id}","starred_url":"https://api.github.com/users/dpla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dpla/subscriptions","organizations_url":"https://api.github.com/users/dpla/orgs","repos_url":"https://api.github.com/users/dpla/repos","events_url":"https://api.github.com/users/dpla/events{/privacy}","received_events_url":"https://api.github.com/users/dpla/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dpla/ingestion3","description":"Metadata ingestion system for Digital Public Library of America","fork":false,"url":"https://api.github.com/repos/dpla/ingestion3","forks_url":"https://api.github.com/repos/dpla/ingestion3/forks","keys_url":"https://api.github.com/repos/dpla/ingestion3/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dpla/ingestion3/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dpla/ingestion3/teams","hooks_url":"https://api.github.com/repos/dpla/ingestion3/hooks","issue_events_url":"https://api.github.com/repos/dpla/ingestion3/issues/events{/number}","events_url":"https://api.github.com/repos/dpla/ingestion3/events","assignees_url":"https://api.github.com/repos/dpla/ingestion3/assignees{/user}","branches_url":"https://api.github.com/repos/dpla/ingestion3/branches{/branch}","tags_url":"https://api.github.com/repos/dpla/ingestion3/tags","blobs_url":"https://api.github.com/repos/dpla/ingestion3/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dpla/ingestion3/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dpla/ingestion3/git/refs{/sha}","trees_url":"https://api.github.com/repos/dpla/ingestion3/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dpla/ingestion3/statuses/{sha}","languages_url":"https://api.github.com/repos/dpla/ingestion3/languages","stargazers_url":"https://api.github.com/repos/dpla/ingestion3/stargazers","contributors_url":"https://api.github.com/repos/dpla/ingestion3/contributors","subscribers_url":"https://api.github.com/repos/dpla/ingestion3/subscribers","subscription_url":"https://api.github.com/repos/dpla/ingestion3/subscription","commits_url":"https://api.github.com/repos/dpla/ingestion3/commits{/sha}","git_commits_url":"https://api.github.com/repos/dpla/ingestion3/git/commits{/sha}","comments_url":"https://api.github.com/repos/dpla/ingestion3/comments{/number}","issue_comment_url":"https://api.github.com/repos/dpla/ingestion3/issues/comments{/number}","contents_url":"https://api.github.com/repos/dpla/ingestion3/contents/{+path}","compare_url":"https://api.github.com/repos/dpla/ingestion3/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dpla/ingestion3/merges","archive_url":"https://api.github.com/repos/dpla/ingestion3/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dpla/ingestion3/downloads","issues_url":"https://api.github.com/repos/dpla/ingestion3/issues{/number}","pulls_url":"https://api.github.com/repos/dpla/ingestion3/pulls{/number}","milestones_url":"https://api.github.com/repos/dpla/ingestion3/milestones{/number}","notifications_url":"https://api.github.com/repos/dpla/ingestion3/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dpla/ingestion3/labels{/name}","releases_url":"https://api.github.com/repos/dpla/ingestion3/releases{/id}","deployments_url":"https://api.github.com/repos/dpla/ingestion3/deployments","created_at":"2017-01-10T19:22:18Z","updated_at":"2018-08-21T15:08:58Z","pushed_at":"2018-08-28T17:09:08Z","git_url":"git://github.com/dpla/ingestion3.git","ssh_url":"git@github.com:dpla/ingestion3.git","clone_url":"https://github.com/dpla/ingestion3.git","svn_url":"https://github.com/dpla/ingestion3","homepage":"https://dp.la","size":1215,"stargazers_count":19,"watchers_count":19,"language":"Scala","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":4,"mirror_url":null,"archived":false,"open_issues_count":2,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":4,"open_issues":2,"watchers":19,"default_branch":"develop"}},"base":{"label":"dpla:develop","ref":"develop","sha":"9bdcffc5ef1b4481725baab4edc71d53bf5125d9","user":{"login":"dpla","id":1252916,"node_id":"MDEyOk9yZ2FuaXphdGlvbjEyNTI5MTY=","avatar_url":"https://avatars2.githubusercontent.com/u/1252916?v=4","gravatar_id":"","url":"https://api.github.com/users/dpla","html_url":"https://github.com/dpla","followers_url":"https://api.github.com/users/dpla/followers","following_url":"https://api.github.com/users/dpla/following{/other_user}","gists_url":"https://api.github.com/users/dpla/gists{/gist_id}","starred_url":"https://api.github.com/users/dpla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dpla/subscriptions","organizations_url":"https://api.github.com/users/dpla/orgs","repos_url":"https://api.github.com/users/dpla/repos","events_url":"https://api.github.com/users/dpla/events{/privacy}","received_events_url":"https://api.github.com/users/dpla/received_events","type":"Organization","site_admin":false},"repo":{"id":78565889,"node_id":"MDEwOlJlcG9zaXRvcnk3ODU2NTg4OQ==","name":"ingestion3","full_name":"dpla/ingestion3","owner":{"login":"dpla","id":1252916,"node_id":"MDEyOk9yZ2FuaXphdGlvbjEyNTI5MTY=","avatar_url":"https://avatars2.githubusercontent.com/u/1252916?v=4","gravatar_id":"","url":"https://api.github.com/users/dpla","html_url":"https://github.com/dpla","followers_url":"https://api.github.com/users/dpla/followers","following_url":"https://api.github.com/users/dpla/following{/other_user}","gists_url":"https://api.github.com/users/dpla/gists{/gist_id}","starred_url":"https://api.github.com/users/dpla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dpla/subscriptions","organizations_url":"https://api.github.com/users/dpla/orgs","repos_url":"https://api.github.com/users/dpla/repos","events_url":"https://api.github.com/users/dpla/events{/privacy}","received_events_url":"https://api.github.com/users/dpla/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dpla/ingestion3","description":"Metadata ingestion system for Digital Public Library of America","fork":false,"url":"https://api.github.com/repos/dpla/ingestion3","forks_url":"https://api.github.com/repos/dpla/ingestion3/forks","keys_url":"https://api.github.com/repos/dpla/ingestion3/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dpla/ingestion3/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dpla/ingestion3/teams","hooks_url":"https://api.github.com/repos/dpla/ingestion3/hooks","issue_events_url":"https://api.github.com/repos/dpla/ingestion3/issues/events{/number}","events_url":"https://api.github.com/repos/dpla/ingestion3/events","assignees_url":"https://api.github.com/repos/dpla/ingestion3/assignees{/user}","branches_url":"https://api.github.com/repos/dpla/ingestion3/branches{/branch}","tags_url":"https://api.github.com/repos/dpla/ingestion3/tags","blobs_url":"https://api.github.com/repos/dpla/ingestion3/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dpla/ingestion3/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dpla/ingestion3/git/refs{/sha}","trees_url":"https://api.github.com/repos/dpla/ingestion3/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dpla/ingestion3/statuses/{sha}","languages_url":"https://api.github.com/repos/dpla/ingestion3/languages","stargazers_url":"https://api.github.com/repos/dpla/ingestion3/stargazers","contributors_url":"https://api.github.com/repos/dpla/ingestion3/contributors","subscribers_url":"https://api.github.com/repos/dpla/ingestion3/subscribers","subscription_url":"https://api.github.com/repos/dpla/ingestion3/subscription","commits_url":"https://api.github.com/repos/dpla/ingestion3/commits{/sha}","git_commits_url":"https://api.github.com/repos/dpla/ingestion3/git/commits{/sha}","comments_url":"https://api.github.com/repos/dpla/ingestion3/comments{/number}","issue_comment_url":"https://api.github.com/repos/dpla/ingestion3/issues/comments{/number}","contents_url":"https://api.github.com/repos/dpla/ingestion3/contents/{+path}","compare_url":"https://api.github.com/repos/dpla/ingestion3/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dpla/ingestion3/merges","archive_url":"https://api.github.com/repos/dpla/ingestion3/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dpla/ingestion3/downloads","issues_url":"https://api.github.com/repos/dpla/ingestion3/issues{/number}","pulls_url":"https://api.github.com/repos/dpla/ingestion3/pulls{/number}","milestones_url":"https://api.github.com/repos/dpla/ingestion3/milestones{/number}","notifications_url":"https://api.github.com/repos/dpla/ingestion3/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dpla/ingestion3/labels{/name}","releases_url":"https://api.github.com/repos/dpla/ingestion3/releases{/id}","deployments_url":"https://api.github.com/repos/dpla/ingestion3/deployments","created_at":"2017-01-10T19:22:18Z","updated_at":"2018-08-21T15:08:58Z","pushed_at":"2018-08-28T17:09:08Z","git_url":"git://github.com/dpla/ingestion3.git","ssh_url":"git@github.com:dpla/ingestion3.git","clone_url":"https://github.com/dpla/ingestion3.git","svn_url":"https://github.com/dpla/ingestion3","homepage":"https://dp.la","size":1215,"stargazers_count":19,"watchers_count":19,"language":"Scala","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":4,"mirror_url":null,"archived":false,"open_issues_count":2,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":4,"open_issues":2,"watchers":19,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/dpla/ingestion3/pulls/199"},"html":{"href":"https://github.com/dpla/ingestion3/pull/199"},"issue":{"href":"https://api.github.com/repos/dpla/ingestion3/issues/199"},"comments":{"href":"https://api.github.com/repos/dpla/ingestion3/issues/199/comments"},"review_comments":{"href":"https://api.github.com/repos/dpla/ingestion3/pulls/199/comments"},"review_comment":{"href":"https://api.github.com/repos/dpla/ingestion3/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/dpla/ingestion3/pulls/199/commits"},"statuses":{"href":"https://api.github.com/repos/dpla/ingestion3/statuses/a6d7ffcb4d2e4597c0d96003aa529eb47292c20e"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"AudreyAltman"}}