{"action":"created","comment":{"url":"https://api.github.com/repos/CartoDB/cartodb/pulls/comments/79881393","id":79881393,"diff_hunk":"@@ -0,0 +1,139 @@\n+# encoding: utf-8\n+\n+require 'carto/connector'\n+require_relative 'exceptions'\n+\n+module CartoDB\n+  module Importer2\n+    # ConnectorRunner runs connector-based imports to import datasets from external databases.\n+    #\n+    # ConnectorRunner has the same public API as Runner, but\n+    # instead of downloading and unpacking files and then processing them with ogr2ogr to import them into\n+    # a user database table, a Connector instance connects directly to a remote database and imports\n+    # the specified dataset into a table in the user's database. Currently FDW are used for this.\n+    #\n+    # A connector runner is defined by a hash of parameters; the `provider` parameter is always required and specifies\n+    # the Connector class, which in turn defines the rest of valid parameters for the connector.\n+    #\n+    class ConnectorRunner\n+\n+      attr_reader :results, :log, :job, :warnings\n+      attr_accessor :stats\n+\n+      def initialize(connector_source, options = {})\n+        @pg_options = options[:pg]\n+        @log        = options[:log] || new_logger\n+        @job        = options[:job] || new_job(@log, @pg_options)\n+        @user       = options[:user]\n+\n+        @id = @job.id\n+        @unique_suffix = @id.delete('-')\n+        @json_params = connector_source\n+        extract_params\n+        @connector = Carto::Connector.new(@params, user: @user, logger: @log)\n+        @results = []\n+        @tracker = nil\n+        @stats = {}\n+        @warnings = {}\n+        @connector.check_availability!","path":"services/importer/lib/importer/connector_runner.rb","position":38,"original_position":38,"commit_id":"bc6ba033ab0d238ba5aa05806745738e2bdc5ccb","original_commit_id":"bc6ba033ab0d238ba5aa05806745738e2bdc5ccb","user":{"login":"ethervoid","id":741240,"avatar_url":"https://avatars.githubusercontent.com/u/741240?v=3","gravatar_id":"","url":"https://api.github.com/users/ethervoid","html_url":"https://github.com/ethervoid","followers_url":"https://api.github.com/users/ethervoid/followers","following_url":"https://api.github.com/users/ethervoid/following{/other_user}","gists_url":"https://api.github.com/users/ethervoid/gists{/gist_id}","starred_url":"https://api.github.com/users/ethervoid/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ethervoid/subscriptions","organizations_url":"https://api.github.com/users/ethervoid/orgs","repos_url":"https://api.github.com/users/ethervoid/repos","events_url":"https://api.github.com/users/ethervoid/events{/privacy}","received_events_url":"https://api.github.com/users/ethervoid/received_events","type":"User","site_admin":false},"body":"Could be nil?","created_at":"2016-09-21T16:57:07Z","updated_at":"2016-09-21T17:00:13Z","html_url":"https://github.com/CartoDB/cartodb/pull/9748#discussion_r79881393","pull_request_url":"https://api.github.com/repos/CartoDB/cartodb/pulls/9748","_links":{"self":{"href":"https://api.github.com/repos/CartoDB/cartodb/pulls/comments/79881393"},"html":{"href":"https://github.com/CartoDB/cartodb/pull/9748#discussion_r79881393"},"pull_request":{"href":"https://api.github.com/repos/CartoDB/cartodb/pulls/9748"}}},"pull_request":{"url":"https://api.github.com/repos/CartoDB/cartodb/pulls/9748","id":85067910,"html_url":"https://github.com/CartoDB/cartodb/pull/9748","diff_url":"https://github.com/CartoDB/cartodb/pull/9748.diff","patch_url":"https://github.com/CartoDB/cartodb/pull/9748.patch","issue_url":"https://api.github.com/repos/CartoDB/cartodb/issues/9748","number":9748,"state":"open","locked":false,"title":"Refactor Connector classes","user":{"login":"jgoizueta","id":5909,"avatar_url":"https://avatars.githubusercontent.com/u/5909?v=3","gravatar_id":"","url":"https://api.github.com/users/jgoizueta","html_url":"https://github.com/jgoizueta","followers_url":"https://api.github.com/users/jgoizueta/followers","following_url":"https://api.github.com/users/jgoizueta/following{/other_user}","gists_url":"https://api.github.com/users/jgoizueta/gists{/gist_id}","starred_url":"https://api.github.com/users/jgoizueta/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jgoizueta/subscriptions","organizations_url":"https://api.github.com/users/jgoizueta/orgs","repos_url":"https://api.github.com/users/jgoizueta/repos","events_url":"https://api.github.com/users/jgoizueta/events{/privacy}","received_events_url":"https://api.github.com/users/jgoizueta/received_events","type":"User","site_admin":false},"body":"The original `CartoDB::Importer2::Connector` class was taking care of two different responsibilities:\r\n\r\n* Execute the table import process using connectors (acting as a substitute for `CartoDB::Importer2::Runner`.\r\n* Implementing the connector logic (based on Providers)\r\n\r\nNow this is separated into two different classes:\r\n\r\n* `CartoDB::Importer2::ConnectorRunner` managing the import process\r\n* `Carto::Connector` handling connector operations via Providers\r\n\r\nWith this separation we can now implement connector-related operations (like list tables) which are not related to the import process in `Carto::Connector`.\r\n\r\nNote that all the Connector stuff that used to be in `services/importer/lib/importer/connector` is now in `lib/carto/connector`. This is all related to the `Carto::Connector` and will not only used from the Import API.\r\n\r\n- [x] Refactor  `Connector` -> `ConnectorRunner`, `Connector`\r\n- [x] Implement limits + app_config limits/availability (per user/provider will come later)\r\n- [x] Add `list_tables` functionality to `Connector`","created_at":"2016-09-13T15:13:06Z","updated_at":"2016-09-21T17:00:13Z","closed_at":null,"merged_at":null,"merge_commit_sha":"274e3577aae218d4d19ad15089d01aced4ab391d","assignee":{"login":"jgoizueta","id":5909,"avatar_url":"https://avatars.githubusercontent.com/u/5909?v=3","gravatar_id":"","url":"https://api.github.com/users/jgoizueta","html_url":"https://github.com/jgoizueta","followers_url":"https://api.github.com/users/jgoizueta/followers","following_url":"https://api.github.com/users/jgoizueta/following{/other_user}","gists_url":"https://api.github.com/users/jgoizueta/gists{/gist_id}","starred_url":"https://api.github.com/users/jgoizueta/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jgoizueta/subscriptions","organizations_url":"https://api.github.com/users/jgoizueta/orgs","repos_url":"https://api.github.com/users/jgoizueta/repos","events_url":"https://api.github.com/users/jgoizueta/events{/privacy}","received_events_url":"https://api.github.com/users/jgoizueta/received_events","type":"User","site_admin":false},"assignees":[{"login":"jgoizueta","id":5909,"avatar_url":"https://avatars.githubusercontent.com/u/5909?v=3","gravatar_id":"","url":"https://api.github.com/users/jgoizueta","html_url":"https://github.com/jgoizueta","followers_url":"https://api.github.com/users/jgoizueta/followers","following_url":"https://api.github.com/users/jgoizueta/following{/other_user}","gists_url":"https://api.github.com/users/jgoizueta/gists{/gist_id}","starred_url":"https://api.github.com/users/jgoizueta/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jgoizueta/subscriptions","organizations_url":"https://api.github.com/users/jgoizueta/orgs","repos_url":"https://api.github.com/users/jgoizueta/repos","events_url":"https://api.github.com/users/jgoizueta/events{/privacy}","received_events_url":"https://api.github.com/users/jgoizueta/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/CartoDB/cartodb/pulls/9748/commits","review_comments_url":"https://api.github.com/repos/CartoDB/cartodb/pulls/9748/comments","review_comment_url":"https://api.github.com/repos/CartoDB/cartodb/pulls/comments{/number}","comments_url":"https://api.github.com/repos/CartoDB/cartodb/issues/9748/comments","statuses_url":"https://api.github.com/repos/CartoDB/cartodb/statuses/bc6ba033ab0d238ba5aa05806745738e2bdc5ccb","head":{"label":"CartoDB:connector_runner","ref":"connector_runner","sha":"bc6ba033ab0d238ba5aa05806745738e2bdc5ccb","user":{"login":"CartoDB","id":1799254,"avatar_url":"https://avatars.githubusercontent.com/u/1799254?v=3","gravatar_id":"","url":"https://api.github.com/users/CartoDB","html_url":"https://github.com/CartoDB","followers_url":"https://api.github.com/users/CartoDB/followers","following_url":"https://api.github.com/users/CartoDB/following{/other_user}","gists_url":"https://api.github.com/users/CartoDB/gists{/gist_id}","starred_url":"https://api.github.com/users/CartoDB/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CartoDB/subscriptions","organizations_url":"https://api.github.com/users/CartoDB/orgs","repos_url":"https://api.github.com/users/CartoDB/repos","events_url":"https://api.github.com/users/CartoDB/events{/privacy}","received_events_url":"https://api.github.com/users/CartoDB/received_events","type":"Organization","site_admin":false},"repo":{"id":4909355,"name":"cartodb","full_name":"CartoDB/cartodb","owner":{"login":"CartoDB","id":1799254,"avatar_url":"https://avatars.githubusercontent.com/u/1799254?v=3","gravatar_id":"","url":"https://api.github.com/users/CartoDB","html_url":"https://github.com/CartoDB","followers_url":"https://api.github.com/users/CartoDB/followers","following_url":"https://api.github.com/users/CartoDB/following{/other_user}","gists_url":"https://api.github.com/users/CartoDB/gists{/gist_id}","starred_url":"https://api.github.com/users/CartoDB/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CartoDB/subscriptions","organizations_url":"https://api.github.com/users/CartoDB/orgs","repos_url":"https://api.github.com/users/CartoDB/repos","events_url":"https://api.github.com/users/CartoDB/events{/privacy}","received_events_url":"https://api.github.com/users/CartoDB/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/CartoDB/cartodb","description":"Location Intelligence & Data Visualization tool","fork":false,"url":"https://api.github.com/repos/CartoDB/cartodb","forks_url":"https://api.github.com/repos/CartoDB/cartodb/forks","keys_url":"https://api.github.com/repos/CartoDB/cartodb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CartoDB/cartodb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CartoDB/cartodb/teams","hooks_url":"https://api.github.com/repos/CartoDB/cartodb/hooks","issue_events_url":"https://api.github.com/repos/CartoDB/cartodb/issues/events{/number}","events_url":"https://api.github.com/repos/CartoDB/cartodb/events","assignees_url":"https://api.github.com/repos/CartoDB/cartodb/assignees{/user}","branches_url":"https://api.github.com/repos/CartoDB/cartodb/branches{/branch}","tags_url":"https://api.github.com/repos/CartoDB/cartodb/tags","blobs_url":"https://api.github.com/repos/CartoDB/cartodb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CartoDB/cartodb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CartoDB/cartodb/git/refs{/sha}","trees_url":"https://api.github.com/repos/CartoDB/cartodb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CartoDB/cartodb/statuses/{sha}","languages_url":"https://api.github.com/repos/CartoDB/cartodb/languages","stargazers_url":"https://api.github.com/repos/CartoDB/cartodb/stargazers","contributors_url":"https://api.github.com/repos/CartoDB/cartodb/contributors","subscribers_url":"https://api.github.com/repos/CartoDB/cartodb/subscribers","subscription_url":"https://api.github.com/repos/CartoDB/cartodb/subscription","commits_url":"https://api.github.com/repos/CartoDB/cartodb/commits{/sha}","git_commits_url":"https://api.github.com/repos/CartoDB/cartodb/git/commits{/sha}","comments_url":"https://api.github.com/repos/CartoDB/cartodb/comments{/number}","issue_comment_url":"https://api.github.com/repos/CartoDB/cartodb/issues/comments{/number}","contents_url":"https://api.github.com/repos/CartoDB/cartodb/contents/{+path}","compare_url":"https://api.github.com/repos/CartoDB/cartodb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CartoDB/cartodb/merges","archive_url":"https://api.github.com/repos/CartoDB/cartodb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CartoDB/cartodb/downloads","issues_url":"https://api.github.com/repos/CartoDB/cartodb/issues{/number}","pulls_url":"https://api.github.com/repos/CartoDB/cartodb/pulls{/number}","milestones_url":"https://api.github.com/repos/CartoDB/cartodb/milestones{/number}","notifications_url":"https://api.github.com/repos/CartoDB/cartodb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CartoDB/cartodb/labels{/name}","releases_url":"https://api.github.com/repos/CartoDB/cartodb/releases{/id}","deployments_url":"https://api.github.com/repos/CartoDB/cartodb/deployments","created_at":"2012-07-05T15:23:45Z","updated_at":"2016-09-20T15:09:54Z","pushed_at":"2016-09-21T16:28:37Z","git_url":"git://github.com/CartoDB/cartodb.git","ssh_url":"git@github.com:CartoDB/cartodb.git","clone_url":"https://github.com/CartoDB/cartodb.git","svn_url":"https://github.com/CartoDB/cartodb","homepage":"http://carto.com","size":400925,"stargazers_count":1264,"watchers_count":1264,"language":"HTML","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":324,"mirror_url":null,"open_issues_count":1098,"forks":324,"open_issues":1098,"watchers":1264,"default_branch":"master"}},"base":{"label":"CartoDB:master","ref":"master","sha":"8318daefaddbf88d55af54238f5325d64076946c","user":{"login":"CartoDB","id":1799254,"avatar_url":"https://avatars.githubusercontent.com/u/1799254?v=3","gravatar_id":"","url":"https://api.github.com/users/CartoDB","html_url":"https://github.com/CartoDB","followers_url":"https://api.github.com/users/CartoDB/followers","following_url":"https://api.github.com/users/CartoDB/following{/other_user}","gists_url":"https://api.github.com/users/CartoDB/gists{/gist_id}","starred_url":"https://api.github.com/users/CartoDB/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CartoDB/subscriptions","organizations_url":"https://api.github.com/users/CartoDB/orgs","repos_url":"https://api.github.com/users/CartoDB/repos","events_url":"https://api.github.com/users/CartoDB/events{/privacy}","received_events_url":"https://api.github.com/users/CartoDB/received_events","type":"Organization","site_admin":false},"repo":{"id":4909355,"name":"cartodb","full_name":"CartoDB/cartodb","owner":{"login":"CartoDB","id":1799254,"avatar_url":"https://avatars.githubusercontent.com/u/1799254?v=3","gravatar_id":"","url":"https://api.github.com/users/CartoDB","html_url":"https://github.com/CartoDB","followers_url":"https://api.github.com/users/CartoDB/followers","following_url":"https://api.github.com/users/CartoDB/following{/other_user}","gists_url":"https://api.github.com/users/CartoDB/gists{/gist_id}","starred_url":"https://api.github.com/users/CartoDB/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CartoDB/subscriptions","organizations_url":"https://api.github.com/users/CartoDB/orgs","repos_url":"https://api.github.com/users/CartoDB/repos","events_url":"https://api.github.com/users/CartoDB/events{/privacy}","received_events_url":"https://api.github.com/users/CartoDB/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/CartoDB/cartodb","description":"Location Intelligence & Data Visualization tool","fork":false,"url":"https://api.github.com/repos/CartoDB/cartodb","forks_url":"https://api.github.com/repos/CartoDB/cartodb/forks","keys_url":"https://api.github.com/repos/CartoDB/cartodb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CartoDB/cartodb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CartoDB/cartodb/teams","hooks_url":"https://api.github.com/repos/CartoDB/cartodb/hooks","issue_events_url":"https://api.github.com/repos/CartoDB/cartodb/issues/events{/number}","events_url":"https://api.github.com/repos/CartoDB/cartodb/events","assignees_url":"https://api.github.com/repos/CartoDB/cartodb/assignees{/user}","branches_url":"https://api.github.com/repos/CartoDB/cartodb/branches{/branch}","tags_url":"https://api.github.com/repos/CartoDB/cartodb/tags","blobs_url":"https://api.github.com/repos/CartoDB/cartodb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CartoDB/cartodb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CartoDB/cartodb/git/refs{/sha}","trees_url":"https://api.github.com/repos/CartoDB/cartodb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CartoDB/cartodb/statuses/{sha}","languages_url":"https://api.github.com/repos/CartoDB/cartodb/languages","stargazers_url":"https://api.github.com/repos/CartoDB/cartodb/stargazers","contributors_url":"https://api.github.com/repos/CartoDB/cartodb/contributors","subscribers_url":"https://api.github.com/repos/CartoDB/cartodb/subscribers","subscription_url":"https://api.github.com/repos/CartoDB/cartodb/subscription","commits_url":"https://api.github.com/repos/CartoDB/cartodb/commits{/sha}","git_commits_url":"https://api.github.com/repos/CartoDB/cartodb/git/commits{/sha}","comments_url":"https://api.github.com/repos/CartoDB/cartodb/comments{/number}","issue_comment_url":"https://api.github.com/repos/CartoDB/cartodb/issues/comments{/number}","contents_url":"https://api.github.com/repos/CartoDB/cartodb/contents/{+path}","compare_url":"https://api.github.com/repos/CartoDB/cartodb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CartoDB/cartodb/merges","archive_url":"https://api.github.com/repos/CartoDB/cartodb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CartoDB/cartodb/downloads","issues_url":"https://api.github.com/repos/CartoDB/cartodb/issues{/number}","pulls_url":"https://api.github.com/repos/CartoDB/cartodb/pulls{/number}","milestones_url":"https://api.github.com/repos/CartoDB/cartodb/milestones{/number}","notifications_url":"https://api.github.com/repos/CartoDB/cartodb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CartoDB/cartodb/labels{/name}","releases_url":"https://api.github.com/repos/CartoDB/cartodb/releases{/id}","deployments_url":"https://api.github.com/repos/CartoDB/cartodb/deployments","created_at":"2012-07-05T15:23:45Z","updated_at":"2016-09-20T15:09:54Z","pushed_at":"2016-09-21T16:28:37Z","git_url":"git://github.com/CartoDB/cartodb.git","ssh_url":"git@github.com:CartoDB/cartodb.git","clone_url":"https://github.com/CartoDB/cartodb.git","svn_url":"https://github.com/CartoDB/cartodb","homepage":"http://carto.com","size":400925,"stargazers_count":1264,"watchers_count":1264,"language":"HTML","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":324,"mirror_url":null,"open_issues_count":1098,"forks":324,"open_issues":1098,"watchers":1264,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/CartoDB/cartodb/pulls/9748"},"html":{"href":"https://github.com/CartoDB/cartodb/pull/9748"},"issue":{"href":"https://api.github.com/repos/CartoDB/cartodb/issues/9748"},"comments":{"href":"https://api.github.com/repos/CartoDB/cartodb/issues/9748/comments"},"review_comments":{"href":"https://api.github.com/repos/CartoDB/cartodb/pulls/9748/comments"},"review_comment":{"href":"https://api.github.com/repos/CartoDB/cartodb/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/CartoDB/cartodb/pulls/9748/commits"},"statuses":{"href":"https://api.github.com/repos/CartoDB/cartodb/statuses/bc6ba033ab0d238ba5aa05806745738e2bdc5ccb"}}}}
{"actor":{"display_login":"ethervoid"}}