{"action":"created","comment":{"url":"https://api.github.com/repos/aamine/bricolage/pulls/comments/55994625","id":55994625,"diff_hunk":"@@ -0,0 +1,151 @@\n+require 'bricolage/datasource'\n+require 'bricolage/commandutils'\n+require 'redis'\n+require 'json'\n+\n+module Bricolage\n+\n+  class RedisDataSource < DataSource\n+    declare_type 'redis'\n+\n+    def initialize(host: 'localhost', port: 6380, **opts)\n+      @host = host\n+      @port = port\n+      @opts = opts\n+    end\n+\n+    attr_reader :host\n+    attr_reader :port\n+    attr_reader :opts\n+\n+    def new_task\n+      RedisTask.new(self)\n+    end\n+\n+    def client\n+      @client = Redis.new(:host => @host, :port => @port, **@opts) unless @client\n+      @client\n+    end\n+  end\n+\n+  class RedisTask < DataSourceTask\n+    def import(src, table, query, key, prefix, encode, delete: false)\n+      add Import.new(src, table, query, key, encode, delete)\n+    end\n+\n+    class Import < Action\n+      def initialize(src, table, query, key, encode, delete)\n+        @src = src\n+        @table = table\n+        @query = query\n+        @key = key\n+        @encode = encode\n+        @delete = delete\n+        @row_count = 0\n+        @success_value = encode_to_hash? ? true : \"OK\"\n+      end\n+\n+      def bind(*args)\n+        @query.bind(*args)\n+      end\n+\n+      def source\n+        @query.stripped_source\n+      end\n+\n+      def prefix\n+        return @prefix unless @prefix.nil?\n+        @prefix = \"#{@table.last.name}_#{@table.last.schema}_\"\n+        ds.logger.info \"Key Pattern: #{@prefix}<#{@key}>\"\n+        @prefix\n+      end\n+\n+      def key_pattern\n+        prefix + \"*\"\n+      end\n+\n+      def encode_to_hash?\n+        @encode == 'hash'\n+      end\n+\n+      def log_write_result(write_futures)\n+        count = 0\n+        dup_count = 0\n+        write_futures.each do |f|\n+          count, dup_count = count_result(f.value, @success_value, count, dup_count)\n+        end\n+        # divide by column_count when hash (which returns row * columns count of results)\n+        count, dup_count = count_hash_rows(count, dup_count) if encode_to_hash?\n+        ds.logger.info \"keys already exist: #{dup_count}\"\n+        ds.logger.info \"keys written: #{count}\"\n+      end\n+\n+      def count_result(result, success_value, count, dup_count)\n+        if result == success_value\n+          count += 1\n+        else\n+          dup_count += 1\n+        end\n+        return count, dup_count\n+      end\n+\n+      def count_hash_rows(count, dup_count)\n+        return count / @column_count, dup_count / @column_count\n+      end\n+\n+      def delete(keys)\n+        ds.logger.info \"Delete: #{key_pattern}\"\n+        ds.client.del(*keys)\n+      end\n+\n+      def write\n+        futures = []\n+        @src.execute_query(source) do |rs|\n+          rs.each do |row|\n+            futures.push(set @encode, prefix+row['id'], row)\n+            @row_count += 1\n+          end\n+          # save number of columns for log_writer_result()\n+          @column_count = rs.first.size unless @column_count\n+        end\n+        futures.flatten\n+      end\n+\n+      def set(type, key, row)\n+        # write only when key does not exist\n+        fs = []\n+        case type\n+        when 'hash'\n+          # set a value for each key:field pair\n+          row.each do |field,value|\n+            fs.push(ds.client.hsetnx key, field, value)\n+          end\n+        when 'json'\n+          fs.push(ds.client.setnx key, JSON.generate(row))","path":"lib/bricolage/redisdatasource.rb","position":124,"original_position":124,"commit_id":"687dd98be4204e959e33acbc6aa604f2527f2399","original_commit_id":"687dd98be4204e959e33acbc6aa604f2527f2399","user":{"login":"aamine","id":1703556,"avatar_url":"https://avatars.githubusercontent.com/u/1703556?v=3","gravatar_id":"","url":"https://api.github.com/users/aamine","html_url":"https://github.com/aamine","followers_url":"https://api.github.com/users/aamine/followers","following_url":"https://api.github.com/users/aamine/following{/other_user}","gists_url":"https://api.github.com/users/aamine/gists{/gist_id}","starred_url":"https://api.github.com/users/aamine/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aamine/subscriptions","organizations_url":"https://api.github.com/users/aamine/orgs","repos_url":"https://api.github.com/users/aamine/repos","events_url":"https://api.github.com/users/aamine/events{/privacy}","received_events_url":"https://api.github.com/users/aamine/received_events","type":"User","site_admin":false},"body":"このクソみたいなカッコ省略はどこから持ってきたわけ？","created_at":"2016-03-14T12:47:34Z","updated_at":"2016-03-14T12:47:34Z","html_url":"https://github.com/aamine/bricolage/pull/54#discussion_r55994625","pull_request_url":"https://api.github.com/repos/aamine/bricolage/pulls/54","_links":{"self":{"href":"https://api.github.com/repos/aamine/bricolage/pulls/comments/55994625"},"html":{"href":"https://github.com/aamine/bricolage/pull/54#discussion_r55994625"},"pull_request":{"href":"https://api.github.com/repos/aamine/bricolage/pulls/54"}}},"pull_request":{"url":"https://api.github.com/repos/aamine/bricolage/pulls/54","id":62710578,"html_url":"https://github.com/aamine/bricolage/pull/54","diff_url":"https://github.com/aamine/bricolage/pull/54.diff","patch_url":"https://github.com/aamine/bricolage/pull/54.patch","issue_url":"https://api.github.com/repos/aamine/bricolage/issues/54","number":54,"state":"open","locked":false,"title":"Support export from PSQLDataSource to Redis","user":{"login":"shimpeko","id":137599,"avatar_url":"https://avatars.githubusercontent.com/u/137599?v=3","gravatar_id":"","url":"https://api.github.com/users/shimpeko","html_url":"https://github.com/shimpeko","followers_url":"https://api.github.com/users/shimpeko/followers","following_url":"https://api.github.com/users/shimpeko/following{/other_user}","gists_url":"https://api.github.com/users/shimpeko/gists{/gist_id}","starred_url":"https://api.github.com/users/shimpeko/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shimpeko/subscriptions","organizations_url":"https://api.github.com/users/shimpeko/orgs","repos_url":"https://api.github.com/users/shimpeko/repos","events_url":"https://api.github.com/users/shimpeko/events{/privacy}","received_events_url":"https://api.github.com/users/shimpeko/received_events","type":"User","site_admin":false},"body":"Create redis-export jobclass to support export from PSQLDataSource to Redis.\r\nRedisDataSource class is also added to define connection information to Redis cluster. \r\n\r\n# redis-export parameter\r\n- src-ds: source data source name (type: psql)\r\n- src-tables: source table (only 1 tables is allowed, used for Redis's key prefix)\r\n- sql-file: source query (optional)\r\n- dest-ds: destination data source name (type: redis)\r\n- key_column: column to use for Redis object key (default: \"id\")\r\n- prefix: Redis object key prefix (default: \"schemaname_tablename_\")\r\n- delete: whether to delete existing key's\r\n  - existing key's will not be overwritten with out delete option.\r\n\r\n# RedisDataSource parameter\r\nsee http://www.rubydoc.info/github/redis/redis-rb/Redis%3Ainitialize\r\n","created_at":"2016-03-14T07:51:31Z","updated_at":"2016-03-14T12:47:34Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7e06065897de25f033094f7d3607dd9dd16837a7","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/aamine/bricolage/pulls/54/commits","review_comments_url":"https://api.github.com/repos/aamine/bricolage/pulls/54/comments","review_comment_url":"https://api.github.com/repos/aamine/bricolage/pulls/comments{/number}","comments_url":"https://api.github.com/repos/aamine/bricolage/issues/54/comments","statuses_url":"https://api.github.com/repos/aamine/bricolage/statuses/687dd98be4204e959e33acbc6aa604f2527f2399","head":{"label":"shimpeko:redis_export","ref":"redis_export","sha":"687dd98be4204e959e33acbc6aa604f2527f2399","user":{"login":"shimpeko","id":137599,"avatar_url":"https://avatars.githubusercontent.com/u/137599?v=3","gravatar_id":"","url":"https://api.github.com/users/shimpeko","html_url":"https://github.com/shimpeko","followers_url":"https://api.github.com/users/shimpeko/followers","following_url":"https://api.github.com/users/shimpeko/following{/other_user}","gists_url":"https://api.github.com/users/shimpeko/gists{/gist_id}","starred_url":"https://api.github.com/users/shimpeko/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shimpeko/subscriptions","organizations_url":"https://api.github.com/users/shimpeko/orgs","repos_url":"https://api.github.com/users/shimpeko/repos","events_url":"https://api.github.com/users/shimpeko/events{/privacy}","received_events_url":"https://api.github.com/users/shimpeko/received_events","type":"User","site_admin":false},"repo":{"id":48536735,"name":"bricolage","full_name":"shimpeko/bricolage","owner":{"login":"shimpeko","id":137599,"avatar_url":"https://avatars.githubusercontent.com/u/137599?v=3","gravatar_id":"","url":"https://api.github.com/users/shimpeko","html_url":"https://github.com/shimpeko","followers_url":"https://api.github.com/users/shimpeko/followers","following_url":"https://api.github.com/users/shimpeko/following{/other_user}","gists_url":"https://api.github.com/users/shimpeko/gists{/gist_id}","starred_url":"https://api.github.com/users/shimpeko/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shimpeko/subscriptions","organizations_url":"https://api.github.com/users/shimpeko/orgs","repos_url":"https://api.github.com/users/shimpeko/repos","events_url":"https://api.github.com/users/shimpeko/events{/privacy}","received_events_url":"https://api.github.com/users/shimpeko/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/shimpeko/bricolage","description":"SQL batch framework for Redshift","fork":true,"url":"https://api.github.com/repos/shimpeko/bricolage","forks_url":"https://api.github.com/repos/shimpeko/bricolage/forks","keys_url":"https://api.github.com/repos/shimpeko/bricolage/keys{/key_id}","collaborators_url":"https://api.github.com/repos/shimpeko/bricolage/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/shimpeko/bricolage/teams","hooks_url":"https://api.github.com/repos/shimpeko/bricolage/hooks","issue_events_url":"https://api.github.com/repos/shimpeko/bricolage/issues/events{/number}","events_url":"https://api.github.com/repos/shimpeko/bricolage/events","assignees_url":"https://api.github.com/repos/shimpeko/bricolage/assignees{/user}","branches_url":"https://api.github.com/repos/shimpeko/bricolage/branches{/branch}","tags_url":"https://api.github.com/repos/shimpeko/bricolage/tags","blobs_url":"https://api.github.com/repos/shimpeko/bricolage/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/shimpeko/bricolage/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/shimpeko/bricolage/git/refs{/sha}","trees_url":"https://api.github.com/repos/shimpeko/bricolage/git/trees{/sha}","statuses_url":"https://api.github.com/repos/shimpeko/bricolage/statuses/{sha}","languages_url":"https://api.github.com/repos/shimpeko/bricolage/languages","stargazers_url":"https://api.github.com/repos/shimpeko/bricolage/stargazers","contributors_url":"https://api.github.com/repos/shimpeko/bricolage/contributors","subscribers_url":"https://api.github.com/repos/shimpeko/bricolage/subscribers","subscription_url":"https://api.github.com/repos/shimpeko/bricolage/subscription","commits_url":"https://api.github.com/repos/shimpeko/bricolage/commits{/sha}","git_commits_url":"https://api.github.com/repos/shimpeko/bricolage/git/commits{/sha}","comments_url":"https://api.github.com/repos/shimpeko/bricolage/comments{/number}","issue_comment_url":"https://api.github.com/repos/shimpeko/bricolage/issues/comments{/number}","contents_url":"https://api.github.com/repos/shimpeko/bricolage/contents/{+path}","compare_url":"https://api.github.com/repos/shimpeko/bricolage/compare/{base}...{head}","merges_url":"https://api.github.com/repos/shimpeko/bricolage/merges","archive_url":"https://api.github.com/repos/shimpeko/bricolage/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/shimpeko/bricolage/downloads","issues_url":"https://api.github.com/repos/shimpeko/bricolage/issues{/number}","pulls_url":"https://api.github.com/repos/shimpeko/bricolage/pulls{/number}","milestones_url":"https://api.github.com/repos/shimpeko/bricolage/milestones{/number}","notifications_url":"https://api.github.com/repos/shimpeko/bricolage/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/shimpeko/bricolage/labels{/name}","releases_url":"https://api.github.com/repos/shimpeko/bricolage/releases{/id}","deployments_url":"https://api.github.com/repos/shimpeko/bricolage/deployments","created_at":"2015-12-24T09:46:25Z","updated_at":"2015-12-24T09:46:26Z","pushed_at":"2016-03-14T10:11:01Z","git_url":"git://github.com/shimpeko/bricolage.git","ssh_url":"git@github.com:shimpeko/bricolage.git","clone_url":"https://github.com/shimpeko/bricolage.git","svn_url":"https://github.com/shimpeko/bricolage","homepage":"","size":5245,"stargazers_count":0,"watchers_count":0,"language":"Ruby","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"aamine:master","ref":"master","sha":"7610894865f088551614ac01b9df7dd547ccfc87","user":{"login":"aamine","id":1703556,"avatar_url":"https://avatars.githubusercontent.com/u/1703556?v=3","gravatar_id":"","url":"https://api.github.com/users/aamine","html_url":"https://github.com/aamine","followers_url":"https://api.github.com/users/aamine/followers","following_url":"https://api.github.com/users/aamine/following{/other_user}","gists_url":"https://api.github.com/users/aamine/gists{/gist_id}","starred_url":"https://api.github.com/users/aamine/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aamine/subscriptions","organizations_url":"https://api.github.com/users/aamine/orgs","repos_url":"https://api.github.com/users/aamine/repos","events_url":"https://api.github.com/users/aamine/events{/privacy}","received_events_url":"https://api.github.com/users/aamine/received_events","type":"User","site_admin":false},"repo":{"id":36477893,"name":"bricolage","full_name":"aamine/bricolage","owner":{"login":"aamine","id":1703556,"avatar_url":"https://avatars.githubusercontent.com/u/1703556?v=3","gravatar_id":"","url":"https://api.github.com/users/aamine","html_url":"https://github.com/aamine","followers_url":"https://api.github.com/users/aamine/followers","following_url":"https://api.github.com/users/aamine/following{/other_user}","gists_url":"https://api.github.com/users/aamine/gists{/gist_id}","starred_url":"https://api.github.com/users/aamine/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aamine/subscriptions","organizations_url":"https://api.github.com/users/aamine/orgs","repos_url":"https://api.github.com/users/aamine/repos","events_url":"https://api.github.com/users/aamine/events{/privacy}","received_events_url":"https://api.github.com/users/aamine/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/aamine/bricolage","description":"AWS-oriented Data Warehouse Framework","fork":false,"url":"https://api.github.com/repos/aamine/bricolage","forks_url":"https://api.github.com/repos/aamine/bricolage/forks","keys_url":"https://api.github.com/repos/aamine/bricolage/keys{/key_id}","collaborators_url":"https://api.github.com/repos/aamine/bricolage/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/aamine/bricolage/teams","hooks_url":"https://api.github.com/repos/aamine/bricolage/hooks","issue_events_url":"https://api.github.com/repos/aamine/bricolage/issues/events{/number}","events_url":"https://api.github.com/repos/aamine/bricolage/events","assignees_url":"https://api.github.com/repos/aamine/bricolage/assignees{/user}","branches_url":"https://api.github.com/repos/aamine/bricolage/branches{/branch}","tags_url":"https://api.github.com/repos/aamine/bricolage/tags","blobs_url":"https://api.github.com/repos/aamine/bricolage/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/aamine/bricolage/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/aamine/bricolage/git/refs{/sha}","trees_url":"https://api.github.com/repos/aamine/bricolage/git/trees{/sha}","statuses_url":"https://api.github.com/repos/aamine/bricolage/statuses/{sha}","languages_url":"https://api.github.com/repos/aamine/bricolage/languages","stargazers_url":"https://api.github.com/repos/aamine/bricolage/stargazers","contributors_url":"https://api.github.com/repos/aamine/bricolage/contributors","subscribers_url":"https://api.github.com/repos/aamine/bricolage/subscribers","subscription_url":"https://api.github.com/repos/aamine/bricolage/subscription","commits_url":"https://api.github.com/repos/aamine/bricolage/commits{/sha}","git_commits_url":"https://api.github.com/repos/aamine/bricolage/git/commits{/sha}","comments_url":"https://api.github.com/repos/aamine/bricolage/comments{/number}","issue_comment_url":"https://api.github.com/repos/aamine/bricolage/issues/comments{/number}","contents_url":"https://api.github.com/repos/aamine/bricolage/contents/{+path}","compare_url":"https://api.github.com/repos/aamine/bricolage/compare/{base}...{head}","merges_url":"https://api.github.com/repos/aamine/bricolage/merges","archive_url":"https://api.github.com/repos/aamine/bricolage/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/aamine/bricolage/downloads","issues_url":"https://api.github.com/repos/aamine/bricolage/issues{/number}","pulls_url":"https://api.github.com/repos/aamine/bricolage/pulls{/number}","milestones_url":"https://api.github.com/repos/aamine/bricolage/milestones{/number}","notifications_url":"https://api.github.com/repos/aamine/bricolage/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/aamine/bricolage/labels{/name}","releases_url":"https://api.github.com/repos/aamine/bricolage/releases{/id}","deployments_url":"https://api.github.com/repos/aamine/bricolage/deployments","created_at":"2015-05-29T02:18:39Z","updated_at":"2016-03-14T07:57:31Z","pushed_at":"2016-03-14T10:11:01Z","git_url":"git://github.com/aamine/bricolage.git","ssh_url":"git@github.com:aamine/bricolage.git","clone_url":"https://github.com/aamine/bricolage.git","svn_url":"https://github.com/aamine/bricolage","homepage":"","size":5252,"stargazers_count":53,"watchers_count":53,"language":"Ruby","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":6,"mirror_url":null,"open_issues_count":30,"forks":6,"open_issues":30,"watchers":53,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/aamine/bricolage/pulls/54"},"html":{"href":"https://github.com/aamine/bricolage/pull/54"},"issue":{"href":"https://api.github.com/repos/aamine/bricolage/issues/54"},"comments":{"href":"https://api.github.com/repos/aamine/bricolage/issues/54/comments"},"review_comments":{"href":"https://api.github.com/repos/aamine/bricolage/pulls/54/comments"},"review_comment":{"href":"https://api.github.com/repos/aamine/bricolage/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/aamine/bricolage/pulls/54/commits"},"statuses":{"href":"https://api.github.com/repos/aamine/bricolage/statuses/687dd98be4204e959e33acbc6aa604f2527f2399"}}}}