{"action":"created","comment":{"url":"https://api.github.com/repos/pinterest/mysql_utils/pulls/comments/75763086","id":75763086,"diff_hunk":"@@ -0,0 +1,173 @@\n+//!    This program has been created to handle an incompatibility between how mysqldump escapes\n+//!    some characters and how hive interprets those escaped chars. It does the following:\n+//!    If you see an 0x5c30 in the input sequence\n+//!    a. and there is no or even number of 0x5c before 0x5c30, translate this 0x5c30 to 0x00\n+//!    b. if there is odd number of 0x5c before 0x5c30, don't do anything.\n+//!    Some sample transforms:\n+//!      0x5c30 => 0x00\n+//!      0x5c5c30 => 0x5c5c30\n+//!      0x5c5c5c30 => 0x5c5c00\n+//!      0x5c5c5c5c30 => 0x5c5c5c5c30\n+//!      0x5c5c5c3030 => 0x5c5c0030\n+//!      0x5c5c5c5c3030 => 0x5c5c5c5c3030\n+//!      0x5c5c5c40 => 0x5c5c5c40\n+//!      0x5c5c5c5c40 => 0x5c5c5c5c40\n+//!    Here is another way to test:\n+//!    - Create table with blob content:  create table MyTest (id integer, value1 varchar(20),\n+//!      content blob, value2 double, primary key(id));\n+//!    - Insert into blob content:  insert into MyTest (id, value1, content, value2)\n+//!      values (1, \"data1\", 0x3020090d0a2227005c30, 2.2);\n+//!    - checking content: select hex(content) from MyTest;\n+//!    - chmod a+rw /tmp/dump\n+//!    - mysqldump -u root --tab=/tmp/dump --single-transaction -- create-options test\n+//!    - see content:  hexdump /tmp/dump/MyTest.txt\n+//!    hexdump of original dump file:\n+//!    0000000 31 09 64 61 74 61 31 09 30 20 5c 09 0d 5c 0a 22\n+//!    0000010 27 5c 30 5c 5c 30 09 32 2e 32 0a\n+//!    000001b\n+//!    hexdump after passing through this program:\n+//!    0000000 31 09 64 61 74 61 31 09 30 20 5c 09 0d 5c 0a 22\n+//!    0000010 27 00 5c 5c 30 09 32 2e 32 0a\n+//!    000001a\n+//!    Author : vamsi Nov 2015\n+\n+#[macro_use]\n+extern crate log;\n+extern crate simplelog;\n+\n+use simplelog::SimpleLogger;\n+use simplelog::LogLevelFilter;\n+\n+use std::io::{BufReader, BufWriter};\n+use std::io;\n+use std::io::ErrorKind;\n+use std::io::prelude::*;\n+\n+#[test]\n+fn test_transform() {\n+    // 0x5c30 => 0x00\n+    let test1 = &[0x5c, 0x30];\n+    let mut input = std::io::Cursor::new(test1);\n+    let mut output: Vec<u8> = Vec::new();\n+    process_input(&mut input, &mut output);\n+    println!(\"output: {:?}\", output);\n+    assert_eq!(output, vec![0x00]);\n+\n+    // TODO: Broken\n+    // 0x5c5c30 => 0x5c5c30 //Broken\n+    let test2 = &[0x5c, 0x5c, 0x30];\n+    let mut input2 = std::io::Cursor::new(test2);\n+    let mut output2: Vec<u8> = Vec::new();\n+    process_input(&mut input2, &mut output2);\n+    println!(\"Output2: {:?}\", output2);\n+    assert_eq!(output2, vec![0x5c, 0x5c, 0x30]);\n+\n+    // 0x5c5c5c30 => 0x5c5c00\n+    let test3 = &[0x5c, 0x5c, 0x5c, 0x30];\n+    let mut input3 = std::io::Cursor::new(test3);\n+    let mut output3: Vec<u8> = Vec::new();\n+    process_input(&mut input3, &mut output3);\n+    println!(\"output3: {:?}\", output3);\n+    assert_eq!(output3, vec![0x5c, 0x5c, 0x00]);\n+\n+    // 0x5c5c5c5c30 => 0x5c5c5c5c30\n+    let test4 = &[0x5c, 0x5c, 0x5c, 0x5c, 0x30];\n+    let mut input4 = std::io::Cursor::new(test4);\n+    let mut output4: Vec<u8> = Vec::new();\n+    process_input(&mut input4, &mut output4);\n+    println!(\"output4: {:?}\", output4);\n+    assert_eq!(output4, vec![0x5c, 0x5c, 0x5c, 0x5c, 0x30]);\n+\n+    // 0x5c5c5c3030 => 0x5c5c0030\n+    let test5 = &[0x5c, 0x5c, 0x5c, 0x30, 0x30];\n+    let mut input5 = std::io::Cursor::new(test5);\n+    let mut output5: Vec<u8> = Vec::new();\n+    process_input(&mut input5, &mut output5);\n+    println!(\"output5: {:?}\", output5);\n+    assert_eq!(output5, vec![0x5c, 0x5c, 0x00, 0x30]);\n+","path":"null-escape/src/main.rs","position":88,"original_position":88,"commit_id":"9afdc2b4e2597cc46e5fb009a8407874fd33bf78","original_commit_id":"9afdc2b4e2597cc46e5fb009a8407874fd33bf78","user":{"login":"jparise","id":10311,"avatar_url":"https://avatars.githubusercontent.com/u/10311?v=3","gravatar_id":"","url":"https://api.github.com/users/jparise","html_url":"https://github.com/jparise","followers_url":"https://api.github.com/users/jparise/followers","following_url":"https://api.github.com/users/jparise/following{/other_user}","gists_url":"https://api.github.com/users/jparise/gists{/gist_id}","starred_url":"https://api.github.com/users/jparise/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jparise/subscriptions","organizations_url":"https://api.github.com/users/jparise/orgs","repos_url":"https://api.github.com/users/jparise/repos","events_url":"https://api.github.com/users/jparise/events{/privacy}","received_events_url":"https://api.github.com/users/jparise/received_events","type":"User","site_admin":false},"body":"Stray newline","created_at":"2016-08-22T21:31:51Z","updated_at":"2016-08-22T21:31:51Z","html_url":"https://github.com/pinterest/mysql_utils/pull/7#discussion_r75763086","pull_request_url":"https://api.github.com/repos/pinterest/mysql_utils/pulls/7","_links":{"self":{"href":"https://api.github.com/repos/pinterest/mysql_utils/pulls/comments/75763086"},"html":{"href":"https://github.com/pinterest/mysql_utils/pull/7#discussion_r75763086"},"pull_request":{"href":"https://api.github.com/repos/pinterest/mysql_utils/pulls/7"}}},"pull_request":{"url":"https://api.github.com/repos/pinterest/mysql_utils/pulls/7","id":81854460,"html_url":"https://github.com/pinterest/mysql_utils/pull/7","diff_url":"https://github.com/pinterest/mysql_utils/pull/7.diff","patch_url":"https://github.com/pinterest/mysql_utils/pull/7.patch","issue_url":"https://api.github.com/repos/pinterest/mysql_utils/issues/7","number":7,"state":"open","locked":false,"title":"Let me Rust that for you","user":{"login":"cholcombe973","id":3058195,"avatar_url":"https://avatars.githubusercontent.com/u/3058195?v=3","gravatar_id":"","url":"https://api.github.com/users/cholcombe973","html_url":"https://github.com/cholcombe973","followers_url":"https://api.github.com/users/cholcombe973/followers","following_url":"https://api.github.com/users/cholcombe973/following{/other_user}","gists_url":"https://api.github.com/users/cholcombe973/gists{/gist_id}","starred_url":"https://api.github.com/users/cholcombe973/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cholcombe973/subscriptions","organizations_url":"https://api.github.com/users/cholcombe973/orgs","repos_url":"https://api.github.com/users/cholcombe973/repos","events_url":"https://api.github.com/users/cholcombe973/events{/privacy}","received_events_url":"https://api.github.com/users/cholcombe973/received_events","type":"User","site_admin":false},"body":"I saw your blog post the other day and thought this would be an exciting opportunity to write some Rust code.  This program as far as I can tell produces identical output to NullEscape.c.  \r\n\r\nIn benchmarking after dropping caches and inserting length 1800 into the content field by following the example in the comments the Rust program is roughly 2x as fast.  I haven't tried to perform any optimization yet.  a.out is the original NullEscape.c with gcc -O3.  null-escape is the Rust program compiled with --release.\r\n```\r\nroot@ip-172-31-23-110:~# echo 3 > /proc/sys/vm/drop_caches\r\nroot@ip-172-31-23-110:~# echo 2 > /proc/sys/vm/drop_caches\r\nroot@ip-172-31-23-110:~# echo 1 > /proc/sys/vm/drop_caches\r\nroot@ip-172-31-23-110:~# time cat /tmp/dump/MyTest.txt | ./a.out > test1\r\n\r\nreal\t0m0.193s\r\nuser\t0m0.003s\r\nsys\t0m0.004s\r\nroot@ip-172-31-23-110:~# echo 3 > /proc/sys/vm/drop_caches\r\nroot@ip-172-31-23-110:~# echo 2 > /proc/sys/vm/drop_caches\r\nroot@ip-172-31-23-110:~# echo 1 > /proc/sys/vm/drop_caches\r\nroot@ip-172-31-23-110:~# time cat /tmp/dump/MyTest.txt | ./null-escape > test1\r\n\r\nreal\t0m0.068s\r\nuser\t0m0.000s\r\nsys\t0m0.007s\r\n```\r\nI reran these benchmarks several times and got similar results.  I don't know how big of a dataset you're running through this tool so it's possible at much larger values the advantage goes away.  In any event though the Rust program is memory safe among other things :)\r\nIf there's interest in using this tool I'd be happy to clean it up a bit so there's no unwraps().  Those can panic.  ","created_at":"2016-08-18T17:20:46Z","updated_at":"2016-08-22T21:31:51Z","closed_at":null,"merged_at":null,"merge_commit_sha":"14cc00a329d94e732b979f5bc1310f8ee74f9502","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/pinterest/mysql_utils/pulls/7/commits","review_comments_url":"https://api.github.com/repos/pinterest/mysql_utils/pulls/7/comments","review_comment_url":"https://api.github.com/repos/pinterest/mysql_utils/pulls/comments{/number}","comments_url":"https://api.github.com/repos/pinterest/mysql_utils/issues/7/comments","statuses_url":"https://api.github.com/repos/pinterest/mysql_utils/statuses/9afdc2b4e2597cc46e5fb009a8407874fd33bf78","head":{"label":"cholcombe973:master","ref":"master","sha":"9afdc2b4e2597cc46e5fb009a8407874fd33bf78","user":{"login":"cholcombe973","id":3058195,"avatar_url":"https://avatars.githubusercontent.com/u/3058195?v=3","gravatar_id":"","url":"https://api.github.com/users/cholcombe973","html_url":"https://github.com/cholcombe973","followers_url":"https://api.github.com/users/cholcombe973/followers","following_url":"https://api.github.com/users/cholcombe973/following{/other_user}","gists_url":"https://api.github.com/users/cholcombe973/gists{/gist_id}","starred_url":"https://api.github.com/users/cholcombe973/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cholcombe973/subscriptions","organizations_url":"https://api.github.com/users/cholcombe973/orgs","repos_url":"https://api.github.com/users/cholcombe973/repos","events_url":"https://api.github.com/users/cholcombe973/events{/privacy}","received_events_url":"https://api.github.com/users/cholcombe973/received_events","type":"User","site_admin":false},"repo":{"id":65859200,"name":"mysql_utils","full_name":"cholcombe973/mysql_utils","owner":{"login":"cholcombe973","id":3058195,"avatar_url":"https://avatars.githubusercontent.com/u/3058195?v=3","gravatar_id":"","url":"https://api.github.com/users/cholcombe973","html_url":"https://github.com/cholcombe973","followers_url":"https://api.github.com/users/cholcombe973/followers","following_url":"https://api.github.com/users/cholcombe973/following{/other_user}","gists_url":"https://api.github.com/users/cholcombe973/gists{/gist_id}","starred_url":"https://api.github.com/users/cholcombe973/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cholcombe973/subscriptions","organizations_url":"https://api.github.com/users/cholcombe973/orgs","repos_url":"https://api.github.com/users/cholcombe973/repos","events_url":"https://api.github.com/users/cholcombe973/events{/privacy}","received_events_url":"https://api.github.com/users/cholcombe973/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/cholcombe973/mysql_utils","description":"Pinterest MySQL Management Tools","fork":true,"url":"https://api.github.com/repos/cholcombe973/mysql_utils","forks_url":"https://api.github.com/repos/cholcombe973/mysql_utils/forks","keys_url":"https://api.github.com/repos/cholcombe973/mysql_utils/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cholcombe973/mysql_utils/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cholcombe973/mysql_utils/teams","hooks_url":"https://api.github.com/repos/cholcombe973/mysql_utils/hooks","issue_events_url":"https://api.github.com/repos/cholcombe973/mysql_utils/issues/events{/number}","events_url":"https://api.github.com/repos/cholcombe973/mysql_utils/events","assignees_url":"https://api.github.com/repos/cholcombe973/mysql_utils/assignees{/user}","branches_url":"https://api.github.com/repos/cholcombe973/mysql_utils/branches{/branch}","tags_url":"https://api.github.com/repos/cholcombe973/mysql_utils/tags","blobs_url":"https://api.github.com/repos/cholcombe973/mysql_utils/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cholcombe973/mysql_utils/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cholcombe973/mysql_utils/git/refs{/sha}","trees_url":"https://api.github.com/repos/cholcombe973/mysql_utils/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cholcombe973/mysql_utils/statuses/{sha}","languages_url":"https://api.github.com/repos/cholcombe973/mysql_utils/languages","stargazers_url":"https://api.github.com/repos/cholcombe973/mysql_utils/stargazers","contributors_url":"https://api.github.com/repos/cholcombe973/mysql_utils/contributors","subscribers_url":"https://api.github.com/repos/cholcombe973/mysql_utils/subscribers","subscription_url":"https://api.github.com/repos/cholcombe973/mysql_utils/subscription","commits_url":"https://api.github.com/repos/cholcombe973/mysql_utils/commits{/sha}","git_commits_url":"https://api.github.com/repos/cholcombe973/mysql_utils/git/commits{/sha}","comments_url":"https://api.github.com/repos/cholcombe973/mysql_utils/comments{/number}","issue_comment_url":"https://api.github.com/repos/cholcombe973/mysql_utils/issues/comments{/number}","contents_url":"https://api.github.com/repos/cholcombe973/mysql_utils/contents/{+path}","compare_url":"https://api.github.com/repos/cholcombe973/mysql_utils/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cholcombe973/mysql_utils/merges","archive_url":"https://api.github.com/repos/cholcombe973/mysql_utils/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cholcombe973/mysql_utils/downloads","issues_url":"https://api.github.com/repos/cholcombe973/mysql_utils/issues{/number}","pulls_url":"https://api.github.com/repos/cholcombe973/mysql_utils/pulls{/number}","milestones_url":"https://api.github.com/repos/cholcombe973/mysql_utils/milestones{/number}","notifications_url":"https://api.github.com/repos/cholcombe973/mysql_utils/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cholcombe973/mysql_utils/labels{/name}","releases_url":"https://api.github.com/repos/cholcombe973/mysql_utils/releases{/id}","deployments_url":"https://api.github.com/repos/cholcombe973/mysql_utils/deployments","created_at":"2016-08-16T22:59:03Z","updated_at":"2016-08-16T22:59:04Z","pushed_at":"2016-08-21T00:01:12Z","git_url":"git://github.com/cholcombe973/mysql_utils.git","ssh_url":"git@github.com:cholcombe973/mysql_utils.git","clone_url":"https://github.com/cholcombe973/mysql_utils.git","svn_url":"https://github.com/cholcombe973/mysql_utils","homepage":"","size":210,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"pinterest:master","ref":"master","sha":"5f22c2452aeb48566f373cc22017a2e8e8bd846f","user":{"login":"pinterest","id":541152,"avatar_url":"https://avatars.githubusercontent.com/u/541152?v=3","gravatar_id":"","url":"https://api.github.com/users/pinterest","html_url":"https://github.com/pinterest","followers_url":"https://api.github.com/users/pinterest/followers","following_url":"https://api.github.com/users/pinterest/following{/other_user}","gists_url":"https://api.github.com/users/pinterest/gists{/gist_id}","starred_url":"https://api.github.com/users/pinterest/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pinterest/subscriptions","organizations_url":"https://api.github.com/users/pinterest/orgs","repos_url":"https://api.github.com/users/pinterest/repos","events_url":"https://api.github.com/users/pinterest/events{/privacy}","received_events_url":"https://api.github.com/users/pinterest/received_events","type":"Organization","site_admin":false},"repo":{"id":44876935,"name":"mysql_utils","full_name":"pinterest/mysql_utils","owner":{"login":"pinterest","id":541152,"avatar_url":"https://avatars.githubusercontent.com/u/541152?v=3","gravatar_id":"","url":"https://api.github.com/users/pinterest","html_url":"https://github.com/pinterest","followers_url":"https://api.github.com/users/pinterest/followers","following_url":"https://api.github.com/users/pinterest/following{/other_user}","gists_url":"https://api.github.com/users/pinterest/gists{/gist_id}","starred_url":"https://api.github.com/users/pinterest/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pinterest/subscriptions","organizations_url":"https://api.github.com/users/pinterest/orgs","repos_url":"https://api.github.com/users/pinterest/repos","events_url":"https://api.github.com/users/pinterest/events{/privacy}","received_events_url":"https://api.github.com/users/pinterest/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/pinterest/mysql_utils","description":"Pinterest MySQL Management Tools","fork":false,"url":"https://api.github.com/repos/pinterest/mysql_utils","forks_url":"https://api.github.com/repos/pinterest/mysql_utils/forks","keys_url":"https://api.github.com/repos/pinterest/mysql_utils/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pinterest/mysql_utils/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pinterest/mysql_utils/teams","hooks_url":"https://api.github.com/repos/pinterest/mysql_utils/hooks","issue_events_url":"https://api.github.com/repos/pinterest/mysql_utils/issues/events{/number}","events_url":"https://api.github.com/repos/pinterest/mysql_utils/events","assignees_url":"https://api.github.com/repos/pinterest/mysql_utils/assignees{/user}","branches_url":"https://api.github.com/repos/pinterest/mysql_utils/branches{/branch}","tags_url":"https://api.github.com/repos/pinterest/mysql_utils/tags","blobs_url":"https://api.github.com/repos/pinterest/mysql_utils/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pinterest/mysql_utils/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pinterest/mysql_utils/git/refs{/sha}","trees_url":"https://api.github.com/repos/pinterest/mysql_utils/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pinterest/mysql_utils/statuses/{sha}","languages_url":"https://api.github.com/repos/pinterest/mysql_utils/languages","stargazers_url":"https://api.github.com/repos/pinterest/mysql_utils/stargazers","contributors_url":"https://api.github.com/repos/pinterest/mysql_utils/contributors","subscribers_url":"https://api.github.com/repos/pinterest/mysql_utils/subscribers","subscription_url":"https://api.github.com/repos/pinterest/mysql_utils/subscription","commits_url":"https://api.github.com/repos/pinterest/mysql_utils/commits{/sha}","git_commits_url":"https://api.github.com/repos/pinterest/mysql_utils/git/commits{/sha}","comments_url":"https://api.github.com/repos/pinterest/mysql_utils/comments{/number}","issue_comment_url":"https://api.github.com/repos/pinterest/mysql_utils/issues/comments{/number}","contents_url":"https://api.github.com/repos/pinterest/mysql_utils/contents/{+path}","compare_url":"https://api.github.com/repos/pinterest/mysql_utils/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pinterest/mysql_utils/merges","archive_url":"https://api.github.com/repos/pinterest/mysql_utils/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pinterest/mysql_utils/downloads","issues_url":"https://api.github.com/repos/pinterest/mysql_utils/issues{/number}","pulls_url":"https://api.github.com/repos/pinterest/mysql_utils/pulls{/number}","milestones_url":"https://api.github.com/repos/pinterest/mysql_utils/milestones{/number}","notifications_url":"https://api.github.com/repos/pinterest/mysql_utils/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pinterest/mysql_utils/labels{/name}","releases_url":"https://api.github.com/repos/pinterest/mysql_utils/releases{/id}","deployments_url":"https://api.github.com/repos/pinterest/mysql_utils/deployments","created_at":"2015-10-24T17:33:19Z","updated_at":"2016-08-21T02:24:03Z","pushed_at":"2016-08-21T00:01:13Z","git_url":"git://github.com/pinterest/mysql_utils.git","ssh_url":"git@github.com:pinterest/mysql_utils.git","clone_url":"https://github.com/pinterest/mysql_utils.git","svn_url":"https://github.com/pinterest/mysql_utils","homepage":"","size":204,"stargazers_count":673,"watchers_count":673,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":84,"mirror_url":null,"open_issues_count":1,"forks":84,"open_issues":1,"watchers":673,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/pinterest/mysql_utils/pulls/7"},"html":{"href":"https://github.com/pinterest/mysql_utils/pull/7"},"issue":{"href":"https://api.github.com/repos/pinterest/mysql_utils/issues/7"},"comments":{"href":"https://api.github.com/repos/pinterest/mysql_utils/issues/7/comments"},"review_comments":{"href":"https://api.github.com/repos/pinterest/mysql_utils/pulls/7/comments"},"review_comment":{"href":"https://api.github.com/repos/pinterest/mysql_utils/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/pinterest/mysql_utils/pulls/7/commits"},"statuses":{"href":"https://api.github.com/repos/pinterest/mysql_utils/statuses/9afdc2b4e2597cc46e5fb009a8407874fd33bf78"}}}}
{"actor":{"display_login":"jparise"}}