{"action":"created","comment":{"url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/pulls/comments/62826626","id":62826626,"diff_hunk":"@@ -50,36 +73,100 @@ func main() {\n \tflag.StringVar(&fPath, \"file\", \"\", \"List of samples (MD5, SHAX, CRITs ID) to copy from CRITs to Totem\")\n \tflag.StringVar(&critsFileServer, \"cfs\", \"\", \"Full URL to your CRITs file server\")\n \tflag.StringVar(&holmesStorage, \"storage\", \"\", \"Full URL to your Holmes-Storage server\")\n+\tflag.StringVar(&directory, \"dir\", \"\", \"Directory of samples to upload\")\n+\tflag.StringVar(&comment, \"comment\", \"\", \"Comment of submitter\")\n+\tflag.StringVar(&source, \"src\", \"\", \"Source of the files\")\n+\tflag.StringVar(&userid, \"uid\", \"-1\", \"User ID of submitter\")\n+\tflag.IntVar(&numWorkers, \"workers\", 1, \"Number of parallel workers\")\n+\tflag.BoolVar(&recursive, \"rec\", false, \"If set, the directory will be iterated recursively\")\n \tflag.Parse()\n \n-\tfmt.Sprintf(\"Copying samples from %s\", fPath)\n+\t//fmt.Sprintf(\"Copying samples from %s\", fPath)\n \n-\tfile, err := os.Open(fPath)\n-\tif err != nil {\n-\t\tfmt.Println(\"Couln't open file containing sample list!\", err.Error())\n-\t\treturn\n+\tc = make(chan string)\n+\tfor i := 0; i < numWorkers; i++ {\n+\t\tgo worker()\n+\t}\n+\n+\tif fPath != \"\"\t{\n+\t\tfile, err := os.Open(fPath)\n+\t\tif err != nil {\n+\t\t\tfmt.Println(\"Couln't open file containing sample list!\", err.Error())\n+\t\t\treturn\n+\t\t}\n+\t\tdefer file.Close()\n+\n+\t\tscanner := bufio.NewScanner(file)\n+\t\tscanner.Split(bufio.ScanLines)\n+\t\t// line by line\n+\t\tfor scanner.Scan() {\n+\t\t\twg.Add(1)\n+\t\t\tc <- scanner.Text()\n+\t\t\t//go copySample(scanner.Text())\n+\t\t}\n \t}\n-\tdefer file.Close()\n \n-\tscanner := bufio.NewScanner(file)\n-\tscanner.Split(bufio.ScanLines)\n+\tmagicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)\n+\tdefer magicmime.Close()\n+\n+\tfullPath, err := filepath.Abs(directory)\n+\n+    if err != nil {\n+        log.Println(\"path error:\", err)\n+        return\n+    }\n+    topLevel = true\n+    err = filepath.Walk(fullPath, walkFn)\n+    if err != nil {\n+        log.Println(\"walk error:\", err)\n+        return\n+    }\n+\n+\twg.Wait()\n+}\n \n-\t// line by line\n-\tfor scanner.Scan() {\n-\t\tcopySample(scanner.Text())\n+func walkFn(path string, fi os.FileInfo, err error) (e error) {\n+\tif fi.IsDir(){\n+\t\tif recursive {\n+\t\t\treturn nil\n+\t\t} else {\n+\t\t\tif topLevel {\n+\t\t\t\ttopLevel = false\n+\t\t\t\treturn nil\n+\t\t\t} else {\n+\t\t\t\treturn filepath.SkipDir\n+\t\t\t}\n+\t\t}\n+\t}\n+\tmimetype, err := magicmime.TypeByFile(path)\n+\tif err != nil {\n+\t\tlog.Println(\"mimetype error (skipping \" + path + \"):\", err)\n+\t\treturn nil\n+\t}\n+\tif strings.Contains(mimetype, \"exec\") {\n+\t\tprint(\"Adding \" + path + \" (\" + mimetype + \")\\n\")\n+\t\twg.Add(1)\n+\t\tc <- path\n+\t\treturn nil\n+\t} else {\n+\t\tprint(\"Skipping \" + path + \" (\" + mimetype + \")\\n\")","path":"push_to_holmes.go","position":146,"original_position":146,"commit_id":"7a636d15692b98f6e082c65a6900be4e750ea7e5","original_commit_id":"7a636d15692b98f6e082c65a6900be4e750ea7e5","user":{"login":"cynexit","id":3159191,"avatar_url":"https://avatars.githubusercontent.com/u/3159191?v=3","gravatar_id":"","url":"https://api.github.com/users/cynexit","html_url":"https://github.com/cynexit","followers_url":"https://api.github.com/users/cynexit/followers","following_url":"https://api.github.com/users/cynexit/following{/other_user}","gists_url":"https://api.github.com/users/cynexit/gists{/gist_id}","starred_url":"https://api.github.com/users/cynexit/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cynexit/subscriptions","organizations_url":"https://api.github.com/users/cynexit/orgs","repos_url":"https://api.github.com/users/cynexit/repos","events_url":"https://api.github.com/users/cynexit/events{/privacy}","received_events_url":"https://api.github.com/users/cynexit/received_events","type":"User","site_admin":false},"body":"Please use log.Println","created_at":"2016-05-11T10:44:02Z","updated_at":"2016-05-11T10:44:02Z","html_url":"https://github.com/HolmesProcessing/Holmes-Toolbox/pull/5#discussion_r62826626","pull_request_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/pulls/5","_links":{"self":{"href":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/pulls/comments/62826626"},"html":{"href":"https://github.com/HolmesProcessing/Holmes-Toolbox/pull/5#discussion_r62826626"},"pull_request":{"href":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/pulls/5"}}},"pull_request":{"url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/pulls/5","id":68777193,"html_url":"https://github.com/HolmesProcessing/Holmes-Toolbox/pull/5","diff_url":"https://github.com/HolmesProcessing/Holmes-Toolbox/pull/5.diff","patch_url":"https://github.com/HolmesProcessing/Holmes-Toolbox/pull/5.patch","issue_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/issues/5","number":5,"state":"open","locked":false,"title":"Uploadscript","user":{"login":"Ma-Shell","id":3225318,"avatar_url":"https://avatars.githubusercontent.com/u/3225318?v=3","gravatar_id":"","url":"https://api.github.com/users/Ma-Shell","html_url":"https://github.com/Ma-Shell","followers_url":"https://api.github.com/users/Ma-Shell/followers","following_url":"https://api.github.com/users/Ma-Shell/following{/other_user}","gists_url":"https://api.github.com/users/Ma-Shell/gists{/gist_id}","starred_url":"https://api.github.com/users/Ma-Shell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Ma-Shell/subscriptions","organizations_url":"https://api.github.com/users/Ma-Shell/orgs","repos_url":"https://api.github.com/users/Ma-Shell/repos","events_url":"https://api.github.com/users/Ma-Shell/events{/privacy}","received_events_url":"https://api.github.com/users/Ma-Shell/received_events","type":"User","site_admin":false},"body":"Improvements to the upload-script:\r\n- Parallellized with a configurable number of worker-threads.\r\n- MIME-type-checking\r\n- Ability to select a local directory for flat or recursive traversal","created_at":"2016-05-03T21:21:36Z","updated_at":"2016-05-11T10:44:02Z","closed_at":null,"merged_at":null,"merge_commit_sha":"930d43259ec2b5f48d3c7d20e7f9ca647166b5ff","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/pulls/5/commits","review_comments_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/pulls/5/comments","review_comment_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/pulls/comments{/number}","comments_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/issues/5/comments","statuses_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/statuses/7a636d15692b98f6e082c65a6900be4e750ea7e5","head":{"label":"Ma-Shell:uploadscript","ref":"uploadscript","sha":"7a636d15692b98f6e082c65a6900be4e750ea7e5","user":{"login":"Ma-Shell","id":3225318,"avatar_url":"https://avatars.githubusercontent.com/u/3225318?v=3","gravatar_id":"","url":"https://api.github.com/users/Ma-Shell","html_url":"https://github.com/Ma-Shell","followers_url":"https://api.github.com/users/Ma-Shell/followers","following_url":"https://api.github.com/users/Ma-Shell/following{/other_user}","gists_url":"https://api.github.com/users/Ma-Shell/gists{/gist_id}","starred_url":"https://api.github.com/users/Ma-Shell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Ma-Shell/subscriptions","organizations_url":"https://api.github.com/users/Ma-Shell/orgs","repos_url":"https://api.github.com/users/Ma-Shell/repos","events_url":"https://api.github.com/users/Ma-Shell/events{/privacy}","received_events_url":"https://api.github.com/users/Ma-Shell/received_events","type":"User","site_admin":false},"repo":{"id":56097551,"name":"Holmes-Toolbox","full_name":"Ma-Shell/Holmes-Toolbox","owner":{"login":"Ma-Shell","id":3225318,"avatar_url":"https://avatars.githubusercontent.com/u/3225318?v=3","gravatar_id":"","url":"https://api.github.com/users/Ma-Shell","html_url":"https://github.com/Ma-Shell","followers_url":"https://api.github.com/users/Ma-Shell/followers","following_url":"https://api.github.com/users/Ma-Shell/following{/other_user}","gists_url":"https://api.github.com/users/Ma-Shell/gists{/gist_id}","starred_url":"https://api.github.com/users/Ma-Shell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Ma-Shell/subscriptions","organizations_url":"https://api.github.com/users/Ma-Shell/orgs","repos_url":"https://api.github.com/users/Ma-Shell/repos","events_url":"https://api.github.com/users/Ma-Shell/events{/privacy}","received_events_url":"https://api.github.com/users/Ma-Shell/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/Ma-Shell/Holmes-Toolbox","description":"helper scripts for managing Holmes","fork":true,"url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox","forks_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/forks","keys_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/teams","hooks_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/hooks","issue_events_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/issues/events{/number}","events_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/events","assignees_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/assignees{/user}","branches_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/branches{/branch}","tags_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/tags","blobs_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/git/refs{/sha}","trees_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/statuses/{sha}","languages_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/languages","stargazers_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/stargazers","contributors_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/contributors","subscribers_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/subscribers","subscription_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/subscription","commits_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/commits{/sha}","git_commits_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/git/commits{/sha}","comments_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/comments{/number}","issue_comment_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/issues/comments{/number}","contents_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/contents/{+path}","compare_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/merges","archive_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/downloads","issues_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/issues{/number}","pulls_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/pulls{/number}","milestones_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/milestones{/number}","notifications_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/labels{/name}","releases_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/releases{/id}","deployments_url":"https://api.github.com/repos/Ma-Shell/Holmes-Toolbox/deployments","created_at":"2016-04-12T21:02:22Z","updated_at":"2016-04-12T21:02:22Z","pushed_at":"2016-05-03T21:17:28Z","git_url":"git://github.com/Ma-Shell/Holmes-Toolbox.git","ssh_url":"git@github.com:Ma-Shell/Holmes-Toolbox.git","clone_url":"https://github.com/Ma-Shell/Holmes-Toolbox.git","svn_url":"https://github.com/Ma-Shell/Holmes-Toolbox","homepage":"https://holmesprocessing.github.io/","size":11,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"HolmesProcessing:master","ref":"master","sha":"fe8425a514ea142e772c552551c2b55896b848db","user":{"login":"HolmesProcessing","id":13543402,"avatar_url":"https://avatars.githubusercontent.com/u/13543402?v=3","gravatar_id":"","url":"https://api.github.com/users/HolmesProcessing","html_url":"https://github.com/HolmesProcessing","followers_url":"https://api.github.com/users/HolmesProcessing/followers","following_url":"https://api.github.com/users/HolmesProcessing/following{/other_user}","gists_url":"https://api.github.com/users/HolmesProcessing/gists{/gist_id}","starred_url":"https://api.github.com/users/HolmesProcessing/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HolmesProcessing/subscriptions","organizations_url":"https://api.github.com/users/HolmesProcessing/orgs","repos_url":"https://api.github.com/users/HolmesProcessing/repos","events_url":"https://api.github.com/users/HolmesProcessing/events{/privacy}","received_events_url":"https://api.github.com/users/HolmesProcessing/received_events","type":"Organization","site_admin":false},"repo":{"id":49013006,"name":"Holmes-Toolbox","full_name":"HolmesProcessing/Holmes-Toolbox","owner":{"login":"HolmesProcessing","id":13543402,"avatar_url":"https://avatars.githubusercontent.com/u/13543402?v=3","gravatar_id":"","url":"https://api.github.com/users/HolmesProcessing","html_url":"https://github.com/HolmesProcessing","followers_url":"https://api.github.com/users/HolmesProcessing/followers","following_url":"https://api.github.com/users/HolmesProcessing/following{/other_user}","gists_url":"https://api.github.com/users/HolmesProcessing/gists{/gist_id}","starred_url":"https://api.github.com/users/HolmesProcessing/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HolmesProcessing/subscriptions","organizations_url":"https://api.github.com/users/HolmesProcessing/orgs","repos_url":"https://api.github.com/users/HolmesProcessing/repos","events_url":"https://api.github.com/users/HolmesProcessing/events{/privacy}","received_events_url":"https://api.github.com/users/HolmesProcessing/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/HolmesProcessing/Holmes-Toolbox","description":"helper scripts for managing Holmes","fork":false,"url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox","forks_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/forks","keys_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/keys{/key_id}","collaborators_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/teams","hooks_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/hooks","issue_events_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/issues/events{/number}","events_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/events","assignees_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/assignees{/user}","branches_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/branches{/branch}","tags_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/tags","blobs_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/git/refs{/sha}","trees_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/git/trees{/sha}","statuses_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/statuses/{sha}","languages_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/languages","stargazers_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/stargazers","contributors_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/contributors","subscribers_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/subscribers","subscription_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/subscription","commits_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/commits{/sha}","git_commits_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/git/commits{/sha}","comments_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/comments{/number}","issue_comment_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/issues/comments{/number}","contents_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/contents/{+path}","compare_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/compare/{base}...{head}","merges_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/merges","archive_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/downloads","issues_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/issues{/number}","pulls_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/pulls{/number}","milestones_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/milestones{/number}","notifications_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/labels{/name}","releases_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/releases{/id}","deployments_url":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/deployments","created_at":"2016-01-04T17:45:09Z","updated_at":"2016-02-17T23:50:39Z","pushed_at":"2016-05-09T19:50:56Z","git_url":"git://github.com/HolmesProcessing/Holmes-Toolbox.git","ssh_url":"git@github.com:HolmesProcessing/Holmes-Toolbox.git","clone_url":"https://github.com/HolmesProcessing/Holmes-Toolbox.git","svn_url":"https://github.com/HolmesProcessing/Holmes-Toolbox","homepage":"https://holmesprocessing.github.io/","size":12,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":4,"mirror_url":null,"open_issues_count":3,"forks":4,"open_issues":3,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/pulls/5"},"html":{"href":"https://github.com/HolmesProcessing/Holmes-Toolbox/pull/5"},"issue":{"href":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/issues/5"},"comments":{"href":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/issues/5/comments"},"review_comments":{"href":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/pulls/5/comments"},"review_comment":{"href":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/pulls/5/commits"},"statuses":{"href":"https://api.github.com/repos/HolmesProcessing/Holmes-Toolbox/statuses/7a636d15692b98f6e082c65a6900be4e750ea7e5"}}}}