{"action":"created","comment":{"url":"https://api.github.com/repos/alpacahq/marketstore/pulls/comments/229884515","pull_request_review_id":170492820,"id":229884515,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIyOTg4NDUxNQ==","diff_hunk":"@@ -149,68 +154,87 @@ type IndirectRecordInfo struct {\n \tIndex, Offset, Len int64\n }\n \n-func WriteBufferToFileIndirect(fp *os.File, buffer offsetIndexBuffer) (err error) {\n-\t// TODO: Incorporate previously written data into new writes - requires re-read of existing into buffer\n+func WriteBufferToFileIndirect(fp *os.File, buffer offsetIndexBuffer, varRecLen int32) (err error) {\n \t/*\n \t\tHere we write the data payload of the buffer to the end of the data file\n+\t\tPrior to writing the new data, we fetch any previously written data and\n+\t\tprepend it to the current data. This implements \"append\"\n \t*/\n-\n \tprimaryOffset := buffer.Offset() // Offset to storage of indirect record info\n \tindex := buffer.Index()\n \tdataToBeWritten := buffer.Payload()\n \tdataLen := int64(len(dataToBeWritten))\n-\n-\t/*\n-\t\tWrite the data at the end of the file\n-\t*/\n-\tendOfFileOffset, _ := fp.Seek(0, os.SEEK_END)\n-\t_, err = fp.Write(dataToBeWritten)\n-\tif err != nil {\n-\t\treturn err\n-\t}\n-\n \t/*\n \t\tNow we write or update the index record\n \t\tFirst we read the file at the index location to see if this is an incremental write\n \t*/\n-\tfp.Seek(primaryOffset, os.SEEK_SET)\n+\tfp.Seek(primaryOffset, stdio.SeekStart)\n \tidBuf := make([]byte, 24) // {Index, Offset, Len}\n-\t_, err = fp.Read(idBuf)\n-\tif err != nil {\n+\tif _, err = fp.Read(idBuf); err != nil {\n \t\treturn err\n \t}\n-\n \tcurrentRecInfo := SwapSliceByte(idBuf, IndirectRecordInfo{}).([]IndirectRecordInfo)[0]\n \t/*\n-\t\tThe default is a new write at the end of the file\n+\t\tRead the data from the previously written location, if it exists\n \t*/\n-\ttargetRecInfo := IndirectRecordInfo{Index: index, Offset: endOfFileOffset, Len: dataLen}\n-\n+\tif currentRecInfo.Index != 0 {\n+\t\tif _, err = fp.Seek(currentRecInfo.Offset, stdio.SeekStart); err != nil {\n+\t\t\treturn err\n+\t\t}\n+\t\toldData := make([]byte, currentRecInfo.Len)\n+\t\tif _, err := fp.Read(oldData); err != nil {\n+\t\t\treturn err\n+\t\t}\n+\t\tdataToBeWritten = append(oldData, dataToBeWritten...)\n+\t\tdataLen = currentRecInfo.Len + dataLen\n+\t}\n \t/*\n-\t\tIf this is a continuation write, we adjust the targetRecInfo accordingly\n+\t\tSort the data by the timestamp to maintain on-disk sorted order\n+\t */\n+\t//insertionSort(dataToBeWritten, varRecLen)\n+\targ1 := (*C.char)(unsafe.Pointer(&dataToBeWritten[0]))\n+\tC.quickSortKeyAtEndUINT32(arg1, C.int64_t(dataLen), C.int64_t(varRecLen))\n+\t/*\n+\t\tWrite the data at the end of the file\n \t*/\n-\tif currentRecInfo.Index != 0 { // If the index from the file is 0, this is a new write\n-\t\tcursor := currentRecInfo.Offset + currentRecInfo.Len\n-\t\tif endOfFileOffset == cursor {\n-\t\t\t// Incremental write\n-\t\t\ttargetRecInfo.Len += currentRecInfo.Len\n-\t\t\ttargetRecInfo.Offset = currentRecInfo.Offset\n-\t\t}\n+\tendOfFileOffset, _ := fp.Seek(0, stdio.SeekEnd)\n+\tif _, err = fp.Write(dataToBeWritten); err != nil {\n+\t\treturn err\n \t}\n \n \t/*\n \t\tWrite the indirect record info at the primaryOffset\n \t*/\n+\ttargetRecInfo := IndirectRecordInfo{Index: index, Offset: endOfFileOffset, Len: dataLen}\n \todata := []int64{targetRecInfo.Index, targetRecInfo.Offset, targetRecInfo.Len}\n \tobuf := SwapSliceData(odata, byte(0)).([]byte)\n \n-\tfp.Seek(-24, os.SEEK_CUR)\n+\tfp.Seek(primaryOffset, stdio.SeekStart)\n \t_, err = fp.Write(obuf)\n-\tif err != nil {\n-\t\treturn err\n-\t}\n+\treturn err\n+}\n \n-\treturn nil\n+func insertionSort(data []byte, rLen int32) {","path":"executor/writer.go","position":121,"original_position":121,"commit_id":"b1e1ce4b45a3e493d434d13f2d383a978907ee2a","original_commit_id":"b1e1ce4b45a3e493d434d13f2d383a978907ee2a","user":{"login":"rocketbitz","id":18317032,"node_id":"MDQ6VXNlcjE4MzE3MDMy","avatar_url":"https://avatars1.githubusercontent.com/u/18317032?v=4","gravatar_id":"","url":"https://api.github.com/users/rocketbitz","html_url":"https://github.com/rocketbitz","followers_url":"https://api.github.com/users/rocketbitz/followers","following_url":"https://api.github.com/users/rocketbitz/following{/other_user}","gists_url":"https://api.github.com/users/rocketbitz/gists{/gist_id}","starred_url":"https://api.github.com/users/rocketbitz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rocketbitz/subscriptions","organizations_url":"https://api.github.com/users/rocketbitz/orgs","repos_url":"https://api.github.com/users/rocketbitz/repos","events_url":"https://api.github.com/users/rocketbitz/events{/privacy}","received_events_url":"https://api.github.com/users/rocketbitz/received_events","type":"User","site_admin":false},"body":"still needed?","created_at":"2018-10-31T22:08:05Z","updated_at":"2018-10-31T22:08:05Z","html_url":"https://github.com/alpacahq/marketstore/pull/134#discussion_r229884515","pull_request_url":"https://api.github.com/repos/alpacahq/marketstore/pulls/134","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/alpacahq/marketstore/pulls/comments/229884515"},"html":{"href":"https://github.com/alpacahq/marketstore/pull/134#discussion_r229884515"},"pull_request":{"href":"https://api.github.com/repos/alpacahq/marketstore/pulls/134"}}},"pull_request":{"url":"https://api.github.com/repos/alpacahq/marketstore/pulls/134","id":227069741,"node_id":"MDExOlB1bGxSZXF1ZXN0MjI3MDY5NzQx","html_url":"https://github.com/alpacahq/marketstore/pull/134","diff_url":"https://github.com/alpacahq/marketstore/pull/134.diff","patch_url":"https://github.com/alpacahq/marketstore/pull/134.patch","issue_url":"https://api.github.com/repos/alpacahq/marketstore/issues/134","number":134,"state":"open","locked":false,"title":"Implement append behavior as the default for variable (tick) data type","user":{"login":"Notargets","id":464941,"node_id":"MDQ6VXNlcjQ2NDk0MQ==","avatar_url":"https://avatars0.githubusercontent.com/u/464941?v=4","gravatar_id":"","url":"https://api.github.com/users/Notargets","html_url":"https://github.com/Notargets","followers_url":"https://api.github.com/users/Notargets/followers","following_url":"https://api.github.com/users/Notargets/following{/other_user}","gists_url":"https://api.github.com/users/Notargets/gists{/gist_id}","starred_url":"https://api.github.com/users/Notargets/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Notargets/subscriptions","organizations_url":"https://api.github.com/users/Notargets/orgs","repos_url":"https://api.github.com/users/Notargets/repos","events_url":"https://api.github.com/users/Notargets/events{/privacy}","received_events_url":"https://api.github.com/users/Notargets/received_events","type":"User","site_admin":false},"body":"","created_at":"2018-10-30T20:27:09Z","updated_at":"2018-10-31T22:08:05Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5da9f65c383c9ce27a5ea3f15714ee5ac5595962","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/alpacahq/marketstore/pulls/134/commits","review_comments_url":"https://api.github.com/repos/alpacahq/marketstore/pulls/134/comments","review_comment_url":"https://api.github.com/repos/alpacahq/marketstore/pulls/comments{/number}","comments_url":"https://api.github.com/repos/alpacahq/marketstore/issues/134/comments","statuses_url":"https://api.github.com/repos/alpacahq/marketstore/statuses/b1e1ce4b45a3e493d434d13f2d383a978907ee2a","head":{"label":"alpacahq:feature/append-ticks-default","ref":"feature/append-ticks-default","sha":"b1e1ce4b45a3e493d434d13f2d383a978907ee2a","user":{"login":"alpacahq","id":30398729,"node_id":"MDEyOk9yZ2FuaXphdGlvbjMwMzk4NzI5","avatar_url":"https://avatars1.githubusercontent.com/u/30398729?v=4","gravatar_id":"","url":"https://api.github.com/users/alpacahq","html_url":"https://github.com/alpacahq","followers_url":"https://api.github.com/users/alpacahq/followers","following_url":"https://api.github.com/users/alpacahq/following{/other_user}","gists_url":"https://api.github.com/users/alpacahq/gists{/gist_id}","starred_url":"https://api.github.com/users/alpacahq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alpacahq/subscriptions","organizations_url":"https://api.github.com/users/alpacahq/orgs","repos_url":"https://api.github.com/users/alpacahq/repos","events_url":"https://api.github.com/users/alpacahq/events{/privacy}","received_events_url":"https://api.github.com/users/alpacahq/received_events","type":"Organization","site_admin":false},"repo":{"id":116546320,"node_id":"MDEwOlJlcG9zaXRvcnkxMTY1NDYzMjA=","name":"marketstore","full_name":"alpacahq/marketstore","private":false,"owner":{"login":"alpacahq","id":30398729,"node_id":"MDEyOk9yZ2FuaXphdGlvbjMwMzk4NzI5","avatar_url":"https://avatars1.githubusercontent.com/u/30398729?v=4","gravatar_id":"","url":"https://api.github.com/users/alpacahq","html_url":"https://github.com/alpacahq","followers_url":"https://api.github.com/users/alpacahq/followers","following_url":"https://api.github.com/users/alpacahq/following{/other_user}","gists_url":"https://api.github.com/users/alpacahq/gists{/gist_id}","starred_url":"https://api.github.com/users/alpacahq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alpacahq/subscriptions","organizations_url":"https://api.github.com/users/alpacahq/orgs","repos_url":"https://api.github.com/users/alpacahq/repos","events_url":"https://api.github.com/users/alpacahq/events{/privacy}","received_events_url":"https://api.github.com/users/alpacahq/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/alpacahq/marketstore","description":"DataFrame Server for Financial Timeseries Data","fork":false,"url":"https://api.github.com/repos/alpacahq/marketstore","forks_url":"https://api.github.com/repos/alpacahq/marketstore/forks","keys_url":"https://api.github.com/repos/alpacahq/marketstore/keys{/key_id}","collaborators_url":"https://api.github.com/repos/alpacahq/marketstore/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/alpacahq/marketstore/teams","hooks_url":"https://api.github.com/repos/alpacahq/marketstore/hooks","issue_events_url":"https://api.github.com/repos/alpacahq/marketstore/issues/events{/number}","events_url":"https://api.github.com/repos/alpacahq/marketstore/events","assignees_url":"https://api.github.com/repos/alpacahq/marketstore/assignees{/user}","branches_url":"https://api.github.com/repos/alpacahq/marketstore/branches{/branch}","tags_url":"https://api.github.com/repos/alpacahq/marketstore/tags","blobs_url":"https://api.github.com/repos/alpacahq/marketstore/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/alpacahq/marketstore/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/alpacahq/marketstore/git/refs{/sha}","trees_url":"https://api.github.com/repos/alpacahq/marketstore/git/trees{/sha}","statuses_url":"https://api.github.com/repos/alpacahq/marketstore/statuses/{sha}","languages_url":"https://api.github.com/repos/alpacahq/marketstore/languages","stargazers_url":"https://api.github.com/repos/alpacahq/marketstore/stargazers","contributors_url":"https://api.github.com/repos/alpacahq/marketstore/contributors","subscribers_url":"https://api.github.com/repos/alpacahq/marketstore/subscribers","subscription_url":"https://api.github.com/repos/alpacahq/marketstore/subscription","commits_url":"https://api.github.com/repos/alpacahq/marketstore/commits{/sha}","git_commits_url":"https://api.github.com/repos/alpacahq/marketstore/git/commits{/sha}","comments_url":"https://api.github.com/repos/alpacahq/marketstore/comments{/number}","issue_comment_url":"https://api.github.com/repos/alpacahq/marketstore/issues/comments{/number}","contents_url":"https://api.github.com/repos/alpacahq/marketstore/contents/{+path}","compare_url":"https://api.github.com/repos/alpacahq/marketstore/compare/{base}...{head}","merges_url":"https://api.github.com/repos/alpacahq/marketstore/merges","archive_url":"https://api.github.com/repos/alpacahq/marketstore/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/alpacahq/marketstore/downloads","issues_url":"https://api.github.com/repos/alpacahq/marketstore/issues{/number}","pulls_url":"https://api.github.com/repos/alpacahq/marketstore/pulls{/number}","milestones_url":"https://api.github.com/repos/alpacahq/marketstore/milestones{/number}","notifications_url":"https://api.github.com/repos/alpacahq/marketstore/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/alpacahq/marketstore/labels{/name}","releases_url":"https://api.github.com/repos/alpacahq/marketstore/releases{/id}","deployments_url":"https://api.github.com/repos/alpacahq/marketstore/deployments","created_at":"2018-01-07T07:27:25Z","updated_at":"2018-10-31T22:07:57Z","pushed_at":"2018-10-31T22:05:02Z","git_url":"git://github.com/alpacahq/marketstore.git","ssh_url":"git@github.com:alpacahq/marketstore.git","clone_url":"https://github.com/alpacahq/marketstore.git","svn_url":"https://github.com/alpacahq/marketstore","homepage":"","size":800,"stargazers_count":564,"watchers_count":564,"language":"Go","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":54,"mirror_url":null,"archived":false,"open_issues_count":24,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":54,"open_issues":24,"watchers":564,"default_branch":"master"}},"base":{"label":"alpacahq:master","ref":"master","sha":"81a1c3be2027ca7f3f1d8c7e7157bf91bead9895","user":{"login":"alpacahq","id":30398729,"node_id":"MDEyOk9yZ2FuaXphdGlvbjMwMzk4NzI5","avatar_url":"https://avatars1.githubusercontent.com/u/30398729?v=4","gravatar_id":"","url":"https://api.github.com/users/alpacahq","html_url":"https://github.com/alpacahq","followers_url":"https://api.github.com/users/alpacahq/followers","following_url":"https://api.github.com/users/alpacahq/following{/other_user}","gists_url":"https://api.github.com/users/alpacahq/gists{/gist_id}","starred_url":"https://api.github.com/users/alpacahq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alpacahq/subscriptions","organizations_url":"https://api.github.com/users/alpacahq/orgs","repos_url":"https://api.github.com/users/alpacahq/repos","events_url":"https://api.github.com/users/alpacahq/events{/privacy}","received_events_url":"https://api.github.com/users/alpacahq/received_events","type":"Organization","site_admin":false},"repo":{"id":116546320,"node_id":"MDEwOlJlcG9zaXRvcnkxMTY1NDYzMjA=","name":"marketstore","full_name":"alpacahq/marketstore","private":false,"owner":{"login":"alpacahq","id":30398729,"node_id":"MDEyOk9yZ2FuaXphdGlvbjMwMzk4NzI5","avatar_url":"https://avatars1.githubusercontent.com/u/30398729?v=4","gravatar_id":"","url":"https://api.github.com/users/alpacahq","html_url":"https://github.com/alpacahq","followers_url":"https://api.github.com/users/alpacahq/followers","following_url":"https://api.github.com/users/alpacahq/following{/other_user}","gists_url":"https://api.github.com/users/alpacahq/gists{/gist_id}","starred_url":"https://api.github.com/users/alpacahq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alpacahq/subscriptions","organizations_url":"https://api.github.com/users/alpacahq/orgs","repos_url":"https://api.github.com/users/alpacahq/repos","events_url":"https://api.github.com/users/alpacahq/events{/privacy}","received_events_url":"https://api.github.com/users/alpacahq/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/alpacahq/marketstore","description":"DataFrame Server for Financial Timeseries Data","fork":false,"url":"https://api.github.com/repos/alpacahq/marketstore","forks_url":"https://api.github.com/repos/alpacahq/marketstore/forks","keys_url":"https://api.github.com/repos/alpacahq/marketstore/keys{/key_id}","collaborators_url":"https://api.github.com/repos/alpacahq/marketstore/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/alpacahq/marketstore/teams","hooks_url":"https://api.github.com/repos/alpacahq/marketstore/hooks","issue_events_url":"https://api.github.com/repos/alpacahq/marketstore/issues/events{/number}","events_url":"https://api.github.com/repos/alpacahq/marketstore/events","assignees_url":"https://api.github.com/repos/alpacahq/marketstore/assignees{/user}","branches_url":"https://api.github.com/repos/alpacahq/marketstore/branches{/branch}","tags_url":"https://api.github.com/repos/alpacahq/marketstore/tags","blobs_url":"https://api.github.com/repos/alpacahq/marketstore/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/alpacahq/marketstore/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/alpacahq/marketstore/git/refs{/sha}","trees_url":"https://api.github.com/repos/alpacahq/marketstore/git/trees{/sha}","statuses_url":"https://api.github.com/repos/alpacahq/marketstore/statuses/{sha}","languages_url":"https://api.github.com/repos/alpacahq/marketstore/languages","stargazers_url":"https://api.github.com/repos/alpacahq/marketstore/stargazers","contributors_url":"https://api.github.com/repos/alpacahq/marketstore/contributors","subscribers_url":"https://api.github.com/repos/alpacahq/marketstore/subscribers","subscription_url":"https://api.github.com/repos/alpacahq/marketstore/subscription","commits_url":"https://api.github.com/repos/alpacahq/marketstore/commits{/sha}","git_commits_url":"https://api.github.com/repos/alpacahq/marketstore/git/commits{/sha}","comments_url":"https://api.github.com/repos/alpacahq/marketstore/comments{/number}","issue_comment_url":"https://api.github.com/repos/alpacahq/marketstore/issues/comments{/number}","contents_url":"https://api.github.com/repos/alpacahq/marketstore/contents/{+path}","compare_url":"https://api.github.com/repos/alpacahq/marketstore/compare/{base}...{head}","merges_url":"https://api.github.com/repos/alpacahq/marketstore/merges","archive_url":"https://api.github.com/repos/alpacahq/marketstore/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/alpacahq/marketstore/downloads","issues_url":"https://api.github.com/repos/alpacahq/marketstore/issues{/number}","pulls_url":"https://api.github.com/repos/alpacahq/marketstore/pulls{/number}","milestones_url":"https://api.github.com/repos/alpacahq/marketstore/milestones{/number}","notifications_url":"https://api.github.com/repos/alpacahq/marketstore/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/alpacahq/marketstore/labels{/name}","releases_url":"https://api.github.com/repos/alpacahq/marketstore/releases{/id}","deployments_url":"https://api.github.com/repos/alpacahq/marketstore/deployments","created_at":"2018-01-07T07:27:25Z","updated_at":"2018-10-31T22:07:57Z","pushed_at":"2018-10-31T22:05:02Z","git_url":"git://github.com/alpacahq/marketstore.git","ssh_url":"git@github.com:alpacahq/marketstore.git","clone_url":"https://github.com/alpacahq/marketstore.git","svn_url":"https://github.com/alpacahq/marketstore","homepage":"","size":800,"stargazers_count":564,"watchers_count":564,"language":"Go","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":54,"mirror_url":null,"archived":false,"open_issues_count":24,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":54,"open_issues":24,"watchers":564,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/alpacahq/marketstore/pulls/134"},"html":{"href":"https://github.com/alpacahq/marketstore/pull/134"},"issue":{"href":"https://api.github.com/repos/alpacahq/marketstore/issues/134"},"comments":{"href":"https://api.github.com/repos/alpacahq/marketstore/issues/134/comments"},"review_comments":{"href":"https://api.github.com/repos/alpacahq/marketstore/pulls/134/comments"},"review_comment":{"href":"https://api.github.com/repos/alpacahq/marketstore/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/alpacahq/marketstore/pulls/134/commits"},"statuses":{"href":"https://api.github.com/repos/alpacahq/marketstore/statuses/b1e1ce4b45a3e493d434d13f2d383a978907ee2a"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"rocketbitz"}}