{"action":"created","comment":{"url":"https://api.github.com/repos/mldbai/mldb/pulls/comments/69006419","id":69006419,"diff_hunk":"@@ -0,0 +1,581 @@\n+// This file is part of MLDB. Copyright 2015 Datacratic. All rights reserved.\n+\n+/** dist_table_procedure.cc                                        -*- C++ -*-\n+    Simon Lemieux, June 2015\n+    Copyright (c) 2015 Datacratic Inc.  All rights reserved.\n+\n+    distTable procedure\n+\n+    Note that this was mainly copied from stats_table_procedure.cc in a hurry.\n+    This will probably be revisited.\n+*/\n+\n+#include \"dist_table_procedure.h\"\n+#include \"types/basic_value_descriptions.h\"\n+#include \"types/distribution_description.h\"\n+#include \"types/map_description.h\"\n+#include \"mldb/rest/in_process_rest_connection.h\"\n+#include \"server/dataset_context.h\"\n+#include \"plugins/matrix.h\"\n+#include \"server/analytics.h\"\n+#include \"ml/value_descriptions.h\"\n+#include \"types/any_impl.h\"\n+#include \"jml/utils/string_functions.h\"\n+#include \"plugins/sql_functions.h\"\n+#include \"sql/execution_pipeline.h\"\n+#include \"server/bound_queries.h\"\n+#include \"mldb/http/http_exception.h\"\n+#include \"mldb/jml/db/persistent.h\"\n+#include \"mldb/types/jml_serialization.h\"\n+#include \"mldb/vfs/filter_streams.h\"\n+#include \"mldb/plugins/sql_config_validator.h\"\n+#include \"mldb/base/parallel.h\"\n+#include \"mldb/types/optional_description.h\"\n+\n+\n+using namespace std;\n+\n+namespace Datacratic {\n+namespace MLDB {\n+\n+inline ML::DB::Store_Writer &\n+operator << (ML::DB::Store_Writer & store, const PathElement & coord)\n+{\n+    return store << Id(coord.toUtf8String());\n+}\n+\n+inline ML::DB::Store_Reader &\n+operator >> (ML::DB::Store_Reader & store, PathElement & coord)\n+{\n+    Id id;\n+    store >> id;\n+    coord = id.toUtf8String();\n+    return store;\n+}\n+\n+inline ML::DB::Store_Writer &\n+operator << (ML::DB::Store_Writer & store, const Path & coords)\n+{\n+    return store << coords.toUtf8String();\n+}\n+\n+inline ML::DB::Store_Reader &\n+operator >> (ML::DB::Store_Reader & store, Path & coords)\n+{\n+    Utf8String str;\n+    store >> str;\n+    coords = Path::parse(str);\n+    return store;\n+}\n+\n+\n+/*****************************************************************************/\n+/* DIST TABLE                                                               */\n+/*****************************************************************************/\n+\n+DistTable::\n+DistTable(const std::string & filename)\n+{\n+    filter_istream stream(filename);\n+    ML::DB::Store_Reader store(stream);\n+    reconstitute(store);\n+}\n+\n+void DistTable::\n+increment(const Utf8String & featureValue, uint outcome, double targetValue) {\n+    auto it = counts[featureValue].find(outcome);\n+    if (it == counts[featureValue].end()) {\n+        counts[featureValue][outcome] = 1;\n+        avgs[featureValue][outcome] = targetValue;\n+        mins[featureValue][outcome] = targetValue;\n+        maxs[featureValue][outcome] = targetValue;\n+        M2s[featureValue][outcome] = 0.;\n+    } else {\n+        uint64_t n = ++counts[featureValue][outcome];\n+\n+        double delta = targetValue - avgs[featureValue][outcome];\n+        avgs[featureValue][outcome] += delta / n;\n+        // For the unbiased std, I'm using Welford's algorithm described here\n+        // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm\n+        double M2 = M2s[featureValue][outcome] =\n+            M2s[featureValue][outcome]\n+            + delta * (targetValue - avgs[featureValue][outcome]);\n+        if (n >= 2) // otherwise unbiased variance doesn't make sense\n+            vars[featureValue][outcome] = M2 / (n - 1);\n+\n+        mins[featureValue][outcome] = std::min(mins[featureValue][outcome],\n+                                               targetValue);\n+        maxs[featureValue][outcome] = std::max(maxs[featureValue][outcome],\n+                                               targetValue);\n+    }\n+}\n+\n+tuple<uint64_t, double, double, double, double>\n+DistTable::\n+getStats(const Utf8String & featureValue, int outcome) const\n+{\n+    // if we don't have anything for that feature and outcome, return\n+    // something dummy\n+    if (!(counts.count(featureValue)\n+            && counts.at(featureValue).count(outcome)))\n+        return make_tuple(0, NAN, NAN, NAN, NAN);\n+\n+    int count = counts.at(featureValue).at(outcome);\n+    ExcAssert(count > 0);\n+    return make_tuple(\n+        counts.at(featureValue).at(outcome),\n+        avgs.at(featureValue).at(outcome),\n+        // unbiased std needs n >= 2\n+        count >= 2 ? sqrt(vars.at(featureValue).at(outcome)) : NAN,\n+        mins.at(featureValue).at(outcome),\n+        maxs.at(featureValue).at(outcome));\n+}\n+\n+void DistTable::\n+save(const std::string & filename) const\n+{\n+    filter_ostream stream(filename);\n+    ML::DB::Store_Writer store(stream);\n+    serialize(store);\n+}\n+\n+void DistTable::\n+serialize(ML::DB::Store_Writer & store) const\n+{\n+    int version = 1;\n+    store << string(\"MLDB Dist Table Binary\")\n+          << version << colName << outcome_names << counts << avgs << vars\n+          << M2s << mins << maxs;\n+}\n+\n+void DistTable::\n+reconstitute(ML::DB::Store_Reader & store)\n+{\n+    int version;\n+    int REQUIRED_V = 1;\n+    std::string name;\n+    store >> name >> version;\n+    if (name != \"MLDB Dist Table Binary\") {\n+        throw HttpReturnException(400, \"File does not appear to be a stats \"\n+                                  \"table model\");\n+    }\n+    if(version!=REQUIRED_V) {\n+        throw HttpReturnException(400, ML::format(\n+                    \"invalid DistTable version! exptected %d, got %d\",\n+                    REQUIRED_V, version));\n+    }\n+\n+    store >> colName >> outcome_names >> counts >> avgs >> vars\n+          >> M2s >> mins >> maxs;\n+}\n+\n+\n+/*****************************************************************************/\n+/* DIST TABLE PROCEDURE CONFIG                                              */\n+/*****************************************************************************/\n+\n+DEFINE_STRUCTURE_DESCRIPTION(DistTableProcedureConfig);\n+\n+DistTableProcedureConfigDescription::\n+DistTableProcedureConfigDescription()\n+{\n+    addField(\"trainingData\", &DistTableProcedureConfig::trainingData,\n+             \"SQL query to select the data on which the rolling operations will \"\n+             \"be performed.\");\n+    addField(\"outputDataset\", &DistTableProcedureConfig::output,\n+             \"Output dataset\",\n+             PolyConfigT<Dataset>().withType(\"sparse.mutable\"));\n+    addField(\"outcomes\", &DistTableProcedureConfig::outcomes,\n+             \"List of expressions to generate the outcomes. Each can be any expression \"\n+             \"involving the columns in the dataset. The type of the outcomes \"\n+             \"must be of type `INTEGER` or `NUMBER`\");\n+    addField(\"distTableFileUrl\", &DistTableProcedureConfig::modelFileUrl,\n+             \"URL where the model file (with extension '.dt') should be saved. \"\n+             \"This file can be loaded by the ![](%%doclink distTable.getStats function). \"\n+             \"This parameter is optional unless the `functionName` parameter is used.\");\n+    addField(\"functionName\", &DistTableProcedureConfig::functionName,\n+             \"If specified, an instance of the ![](%%doclink distTable.getStats function) \"\n+             \"of this name will be created using the trained stats tables. Note that to use \"","path":"plugins/dist_table_procedure.cc","position":198,"original_position":198,"commit_id":"f82f39a2ba4a027e098856d97cc46f3037eb99fa","original_commit_id":"f82f39a2ba4a027e098856d97cc46f3037eb99fa","user":{"login":"mailletf","id":569421,"avatar_url":"https://avatars.githubusercontent.com/u/569421?v=3","gravatar_id":"","url":"https://api.github.com/users/mailletf","html_url":"https://github.com/mailletf","followers_url":"https://api.github.com/users/mailletf/followers","following_url":"https://api.github.com/users/mailletf/following{/other_user}","gists_url":"https://api.github.com/users/mailletf/gists{/gist_id}","starred_url":"https://api.github.com/users/mailletf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mailletf/subscriptions","organizations_url":"https://api.github.com/users/mailletf/orgs","repos_url":"https://api.github.com/users/mailletf/repos","events_url":"https://api.github.com/users/mailletf/events{/privacy}","received_events_url":"https://api.github.com/users/mailletf/received_events","type":"User","site_admin":false},"body":"trained stats tables -> trained dist tables","created_at":"2016-06-29T18:55:02Z","updated_at":"2016-06-29T18:55:02Z","html_url":"https://github.com/mldbai/mldb/pull/471#discussion_r69006419","pull_request_url":"https://api.github.com/repos/mldbai/mldb/pulls/471","_links":{"self":{"href":"https://api.github.com/repos/mldbai/mldb/pulls/comments/69006419"},"html":{"href":"https://github.com/mldbai/mldb/pull/471#discussion_r69006419"},"pull_request":{"href":"https://api.github.com/repos/mldbai/mldb/pulls/471"}}},"pull_request":{"url":"https://api.github.com/repos/mldbai/mldb/pulls/471","id":75691534,"html_url":"https://github.com/mldbai/mldb/pull/471","diff_url":"https://github.com/mldbai/mldb/pull/471.diff","patch_url":"https://github.com/mldbai/mldb/pull/471.patch","issue_url":"https://api.github.com/repos/mldbai/mldb/issues/471","number":471,"state":"open","locked":false,"title":"Mldb 1750 dist tables","user":{"login":"simlmx","id":1105380,"avatar_url":"https://avatars.githubusercontent.com/u/1105380?v=3","gravatar_id":"","url":"https://api.github.com/users/simlmx","html_url":"https://github.com/simlmx","followers_url":"https://api.github.com/users/simlmx/followers","following_url":"https://api.github.com/users/simlmx/following{/other_user}","gists_url":"https://api.github.com/users/simlmx/gists{/gist_id}","starred_url":"https://api.github.com/users/simlmx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/simlmx/subscriptions","organizations_url":"https://api.github.com/users/simlmx/orgs","repos_url":"https://api.github.com/users/simlmx/repos","events_url":"https://api.github.com/users/simlmx/events{/privacy}","received_events_url":"https://api.github.com/users/simlmx/received_events","type":"User","site_admin":false},"body":"First version of distribution tables, the numeric-value version of the stats tables.","created_at":"2016-06-29T17:48:30Z","updated_at":"2016-06-29T18:55:02Z","closed_at":null,"merged_at":null,"merge_commit_sha":"57dfe3e0c392b570814bd1f75975eae3e3580a3b","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/mldbai/mldb/pulls/471/commits","review_comments_url":"https://api.github.com/repos/mldbai/mldb/pulls/471/comments","review_comment_url":"https://api.github.com/repos/mldbai/mldb/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mldbai/mldb/issues/471/comments","statuses_url":"https://api.github.com/repos/mldbai/mldb/statuses/f82f39a2ba4a027e098856d97cc46f3037eb99fa","head":{"label":"mldbai:MLDB-1750-dist-tables","ref":"MLDB-1750-dist-tables","sha":"f82f39a2ba4a027e098856d97cc46f3037eb99fa","user":{"login":"mldbai","id":11510172,"avatar_url":"https://avatars.githubusercontent.com/u/11510172?v=3","gravatar_id":"","url":"https://api.github.com/users/mldbai","html_url":"https://github.com/mldbai","followers_url":"https://api.github.com/users/mldbai/followers","following_url":"https://api.github.com/users/mldbai/following{/other_user}","gists_url":"https://api.github.com/users/mldbai/gists{/gist_id}","starred_url":"https://api.github.com/users/mldbai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mldbai/subscriptions","organizations_url":"https://api.github.com/users/mldbai/orgs","repos_url":"https://api.github.com/users/mldbai/repos","events_url":"https://api.github.com/users/mldbai/events{/privacy}","received_events_url":"https://api.github.com/users/mldbai/received_events","type":"Organization","site_admin":false},"repo":{"id":47634692,"name":"mldb","full_name":"mldbai/mldb","owner":{"login":"mldbai","id":11510172,"avatar_url":"https://avatars.githubusercontent.com/u/11510172?v=3","gravatar_id":"","url":"https://api.github.com/users/mldbai","html_url":"https://github.com/mldbai","followers_url":"https://api.github.com/users/mldbai/followers","following_url":"https://api.github.com/users/mldbai/following{/other_user}","gists_url":"https://api.github.com/users/mldbai/gists{/gist_id}","starred_url":"https://api.github.com/users/mldbai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mldbai/subscriptions","organizations_url":"https://api.github.com/users/mldbai/orgs","repos_url":"https://api.github.com/users/mldbai/repos","events_url":"https://api.github.com/users/mldbai/events{/privacy}","received_events_url":"https://api.github.com/users/mldbai/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mldbai/mldb","description":"MLDB is the Machine Learning Database","fork":false,"url":"https://api.github.com/repos/mldbai/mldb","forks_url":"https://api.github.com/repos/mldbai/mldb/forks","keys_url":"https://api.github.com/repos/mldbai/mldb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mldbai/mldb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mldbai/mldb/teams","hooks_url":"https://api.github.com/repos/mldbai/mldb/hooks","issue_events_url":"https://api.github.com/repos/mldbai/mldb/issues/events{/number}","events_url":"https://api.github.com/repos/mldbai/mldb/events","assignees_url":"https://api.github.com/repos/mldbai/mldb/assignees{/user}","branches_url":"https://api.github.com/repos/mldbai/mldb/branches{/branch}","tags_url":"https://api.github.com/repos/mldbai/mldb/tags","blobs_url":"https://api.github.com/repos/mldbai/mldb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mldbai/mldb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mldbai/mldb/git/refs{/sha}","trees_url":"https://api.github.com/repos/mldbai/mldb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mldbai/mldb/statuses/{sha}","languages_url":"https://api.github.com/repos/mldbai/mldb/languages","stargazers_url":"https://api.github.com/repos/mldbai/mldb/stargazers","contributors_url":"https://api.github.com/repos/mldbai/mldb/contributors","subscribers_url":"https://api.github.com/repos/mldbai/mldb/subscribers","subscription_url":"https://api.github.com/repos/mldbai/mldb/subscription","commits_url":"https://api.github.com/repos/mldbai/mldb/commits{/sha}","git_commits_url":"https://api.github.com/repos/mldbai/mldb/git/commits{/sha}","comments_url":"https://api.github.com/repos/mldbai/mldb/comments{/number}","issue_comment_url":"https://api.github.com/repos/mldbai/mldb/issues/comments{/number}","contents_url":"https://api.github.com/repos/mldbai/mldb/contents/{+path}","compare_url":"https://api.github.com/repos/mldbai/mldb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mldbai/mldb/merges","archive_url":"https://api.github.com/repos/mldbai/mldb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mldbai/mldb/downloads","issues_url":"https://api.github.com/repos/mldbai/mldb/issues{/number}","pulls_url":"https://api.github.com/repos/mldbai/mldb/pulls{/number}","milestones_url":"https://api.github.com/repos/mldbai/mldb/milestones{/number}","notifications_url":"https://api.github.com/repos/mldbai/mldb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mldbai/mldb/labels{/name}","releases_url":"https://api.github.com/repos/mldbai/mldb/releases{/id}","deployments_url":"https://api.github.com/repos/mldbai/mldb/deployments","created_at":"2015-12-08T16:34:16Z","updated_at":"2016-06-29T16:49:23Z","pushed_at":"2016-06-29T18:43:05Z","git_url":"git://github.com/mldbai/mldb.git","ssh_url":"git@github.com:mldbai/mldb.git","clone_url":"https://github.com/mldbai/mldb.git","svn_url":"https://github.com/mldbai/mldb","homepage":"http://mldb.ai","size":22251,"stargazers_count":170,"watchers_count":170,"language":"C++","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":11,"mirror_url":null,"open_issues_count":7,"forks":11,"open_issues":7,"watchers":170,"default_branch":"master"}},"base":{"label":"mldbai:master","ref":"master","sha":"9c4ff90f87fa0c4ba5f005853d0daf060a579579","user":{"login":"mldbai","id":11510172,"avatar_url":"https://avatars.githubusercontent.com/u/11510172?v=3","gravatar_id":"","url":"https://api.github.com/users/mldbai","html_url":"https://github.com/mldbai","followers_url":"https://api.github.com/users/mldbai/followers","following_url":"https://api.github.com/users/mldbai/following{/other_user}","gists_url":"https://api.github.com/users/mldbai/gists{/gist_id}","starred_url":"https://api.github.com/users/mldbai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mldbai/subscriptions","organizations_url":"https://api.github.com/users/mldbai/orgs","repos_url":"https://api.github.com/users/mldbai/repos","events_url":"https://api.github.com/users/mldbai/events{/privacy}","received_events_url":"https://api.github.com/users/mldbai/received_events","type":"Organization","site_admin":false},"repo":{"id":47634692,"name":"mldb","full_name":"mldbai/mldb","owner":{"login":"mldbai","id":11510172,"avatar_url":"https://avatars.githubusercontent.com/u/11510172?v=3","gravatar_id":"","url":"https://api.github.com/users/mldbai","html_url":"https://github.com/mldbai","followers_url":"https://api.github.com/users/mldbai/followers","following_url":"https://api.github.com/users/mldbai/following{/other_user}","gists_url":"https://api.github.com/users/mldbai/gists{/gist_id}","starred_url":"https://api.github.com/users/mldbai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mldbai/subscriptions","organizations_url":"https://api.github.com/users/mldbai/orgs","repos_url":"https://api.github.com/users/mldbai/repos","events_url":"https://api.github.com/users/mldbai/events{/privacy}","received_events_url":"https://api.github.com/users/mldbai/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mldbai/mldb","description":"MLDB is the Machine Learning Database","fork":false,"url":"https://api.github.com/repos/mldbai/mldb","forks_url":"https://api.github.com/repos/mldbai/mldb/forks","keys_url":"https://api.github.com/repos/mldbai/mldb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mldbai/mldb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mldbai/mldb/teams","hooks_url":"https://api.github.com/repos/mldbai/mldb/hooks","issue_events_url":"https://api.github.com/repos/mldbai/mldb/issues/events{/number}","events_url":"https://api.github.com/repos/mldbai/mldb/events","assignees_url":"https://api.github.com/repos/mldbai/mldb/assignees{/user}","branches_url":"https://api.github.com/repos/mldbai/mldb/branches{/branch}","tags_url":"https://api.github.com/repos/mldbai/mldb/tags","blobs_url":"https://api.github.com/repos/mldbai/mldb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mldbai/mldb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mldbai/mldb/git/refs{/sha}","trees_url":"https://api.github.com/repos/mldbai/mldb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mldbai/mldb/statuses/{sha}","languages_url":"https://api.github.com/repos/mldbai/mldb/languages","stargazers_url":"https://api.github.com/repos/mldbai/mldb/stargazers","contributors_url":"https://api.github.com/repos/mldbai/mldb/contributors","subscribers_url":"https://api.github.com/repos/mldbai/mldb/subscribers","subscription_url":"https://api.github.com/repos/mldbai/mldb/subscription","commits_url":"https://api.github.com/repos/mldbai/mldb/commits{/sha}","git_commits_url":"https://api.github.com/repos/mldbai/mldb/git/commits{/sha}","comments_url":"https://api.github.com/repos/mldbai/mldb/comments{/number}","issue_comment_url":"https://api.github.com/repos/mldbai/mldb/issues/comments{/number}","contents_url":"https://api.github.com/repos/mldbai/mldb/contents/{+path}","compare_url":"https://api.github.com/repos/mldbai/mldb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mldbai/mldb/merges","archive_url":"https://api.github.com/repos/mldbai/mldb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mldbai/mldb/downloads","issues_url":"https://api.github.com/repos/mldbai/mldb/issues{/number}","pulls_url":"https://api.github.com/repos/mldbai/mldb/pulls{/number}","milestones_url":"https://api.github.com/repos/mldbai/mldb/milestones{/number}","notifications_url":"https://api.github.com/repos/mldbai/mldb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mldbai/mldb/labels{/name}","releases_url":"https://api.github.com/repos/mldbai/mldb/releases{/id}","deployments_url":"https://api.github.com/repos/mldbai/mldb/deployments","created_at":"2015-12-08T16:34:16Z","updated_at":"2016-06-29T16:49:23Z","pushed_at":"2016-06-29T18:43:05Z","git_url":"git://github.com/mldbai/mldb.git","ssh_url":"git@github.com:mldbai/mldb.git","clone_url":"https://github.com/mldbai/mldb.git","svn_url":"https://github.com/mldbai/mldb","homepage":"http://mldb.ai","size":22251,"stargazers_count":170,"watchers_count":170,"language":"C++","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":11,"mirror_url":null,"open_issues_count":7,"forks":11,"open_issues":7,"watchers":170,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mldbai/mldb/pulls/471"},"html":{"href":"https://github.com/mldbai/mldb/pull/471"},"issue":{"href":"https://api.github.com/repos/mldbai/mldb/issues/471"},"comments":{"href":"https://api.github.com/repos/mldbai/mldb/issues/471/comments"},"review_comments":{"href":"https://api.github.com/repos/mldbai/mldb/pulls/471/comments"},"review_comment":{"href":"https://api.github.com/repos/mldbai/mldb/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mldbai/mldb/pulls/471/commits"},"statuses":{"href":"https://api.github.com/repos/mldbai/mldb/statuses/f82f39a2ba4a027e098856d97cc46f3037eb99fa"}}}}
{"actor":{"display_login":"mailletf"}}