{"action":"created","comment":{"url":"https://api.github.com/repos/src-d/ml/pulls/comments/171502334","pull_request_review_id":100360552,"id":171502334,"diff_hunk":"@@ -88,52 +41,29 @@ def add_engine_args(my_parser, default_packages=None):\n                            help=\"Print the PySpark execution plans.\")\n \n \n-def create_spark(session_name,\n-                 spark=SparkDefault.MASTER_ADDRESS,\n-                 spark_local_dir=SparkDefault.LOCAL_DIR,\n-                 config=SparkDefault.CONFIG,\n-                 packages=SparkDefault.PACKAGES,\n-                 spark_log_level=SparkDefault.LOG_LEVEL,\n-                 **_):  # **kwargs are discarded for convenience\n-    log = logging.getLogger(\"spark\")\n-    log.info(\"Starting %s on %s\", session_name, spark)\n-    builder = SparkSession.builder.master(spark).appName(session_name)\n-    builder = builder.config(\n-        \"spark.jars.packages\", \",\".join(packages))\n-    builder = builder.config(\"spark.local.dir\", spark_local_dir)\n-    for cfg in config:\n-        builder = builder.config(*cfg.split(\"=\", 1))\n-    session = builder.getOrCreate()\n-    session.sparkContext.setLogLevel(spark_log_level)\n-    # Hide py4j verbose logging (It appears in travis mostly)\n-    logging.getLogger(\"py4j\").setLevel(logging.WARNING)\n-    return session\n+def add_engine_dependencies(engine=EngineDefault.VERSION, config=None, packages=None):\n+    # TODO(vmarkovtsev): figure out why is this option needed\n+    if engine is not None:\n+        config.append(\"spark.tech.sourced.engine.cleanup.skip=true\")\n+        packages.append(\"tech.sourced:engine:\" + engine)\n+\n+\n+def add_bblfsh_dependencies(bblfsh, config=None):\n+    if bblfsh is not None:\n+        config.append(\"spark.tech.sourced.bblfsh.grpc.host=\" + bblfsh)\n \n \n def create_engine(session_name, repositories,\n                   bblfsh=EngineDefault.BBLFSH,\n                   engine=EngineDefault.VERSION,\n                   config=None, packages=None, memory=\"\",\n                   repository_format=\"siva\", **spark_kwargs):\n-    if config is None:\n-        config = []\n-    if packages is None:\n-        packages = []\n-    config.append(\"spark.tech.sourced.bblfsh.grpc.host=\" + bblfsh)\n-    # TODO(vmarkovtsev): figure out why is this option needed\n-    config.append(\"spark.tech.sourced.engine.cleanup.skip=true\")\n-    packages.append(\"tech.sourced:engine:\" + engine)\n-    memory_conf = []\n-    if memory:\n-        memory = memory.split(\",\")\n-        err = \"Expected 3 memory parameters but got {}. \" \\\n-              \"Please check --help to see how -m/--memory should be used.\"\n-        assert len(memory) == 3, err.format(len(memory))\n-        memory_conf.append(\"spark.executor.memory=\" + memory[0])\n-        memory_conf.append(\"spark.driver.memory=\" + memory[1])\n-        memory_conf.append(\"spark.driver.maxResultSize=\" + memory[2])\n-    session = create_spark(session_name, config=config + memory_conf, packages=packages,\n-                           **spark_kwargs)\n+    config, packages = spark_config(config=config, packages=packages, memory=memory)\n+","path":"sourced/ml/utils/engine.py","position":119,"original_position":119,"commit_id":"e6f2abccc9abd00dcc8ba43448e9f23ba3593ab2","original_commit_id":"e6f2abccc9abd00dcc8ba43448e9f23ba3593ab2","user":{"login":"vmarkovtsev","id":2793551,"avatar_url":"https://avatars3.githubusercontent.com/u/2793551?v=4","gravatar_id":"","url":"https://api.github.com/users/vmarkovtsev","html_url":"https://github.com/vmarkovtsev","followers_url":"https://api.github.com/users/vmarkovtsev/followers","following_url":"https://api.github.com/users/vmarkovtsev/following{/other_user}","gists_url":"https://api.github.com/users/vmarkovtsev/gists{/gist_id}","starred_url":"https://api.github.com/users/vmarkovtsev/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vmarkovtsev/subscriptions","organizations_url":"https://api.github.com/users/vmarkovtsev/orgs","repos_url":"https://api.github.com/users/vmarkovtsev/repos","events_url":"https://api.github.com/users/vmarkovtsev/events{/privacy}","received_events_url":"https://api.github.com/users/vmarkovtsev/received_events","type":"User","site_admin":false},"body":"rm \\n","created_at":"2018-03-01T09:30:16Z","updated_at":"2018-03-01T09:31:51Z","html_url":"https://github.com/src-d/ml/pull/187#discussion_r171502334","pull_request_url":"https://api.github.com/repos/src-d/ml/pulls/187","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/src-d/ml/pulls/comments/171502334"},"html":{"href":"https://github.com/src-d/ml/pull/187#discussion_r171502334"},"pull_request":{"href":"https://api.github.com/repos/src-d/ml/pulls/187"}}},"pull_request":{"url":"https://api.github.com/repos/src-d/ml/pulls/187","id":171717323,"html_url":"https://github.com/src-d/ml/pull/187","diff_url":"https://github.com/src-d/ml/pull/187.diff","patch_url":"https://github.com/src-d/ml/pull/187.patch","issue_url":"https://api.github.com/repos/src-d/ml/issues/187","number":187,"state":"open","locked":false,"title":"Separate spark functionality from engine","user":{"login":"EgorBu","id":2606702,"avatar_url":"https://avatars0.githubusercontent.com/u/2606702?v=4","gravatar_id":"","url":"https://api.github.com/users/EgorBu","html_url":"https://github.com/EgorBu","followers_url":"https://api.github.com/users/EgorBu/followers","following_url":"https://api.github.com/users/EgorBu/following{/other_user}","gists_url":"https://api.github.com/users/EgorBu/gists{/gist_id}","starred_url":"https://api.github.com/users/EgorBu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/EgorBu/subscriptions","organizations_url":"https://api.github.com/users/EgorBu/orgs","repos_url":"https://api.github.com/users/EgorBu/repos","events_url":"https://api.github.com/users/EgorBu/events{/privacy}","received_events_url":"https://api.github.com/users/EgorBu/received_events","type":"User","site_admin":false},"body":"Make able to use repos2bow with parquet files as input.\r\nSigned-off-by: egor <egor@sourced.tech>","created_at":"2018-02-27T16:37:02Z","updated_at":"2018-03-01T09:31:51Z","closed_at":null,"merged_at":null,"merge_commit_sha":"0189375e67dcbba3b4c72c75f078e01a74c19e10","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/src-d/ml/pulls/187/commits","review_comments_url":"https://api.github.com/repos/src-d/ml/pulls/187/comments","review_comment_url":"https://api.github.com/repos/src-d/ml/pulls/comments{/number}","comments_url":"https://api.github.com/repos/src-d/ml/issues/187/comments","statuses_url":"https://api.github.com/repos/src-d/ml/statuses/e6f2abccc9abd00dcc8ba43448e9f23ba3593ab2","head":{"label":"EgorBu:master","ref":"master","sha":"e6f2abccc9abd00dcc8ba43448e9f23ba3593ab2","user":{"login":"EgorBu","id":2606702,"avatar_url":"https://avatars0.githubusercontent.com/u/2606702?v=4","gravatar_id":"","url":"https://api.github.com/users/EgorBu","html_url":"https://github.com/EgorBu","followers_url":"https://api.github.com/users/EgorBu/followers","following_url":"https://api.github.com/users/EgorBu/following{/other_user}","gists_url":"https://api.github.com/users/EgorBu/gists{/gist_id}","starred_url":"https://api.github.com/users/EgorBu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/EgorBu/subscriptions","organizations_url":"https://api.github.com/users/EgorBu/orgs","repos_url":"https://api.github.com/users/EgorBu/repos","events_url":"https://api.github.com/users/EgorBu/events{/privacy}","received_events_url":"https://api.github.com/users/EgorBu/received_events","type":"User","site_admin":false},"repo":{"id":96125842,"name":"ml","full_name":"EgorBu/ml","owner":{"login":"EgorBu","id":2606702,"avatar_url":"https://avatars0.githubusercontent.com/u/2606702?v=4","gravatar_id":"","url":"https://api.github.com/users/EgorBu","html_url":"https://github.com/EgorBu","followers_url":"https://api.github.com/users/EgorBu/followers","following_url":"https://api.github.com/users/EgorBu/following{/other_user}","gists_url":"https://api.github.com/users/EgorBu/gists{/gist_id}","starred_url":"https://api.github.com/users/EgorBu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/EgorBu/subscriptions","organizations_url":"https://api.github.com/users/EgorBu/orgs","repos_url":"https://api.github.com/users/EgorBu/repos","events_url":"https://api.github.com/users/EgorBu/events{/privacy}","received_events_url":"https://api.github.com/users/EgorBu/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/EgorBu/ml","description":null,"fork":true,"url":"https://api.github.com/repos/EgorBu/ml","forks_url":"https://api.github.com/repos/EgorBu/ml/forks","keys_url":"https://api.github.com/repos/EgorBu/ml/keys{/key_id}","collaborators_url":"https://api.github.com/repos/EgorBu/ml/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/EgorBu/ml/teams","hooks_url":"https://api.github.com/repos/EgorBu/ml/hooks","issue_events_url":"https://api.github.com/repos/EgorBu/ml/issues/events{/number}","events_url":"https://api.github.com/repos/EgorBu/ml/events","assignees_url":"https://api.github.com/repos/EgorBu/ml/assignees{/user}","branches_url":"https://api.github.com/repos/EgorBu/ml/branches{/branch}","tags_url":"https://api.github.com/repos/EgorBu/ml/tags","blobs_url":"https://api.github.com/repos/EgorBu/ml/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/EgorBu/ml/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/EgorBu/ml/git/refs{/sha}","trees_url":"https://api.github.com/repos/EgorBu/ml/git/trees{/sha}","statuses_url":"https://api.github.com/repos/EgorBu/ml/statuses/{sha}","languages_url":"https://api.github.com/repos/EgorBu/ml/languages","stargazers_url":"https://api.github.com/repos/EgorBu/ml/stargazers","contributors_url":"https://api.github.com/repos/EgorBu/ml/contributors","subscribers_url":"https://api.github.com/repos/EgorBu/ml/subscribers","subscription_url":"https://api.github.com/repos/EgorBu/ml/subscription","commits_url":"https://api.github.com/repos/EgorBu/ml/commits{/sha}","git_commits_url":"https://api.github.com/repos/EgorBu/ml/git/commits{/sha}","comments_url":"https://api.github.com/repos/EgorBu/ml/comments{/number}","issue_comment_url":"https://api.github.com/repos/EgorBu/ml/issues/comments{/number}","contents_url":"https://api.github.com/repos/EgorBu/ml/contents/{+path}","compare_url":"https://api.github.com/repos/EgorBu/ml/compare/{base}...{head}","merges_url":"https://api.github.com/repos/EgorBu/ml/merges","archive_url":"https://api.github.com/repos/EgorBu/ml/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/EgorBu/ml/downloads","issues_url":"https://api.github.com/repos/EgorBu/ml/issues{/number}","pulls_url":"https://api.github.com/repos/EgorBu/ml/pulls{/number}","milestones_url":"https://api.github.com/repos/EgorBu/ml/milestones{/number}","notifications_url":"https://api.github.com/repos/EgorBu/ml/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/EgorBu/ml/labels{/name}","releases_url":"https://api.github.com/repos/EgorBu/ml/releases{/id}","deployments_url":"https://api.github.com/repos/EgorBu/ml/deployments","created_at":"2017-07-03T15:34:15Z","updated_at":"2017-11-15T14:38:27Z","pushed_at":"2018-02-27T16:36:09Z","git_url":"git://github.com/EgorBu/ml.git","ssh_url":"git@github.com:EgorBu/ml.git","clone_url":"https://github.com/EgorBu/ml.git","svn_url":"https://github.com/EgorBu/ml","homepage":null,"size":27120,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"other","name":"Other","spdx_id":null,"url":null},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"src-d:master","ref":"master","sha":"eb1f8b91ed48058b654689ae4cc834e2a6ae8d79","user":{"login":"src-d","id":15128793,"avatar_url":"https://avatars2.githubusercontent.com/u/15128793?v=4","gravatar_id":"","url":"https://api.github.com/users/src-d","html_url":"https://github.com/src-d","followers_url":"https://api.github.com/users/src-d/followers","following_url":"https://api.github.com/users/src-d/following{/other_user}","gists_url":"https://api.github.com/users/src-d/gists{/gist_id}","starred_url":"https://api.github.com/users/src-d/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/src-d/subscriptions","organizations_url":"https://api.github.com/users/src-d/orgs","repos_url":"https://api.github.com/users/src-d/repos","events_url":"https://api.github.com/users/src-d/events{/privacy}","received_events_url":"https://api.github.com/users/src-d/received_events","type":"Organization","site_admin":false},"repo":{"id":94111019,"name":"ml","full_name":"src-d/ml","owner":{"login":"src-d","id":15128793,"avatar_url":"https://avatars2.githubusercontent.com/u/15128793?v=4","gravatar_id":"","url":"https://api.github.com/users/src-d","html_url":"https://github.com/src-d","followers_url":"https://api.github.com/users/src-d/followers","following_url":"https://api.github.com/users/src-d/following{/other_user}","gists_url":"https://api.github.com/users/src-d/gists{/gist_id}","starred_url":"https://api.github.com/users/src-d/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/src-d/subscriptions","organizations_url":"https://api.github.com/users/src-d/orgs","repos_url":"https://api.github.com/users/src-d/repos","events_url":"https://api.github.com/users/src-d/events{/privacy}","received_events_url":"https://api.github.com/users/src-d/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/src-d/ml","description":"sourced.ml is a library to build and apply machine learning models on top of Universal Abstract Syntax Trees","fork":false,"url":"https://api.github.com/repos/src-d/ml","forks_url":"https://api.github.com/repos/src-d/ml/forks","keys_url":"https://api.github.com/repos/src-d/ml/keys{/key_id}","collaborators_url":"https://api.github.com/repos/src-d/ml/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/src-d/ml/teams","hooks_url":"https://api.github.com/repos/src-d/ml/hooks","issue_events_url":"https://api.github.com/repos/src-d/ml/issues/events{/number}","events_url":"https://api.github.com/repos/src-d/ml/events","assignees_url":"https://api.github.com/repos/src-d/ml/assignees{/user}","branches_url":"https://api.github.com/repos/src-d/ml/branches{/branch}","tags_url":"https://api.github.com/repos/src-d/ml/tags","blobs_url":"https://api.github.com/repos/src-d/ml/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/src-d/ml/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/src-d/ml/git/refs{/sha}","trees_url":"https://api.github.com/repos/src-d/ml/git/trees{/sha}","statuses_url":"https://api.github.com/repos/src-d/ml/statuses/{sha}","languages_url":"https://api.github.com/repos/src-d/ml/languages","stargazers_url":"https://api.github.com/repos/src-d/ml/stargazers","contributors_url":"https://api.github.com/repos/src-d/ml/contributors","subscribers_url":"https://api.github.com/repos/src-d/ml/subscribers","subscription_url":"https://api.github.com/repos/src-d/ml/subscription","commits_url":"https://api.github.com/repos/src-d/ml/commits{/sha}","git_commits_url":"https://api.github.com/repos/src-d/ml/git/commits{/sha}","comments_url":"https://api.github.com/repos/src-d/ml/comments{/number}","issue_comment_url":"https://api.github.com/repos/src-d/ml/issues/comments{/number}","contents_url":"https://api.github.com/repos/src-d/ml/contents/{+path}","compare_url":"https://api.github.com/repos/src-d/ml/compare/{base}...{head}","merges_url":"https://api.github.com/repos/src-d/ml/merges","archive_url":"https://api.github.com/repos/src-d/ml/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/src-d/ml/downloads","issues_url":"https://api.github.com/repos/src-d/ml/issues{/number}","pulls_url":"https://api.github.com/repos/src-d/ml/pulls{/number}","milestones_url":"https://api.github.com/repos/src-d/ml/milestones{/number}","notifications_url":"https://api.github.com/repos/src-d/ml/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/src-d/ml/labels{/name}","releases_url":"https://api.github.com/repos/src-d/ml/releases{/id}","deployments_url":"https://api.github.com/repos/src-d/ml/deployments","created_at":"2017-06-12T15:21:41Z","updated_at":"2018-02-27T06:51:27Z","pushed_at":"2018-03-01T02:12:24Z","git_url":"git://github.com/src-d/ml.git","ssh_url":"git@github.com:src-d/ml.git","clone_url":"https://github.com/src-d/ml.git","svn_url":"https://github.com/src-d/ml","homepage":"","size":27114,"stargazers_count":29,"watchers_count":29,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":15,"mirror_url":null,"archived":false,"open_issues_count":4,"license":{"key":"other","name":"Other","spdx_id":null,"url":null},"forks":15,"open_issues":4,"watchers":29,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/src-d/ml/pulls/187"},"html":{"href":"https://github.com/src-d/ml/pull/187"},"issue":{"href":"https://api.github.com/repos/src-d/ml/issues/187"},"comments":{"href":"https://api.github.com/repos/src-d/ml/issues/187/comments"},"review_comments":{"href":"https://api.github.com/repos/src-d/ml/pulls/187/comments"},"review_comment":{"href":"https://api.github.com/repos/src-d/ml/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/src-d/ml/pulls/187/commits"},"statuses":{"href":"https://api.github.com/repos/src-d/ml/statuses/e6f2abccc9abd00dcc8ba43448e9f23ba3593ab2"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"vmarkovtsev"}}