{"action":"created","comment":{"url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/pulls/comments/221178365","pull_request_review_id":159765411,"id":221178365,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIyMTE3ODM2NQ==","diff_hunk":"@@ -0,0 +1,173 @@\n+import os\n+import re\n+import yaml\n+import argparse\n+\n+from bimaterializedviewsautomation.parsing_utils import is_incremental\n+\n+current_path = os.path.abspath(os.path.join(__file__, os.pardir))\n+VIEWS_BASE = os.path.join(current_path, '..', '..', '..', 'bi-materialized-views')\n+VIEWS_DB = 'materialized_views'\n+\n+\n+def schedule_interval_weekly(schedule_time):\n+    hour = 20\n+    minute = 00\n+    week_day = schedule_time[:3].upper()\n+    return '{minute} {hour} * * {week_day}'.format(\n+        minute=minute,\n+        hour=hour,\n+        week_day=week_day\n+    )\n+\n+\n+def match_schedule_weekly(script_text):\n+    script_text_lower = script_text.lower()\n+    weekly = re.search(\n+        '@schedule weekly (?P<time>monday|tuesday|wednesday|thursday|friday|saturday|sunday)',\n+        script_text_lower\n+    )\n+    if weekly:\n+        return weekly.group('time')\n+    else:\n+        return None\n+\n+\n+def match_schedule_daily(script_text):\n+    script_text_lower = script_text.lower()\n+    daily = re.search(\n+        '@schedule daily (?P<time>morning|midday|evening)',\n+        script_text_lower\n+    )\n+    if daily:\n+        return daily.group('time')\n+    else:\n+        return None\n+\n+\n+def schedule_interval_daily(schedule_time):\n+    minute = 00\n+    if schedule_time == 'morning':\n+        hour = 06\n+    elif schedule_time == 'midday':\n+        hour = 12\n+    elif schedule_time == 'evening':\n+        hour = 19\n+    else:\n+        raise ValueError('Unknown date')\n+\n+    return '{minute} {hour} * * *'.format(\n+        minute=minute,\n+        hour=hour,\n+    )\n+\n+\n+def is_marked_for_impala(script_text):\n+    return len(re.findall(\"@%s\" % 'impala', script_text)) > 0\n+\n+\n+def read_mat_views(delete=False):\n+    mat_views = {}\n+    mat_views_path = os.path.join(VIEWS_BASE, VIEWS_DB)\n+    for script_file_name in os.listdir(mat_views_path):\n+        script_file_full_path = os.path.join(mat_views_path, script_file_name)\n+        if os.path.isdir(script_file_full_path):\n+            continue\n+        with open(script_file_full_path, \"r\") as query_file:\n+            text = query_file.read()\n+        incremental = is_incremental(text)\n+        is_impala = is_marked_for_impala(text)\n+        weekly = match_schedule_weekly(text)\n+        daily = match_schedule_daily(text)\n+        if weekly:\n+            schedule = schedule_interval_weekly(weekly)\n+            name = \"weekly-%s\" % weekly\n+        elif daily:\n+            schedule = schedule_interval_daily(daily)\n+            name = \"daily-%s\" % daily\n+        else:\n+            raise ValueError('Unrecognized time in mat view %s' % script_file_name)\n+        if name not in mat_views:\n+            mat_views[name] = {'schedule': schedule, 'views': []}\n+        mat_views[name]['views'].append((text, incremental, is_impala, script_file_name.split('.')[0],))\n+        if delete:\n+            os.remove(script_file_full_path)\n+    return mat_views\n+\n+\n+def create_yml(name, schedule, entries):\n+    yml_data = [{\n+        'etl_name': name,\n+        'desc': None,\n+        'owners': None,\n+        'schedule': schedule,\n+        'tasks': []\n+    }]\n+\n+    for text, incremental, is_impala, view_name in entries:\n+        yml_data[0]['tasks'].append({\n+            'name': view_name,\n+            'description': None,\n+            'inputs': [],\n+            'outputs': [{\n+                'type': 'materialized_view',\n+                'table': view_name\n+            }],\n+            'job': {\n+                'type': 'materialized_view',\n+                'incremental': incremental,\n+                'sql': '%s.sql' % view_name,\n+                'impala': is_impala\n+            }\n+        })\n+\n+    return yaml.dump(yml_data)","path":"bimaterializedviewsautomation/manual_scripts/recreate_pipelines_directories.py","position":124,"original_position":124,"commit_id":"bf5b732c9bf2324025e3d52f9f09c8cfc1af918d","original_commit_id":"bf5b732c9bf2324025e3d52f9f09c8cfc1af918d","user":{"login":"siklosid","id":11415774,"node_id":"MDQ6VXNlcjExNDE1Nzc0","avatar_url":"https://avatars0.githubusercontent.com/u/11415774?v=4","gravatar_id":"","url":"https://api.github.com/users/siklosid","html_url":"https://github.com/siklosid","followers_url":"https://api.github.com/users/siklosid/followers","following_url":"https://api.github.com/users/siklosid/following{/other_user}","gists_url":"https://api.github.com/users/siklosid/gists{/gist_id}","starred_url":"https://api.github.com/users/siklosid/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/siklosid/subscriptions","organizations_url":"https://api.github.com/users/siklosid/orgs","repos_url":"https://api.github.com/users/siklosid/repos","events_url":"https://api.github.com/users/siklosid/events{/privacy}","received_events_url":"https://api.github.com/users/siklosid/received_events","type":"User","site_admin":false},"body":"This is ok to merge, but can we recreate the yaml's with this parameter? https://stackoverflow.com/questions/12246605/python-yaml-controlling-output-format","created_at":"2018-09-28T08:34:45Z","updated_at":"2018-09-28T08:34:45Z","html_url":"https://github.com/hellofresh/bi-materialized-views-automation/pull/42#discussion_r221178365","pull_request_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/pulls/42","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/pulls/comments/221178365"},"html":{"href":"https://github.com/hellofresh/bi-materialized-views-automation/pull/42#discussion_r221178365"},"pull_request":{"href":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/pulls/42"}}},"pull_request":{"url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/pulls/42","id":218717864,"node_id":"MDExOlB1bGxSZXF1ZXN0MjE4NzE3ODY0","html_url":"https://github.com/hellofresh/bi-materialized-views-automation/pull/42","diff_url":"https://github.com/hellofresh/bi-materialized-views-automation/pull/42.diff","patch_url":"https://github.com/hellofresh/bi-materialized-views-automation/pull/42.patch","issue_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/issues/42","number":42,"state":"open","locked":false,"title":"Tmp solution no topological order","user":{"login":"sofiabravo103","id":1728402,"node_id":"MDQ6VXNlcjE3Mjg0MDI=","avatar_url":"https://avatars2.githubusercontent.com/u/1728402?v=4","gravatar_id":"","url":"https://api.github.com/users/sofiabravo103","html_url":"https://github.com/sofiabravo103","followers_url":"https://api.github.com/users/sofiabravo103/followers","following_url":"https://api.github.com/users/sofiabravo103/following{/other_user}","gists_url":"https://api.github.com/users/sofiabravo103/gists{/gist_id}","starred_url":"https://api.github.com/users/sofiabravo103/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sofiabravo103/subscriptions","organizations_url":"https://api.github.com/users/sofiabravo103/orgs","repos_url":"https://api.github.com/users/sofiabravo103/repos","events_url":"https://api.github.com/users/sofiabravo103/events{/privacy}","received_events_url":"https://api.github.com/users/sofiabravo103/received_events","type":"User","site_admin":false},"body":"Automation to work with the new file structure for bi-materialized-views. Creating views in order is not implemented, so to make the checks pass in bi-materialized-views under this automation the views must be created manually before the test. ","created_at":"2018-09-27T16:38:37Z","updated_at":"2018-09-28T08:34:45Z","closed_at":null,"merged_at":null,"merge_commit_sha":"320b457df29b947b00aa5efda56a365c3b8c5e6f","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/pulls/42/commits","review_comments_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/pulls/42/comments","review_comment_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/pulls/comments{/number}","comments_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/issues/42/comments","statuses_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/statuses/bf5b732c9bf2324025e3d52f9f09c8cfc1af918d","head":{"label":"hellofresh:tmp_solution_no_topological_order","ref":"tmp_solution_no_topological_order","sha":"bf5b732c9bf2324025e3d52f9f09c8cfc1af918d","user":{"login":"hellofresh","id":4430027,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ0MzAwMjc=","avatar_url":"https://avatars1.githubusercontent.com/u/4430027?v=4","gravatar_id":"","url":"https://api.github.com/users/hellofresh","html_url":"https://github.com/hellofresh","followers_url":"https://api.github.com/users/hellofresh/followers","following_url":"https://api.github.com/users/hellofresh/following{/other_user}","gists_url":"https://api.github.com/users/hellofresh/gists{/gist_id}","starred_url":"https://api.github.com/users/hellofresh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hellofresh/subscriptions","organizations_url":"https://api.github.com/users/hellofresh/orgs","repos_url":"https://api.github.com/users/hellofresh/repos","events_url":"https://api.github.com/users/hellofresh/events{/privacy}","received_events_url":"https://api.github.com/users/hellofresh/received_events","type":"Organization","site_admin":false},"repo":{"id":143032758,"node_id":"MDEwOlJlcG9zaXRvcnkxNDMwMzI3NTg=","name":"bi-materialized-views-automation","full_name":"hellofresh/bi-materialized-views-automation","private":false,"owner":{"login":"hellofresh","id":4430027,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ0MzAwMjc=","avatar_url":"https://avatars1.githubusercontent.com/u/4430027?v=4","gravatar_id":"","url":"https://api.github.com/users/hellofresh","html_url":"https://github.com/hellofresh","followers_url":"https://api.github.com/users/hellofresh/followers","following_url":"https://api.github.com/users/hellofresh/following{/other_user}","gists_url":"https://api.github.com/users/hellofresh/gists{/gist_id}","starred_url":"https://api.github.com/users/hellofresh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hellofresh/subscriptions","organizations_url":"https://api.github.com/users/hellofresh/orgs","repos_url":"https://api.github.com/users/hellofresh/repos","events_url":"https://api.github.com/users/hellofresh/events{/privacy}","received_events_url":"https://api.github.com/users/hellofresh/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/hellofresh/bi-materialized-views-automation","description":"bi-materialized-views-automation","fork":false,"url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation","forks_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/forks","keys_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/teams","hooks_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/hooks","issue_events_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/issues/events{/number}","events_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/events","assignees_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/assignees{/user}","branches_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/branches{/branch}","tags_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/tags","blobs_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/git/refs{/sha}","trees_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/statuses/{sha}","languages_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/languages","stargazers_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/stargazers","contributors_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/contributors","subscribers_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/subscribers","subscription_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/subscription","commits_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/commits{/sha}","git_commits_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/git/commits{/sha}","comments_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/comments{/number}","issue_comment_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/issues/comments{/number}","contents_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/contents/{+path}","compare_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/merges","archive_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/downloads","issues_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/issues{/number}","pulls_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/pulls{/number}","milestones_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/milestones{/number}","notifications_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/labels{/name}","releases_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/releases{/id}","deployments_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/deployments","created_at":"2018-07-31T15:24:37Z","updated_at":"2018-09-26T13:39:14Z","pushed_at":"2018-09-27T16:47:10Z","git_url":"git://github.com/hellofresh/bi-materialized-views-automation.git","ssh_url":"git@github.com:hellofresh/bi-materialized-views-automation.git","clone_url":"https://github.com/hellofresh/bi-materialized-views-automation.git","svn_url":"https://github.com/hellofresh/bi-materialized-views-automation","homepage":null,"size":160,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":4,"license":null,"forks":0,"open_issues":4,"watchers":0,"default_branch":"master"}},"base":{"label":"hellofresh:v3_pipelines","ref":"v3_pipelines","sha":"34d4b0c294f356a7c4ed01740a9ddb32f663ac02","user":{"login":"hellofresh","id":4430027,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ0MzAwMjc=","avatar_url":"https://avatars1.githubusercontent.com/u/4430027?v=4","gravatar_id":"","url":"https://api.github.com/users/hellofresh","html_url":"https://github.com/hellofresh","followers_url":"https://api.github.com/users/hellofresh/followers","following_url":"https://api.github.com/users/hellofresh/following{/other_user}","gists_url":"https://api.github.com/users/hellofresh/gists{/gist_id}","starred_url":"https://api.github.com/users/hellofresh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hellofresh/subscriptions","organizations_url":"https://api.github.com/users/hellofresh/orgs","repos_url":"https://api.github.com/users/hellofresh/repos","events_url":"https://api.github.com/users/hellofresh/events{/privacy}","received_events_url":"https://api.github.com/users/hellofresh/received_events","type":"Organization","site_admin":false},"repo":{"id":143032758,"node_id":"MDEwOlJlcG9zaXRvcnkxNDMwMzI3NTg=","name":"bi-materialized-views-automation","full_name":"hellofresh/bi-materialized-views-automation","private":false,"owner":{"login":"hellofresh","id":4430027,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ0MzAwMjc=","avatar_url":"https://avatars1.githubusercontent.com/u/4430027?v=4","gravatar_id":"","url":"https://api.github.com/users/hellofresh","html_url":"https://github.com/hellofresh","followers_url":"https://api.github.com/users/hellofresh/followers","following_url":"https://api.github.com/users/hellofresh/following{/other_user}","gists_url":"https://api.github.com/users/hellofresh/gists{/gist_id}","starred_url":"https://api.github.com/users/hellofresh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hellofresh/subscriptions","organizations_url":"https://api.github.com/users/hellofresh/orgs","repos_url":"https://api.github.com/users/hellofresh/repos","events_url":"https://api.github.com/users/hellofresh/events{/privacy}","received_events_url":"https://api.github.com/users/hellofresh/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/hellofresh/bi-materialized-views-automation","description":"bi-materialized-views-automation","fork":false,"url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation","forks_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/forks","keys_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/teams","hooks_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/hooks","issue_events_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/issues/events{/number}","events_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/events","assignees_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/assignees{/user}","branches_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/branches{/branch}","tags_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/tags","blobs_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/git/refs{/sha}","trees_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/statuses/{sha}","languages_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/languages","stargazers_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/stargazers","contributors_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/contributors","subscribers_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/subscribers","subscription_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/subscription","commits_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/commits{/sha}","git_commits_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/git/commits{/sha}","comments_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/comments{/number}","issue_comment_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/issues/comments{/number}","contents_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/contents/{+path}","compare_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/merges","archive_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/downloads","issues_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/issues{/number}","pulls_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/pulls{/number}","milestones_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/milestones{/number}","notifications_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/labels{/name}","releases_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/releases{/id}","deployments_url":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/deployments","created_at":"2018-07-31T15:24:37Z","updated_at":"2018-09-26T13:39:14Z","pushed_at":"2018-09-27T16:47:10Z","git_url":"git://github.com/hellofresh/bi-materialized-views-automation.git","ssh_url":"git@github.com:hellofresh/bi-materialized-views-automation.git","clone_url":"https://github.com/hellofresh/bi-materialized-views-automation.git","svn_url":"https://github.com/hellofresh/bi-materialized-views-automation","homepage":null,"size":160,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":4,"license":null,"forks":0,"open_issues":4,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/pulls/42"},"html":{"href":"https://github.com/hellofresh/bi-materialized-views-automation/pull/42"},"issue":{"href":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/issues/42"},"comments":{"href":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/issues/42/comments"},"review_comments":{"href":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/pulls/42/comments"},"review_comment":{"href":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/pulls/42/commits"},"statuses":{"href":"https://api.github.com/repos/hellofresh/bi-materialized-views-automation/statuses/bf5b732c9bf2324025e3d52f9f09c8cfc1af918d"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"siklosid"}}