{"action":"created","comment":{"url":"https://api.github.com/repos/apache/incubator-airflow/pulls/comments/242232376","pull_request_review_id":185696016,"id":242232376,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDI0MjIzMjM3Ng==","diff_hunk":"@@ -563,29 +442,31 @@ def helper():\n         p.start()\n         return p\n \n-    def harvest_simple_dags(self):\n+    def harvest_dags(self):\n         \"\"\"\n         Harvest DAG parsing results from result queue and sync metadata from stat queue.\n-        :return: List of parsing result in SimpleDag format.\n+        :return: DagBag\n         \"\"\"\n         # Metadata and results to be harvested can be inconsistent,\n         # but it should not be a big problem.\n         self._sync_metadata()\n         # Heartbeating after syncing metadata so we do not restart manager\n         # if it processed all files for max_run times and exit normally.\n         self._heartbeat_manager()\n-        simple_dags = []\n+        dags = []\n         # multiprocessing.Queue().qsize will not work on MacOS.\n         if sys.platform == \"darwin\":\n             qsize = self._result_count\n         else:\n             qsize = self._result_queue.qsize()\n         for _ in range(qsize):\n-            simple_dags.append(self._result_queue.get())\n+            dags.append(self._result_queue.get())\n \n         self._result_count = 0\n \n-        return simple_dags\n+        for dag in dags:\n+            self.dag_bag.dags[dag.dag_id] = dag","path":"airflow/utils/dag_processing.py","position":200,"original_position":200,"commit_id":"eef41ad2de70e204fbfa23482745823257276a59","original_commit_id":"eef41ad2de70e204fbfa23482745823257276a59","user":{"login":"ashb","id":34150,"node_id":"MDQ6VXNlcjM0MTUw","avatar_url":"https://avatars2.githubusercontent.com/u/34150?v=4","gravatar_id":"","url":"https://api.github.com/users/ashb","html_url":"https://github.com/ashb","followers_url":"https://api.github.com/users/ashb/followers","following_url":"https://api.github.com/users/ashb/following{/other_user}","gists_url":"https://api.github.com/users/ashb/gists{/gist_id}","starred_url":"https://api.github.com/users/ashb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ashb/subscriptions","organizations_url":"https://api.github.com/users/ashb/orgs","repos_url":"https://api.github.com/users/ashb/repos","events_url":"https://api.github.com/users/ashb/events{/privacy}","received_events_url":"https://api.github.com/users/ashb/received_events","type":"User","site_admin":false},"body":"I think there's a `dag_bag.bag_dag()` method that we should use instead?","created_at":"2018-12-17T17:02:23Z","updated_at":"2018-12-17T17:02:23Z","html_url":"https://github.com/apache/incubator-airflow/pull/4328#discussion_r242232376","pull_request_url":"https://api.github.com/repos/apache/incubator-airflow/pulls/4328","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-airflow/pulls/comments/242232376"},"html":{"href":"https://github.com/apache/incubator-airflow/pull/4328#discussion_r242232376"},"pull_request":{"href":"https://api.github.com/repos/apache/incubator-airflow/pulls/4328"}}},"pull_request":{"url":"https://api.github.com/repos/apache/incubator-airflow/pulls/4328","id":238986113,"node_id":"MDExOlB1bGxSZXF1ZXN0MjM4OTg2MTEz","html_url":"https://github.com/apache/incubator-airflow/pull/4328","diff_url":"https://github.com/apache/incubator-airflow/pull/4328.diff","patch_url":"https://github.com/apache/incubator-airflow/pull/4328.patch","issue_url":"https://api.github.com/repos/apache/incubator-airflow/issues/4328","number":4328,"state":"open","locked":false,"title":"AIRFLOW-3530 - No re-parsing of dag files","user":{"login":"ffinfo","id":2419555,"node_id":"MDQ6VXNlcjI0MTk1NTU=","avatar_url":"https://avatars0.githubusercontent.com/u/2419555?v=4","gravatar_id":"","url":"https://api.github.com/users/ffinfo","html_url":"https://github.com/ffinfo","followers_url":"https://api.github.com/users/ffinfo/followers","following_url":"https://api.github.com/users/ffinfo/following{/other_user}","gists_url":"https://api.github.com/users/ffinfo/gists{/gist_id}","starred_url":"https://api.github.com/users/ffinfo/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ffinfo/subscriptions","organizations_url":"https://api.github.com/users/ffinfo/orgs","repos_url":"https://api.github.com/users/ffinfo/repos","events_url":"https://api.github.com/users/ffinfo/events{/privacy}","received_events_url":"https://api.github.com/users/ffinfo/received_events","type":"User","site_admin":false},"body":"Make sure you have checked _all_ steps below.\r\n\r\n### Jira\r\n\r\n- [x] My PR addresses the following [Airflow Jira]\r\n  - https://issues.apache.org/jira/browse/AIRFLOW-3530\r\n\r\n### Description\r\n\r\nThis will parse the dag files only when a change is detected. Current implementation reads/parse the python files on each heartbeat of the scheduler.\r\n\r\n### Tests\r\n\r\n- [x] My PR adds the following unit tests __OR__ does not need testing for this extremely good reason:\r\n\r\n### Commits\r\n\r\n- [ ] My commits all reference Jira issues in their subject lines, and I have squashed multiple commits if they address the same issue. In addition, my commits follow the guidelines from \"[How to write a good git commit message](http://chris.beams.io/posts/git-commit/)\":\r\n  1. Subject is separated from body by a blank line\r\n  1. Subject is limited to 50 characters (not including Jira issue reference)\r\n  1. Subject does not end with a period\r\n  1. Subject uses the imperative mood (\"add\", not \"adding\")\r\n  1. Body wraps at 72 characters\r\n  1. Body explains \"what\" and \"why\", not \"how\"\r\n\r\n### Documentation\r\n\r\n  - [ ] All the public functions and the classes in the PR contain docstrings that explain what it does\r\n\r\n### Code Quality\r\n\r\n- [x] Passes `flake8`","created_at":"2018-12-16T19:45:37Z","updated_at":"2018-12-17T17:02:23Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6ecef2c55dac17a2de72550ce24700434512b2ca","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/incubator-airflow/pulls/4328/commits","review_comments_url":"https://api.github.com/repos/apache/incubator-airflow/pulls/4328/comments","review_comment_url":"https://api.github.com/repos/apache/incubator-airflow/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/incubator-airflow/issues/4328/comments","statuses_url":"https://api.github.com/repos/apache/incubator-airflow/statuses/eef41ad2de70e204fbfa23482745823257276a59","head":{"label":"ffinfo:recuce_parsing","ref":"recuce_parsing","sha":"eef41ad2de70e204fbfa23482745823257276a59","user":{"login":"ffinfo","id":2419555,"node_id":"MDQ6VXNlcjI0MTk1NTU=","avatar_url":"https://avatars0.githubusercontent.com/u/2419555?v=4","gravatar_id":"","url":"https://api.github.com/users/ffinfo","html_url":"https://github.com/ffinfo","followers_url":"https://api.github.com/users/ffinfo/followers","following_url":"https://api.github.com/users/ffinfo/following{/other_user}","gists_url":"https://api.github.com/users/ffinfo/gists{/gist_id}","starred_url":"https://api.github.com/users/ffinfo/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ffinfo/subscriptions","organizations_url":"https://api.github.com/users/ffinfo/orgs","repos_url":"https://api.github.com/users/ffinfo/repos","events_url":"https://api.github.com/users/ffinfo/events{/privacy}","received_events_url":"https://api.github.com/users/ffinfo/received_events","type":"User","site_admin":false},"repo":{"id":159806891,"node_id":"MDEwOlJlcG9zaXRvcnkxNTk4MDY4OTE=","name":"incubator-airflow","full_name":"ffinfo/incubator-airflow","private":false,"owner":{"login":"ffinfo","id":2419555,"node_id":"MDQ6VXNlcjI0MTk1NTU=","avatar_url":"https://avatars0.githubusercontent.com/u/2419555?v=4","gravatar_id":"","url":"https://api.github.com/users/ffinfo","html_url":"https://github.com/ffinfo","followers_url":"https://api.github.com/users/ffinfo/followers","following_url":"https://api.github.com/users/ffinfo/following{/other_user}","gists_url":"https://api.github.com/users/ffinfo/gists{/gist_id}","starred_url":"https://api.github.com/users/ffinfo/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ffinfo/subscriptions","organizations_url":"https://api.github.com/users/ffinfo/orgs","repos_url":"https://api.github.com/users/ffinfo/repos","events_url":"https://api.github.com/users/ffinfo/events{/privacy}","received_events_url":"https://api.github.com/users/ffinfo/received_events","type":"User","site_admin":false},"html_url":"https://github.com/ffinfo/incubator-airflow","description":"Apache Airflow (Incubating)","fork":true,"url":"https://api.github.com/repos/ffinfo/incubator-airflow","forks_url":"https://api.github.com/repos/ffinfo/incubator-airflow/forks","keys_url":"https://api.github.com/repos/ffinfo/incubator-airflow/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ffinfo/incubator-airflow/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ffinfo/incubator-airflow/teams","hooks_url":"https://api.github.com/repos/ffinfo/incubator-airflow/hooks","issue_events_url":"https://api.github.com/repos/ffinfo/incubator-airflow/issues/events{/number}","events_url":"https://api.github.com/repos/ffinfo/incubator-airflow/events","assignees_url":"https://api.github.com/repos/ffinfo/incubator-airflow/assignees{/user}","branches_url":"https://api.github.com/repos/ffinfo/incubator-airflow/branches{/branch}","tags_url":"https://api.github.com/repos/ffinfo/incubator-airflow/tags","blobs_url":"https://api.github.com/repos/ffinfo/incubator-airflow/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ffinfo/incubator-airflow/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ffinfo/incubator-airflow/git/refs{/sha}","trees_url":"https://api.github.com/repos/ffinfo/incubator-airflow/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ffinfo/incubator-airflow/statuses/{sha}","languages_url":"https://api.github.com/repos/ffinfo/incubator-airflow/languages","stargazers_url":"https://api.github.com/repos/ffinfo/incubator-airflow/stargazers","contributors_url":"https://api.github.com/repos/ffinfo/incubator-airflow/contributors","subscribers_url":"https://api.github.com/repos/ffinfo/incubator-airflow/subscribers","subscription_url":"https://api.github.com/repos/ffinfo/incubator-airflow/subscription","commits_url":"https://api.github.com/repos/ffinfo/incubator-airflow/commits{/sha}","git_commits_url":"https://api.github.com/repos/ffinfo/incubator-airflow/git/commits{/sha}","comments_url":"https://api.github.com/repos/ffinfo/incubator-airflow/comments{/number}","issue_comment_url":"https://api.github.com/repos/ffinfo/incubator-airflow/issues/comments{/number}","contents_url":"https://api.github.com/repos/ffinfo/incubator-airflow/contents/{+path}","compare_url":"https://api.github.com/repos/ffinfo/incubator-airflow/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ffinfo/incubator-airflow/merges","archive_url":"https://api.github.com/repos/ffinfo/incubator-airflow/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ffinfo/incubator-airflow/downloads","issues_url":"https://api.github.com/repos/ffinfo/incubator-airflow/issues{/number}","pulls_url":"https://api.github.com/repos/ffinfo/incubator-airflow/pulls{/number}","milestones_url":"https://api.github.com/repos/ffinfo/incubator-airflow/milestones{/number}","notifications_url":"https://api.github.com/repos/ffinfo/incubator-airflow/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ffinfo/incubator-airflow/labels{/name}","releases_url":"https://api.github.com/repos/ffinfo/incubator-airflow/releases{/id}","deployments_url":"https://api.github.com/repos/ffinfo/incubator-airflow/deployments","created_at":"2018-11-30T10:24:23Z","updated_at":"2018-11-30T10:24:29Z","pushed_at":"2018-12-17T13:28:16Z","git_url":"git://github.com/ffinfo/incubator-airflow.git","ssh_url":"git@github.com:ffinfo/incubator-airflow.git","clone_url":"https://github.com/ffinfo/incubator-airflow.git","svn_url":"https://github.com/ffinfo/incubator-airflow","homepage":"https://airflow.apache.org","size":28735,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"1023e8c1bc9e046b32442cacd8d046a467b9bbc9","user":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":33884891,"node_id":"MDEwOlJlcG9zaXRvcnkzMzg4NDg5MQ==","name":"incubator-airflow","full_name":"apache/incubator-airflow","private":false,"owner":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/apache/incubator-airflow","description":"Apache Airflow (Incubating)","fork":false,"url":"https://api.github.com/repos/apache/incubator-airflow","forks_url":"https://api.github.com/repos/apache/incubator-airflow/forks","keys_url":"https://api.github.com/repos/apache/incubator-airflow/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/incubator-airflow/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/incubator-airflow/teams","hooks_url":"https://api.github.com/repos/apache/incubator-airflow/hooks","issue_events_url":"https://api.github.com/repos/apache/incubator-airflow/issues/events{/number}","events_url":"https://api.github.com/repos/apache/incubator-airflow/events","assignees_url":"https://api.github.com/repos/apache/incubator-airflow/assignees{/user}","branches_url":"https://api.github.com/repos/apache/incubator-airflow/branches{/branch}","tags_url":"https://api.github.com/repos/apache/incubator-airflow/tags","blobs_url":"https://api.github.com/repos/apache/incubator-airflow/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/incubator-airflow/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/incubator-airflow/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/incubator-airflow/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/incubator-airflow/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/incubator-airflow/languages","stargazers_url":"https://api.github.com/repos/apache/incubator-airflow/stargazers","contributors_url":"https://api.github.com/repos/apache/incubator-airflow/contributors","subscribers_url":"https://api.github.com/repos/apache/incubator-airflow/subscribers","subscription_url":"https://api.github.com/repos/apache/incubator-airflow/subscription","commits_url":"https://api.github.com/repos/apache/incubator-airflow/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/incubator-airflow/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/incubator-airflow/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/incubator-airflow/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/incubator-airflow/contents/{+path}","compare_url":"https://api.github.com/repos/apache/incubator-airflow/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/incubator-airflow/merges","archive_url":"https://api.github.com/repos/apache/incubator-airflow/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/incubator-airflow/downloads","issues_url":"https://api.github.com/repos/apache/incubator-airflow/issues{/number}","pulls_url":"https://api.github.com/repos/apache/incubator-airflow/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/incubator-airflow/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/incubator-airflow/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/incubator-airflow/labels{/name}","releases_url":"https://api.github.com/repos/apache/incubator-airflow/releases{/id}","deployments_url":"https://api.github.com/repos/apache/incubator-airflow/deployments","created_at":"2015-04-13T18:04:58Z","updated_at":"2018-12-17T12:03:30Z","pushed_at":"2018-12-17T15:51:37Z","git_url":"git://github.com/apache/incubator-airflow.git","ssh_url":"git@github.com:apache/incubator-airflow.git","clone_url":"https://github.com/apache/incubator-airflow.git","svn_url":"https://github.com/apache/incubator-airflow","homepage":"https://airflow.apache.org","size":28831,"stargazers_count":10290,"watchers_count":10290,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3601,"mirror_url":null,"archived":false,"open_issues_count":316,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":3601,"open_issues":316,"watchers":10290,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-airflow/pulls/4328"},"html":{"href":"https://github.com/apache/incubator-airflow/pull/4328"},"issue":{"href":"https://api.github.com/repos/apache/incubator-airflow/issues/4328"},"comments":{"href":"https://api.github.com/repos/apache/incubator-airflow/issues/4328/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/incubator-airflow/pulls/4328/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/incubator-airflow/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/incubator-airflow/pulls/4328/commits"},"statuses":{"href":"https://api.github.com/repos/apache/incubator-airflow/statuses/eef41ad2de70e204fbfa23482745823257276a59"}},"author_association":"NONE"}}
{"actor":{"display_login":"ashb"}}