{"action":"created","comment":{"url":"https://api.github.com/repos/google/ggrc-core/pulls/comments/73018331","id":73018331,"diff_hunk":"@@ -1159,57 +1200,69 @@ def _check_post_permissions(self, objects):\n       Forbidden error if user does not have create permission for all objects\n       in the objects list.\n     \"\"\"\n-    with benchmark(\"Check create permissions\"):\n-      for obj in objects:\n-        if not permissions.is_allowed_create_for(obj):\n-          # json_create sometimes adds objects to session, so we need to\n-          # make sure the session is cleared\n-          db.session.expunge_all()\n-          raise Forbidden()\n+    for obj in objects:\n+      if not permissions.is_allowed_create_for(obj):\n+        # json_create sometimes adds objects to session, so we need to\n+        # make sure the session is cleared\n+        db.session.expunge_all()\n+        raise Forbidden()\n \n   def collection_post_loop(self, body, res, no_result, running_async):\n-    for wrapped_src in body:\n-      if running_async:\n-        time.sleep(settings.BACKGROUND_COLLECTION_POST_SLEEP)\n+    \"\"\"Handle all posted objects.\n \n-      src = self._unwrap_collection_post_src(wrapped_src)\n-      obj = self._get_model_instance(src)\n-\n-      with benchmark(\"Deserialize object\"):\n-        self.json_create(obj, src)\n-      with benchmark(\"Send model POSTed event\"):\n-        self.model_posted.send(obj.__class__, obj=obj, src=src, service=self)\n-\n-      self._check_post_permissions([obj])\n-\n-      obj.modified_by = get_current_user()\n+    Args:\n+      body: list of dictionaries containing json object representations.\n+      res: List that will get responses appended to it.\n+      no_result: Flag for suppressing results.\n+      running_async: Flag for async jobs.\n+    \"\"\"\n \n-      with benchmark(\"Get modified objects\"):\n-        modified_objects = get_modified_objects(db.session)\n-      with benchmark(\"Update custom attribute values\"):\n-        set_ids_for_new_custom_attributes(modified_objects.new, obj)\n+    with benchmark(\"Generate objects\"):\n+      objects = []\n+      for wrapped_src in body:\n+        if running_async:\n+          time.sleep(settings.BACKGROUND_COLLECTION_POST_SLEEP)","path":"src/ggrc/services/common.py","position":199,"original_position":199,"commit_id":"2defd151c2045cf68a1d5fd768deb3523d22be0b","original_commit_id":"2defd151c2045cf68a1d5fd768deb3523d22be0b","user":{"login":"Smotko","id":513444,"avatar_url":"https://avatars.githubusercontent.com/u/513444?v=3","gravatar_id":"","url":"https://api.github.com/users/Smotko","html_url":"https://github.com/Smotko","followers_url":"https://api.github.com/users/Smotko/followers","following_url":"https://api.github.com/users/Smotko/following{/other_user}","gists_url":"https://api.github.com/users/Smotko/gists{/gist_id}","starred_url":"https://api.github.com/users/Smotko/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Smotko/subscriptions","organizations_url":"https://api.github.com/users/Smotko/orgs","repos_url":"https://api.github.com/users/Smotko/repos","events_url":"https://api.github.com/users/Smotko/events{/privacy}","received_events_url":"https://api.github.com/users/Smotko/received_events","type":"User","site_admin":false},"body":"We should remove this sleep or at least move it to automappings generation.","created_at":"2016-08-01T17:29:27Z","updated_at":"2016-08-01T17:29:27Z","html_url":"https://github.com/google/ggrc-core/pull/4212#discussion_r73018331","pull_request_url":"https://api.github.com/repos/google/ggrc-core/pulls/4212","_links":{"self":{"href":"https://api.github.com/repos/google/ggrc-core/pulls/comments/73018331"},"html":{"href":"https://github.com/google/ggrc-core/pull/4212#discussion_r73018331"},"pull_request":{"href":"https://api.github.com/repos/google/ggrc-core/pulls/4212"}}},"pull_request":{"url":"https://api.github.com/repos/google/ggrc-core/pulls/4212","id":78891873,"html_url":"https://github.com/google/ggrc-core/pull/4212","diff_url":"https://github.com/google/ggrc-core/pull/4212.diff","patch_url":"https://github.com/google/ggrc-core/pull/4212.patch","issue_url":"https://api.github.com/repos/google/ggrc-core/issues/4212","number":4212,"state":"open","locked":false,"title":"QUICK-FIX Split collection post loop","user":{"login":"zidarsk8","id":566311,"avatar_url":"https://avatars.githubusercontent.com/u/566311?v=3","gravatar_id":"","url":"https://api.github.com/users/zidarsk8","html_url":"https://github.com/zidarsk8","followers_url":"https://api.github.com/users/zidarsk8/followers","following_url":"https://api.github.com/users/zidarsk8/following{/other_user}","gists_url":"https://api.github.com/users/zidarsk8/gists{/gist_id}","starred_url":"https://api.github.com/users/zidarsk8/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zidarsk8/subscriptions","organizations_url":"https://api.github.com/users/zidarsk8/orgs","repos_url":"https://api.github.com/users/zidarsk8/repos","events_url":"https://api.github.com/users/zidarsk8/events{/privacy}","received_events_url":"https://api.github.com/users/zidarsk8/received_events","type":"User","site_admin":false},"body":"~~This is continuation of https://github.com/google/ggrc-core/pull/4209 and will be rebased once that PR is merged.~~\r\n~~When reviewing just checks commits from \"Change CA set_ids return condition\" on.~~\r\n~~PS: this PR should improve collection post performance when batching multiple objects~~\r\n\r\nMost notable change in this PR is that collection post no longer fails on single bad object and adds all the good ones, but rather if any object in the collection is bad, none of them get inserted. This is the main source of the performance improvements, by batching multiple operations and queries.\r\n\r\nPlease pay attention to all object generatios, revision generation (on admin panel), lhn entries, and anything else you can think of.\r\n\r\nTesting this is mostly, creating a lot of objects and checking that they appear where they should, Try special objects such as custom attributes, objects with custom attributes, people, and so on.\r\n\r\nAlso pay attention on performance, if it seems slower than on develop, you should make a comment and the issues should be investigated.","created_at":"2016-07-26T17:25:58Z","updated_at":"2016-08-01T17:29:27Z","closed_at":null,"merged_at":null,"merge_commit_sha":"579b22fc7af1a71682840225b0b549c84ba7b327","assignee":null,"assignees":[],"milestone":{"url":"https://api.github.com/repos/google/ggrc-core/milestones/21","html_url":"https://github.com/google/ggrc-core/milestones/0.9.7-Quince","labels_url":"https://api.github.com/repos/google/ggrc-core/milestones/21/labels","id":1845906,"number":21,"title":"0.9.7-Quince","description":"","creator":{"login":"Smotko","id":513444,"avatar_url":"https://avatars.githubusercontent.com/u/513444?v=3","gravatar_id":"","url":"https://api.github.com/users/Smotko","html_url":"https://github.com/Smotko","followers_url":"https://api.github.com/users/Smotko/followers","following_url":"https://api.github.com/users/Smotko/following{/other_user}","gists_url":"https://api.github.com/users/Smotko/gists{/gist_id}","starred_url":"https://api.github.com/users/Smotko/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Smotko/subscriptions","organizations_url":"https://api.github.com/users/Smotko/orgs","repos_url":"https://api.github.com/users/Smotko/repos","events_url":"https://api.github.com/users/Smotko/events{/privacy}","received_events_url":"https://api.github.com/users/Smotko/received_events","type":"User","site_admin":false},"open_issues":13,"closed_issues":161,"state":"open","created_at":"2016-06-23T17:56:25Z","updated_at":"2016-08-01T15:41:30Z","due_on":"2016-08-14T22:00:00Z","closed_at":null},"commits_url":"https://api.github.com/repos/google/ggrc-core/pulls/4212/commits","review_comments_url":"https://api.github.com/repos/google/ggrc-core/pulls/4212/comments","review_comment_url":"https://api.github.com/repos/google/ggrc-core/pulls/comments{/number}","comments_url":"https://api.github.com/repos/google/ggrc-core/issues/4212/comments","statuses_url":"https://api.github.com/repos/google/ggrc-core/statuses/2defd151c2045cf68a1d5fd768deb3523d22be0b","head":{"label":"zidarsk8:split-collection-post-loop","ref":"split-collection-post-loop","sha":"2defd151c2045cf68a1d5fd768deb3523d22be0b","user":{"login":"zidarsk8","id":566311,"avatar_url":"https://avatars.githubusercontent.com/u/566311?v=3","gravatar_id":"","url":"https://api.github.com/users/zidarsk8","html_url":"https://github.com/zidarsk8","followers_url":"https://api.github.com/users/zidarsk8/followers","following_url":"https://api.github.com/users/zidarsk8/following{/other_user}","gists_url":"https://api.github.com/users/zidarsk8/gists{/gist_id}","starred_url":"https://api.github.com/users/zidarsk8/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zidarsk8/subscriptions","organizations_url":"https://api.github.com/users/zidarsk8/orgs","repos_url":"https://api.github.com/users/zidarsk8/repos","events_url":"https://api.github.com/users/zidarsk8/events{/privacy}","received_events_url":"https://api.github.com/users/zidarsk8/received_events","type":"User","site_admin":false},"repo":{"id":27515613,"name":"ggrc-core","full_name":"zidarsk8/ggrc-core","owner":{"login":"zidarsk8","id":566311,"avatar_url":"https://avatars.githubusercontent.com/u/566311?v=3","gravatar_id":"","url":"https://api.github.com/users/zidarsk8","html_url":"https://github.com/zidarsk8","followers_url":"https://api.github.com/users/zidarsk8/followers","following_url":"https://api.github.com/users/zidarsk8/following{/other_user}","gists_url":"https://api.github.com/users/zidarsk8/gists{/gist_id}","starred_url":"https://api.github.com/users/zidarsk8/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zidarsk8/subscriptions","organizations_url":"https://api.github.com/users/zidarsk8/orgs","repos_url":"https://api.github.com/users/zidarsk8/repos","events_url":"https://api.github.com/users/zidarsk8/events{/privacy}","received_events_url":"https://api.github.com/users/zidarsk8/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/zidarsk8/ggrc-core","description":"GGRC Core","fork":true,"url":"https://api.github.com/repos/zidarsk8/ggrc-core","forks_url":"https://api.github.com/repos/zidarsk8/ggrc-core/forks","keys_url":"https://api.github.com/repos/zidarsk8/ggrc-core/keys{/key_id}","collaborators_url":"https://api.github.com/repos/zidarsk8/ggrc-core/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/zidarsk8/ggrc-core/teams","hooks_url":"https://api.github.com/repos/zidarsk8/ggrc-core/hooks","issue_events_url":"https://api.github.com/repos/zidarsk8/ggrc-core/issues/events{/number}","events_url":"https://api.github.com/repos/zidarsk8/ggrc-core/events","assignees_url":"https://api.github.com/repos/zidarsk8/ggrc-core/assignees{/user}","branches_url":"https://api.github.com/repos/zidarsk8/ggrc-core/branches{/branch}","tags_url":"https://api.github.com/repos/zidarsk8/ggrc-core/tags","blobs_url":"https://api.github.com/repos/zidarsk8/ggrc-core/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/zidarsk8/ggrc-core/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/zidarsk8/ggrc-core/git/refs{/sha}","trees_url":"https://api.github.com/repos/zidarsk8/ggrc-core/git/trees{/sha}","statuses_url":"https://api.github.com/repos/zidarsk8/ggrc-core/statuses/{sha}","languages_url":"https://api.github.com/repos/zidarsk8/ggrc-core/languages","stargazers_url":"https://api.github.com/repos/zidarsk8/ggrc-core/stargazers","contributors_url":"https://api.github.com/repos/zidarsk8/ggrc-core/contributors","subscribers_url":"https://api.github.com/repos/zidarsk8/ggrc-core/subscribers","subscription_url":"https://api.github.com/repos/zidarsk8/ggrc-core/subscription","commits_url":"https://api.github.com/repos/zidarsk8/ggrc-core/commits{/sha}","git_commits_url":"https://api.github.com/repos/zidarsk8/ggrc-core/git/commits{/sha}","comments_url":"https://api.github.com/repos/zidarsk8/ggrc-core/comments{/number}","issue_comment_url":"https://api.github.com/repos/zidarsk8/ggrc-core/issues/comments{/number}","contents_url":"https://api.github.com/repos/zidarsk8/ggrc-core/contents/{+path}","compare_url":"https://api.github.com/repos/zidarsk8/ggrc-core/compare/{base}...{head}","merges_url":"https://api.github.com/repos/zidarsk8/ggrc-core/merges","archive_url":"https://api.github.com/repos/zidarsk8/ggrc-core/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/zidarsk8/ggrc-core/downloads","issues_url":"https://api.github.com/repos/zidarsk8/ggrc-core/issues{/number}","pulls_url":"https://api.github.com/repos/zidarsk8/ggrc-core/pulls{/number}","milestones_url":"https://api.github.com/repos/zidarsk8/ggrc-core/milestones{/number}","notifications_url":"https://api.github.com/repos/zidarsk8/ggrc-core/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/zidarsk8/ggrc-core/labels{/name}","releases_url":"https://api.github.com/repos/zidarsk8/ggrc-core/releases{/id}","deployments_url":"https://api.github.com/repos/zidarsk8/ggrc-core/deployments","created_at":"2014-12-04T00:47:01Z","updated_at":"2016-01-11T15:13:48Z","pushed_at":"2016-08-01T14:00:01Z","git_url":"git://github.com/zidarsk8/ggrc-core.git","ssh_url":"git@github.com:zidarsk8/ggrc-core.git","clone_url":"https://github.com/zidarsk8/ggrc-core.git","svn_url":"https://github.com/zidarsk8/ggrc-core","homepage":null,"size":53682,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"develop"}},"base":{"label":"google:develop","ref":"develop","sha":"0790b632dab2d64d54caab150bbb790b7311404e","user":{"login":"google","id":1342004,"avatar_url":"https://avatars.githubusercontent.com/u/1342004?v=3","gravatar_id":"","url":"https://api.github.com/users/google","html_url":"https://github.com/google","followers_url":"https://api.github.com/users/google/followers","following_url":"https://api.github.com/users/google/following{/other_user}","gists_url":"https://api.github.com/users/google/gists{/gist_id}","starred_url":"https://api.github.com/users/google/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/google/subscriptions","organizations_url":"https://api.github.com/users/google/orgs","repos_url":"https://api.github.com/users/google/repos","events_url":"https://api.github.com/users/google/events{/privacy}","received_events_url":"https://api.github.com/users/google/received_events","type":"Organization","site_admin":false},"repo":{"id":9304637,"name":"ggrc-core","full_name":"google/ggrc-core","owner":{"login":"google","id":1342004,"avatar_url":"https://avatars.githubusercontent.com/u/1342004?v=3","gravatar_id":"","url":"https://api.github.com/users/google","html_url":"https://github.com/google","followers_url":"https://api.github.com/users/google/followers","following_url":"https://api.github.com/users/google/following{/other_user}","gists_url":"https://api.github.com/users/google/gists{/gist_id}","starred_url":"https://api.github.com/users/google/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/google/subscriptions","organizations_url":"https://api.github.com/users/google/orgs","repos_url":"https://api.github.com/users/google/repos","events_url":"https://api.github.com/users/google/events{/privacy}","received_events_url":"https://api.github.com/users/google/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/google/ggrc-core","description":"GGRC Core","fork":false,"url":"https://api.github.com/repos/google/ggrc-core","forks_url":"https://api.github.com/repos/google/ggrc-core/forks","keys_url":"https://api.github.com/repos/google/ggrc-core/keys{/key_id}","collaborators_url":"https://api.github.com/repos/google/ggrc-core/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/google/ggrc-core/teams","hooks_url":"https://api.github.com/repos/google/ggrc-core/hooks","issue_events_url":"https://api.github.com/repos/google/ggrc-core/issues/events{/number}","events_url":"https://api.github.com/repos/google/ggrc-core/events","assignees_url":"https://api.github.com/repos/google/ggrc-core/assignees{/user}","branches_url":"https://api.github.com/repos/google/ggrc-core/branches{/branch}","tags_url":"https://api.github.com/repos/google/ggrc-core/tags","blobs_url":"https://api.github.com/repos/google/ggrc-core/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/google/ggrc-core/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/google/ggrc-core/git/refs{/sha}","trees_url":"https://api.github.com/repos/google/ggrc-core/git/trees{/sha}","statuses_url":"https://api.github.com/repos/google/ggrc-core/statuses/{sha}","languages_url":"https://api.github.com/repos/google/ggrc-core/languages","stargazers_url":"https://api.github.com/repos/google/ggrc-core/stargazers","contributors_url":"https://api.github.com/repos/google/ggrc-core/contributors","subscribers_url":"https://api.github.com/repos/google/ggrc-core/subscribers","subscription_url":"https://api.github.com/repos/google/ggrc-core/subscription","commits_url":"https://api.github.com/repos/google/ggrc-core/commits{/sha}","git_commits_url":"https://api.github.com/repos/google/ggrc-core/git/commits{/sha}","comments_url":"https://api.github.com/repos/google/ggrc-core/comments{/number}","issue_comment_url":"https://api.github.com/repos/google/ggrc-core/issues/comments{/number}","contents_url":"https://api.github.com/repos/google/ggrc-core/contents/{+path}","compare_url":"https://api.github.com/repos/google/ggrc-core/compare/{base}...{head}","merges_url":"https://api.github.com/repos/google/ggrc-core/merges","archive_url":"https://api.github.com/repos/google/ggrc-core/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/google/ggrc-core/downloads","issues_url":"https://api.github.com/repos/google/ggrc-core/issues{/number}","pulls_url":"https://api.github.com/repos/google/ggrc-core/pulls{/number}","milestones_url":"https://api.github.com/repos/google/ggrc-core/milestones{/number}","notifications_url":"https://api.github.com/repos/google/ggrc-core/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/google/ggrc-core/labels{/name}","releases_url":"https://api.github.com/repos/google/ggrc-core/releases{/id}","deployments_url":"https://api.github.com/repos/google/ggrc-core/deployments","created_at":"2013-04-08T19:16:28Z","updated_at":"2016-08-01T10:45:37Z","pushed_at":"2016-08-01T17:18:12Z","git_url":"git://github.com/google/ggrc-core.git","ssh_url":"git@github.com:google/ggrc-core.git","clone_url":"https://github.com/google/ggrc-core.git","svn_url":"https://github.com/google/ggrc-core","homepage":null,"size":64278,"stargazers_count":116,"watchers_count":116,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":86,"mirror_url":null,"open_issues_count":14,"forks":86,"open_issues":14,"watchers":116,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/google/ggrc-core/pulls/4212"},"html":{"href":"https://github.com/google/ggrc-core/pull/4212"},"issue":{"href":"https://api.github.com/repos/google/ggrc-core/issues/4212"},"comments":{"href":"https://api.github.com/repos/google/ggrc-core/issues/4212/comments"},"review_comments":{"href":"https://api.github.com/repos/google/ggrc-core/pulls/4212/comments"},"review_comment":{"href":"https://api.github.com/repos/google/ggrc-core/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/google/ggrc-core/pulls/4212/commits"},"statuses":{"href":"https://api.github.com/repos/google/ggrc-core/statuses/2defd151c2045cf68a1d5fd768deb3523d22be0b"}}}}
{"actor":{"display_login":"Smotko"}}