{"action":"created","comment":{"url":"https://api.github.com/repos/edx/edx-platform/pulls/comments/94137165","pull_request_review_id":14667870,"id":94137165,"diff_hunk":"@@ -158,43 +158,45 @@ def _update_subsection_grades(\n     that those subsection grades were updated.\n     \"\"\"\n     student = User.objects.get(id=user_id)\n-    course_structure = get_course_blocks(student, modulestore().make_course_usage_key(course_key))\n-    subsections_to_update = course_structure.get_transformer_block_field(\n-        scored_block_usage_key,\n-        GradesTransformer,\n-        'subsections',\n-        set(),\n-    )\n-\n-    course = modulestore().get_course(course_key, depth=0)\n-    subsection_grade_factory = SubsectionGradeFactory(student, course, course_structure)\n-\n-    try:\n-        for subsection_usage_key in subsections_to_update:\n-            if subsection_usage_key in course_structure:\n-                subsection_grade = subsection_grade_factory.update(\n-                    course_structure[subsection_usage_key],\n-                    only_if_higher,\n-                )\n-                SUBSECTION_SCORE_CHANGED.send(\n-                    sender=recalculate_subsection_grade,\n-                    course=course,\n-                    course_structure=course_structure,\n-                    user=student,\n-                    subsection_grade=subsection_grade,\n-                )\n-\n-    except DatabaseError as exc:\n-        raise _retry_recalculate_subsection_grade(\n-            user_id,\n-            course_id,\n-            usage_id,\n-            only_if_higher,\n-            expected_modified_time,\n-            score_deleted,\n-            exc,\n+    store = modulestore()\n+    with store.bulk_operations(course_key):\n+        course_structure = get_course_blocks(student, modulestore().make_course_usage_key(course_key))","path":"lms/djangoapps/grades/tasks.py","position":41,"original_position":41,"commit_id":"84f4ef189531581900b72af7ebd4fc8682a02656","original_commit_id":"84f4ef189531581900b72af7ebd4fc8682a02656","user":{"login":"nasthagiri","id":6772826,"avatar_url":"https://avatars.githubusercontent.com/u/6772826?v=3","gravatar_id":"","url":"https://api.github.com/users/nasthagiri","html_url":"https://github.com/nasthagiri","followers_url":"https://api.github.com/users/nasthagiri/followers","following_url":"https://api.github.com/users/nasthagiri/following{/other_user}","gists_url":"https://api.github.com/users/nasthagiri/gists{/gist_id}","starred_url":"https://api.github.com/users/nasthagiri/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nasthagiri/subscriptions","organizations_url":"https://api.github.com/users/nasthagiri/orgs","repos_url":"https://api.github.com/users/nasthagiri/repos","events_url":"https://api.github.com/users/nasthagiri/events{/privacy}","received_events_url":"https://api.github.com/users/nasthagiri/received_events","type":"User","site_admin":false},"body":"Good catch.  Thanks.","created_at":"2016-12-29T13:48:17Z","updated_at":"2016-12-29T13:48:17Z","html_url":"https://github.com/edx/edx-platform/pull/14213#discussion_r94137165","pull_request_url":"https://api.github.com/repos/edx/edx-platform/pulls/14213","_links":{"self":{"href":"https://api.github.com/repos/edx/edx-platform/pulls/comments/94137165"},"html":{"href":"https://github.com/edx/edx-platform/pull/14213#discussion_r94137165"},"pull_request":{"href":"https://api.github.com/repos/edx/edx-platform/pulls/14213"}}},"pull_request":{"url":"https://api.github.com/repos/edx/edx-platform/pulls/14213","id":99259314,"html_url":"https://github.com/edx/edx-platform/pull/14213","diff_url":"https://github.com/edx/edx-platform/pull/14213.diff","patch_url":"https://github.com/edx/edx-platform/pull/14213.patch","issue_url":"https://api.github.com/repos/edx/edx-platform/issues/14213","number":14213,"state":"open","locked":false,"title":"Pre-load Definitions in Split Modulestore when accessing all blocks","user":{"login":"nasthagiri","id":6772826,"avatar_url":"https://avatars.githubusercontent.com/u/6772826?v=3","gravatar_id":"","url":"https://api.github.com/users/nasthagiri","html_url":"https://github.com/nasthagiri","followers_url":"https://api.github.com/users/nasthagiri/followers","following_url":"https://api.github.com/users/nasthagiri/following{/other_user}","gists_url":"https://api.github.com/users/nasthagiri/gists{/gist_id}","starred_url":"https://api.github.com/users/nasthagiri/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nasthagiri/subscriptions","organizations_url":"https://api.github.com/users/nasthagiri/orgs","repos_url":"https://api.github.com/users/nasthagiri/repos","events_url":"https://api.github.com/users/nasthagiri/events{/privacy}","received_events_url":"https://api.github.com/users/nasthagiri/received_events","type":"User","site_admin":false},"body":"## [TNL-6222](https://openedx.atlassian.net/browse/TNL-6222)\r\n\r\n### Description\r\n\r\n#### TLDR; \r\nThis PR addresses the excessive calls to `modulestore.definitions find_one` and `modulestore.active_versions find_one` in `lms.djangoapps.grades.tasks.recalculate_subsection_grade_v2`.  It also reduces the number of calls to  `modulestore.definitions find_one` in `courseware.views.index:CoursewareIndex.get`.\r\n\r\n#### Details\r\nWhen monitoring the staged rollout of Persistent Grades, we noticed that the celery task for updating grades was making unexpected number of mongo queries - some even up to 2,000 mongo queries!  \r\n\r\nWe were able to reduce the extra calls to `modulestore.active_versions find_one` by surrounding the task with a transactional `bulk_operations` context manager.  Those grew linearly by the number of subsections in the course.\r\n\r\nHowever, there were still excessive calls to `modulestore.definitions find_one` (growing linearly by the number of problems in the course).  After further testing, we narrowed down the issue to: (1) cache misses on the Block Structure and (2) Split Modulestore courses.\r\n\r\nFor Block Structures, we expected all course blocks to be pre-loaded in a few (4) queries since we loaded the course with `depth=None`.  Apparently, there is another parameter that we should have also set: `lazy=False`.  So this PR corrects the Block Structure's call to the modulestore to pre-load all definitions since they are needed during the \"collect\" phase.\r\n\r\nAdditionally, when investigating this issue, we noticed that loading the Sequential on the Courseware page had the same issue (since `lazy=False` was not specified along with `depth=None`, although definitions of all descendant blocks of the Sequential need to be accessed).  So this PR updates Courseware to do so, and adds query count tests to validate the change.\r\n\r\nFinally, the implementation of the Split Modulestore had to be updated to:\r\n\r\n1. Support pre-loading of Definitions with the `lazy=False` parameter even after the runtime for the course had been previously cached.  This allows callers to be agnostic to whether a course is pre-fetched without the `depth=None, lazy=False` parameters, but then later accessed with those parameters. \r\n\r\n2. Fix a bug in `get_definitions` so the `bulk_write_record.definitions_in_db` object is also updated.  Otherwise, the Mongo collection was mistakenly \"Inserted\" with pre-fetched records at the end of the bulk-operation - since the code incorrectly calculated that those definitions didn't pre-exist in the database.\r\n\r\n**Note:** There are still linearly growing SQL queries that will be addressed by a separate ticket: https://openedx.atlassian.net/browse/TNL-6225\r\n\r\n### Reviewers\r\nIf you've been tagged for review, please check your corresponding box once you've given the :+1:.\r\n- [ ] Code review: @doctoryes and/or @ormsbee \r\n- [ ] Code review: @sanfordstudent \r\n\r\nFYI: @jcdyer \r\n\r\n### Post-review\r\n- [ ] Rebase and squash commits","created_at":"2016-12-23T16:16:53Z","updated_at":"2016-12-29T13:48:17Z","closed_at":null,"merged_at":null,"merge_commit_sha":"716690ceb848ef2e528ee606c48d6c1bb563ae90","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/edx/edx-platform/pulls/14213/commits","review_comments_url":"https://api.github.com/repos/edx/edx-platform/pulls/14213/comments","review_comment_url":"https://api.github.com/repos/edx/edx-platform/pulls/comments{/number}","comments_url":"https://api.github.com/repos/edx/edx-platform/issues/14213/comments","statuses_url":"https://api.github.com/repos/edx/edx-platform/statuses/84f4ef189531581900b72af7ebd4fc8682a02656","head":{"label":"edx:beryl/mongocrazy","ref":"beryl/mongocrazy","sha":"84f4ef189531581900b72af7ebd4fc8682a02656","user":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"repo":{"id":10391073,"name":"edx-platform","full_name":"edx/edx-platform","owner":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/edx/edx-platform","description":"The Open edX platform, the software that powers edX!","fork":false,"url":"https://api.github.com/repos/edx/edx-platform","forks_url":"https://api.github.com/repos/edx/edx-platform/forks","keys_url":"https://api.github.com/repos/edx/edx-platform/keys{/key_id}","collaborators_url":"https://api.github.com/repos/edx/edx-platform/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/edx/edx-platform/teams","hooks_url":"https://api.github.com/repos/edx/edx-platform/hooks","issue_events_url":"https://api.github.com/repos/edx/edx-platform/issues/events{/number}","events_url":"https://api.github.com/repos/edx/edx-platform/events","assignees_url":"https://api.github.com/repos/edx/edx-platform/assignees{/user}","branches_url":"https://api.github.com/repos/edx/edx-platform/branches{/branch}","tags_url":"https://api.github.com/repos/edx/edx-platform/tags","blobs_url":"https://api.github.com/repos/edx/edx-platform/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/edx/edx-platform/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/edx/edx-platform/git/refs{/sha}","trees_url":"https://api.github.com/repos/edx/edx-platform/git/trees{/sha}","statuses_url":"https://api.github.com/repos/edx/edx-platform/statuses/{sha}","languages_url":"https://api.github.com/repos/edx/edx-platform/languages","stargazers_url":"https://api.github.com/repos/edx/edx-platform/stargazers","contributors_url":"https://api.github.com/repos/edx/edx-platform/contributors","subscribers_url":"https://api.github.com/repos/edx/edx-platform/subscribers","subscription_url":"https://api.github.com/repos/edx/edx-platform/subscription","commits_url":"https://api.github.com/repos/edx/edx-platform/commits{/sha}","git_commits_url":"https://api.github.com/repos/edx/edx-platform/git/commits{/sha}","comments_url":"https://api.github.com/repos/edx/edx-platform/comments{/number}","issue_comment_url":"https://api.github.com/repos/edx/edx-platform/issues/comments{/number}","contents_url":"https://api.github.com/repos/edx/edx-platform/contents/{+path}","compare_url":"https://api.github.com/repos/edx/edx-platform/compare/{base}...{head}","merges_url":"https://api.github.com/repos/edx/edx-platform/merges","archive_url":"https://api.github.com/repos/edx/edx-platform/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/edx/edx-platform/downloads","issues_url":"https://api.github.com/repos/edx/edx-platform/issues{/number}","pulls_url":"https://api.github.com/repos/edx/edx-platform/pulls{/number}","milestones_url":"https://api.github.com/repos/edx/edx-platform/milestones{/number}","notifications_url":"https://api.github.com/repos/edx/edx-platform/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/edx/edx-platform/labels{/name}","releases_url":"https://api.github.com/repos/edx/edx-platform/releases{/id}","deployments_url":"https://api.github.com/repos/edx/edx-platform/deployments","created_at":"2013-05-30T20:20:38Z","updated_at":"2016-12-29T10:19:51Z","pushed_at":"2016-12-29T13:29:57Z","git_url":"git://github.com/edx/edx-platform.git","ssh_url":"git@github.com:edx/edx-platform.git","clone_url":"https://github.com/edx/edx-platform.git","svn_url":"https://github.com/edx/edx-platform","homepage":"http://open.edx.org/","size":770311,"stargazers_count":2909,"watchers_count":2909,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":1721,"mirror_url":null,"open_issues_count":167,"forks":1721,"open_issues":167,"watchers":2909,"default_branch":"master"}},"base":{"label":"edx:master","ref":"master","sha":"9e5139dcb010326c9aa3480eb6355f87b72ab12d","user":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"repo":{"id":10391073,"name":"edx-platform","full_name":"edx/edx-platform","owner":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/edx/edx-platform","description":"The Open edX platform, the software that powers edX!","fork":false,"url":"https://api.github.com/repos/edx/edx-platform","forks_url":"https://api.github.com/repos/edx/edx-platform/forks","keys_url":"https://api.github.com/repos/edx/edx-platform/keys{/key_id}","collaborators_url":"https://api.github.com/repos/edx/edx-platform/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/edx/edx-platform/teams","hooks_url":"https://api.github.com/repos/edx/edx-platform/hooks","issue_events_url":"https://api.github.com/repos/edx/edx-platform/issues/events{/number}","events_url":"https://api.github.com/repos/edx/edx-platform/events","assignees_url":"https://api.github.com/repos/edx/edx-platform/assignees{/user}","branches_url":"https://api.github.com/repos/edx/edx-platform/branches{/branch}","tags_url":"https://api.github.com/repos/edx/edx-platform/tags","blobs_url":"https://api.github.com/repos/edx/edx-platform/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/edx/edx-platform/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/edx/edx-platform/git/refs{/sha}","trees_url":"https://api.github.com/repos/edx/edx-platform/git/trees{/sha}","statuses_url":"https://api.github.com/repos/edx/edx-platform/statuses/{sha}","languages_url":"https://api.github.com/repos/edx/edx-platform/languages","stargazers_url":"https://api.github.com/repos/edx/edx-platform/stargazers","contributors_url":"https://api.github.com/repos/edx/edx-platform/contributors","subscribers_url":"https://api.github.com/repos/edx/edx-platform/subscribers","subscription_url":"https://api.github.com/repos/edx/edx-platform/subscription","commits_url":"https://api.github.com/repos/edx/edx-platform/commits{/sha}","git_commits_url":"https://api.github.com/repos/edx/edx-platform/git/commits{/sha}","comments_url":"https://api.github.com/repos/edx/edx-platform/comments{/number}","issue_comment_url":"https://api.github.com/repos/edx/edx-platform/issues/comments{/number}","contents_url":"https://api.github.com/repos/edx/edx-platform/contents/{+path}","compare_url":"https://api.github.com/repos/edx/edx-platform/compare/{base}...{head}","merges_url":"https://api.github.com/repos/edx/edx-platform/merges","archive_url":"https://api.github.com/repos/edx/edx-platform/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/edx/edx-platform/downloads","issues_url":"https://api.github.com/repos/edx/edx-platform/issues{/number}","pulls_url":"https://api.github.com/repos/edx/edx-platform/pulls{/number}","milestones_url":"https://api.github.com/repos/edx/edx-platform/milestones{/number}","notifications_url":"https://api.github.com/repos/edx/edx-platform/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/edx/edx-platform/labels{/name}","releases_url":"https://api.github.com/repos/edx/edx-platform/releases{/id}","deployments_url":"https://api.github.com/repos/edx/edx-platform/deployments","created_at":"2013-05-30T20:20:38Z","updated_at":"2016-12-29T10:19:51Z","pushed_at":"2016-12-29T13:29:57Z","git_url":"git://github.com/edx/edx-platform.git","ssh_url":"git@github.com:edx/edx-platform.git","clone_url":"https://github.com/edx/edx-platform.git","svn_url":"https://github.com/edx/edx-platform","homepage":"http://open.edx.org/","size":770311,"stargazers_count":2909,"watchers_count":2909,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":1721,"mirror_url":null,"open_issues_count":167,"forks":1721,"open_issues":167,"watchers":2909,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/edx/edx-platform/pulls/14213"},"html":{"href":"https://github.com/edx/edx-platform/pull/14213"},"issue":{"href":"https://api.github.com/repos/edx/edx-platform/issues/14213"},"comments":{"href":"https://api.github.com/repos/edx/edx-platform/issues/14213/comments"},"review_comments":{"href":"https://api.github.com/repos/edx/edx-platform/pulls/14213/comments"},"review_comment":{"href":"https://api.github.com/repos/edx/edx-platform/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/edx/edx-platform/pulls/14213/commits"},"statuses":{"href":"https://api.github.com/repos/edx/edx-platform/statuses/84f4ef189531581900b72af7ebd4fc8682a02656"}}}}
{"actor":{"display_login":"nasthagiri"}}