{"action":"created","comment":{"url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments/85474908","pull_request_review_id":6189435,"id":85474908,"diff_hunk":"@@ -0,0 +1,533 @@\n+\"\"\"\n+Store course block details sourced from the Course Blocks API into a hive table.\n+\n+See LoadAllCourseBlocksWorkflow for details.\n+\"\"\"\n+\n+import logging\n+import json\n+import datetime\n+import luigi\n+from requests.exceptions import HTTPError\n+\n+from edx.analytics.tasks.mapreduce import MapReduceJobTask, MapReduceJobTaskMixin\n+from edx.analytics.tasks.url import get_target_from_url, url_path_join\n+from edx.analytics.tasks.util.edx_api_client import EdxApiClient\n+from edx.analytics.tasks.util.hive import (\n+    WarehouseMixin, BareHiveTableTask, HivePartitionTask,\n+)\n+from edx.analytics.tasks.util.overwrite import OverwriteOutputMixin\n+from edx.analytics.tasks.util.record import Record, BooleanField, StringField, IntegerField, DEFAULT_NULL_VALUE\n+from edx.analytics.tasks.decorators import workflow_entry_point\n+from edx.analytics.tasks.course_list import TimestampPartitionMixin, CourseListPartitionTask\n+\n+\n+log = logging.getLogger(__name__)\n+\n+\n+class CourseBlockRecord(Record):\n+    \"\"\"\n+    Represents a course block as fetched from the edX Course Blocks REST API, augmented with details about its\n+    position in the course.\n+    \"\"\"\n+    block_id = StringField(length=255, nullable=False, description='Block identifier.')\n+    course_id = StringField(length=255, nullable=False, description='Identifier for the course containing the block.')\n+    block_type = StringField(length=255, nullable=False, description='Block type.')\n+    display_name = StringField(length=255, nullable=False, description='User-facing title of the block.')","path":"edx/analytics/tasks/course_blocks.py","position":36,"original_position":36,"commit_id":"a500b105b22420b706b30d93f37b136a48064382","original_commit_id":"a500b105b22420b706b30d93f37b136a48064382","user":{"login":"pomegranited","id":7556571,"avatar_url":"https://avatars.githubusercontent.com/u/7556571?v=3","gravatar_id":"","url":"https://api.github.com/users/pomegranited","html_url":"https://github.com/pomegranited","followers_url":"https://api.github.com/users/pomegranited/followers","following_url":"https://api.github.com/users/pomegranited/following{/other_user}","gists_url":"https://api.github.com/users/pomegranited/gists{/gist_id}","starred_url":"https://api.github.com/users/pomegranited/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pomegranited/subscriptions","organizations_url":"https://api.github.com/users/pomegranited/orgs","repos_url":"https://api.github.com/users/pomegranited/repos","events_url":"https://api.github.com/users/pomegranited/events{/privacy}","received_events_url":"https://api.github.com/users/pomegranited/received_events","type":"User","site_admin":false},"body":"Should also truncate the `display_name` values here.","created_at":"2016-10-28T05:46:09Z","updated_at":"2016-10-28T05:46:09Z","html_url":"https://github.com/edx/edx-analytics-pipeline/pull/274#discussion_r85474908","pull_request_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274","_links":{"self":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments/85474908"},"html":{"href":"https://github.com/edx/edx-analytics-pipeline/pull/274#discussion_r85474908"},"pull_request":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274"}}},"pull_request":{"url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274","id":80062581,"html_url":"https://github.com/edx/edx-analytics-pipeline/pull/274","diff_url":"https://github.com/edx/edx-analytics-pipeline/pull/274.diff","patch_url":"https://github.com/edx/edx-analytics-pipeline/pull/274.patch","issue_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/274","number":274,"state":"open","locked":false,"title":"Adds tasks to create Problem Response reports with Course Block records","user":{"login":"pomegranited","id":7556571,"avatar_url":"https://avatars.githubusercontent.com/u/7556571?v=3","gravatar_id":"","url":"https://api.github.com/users/pomegranited","html_url":"https://github.com/pomegranited","followers_url":"https://api.github.com/users/pomegranited/followers","following_url":"https://api.github.com/users/pomegranited/following{/other_user}","gists_url":"https://api.github.com/users/pomegranited/gists{/gist_id}","starred_url":"https://api.github.com/users/pomegranited/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pomegranited/subscriptions","organizations_url":"https://api.github.com/users/pomegranited/orgs","repos_url":"https://api.github.com/users/pomegranited/repos","events_url":"https://api.github.com/users/pomegranited/events{/privacy}","received_events_url":"https://api.github.com/users/pomegranited/received_events","type":"User","site_admin":false},"body":"This change adds a set of tasks which read `problem_check` events from the tracking logs, collate the \"latest problem response\" data into a Hive table, and generate CSV reports from this data.\r\n\r\n**Discussions**: This change is related to [edx-analytics-data-api PR #128](https://github.com/edx/edx-analytics-data-api/pull/128), which has been discussed extensively with the Analytics team.\r\n\r\n**Dependencies**: None\r\n\r\n**Sandbox URL**: TBD\r\n\r\n**Partner information**: 3rd party-hosted open edX instance\r\n\r\n**Testing instructions**:\r\n\r\n1. See [edx-analytics-data-api PR 128](https://github.com/edx/edx-analytics-data-api/pull/128) for analyticstack test setup instructions.  \r\n\r\n    Notes:\r\n    * You don't need the Analytics API setup, or the custom ElasticSearch.\r\n    * When installing the analytics-pipeline, take care to install this PR's branch:\r\n\r\n    ```\r\nexport PIPELINE_REPO=\"https://github.com/open-craft/edx-analytics-pipeline.git\"\r\nexport PIPELINE_BRANCH=\"jill/latest-problem-response\"\r\n```\r\n   * And run this `launch-task` instead of `ImportEnrollmentsIntoMysql` or `ModuleEngagementWorkflowTask`:\r\n\r\n   ```\r\nProblemResponseReportWorkflow \\\r\n            --interval-start 2016-01-01  \\\r\n            --local-scheduler --n-reduce-tasks 2 --remote-log-level DEBUG \\\r\n            --output-root /edx/var/reports/ \\\r\n            --marker hdfs://localhost:9000/reports/marker`date +%s` \\\r\n            --hive-overwrite\r\n```\r\n   * You'll need to create `/edx/var/reports`, and ensure the hadoop user can write to it.\r\n   * Use the `--hive-overwrite` parameter to see data changes if tracking logs change.\r\n2. To view the problem response data, query the hive table created by the task:\r\n\r\n    ```\r\nsudo su hadoop\r\nhive\r\nhive> select * from problem_response_latest;\r\n```\r\n3. To see the CSV reports, check `/edx/var/reports`.  There should be one file created there per course.\r\n3. You can add more problem attempts via the LMS, and see them reflected in the reports by re-running the task.\r\n\r\n    This task does not depend on any other enrolment or user profile data, so you can also dump any tracking.log files into hdfs and re-run the task, e.g.\r\n\r\n    ```\r\nsudo su hadoop\r\nhdfs dfs -put -f /edx/src/tracking.log-real-data-from-course hdfs://localhost:9000/data/tracking.log-REAL\r\n```\r\n\r\n4. Run unit tests:\r\n\r\n    ```\r\nvagrant ssh\r\ncd /var/lib/analytics-tasks/analyticstack/repo\r\nsource ../venv/bin/activate\r\nexport WHEEL_URL=http://edx-wheelhouse.s3-website-us-east-1.amazonaws.com/Ubuntu/precise\r\nexport WHEEL_PYVER=python2.7\r\nmake coverage\r\n```\r\n\r\n**Reviewers**\r\n- [x] @haikuginger \r\n- [ ] @mulby \r\n- [ ] @brianhw ","created_at":"2016-08-04T11:31:17Z","updated_at":"2016-10-28T05:46:09Z","closed_at":null,"merged_at":null,"merge_commit_sha":"8ac6f1cd276e65c3840805c009b45ab81b53b199","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274/commits","review_comments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274/comments","review_comment_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments{/number}","comments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/274/comments","statuses_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/statuses/a500b105b22420b706b30d93f37b136a48064382","head":{"label":"open-craft:jill/latest-problem-response","ref":"jill/latest-problem-response","sha":"a500b105b22420b706b30d93f37b136a48064382","user":{"login":"open-craft","id":7414786,"avatar_url":"https://avatars.githubusercontent.com/u/7414786?v=3","gravatar_id":"","url":"https://api.github.com/users/open-craft","html_url":"https://github.com/open-craft","followers_url":"https://api.github.com/users/open-craft/followers","following_url":"https://api.github.com/users/open-craft/following{/other_user}","gists_url":"https://api.github.com/users/open-craft/gists{/gist_id}","starred_url":"https://api.github.com/users/open-craft/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/open-craft/subscriptions","organizations_url":"https://api.github.com/users/open-craft/orgs","repos_url":"https://api.github.com/users/open-craft/repos","events_url":"https://api.github.com/users/open-craft/events{/privacy}","received_events_url":"https://api.github.com/users/open-craft/received_events","type":"Organization","site_admin":false},"repo":{"id":38666315,"name":"edx-analytics-pipeline","full_name":"open-craft/edx-analytics-pipeline","owner":{"login":"open-craft","id":7414786,"avatar_url":"https://avatars.githubusercontent.com/u/7414786?v=3","gravatar_id":"","url":"https://api.github.com/users/open-craft","html_url":"https://github.com/open-craft","followers_url":"https://api.github.com/users/open-craft/followers","following_url":"https://api.github.com/users/open-craft/following{/other_user}","gists_url":"https://api.github.com/users/open-craft/gists{/gist_id}","starred_url":"https://api.github.com/users/open-craft/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/open-craft/subscriptions","organizations_url":"https://api.github.com/users/open-craft/orgs","repos_url":"https://api.github.com/users/open-craft/repos","events_url":"https://api.github.com/users/open-craft/events{/privacy}","received_events_url":"https://api.github.com/users/open-craft/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/open-craft/edx-analytics-pipeline","description":null,"fork":true,"url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline","forks_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/forks","keys_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/teams","hooks_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/hooks","issue_events_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/issues/events{/number}","events_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/events","assignees_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/assignees{/user}","branches_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/branches{/branch}","tags_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/tags","blobs_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/git/refs{/sha}","trees_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/statuses/{sha}","languages_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/languages","stargazers_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/stargazers","contributors_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/contributors","subscribers_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/subscribers","subscription_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/subscription","commits_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/commits{/sha}","git_commits_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/git/commits{/sha}","comments_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/comments{/number}","issue_comment_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/issues/comments{/number}","contents_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/contents/{+path}","compare_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/merges","archive_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/downloads","issues_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/issues{/number}","pulls_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/pulls{/number}","milestones_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/milestones{/number}","notifications_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/labels{/name}","releases_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/releases{/id}","deployments_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/deployments","created_at":"2015-07-07T05:33:39Z","updated_at":"2015-07-07T05:33:40Z","pushed_at":"2016-10-04T01:40:38Z","git_url":"git://github.com/open-craft/edx-analytics-pipeline.git","ssh_url":"git@github.com:open-craft/edx-analytics-pipeline.git","clone_url":"https://github.com/open-craft/edx-analytics-pipeline.git","svn_url":"https://github.com/open-craft/edx-analytics-pipeline","homepage":null,"size":2780,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":2,"forks":0,"open_issues":2,"watchers":0,"default_branch":"master"}},"base":{"label":"edx:master","ref":"master","sha":"e63396c92335830fbbfc3f1f68e6a3159e17e2e2","user":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"repo":{"id":22929261,"name":"edx-analytics-pipeline","full_name":"edx/edx-analytics-pipeline","owner":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/edx/edx-analytics-pipeline","description":null,"fork":false,"url":"https://api.github.com/repos/edx/edx-analytics-pipeline","forks_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/forks","keys_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/teams","hooks_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/hooks","issue_events_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/events{/number}","events_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/events","assignees_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/assignees{/user}","branches_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/branches{/branch}","tags_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/tags","blobs_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/refs{/sha}","trees_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/statuses/{sha}","languages_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/languages","stargazers_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/stargazers","contributors_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/contributors","subscribers_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/subscribers","subscription_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/subscription","commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/commits{/sha}","git_commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/commits{/sha}","comments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/comments{/number}","issue_comment_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/comments{/number}","contents_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/contents/{+path}","compare_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/merges","archive_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/downloads","issues_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues{/number}","pulls_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls{/number}","milestones_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/milestones{/number}","notifications_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/labels{/name}","releases_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/releases{/id}","deployments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/deployments","created_at":"2014-08-13T19:57:17Z","updated_at":"2016-10-27T18:32:45Z","pushed_at":"2016-10-27T14:24:38Z","git_url":"git://github.com/edx/edx-analytics-pipeline.git","ssh_url":"git@github.com:edx/edx-analytics-pipeline.git","clone_url":"https://github.com/edx/edx-analytics-pipeline.git","svn_url":"https://github.com/edx/edx-analytics-pipeline","homepage":null,"size":4290,"stargazers_count":31,"watchers_count":31,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":49,"mirror_url":null,"open_issues_count":14,"forks":49,"open_issues":14,"watchers":31,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274"},"html":{"href":"https://github.com/edx/edx-analytics-pipeline/pull/274"},"issue":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/274"},"comments":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/274/comments"},"review_comments":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274/comments"},"review_comment":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274/commits"},"statuses":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/statuses/a500b105b22420b706b30d93f37b136a48064382"}}}}
{"actor":{"display_login":"pomegranited"}}