{"action":"created","comment":{"url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments/74697464","id":74697464,"diff_hunk":"@@ -0,0 +1,562 @@\n+\"\"\"\n+Luigi tasks for extracting the latest problem response data from tracking log files.\n+\"\"\"\n+import re\n+import csv\n+import ast\n+import json\n+from datetime import datetime\n+import logging\n+import luigi\n+\n+from edx.analytics.tasks.mapreduce import MapReduceJobTask, MapReduceJobTaskMixin, MultiOutputMapReduceJobTask\n+from edx.analytics.tasks.pathutil import EventLogSelectionDownstreamMixin, EventLogSelectionMixin\n+from edx.analytics.tasks.url import get_target_from_url, url_path_join\n+from edx.analytics.tasks.util.overwrite import OverwriteOutputMixin\n+from edx.analytics.tasks.decorators import workflow_entry_point\n+from edx.analytics.tasks.util.hive import BareHiveTableTask\n+from edx.analytics.tasks.util.opaque_key_util import get_filename_safe_course_id\n+from edx.analytics.tasks.util.record import (\n+    Record, StringField, StringListField, IntegerField, DateTimeField, FloatField, BooleanField,\n+)\n+\n+from edx.analytics.tasks.answer_dist import ProblemCheckEventMixin, get_problem_check_event\n+\n+log = logging.getLogger(__name__)\n+\n+\n+class ProblemResponseRecord(Record):\n+    \"\"\"\n+    Record containing the data for a single user's response to a problem, in a given date range.\n+\n+    If there are multiple questions in a problem, they are spread over separate ProblemResponseRecords.\n+    \"\"\"\n+    # Fields that provide the unique key for each record\n+    course_id = StringField(description='Course containing the problem.')\n+    answer_id = StringField(description='Learner\\'s answer ID.')\n+\n+    # Remaining data fields\n+    problem_id = StringField(description='Problem\\'s block usage ID.')\n+    problem = StringField(description='Problem display name, at time of answering.')\n+    username = StringField(description='Learner\\'s username.')\n+    question = StringField(description='Question\\'s display name, at time of answering.')\n+    score = FloatField(description='Score achieved by the learner.')\n+    max_score = FloatField(description='Maximum possible score for the problem.')\n+    correct = BooleanField(nullable=True, description='True if all answers are correct; '\n+                                                      'False if any answers are not correct; '\n+                                                      'None if any answers have unknown correctness.')\n+    answer = StringListField(description='List of answers the user chose for the question.')\n+    total_attempts = IntegerField(description='Total number of attempts the user has made on the problem.')\n+    first_attempt_date = DateTimeField(description='date/time of the first attempt the user has made on the problem.')\n+    last_attempt_date = DateTimeField(description='date/time of the last attempt the user has made on the problem.')\n+\n+    def to_key_value_tuples(self, num_keys=2):\n+        \"\"\"\n+        Return two string tuples:\n+            * first \"num_key\" field values\n+            * the remaining field values\n+        \"\"\"\n+        string_tuple = self.to_string_tuple()\n+        return (string_tuple[:num_keys],\n+                string_tuple[num_keys:])\n+\n+\n+class ProblemResponseTableMixin(EventLogSelectionDownstreamMixin,\n+                                MapReduceJobTaskMixin):\n+    \"\"\"\n+    Common parameters passed through the problem response workflow.\n+    \"\"\"\n+    # Make the interval be optional:\n+    interval = luigi.DateIntervalParameter(\n+        default=None,\n+        description='The range of dates to export logs for. '\n+        'If not specified, `interval_start` and `interval_end` are used to construct the `interval`.',\n+    )\n+\n+    # Define optional parameters, to be used if 'interval' is not defined.\n+    interval_start = luigi.DateParameter(\n+        config_path={'section': 'problem_response', 'name': 'interval_start'},\n+        significant=False,\n+        description='The start date to export logs for.  Ignored if `interval` is provided.',\n+    )\n+    interval_end = luigi.DateParameter(\n+        default=datetime.utcnow().date(),\n+        significant=False,\n+        description='The end date to export logs for.  Ignored if `interval` is provided. '\n+        'Default is today, UTC.',\n+    )\n+\n+    def __init__(self, *args, **kwargs):\n+        super(ProblemResponseTableMixin, self).__init__(*args, **kwargs)\n+\n+        if not self.interval:\n+            self.interval = luigi.date_interval.Custom(self.interval_start, self.interval_end)\n+\n+\n+class LatestProblemResponseTableTask(ProblemResponseTableMixin,\n+                                     BareHiveTableTask):\n+    \"\"\"\n+    A hive table containing the latest problem response data.\n+    \"\"\"\n+    partition_by = None\n+    table = 'problem_response_latest'\n+\n+    @property\n+    def columns(self):\n+        return ProblemResponseRecord.get_hive_schema()\n+\n+    @property\n+    def output_root(self):\n+        \"\"\"Use the table location path for the output root.\"\"\"\n+        return self.table_location\n+\n+    def requires(self):\n+        return LatestProblemResponseDataTask(\n+            n_reduce_tasks=self.n_reduce_tasks,\n+            source=self.source,\n+            interval=self.interval,\n+            pattern=self.pattern,\n+            output_root=self.output_root,\n+            overwrite=self.overwrite\n+        )\n+\n+\n+class ProblemResponseDataMixin(object):\n+    \"\"\"\n+    Tasks that run the problem response data on the hadoop cluster need these extra modules.\n+    \"\"\"\n+    def extra_modules(self):\n+        \"\"\"\n+        Extra modules needed by the hadoop cluster when processing tracking logs.\n+        \"\"\"\n+        import html5lib\n+        return [html5lib,]\n+\n+\n+class LatestProblemResponseDataTask(EventLogSelectionMixin,\n+                                    ProblemCheckEventMixin,\n+                                    ProblemResponseDataMixin,\n+                                    OverwriteOutputMixin,\n+                                    MapReduceJobTask):\n+    \"\"\"\n+    Process the event log and gather the latest problem_check events.\n+\n+    This emits one record for each answer in the latest problem_check event for each user,\n+    for each problem in a course.\n+\n+    This task is intended to be run over all the tracking log data.\n+    \"\"\"\n+    output_root = luigi.Parameter(\n+        description='URL pointing to the folder where the problem response data should be stored.',\n+    )\n+    clean_text_regex = luigi.Parameter(\n+        default=r'(?:<choicehint.*?</choicehint>)|(?:<choicehint.*?\\>)',\n+        config_path={'section': 'problem-response', 'name': 'clean_text_regex'},\n+        description='Regex string used to strip unwanted HTML tags or other strings from text.',\n+    )\n+\n+    # Write the output directly to the final destination and rely on the _SUCCESS file to\n+    # indicate whether or not it is complete. Note that this is a custom extension to luigi.\n+    enable_direct_output = True\n+\n+    def __init__(self, *args, **kwargs):\n+        super(LatestProblemResponseDataTask, self).__init__(*args, **kwargs)\n+        if self.clean_text_regex is not None:\n+            self.clean_text_regex = re.compile(self.clean_text_regex)\n+\n+    def mapper(self, line):\n+        \"\"\"\n+        Generates output values for explicit problem_check events.\n+\n+        Args:\n+            line: text line from a tracking event log.\n+\n+        Yields:\n+            (course_id, problem_id, username), (timestamp, problem_check_info)\n+\n+            See answer_dist.ProblemCheckEventMixin.mapper for details.\n+\n+        Example:\n+                (DemoX-Demo_Course-T1, i4x://edX/DemoX/Demo_Course/problem/PS1_P1, dummy_username),\n+                (2013-09-10T00:01:05.123456, blah)\n+        \"\"\"\n+\n+        # Filter events on date interval\n+        value = self.get_event_and_date_string(line)\n+        if value is None:\n+            return\n+        event, _date_string = value\n+\n+        # Username is required\n+        username = event.get('username', '').strip()\n+        if not username:\n+            return\n+\n+        # We are only interested in server events, not browser events.\n+        event_source = event.get('event_source')\n+        if event_source is None:\n+            log.error(\"encountered event with no event_source: %s\", event)\n+            return\n+        if event_source != 'server':\n+            return\n+\n+        # Parse the event as a problem_check event\n+        parsed_tuple_or_none = get_problem_check_event(event)\n+        if parsed_tuple_or_none is not None:\n+            yield parsed_tuple_or_none\n+\n+    def reducer(self, key, values):\n+        \"\"\"\n+        Calculate a ProblemResponseRecord from the most recently submitted\n+        response to a problem in a course.\n+\n+        If the problem response contains multiple \"submissions\"\n+        (i.e. multiple questions), they will be split into separate\n+        ProblemResponseRecords.\n+\n+        Args:\n+            key:  (course_id, problem_id, username)\n+            values:  iterator of (attempt_date, problem_check_json)\n+\n+            See ProblemCheckEventMixin.mapper for details.\n+\n+        Yields:\n+            A key/value tuple for each of the latest problem attempt\n+            \"submissions\", annotated with the aggregated total_attempts,\n+            first_attempt_date, and last_attempt_date.\n+\n+            ((course_id, answer_id),\n+             (problem_id, problem, username, question, score, max_score, correct, answer,\n+              total_attempts, first_attempt_date, last_attempt_date))\n+        \"\"\"\n+        # Parse the map key\n+        (course_id, problem_id, username) = key\n+\n+        # Sort input values (by timestamp) to easily detect the first\n+        # and most recent answer to a problem by a particular user.\n+        # Note that this assumes the timestamp values (strings) are in\n+        # ISO representation, so that the tuples will be ordered in\n+        # ascending time value.\n+        values = sorted(values)\n+        if not values:\n+            return\n+\n+        # Get the first entry.\n+        first_attempt_date, _first_response = values[0]\n+\n+        # Get the last entry\n+        last_attempt_date, latest_response = values[-1]\n+\n+        # Get attempt count\n+        total_attempts = len(values)\n+\n+        # Generate a single response record from each answer submission\n+        for answer in self.get_answer_data(latest_response):\n+            latest_response_record = ProblemResponseRecord(\n+                course_id=course_id,\n+                answer_id=answer.get('answer_id'),\n+                problem_id=problem_id,\n+                problem=answer.get('problem', ''),\n+                username=username,\n+                question=answer.get('question', ''),\n+                score=answer.get('grade', 0),\n+                max_score=answer.get('max_grade', 0),\n+                correct=answer.get('correct', None),\n+                answer=answer.get('answer', ''),\n+                total_attempts=total_attempts,\n+                first_attempt_date=first_attempt_date,\n+                last_attempt_date=last_attempt_date\n+            )\n+\n+            yield latest_response_record.to_key_value_tuples()\n+\n+    def _clean_string(self, string):\n+        \"\"\"Remove unwanted characters from the given string or list of strings.\"\"\"\n+\n+        # Handle lists of strings\n+        if isinstance(string, (list, tuple)):\n+            for idx, substring in enumerate(string):\n+                string[idx] = self._clean_string(substring)\n+            return string\n+\n+        # Replace multiple whitespaces (including newlines) with a single space\n+        string = re.sub(r'\\s+', r' ', string)\n+\n+        # Remove any unwanted tags\n+        if self.clean_text_regex is not None:\n+            string = self.clean_text_regex.sub(r'', string)\n+\n+        return string\n+\n+    def get_answer_data(self, event_string):\n+        \"\"\"\n+        Concatenate the answer text and \"correctness\" information for all\n+        the answers parsed from the given event_string.\n+\n+        Args:\n+            event_string: a JSON-encoded string version of the event's data\n+\n+        Yields:\n+            A record for each submitted answer, containing:\n+\n+            * course_id: identifier for the course containing the question\n+            * problem: display text for the problem\n+            * question: display text for the question\n+            * grade: score given for the attempt\n+            * max_grade: maximum score given for the attempt\n+            * correct: A nullable, boolean representation of the \"correctness\" value:\n+                * True if \"correct\"\n+                * False if \"incorrect\"\n+                * None if unspecified or \"unknown\"\n+            * answer: concatenated string of answer values, with <choicehint> etc removed.\n+        \"\"\"\n+        # Process each submitted answer for the current problem\n+        for answer in self._generate_answers(event_string, 'unused'):\n+            (course_id, answer_id), (_timestamp, answer_json) = answer\n+            answer_data = json.loads(answer_json)\n+\n+            problem = answer_data.get('problem_display_name', '')\n+            question = answer_data.get('question', '')\n+\n+            # Answers can be correct, incorrect, or unknown\n+            correct_map = answer_data.get('answer_correct_map', {})\n+            correctness = correct_map.get('correctness')\n+            if correctness == 'correct':\n+                correct = True\n+            elif correctness == 'incorrect':\n+                correct = False\n+            else:\n+                correct = None\n+\n+            # Answer text may have been given, or maybe just the answer IDs\n+            answer = answer_data.get('answer', answer_data.get('answer_value_id', ''))\n+\n+            # Yield each processed submission\n+            yield dict(\n+                course_id=course_id,\n+                answer_id=answer_id,\n+                problem=self._clean_string(problem),\n+                question=self._clean_string(question),\n+                grade=answer_data.get('grade'),\n+                max_grade=answer_data.get('max_grade'),\n+                correct=correct,\n+                answer=self._clean_string(answer),\n+            )\n+\n+    def output(self):\n+        return get_target_from_url(self.output_root)\n+\n+    def complete(self):\n+        \"\"\"\n+        The current task is complete if no overwrite was requested,\n+        and the output_root/_SUCCESS file is present.\n+        \"\"\"\n+        if super(LatestProblemResponseDataTask, self).complete():\n+            return get_target_from_url(url_path_join(self.output_root, '_SUCCESS')).exists()\n+        return False\n+\n+    def run(self):\n+        \"\"\"\n+        Clear out output if data is incomplete, or if overwrite requested.\n+        \"\"\"\n+        if not self.complete():\n+            self.remove_output_on_overwrite()\n+\n+        super(LatestProblemResponseDataTask, self).run()\n+\n+\n+class ProblemResponseReportTask(ProblemResponseDataMixin,\n+                                MultiOutputMapReduceJobTask):\n+    \"\"\"\n+    Task which generates one report per course from the input problem response records.\n+\n+    ProblemResponseRecords are mapped by course_id, and each course is written to a separate file.\n+    \"\"\"\n+    input_root = luigi.Parameter(\n+        description=\"URL pointing to the folder of problem response records to include in the reports.\",\n+    )\n+    report_filename_template = luigi.Parameter(\n+        config_path={'section': 'problem-response', 'name': 'report_filename_template'},\n+        default='{course_id}_problem_response_{timestamp}.csv',","path":"edx/analytics/tasks/problem_response.py","position":null,"original_position":380,"commit_id":"25674d8bbd8e90410de529f3207bd0a4d4f4db5a","original_commit_id":"ce7d698c312f595858666e7e72f77c8525721f0d","user":{"login":"pomegranited","id":7556571,"avatar_url":"https://avatars.githubusercontent.com/u/7556571?v=3","gravatar_id":"","url":"https://api.github.com/users/pomegranited","html_url":"https://github.com/pomegranited","followers_url":"https://api.github.com/users/pomegranited/followers","following_url":"https://api.github.com/users/pomegranited/following{/other_user}","gists_url":"https://api.github.com/users/pomegranited/gists{/gist_id}","starred_url":"https://api.github.com/users/pomegranited/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pomegranited/subscriptions","organizations_url":"https://api.github.com/users/pomegranited/orgs","repos_url":"https://api.github.com/users/pomegranited/repos","events_url":"https://api.github.com/users/pomegranited/events{/privacy}","received_events_url":"https://api.github.com/users/pomegranited/received_events","type":"User","site_admin":false},"body":"@haikuginger & @bradenmacdonald \r\n\r\nI totally agree that we need consistent file naming across all of these changes.  Configuration is the most challenging part of the analytics setup, and I don't want to do anything to make it more so!\r\n\r\nSo I vote to use this as the default everywhere: `{course_id}_{report_name}.csv`, where the `{report_name}` in this case is `problem_response`.\r\n\r\nReasons:\r\n\r\n* `{course_id}`: Sanitized as filename safe, as discussed.  I'm working on a [change to opaque-keys](https://github.com/edx/opaque-keys/pull/74) to share the logic for this, but it's still [under discussion](https://groups.google.com/forum/#!topic/edx-code/d52LqI0xYv4).\r\n* No subdirectories: Some storage mechanisms require subdirectories to exist before files can be created inside them.  S3 doesn't care, since it's not really a file system, but we can't assume S3.\r\n* No timestamps:  The report endpoint needs to look in a single location for the \"most recent\" file, and there's no current use case for keeping old reports.  With S3, it's unfortunately not that easy to find the \"most recent file\" from a list of files.  You have to fetch the whole list (1,000 max at a time), and sort them yourself, which seems like more effort than it's worth.  I'm happy to discuss alternatives, though.\r\n* .csv extension: because it's a CSV.\r\n\r\nI've updated this PR to reflect this: [ab61d8d](https://github.com/edx/edx-analytics-pipeline/pull/274/commits/ab61d8dee0101f086ca75ce87624574cb38b701a)\r\n\r\n@haikuginger, if you agree, could you please update the configuration and data API PRs?","created_at":"2016-08-14T07:54:29Z","updated_at":"2016-08-14T07:54:29Z","html_url":"https://github.com/edx/edx-analytics-pipeline/pull/274#discussion_r74697464","pull_request_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274","_links":{"self":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments/74697464"},"html":{"href":"https://github.com/edx/edx-analytics-pipeline/pull/274#discussion_r74697464"},"pull_request":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274"}}},"pull_request":{"url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274","id":80062581,"html_url":"https://github.com/edx/edx-analytics-pipeline/pull/274","diff_url":"https://github.com/edx/edx-analytics-pipeline/pull/274.diff","patch_url":"https://github.com/edx/edx-analytics-pipeline/pull/274.patch","issue_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/274","number":274,"state":"open","locked":false,"title":"Adds tasks to create Problem Response reports","user":{"login":"pomegranited","id":7556571,"avatar_url":"https://avatars.githubusercontent.com/u/7556571?v=3","gravatar_id":"","url":"https://api.github.com/users/pomegranited","html_url":"https://github.com/pomegranited","followers_url":"https://api.github.com/users/pomegranited/followers","following_url":"https://api.github.com/users/pomegranited/following{/other_user}","gists_url":"https://api.github.com/users/pomegranited/gists{/gist_id}","starred_url":"https://api.github.com/users/pomegranited/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pomegranited/subscriptions","organizations_url":"https://api.github.com/users/pomegranited/orgs","repos_url":"https://api.github.com/users/pomegranited/repos","events_url":"https://api.github.com/users/pomegranited/events{/privacy}","received_events_url":"https://api.github.com/users/pomegranited/received_events","type":"User","site_admin":false},"body":"This change adds a set of tasks which read `problem_check` events from the tracking logs, collate the \"latest problem response\" data into a Hive table, and generate CSV reports from this data.\r\n\r\n**Discussions**: This change is related to [edx-analytics-data-api PR #128](https://github.com/edx/edx-analytics-data-api/pull/128), which has been discussed extensively with the Analytics team.\r\n\r\n**Dependencies**: None\r\n\r\n**Sandbox URL**: TBD\r\n\r\n**Partner information**: 3rd party-hosted open edX instance\r\n\r\n**Testing instructions**:\r\n\r\n1. See [edx-analytics-data-api PR 128](https://github.com/edx/edx-analytics-data-api/pull/128) for analyticstack test setup instructions.  \r\n\r\n    Notes:\r\n    * You don't need the Analytics API setup, or the custom ElasticSearch.\r\n    * When installing the analytics-pipeline, take care to install this PR's branch:\r\n\r\n    ```\r\nexport PIPELINE_REPO=\"https://github.com/open-craft/edx-analytics-pipeline.git\"\r\nexport PIPELINE_BRANCH=\"jill/latest-problem-response\"\r\n```\r\n   * And run this `launch-task` instead of `ImportEnrollmentsIntoMysql` or `ModuleEngagementWorkflowTask`:\r\n\r\n   ```\r\nProblemResponseReportWorkflow \\\r\n            --interval-start 2016-01-01  \\\r\n            --local-scheduler --n-reduce-tasks 2 --remote-log-level DEBUG \\\r\n            --output-root /edx/var/reports/ \\\r\n            --marker hdfs://localhost:9000/reports/marker`date +%s` \\\r\n            --hive-overwrite\r\n```\r\n   * You'll need to create `/edx/var/reports`, and ensure the hadoop user can write to it.\r\n   * Use the `--hive-overwrite` parameter to see data changes if tracking logs change.\r\n2. To view the problem response data, query the hive table created by the task:\r\n\r\n    ```\r\nsudo su hadoop\r\nhive\r\nhive> select * from problem_response_latest;\r\n```\r\n3. To see the CSV reports, check `/edx/var/reports`.  There should be one file created there per course.\r\n3. You can add more problem attempts via the LMS, and see them reflected in the reports by re-running the task.\r\n\r\n    This task does not depend on any other enrolment or user profile data, so you can also dump any tracking.log files into hdfs and re-run the task, e.g.\r\n\r\n    ```\r\nsudo su hadoop\r\nhdfs dfs -put -f /edx/src/tracking.log-real-data-from-course hdfs://localhost:9000/data/tracking.log-REAL\r\n```\r\n\r\n4. Run unit tests:\r\n\r\n    ```\r\nvagrant ssh\r\ncd /var/lib/analytics-tasks/analyticstack/repo\r\nsource ../venv/bin/activate\r\nexport WHEEL_URL=http://edx-wheelhouse.s3-website-us-east-1.amazonaws.com/Ubuntu/precise\r\nexport WHEEL_PYVER=python2.7\r\nmake coverage\r\n```\r\n\r\n**Author notes and concerns**:\r\n\r\n1. I'm not sure how best to support updates to this data.\r\n\r\n    The tasks are modelled after answer_dist.AnswerDistribution, which may be configured to run over all the tracking data all the time.  And we can run with the `--overwrite-hive` flag turned on to keep the Hive table updated, but this seems really extreme, and perhaps unsustainable?\r\n2. Need to add acceptance tests.\r\n\r\n**Reviewers**\r\n- [x] @haikuginger \r\n- [ ] edX reviewer[s] TBD","created_at":"2016-08-04T11:31:17Z","updated_at":"2016-08-14T07:54:29Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5c1d710a2aac46d4f939709968e3ff995df10867","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274/commits","review_comments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274/comments","review_comment_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments{/number}","comments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/274/comments","statuses_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/statuses/25674d8bbd8e90410de529f3207bd0a4d4f4db5a","head":{"label":"open-craft:jill/latest-problem-response","ref":"jill/latest-problem-response","sha":"25674d8bbd8e90410de529f3207bd0a4d4f4db5a","user":{"login":"open-craft","id":7414786,"avatar_url":"https://avatars.githubusercontent.com/u/7414786?v=3","gravatar_id":"","url":"https://api.github.com/users/open-craft","html_url":"https://github.com/open-craft","followers_url":"https://api.github.com/users/open-craft/followers","following_url":"https://api.github.com/users/open-craft/following{/other_user}","gists_url":"https://api.github.com/users/open-craft/gists{/gist_id}","starred_url":"https://api.github.com/users/open-craft/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/open-craft/subscriptions","organizations_url":"https://api.github.com/users/open-craft/orgs","repos_url":"https://api.github.com/users/open-craft/repos","events_url":"https://api.github.com/users/open-craft/events{/privacy}","received_events_url":"https://api.github.com/users/open-craft/received_events","type":"Organization","site_admin":false},"repo":{"id":38666315,"name":"edx-analytics-pipeline","full_name":"open-craft/edx-analytics-pipeline","owner":{"login":"open-craft","id":7414786,"avatar_url":"https://avatars.githubusercontent.com/u/7414786?v=3","gravatar_id":"","url":"https://api.github.com/users/open-craft","html_url":"https://github.com/open-craft","followers_url":"https://api.github.com/users/open-craft/followers","following_url":"https://api.github.com/users/open-craft/following{/other_user}","gists_url":"https://api.github.com/users/open-craft/gists{/gist_id}","starred_url":"https://api.github.com/users/open-craft/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/open-craft/subscriptions","organizations_url":"https://api.github.com/users/open-craft/orgs","repos_url":"https://api.github.com/users/open-craft/repos","events_url":"https://api.github.com/users/open-craft/events{/privacy}","received_events_url":"https://api.github.com/users/open-craft/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/open-craft/edx-analytics-pipeline","description":"","fork":true,"url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline","forks_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/forks","keys_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/teams","hooks_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/hooks","issue_events_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/issues/events{/number}","events_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/events","assignees_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/assignees{/user}","branches_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/branches{/branch}","tags_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/tags","blobs_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/git/refs{/sha}","trees_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/statuses/{sha}","languages_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/languages","stargazers_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/stargazers","contributors_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/contributors","subscribers_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/subscribers","subscription_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/subscription","commits_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/commits{/sha}","git_commits_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/git/commits{/sha}","comments_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/comments{/number}","issue_comment_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/issues/comments{/number}","contents_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/contents/{+path}","compare_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/merges","archive_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/downloads","issues_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/issues{/number}","pulls_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/pulls{/number}","milestones_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/milestones{/number}","notifications_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/labels{/name}","releases_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/releases{/id}","deployments_url":"https://api.github.com/repos/open-craft/edx-analytics-pipeline/deployments","created_at":"2015-07-07T05:33:39Z","updated_at":"2015-07-07T05:33:40Z","pushed_at":"2016-08-14T07:53:01Z","git_url":"git://github.com/open-craft/edx-analytics-pipeline.git","ssh_url":"git@github.com:open-craft/edx-analytics-pipeline.git","clone_url":"https://github.com/open-craft/edx-analytics-pipeline.git","svn_url":"https://github.com/open-craft/edx-analytics-pipeline","homepage":null,"size":2560,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":2,"forks":0,"open_issues":2,"watchers":0,"default_branch":"master"}},"base":{"label":"edx:master","ref":"master","sha":"891a6935a7f9921f9033e8aac679aa6288294014","user":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"repo":{"id":22929261,"name":"edx-analytics-pipeline","full_name":"edx/edx-analytics-pipeline","owner":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/edx/edx-analytics-pipeline","description":"","fork":false,"url":"https://api.github.com/repos/edx/edx-analytics-pipeline","forks_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/forks","keys_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/teams","hooks_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/hooks","issue_events_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/events{/number}","events_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/events","assignees_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/assignees{/user}","branches_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/branches{/branch}","tags_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/tags","blobs_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/refs{/sha}","trees_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/statuses/{sha}","languages_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/languages","stargazers_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/stargazers","contributors_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/contributors","subscribers_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/subscribers","subscription_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/subscription","commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/commits{/sha}","git_commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/commits{/sha}","comments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/comments{/number}","issue_comment_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/comments{/number}","contents_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/contents/{+path}","compare_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/merges","archive_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/downloads","issues_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues{/number}","pulls_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls{/number}","milestones_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/milestones{/number}","notifications_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/labels{/name}","releases_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/releases{/id}","deployments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/deployments","created_at":"2014-08-13T19:57:17Z","updated_at":"2016-07-28T11:45:35Z","pushed_at":"2016-08-14T07:53:02Z","git_url":"git://github.com/edx/edx-analytics-pipeline.git","ssh_url":"git@github.com:edx/edx-analytics-pipeline.git","clone_url":"https://github.com/edx/edx-analytics-pipeline.git","svn_url":"https://github.com/edx/edx-analytics-pipeline","homepage":null,"size":3939,"stargazers_count":27,"watchers_count":27,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":46,"mirror_url":null,"open_issues_count":13,"forks":46,"open_issues":13,"watchers":27,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274"},"html":{"href":"https://github.com/edx/edx-analytics-pipeline/pull/274"},"issue":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/274"},"comments":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/274/comments"},"review_comments":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274/comments"},"review_comment":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/274/commits"},"statuses":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/statuses/25674d8bbd8e90410de529f3207bd0a4d4f4db5a"}}}}
{"actor":{"display_login":"pomegranited"}}