{"action":"created","comment":{"url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments/57232640","id":57232640,"diff_hunk":"@@ -276,85 +298,885 @@ def insert_source_task(self):\n         return partition_task.data_task\n \n \n-class ModuleEngagementVerticaTask(ModuleEngagementDownstreamMixin, VerticaCopyTask):\n+class ModuleEngagementIntervalTask(\n+    MapReduceJobTaskMixin, EventLogSelectionDownstreamMixin, WarehouseMixin,\n+    OverwriteOutputMixin, luigi.WrapperTask\n+):\n+    \"\"\"Compute engagement information over a range of dates and insert the results into Hive and MySQL\"\"\"\n+\n+    overwrite_last_n_days = luigi.IntParameter(\n+        config_path={'section': 'module-engagement', 'name': 'overwrite_last_n_days'},\n+        default=2\n+    )\n+\n+    def requires(self):\n+        for date_index, date in enumerate(reversed([d for d in self.interval])):\n+            should_overwrite = self.overwrite\n+            if date_index < self.overwrite_last_n_days:\n+                should_overwrite = True\n+\n+            yield ModuleEngagementPartitionTask(\n+                date=date,\n+                n_reduce_tasks=self.n_reduce_tasks,\n+                warehouse_path=self.warehouse_path,\n+                overwrite=should_overwrite,\n+            )\n+            yield ModuleEngagementMysqlTask(\n+                date=date,\n+                n_reduce_tasks=self.n_reduce_tasks,\n+                warehouse_path=self.warehouse_path,\n+                overwrite=should_overwrite,\n+            )\n+\n+    def output(self):\n+        return [task.output() for task in self.requires()]\n+\n+    def get_raw_data_tasks(self):\n+        \"\"\"\n+        A generator that iterates through all tasks used to generate the data in each partition in the interval.\n+\n+        This can be used by downstream map reduce jobs to read all of the raw data.\n+        \"\"\"\n+        for task in self.requires():\n+            if isinstance(task, ModuleEngagementPartitionTask):\n+                yield task.data_task\n+\n+\n+class ModuleEngagementSummaryRecord(Record):\n+    \"\"\"\n+    Summarizes a user's engagement with a particular course on a particular day with simple counts of activity.\n+    \"\"\"\n+\n+    course_id = StringField()\n+    username = StringField()\n+    start_date = DateField()\n+    end_date = DateField()\n+    problem_attempts = IntegerField(is_metric=True)\n+    problems_attempted = IntegerField(is_metric=True)\n+    problems_completed = IntegerField(is_metric=True)\n+    problem_attempts_per_completed = FloatField(is_metric=True)\n+    videos_viewed = IntegerField(is_metric=True)\n+    discussion_contributions = IntegerField(is_metric=True)\n+    days_active = IntegerField()\n+\n+    def get_metrics(self):\n+        \"\"\"\n+        A generator that returns all fields that are metrics.\n+\n+        Returns: A generator of tuples whose first element is the metric name, and the second is the value of the metric\n+            for this particular record.\n+        \"\"\"\n+        for field_name, field_obj in self.get_fields().items():\n+            if getattr(field_obj, 'is_metric', False):\n+                yield field_name, getattr(self, field_name)\n+\n+\n+class ModuleEngagementSummaryRecordBuilder(object):\n+    \"\"\"Gather the data needed to emit a sparse weekly course engagement record\"\"\"\n+\n+    def __init__(self):\n+        self.problem_attempts = 0\n+        self.problems_attempted = set()\n+        self.problems_completed = set()\n+        self.videos_viewed = set()\n+        self.discussion_contributions = 0\n+        self.days_active = set()\n+\n+\n+class ModuleEngagementSummaryDataTask(\n+    ModuleEngagementDownstreamMixin, OverwriteOutputMixin, MapReduceJobTask\n+):\n     \"\"\"\n-    Replicate the hive table in Vertica.\n+    Store a summary of student engagement with their courses on particular dates aggregated using a sliding window.\n \n-    Note that it is updated incrementally.\n+    Only emits a record if the user did something in the course on that particular day, this dramatically reduces the\n+    volume of data in this table and keeps it manageable.\n+\n+    Currently this analyzes a sliding one week window of data to generate the summary.\n     \"\"\"\n \n+    # NOTE: We could have done this in Hive, but this MR job is much more simple than the necessary Hive query.\n+    # particularly given the nuance of how some of the metrics are aggregated (like attempts per completion).\n+\n+    output_root = luigi.Parameter()\n+\n+    def __init__(self, *args, **kwargs):\n+        super(ModuleEngagementSummaryDataTask, self).__init__(*args, **kwargs)\n+\n+        start_date = self.date - datetime.timedelta(weeks=1)\n+        self.interval = date_interval.Custom(start_date, self.date)\n+\n+    def requires_local(self):\n+        return ModuleEngagementIntervalTask(\n+            interval=self.interval,\n+            n_reduce_tasks=self.n_reduce_tasks,\n+            warehouse_path=self.warehouse_path,\n+        )\n+\n+    def requires_hadoop(self):\n+        # The hadoop task only wants to read the raw Hive partitions, so only use them as input to the job.\n+        return list(self.requires_local().get_raw_data_tasks())\n+\n+    def mapper(self, line):\n+        record = ModuleEngagementRecord.from_tsv(line)\n+        yield ((record.course_id, record.username), line.rstrip('\\r\\n'))\n+\n+    def reducer(self, key, lines):\n+        \"\"\"Calculate counts for events corresponding to user and course in a given time period.\"\"\"\n+        course_id, username = key\n+\n+        output_record = ModuleEngagementSummaryRecordBuilder()\n+        for line in lines:\n+            record = ModuleEngagementRecord.from_tsv(line)\n+\n+            output_record.days_active.add(record.date)\n+\n+            count = int(record.count)\n+            if record.entity_type == 'problem':\n+                if record.event == 'attempted':\n+                    output_record.problem_attempts += count\n+                    output_record.problems_attempted.add(record.entity_id)\n+                elif record.event == 'completed':\n+                    output_record.problems_completed.add(record.entity_id)\n+            elif record.entity_type == 'video':\n+                if record.event == 'viewed':\n+                    output_record.videos_viewed.add(record.entity_id)\n+            elif record.entity_type == 'discussion':\n+                output_record.discussion_contributions += count\n+            else:\n+                log.warn('Unrecognized entity type: %s', record.entity_type)\n+\n+        attempts_per_completion = self.compute_attempts_per_completion(\n+            output_record.problem_attempts,\n+            len(output_record.problems_completed)\n+        )\n+\n+        yield ModuleEngagementSummaryRecord(\n+            course_id,\n+            username,\n+            self.interval.date_a,\n+            self.interval.date_b,\n+            output_record.problem_attempts,\n+            len(output_record.problems_attempted),\n+            len(output_record.problems_completed),\n+            attempts_per_completion,\n+            len(output_record.videos_viewed),\n+            output_record.discussion_contributions,\n+            len(output_record.days_active)\n+        ).to_string_tuple()\n+\n+    @staticmethod\n+    def compute_attempts_per_completion(num_problem_attempts, num_problems_completed):\n+        \"\"\"\n+        The ratio of attempts per correct problem submission is an indicator of how much a student is struggling.\n+\n+        If a student has not completed any problems a value of float('inf') is returned.\n+        \"\"\"\n+        if num_problems_completed > 0:\n+            attempts_per_completion = float(num_problem_attempts) / num_problems_completed\n+        else:\n+            attempts_per_completion = float('inf')\n+        return attempts_per_completion\n+\n+    def output(self):\n+        return get_target_from_url(self.output_root)\n+\n+    def run(self):\n+        self.remove_output_on_overwrite()\n+        return super(ModuleEngagementSummaryDataTask, self).run()\n+\n+\n+class ModuleEngagementSummaryTableTask(BareHiveTableTask):\n+    \"\"\"The hive table for this summary of engagement data.\"\"\"\n+\n     @property\n-    def insert_source_task(self):\n-        partition_task = ModuleEngagementPartitionTask(\n+    def partition_by(self):\n+        return 'dt'\n+\n+    @property\n+    def table(self):\n+        return 'module_engagement_summary'\n+\n+    @property\n+    def columns(self):\n+        return ModuleEngagementSummaryRecord.get_hive_schema()\n+\n+\n+class ModuleEngagementSummaryPartitionTask(\n+    ModuleEngagementDownstreamMixin, HivePartitionTask\n+):\n+    \"\"\"The hive partition for this summary of engagement data.\"\"\"\n+\n+    @property\n+    def partition_value(self):\n+        \"\"\"Partition based on the end date of the sliding week-long interval.\"\"\"\n+        return self.date.isoformat()  # pylint: disable=no-member\n+\n+    @property\n+    def hive_table_task(self):\n+        return ModuleEngagementSummaryTableTask(\n+            warehouse_path=self.warehouse_path,\n+            overwrite=self.overwrite,\n+        )\n+\n+    @property\n+    def data_task(self):\n+        return ModuleEngagementSummaryDataTask(\n+            date=self.date,\n+            n_reduce_tasks=self.n_reduce_tasks,\n+            output_root=self.partition_location,\n+        )\n+\n+\n+class ModuleEngagementSummaryMetricRangeRecord(Record):\n+    \"\"\"\n+    Metrics are analyzed to determine interesting ranges.\n+\n+    This could really be any arbitrary range, maybe just the 3rd percentile, or perhaps it could define two groups,\n+    below and above the median.\n+    \"\"\"\n+\n+    course_id = StringField(length=255, nullable=False)\n+    start_date = DateField()\n+    end_date = DateField()\n+    metric = StringField(length=50, nullable=False)\n+    range_type = StringField(length=50, nullable=False)\n+    low_value = FloatField()\n+    high_value = FloatField()\n+\n+\n+METRIC_RANGE_HIGH = 'high'\n+METRIC_RANGE_LOW = 'low'\n+\n+\n+class ModuleEngagementSummaryMetricRangesDataTask(\n+    ModuleEngagementDownstreamMixin, OverwriteOutputMixin, MapReduceJobTask\n+):\n+    \"\"\"\n+    Summarize the metrics of interest and persist the interesting ranges.\n+\n+    This task currently computes \"low\" and \"high\" ranges. The \"low\" range is defined as the [min(), 15th percentile)\n+    interval. The \"high\" range is defined as the [85th percentile, max()). There is an implied \"middle\" range that is\n+    not persisted since it is fully defined by the low and the high. Note that all intervals are left-closed.\n+    \"\"\"\n+\n+    EXPAND_RANGE_BY = 0.1\n+\n+    output_root = luigi.Parameter()\n+    low_percentile = luigi.FloatParameter(default=15.0)\n+    high_percentile = luigi.FloatParameter(default=85.0)\n+\n+    def requires(self):\n+        # NOTE: The hadoop job needs the raw data to use as input, not the hive partition metadata, which is the output\n+        # of the partition task.\n+        partition_task = ModuleEngagementSummaryPartitionTask(\n             date=self.date,\n             n_reduce_tasks=self.n_reduce_tasks,\n-            overwrite=self.overwrite,\n         )\n         return partition_task.data_task\n \n+    def mapper(self, line):\n+        record = ModuleEngagementSummaryRecord.from_tsv(line)\n+        yield record.course_id, line.rstrip('\\n')\n+\n+    def reducer(self, course_id, lines):\n+        \"\"\"\n+        Analyze all summary records for particular course.\n+\n+        This will include all students performed any activity of interest in the past week.\n+        \"\"\"\n+        metric_values = defaultdict(list)\n+\n+        first_record = None\n+        for line in lines:\n+            record = ModuleEngagementSummaryRecord.from_tsv(line)\n+            if first_record is None:\n+                # There is some information we need to copy out of the summary records, so just grab one of them. There\n+                # will be at least one, or else the reduce function would have never been called.\n+                first_record = record\n+            for metric, value in record.get_metrics():\n+                if value != 0:\n+                    # NOTE: a lot of people don't participate, so don't include them in the analysis. Otherwise it would\n+                    # look like doing *anything* in the course meant you were doing really well.\n+                    metric_values[metric].append(value)\n+\n+        for metric in sorted(metric_values):\n+            values = metric_values[metric]\n+            range_values = numpy.percentile(  # pylint: disable=no-member\n+                values, [0.0, self.low_percentile, self.high_percentile, 100.0]\n+            )\n+            min_value = range_values[0] - self.EXPAND_RANGE_BY\n+            max_value = range_values[3] + self.EXPAND_RANGE_BY\n+            ranges = [\n+                (METRIC_RANGE_LOW, min_value, range_values[1]),\n+                (METRIC_RANGE_HIGH, range_values[2], max_value),\n+            ]\n+            for range_type, low_value, high_value in ranges:\n+                yield ModuleEngagementSummaryMetricRangeRecord(\n+                    course_id=course_id,\n+                    start_date=first_record.start_date,\n+                    end_date=first_record.end_date,\n+                    metric=metric,\n+                    range_type=range_type,\n+                    low_value=low_value,\n+                    high_value=high_value\n+                ).to_string_tuple()\n+\n+    def output(self):\n+        return get_target_from_url(self.output_root)\n+\n+    def extra_modules(self):\n+        return [numpy]\n+\n+    def run(self):\n+        self.remove_output_on_overwrite()\n+        return super(ModuleEngagementSummaryMetricRangesDataTask, self).run()\n+\n+\n+class ModuleEngagementSummaryMetricRangesTableTask(BareHiveTableTask):\n+    \"\"\"Hive table for metric ranges\"\"\"\n+\n+    @property\n+    def partition_by(self):\n+        return 'dt'\n+\n+    @property\n+    def table(self):\n+        return 'module_engagement_metric_ranges'\n+\n+    @property\n+    def columns(self):\n+        return ModuleEngagementSummaryMetricRangeRecord.get_hive_schema()\n+\n+\n+class ModuleEngagementSummaryMetricRangesPartitionTask(\n+    ModuleEngagementDownstreamMixin, HivePartitionTask\n+):\n+    \"\"\"Hive partition for metric ranges\"\"\"\n+\n+    @property\n+    def partition_value(self):\n+        \"\"\"Use the end date as the partition identifier\"\"\"\n+        return self.date.isoformat()  # pylint: disable=no-member\n+\n+    @property\n+    def hive_table_task(self):\n+        return ModuleEngagementSummaryMetricRangesTableTask(\n+            warehouse_path=self.warehouse_path,\n+            overwrite=self.overwrite,\n+        )\n+\n+    @property\n+    def data_task(self):\n+        return ModuleEngagementSummaryMetricRangesDataTask(\n+            date=self.date,\n+            n_reduce_tasks=self.n_reduce_tasks,\n+            output_root=self.partition_location,\n+        )\n+\n+\n+class ModuleEngagementSummaryMetricRangesMysqlTask(ModuleEngagementDownstreamMixin, MysqlInsertTask):\n+    \"\"\"Result store storage for the metric ranges.\"\"\"\n+\n+    overwrite = luigi.BooleanParameter(default=True)\n+\n     @property\n     def table(self):\n-        return 'f_module_engagement'\n+        return \"module_engagement_metric_ranges\"\n \n     @property\n-    def default_columns(self):\n-        \"\"\"List of tuples defining name and definition of automatically-filled columns.\"\"\"\n+    def auto_primary_key(self):\n         return None\n \n     @property\n     def columns(self):\n-        return ModuleEngagementRecord.get_sql_schema()\n+        return ModuleEngagementSummaryMetricRangeRecord.get_sql_schema()\n+\n+    @property\n+    def keys(self):\n+        # Store the records in order by course_id to allow for very fast access for point queries by course.\n+        return [\n+            ('PRIMARY KEY', ['course_id', 'metric', 'range_type'])\n+        ]\n+\n+    @property\n+    def insert_source_task(self):\n+        partition_task = ModuleEngagementSummaryMetricRangesPartitionTask(\n+            date=self.date,\n+            n_reduce_tasks=self.n_reduce_tasks,\n+            overwrite=self.overwrite,\n+        )\n+        return partition_task.data_task\n \n \n-class OptionalVerticaMixin(object):\n+class ModuleEngagementUserSegmentRecord(Record):\n     \"\"\"\n-    If a vertica connection is present, replicate the data there. Otherwise, don't require those insertion tasks.\n+    Maps a user's activity in a course to various segments.\n     \"\"\"\n \n-    vertica_schema = luigi.Parameter(default=None)\n-    vertica_credentials = luigi.Parameter(default=None)\n+    course_id = StringField()\n+    username = StringField()\n+    start_date = DateField()\n+    end_date = DateField()\n+    segment = StringField()\n+    reason = StringField()\n \n-    def __init__(self, *args, **kwargs):\n-        super(OptionalVerticaMixin, self).__init__(*args, **kwargs)\n \n-        if not self.vertica_credentials:\n-            self.vertica_credentials = get_config().get('vertica-export', 'credentials', None)\n+SEGMENT_HIGHLY_ENGAGED = 'highly_engaged'\n+SEGMENT_STRUGGLING = 'struggling'\n \n-        if not self.vertica_schema:\n-            self.vertica_schema = get_config().get('vertica-export', 'schema', None)\n \n-        self.vertica_enabled = self.vertica_credentials and self.vertica_schema\n+class ModuleEngagementUserSegmentDataTask(\n+    ModuleEngagementDownstreamMixin, OverwriteOutputMixin, MapReduceJobTask\n+):\n+    \"\"\"\n+    Segment the user population in each course using their activity data.\n \n+    Note that a particular user may belong to many different segments. Also note that this will not be an exhaustive\n+    list of segments. Some segments are based on the absence of activity data, which cannot be computed here since this\n+    job only reads records for users that performed *some* activity in the course.\n \n-class ModuleEngagementIntervalTask(\n-    MapReduceJobTaskMixin, EventLogSelectionDownstreamMixin, WarehouseMixin, OptionalVerticaMixin,\n-    OverwriteOutputMixin, luigi.WrapperTask\n+    Looks at the last week of activity summary records for each user and assigns the following segments:\n+\n+        struggling: The user has a value for problem_attempts_per_completed that is in the top 15% of all non-zero\n+            values.\n+        highly_engaged: The user is in the top 15% for any of the metrics except problem_attempts\n+            and problem_attempts_per_completed.\n+    \"\"\"\n+\n+    output_root = luigi.Parameter()\n+\n+    def requires_local(self):\n+        # NOTE: Most of this table is pickled on the master node and distributed to the slaves via the distributed\n+        # cache.\n+        mysql_task = ModuleEngagementSummaryMetricRangesMysqlTask(\n+            date=self.date,\n+            n_reduce_tasks=self.n_reduce_tasks,\n+        )\n+        return {\n+            'range_data': mysql_task.insert_source_task,\n+            'range_mysql': mysql_task,\n+        }\n+\n+    def requires_hadoop(self):\n+        return ModuleEngagementSummaryPartitionTask(\n+            date=self.date,\n+            n_reduce_tasks=self.n_reduce_tasks,\n+        ).data_task\n+\n+    def init_local(self):\n+        super(ModuleEngagementUserSegmentDataTask, self).init_local()\n+        self.high_metric_ranges = defaultdict(dict)\n+\n+        # Grab all of the \"high\" ranges since those are the only ones we are going to use. This will load\n+        # O(num_courses * num_metrics) records into memory, and then into the pickled job instance. This should be a\n+        # relatively small number (thousands).\n+        with self.input_local()['range_data'].open('r') as metric_ranges_target:\n+            for line in metric_ranges_target:\n+                range_record = ModuleEngagementSummaryMetricRangeRecord.from_tsv(line)\n+                if range_record.range_type == METRIC_RANGE_HIGH:\n+                    self.high_metric_ranges[range_record.course_id][range_record.metric] = range_record\n+\n+    def mapper(self, line):\n+        record = ModuleEngagementSummaryRecord.from_tsv(line)\n+        yield (record.course_id, record.username), line.rstrip('\\n')\n+\n+    def reducer(self, key, lines):\n+        course_id, username = key\n+\n+        records = [ModuleEngagementSummaryRecord.from_tsv(line) for line in lines]\n+\n+        # Maps segment names to a set of \"reasons\" that tell why the user was placed in that segment\n+        segments = defaultdict(set)\n+        most_recent_summary = records[-1]","path":"edx/analytics/tasks/module_engagement.py","position":650,"original_position":650,"commit_id":"d1139f279f4b1e9bf186a132331a2959ec390dca","original_commit_id":"d1139f279f4b1e9bf186a132331a2959ec390dca","user":{"login":"brianhw","id":2534095,"avatar_url":"https://avatars.githubusercontent.com/u/2534095?v=3","gravatar_id":"","url":"https://api.github.com/users/brianhw","html_url":"https://github.com/brianhw","followers_url":"https://api.github.com/users/brianhw/followers","following_url":"https://api.github.com/users/brianhw/following{/other_user}","gists_url":"https://api.github.com/users/brianhw/gists{/gist_id}","starred_url":"https://api.github.com/users/brianhw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/brianhw/subscriptions","organizations_url":"https://api.github.com/users/brianhw/orgs","repos_url":"https://api.github.com/users/brianhw/repos","events_url":"https://api.github.com/users/brianhw/events{/privacy}","received_events_url":"https://api.github.com/users/brianhw/received_events","type":"User","site_admin":false},"body":"I don't understand the name of this.  Do you assume that the records presented to the user are in sorted order, from least recent to most recent?","created_at":"2016-03-23T20:38:56Z","updated_at":"2016-03-23T20:38:56Z","html_url":"https://github.com/edx/edx-analytics-pipeline/pull/215#discussion_r57232640","pull_request_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/215","_links":{"self":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments/57232640"},"html":{"href":"https://github.com/edx/edx-analytics-pipeline/pull/215#discussion_r57232640"},"pull_request":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/215"}}},"pull_request":{"url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/215","id":62494815,"html_url":"https://github.com/edx/edx-analytics-pipeline/pull/215","diff_url":"https://github.com/edx/edx-analytics-pipeline/pull/215.diff","patch_url":"https://github.com/edx/edx-analytics-pipeline/pull/215.patch","issue_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/215","number":215,"state":"open","locked":false,"title":"WIP: Generate a searchable student roster","user":{"login":"mulby","id":627177,"avatar_url":"https://avatars.githubusercontent.com/u/627177?v=3","gravatar_id":"","url":"https://api.github.com/users/mulby","html_url":"https://github.com/mulby","followers_url":"https://api.github.com/users/mulby/followers","following_url":"https://api.github.com/users/mulby/following{/other_user}","gists_url":"https://api.github.com/users/mulby/gists{/gist_id}","starred_url":"https://api.github.com/users/mulby/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mulby/subscriptions","organizations_url":"https://api.github.com/users/mulby/orgs","repos_url":"https://api.github.com/users/mulby/repos","events_url":"https://api.github.com/users/mulby/events{/privacy}","received_events_url":"https://api.github.com/users/mulby/received_events","type":"User","site_admin":false},"body":"@brianhw - in case you have time to start review when I'm gone, I figured I would at least post this.\r\n\r\nOpen to suggestions about how to make this easier to review.\r\n\r\nTODO:\r\n\r\n- [ ] finish tests cases for elasticsearch_load\r\n- [ ] lots of docstrings and comments!\r\n- [ ] document the sort ordering scheme\r\n- [ ] pep8, pylint\r\n- [ ] make sure the acceptance test isn't broken (I haven't run it in a while)\r\n\r\nOther optional tasks:\r\n- [ ] Handle the case when there is no tracking logs for a particular date. This currently causes a failure, which makes testing on devstack difficult.\r\n- [ ] The big hive query runs and overwrites any existing data if the underlying data exists but the hive partition does not. This can happen fairly frequently during testing if you use a new cluster with existing data.","created_at":"2016-03-11T03:57:46Z","updated_at":"2016-03-23T20:38:56Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2c924135f8af876660a4f70b0a0db8e14d92c0e0","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/215/commits","review_comments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/215/comments","review_comment_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments{/number}","comments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/215/comments","statuses_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/statuses/d1139f279f4b1e9bf186a132331a2959ec390dca","head":{"label":"edx:gabe/la-roster","ref":"gabe/la-roster","sha":"d1139f279f4b1e9bf186a132331a2959ec390dca","user":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"repo":{"id":22929261,"name":"edx-analytics-pipeline","full_name":"edx/edx-analytics-pipeline","owner":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/edx/edx-analytics-pipeline","description":"","fork":false,"url":"https://api.github.com/repos/edx/edx-analytics-pipeline","forks_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/forks","keys_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/teams","hooks_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/hooks","issue_events_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/events{/number}","events_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/events","assignees_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/assignees{/user}","branches_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/branches{/branch}","tags_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/tags","blobs_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/refs{/sha}","trees_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/statuses/{sha}","languages_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/languages","stargazers_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/stargazers","contributors_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/contributors","subscribers_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/subscribers","subscription_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/subscription","commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/commits{/sha}","git_commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/commits{/sha}","comments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/comments{/number}","issue_comment_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/comments{/number}","contents_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/contents/{+path}","compare_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/merges","archive_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/downloads","issues_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues{/number}","pulls_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls{/number}","milestones_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/milestones{/number}","notifications_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/labels{/name}","releases_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/releases{/id}","deployments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/deployments","created_at":"2014-08-13T19:57:17Z","updated_at":"2016-03-15T11:47:11Z","pushed_at":"2016-03-23T15:13:22Z","git_url":"git://github.com/edx/edx-analytics-pipeline.git","ssh_url":"git@github.com:edx/edx-analytics-pipeline.git","clone_url":"https://github.com/edx/edx-analytics-pipeline.git","svn_url":"https://github.com/edx/edx-analytics-pipeline","homepage":null,"size":3179,"stargazers_count":21,"watchers_count":21,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":34,"mirror_url":null,"open_issues_count":13,"forks":34,"open_issues":13,"watchers":21,"default_branch":"master"}},"base":{"label":"edx:master","ref":"master","sha":"9efd9fa2b2eda31d9db4aad68355c3f3abe0c0eb","user":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"repo":{"id":22929261,"name":"edx-analytics-pipeline","full_name":"edx/edx-analytics-pipeline","owner":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/edx/edx-analytics-pipeline","description":"","fork":false,"url":"https://api.github.com/repos/edx/edx-analytics-pipeline","forks_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/forks","keys_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/teams","hooks_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/hooks","issue_events_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/events{/number}","events_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/events","assignees_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/assignees{/user}","branches_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/branches{/branch}","tags_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/tags","blobs_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/refs{/sha}","trees_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/statuses/{sha}","languages_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/languages","stargazers_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/stargazers","contributors_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/contributors","subscribers_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/subscribers","subscription_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/subscription","commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/commits{/sha}","git_commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/commits{/sha}","comments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/comments{/number}","issue_comment_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/comments{/number}","contents_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/contents/{+path}","compare_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/merges","archive_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/downloads","issues_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues{/number}","pulls_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls{/number}","milestones_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/milestones{/number}","notifications_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/labels{/name}","releases_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/releases{/id}","deployments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/deployments","created_at":"2014-08-13T19:57:17Z","updated_at":"2016-03-15T11:47:11Z","pushed_at":"2016-03-23T15:13:22Z","git_url":"git://github.com/edx/edx-analytics-pipeline.git","ssh_url":"git@github.com:edx/edx-analytics-pipeline.git","clone_url":"https://github.com/edx/edx-analytics-pipeline.git","svn_url":"https://github.com/edx/edx-analytics-pipeline","homepage":null,"size":3179,"stargazers_count":21,"watchers_count":21,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":34,"mirror_url":null,"open_issues_count":13,"forks":34,"open_issues":13,"watchers":21,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/215"},"html":{"href":"https://github.com/edx/edx-analytics-pipeline/pull/215"},"issue":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/215"},"comments":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/215/comments"},"review_comments":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/215/comments"},"review_comment":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/215/commits"},"statuses":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/statuses/d1139f279f4b1e9bf186a132331a2959ec390dca"}}}}