{"action":"created","comment":{"url":"https://api.github.com/repos/ironmussa/Optimus/pulls/comments/217130153","pull_request_review_id":154771111,"id":217130153,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIxNzEzMDE1Mw==","diff_hunk":"@@ -0,0 +1,420 @@\n+import csv\n+import json\n+import logging\n+\n+import pandas as pd\n+import requests\n+from backoff import on_exception, expo\n+from pymongo import MongoClient\n+from pyspark.sql.functions import DataFrame\n+from pyspark.sql.functions import pandas_udf, PandasUDFType\n+from ratelimit import limits, RateLimitException\n+from tqdm import tqdm_notebook\n+\n+from optimus.helpers.checkit import is_function, is_\n+\n+# Temporal col used to create a temporal ID to join the enriched data in mongo with the dataframe.\n+COL_ID = \"jazz_id\"\n+COL_RESULTS = \"jazz_results\"\n+\n+\n+class Enricher:\n+    \"\"\"\n+    Enrich data from a Pandas or Spark dataframe\n+    \"\"\"\n+\n+    def __init__(self, op=None, host=\"localhost\", port=27017, db_name=\"jazz\", collection_name=\"data\", *args,\n+                 **kwargs):\n+        \"\"\"\n+\n+        :param host: Mongo server host\n+        :param port: Mongo server port\n+        :param db_name: Mongo server database\n+        :param collection_name: Mongo serverg collection\n+        :param op: optimus instance\n+        :param args:\n+        :param kwargs:\n+        \"\"\"\n+        logging.basicConfig(format=\"%(message)s\", level=logging.INFO)\n+\n+        self.host = host\n+        self.port = port\n+        self.db_name = db_name\n+        self.collection_name = collection_name\n+        self.client = MongoClient(host, port, *args, **kwargs)\n+        self.op = op\n+\n+    def send(self, df):\n+        \"\"\"\n+        Send the dataframe to the mongo collection\n+        :param df: dataframe to be send to the enricher\n+        :return:\n+        \"\"\"\n+\n+        if is_(df, pd.DataFrame):\n+            self.get_collection(self.collection_name).insert_many(df.to_dict(\"records\"))\n+        elif is_(df, DataFrame):\n+            df.save.mongo(self.host, self.port, self.db_name, self.collection_name)\n+        else:\n+            raise Exception(\"df must by a Spark Dataframe or Pandas Dataframe\")\n+\n+    def run(self, df, collection_name=None, func_request=None, func_response=None, return_type=\"json\", calls=60,\n+            period=60, max_tries=8):\n+        \"\"\"\n+        Read a the url key from a mongo collection an make a request to a service\n+        :param df: Dataframe to me loaded to the enricher collection.\n+        :param collection_name: Custom collection to save the data.\n+        :param func_request: help to create a custom request\n+        :param func_response: help to create a custom response\n+        :param return_type:\n+        :param calls: how many call can you make\n+        :param period: in which period ot time can the call be made\n+        :param max_tries: how many retries should we do\n+        :return:\n+        \"\"\"\n+\n+        # Load the dataframe data in the enricher\n+        if is_(df, DataFrame):\n+            df = df.create_id(COL_ID)\n+\n+        # Load the dataframe data in the enricher\n+        self.send(df)\n+\n+        if collection_name is None:\n+            collection_name = self.collection_name\n+        collection = self.get_collection(collection_name)\n+\n+        # Get data that is not yet enriched\n+        cursor = collection.find({COL_RESULTS: {\"$exists\": False}})\n+\n+        total_docs = cursor.count(True)\n+\n+        if func_request is None:\n+            func_request = requests.get\n+        collection = self.get_collection(collection_name)\n+\n+        @on_exception(expo, RateLimitException, max_tries=max_tries)\n+        @limits(calls=calls, period=period)\n+        def _func_request(v):\n+            return func_request(v)\n+\n+        if total_docs > 0:\n+            for c in tqdm_notebook(cursor, total=total_docs, desc='Processing...'):\n+\n+                # Send request to the API\n+                response = _func_request(c)\n+\n+                mongo_id = c[\"_id\"]\n+\n+                if response.status_code == 200:\n+                    if return_type == \"json\":\n+                        response = json.loads(response.text)\n+                    elif return_type == \"text\":\n+                        response = response.text\n+\n+                    # Process the result with an external function\n+                    if is_function(func_response):\n+                        response = func_response(response)\n+\n+                    # Update the mongo id with the result\n+                    collection.find_and_modify(query={\"_id\": mongo_id},\n+                                               update={\"$set\": {COL_RESULTS: response}},\n+                                               upsert=False, full_response=True)\n+                else:\n+                    # The response key will remain blank so we can filter it to try in future request\n+                    logging.info(response.status_code)\n+\n+            # Append the data in enrichment to the dataframe\n+\n+            logging.info(\"Appending collection info into the dataframe\")\n+            # TODO: An elegant way to handle pickling?\n+            # take care to the pickling\n+            host = self.host\n+            port = self.port\n+            db_name = self.db_name\n+\n+            @pandas_udf('string', PandasUDFType.SCALAR)\n+            def func(value):\n+                # More about pickling\n+                from pymongo import MongoClient\n+                _client = MongoClient(host, port)\n+                _db = _client[db_name]\n+                _collection = _db[collection_name]\n+\n+                def func_serie(serie):\n+                    _cursor = _collection.find_one({COL_ID: serie}, projection={\"_id\": 0, COL_RESULTS: 1})\n+                    return _cursor[COL_RESULTS]\n+\n+                return value.apply(func_serie)\n+\n+            df = df.withColumn(COL_RESULTS, func(df[COL_ID])).cols.drop(COL_ID).run()\n+\n+            # If the process is finished, flush the Mongo collection\n+            self.flush()\n+            return df\n+        else:\n+            print(\"No records available to process\")\n+\n+    def count(self):\n+        \"\"\"\n+        Count number of documents in a collections\n+        :return:\n+        \"\"\"\n+        collection = self.get_collection(self.collection_name)\n+        cursor = collection.find()\n+        return cursor.count(True)\n+\n+    def flush(self):\n+        \"\"\"\n+        Flush the enricher default collection\n+        :return:\n+        \"\"\"\n+        count = self.count()\n+        self.drop_collection(self.collection_name)\n+        logging.info(\"Removed {count} documents\".format(count=count))\n+\n+    def collection_exists(self, collection_name):\n+        \"\"\"\n+        Check if a collection exist\n+        :param collection_name:\n+        :return:\n+        \"\"\"\n+        if collection_name in self.get_db().collection_names():\n+            return True\n+        else:\n+            return False\n+\n+    def db_exists(self, db_name):\n+        \"\"\"\n+        Check if a collection exist\n+        :param db_name:\n+        :return:\n+        \"\"\"\n+        if db_name in self.client.list_database_names():\n+            return True\n+        else:\n+            return False\n+\n+    def get_db(self):\n+        return self.client[self.db_name]\n+\n+    def get_collection(self, collection_name):\n+        \"\"\"\n+\n+        :param collection_name:\n+        :return:\n+        \"\"\"\n+        collection = self.get_db()[collection_name]\n+        return collection\n+\n+    def copy_collection(self, source_name, dest_name):\n+        \"\"\"\n+        Copy Collection\n+        :param source_name:\n+        :param dest_name:\n+        :return:\n+        \"\"\"\n+\n+        source = self.db[source_name]\n+\n+        logging.info('Dropping', dest_name, 'collection')\n+        self.db[dest_name].drop()\n+        # if data exist in the collection drop it\n+\n+        pipeline = [{\"$match\": {}},\n+                    {\"$out\": dest_name},\n+                    ]\n+        logging.info('Copying', source_name, 'collection to', dest_name, 'collection ...')\n+\n+        source.aggregate(pipeline)\n+        logging.info('Done')\n+\n+    def head(self, collection_name, n=10):\n+        \"\"\"\n+        Print n first documents\n+        :param collection_name:\n+        :param n:\n+        :return:\n+        \"\"\"\n+\n+        # try to bring a cursor from a collection\n+        cursor = self.get_collection(collection_name).find({}).limit(n)\n+        print(\"Total documents:\" + str(cursor.count()))\n+        for c in cursor:\n+            print(c)\n+\n+    def get_keys(self, collection_name=None):\n+        \"\"\"\n+        Show keys in collection\n+        :param collection_name:\n+        :return:\n+        \"\"\"\n+\n+        if not self.collection_exists(self.collection_name):\n+            raise Exception(\"Collection {collection_name} not exist\".format(collection_name=collection_name))\n+\n+        source = None\n+        if collection_name is None:\n+            source = self.get_collection(self.collection_name)\n+\n+        results = source.aggregate([\n+            {\"$project\": {\"arrayofkeyvalue\": {\"$objectToArray\": \"$$ROOT\"}}},\n+            {\"$unwind\": \"$arrayofkeyvalue\"},\n+            {\"$group\": {\"_id\": None, \"allkeys\": {\"$addToSet\": \"$arrayofkeyvalue.k\"}}}\n+        ])\n+\n+        results = list(results)[0]['allkeys']\n+        return results\n+\n+    def show_collections(self, db):\n+        \"\"\"\n+        Show collections in a database\n+        :return:\n+        \"\"\"\n+        return self.client[db].collection_names()\n+\n+        # d = dict((db, [collection for collection in self.client[db].collection_names()])\n+        #         for db in self.client.list_database_names())\n+        # print(json.dumps(d))\n+\n+    @staticmethod\n+    def drop_keys(collection_name, keys):\n+        \"\"\"\n+        Drop key in collection\n+        :return:\n+        \"\"\"\n+        for key in tqdm_notebook(keys, desc='Processing cols'):\n+            logging.info('Dropping', key, 'field')\n+            collection_name.update_many({}, {'$unset': {key: 1}})\n+\n+    def drop_collection(self, collection_name):\n+        \"\"\"\n+        Drop  a collection\n+        :param collection_name:\n+        :return:\n+        \"\"\"\n+        if collection_name is None:\n+            collection_name = self.collection_name\n+        self.get_collection(collection_name).drop()\n+\n+    def save_to_csv(self, filename, collection_name=None, projection=None, limit=0):\n+        \"\"\"\n+        Save collection to csv\n+        :param filename: Output filename\n+        :param collection_name: custom collection to save\n+        :param projection: Filter the keys on csv output\n+        :param limit: Limit the number to record in the output file\n+        :return:\n+        \"\"\"\n+\n+        if collection_name is None:\n+            collection_name = self.collection_name\n+\n+        collection = self.get_collection(collection_name)\n+\n+        try:\n+            file = open(filename, \"w\", newline='')\n+            csv_write = csv.writer(file, delimiter=\";\", quotechar=\"|\", quoting=csv.QUOTE_MINIMAL)\n+        except IOError:\n+            raise Exception(\"Could not read file {filename}\".format(filename=filename))\n+\n+        if projection is None:\n+            projection = {}\n+        projection[\"_id\"] = 0\n+\n+        try:\n+            # Save csv header\n+            for header in collection.find({}, projection).limit(1):\n+                csv_write.writerow(header.keys())\n+\n+            # Save csv body\n+            documents = collection.find({}, projection).limit(limit)\n+            count = documents.count(True)\n+\n+            # Save csv body\n+            for document in tqdm_notebook(documents, total=count, desc='Saving...'):\n+                # Get a json, transform it to str and return a semicolon separated string\n+\n+                result = list(map((lambda x: str(x)), document.values()))\n+                csv_write.writerow(result)\n+        except IOError:\n+            raise Exception(\"Could not write in {filename}\".format(filename=filename))\n+\n+        file.close()\n+\n+    def insert_to_collection(self, cursor, dest_collection_name, drop=False):\n+        \"\"\"\n+        Insert a cursor into a collection\n+        :param cursor:\n+        :param dest_collection_name:\n+        :param drop:\n+        :return:\n+        \"\"\"\n+        dest_collection = self.get_collection(dest_collection_name)\n+        if drop:\n+            dest_collection.drop()\n+\n+        if \"count\" in dir(cursor):\n+            count = cursor.count()\n+        else:\n+            count = 1\n+\n+        for c in tqdm_notebook(cursor, total=count, desc='Saving Collection'):\n+            dest_collection.insert_one(c)\n+\n+    def create_missing_fields(self, cols, collection_name=None):\n+        \"\"\"\n+        Helper function to fill missing keys in a json\n+        :param cols:\n+        :param collection_name:\n+        :return:\n+        \"\"\"\n+\n+        if collection_name is not None:\n+            source = self.get_collection(collection_name)\n+        else:\n+            source = self.collection\n+\n+        for c in tqdm_notebook(cols, total=len(cols), desc='Processing cols'):\n+            logging.info('Inserting', c)\n+            if c:\n+                source.update_many(\n+                    {c: {'$exists': False}},\n+                    {'$set':\n+                        {\n+                            c: None,\n+                        }\n+                    }\n+                );\n+            else:\n+                logging.info('Field', c, 'could not be added')\n+\n+    def cast(self, collection_name, field, convert_to):\n+        \"\"\"\n+\n+        :param collection_name:\n+        :param field:\n+        :param convert_to:\n+        :return:\n+        \"\"\"\n+        collection = self.get_collection(collection_name)\n+        cursor = collection.find({field: {'$exists': True}}).limit(0)\n+        desc = 'Converting', field, 'to', convert_to\n+\n+        if convert_to == 'int':\n+            data_type = float\n+        elif convert_to == 'float':\n+            data_type = int\n+        elif convert_to == 'string':\n+            data_type = str\n+        else:\n+            raise ValueError('Only int, float or string accepted in field param', field, 'value present')\n+\n+        for c in tqdm_notebook(cursor, total=cursor.count(), desc='Processing records'):\n+            try:\n+                val = c[field]\n+                val = data_type(c[field])\n+                collection.update_one({'_id': c['_id']}, {'$set': {field: val}})\n+\n+            except ValueError:\n+                logging.info('Could not convert \"', val, '\" to', convert_to)","path":"optimus/enricher.py","position":420,"original_position":420,"commit_id":"accc59fa0e5f7f7153775c5882b2bfb63b07423b","original_commit_id":"accc59fa0e5f7f7153775c5882b2bfb63b07423b","user":{"login":"codacy-bot","id":19940114,"node_id":"MDQ6VXNlcjE5OTQwMTE0","avatar_url":"https://avatars1.githubusercontent.com/u/19940114?v=4","gravatar_id":"","url":"https://api.github.com/users/codacy-bot","html_url":"https://github.com/codacy-bot","followers_url":"https://api.github.com/users/codacy-bot/followers","following_url":"https://api.github.com/users/codacy-bot/following{/other_user}","gists_url":"https://api.github.com/users/codacy-bot/gists{/gist_id}","starred_url":"https://api.github.com/users/codacy-bot/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/codacy-bot/subscriptions","organizations_url":"https://api.github.com/users/codacy-bot/orgs","repos_url":"https://api.github.com/users/codacy-bot/repos","events_url":"https://api.github.com/users/codacy-bot/events{/privacy}","received_events_url":"https://api.github.com/users/codacy-bot/received_events","type":"User","site_admin":false},"body":"![Codacy](https://app.codacy.com/assets/images/favicon.png) Issue found: [Too many arguments for logging format string](https://app.codacy.com/app/favio.vazquezp/Optimus/pullRequest?prid=2159064)","created_at":"2018-09-12T17:54:29Z","updated_at":"2018-09-12T17:54:29Z","html_url":"https://github.com/ironmussa/Optimus/pull/326#discussion_r217130153","pull_request_url":"https://api.github.com/repos/ironmussa/Optimus/pulls/326","author_association":"NONE","_links":{"self":{"href":"https://api.github.com/repos/ironmussa/Optimus/pulls/comments/217130153"},"html":{"href":"https://github.com/ironmussa/Optimus/pull/326#discussion_r217130153"},"pull_request":{"href":"https://api.github.com/repos/ironmussa/Optimus/pulls/326"}}},"pull_request":{"url":"https://api.github.com/repos/ironmussa/Optimus/pulls/326","id":213998940,"node_id":"MDExOlB1bGxSZXF1ZXN0MjEzOTk4OTQw","html_url":"https://github.com/ironmussa/Optimus/pull/326","diff_url":"https://github.com/ironmussa/Optimus/pull/326.diff","patch_url":"https://github.com/ironmussa/Optimus/pull/326.patch","issue_url":"https://api.github.com/repos/ironmussa/Optimus/issues/326","number":326,"state":"open","locked":false,"title":"Optimus 2.0.7","user":{"login":"FavioVazquez","id":10162068,"node_id":"MDQ6VXNlcjEwMTYyMDY4","avatar_url":"https://avatars2.githubusercontent.com/u/10162068?v=4","gravatar_id":"","url":"https://api.github.com/users/FavioVazquez","html_url":"https://github.com/FavioVazquez","followers_url":"https://api.github.com/users/FavioVazquez/followers","following_url":"https://api.github.com/users/FavioVazquez/following{/other_user}","gists_url":"https://api.github.com/users/FavioVazquez/gists{/gist_id}","starred_url":"https://api.github.com/users/FavioVazquez/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/FavioVazquez/subscriptions","organizations_url":"https://api.github.com/users/FavioVazquez/orgs","repos_url":"https://api.github.com/users/FavioVazquez/repos","events_url":"https://api.github.com/users/FavioVazquez/events{/privacy}","received_events_url":"https://api.github.com/users/FavioVazquez/received_events","type":"User","site_admin":false},"body":"Improvements for profiler and several functions","created_at":"2018-09-07T17:57:34Z","updated_at":"2018-09-12T17:54:29Z","closed_at":null,"merged_at":null,"merge_commit_sha":"72f36115554ef7e8c15f716abd85003c82774dfd","assignee":{"login":"FavioVazquez","id":10162068,"node_id":"MDQ6VXNlcjEwMTYyMDY4","avatar_url":"https://avatars2.githubusercontent.com/u/10162068?v=4","gravatar_id":"","url":"https://api.github.com/users/FavioVazquez","html_url":"https://github.com/FavioVazquez","followers_url":"https://api.github.com/users/FavioVazquez/followers","following_url":"https://api.github.com/users/FavioVazquez/following{/other_user}","gists_url":"https://api.github.com/users/FavioVazquez/gists{/gist_id}","starred_url":"https://api.github.com/users/FavioVazquez/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/FavioVazquez/subscriptions","organizations_url":"https://api.github.com/users/FavioVazquez/orgs","repos_url":"https://api.github.com/users/FavioVazquez/repos","events_url":"https://api.github.com/users/FavioVazquez/events{/privacy}","received_events_url":"https://api.github.com/users/FavioVazquez/received_events","type":"User","site_admin":false},"assignees":[{"login":"FavioVazquez","id":10162068,"node_id":"MDQ6VXNlcjEwMTYyMDY4","avatar_url":"https://avatars2.githubusercontent.com/u/10162068?v=4","gravatar_id":"","url":"https://api.github.com/users/FavioVazquez","html_url":"https://github.com/FavioVazquez","followers_url":"https://api.github.com/users/FavioVazquez/followers","following_url":"https://api.github.com/users/FavioVazquez/following{/other_user}","gists_url":"https://api.github.com/users/FavioVazquez/gists{/gist_id}","starred_url":"https://api.github.com/users/FavioVazquez/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/FavioVazquez/subscriptions","organizations_url":"https://api.github.com/users/FavioVazquez/orgs","repos_url":"https://api.github.com/users/FavioVazquez/repos","events_url":"https://api.github.com/users/FavioVazquez/events{/privacy}","received_events_url":"https://api.github.com/users/FavioVazquez/received_events","type":"User","site_admin":false}],"requested_reviewers":[{"login":"argenisleon","id":37144,"node_id":"MDQ6VXNlcjM3MTQ0","avatar_url":"https://avatars0.githubusercontent.com/u/37144?v=4","gravatar_id":"","url":"https://api.github.com/users/argenisleon","html_url":"https://github.com/argenisleon","followers_url":"https://api.github.com/users/argenisleon/followers","following_url":"https://api.github.com/users/argenisleon/following{/other_user}","gists_url":"https://api.github.com/users/argenisleon/gists{/gist_id}","starred_url":"https://api.github.com/users/argenisleon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/argenisleon/subscriptions","organizations_url":"https://api.github.com/users/argenisleon/orgs","repos_url":"https://api.github.com/users/argenisleon/repos","events_url":"https://api.github.com/users/argenisleon/events{/privacy}","received_events_url":"https://api.github.com/users/argenisleon/received_events","type":"User","site_admin":false}],"requested_teams":[],"labels":[{"id":647151202,"node_id":"MDU6TGFiZWw2NDcxNTEyMDI=","url":"https://api.github.com/repos/ironmussa/Optimus/labels/documentation","name":"documentation","color":"0052cc","default":false},{"id":645896929,"node_id":"MDU6TGFiZWw2NDU4OTY5Mjk=","url":"https://api.github.com/repos/ironmussa/Optimus/labels/enhancement","name":"enhancement","color":"84b6eb","default":true},{"id":721871911,"node_id":"MDU6TGFiZWw3MjE4NzE5MTE=","url":"https://api.github.com/repos/ironmussa/Optimus/labels/testing","name":"testing","color":"d93f0b","default":false}],"milestone":null,"commits_url":"https://api.github.com/repos/ironmussa/Optimus/pulls/326/commits","review_comments_url":"https://api.github.com/repos/ironmussa/Optimus/pulls/326/comments","review_comment_url":"https://api.github.com/repos/ironmussa/Optimus/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ironmussa/Optimus/issues/326/comments","statuses_url":"https://api.github.com/repos/ironmussa/Optimus/statuses/accc59fa0e5f7f7153775c5882b2bfb63b07423b","head":{"label":"ironmussa:develop","ref":"develop","sha":"accc59fa0e5f7f7153775c5882b2bfb63b07423b","user":{"login":"ironmussa","id":30134402,"node_id":"MDEyOk9yZ2FuaXphdGlvbjMwMTM0NDAy","avatar_url":"https://avatars2.githubusercontent.com/u/30134402?v=4","gravatar_id":"","url":"https://api.github.com/users/ironmussa","html_url":"https://github.com/ironmussa","followers_url":"https://api.github.com/users/ironmussa/followers","following_url":"https://api.github.com/users/ironmussa/following{/other_user}","gists_url":"https://api.github.com/users/ironmussa/gists{/gist_id}","starred_url":"https://api.github.com/users/ironmussa/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ironmussa/subscriptions","organizations_url":"https://api.github.com/users/ironmussa/orgs","repos_url":"https://api.github.com/users/ironmussa/repos","events_url":"https://api.github.com/users/ironmussa/events{/privacy}","received_events_url":"https://api.github.com/users/ironmussa/received_events","type":"Organization","site_admin":false},"repo":{"id":97071697,"node_id":"MDEwOlJlcG9zaXRvcnk5NzA3MTY5Nw==","name":"Optimus","full_name":"ironmussa/Optimus","owner":{"login":"ironmussa","id":30134402,"node_id":"MDEyOk9yZ2FuaXphdGlvbjMwMTM0NDAy","avatar_url":"https://avatars2.githubusercontent.com/u/30134402?v=4","gravatar_id":"","url":"https://api.github.com/users/ironmussa","html_url":"https://github.com/ironmussa","followers_url":"https://api.github.com/users/ironmussa/followers","following_url":"https://api.github.com/users/ironmussa/following{/other_user}","gists_url":"https://api.github.com/users/ironmussa/gists{/gist_id}","starred_url":"https://api.github.com/users/ironmussa/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ironmussa/subscriptions","organizations_url":"https://api.github.com/users/ironmussa/orgs","repos_url":"https://api.github.com/users/ironmussa/repos","events_url":"https://api.github.com/users/ironmussa/events{/privacy}","received_events_url":"https://api.github.com/users/ironmussa/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ironmussa/Optimus","description":":truck: Agile Data Science Workflows made easy with Python and Spark.","fork":false,"url":"https://api.github.com/repos/ironmussa/Optimus","forks_url":"https://api.github.com/repos/ironmussa/Optimus/forks","keys_url":"https://api.github.com/repos/ironmussa/Optimus/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ironmussa/Optimus/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ironmussa/Optimus/teams","hooks_url":"https://api.github.com/repos/ironmussa/Optimus/hooks","issue_events_url":"https://api.github.com/repos/ironmussa/Optimus/issues/events{/number}","events_url":"https://api.github.com/repos/ironmussa/Optimus/events","assignees_url":"https://api.github.com/repos/ironmussa/Optimus/assignees{/user}","branches_url":"https://api.github.com/repos/ironmussa/Optimus/branches{/branch}","tags_url":"https://api.github.com/repos/ironmussa/Optimus/tags","blobs_url":"https://api.github.com/repos/ironmussa/Optimus/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ironmussa/Optimus/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ironmussa/Optimus/git/refs{/sha}","trees_url":"https://api.github.com/repos/ironmussa/Optimus/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ironmussa/Optimus/statuses/{sha}","languages_url":"https://api.github.com/repos/ironmussa/Optimus/languages","stargazers_url":"https://api.github.com/repos/ironmussa/Optimus/stargazers","contributors_url":"https://api.github.com/repos/ironmussa/Optimus/contributors","subscribers_url":"https://api.github.com/repos/ironmussa/Optimus/subscribers","subscription_url":"https://api.github.com/repos/ironmussa/Optimus/subscription","commits_url":"https://api.github.com/repos/ironmussa/Optimus/commits{/sha}","git_commits_url":"https://api.github.com/repos/ironmussa/Optimus/git/commits{/sha}","comments_url":"https://api.github.com/repos/ironmussa/Optimus/comments{/number}","issue_comment_url":"https://api.github.com/repos/ironmussa/Optimus/issues/comments{/number}","contents_url":"https://api.github.com/repos/ironmussa/Optimus/contents/{+path}","compare_url":"https://api.github.com/repos/ironmussa/Optimus/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ironmussa/Optimus/merges","archive_url":"https://api.github.com/repos/ironmussa/Optimus/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ironmussa/Optimus/downloads","issues_url":"https://api.github.com/repos/ironmussa/Optimus/issues{/number}","pulls_url":"https://api.github.com/repos/ironmussa/Optimus/pulls{/number}","milestones_url":"https://api.github.com/repos/ironmussa/Optimus/milestones{/number}","notifications_url":"https://api.github.com/repos/ironmussa/Optimus/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ironmussa/Optimus/labels{/name}","releases_url":"https://api.github.com/repos/ironmussa/Optimus/releases{/id}","deployments_url":"https://api.github.com/repos/ironmussa/Optimus/deployments","created_at":"2017-07-13T02:31:18Z","updated_at":"2018-09-10T15:18:56Z","pushed_at":"2018-09-12T16:46:47Z","git_url":"git://github.com/ironmussa/Optimus.git","ssh_url":"git@github.com:ironmussa/Optimus.git","clone_url":"https://github.com/ironmussa/Optimus.git","svn_url":"https://github.com/ironmussa/Optimus","homepage":"https://hioptimus.com","size":8040,"stargazers_count":339,"watchers_count":339,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":88,"mirror_url":null,"archived":false,"open_issues_count":9,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":88,"open_issues":9,"watchers":339,"default_branch":"master"}},"base":{"label":"ironmussa:master","ref":"master","sha":"c6589099e1d8b43bedd14f31cc95e4f622771836","user":{"login":"ironmussa","id":30134402,"node_id":"MDEyOk9yZ2FuaXphdGlvbjMwMTM0NDAy","avatar_url":"https://avatars2.githubusercontent.com/u/30134402?v=4","gravatar_id":"","url":"https://api.github.com/users/ironmussa","html_url":"https://github.com/ironmussa","followers_url":"https://api.github.com/users/ironmussa/followers","following_url":"https://api.github.com/users/ironmussa/following{/other_user}","gists_url":"https://api.github.com/users/ironmussa/gists{/gist_id}","starred_url":"https://api.github.com/users/ironmussa/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ironmussa/subscriptions","organizations_url":"https://api.github.com/users/ironmussa/orgs","repos_url":"https://api.github.com/users/ironmussa/repos","events_url":"https://api.github.com/users/ironmussa/events{/privacy}","received_events_url":"https://api.github.com/users/ironmussa/received_events","type":"Organization","site_admin":false},"repo":{"id":97071697,"node_id":"MDEwOlJlcG9zaXRvcnk5NzA3MTY5Nw==","name":"Optimus","full_name":"ironmussa/Optimus","owner":{"login":"ironmussa","id":30134402,"node_id":"MDEyOk9yZ2FuaXphdGlvbjMwMTM0NDAy","avatar_url":"https://avatars2.githubusercontent.com/u/30134402?v=4","gravatar_id":"","url":"https://api.github.com/users/ironmussa","html_url":"https://github.com/ironmussa","followers_url":"https://api.github.com/users/ironmussa/followers","following_url":"https://api.github.com/users/ironmussa/following{/other_user}","gists_url":"https://api.github.com/users/ironmussa/gists{/gist_id}","starred_url":"https://api.github.com/users/ironmussa/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ironmussa/subscriptions","organizations_url":"https://api.github.com/users/ironmussa/orgs","repos_url":"https://api.github.com/users/ironmussa/repos","events_url":"https://api.github.com/users/ironmussa/events{/privacy}","received_events_url":"https://api.github.com/users/ironmussa/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ironmussa/Optimus","description":":truck: Agile Data Science Workflows made easy with Python and Spark.","fork":false,"url":"https://api.github.com/repos/ironmussa/Optimus","forks_url":"https://api.github.com/repos/ironmussa/Optimus/forks","keys_url":"https://api.github.com/repos/ironmussa/Optimus/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ironmussa/Optimus/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ironmussa/Optimus/teams","hooks_url":"https://api.github.com/repos/ironmussa/Optimus/hooks","issue_events_url":"https://api.github.com/repos/ironmussa/Optimus/issues/events{/number}","events_url":"https://api.github.com/repos/ironmussa/Optimus/events","assignees_url":"https://api.github.com/repos/ironmussa/Optimus/assignees{/user}","branches_url":"https://api.github.com/repos/ironmussa/Optimus/branches{/branch}","tags_url":"https://api.github.com/repos/ironmussa/Optimus/tags","blobs_url":"https://api.github.com/repos/ironmussa/Optimus/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ironmussa/Optimus/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ironmussa/Optimus/git/refs{/sha}","trees_url":"https://api.github.com/repos/ironmussa/Optimus/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ironmussa/Optimus/statuses/{sha}","languages_url":"https://api.github.com/repos/ironmussa/Optimus/languages","stargazers_url":"https://api.github.com/repos/ironmussa/Optimus/stargazers","contributors_url":"https://api.github.com/repos/ironmussa/Optimus/contributors","subscribers_url":"https://api.github.com/repos/ironmussa/Optimus/subscribers","subscription_url":"https://api.github.com/repos/ironmussa/Optimus/subscription","commits_url":"https://api.github.com/repos/ironmussa/Optimus/commits{/sha}","git_commits_url":"https://api.github.com/repos/ironmussa/Optimus/git/commits{/sha}","comments_url":"https://api.github.com/repos/ironmussa/Optimus/comments{/number}","issue_comment_url":"https://api.github.com/repos/ironmussa/Optimus/issues/comments{/number}","contents_url":"https://api.github.com/repos/ironmussa/Optimus/contents/{+path}","compare_url":"https://api.github.com/repos/ironmussa/Optimus/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ironmussa/Optimus/merges","archive_url":"https://api.github.com/repos/ironmussa/Optimus/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ironmussa/Optimus/downloads","issues_url":"https://api.github.com/repos/ironmussa/Optimus/issues{/number}","pulls_url":"https://api.github.com/repos/ironmussa/Optimus/pulls{/number}","milestones_url":"https://api.github.com/repos/ironmussa/Optimus/milestones{/number}","notifications_url":"https://api.github.com/repos/ironmussa/Optimus/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ironmussa/Optimus/labels{/name}","releases_url":"https://api.github.com/repos/ironmussa/Optimus/releases{/id}","deployments_url":"https://api.github.com/repos/ironmussa/Optimus/deployments","created_at":"2017-07-13T02:31:18Z","updated_at":"2018-09-10T15:18:56Z","pushed_at":"2018-09-12T16:46:47Z","git_url":"git://github.com/ironmussa/Optimus.git","ssh_url":"git@github.com:ironmussa/Optimus.git","clone_url":"https://github.com/ironmussa/Optimus.git","svn_url":"https://github.com/ironmussa/Optimus","homepage":"https://hioptimus.com","size":8040,"stargazers_count":339,"watchers_count":339,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":88,"mirror_url":null,"archived":false,"open_issues_count":9,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":88,"open_issues":9,"watchers":339,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ironmussa/Optimus/pulls/326"},"html":{"href":"https://github.com/ironmussa/Optimus/pull/326"},"issue":{"href":"https://api.github.com/repos/ironmussa/Optimus/issues/326"},"comments":{"href":"https://api.github.com/repos/ironmussa/Optimus/issues/326/comments"},"review_comments":{"href":"https://api.github.com/repos/ironmussa/Optimus/pulls/326/comments"},"review_comment":{"href":"https://api.github.com/repos/ironmussa/Optimus/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ironmussa/Optimus/pulls/326/commits"},"statuses":{"href":"https://api.github.com/repos/ironmussa/Optimus/statuses/accc59fa0e5f7f7153775c5882b2bfb63b07423b"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"codacy-bot"}}