{"action":"created","comment":{"url":"https://api.github.com/repos/grimoirelab/perceval/pulls/comments/167208652","pull_request_review_id":95396037,"id":167208652,"diff_hunk":"@@ -342,3 +343,123 @@ def _count_table_rows(self, table_name):\n             cursor.close()\n \n         return row[0]\n+\n+\n+class ArchiveManager:\n+    \"\"\"Manager for handling archives in Perceval.\n+\n+    This class manages the creation, deletion and access of `Archive`\n+    objects. Archives are stored under `dirpath` directory, using\n+    a random SHA1 for each file. The first byte of the hashcode will\n+    be the name of the subdirectory; the remaining bytes, the archive\n+    name.\n+\n+    :param: dirpath: path where the archives are stored\n+    \"\"\"\n+\n+    STORAGE_EXT = '.sqlite3'\n+\n+    def __init__(self, dirpath):\n+        self.dirpath = dirpath\n+\n+        if not os.path.exists(self.dirpath):\n+            os.makedirs(self.dirpath)\n+\n+    def create_archive(self):\n+        \"\"\"Create a new archive.\n+\n+        The method creates in the filesystem a brand new archive with\n+        a random SHA1 as its name. The first byte of the hashcode will\n+        be the name of the subdirectory; the remaining bytes, the\n+        archive name.\n+\n+        :returns: a new `Archive` object\n+\n+        :raises ArchiveManagerError: when an error occurs creating the\n+            new archive\n+        \"\"\"\n+        hashcode = uuid.uuid4().hex\n+        archive_dir = os.path.join(self.dirpath, hashcode[0:2])\n+        archive_name = hashcode[2:] + self.STORAGE_EXT\n+        archive_path = os.path.join(archive_dir, archive_name)\n+\n+        if not os.path.exists(archive_dir):\n+            os.makedirs(archive_dir)\n+\n+        try:\n+            archive = Archive.create(archive_path)\n+        except ArchiveError as e:\n+            raise ArchiveManagerError(cause=str(e))\n+\n+        return archive\n+\n+    def remove_archive(self, archive_path):\n+        \"\"\"Remove an archive.\n+\n+        This method deletes from the filesystem the archive stored\n+        in `archive_path`.\n+\n+        :param archive_path: path to the archive\n+\n+        :raises ArchiveManangerError: when an error occurs removing the\n+            archive\n+        \"\"\"\n+        try:\n+            Archive(archive_path)\n+        except ArchiveError as e:\n+            raise ArchiveManagerError(cause=str(e))\n+\n+        os.remove(archive_path)\n+\n+    def search(self, origin, backend_name, category, archived_after):\n+        \"\"\"Search archives.\n+\n+        Get the archives which store data based on the given parameters.\n+        These parameters define which the origin was (`origin`), how data\n+        was fetched (`backend_name`) and data type ('category').\n+        Only those archives created on or after `archived_after` will be\n+        returned.\n+\n+        The method returns a list with the file paths to those archives.\n+        The list is sorted by the date of creation of each archive.\n+\n+        :param origin: data origin\n+        :param backend_name: backed used to fetch data\n+        :param category: type of the items fetched by the backend\n+        :param archived_after: get archives created on or after this date\n+\n+        :returns: a list with archive names which match the search criteria\n+        \"\"\"\n+        archives = self._search_archives(origin, backend_name,\n+                                         category, archived_after)\n+        archives = [(fp, date) for fp, date in archives]\n+        archives = [fp for fp, _ in sorted(archives, key=lambda x: x[1])]\n+\n+        return archives\n+\n+    def _search_archives(self, origin, backend_name, category, archived_after):\n+        \"\"\"Search archives using filters.\"\"\"\n+\n+        for archive_path in self._search_files():\n+            try:\n+                archive = Archive(archive_path)\n+            except ArchiveError:\n+                continue\n+\n+            match = archive.origin == origin and \\\n+                archive.backend_name == backend_name and \\\n+                archive.category == category and \\\n+                archive.created_on >= archived_after\n+\n+            if not match:\n+                continue\n+\n+            yield archive_path, archive.created_on\n+\n+    def _search_files(self):\n+        \"\"\"Retrieve the file paths stored under the base path.\"\"\"\n+\n+        for root, _, files in os.walk(self.dirpath):\n+            for filename in files:\n+                location = os.path.join(root, filename)\n+                yield location","path":"perceval/archive.py","position":218,"original_position":218,"commit_id":"ba64fcb18cebdc987aecd0273e7639dd3787e22c","original_commit_id":"ba64fcb18cebdc987aecd0273e7639dd3787e22c","user":{"login":"sduenas","id":833352,"avatar_url":"https://avatars3.githubusercontent.com/u/833352?v=4","gravatar_id":"","url":"https://api.github.com/users/sduenas","html_url":"https://github.com/sduenas","followers_url":"https://api.github.com/users/sduenas/followers","following_url":"https://api.github.com/users/sduenas/following{/other_user}","gists_url":"https://api.github.com/users/sduenas/gists{/gist_id}","starred_url":"https://api.github.com/users/sduenas/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sduenas/subscriptions","organizations_url":"https://api.github.com/users/sduenas/orgs","repos_url":"https://api.github.com/users/sduenas/repos","events_url":"https://api.github.com/users/sduenas/events{/privacy}","received_events_url":"https://api.github.com/users/sduenas/received_events","type":"User","site_admin":false},"body":"Probably that makes sense but as you say we can keep it as is and add other features in a near future.","created_at":"2018-02-09T11:52:44Z","updated_at":"2018-02-09T11:52:45Z","html_url":"https://github.com/grimoirelab/perceval/pull/317#discussion_r167208652","pull_request_url":"https://api.github.com/repos/grimoirelab/perceval/pulls/317","author_association":"OWNER","_links":{"self":{"href":"https://api.github.com/repos/grimoirelab/perceval/pulls/comments/167208652"},"html":{"href":"https://github.com/grimoirelab/perceval/pull/317#discussion_r167208652"},"pull_request":{"href":"https://api.github.com/repos/grimoirelab/perceval/pulls/317"}},"in_reply_to_id":167126476},"pull_request":{"url":"https://api.github.com/repos/grimoirelab/perceval/pulls/317","id":168127577,"html_url":"https://github.com/grimoirelab/perceval/pull/317","diff_url":"https://github.com/grimoirelab/perceval/pull/317.diff","patch_url":"https://github.com/grimoirelab/perceval/pull/317.patch","issue_url":"https://api.github.com/repos/grimoirelab/perceval/issues/317","number":317,"state":"open","locked":false,"title":"Add archive manager","user":{"login":"sduenas","id":833352,"avatar_url":"https://avatars3.githubusercontent.com/u/833352?v=4","gravatar_id":"","url":"https://api.github.com/users/sduenas","html_url":"https://github.com/sduenas","followers_url":"https://api.github.com/users/sduenas/followers","following_url":"https://api.github.com/users/sduenas/following{/other_user}","gists_url":"https://api.github.com/users/sduenas/gists{/gist_id}","starred_url":"https://api.github.com/users/sduenas/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sduenas/subscriptions","organizations_url":"https://api.github.com/users/sduenas/orgs","repos_url":"https://api.github.com/users/sduenas/repos","events_url":"https://api.github.com/users/sduenas/events{/privacy}","received_events_url":"https://api.github.com/users/sduenas/received_events","type":"User","site_admin":false},"body":"The idea behind the archive manager is to handle a set of `Archive` objects in Perceval. This class manages the creation, deletion and access to this objects","created_at":"2018-02-09T00:50:09Z","updated_at":"2018-02-09T11:52:45Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ef11b34eaf907fe3858584a9f3bb55385230228b","assignee":null,"assignees":[],"requested_reviewers":[{"login":"valeriocos","id":6515067,"avatar_url":"https://avatars2.githubusercontent.com/u/6515067?v=4","gravatar_id":"","url":"https://api.github.com/users/valeriocos","html_url":"https://github.com/valeriocos","followers_url":"https://api.github.com/users/valeriocos/followers","following_url":"https://api.github.com/users/valeriocos/following{/other_user}","gists_url":"https://api.github.com/users/valeriocos/gists{/gist_id}","starred_url":"https://api.github.com/users/valeriocos/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/valeriocos/subscriptions","organizations_url":"https://api.github.com/users/valeriocos/orgs","repos_url":"https://api.github.com/users/valeriocos/repos","events_url":"https://api.github.com/users/valeriocos/events{/privacy}","received_events_url":"https://api.github.com/users/valeriocos/received_events","type":"User","site_admin":false}],"requested_teams":[],"milestone":null,"commits_url":"https://api.github.com/repos/grimoirelab/perceval/pulls/317/commits","review_comments_url":"https://api.github.com/repos/grimoirelab/perceval/pulls/317/comments","review_comment_url":"https://api.github.com/repos/grimoirelab/perceval/pulls/comments{/number}","comments_url":"https://api.github.com/repos/grimoirelab/perceval/issues/317/comments","statuses_url":"https://api.github.com/repos/grimoirelab/perceval/statuses/ba64fcb18cebdc987aecd0273e7639dd3787e22c","head":{"label":"sduenas:archive-manager","ref":"archive-manager","sha":"ba64fcb18cebdc987aecd0273e7639dd3787e22c","user":{"login":"sduenas","id":833352,"avatar_url":"https://avatars3.githubusercontent.com/u/833352?v=4","gravatar_id":"","url":"https://api.github.com/users/sduenas","html_url":"https://github.com/sduenas","followers_url":"https://api.github.com/users/sduenas/followers","following_url":"https://api.github.com/users/sduenas/following{/other_user}","gists_url":"https://api.github.com/users/sduenas/gists{/gist_id}","starred_url":"https://api.github.com/users/sduenas/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sduenas/subscriptions","organizations_url":"https://api.github.com/users/sduenas/orgs","repos_url":"https://api.github.com/users/sduenas/repos","events_url":"https://api.github.com/users/sduenas/events{/privacy}","received_events_url":"https://api.github.com/users/sduenas/received_events","type":"User","site_admin":false},"repo":{"id":102633322,"name":"perceval","full_name":"sduenas/perceval","owner":{"login":"sduenas","id":833352,"avatar_url":"https://avatars3.githubusercontent.com/u/833352?v=4","gravatar_id":"","url":"https://api.github.com/users/sduenas","html_url":"https://github.com/sduenas","followers_url":"https://api.github.com/users/sduenas/followers","following_url":"https://api.github.com/users/sduenas/following{/other_user}","gists_url":"https://api.github.com/users/sduenas/gists{/gist_id}","starred_url":"https://api.github.com/users/sduenas/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sduenas/subscriptions","organizations_url":"https://api.github.com/users/sduenas/orgs","repos_url":"https://api.github.com/users/sduenas/repos","events_url":"https://api.github.com/users/sduenas/events{/privacy}","received_events_url":"https://api.github.com/users/sduenas/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/sduenas/perceval","description":"Send Sir Perceval on a quest to retrieve and gather data from software repositories.","fork":true,"url":"https://api.github.com/repos/sduenas/perceval","forks_url":"https://api.github.com/repos/sduenas/perceval/forks","keys_url":"https://api.github.com/repos/sduenas/perceval/keys{/key_id}","collaborators_url":"https://api.github.com/repos/sduenas/perceval/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/sduenas/perceval/teams","hooks_url":"https://api.github.com/repos/sduenas/perceval/hooks","issue_events_url":"https://api.github.com/repos/sduenas/perceval/issues/events{/number}","events_url":"https://api.github.com/repos/sduenas/perceval/events","assignees_url":"https://api.github.com/repos/sduenas/perceval/assignees{/user}","branches_url":"https://api.github.com/repos/sduenas/perceval/branches{/branch}","tags_url":"https://api.github.com/repos/sduenas/perceval/tags","blobs_url":"https://api.github.com/repos/sduenas/perceval/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/sduenas/perceval/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/sduenas/perceval/git/refs{/sha}","trees_url":"https://api.github.com/repos/sduenas/perceval/git/trees{/sha}","statuses_url":"https://api.github.com/repos/sduenas/perceval/statuses/{sha}","languages_url":"https://api.github.com/repos/sduenas/perceval/languages","stargazers_url":"https://api.github.com/repos/sduenas/perceval/stargazers","contributors_url":"https://api.github.com/repos/sduenas/perceval/contributors","subscribers_url":"https://api.github.com/repos/sduenas/perceval/subscribers","subscription_url":"https://api.github.com/repos/sduenas/perceval/subscription","commits_url":"https://api.github.com/repos/sduenas/perceval/commits{/sha}","git_commits_url":"https://api.github.com/repos/sduenas/perceval/git/commits{/sha}","comments_url":"https://api.github.com/repos/sduenas/perceval/comments{/number}","issue_comment_url":"https://api.github.com/repos/sduenas/perceval/issues/comments{/number}","contents_url":"https://api.github.com/repos/sduenas/perceval/contents/{+path}","compare_url":"https://api.github.com/repos/sduenas/perceval/compare/{base}...{head}","merges_url":"https://api.github.com/repos/sduenas/perceval/merges","archive_url":"https://api.github.com/repos/sduenas/perceval/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/sduenas/perceval/downloads","issues_url":"https://api.github.com/repos/sduenas/perceval/issues{/number}","pulls_url":"https://api.github.com/repos/sduenas/perceval/pulls{/number}","milestones_url":"https://api.github.com/repos/sduenas/perceval/milestones{/number}","notifications_url":"https://api.github.com/repos/sduenas/perceval/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/sduenas/perceval/labels{/name}","releases_url":"https://api.github.com/repos/sduenas/perceval/releases{/id}","deployments_url":"https://api.github.com/repos/sduenas/perceval/deployments","created_at":"2017-09-06T16:39:40Z","updated_at":"2017-09-06T16:39:42Z","pushed_at":"2018-02-09T00:46:51Z","git_url":"git://github.com/sduenas/perceval.git","ssh_url":"git@github.com:sduenas/perceval.git","clone_url":"https://github.com/sduenas/perceval.git","svn_url":"https://github.com/sduenas/perceval","homepage":null,"size":1800,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":null,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"grimoirelab:master","ref":"master","sha":"9e6ae8b47d063d1355699d9fa8ebd9544a063989","user":{"login":"grimoirelab","id":16151805,"avatar_url":"https://avatars0.githubusercontent.com/u/16151805?v=4","gravatar_id":"","url":"https://api.github.com/users/grimoirelab","html_url":"https://github.com/grimoirelab","followers_url":"https://api.github.com/users/grimoirelab/followers","following_url":"https://api.github.com/users/grimoirelab/following{/other_user}","gists_url":"https://api.github.com/users/grimoirelab/gists{/gist_id}","starred_url":"https://api.github.com/users/grimoirelab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grimoirelab/subscriptions","organizations_url":"https://api.github.com/users/grimoirelab/orgs","repos_url":"https://api.github.com/users/grimoirelab/repos","events_url":"https://api.github.com/users/grimoirelab/events{/privacy}","received_events_url":"https://api.github.com/users/grimoirelab/received_events","type":"Organization","site_admin":false},"repo":{"id":47415120,"name":"perceval","full_name":"grimoirelab/perceval","owner":{"login":"grimoirelab","id":16151805,"avatar_url":"https://avatars0.githubusercontent.com/u/16151805?v=4","gravatar_id":"","url":"https://api.github.com/users/grimoirelab","html_url":"https://github.com/grimoirelab","followers_url":"https://api.github.com/users/grimoirelab/followers","following_url":"https://api.github.com/users/grimoirelab/following{/other_user}","gists_url":"https://api.github.com/users/grimoirelab/gists{/gist_id}","starred_url":"https://api.github.com/users/grimoirelab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grimoirelab/subscriptions","organizations_url":"https://api.github.com/users/grimoirelab/orgs","repos_url":"https://api.github.com/users/grimoirelab/repos","events_url":"https://api.github.com/users/grimoirelab/events{/privacy}","received_events_url":"https://api.github.com/users/grimoirelab/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/grimoirelab/perceval","description":"Send Sir Perceval on a quest to retrieve and gather data from software repositories.","fork":false,"url":"https://api.github.com/repos/grimoirelab/perceval","forks_url":"https://api.github.com/repos/grimoirelab/perceval/forks","keys_url":"https://api.github.com/repos/grimoirelab/perceval/keys{/key_id}","collaborators_url":"https://api.github.com/repos/grimoirelab/perceval/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/grimoirelab/perceval/teams","hooks_url":"https://api.github.com/repos/grimoirelab/perceval/hooks","issue_events_url":"https://api.github.com/repos/grimoirelab/perceval/issues/events{/number}","events_url":"https://api.github.com/repos/grimoirelab/perceval/events","assignees_url":"https://api.github.com/repos/grimoirelab/perceval/assignees{/user}","branches_url":"https://api.github.com/repos/grimoirelab/perceval/branches{/branch}","tags_url":"https://api.github.com/repos/grimoirelab/perceval/tags","blobs_url":"https://api.github.com/repos/grimoirelab/perceval/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/grimoirelab/perceval/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/grimoirelab/perceval/git/refs{/sha}","trees_url":"https://api.github.com/repos/grimoirelab/perceval/git/trees{/sha}","statuses_url":"https://api.github.com/repos/grimoirelab/perceval/statuses/{sha}","languages_url":"https://api.github.com/repos/grimoirelab/perceval/languages","stargazers_url":"https://api.github.com/repos/grimoirelab/perceval/stargazers","contributors_url":"https://api.github.com/repos/grimoirelab/perceval/contributors","subscribers_url":"https://api.github.com/repos/grimoirelab/perceval/subscribers","subscription_url":"https://api.github.com/repos/grimoirelab/perceval/subscription","commits_url":"https://api.github.com/repos/grimoirelab/perceval/commits{/sha}","git_commits_url":"https://api.github.com/repos/grimoirelab/perceval/git/commits{/sha}","comments_url":"https://api.github.com/repos/grimoirelab/perceval/comments{/number}","issue_comment_url":"https://api.github.com/repos/grimoirelab/perceval/issues/comments{/number}","contents_url":"https://api.github.com/repos/grimoirelab/perceval/contents/{+path}","compare_url":"https://api.github.com/repos/grimoirelab/perceval/compare/{base}...{head}","merges_url":"https://api.github.com/repos/grimoirelab/perceval/merges","archive_url":"https://api.github.com/repos/grimoirelab/perceval/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/grimoirelab/perceval/downloads","issues_url":"https://api.github.com/repos/grimoirelab/perceval/issues{/number}","pulls_url":"https://api.github.com/repos/grimoirelab/perceval/pulls{/number}","milestones_url":"https://api.github.com/repos/grimoirelab/perceval/milestones{/number}","notifications_url":"https://api.github.com/repos/grimoirelab/perceval/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/grimoirelab/perceval/labels{/name}","releases_url":"https://api.github.com/repos/grimoirelab/perceval/releases{/id}","deployments_url":"https://api.github.com/repos/grimoirelab/perceval/deployments","created_at":"2015-12-04T16:20:11Z","updated_at":"2018-02-06T07:55:59Z","pushed_at":"2018-02-09T00:50:09Z","git_url":"git://github.com/grimoirelab/perceval.git","ssh_url":"git@github.com:grimoirelab/perceval.git","clone_url":"https://github.com/grimoirelab/perceval.git","svn_url":"https://github.com/grimoirelab/perceval","homepage":null,"size":1790,"stargazers_count":77,"watchers_count":77,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":36,"mirror_url":null,"archived":false,"open_issues_count":33,"license":{"key":"gpl-3.0","name":"GNU General Public License v3.0","spdx_id":"GPL-3.0","url":"https://api.github.com/licenses/gpl-3.0"},"forks":36,"open_issues":33,"watchers":77,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/grimoirelab/perceval/pulls/317"},"html":{"href":"https://github.com/grimoirelab/perceval/pull/317"},"issue":{"href":"https://api.github.com/repos/grimoirelab/perceval/issues/317"},"comments":{"href":"https://api.github.com/repos/grimoirelab/perceval/issues/317/comments"},"review_comments":{"href":"https://api.github.com/repos/grimoirelab/perceval/pulls/317/comments"},"review_comment":{"href":"https://api.github.com/repos/grimoirelab/perceval/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/grimoirelab/perceval/pulls/317/commits"},"statuses":{"href":"https://api.github.com/repos/grimoirelab/perceval/statuses/ba64fcb18cebdc987aecd0273e7639dd3787e22c"}},"author_association":"OWNER"}}
{"actor":{"display_login":"sduenas"}}