{"action":"created","comment":{"url":"https://api.github.com/repos/codice/ddf/pulls/comments/52557282","id":52557282,"diff_hunk":"@@ -0,0 +1,194 @@\n+/**\n+ * Copyright (c) Codice Foundation\n+ * <p>\n+ * This is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser\n+ * General Public License as published by the Free Software Foundation, either version 3 of the\n+ * License, or any later version.\n+ * <p>\n+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without\n+ * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU\n+ * Lesser General Public License for more details. A copy of the GNU Lesser General Public License\n+ * is distributed along with this program and can be found at\n+ * <http://www.gnu.org/licenses/lgpl.html>.\n+ */\n+package org.codice.ddf.catalog.migratable.impl;\n+\n+import java.io.File;\n+import java.util.List;\n+import java.util.concurrent.ExecutorService;\n+import java.util.concurrent.TimeUnit;\n+import java.util.concurrent.atomic.AtomicReference;\n+\n+import org.codice.ddf.migration.MigrationException;\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+\n+import com.google.common.util.concurrent.FutureCallback;\n+import com.google.common.util.concurrent.Futures;\n+import com.google.common.util.concurrent.ListenableFuture;\n+import com.google.common.util.concurrent.ListeningExecutorService;\n+import com.google.common.util.concurrent.MoreExecutors;\n+\n+import ddf.catalog.data.Result;\n+\n+/**\n+ * Encapsulates all threading state for the catalog migration process and allows inspection of\n+ * its progress through a public API to make testing easier.\n+ */\n+class MigrationTaskManager {\n+    private static final Logger LOGGER = LoggerFactory.getLogger(MigrationTaskManager.class);\n+\n+    private final ListeningExecutorService taskExecutor;\n+\n+    private final AtomicReference<Throwable> taskFailure;\n+\n+    private final MigrationFileWriter fileWriter;\n+\n+    private final CatalogMigratableConfig catalogConfig;\n+\n+    /**\n+     * The creation of a task manager requires the passing of migratable configurations so that\n+     * task behavior can be explicitly defined (i.e. where to export, how many metacards per file,\n+     * and other useful data).\n+     *\n+     * @param config The set of configuration options to use.\n+     */\n+    public MigrationTaskManager(final CatalogMigratableConfig config,\n+            final MigrationFileWriter fileWriter, final ExecutorService executor) {\n+        this.taskExecutor = MoreExecutors.listeningDecorator(executor);\n+        this.taskFailure = createAtomicReference();\n+        this.catalogConfig = config;\n+        this.fileWriter = fileWriter;\n+    }\n+\n+    AtomicReference<Throwable> createAtomicReference() {\n+        return new AtomicReference<>(null);\n+    }\n+\n+    /**\n+     * Must be called prior to exportFinish and exportMetacardQuery to initilize the\n+     * export directory.\n+     *\n+     * @throws MigrationException thrown if the setup fails\n+     */\n+    public void exportSetup() throws MigrationException {\n+        fileWriter.init(this.catalogConfig.getExportPath());\n+    }\n+\n+    /**\n+     * Must be called after exportSetup and all calls to exportMetacardQuery. This will attempt\n+     * to gracefully shutdown the executor service and handle any exceptions that might have\n+     * occured in the asynchronous threads.\n+     * <p>\n+     * If you are using {@link MigrationTaskManager}; that is, you have constructed an instance,\n+     * you must call this method to gracefully release resources and shutdown.\n+     *\n+     * @throws MigrationException thrown if some of the some of the metacards couldn't be exported\n+     */\n+    public void exportFinish() throws MigrationException {\n+        LOGGER.info(\"Attempting to shutdown catalog export\");\n+        taskExecutor.shutdown();\n+        boolean isGracefulTermination = false;\n+        try {\n+            isGracefulTermination = taskExecutor.awaitTermination(1L, TimeUnit.MINUTES);\n+        } catch (InterruptedException e) {\n+            LOGGER.warn(\"Executor was interrupted during shutdown: \" + e.getMessage(), e);\n+        } finally {\n+            if (!isGracefulTermination) {\n+                taskExecutor.shutdownNow();\n+            }\n+\n+            checkForFailures();\n+        }\n+    }\n+\n+    /**\n+     * Creates a sublist from the given results and a destination file for metacards based on\n+     * the {@link CatalogMigratableConfig} object used by this task manager. An asynchronous\n+     * task is started for exporting the metacards and is deferred to the appropriate instance\n+     * of {@link MigrationFileWriter} for processing.\n+     *\n+     * @param results          The results of a catalog query that need to be exported.\n+     * @param exportGroupCount The group or page number of the query. This value is not monitored\n+     *                         or audited for repeats and is strictly for record keeping by naming\n+     *                         the resulting export files appropriately.\n+     * @throws MigrationException Thrown if one of the writing threads fails or throws an exception\n+     *                            itself.\n+     */\n+    public void exportMetacardQuery(final List<Result> results, long exportGroupCount)\n+            throws MigrationException {\n+        for (int i = 0; i < results.size(); i += catalogConfig.getExportCardsPerFile()) {\n+            final List<Result> fileResults = results.subList(i,\n+                    Math.min((i + catalogConfig.getExportCardsPerFile()), results.size()));\n+            final File exportFile = catalogConfig.getExportPath()\n+                    .resolve(makeFileName(exportGroupCount, i))\n+                    .toFile();\n+\n+            CatalogWriterCallable writerCallable = createWriterCallable(exportFile,\n+                    fileResults,\n+                    fileWriter);\n+\n+            ListenableFuture<Void> task = taskExecutor.submit(writerCallable);\n+\n+            Futures.addCallback(task, createFutureCallback());\n+        }\n+\n+        checkForFailures();\n+\n+        //        List<Future<Void>> finishedTasks = new ArrayList<>();\n+        //\n+        //        for (Future<Void> possibleFinishedTask : taskData) {\n+        //            if (possibleFinishedTask.isDone()) {\n+        //                finishedTasks.add(possibleFinishedTask);\n+        //            }\n+        //        }\n+        //\n+        //        for (Future<Void> finishedTask : finishedTasks) {\n+        //            try {\n+        //                finishedTask.get();\n+        //            } catch (ExecutionException | InterruptedException | RuntimeException e) {\n+        //                LOGGER.error(\"Task in manager encountered an error: {}\", e.getMessage(), e);\n+        //                throw new MigrationException(\"Catalog could not export metacards\", e);\n+        //            } finally {\n+        //                taskData.remove(finishedTask);\n+        //            }\n+        //        }\n+    }\n+\n+    AtomicReference<Throwable> getAtomicReference() {","path":"catalog/core/catalog-core-migratable/src/main/java/org/codice/ddf/catalog/migratable/impl/MigrationTaskManager.java","position":158,"original_position":158,"commit_id":"94be039476e0c9ccf8422e85388afc98b3e0e168","original_commit_id":"94be039476e0c9ccf8422e85388afc98b3e0e168","user":{"login":"rzwiefel","id":9013416,"avatar_url":"https://avatars.githubusercontent.com/u/9013416?v=3","gravatar_id":"","url":"https://api.github.com/users/rzwiefel","html_url":"https://github.com/rzwiefel","followers_url":"https://api.github.com/users/rzwiefel/followers","following_url":"https://api.github.com/users/rzwiefel/following{/other_user}","gists_url":"https://api.github.com/users/rzwiefel/gists{/gist_id}","starred_url":"https://api.github.com/users/rzwiefel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rzwiefel/subscriptions","organizations_url":"https://api.github.com/users/rzwiefel/orgs","repos_url":"https://api.github.com/users/rzwiefel/repos","events_url":"https://api.github.com/users/rzwiefel/events{/privacy}","received_events_url":"https://api.github.com/users/rzwiefel/received_events","type":"User","site_admin":false},"body":"This should be `getTaskFailure`","created_at":"2016-02-11T02:05:02Z","updated_at":"2016-02-11T02:05:02Z","html_url":"https://github.com/codice/ddf/pull/526#discussion_r52557282","pull_request_url":"https://api.github.com/repos/codice/ddf/pulls/526","_links":{"self":{"href":"https://api.github.com/repos/codice/ddf/pulls/comments/52557282"},"html":{"href":"https://github.com/codice/ddf/pull/526#discussion_r52557282"},"pull_request":{"href":"https://api.github.com/repos/codice/ddf/pulls/526"}}},"pull_request":{"url":"https://api.github.com/repos/codice/ddf/pulls/526","id":58635333,"html_url":"https://github.com/codice/ddf/pull/526","diff_url":"https://github.com/codice/ddf/pull/526.diff","patch_url":"https://github.com/codice/ddf/pull/526.patch","issue_url":"https://api.github.com/repos/codice/ddf/issues/526","number":526,"state":"open","locked":false,"title":"DDF-1824: Added ability for catalog (metacard) export","user":{"login":"Lambeaux","id":7215534,"avatar_url":"https://avatars.githubusercontent.com/u/7215534?v=3","gravatar_id":"","url":"https://api.github.com/users/Lambeaux","html_url":"https://github.com/Lambeaux","followers_url":"https://api.github.com/users/Lambeaux/followers","following_url":"https://api.github.com/users/Lambeaux/following{/other_user}","gists_url":"https://api.github.com/users/Lambeaux/gists{/gist_id}","starred_url":"https://api.github.com/users/Lambeaux/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Lambeaux/subscriptions","organizations_url":"https://api.github.com/users/Lambeaux/orgs","repos_url":"https://api.github.com/users/Lambeaux/repos","events_url":"https://api.github.com/users/Lambeaux/events{/privacy}","received_events_url":"https://api.github.com/users/Lambeaux/received_events","type":"User","site_admin":false},"body":"- New files introduced\r\n- All objects unit tested\r\n- Single iTest added\r\n- Some comments need addressing","created_at":"2016-02-08T16:50:15Z","updated_at":"2016-02-11T02:05:03Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a33528ec8ac0b8112e4a4afa5e811d2b4881e704","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/codice/ddf/pulls/526/commits","review_comments_url":"https://api.github.com/repos/codice/ddf/pulls/526/comments","review_comment_url":"https://api.github.com/repos/codice/ddf/pulls/comments{/number}","comments_url":"https://api.github.com/repos/codice/ddf/issues/526/comments","statuses_url":"https://api.github.com/repos/codice/ddf/statuses/94be039476e0c9ccf8422e85388afc98b3e0e168","head":{"label":"Lambeaux:DDF-1824","ref":"DDF-1824","sha":"94be039476e0c9ccf8422e85388afc98b3e0e168","user":{"login":"Lambeaux","id":7215534,"avatar_url":"https://avatars.githubusercontent.com/u/7215534?v=3","gravatar_id":"","url":"https://api.github.com/users/Lambeaux","html_url":"https://github.com/Lambeaux","followers_url":"https://api.github.com/users/Lambeaux/followers","following_url":"https://api.github.com/users/Lambeaux/following{/other_user}","gists_url":"https://api.github.com/users/Lambeaux/gists{/gist_id}","starred_url":"https://api.github.com/users/Lambeaux/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Lambeaux/subscriptions","organizations_url":"https://api.github.com/users/Lambeaux/orgs","repos_url":"https://api.github.com/users/Lambeaux/repos","events_url":"https://api.github.com/users/Lambeaux/events{/privacy}","received_events_url":"https://api.github.com/users/Lambeaux/received_events","type":"User","site_admin":false},"repo":{"id":47577497,"name":"ddf","full_name":"Lambeaux/ddf","owner":{"login":"Lambeaux","id":7215534,"avatar_url":"https://avatars.githubusercontent.com/u/7215534?v=3","gravatar_id":"","url":"https://api.github.com/users/Lambeaux","html_url":"https://github.com/Lambeaux","followers_url":"https://api.github.com/users/Lambeaux/followers","following_url":"https://api.github.com/users/Lambeaux/following{/other_user}","gists_url":"https://api.github.com/users/Lambeaux/gists{/gist_id}","starred_url":"https://api.github.com/users/Lambeaux/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Lambeaux/subscriptions","organizations_url":"https://api.github.com/users/Lambeaux/orgs","repos_url":"https://api.github.com/users/Lambeaux/repos","events_url":"https://api.github.com/users/Lambeaux/events{/privacy}","received_events_url":"https://api.github.com/users/Lambeaux/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/Lambeaux/ddf","description":"The Distributed Data Framework - Distribution, SDK, and Test","fork":true,"url":"https://api.github.com/repos/Lambeaux/ddf","forks_url":"https://api.github.com/repos/Lambeaux/ddf/forks","keys_url":"https://api.github.com/repos/Lambeaux/ddf/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Lambeaux/ddf/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Lambeaux/ddf/teams","hooks_url":"https://api.github.com/repos/Lambeaux/ddf/hooks","issue_events_url":"https://api.github.com/repos/Lambeaux/ddf/issues/events{/number}","events_url":"https://api.github.com/repos/Lambeaux/ddf/events","assignees_url":"https://api.github.com/repos/Lambeaux/ddf/assignees{/user}","branches_url":"https://api.github.com/repos/Lambeaux/ddf/branches{/branch}","tags_url":"https://api.github.com/repos/Lambeaux/ddf/tags","blobs_url":"https://api.github.com/repos/Lambeaux/ddf/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Lambeaux/ddf/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Lambeaux/ddf/git/refs{/sha}","trees_url":"https://api.github.com/repos/Lambeaux/ddf/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Lambeaux/ddf/statuses/{sha}","languages_url":"https://api.github.com/repos/Lambeaux/ddf/languages","stargazers_url":"https://api.github.com/repos/Lambeaux/ddf/stargazers","contributors_url":"https://api.github.com/repos/Lambeaux/ddf/contributors","subscribers_url":"https://api.github.com/repos/Lambeaux/ddf/subscribers","subscription_url":"https://api.github.com/repos/Lambeaux/ddf/subscription","commits_url":"https://api.github.com/repos/Lambeaux/ddf/commits{/sha}","git_commits_url":"https://api.github.com/repos/Lambeaux/ddf/git/commits{/sha}","comments_url":"https://api.github.com/repos/Lambeaux/ddf/comments{/number}","issue_comment_url":"https://api.github.com/repos/Lambeaux/ddf/issues/comments{/number}","contents_url":"https://api.github.com/repos/Lambeaux/ddf/contents/{+path}","compare_url":"https://api.github.com/repos/Lambeaux/ddf/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Lambeaux/ddf/merges","archive_url":"https://api.github.com/repos/Lambeaux/ddf/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Lambeaux/ddf/downloads","issues_url":"https://api.github.com/repos/Lambeaux/ddf/issues{/number}","pulls_url":"https://api.github.com/repos/Lambeaux/ddf/pulls{/number}","milestones_url":"https://api.github.com/repos/Lambeaux/ddf/milestones{/number}","notifications_url":"https://api.github.com/repos/Lambeaux/ddf/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Lambeaux/ddf/labels{/name}","releases_url":"https://api.github.com/repos/Lambeaux/ddf/releases{/id}","deployments_url":"https://api.github.com/repos/Lambeaux/ddf/deployments","created_at":"2015-12-07T20:30:44Z","updated_at":"2016-01-11T17:33:30Z","pushed_at":"2016-02-10T22:33:34Z","git_url":"git://github.com/Lambeaux/ddf.git","ssh_url":"git@github.com:Lambeaux/ddf.git","clone_url":"https://github.com/Lambeaux/ddf.git","svn_url":"https://github.com/Lambeaux/ddf","homepage":"","size":129712,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"codice:master","ref":"master","sha":"5d2bc370c3c6b7511ccb85c75f5083511ac0dc62","user":{"login":"codice","id":2386734,"avatar_url":"https://avatars.githubusercontent.com/u/2386734?v=3","gravatar_id":"","url":"https://api.github.com/users/codice","html_url":"https://github.com/codice","followers_url":"https://api.github.com/users/codice/followers","following_url":"https://api.github.com/users/codice/following{/other_user}","gists_url":"https://api.github.com/users/codice/gists{/gist_id}","starred_url":"https://api.github.com/users/codice/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/codice/subscriptions","organizations_url":"https://api.github.com/users/codice/orgs","repos_url":"https://api.github.com/users/codice/repos","events_url":"https://api.github.com/users/codice/events{/privacy}","received_events_url":"https://api.github.com/users/codice/received_events","type":"Organization","site_admin":false},"repo":{"id":15173563,"name":"ddf","full_name":"codice/ddf","owner":{"login":"codice","id":2386734,"avatar_url":"https://avatars.githubusercontent.com/u/2386734?v=3","gravatar_id":"","url":"https://api.github.com/users/codice","html_url":"https://github.com/codice","followers_url":"https://api.github.com/users/codice/followers","following_url":"https://api.github.com/users/codice/following{/other_user}","gists_url":"https://api.github.com/users/codice/gists{/gist_id}","starred_url":"https://api.github.com/users/codice/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/codice/subscriptions","organizations_url":"https://api.github.com/users/codice/orgs","repos_url":"https://api.github.com/users/codice/repos","events_url":"https://api.github.com/users/codice/events{/privacy}","received_events_url":"https://api.github.com/users/codice/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/codice/ddf","description":"The Distributed Data Framework - Federated Open Geospatial Consortium (OGC) filter powered metadata catalog with SAML Web SSO secured container","fork":false,"url":"https://api.github.com/repos/codice/ddf","forks_url":"https://api.github.com/repos/codice/ddf/forks","keys_url":"https://api.github.com/repos/codice/ddf/keys{/key_id}","collaborators_url":"https://api.github.com/repos/codice/ddf/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/codice/ddf/teams","hooks_url":"https://api.github.com/repos/codice/ddf/hooks","issue_events_url":"https://api.github.com/repos/codice/ddf/issues/events{/number}","events_url":"https://api.github.com/repos/codice/ddf/events","assignees_url":"https://api.github.com/repos/codice/ddf/assignees{/user}","branches_url":"https://api.github.com/repos/codice/ddf/branches{/branch}","tags_url":"https://api.github.com/repos/codice/ddf/tags","blobs_url":"https://api.github.com/repos/codice/ddf/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/codice/ddf/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/codice/ddf/git/refs{/sha}","trees_url":"https://api.github.com/repos/codice/ddf/git/trees{/sha}","statuses_url":"https://api.github.com/repos/codice/ddf/statuses/{sha}","languages_url":"https://api.github.com/repos/codice/ddf/languages","stargazers_url":"https://api.github.com/repos/codice/ddf/stargazers","contributors_url":"https://api.github.com/repos/codice/ddf/contributors","subscribers_url":"https://api.github.com/repos/codice/ddf/subscribers","subscription_url":"https://api.github.com/repos/codice/ddf/subscription","commits_url":"https://api.github.com/repos/codice/ddf/commits{/sha}","git_commits_url":"https://api.github.com/repos/codice/ddf/git/commits{/sha}","comments_url":"https://api.github.com/repos/codice/ddf/comments{/number}","issue_comment_url":"https://api.github.com/repos/codice/ddf/issues/comments{/number}","contents_url":"https://api.github.com/repos/codice/ddf/contents/{+path}","compare_url":"https://api.github.com/repos/codice/ddf/compare/{base}...{head}","merges_url":"https://api.github.com/repos/codice/ddf/merges","archive_url":"https://api.github.com/repos/codice/ddf/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/codice/ddf/downloads","issues_url":"https://api.github.com/repos/codice/ddf/issues{/number}","pulls_url":"https://api.github.com/repos/codice/ddf/pulls{/number}","milestones_url":"https://api.github.com/repos/codice/ddf/milestones{/number}","notifications_url":"https://api.github.com/repos/codice/ddf/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/codice/ddf/labels{/name}","releases_url":"https://api.github.com/repos/codice/ddf/releases{/id}","deployments_url":"https://api.github.com/repos/codice/ddf/deployments","created_at":"2013-12-13T20:38:28Z","updated_at":"2016-02-02T16:34:27Z","pushed_at":"2016-02-11T00:34:00Z","git_url":"git://github.com/codice/ddf.git","ssh_url":"git@github.com:codice/ddf.git","clone_url":"https://github.com/codice/ddf.git","svn_url":"https://github.com/codice/ddf","homepage":"","size":136798,"stargazers_count":32,"watchers_count":32,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":68,"mirror_url":null,"open_issues_count":10,"forks":68,"open_issues":10,"watchers":32,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/codice/ddf/pulls/526"},"html":{"href":"https://github.com/codice/ddf/pull/526"},"issue":{"href":"https://api.github.com/repos/codice/ddf/issues/526"},"comments":{"href":"https://api.github.com/repos/codice/ddf/issues/526/comments"},"review_comments":{"href":"https://api.github.com/repos/codice/ddf/pulls/526/comments"},"review_comment":{"href":"https://api.github.com/repos/codice/ddf/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/codice/ddf/pulls/526/commits"},"statuses":{"href":"https://api.github.com/repos/codice/ddf/statuses/94be039476e0c9ccf8422e85388afc98b3e0e168"}}}}