{"action":"created","comment":{"url":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments/49482525","id":49482525,"diff_hunk":"@@ -605,40 +630,165 @@ private void maybeFSyncTranslogs() {\n         }\n     }\n \n+    private void maybeRefreshEngine() {\n+        if (indexSettings.getRefreshInterval().millis() > 0) {\n+            for (IndexShard shard : this.shards.values()) {\n+                switch (shard.state()) {\n+                    case CREATED:\n+                    case RECOVERING:\n+                    case CLOSED:\n+                        continue;\n+                    case POST_RECOVERY:\n+                    case STARTED:\n+                    case RELOCATED:\n+                        try {\n+                            shard.refresh(\"schedule\");\n+                        } catch (EngineClosedException | AlreadyClosedException ex) {\n+                            // fine - continue;\n+                        }\n+                        continue;\n+                    default:\n+                        throw new IllegalStateException(\"unknown state: \" + shard.state());\n+                }\n+            }\n+        }\n+    }\n \n-    /**\n-     * FSyncs the translog for all shards of this index in a defined interval.\n-     */\n-    final static class AsyncTranslogFSync implements Runnable {\n-        private final IndexService indexService;\n-        private final ThreadPool threadPool;\n+    static abstract class BaseAsyncTask implements Runnable, Closeable {\n+        protected final IndexService indexService;\n+        protected final ThreadPool threadPool;\n+        private final TimeValue interval;\n+        private ScheduledFuture<?> scheduledFuture;\n+        private final AtomicBoolean closed = new AtomicBoolean(false);\n+        private volatile Exception lastThrownException;\n \n-        AsyncTranslogFSync(IndexService indexService, ThreadPool threadPool) {\n+        BaseAsyncTask(IndexService indexService, ThreadPool threadPool, TimeValue interval) {\n             this.indexService = indexService;\n             this.threadPool = threadPool;\n+            this.interval = interval;\n+            onTaskCompletion();\n         }\n \n-        boolean mustRun() {\n+        boolean mustReschedule() {\n             // don't re-schedule if its closed or if we dont' have a single shard here..., we are done\n-            return (indexService.closed.get() || indexService.shards.isEmpty()) == false;\n+            return (indexService.closed.get() || indexService.shards.isEmpty()) == false\n+                && closed.get() == false && interval.millis() > 0;\n         }\n \n-        void schedule() {\n-            threadPool.schedule(indexService.getIndexSettings().getTranslogSyncInterval(), ThreadPool.Names.SAME, AsyncTranslogFSync.this);\n+        private synchronized void onTaskCompletion() {\n+            if (mustReschedule()) {\n+                indexService.logger.debug(\"scheduling {} every {}\", toString(), interval);\n+                this.scheduledFuture = threadPool.schedule(interval, getThreadPool(), BaseAsyncTask.this);\n+            } else {\n+                indexService.logger.debug(\"scheduled {} disabled\", toString());\n+                this.scheduledFuture = null;\n+            }\n         }\n \n-        @Override\n-        public void run() {\n-            if (mustRun()) {\n-                threadPool.executor(ThreadPool.Names.FLUSH).execute(() -> {\n-                    indexService.maybeFSyncTranslogs();\n-                    if (mustRun()) {\n-                        schedule();\n+        public final void run() {\n+            try {\n+                runInternal();\n+            } catch (Exception ex) {\n+                if (lastThrownException == null || sameException(lastThrownException, ex) == false) {\n+                    // prevent the annoying fact of logging the same stuff all the time with an interval of 1 sec will spam all your logs\n+                    indexService.logger.warn(\"failed to run task {} - supressing re-occuring exceptions unless the exception changes\", ex, toString());\n+                    lastThrownException = ex;\n+                }\n+            } finally {\n+                onTaskCompletion();\n+            }\n+        }\n+\n+        private static boolean sameException(Exception left, Exception right) {\n+            if (left.getClass() == right.getClass()) {\n+                if ((left.getMessage() != null && left.getMessage().equals(right.getMessage()))\n+                    || left.getMessage() == right.getMessage()) {\n+                    StackTraceElement[] stackTraceLeft = left.getStackTrace();\n+                    StackTraceElement[] stackTraceRight = right.getStackTrace();\n+                    if (stackTraceLeft.length == stackTraceRight.length) {\n+                        for (int i = 0; i < stackTraceLeft.length; i++) {\n+                            if (stackTraceLeft[i].equals(stackTraceRight[i]) == false) {\n+                                return false;\n+                            }\n+                        }\n+                        return true;\n                     }\n-                });\n+                }\n+            }\n+            return false;\n+        }\n+\n+        protected abstract void runInternal();\n+\n+        protected String getThreadPool() {\n+            return ThreadPool.Names.SAME;\n+        }\n+\n+        @Override\n+        public void close() {\n+            if (closed.compareAndSet(false, true)) {\n+                FutureUtils.cancel(scheduledFuture);\n+                scheduledFuture = null;\n             }\n         }\n+\n+        TimeValue getInterval() {\n+            return interval;\n+        }\n+\n+        boolean isClosed() {\n+            return this.closed.get();\n+        }\n     }\n \n+    /**\n+     * FSyncs the translog for all shards of this index in a defined interval.\n+     */\n+    final static class AsyncTranslogFSync extends BaseAsyncTask {\n+\n+        AsyncTranslogFSync(IndexService indexService, ThreadPool threadPool) {\n+            super(indexService, threadPool, indexService.getIndexSettings().getTranslogSyncInterval());\n+        }\n+\n+        protected String getThreadPool() {\n+            return ThreadPool.Names.FLUSH;\n+        }\n+        @Override\n+        protected void runInternal() {\n+            indexService.maybeFSyncTranslogs();\n+        }\n+\n+        @Override\n+        public String toString() {\n+            return \"translog_sync\";\n+        }\n+    }\n+\n+    final class RefreshTasks extends BaseAsyncTask {","path":"core/src/main/java/org/elasticsearch/index/IndexService.java","position":244,"original_position":244,"commit_id":"59211927b6d9a41f1c89fa8f27aed1270da7c17d","original_commit_id":"59211927b6d9a41f1c89fa8f27aed1270da7c17d","user":{"login":"mikemccand","id":796508,"avatar_url":"https://avatars.githubusercontent.com/u/796508?v=3","gravatar_id":"","url":"https://api.github.com/users/mikemccand","html_url":"https://github.com/mikemccand","followers_url":"https://api.github.com/users/mikemccand/followers","following_url":"https://api.github.com/users/mikemccand/following{/other_user}","gists_url":"https://api.github.com/users/mikemccand/gists{/gist_id}","starred_url":"https://api.github.com/users/mikemccand/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mikemccand/subscriptions","organizations_url":"https://api.github.com/users/mikemccand/orgs","repos_url":"https://api.github.com/users/mikemccand/repos","events_url":"https://api.github.com/users/mikemccand/events{/privacy}","received_events_url":"https://api.github.com/users/mikemccand/received_events","type":"User","site_admin":false},"body":"Maybe rename to `AysyncRefreshTask`?","created_at":"2016-01-12T17:07:42Z","updated_at":"2016-01-12T17:07:42Z","html_url":"https://github.com/elastic/elasticsearch/pull/15933#discussion_r49482525","pull_request_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/15933","_links":{"self":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments/49482525"},"html":{"href":"https://github.com/elastic/elasticsearch/pull/15933#discussion_r49482525"},"pull_request":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/15933"}}},"pull_request":{"url":"https://api.github.com/repos/elastic/elasticsearch/pulls/15933","id":55784285,"html_url":"https://github.com/elastic/elasticsearch/pull/15933","diff_url":"https://github.com/elastic/elasticsearch/pull/15933.diff","patch_url":"https://github.com/elastic/elasticsearch/pull/15933.patch","issue_url":"https://api.github.com/repos/elastic/elasticsearch/issues/15933","number":15933,"state":"open","locked":false,"title":"Move RefreshTask into IndexService and use since task per index","user":{"login":"s1monw","id":973334,"avatar_url":"https://avatars.githubusercontent.com/u/973334?v=3","gravatar_id":"","url":"https://api.github.com/users/s1monw","html_url":"https://github.com/s1monw","followers_url":"https://api.github.com/users/s1monw/followers","following_url":"https://api.github.com/users/s1monw/following{/other_user}","gists_url":"https://api.github.com/users/s1monw/gists{/gist_id}","starred_url":"https://api.github.com/users/s1monw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s1monw/subscriptions","organizations_url":"https://api.github.com/users/s1monw/orgs","repos_url":"https://api.github.com/users/s1monw/repos","events_url":"https://api.github.com/users/s1monw/events{/privacy}","received_events_url":"https://api.github.com/users/s1monw/received_events","type":"User","site_admin":false},"body":"`refresh_interval` is a per index setting but we interpret and maintain it per shard. This\r\nchange moves the refresh task outside of IndexShard to the IndexService where it logically belongs\r\nand reuses scheduling infrastructure used for translog fsync (async commit).\r\n\r\nThis change will use the same task for all shards of an index while previously we used on thread/task\r\nper shard to refresh. This will also prevent too many concurrent refreshes if there are many indices and\r\nshards allocated on a single node.","created_at":"2016-01-12T16:47:35Z","updated_at":"2016-01-12T17:07:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7d12a22cd2790139ac51dbb7e0dd97e781cfc7fe","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/15933/commits","review_comments_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/15933/comments","review_comment_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments{/number}","comments_url":"https://api.github.com/repos/elastic/elasticsearch/issues/15933/comments","statuses_url":"https://api.github.com/repos/elastic/elasticsearch/statuses/59211927b6d9a41f1c89fa8f27aed1270da7c17d","head":{"label":"s1monw:move_refresh_into_index_service","ref":"move_refresh_into_index_service","sha":"59211927b6d9a41f1c89fa8f27aed1270da7c17d","user":{"login":"s1monw","id":973334,"avatar_url":"https://avatars.githubusercontent.com/u/973334?v=3","gravatar_id":"","url":"https://api.github.com/users/s1monw","html_url":"https://github.com/s1monw","followers_url":"https://api.github.com/users/s1monw/followers","following_url":"https://api.github.com/users/s1monw/following{/other_user}","gists_url":"https://api.github.com/users/s1monw/gists{/gist_id}","starred_url":"https://api.github.com/users/s1monw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s1monw/subscriptions","organizations_url":"https://api.github.com/users/s1monw/orgs","repos_url":"https://api.github.com/users/s1monw/repos","events_url":"https://api.github.com/users/s1monw/events{/privacy}","received_events_url":"https://api.github.com/users/s1monw/received_events","type":"User","site_admin":false},"repo":{"id":4663744,"name":"elasticsearch","full_name":"s1monw/elasticsearch","owner":{"login":"s1monw","id":973334,"avatar_url":"https://avatars.githubusercontent.com/u/973334?v=3","gravatar_id":"","url":"https://api.github.com/users/s1monw","html_url":"https://github.com/s1monw","followers_url":"https://api.github.com/users/s1monw/followers","following_url":"https://api.github.com/users/s1monw/following{/other_user}","gists_url":"https://api.github.com/users/s1monw/gists{/gist_id}","starred_url":"https://api.github.com/users/s1monw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s1monw/subscriptions","organizations_url":"https://api.github.com/users/s1monw/orgs","repos_url":"https://api.github.com/users/s1monw/repos","events_url":"https://api.github.com/users/s1monw/events{/privacy}","received_events_url":"https://api.github.com/users/s1monw/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/s1monw/elasticsearch","description":"Open Source, Distributed, RESTful Search Engine","fork":true,"url":"https://api.github.com/repos/s1monw/elasticsearch","forks_url":"https://api.github.com/repos/s1monw/elasticsearch/forks","keys_url":"https://api.github.com/repos/s1monw/elasticsearch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/s1monw/elasticsearch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/s1monw/elasticsearch/teams","hooks_url":"https://api.github.com/repos/s1monw/elasticsearch/hooks","issue_events_url":"https://api.github.com/repos/s1monw/elasticsearch/issues/events{/number}","events_url":"https://api.github.com/repos/s1monw/elasticsearch/events","assignees_url":"https://api.github.com/repos/s1monw/elasticsearch/assignees{/user}","branches_url":"https://api.github.com/repos/s1monw/elasticsearch/branches{/branch}","tags_url":"https://api.github.com/repos/s1monw/elasticsearch/tags","blobs_url":"https://api.github.com/repos/s1monw/elasticsearch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/s1monw/elasticsearch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/s1monw/elasticsearch/git/refs{/sha}","trees_url":"https://api.github.com/repos/s1monw/elasticsearch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/s1monw/elasticsearch/statuses/{sha}","languages_url":"https://api.github.com/repos/s1monw/elasticsearch/languages","stargazers_url":"https://api.github.com/repos/s1monw/elasticsearch/stargazers","contributors_url":"https://api.github.com/repos/s1monw/elasticsearch/contributors","subscribers_url":"https://api.github.com/repos/s1monw/elasticsearch/subscribers","subscription_url":"https://api.github.com/repos/s1monw/elasticsearch/subscription","commits_url":"https://api.github.com/repos/s1monw/elasticsearch/commits{/sha}","git_commits_url":"https://api.github.com/repos/s1monw/elasticsearch/git/commits{/sha}","comments_url":"https://api.github.com/repos/s1monw/elasticsearch/comments{/number}","issue_comment_url":"https://api.github.com/repos/s1monw/elasticsearch/issues/comments{/number}","contents_url":"https://api.github.com/repos/s1monw/elasticsearch/contents/{+path}","compare_url":"https://api.github.com/repos/s1monw/elasticsearch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/s1monw/elasticsearch/merges","archive_url":"https://api.github.com/repos/s1monw/elasticsearch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/s1monw/elasticsearch/downloads","issues_url":"https://api.github.com/repos/s1monw/elasticsearch/issues{/number}","pulls_url":"https://api.github.com/repos/s1monw/elasticsearch/pulls{/number}","milestones_url":"https://api.github.com/repos/s1monw/elasticsearch/milestones{/number}","notifications_url":"https://api.github.com/repos/s1monw/elasticsearch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/s1monw/elasticsearch/labels{/name}","releases_url":"https://api.github.com/repos/s1monw/elasticsearch/releases{/id}","created_at":"2012-06-14T13:44:13Z","updated_at":"2015-05-07T08:25:29Z","pushed_at":"2016-01-12T16:47:01Z","git_url":"git://github.com/s1monw/elasticsearch.git","ssh_url":"git@github.com:s1monw/elasticsearch.git","clone_url":"https://github.com/s1monw/elasticsearch.git","svn_url":"https://github.com/s1monw/elasticsearch","homepage":"http://elasticsearch.org","size":191433,"stargazers_count":1,"watchers_count":1,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"elastic:master","ref":"master","sha":"7846e5ae99c99466de1bc5972b03ff193f84c052","user":{"login":"elastic","id":6764390,"avatar_url":"https://avatars.githubusercontent.com/u/6764390?v=3","gravatar_id":"","url":"https://api.github.com/users/elastic","html_url":"https://github.com/elastic","followers_url":"https://api.github.com/users/elastic/followers","following_url":"https://api.github.com/users/elastic/following{/other_user}","gists_url":"https://api.github.com/users/elastic/gists{/gist_id}","starred_url":"https://api.github.com/users/elastic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elastic/subscriptions","organizations_url":"https://api.github.com/users/elastic/orgs","repos_url":"https://api.github.com/users/elastic/repos","events_url":"https://api.github.com/users/elastic/events{/privacy}","received_events_url":"https://api.github.com/users/elastic/received_events","type":"Organization","site_admin":false},"repo":{"id":507775,"name":"elasticsearch","full_name":"elastic/elasticsearch","owner":{"login":"elastic","id":6764390,"avatar_url":"https://avatars.githubusercontent.com/u/6764390?v=3","gravatar_id":"","url":"https://api.github.com/users/elastic","html_url":"https://github.com/elastic","followers_url":"https://api.github.com/users/elastic/followers","following_url":"https://api.github.com/users/elastic/following{/other_user}","gists_url":"https://api.github.com/users/elastic/gists{/gist_id}","starred_url":"https://api.github.com/users/elastic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elastic/subscriptions","organizations_url":"https://api.github.com/users/elastic/orgs","repos_url":"https://api.github.com/users/elastic/repos","events_url":"https://api.github.com/users/elastic/events{/privacy}","received_events_url":"https://api.github.com/users/elastic/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/elastic/elasticsearch","description":"Open Source, Distributed, RESTful Search Engine","fork":false,"url":"https://api.github.com/repos/elastic/elasticsearch","forks_url":"https://api.github.com/repos/elastic/elasticsearch/forks","keys_url":"https://api.github.com/repos/elastic/elasticsearch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/elastic/elasticsearch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/elastic/elasticsearch/teams","hooks_url":"https://api.github.com/repos/elastic/elasticsearch/hooks","issue_events_url":"https://api.github.com/repos/elastic/elasticsearch/issues/events{/number}","events_url":"https://api.github.com/repos/elastic/elasticsearch/events","assignees_url":"https://api.github.com/repos/elastic/elasticsearch/assignees{/user}","branches_url":"https://api.github.com/repos/elastic/elasticsearch/branches{/branch}","tags_url":"https://api.github.com/repos/elastic/elasticsearch/tags","blobs_url":"https://api.github.com/repos/elastic/elasticsearch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/elastic/elasticsearch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/elastic/elasticsearch/git/refs{/sha}","trees_url":"https://api.github.com/repos/elastic/elasticsearch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/elastic/elasticsearch/statuses/{sha}","languages_url":"https://api.github.com/repos/elastic/elasticsearch/languages","stargazers_url":"https://api.github.com/repos/elastic/elasticsearch/stargazers","contributors_url":"https://api.github.com/repos/elastic/elasticsearch/contributors","subscribers_url":"https://api.github.com/repos/elastic/elasticsearch/subscribers","subscription_url":"https://api.github.com/repos/elastic/elasticsearch/subscription","commits_url":"https://api.github.com/repos/elastic/elasticsearch/commits{/sha}","git_commits_url":"https://api.github.com/repos/elastic/elasticsearch/git/commits{/sha}","comments_url":"https://api.github.com/repos/elastic/elasticsearch/comments{/number}","issue_comment_url":"https://api.github.com/repos/elastic/elasticsearch/issues/comments{/number}","contents_url":"https://api.github.com/repos/elastic/elasticsearch/contents/{+path}","compare_url":"https://api.github.com/repos/elastic/elasticsearch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/elastic/elasticsearch/merges","archive_url":"https://api.github.com/repos/elastic/elasticsearch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/elastic/elasticsearch/downloads","issues_url":"https://api.github.com/repos/elastic/elasticsearch/issues{/number}","pulls_url":"https://api.github.com/repos/elastic/elasticsearch/pulls{/number}","milestones_url":"https://api.github.com/repos/elastic/elasticsearch/milestones{/number}","notifications_url":"https://api.github.com/repos/elastic/elasticsearch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/elastic/elasticsearch/labels{/name}","releases_url":"https://api.github.com/repos/elastic/elasticsearch/releases{/id}","created_at":"2010-02-08T13:20:56Z","updated_at":"2016-01-12T16:18:46Z","pushed_at":"2016-01-12T16:47:37Z","git_url":"git://github.com/elastic/elasticsearch.git","ssh_url":"git@github.com:elastic/elasticsearch.git","clone_url":"https://github.com/elastic/elasticsearch.git","svn_url":"https://github.com/elastic/elasticsearch","homepage":"https://www.elastic.co/products/elasticsearch","size":236455,"stargazers_count":14346,"watchers_count":14346,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":4787,"mirror_url":null,"open_issues_count":1255,"forks":4787,"open_issues":1255,"watchers":14346,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/15933"},"html":{"href":"https://github.com/elastic/elasticsearch/pull/15933"},"issue":{"href":"https://api.github.com/repos/elastic/elasticsearch/issues/15933"},"comments":{"href":"https://api.github.com/repos/elastic/elasticsearch/issues/15933/comments"},"review_comments":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/15933/comments"},"review_comment":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/15933/commits"},"statuses":{"href":"https://api.github.com/repos/elastic/elasticsearch/statuses/59211927b6d9a41f1c89fa8f27aed1270da7c17d"}}}}