{"action":"created","comment":{"url":"https://api.github.com/repos/Normation/rudder/pulls/comments/60389582","id":60389582,"diff_hunk":"@@ -87,120 +90,183 @@ class AutomaticReportLogger(\n   private class LAAutomaticReportLogger extends LiftActor with Loggable {\n \n     /*\n-     * Last id processed\n-     */\n-    private[this] var lastId   = propertyRepository.getReportLoggerLastId\n-    /*\n      * List of all reports kind processed by the logger\n      */\n-    private[this] var Reportskind = List(Reports.LOG_REPAIRED,Reports.RESULT_ERROR,Reports.RESULT_REPAIRED,Reports.LOG_WARN)\n-\n-    private[this] val reportLineTemplate = \"[%s] N: %s S: [%s] R: %s D: %s T: %s C: [%s] V: [%s] %s\"\n+    private[this] val reportsKind = List(Reports.LOG_REPAIRED, Reports.RESULT_ERROR, Reports.RESULT_REPAIRED, Reports.LOG_WARN)\n \n     override protected def messageHandler = {\n       case StartAutomaticReporting =>\n-\n-\n-      lastId match {\n-        // Report logger was not running before, try to log the last hundred reports and initialize lastId\n-        case Empty =>\n-          logger.warn(\"Automatic report logger has never run, logging latest 100 non compliant reports\")\n-          reportsRepository.getLastHundredErrorReports(Reportskind) match {\n-            case Full(hundredReports) =>\n+        propertyRepository.getReportLoggerLastId match {\n+          // Report logger was not running before, try to log the last hundred reports and initialize lastId\n+          case Empty =>\n+            logger.warn(\"Automatic report logger has never run, logging latest 100 non compliant reports\")\n+            val isSuccess = for {\n+              hundredReports <- reportsRepository.getLastHundredErrorReports(reportsKind)\n+              nodes          <- nodeInfoService.getAll\n+              rules          <- ruleRepository.getAll(true)\n+              directives     <- directiveRepository.getFullDirectiveLibrary()\n+            } yield {\n               val id = hundredReports.headOption match {\n                 // None means this is a new rudder without any reports, don't log anything, current id is 0\n                 case None =>\n                   logger.warn(\"There is no reports to log\")\n                   0\n \n-                  // return last id and report the latest 100 non compliant reports\n+                // return last id and report the latest 100 non compliant reports\n                 case Some(report) =>\n-                  logReports(hundredReports.map(_._1).reverse)\n-                  report._2\n+                  logReports(hundredReports.reverse, nodes, rules.map(r => (r.id, r)).toMap, directives)\n+                  report._1\n               }\n               updateLastId(id)\n+            }\n \n-            case eb:EmptyBox =>\n-              logger.error(\"report logger could not fetch latest 100 non compliant reports, retry on next run\",eb)\n-          }\n-\n-        case Full(last) =>\n-          logger.trace(\"***** get current log id\")\n-\n-          reportsRepository.getHighestId match {\n-            case Full(currentId) if (currentId > last) =>\n-              logger.trace(\"***** log report beetween ids %d and %d\".format(last,currentId))\n-              reportsRepository.getErrorReportsBeetween(last,currentId,Reportskind) match {\n-                case Full(reports) =>\n-                  logReports(reports)\n-                  updateLastId(currentId)\n-                case eb:EmptyBox =>\n-                  logger.error(\"report logger could not fetch latest non compliant reports, try on next run\",eb)\n-              }\n+            isSuccess match {\n+              case eb:EmptyBox =>\n+                logger.error(\"report logger could not fetch latest 100 non compliant reports, retry on next run\",eb)\n+              case Full(x) => //\n+            }\n \n-            case _ =>\n-              logger.trace(\"***** no reports to log\")\n-          }\n \n-        case Failure(message,_,_) =>\n-          logger.error(\"could not fetch last id, don't log anything, wait next run, cause is %s\".format(message))\n-      }\n+          case Full(lastId) =>\n+            logger.trace(\"***** get current highest non-compliant log id\")\n+            val highest = reportsRepository.getHighestId()\n+            logger.trace(s\"***** highest non-compliant log id = ${highest} and last processed id = ${lastId}\")","path":"rudder-core/src/main/scala/com/normation/rudder/batch/AutomaticReportLogger.scala","position":95,"original_position":95,"commit_id":"52def88ed026688125aa42df1ca6553b8687971f","original_commit_id":"52def88ed026688125aa42df1ca6553b8687971f","user":{"login":"VinceMacBuche","id":1238978,"avatar_url":"https://avatars.githubusercontent.com/u/1238978?v=3","gravatar_id":"","url":"https://api.github.com/users/VinceMacBuche","html_url":"https://github.com/VinceMacBuche","followers_url":"https://api.github.com/users/VinceMacBuche/followers","following_url":"https://api.github.com/users/VinceMacBuche/following{/other_user}","gists_url":"https://api.github.com/users/VinceMacBuche/gists{/gist_id}","starred_url":"https://api.github.com/users/VinceMacBuche/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/VinceMacBuche/subscriptions","organizations_url":"https://api.github.com/users/VinceMacBuche/orgs","repos_url":"https://api.github.com/users/VinceMacBuche/repos","events_url":"https://api.github.com/users/VinceMacBuche/events{/privacy}","received_events_url":"https://api.github.com/users/VinceMacBuche/received_events","type":"User","site_admin":false},"body":"Full(id) :( ","created_at":"2016-04-20T11:13:21Z","updated_at":"2016-04-20T11:13:21Z","html_url":"https://github.com/Normation/rudder/pull/1079#discussion_r60389582","pull_request_url":"https://api.github.com/repos/Normation/rudder/pulls/1079","_links":{"self":{"href":"https://api.github.com/repos/Normation/rudder/pulls/comments/60389582"},"html":{"href":"https://github.com/Normation/rudder/pull/1079#discussion_r60389582"},"pull_request":{"href":"https://api.github.com/repos/Normation/rudder/pulls/1079"}}},"pull_request":{"url":"https://api.github.com/repos/Normation/rudder/pulls/1079","id":67029645,"html_url":"https://github.com/Normation/rudder/pull/1079","diff_url":"https://github.com/Normation/rudder/pull/1079.diff","patch_url":"https://github.com/Normation/rudder/pull/1079.patch","issue_url":"https://api.github.com/repos/Normation/rudder/issues/1079","number":1079,"state":"open","locked":false,"title":"Fixes #8195: Correct memory profile and efficiency of error report logger","user":{"login":"fanf","id":44649,"avatar_url":"https://avatars.githubusercontent.com/u/44649?v=3","gravatar_id":"","url":"https://api.github.com/users/fanf","html_url":"https://github.com/fanf","followers_url":"https://api.github.com/users/fanf/followers","following_url":"https://api.github.com/users/fanf/following{/other_user}","gists_url":"https://api.github.com/users/fanf/gists{/gist_id}","starred_url":"https://api.github.com/users/fanf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fanf/subscriptions","organizations_url":"https://api.github.com/users/fanf/orgs","repos_url":"https://api.github.com/users/fanf/repos","events_url":"https://api.github.com/users/fanf/events{/privacy}","received_events_url":"https://api.github.com/users/fanf/received_events","type":"User","site_admin":false},"body":"https://www.rudder-project.org/redmine/issues/8195\r\n\r\nSome information: \r\n- a lot of modification in the ReportsJdbcRepository are cosmetic and just help see what are the differences between queries\r\n- the previous code was doing 3 i/o (get directive, rule, technique) for each log to write, which can be made more efficient by just getting all the rules/directives/techniques. There are already in Rudder memory, that doesn't change the memory profile of the app, but it won a LOT of time (from > 15min to ~10s for writting 250 000 lines)\r\n- the whole process is split in little batch of 10 000 non-compliant-reports. I didn't see much interest in making them bigger, since it could help only for some fixes cost like getting the connection from pool, which is clearly not a big problem. \r\n\r\nOne question remains: the actual code has a kind of local cache for the last non-compliant-report id processed. It was like that, and I didn't changed it, but I'm not sure about the gain compared to just getting it from the DB. And it may lead to inconsistencies if the value in DB is updated from an other process (like an human doing tests, or wanting to produce again the non-compliant-report from a given id). ","created_at":"2016-04-19T14:36:04Z","updated_at":"2016-04-20T11:13:21Z","closed_at":null,"merged_at":null,"merge_commit_sha":"d150b2aa050e49851e51cd32ebacf5c863a0ac29","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/Normation/rudder/pulls/1079/commits","review_comments_url":"https://api.github.com/repos/Normation/rudder/pulls/1079/comments","review_comment_url":"https://api.github.com/repos/Normation/rudder/pulls/comments{/number}","comments_url":"https://api.github.com/repos/Normation/rudder/issues/1079/comments","statuses_url":"https://api.github.com/repos/Normation/rudder/statuses/52def88ed026688125aa42df1ca6553b8687971f","head":{"label":"fanf:bug_8195/correct_memory_profile_and_efficiency_of_error_report_logger","ref":"bug_8195/correct_memory_profile_and_efficiency_of_error_report_logger","sha":"52def88ed026688125aa42df1ca6553b8687971f","user":{"login":"fanf","id":44649,"avatar_url":"https://avatars.githubusercontent.com/u/44649?v=3","gravatar_id":"","url":"https://api.github.com/users/fanf","html_url":"https://github.com/fanf","followers_url":"https://api.github.com/users/fanf/followers","following_url":"https://api.github.com/users/fanf/following{/other_user}","gists_url":"https://api.github.com/users/fanf/gists{/gist_id}","starred_url":"https://api.github.com/users/fanf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fanf/subscriptions","organizations_url":"https://api.github.com/users/fanf/orgs","repos_url":"https://api.github.com/users/fanf/repos","events_url":"https://api.github.com/users/fanf/events{/privacy}","received_events_url":"https://api.github.com/users/fanf/received_events","type":"User","site_admin":false},"repo":{"id":6705258,"name":"rudder","full_name":"fanf/rudder","owner":{"login":"fanf","id":44649,"avatar_url":"https://avatars.githubusercontent.com/u/44649?v=3","gravatar_id":"","url":"https://api.github.com/users/fanf","html_url":"https://github.com/fanf","followers_url":"https://api.github.com/users/fanf/followers","following_url":"https://api.github.com/users/fanf/following{/other_user}","gists_url":"https://api.github.com/users/fanf/gists{/gist_id}","starred_url":"https://api.github.com/users/fanf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fanf/subscriptions","organizations_url":"https://api.github.com/users/fanf/orgs","repos_url":"https://api.github.com/users/fanf/repos","events_url":"https://api.github.com/users/fanf/events{/privacy}","received_events_url":"https://api.github.com/users/fanf/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/fanf/rudder","description":"","fork":true,"url":"https://api.github.com/repos/fanf/rudder","forks_url":"https://api.github.com/repos/fanf/rudder/forks","keys_url":"https://api.github.com/repos/fanf/rudder/keys{/key_id}","collaborators_url":"https://api.github.com/repos/fanf/rudder/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/fanf/rudder/teams","hooks_url":"https://api.github.com/repos/fanf/rudder/hooks","issue_events_url":"https://api.github.com/repos/fanf/rudder/issues/events{/number}","events_url":"https://api.github.com/repos/fanf/rudder/events","assignees_url":"https://api.github.com/repos/fanf/rudder/assignees{/user}","branches_url":"https://api.github.com/repos/fanf/rudder/branches{/branch}","tags_url":"https://api.github.com/repos/fanf/rudder/tags","blobs_url":"https://api.github.com/repos/fanf/rudder/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/fanf/rudder/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/fanf/rudder/git/refs{/sha}","trees_url":"https://api.github.com/repos/fanf/rudder/git/trees{/sha}","statuses_url":"https://api.github.com/repos/fanf/rudder/statuses/{sha}","languages_url":"https://api.github.com/repos/fanf/rudder/languages","stargazers_url":"https://api.github.com/repos/fanf/rudder/stargazers","contributors_url":"https://api.github.com/repos/fanf/rudder/contributors","subscribers_url":"https://api.github.com/repos/fanf/rudder/subscribers","subscription_url":"https://api.github.com/repos/fanf/rudder/subscription","commits_url":"https://api.github.com/repos/fanf/rudder/commits{/sha}","git_commits_url":"https://api.github.com/repos/fanf/rudder/git/commits{/sha}","comments_url":"https://api.github.com/repos/fanf/rudder/comments{/number}","issue_comment_url":"https://api.github.com/repos/fanf/rudder/issues/comments{/number}","contents_url":"https://api.github.com/repos/fanf/rudder/contents/{+path}","compare_url":"https://api.github.com/repos/fanf/rudder/compare/{base}...{head}","merges_url":"https://api.github.com/repos/fanf/rudder/merges","archive_url":"https://api.github.com/repos/fanf/rudder/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/fanf/rudder/downloads","issues_url":"https://api.github.com/repos/fanf/rudder/issues{/number}","pulls_url":"https://api.github.com/repos/fanf/rudder/pulls{/number}","milestones_url":"https://api.github.com/repos/fanf/rudder/milestones{/number}","notifications_url":"https://api.github.com/repos/fanf/rudder/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/fanf/rudder/labels{/name}","releases_url":"https://api.github.com/repos/fanf/rudder/releases{/id}","deployments_url":"https://api.github.com/repos/fanf/rudder/deployments","created_at":"2012-11-15T13:39:53Z","updated_at":"2014-06-06T15:06:41Z","pushed_at":"2016-04-20T10:58:07Z","git_url":"git://github.com/fanf/rudder.git","ssh_url":"git@github.com:fanf/rudder.git","clone_url":"https://github.com/fanf/rudder.git","svn_url":"https://github.com/fanf/rudder","homepage":"http://www.rudder-project.org/","size":18792,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"Normation:branches/rudder/3.0","ref":"branches/rudder/3.0","sha":"81a75b4f6acc5429893d8404b50b35f08743004d","user":{"login":"Normation","id":695105,"avatar_url":"https://avatars.githubusercontent.com/u/695105?v=3","gravatar_id":"","url":"https://api.github.com/users/Normation","html_url":"https://github.com/Normation","followers_url":"https://api.github.com/users/Normation/followers","following_url":"https://api.github.com/users/Normation/following{/other_user}","gists_url":"https://api.github.com/users/Normation/gists{/gist_id}","starred_url":"https://api.github.com/users/Normation/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Normation/subscriptions","organizations_url":"https://api.github.com/users/Normation/orgs","repos_url":"https://api.github.com/users/Normation/repos","events_url":"https://api.github.com/users/Normation/events{/privacy}","received_events_url":"https://api.github.com/users/Normation/received_events","type":"Organization","site_admin":false},"repo":{"id":2528254,"name":"rudder","full_name":"Normation/rudder","owner":{"login":"Normation","id":695105,"avatar_url":"https://avatars.githubusercontent.com/u/695105?v=3","gravatar_id":"","url":"https://api.github.com/users/Normation","html_url":"https://github.com/Normation","followers_url":"https://api.github.com/users/Normation/followers","following_url":"https://api.github.com/users/Normation/following{/other_user}","gists_url":"https://api.github.com/users/Normation/gists{/gist_id}","starred_url":"https://api.github.com/users/Normation/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Normation/subscriptions","organizations_url":"https://api.github.com/users/Normation/orgs","repos_url":"https://api.github.com/users/Normation/repos","events_url":"https://api.github.com/users/Normation/events{/privacy}","received_events_url":"https://api.github.com/users/Normation/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Normation/rudder","description":"","fork":false,"url":"https://api.github.com/repos/Normation/rudder","forks_url":"https://api.github.com/repos/Normation/rudder/forks","keys_url":"https://api.github.com/repos/Normation/rudder/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Normation/rudder/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Normation/rudder/teams","hooks_url":"https://api.github.com/repos/Normation/rudder/hooks","issue_events_url":"https://api.github.com/repos/Normation/rudder/issues/events{/number}","events_url":"https://api.github.com/repos/Normation/rudder/events","assignees_url":"https://api.github.com/repos/Normation/rudder/assignees{/user}","branches_url":"https://api.github.com/repos/Normation/rudder/branches{/branch}","tags_url":"https://api.github.com/repos/Normation/rudder/tags","blobs_url":"https://api.github.com/repos/Normation/rudder/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Normation/rudder/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Normation/rudder/git/refs{/sha}","trees_url":"https://api.github.com/repos/Normation/rudder/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Normation/rudder/statuses/{sha}","languages_url":"https://api.github.com/repos/Normation/rudder/languages","stargazers_url":"https://api.github.com/repos/Normation/rudder/stargazers","contributors_url":"https://api.github.com/repos/Normation/rudder/contributors","subscribers_url":"https://api.github.com/repos/Normation/rudder/subscribers","subscription_url":"https://api.github.com/repos/Normation/rudder/subscription","commits_url":"https://api.github.com/repos/Normation/rudder/commits{/sha}","git_commits_url":"https://api.github.com/repos/Normation/rudder/git/commits{/sha}","comments_url":"https://api.github.com/repos/Normation/rudder/comments{/number}","issue_comment_url":"https://api.github.com/repos/Normation/rudder/issues/comments{/number}","contents_url":"https://api.github.com/repos/Normation/rudder/contents/{+path}","compare_url":"https://api.github.com/repos/Normation/rudder/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Normation/rudder/merges","archive_url":"https://api.github.com/repos/Normation/rudder/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Normation/rudder/downloads","issues_url":"https://api.github.com/repos/Normation/rudder/issues{/number}","pulls_url":"https://api.github.com/repos/Normation/rudder/pulls{/number}","milestones_url":"https://api.github.com/repos/Normation/rudder/milestones{/number}","notifications_url":"https://api.github.com/repos/Normation/rudder/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Normation/rudder/labels{/name}","releases_url":"https://api.github.com/repos/Normation/rudder/releases{/id}","deployments_url":"https://api.github.com/repos/Normation/rudder/deployments","created_at":"2011-10-06T19:27:50Z","updated_at":"2016-02-09T10:15:04Z","pushed_at":"2016-04-20T10:58:08Z","git_url":"git://github.com/Normation/rudder.git","ssh_url":"git@github.com:Normation/rudder.git","clone_url":"https://github.com/Normation/rudder.git","svn_url":"https://github.com/Normation/rudder","homepage":"http://www.rudder-project.org/","size":18178,"stargazers_count":58,"watchers_count":58,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":18,"mirror_url":null,"open_issues_count":6,"forks":18,"open_issues":6,"watchers":58,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/Normation/rudder/pulls/1079"},"html":{"href":"https://github.com/Normation/rudder/pull/1079"},"issue":{"href":"https://api.github.com/repos/Normation/rudder/issues/1079"},"comments":{"href":"https://api.github.com/repos/Normation/rudder/issues/1079/comments"},"review_comments":{"href":"https://api.github.com/repos/Normation/rudder/pulls/1079/comments"},"review_comment":{"href":"https://api.github.com/repos/Normation/rudder/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/Normation/rudder/pulls/1079/commits"},"statuses":{"href":"https://api.github.com/repos/Normation/rudder/statuses/52def88ed026688125aa42df1ca6553b8687971f"}}}}