{"action":"created","comment":{"url":"https://api.github.com/repos/onyxim/otus/pulls/comments/211860705","pull_request_review_id":148368725,"id":211860705,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIxMTg2MDcwNQ==","diff_hunk":"@@ -6,16 +6,329 @@\n #                     '$status $body_bytes_sent \"$http_referer\" '\n #                     '\"$http_user_agent\" \"$http_x_forwarded_for\" \"$http_X_REQUEST_ID\" \"$http_X_RB_USER\" '\n #                     '$request_time';\n+import argparse\n+import gzip\n+import json\n+import logging\n+import os\n+import re\n+from collections import namedtuple\n+from configparser import ConfigParser\n+from datetime import datetime\n+from itertools import chain\n+from statistics import median\n+from string import Template\n \n-config = {\n-    \"REPORT_SIZE\": 1000,\n-    \"REPORT_DIR\": \"./reports\",\n-    \"LOG_DIR\": \"./log\"\n+logging.basicConfig(format='[%(asctime)s] %(levelname).1s %(message)s', level=logging.INFO)\n+\n+DEFAULT_SECTION_NAME = 'SETTINGS'\n+\n+\n+class SettingsKeys:\n+    report_size = \"report_size\"\n+    reports_dir = \"report_dir\"\n+    nginx_logs_dir = \"nginx_log_dir\"\n+    analyzer_log_file = \"analyzer_logs_file\"\n+    parse_success_threshold_percent = \"parse_success_threshold_percent\"\n+\n+\n+INT_VALUES = [SettingsKeys.report_size, SettingsKeys.parse_success_threshold_percent]\n+\n+DEFAULT_SETTINGS = {\n+    SettingsKeys.report_size: 1000,\n+    SettingsKeys.reports_dir: \"./reports\",\n+    SettingsKeys.nginx_logs_dir: \"./log\",\n+    SettingsKeys.analyzer_log_file: None,\n+    SettingsKeys.parse_success_threshold_percent: 100,\n }\n \n+DEFAULT_CONFIG_PATH = \"./config.ini\"\n \n-def main():\n+NGINX_LOG_NAME_RE = r\"\"\"\n+^nginx-access-ui.log-       # обязательный префикс\n+(?P<date>\\d{8})              # дата, ровно 8 цифр подряд\n+(?P<ext>\\.gz|)$              # в конце либо .gz либо пусто\n+\"\"\"\n+regex_name = re.compile(NGINX_LOG_NAME_RE, re.VERBOSE)\n+\n+NGINX_LINE_LOG_RE = r\"\"\"\n+^(?:\\S+)\\s                          # $remote_addr\n+(?:\\S+)\\s{2}                        # $remote_user\n+(?:\\S+)\\s                           # $http_x_real_ip\n+\\[(?:\\S+\\s\\S+)\\]\\s                  # $time_local\n+\n+\"(?:                                # $request\n+\\S+\\s(?P<url>\\S+)\\s\\S+              # Либо получаем нормальный url по шаблону. Именновая capture group.\n+|0                                  # Либо получаем 0, тогда приедет None в groupsdict\n+)\"\\s                                # конец non-capture group\n+                                    \n+(?:\\S+)\\s                           # $status\n+(?:\\S+)\\s                           # $body_bytes_sent\n+\"(?:\\S+)\"\\s                         # $http_referer\n+\"(?:[^\"]+)\"\\s                       # $http_user_agent\n+\"(?:\\S+)\"\\s                         # $http_x_forwarded_for\n+\"(?:\\S+)\"\\s                         # $http_X_REQUEST_ID\n+\"(?:\\S+)\"\\s                         # $http_X_RB_USER\n+(?P<time>\\d+\\.\\d{3})$               # $request_time и еще нужно время, приезжает в виде \"0.867\"\n+\"\"\"\n+regex_line_from_log = re.compile(NGINX_LINE_LOG_RE, re.VERBOSE)\n+\n+REPORT_NAME_TEMPLATE = 'report-{}.html'  # df\n+\n+\n+class MyException(Exception):\n     pass\n \n+\n+class ErrorMessages:\n+    more_than_one_error = 'There are {} more than one default section \"{}\" in the config file.'\n+    wrong_section_name = 'Section \"{}\" not equal to default section name \"{}\"'\n+    no_section = 'There is no a section in the config file or section is empty.'\n+    not_in_allowed_keys = 'There are not allowed keys {} in the config file.'\n+    wrong_date_conversion = 'Date \"{}\" is invalid in filename \"{}\"'\n+    directory_not_exist = 'Directory \"{}\" not exist. Maybe it wrong?'\n+    log_encoding_error = 'There is an encoding error in the logfile \"{}\". File has to be properly utf-8 encoding. ' \\\n+                         'Log file skipped.'\n+    log_file_open_problem = 'Can\\'t open log file \"{}\". Going to the next one.'\n+    line_not_matched = 'Couldn\\'t parse next line:\\n'\n+    threshold_not_reached = 'The quality parsing threshold {} is not reached and currently is {}. Log file skipped.'\n+    probably_duplicate = 'There is already another file for the date \"{}\" has planned to parse. ' \\\n+                         'Probably duplicate gz archive or plain log?'\n+\n+\n+def check_config(config):\n+    sections = config.sections()\n+    if not sections:\n+        raise MyException(ErrorMessages.no_section)\n+    elif len(sections) > 1:\n+        raise MyException(ErrorMessages.more_than_one_error.format(sections, DEFAULT_SECTION_NAME))\n+    elif sections[0] != DEFAULT_SECTION_NAME:\n+        raise MyException(ErrorMessages.wrong_section_name.format(sections[0], DEFAULT_SECTION_NAME))\n+    settings_from_config = dict(config[DEFAULT_SECTION_NAME])\n+    not_allowed = set(settings_from_config.keys()).difference(DEFAULT_SETTINGS.keys())\n+    if not_allowed:\n+        raise MyException(ErrorMessages.not_in_allowed_keys.format(not_allowed))\n+    for v in INT_VALUES:\n+        current_value = settings_from_config[v]\n+        settings_from_config[v] = int(current_value)\n+    return settings_from_config\n+\n+\n+def update_config(config_path):\n+    try:\n+        config = ConfigParser()\n+        with open(config_path) as config_file:\n+            logging.info('Open file {}'.format(config_path))\n+            config.read_file(config_file)\n+            return check_config(config)\n+\n+    except OSError:\n+        raise MyException('Wrong path \"{}\" to config file! The script will be stopped.'.format(config_path))\n+\n+\n+log_params = namedtuple('log_name', ('date_str', 'date', 'report_file_path', 'log_file_path', 'ext'))\n+\n+\n+def check_result_log_name(result, nginx_logs_path, reports_path, existed_reports):\n+    if not result:\n+        return False\n+    re_dict = result.groupdict()\n+    date_str = re_dict['date']\n+    try:\n+        log_date = datetime.strptime(date_str, '%Y%m%d')\n+    except ValueError:\n+        logging.error(ErrorMessages.wrong_date_conversion.format(date_str, result.string))\n+        return False\n+    date_str_report = log_date.strftime('%Y.%m.%d')\n+    report_name = REPORT_NAME_TEMPLATE.format(date_str_report)\n+    # Можно конечно проверять наличие файла через os.path.isfile, но так не будет системного вызова\n+    if report_name in existed_reports:\n+        # Значит уже существует отчет на лог с полученным именем\n+        return False\n+    report_file_path = reports_path + '/' + report_name\n+    return log_params(date_str, log_date, report_file_path, nginx_logs_path + '/' + result.string, re_dict['ext'])\n+\n+\n+def get_logs_names_to_parse(nginx_logs_path, reports_path):\n+    max_date = None\n+    logs_registry = {}\n+    existed_reports = set(os.listdir(reports_path))\n+    for entry in os.listdir(nginx_logs_path):","path":"01_log/log_analyzer.py","position":156,"original_position":156,"commit_id":"c7dc3ba36d048c08afbedf93ee2f65d5a856cc10","original_commit_id":"c7dc3ba36d048c08afbedf93ee2f65d5a856cc10","user":{"login":"VerusK","id":1921825,"node_id":"MDQ6VXNlcjE5MjE4MjU=","avatar_url":"https://avatars1.githubusercontent.com/u/1921825?v=4","gravatar_id":"","url":"https://api.github.com/users/VerusK","html_url":"https://github.com/VerusK","followers_url":"https://api.github.com/users/VerusK/followers","following_url":"https://api.github.com/users/VerusK/following{/other_user}","gists_url":"https://api.github.com/users/VerusK/gists{/gist_id}","starred_url":"https://api.github.com/users/VerusK/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/VerusK/subscriptions","organizations_url":"https://api.github.com/users/VerusK/orgs","repos_url":"https://api.github.com/users/VerusK/repos","events_url":"https://api.github.com/users/VerusK/events{/privacy}","received_events_url":"https://api.github.com/users/VerusK/received_events","type":"User","site_admin":false},"body":"когда в директории будет 1кк файлов, вдруг, то тут все приляжет :D","created_at":"2018-08-22T07:58:01Z","updated_at":"2018-08-22T07:58:01Z","html_url":"https://github.com/onyxim/otus/pull/1#discussion_r211860705","pull_request_url":"https://api.github.com/repos/onyxim/otus/pulls/1","author_association":"NONE","_links":{"self":{"href":"https://api.github.com/repos/onyxim/otus/pulls/comments/211860705"},"html":{"href":"https://github.com/onyxim/otus/pull/1#discussion_r211860705"},"pull_request":{"href":"https://api.github.com/repos/onyxim/otus/pulls/1"}}},"pull_request":{"url":"https://api.github.com/repos/onyxim/otus/pulls/1","id":210046763,"node_id":"MDExOlB1bGxSZXF1ZXN0MjEwMDQ2NzYz","html_url":"https://github.com/onyxim/otus/pull/1","diff_url":"https://github.com/onyxim/otus/pull/1.diff","patch_url":"https://github.com/onyxim/otus/pull/1.patch","issue_url":"https://api.github.com/repos/onyxim/otus/issues/1","number":1,"state":"open","locked":false,"title":"01_log","user":{"login":"onyxim","id":29899080,"node_id":"MDQ6VXNlcjI5ODk5MDgw","avatar_url":"https://avatars3.githubusercontent.com/u/29899080?v=4","gravatar_id":"","url":"https://api.github.com/users/onyxim","html_url":"https://github.com/onyxim","followers_url":"https://api.github.com/users/onyxim/followers","following_url":"https://api.github.com/users/onyxim/following{/other_user}","gists_url":"https://api.github.com/users/onyxim/gists{/gist_id}","starred_url":"https://api.github.com/users/onyxim/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/onyxim/subscriptions","organizations_url":"https://api.github.com/users/onyxim/orgs","repos_url":"https://api.github.com/users/onyxim/repos","events_url":"https://api.github.com/users/onyxim/events{/privacy}","received_events_url":"https://api.github.com/users/onyxim/received_events","type":"User","site_admin":false},"body":"","created_at":"2018-08-22T07:48:38Z","updated_at":"2018-08-22T07:58:01Z","closed_at":null,"merged_at":null,"merge_commit_sha":"03584b72ad6f3732e6dce05a720eaab2e535645b","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/onyxim/otus/pulls/1/commits","review_comments_url":"https://api.github.com/repos/onyxim/otus/pulls/1/comments","review_comment_url":"https://api.github.com/repos/onyxim/otus/pulls/comments{/number}","comments_url":"https://api.github.com/repos/onyxim/otus/issues/1/comments","statuses_url":"https://api.github.com/repos/onyxim/otus/statuses/c7dc3ba36d048c08afbedf93ee2f65d5a856cc10","head":{"label":"onyxim:01_log","ref":"01_log","sha":"c7dc3ba36d048c08afbedf93ee2f65d5a856cc10","user":{"login":"onyxim","id":29899080,"node_id":"MDQ6VXNlcjI5ODk5MDgw","avatar_url":"https://avatars3.githubusercontent.com/u/29899080?v=4","gravatar_id":"","url":"https://api.github.com/users/onyxim","html_url":"https://github.com/onyxim","followers_url":"https://api.github.com/users/onyxim/followers","following_url":"https://api.github.com/users/onyxim/following{/other_user}","gists_url":"https://api.github.com/users/onyxim/gists{/gist_id}","starred_url":"https://api.github.com/users/onyxim/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/onyxim/subscriptions","organizations_url":"https://api.github.com/users/onyxim/orgs","repos_url":"https://api.github.com/users/onyxim/repos","events_url":"https://api.github.com/users/onyxim/events{/privacy}","received_events_url":"https://api.github.com/users/onyxim/received_events","type":"User","site_admin":false},"repo":{"id":145674409,"node_id":"MDEwOlJlcG9zaXRvcnkxNDU2NzQ0MDk=","name":"otus","full_name":"onyxim/otus","owner":{"login":"onyxim","id":29899080,"node_id":"MDQ6VXNlcjI5ODk5MDgw","avatar_url":"https://avatars3.githubusercontent.com/u/29899080?v=4","gravatar_id":"","url":"https://api.github.com/users/onyxim","html_url":"https://github.com/onyxim","followers_url":"https://api.github.com/users/onyxim/followers","following_url":"https://api.github.com/users/onyxim/following{/other_user}","gists_url":"https://api.github.com/users/onyxim/gists{/gist_id}","starred_url":"https://api.github.com/users/onyxim/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/onyxim/subscriptions","organizations_url":"https://api.github.com/users/onyxim/orgs","repos_url":"https://api.github.com/users/onyxim/repos","events_url":"https://api.github.com/users/onyxim/events{/privacy}","received_events_url":"https://api.github.com/users/onyxim/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/onyxim/otus","description":null,"fork":false,"url":"https://api.github.com/repos/onyxim/otus","forks_url":"https://api.github.com/repos/onyxim/otus/forks","keys_url":"https://api.github.com/repos/onyxim/otus/keys{/key_id}","collaborators_url":"https://api.github.com/repos/onyxim/otus/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/onyxim/otus/teams","hooks_url":"https://api.github.com/repos/onyxim/otus/hooks","issue_events_url":"https://api.github.com/repos/onyxim/otus/issues/events{/number}","events_url":"https://api.github.com/repos/onyxim/otus/events","assignees_url":"https://api.github.com/repos/onyxim/otus/assignees{/user}","branches_url":"https://api.github.com/repos/onyxim/otus/branches{/branch}","tags_url":"https://api.github.com/repos/onyxim/otus/tags","blobs_url":"https://api.github.com/repos/onyxim/otus/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/onyxim/otus/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/onyxim/otus/git/refs{/sha}","trees_url":"https://api.github.com/repos/onyxim/otus/git/trees{/sha}","statuses_url":"https://api.github.com/repos/onyxim/otus/statuses/{sha}","languages_url":"https://api.github.com/repos/onyxim/otus/languages","stargazers_url":"https://api.github.com/repos/onyxim/otus/stargazers","contributors_url":"https://api.github.com/repos/onyxim/otus/contributors","subscribers_url":"https://api.github.com/repos/onyxim/otus/subscribers","subscription_url":"https://api.github.com/repos/onyxim/otus/subscription","commits_url":"https://api.github.com/repos/onyxim/otus/commits{/sha}","git_commits_url":"https://api.github.com/repos/onyxim/otus/git/commits{/sha}","comments_url":"https://api.github.com/repos/onyxim/otus/comments{/number}","issue_comment_url":"https://api.github.com/repos/onyxim/otus/issues/comments{/number}","contents_url":"https://api.github.com/repos/onyxim/otus/contents/{+path}","compare_url":"https://api.github.com/repos/onyxim/otus/compare/{base}...{head}","merges_url":"https://api.github.com/repos/onyxim/otus/merges","archive_url":"https://api.github.com/repos/onyxim/otus/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/onyxim/otus/downloads","issues_url":"https://api.github.com/repos/onyxim/otus/issues{/number}","pulls_url":"https://api.github.com/repos/onyxim/otus/pulls{/number}","milestones_url":"https://api.github.com/repos/onyxim/otus/milestones{/number}","notifications_url":"https://api.github.com/repos/onyxim/otus/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/onyxim/otus/labels{/name}","releases_url":"https://api.github.com/repos/onyxim/otus/releases{/id}","deployments_url":"https://api.github.com/repos/onyxim/otus/deployments","created_at":"2018-08-22T07:42:29Z","updated_at":"2018-08-22T07:50:50Z","pushed_at":"2018-08-22T07:50:49Z","git_url":"git://github.com/onyxim/otus.git","ssh_url":"git@github.com:onyxim/otus.git","clone_url":"https://github.com/onyxim/otus.git","svn_url":"https://github.com/onyxim/otus","homepage":null,"size":0,"stargazers_count":0,"watchers_count":0,"language":"HTML","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":null,"forks":0,"open_issues":1,"watchers":0,"default_branch":"01_log"}},"base":{"label":"onyxim:master","ref":"master","sha":"398e0a8cccfb202ba50107bd62381a0fb4312849","user":{"login":"onyxim","id":29899080,"node_id":"MDQ6VXNlcjI5ODk5MDgw","avatar_url":"https://avatars3.githubusercontent.com/u/29899080?v=4","gravatar_id":"","url":"https://api.github.com/users/onyxim","html_url":"https://github.com/onyxim","followers_url":"https://api.github.com/users/onyxim/followers","following_url":"https://api.github.com/users/onyxim/following{/other_user}","gists_url":"https://api.github.com/users/onyxim/gists{/gist_id}","starred_url":"https://api.github.com/users/onyxim/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/onyxim/subscriptions","organizations_url":"https://api.github.com/users/onyxim/orgs","repos_url":"https://api.github.com/users/onyxim/repos","events_url":"https://api.github.com/users/onyxim/events{/privacy}","received_events_url":"https://api.github.com/users/onyxim/received_events","type":"User","site_admin":false},"repo":{"id":145674409,"node_id":"MDEwOlJlcG9zaXRvcnkxNDU2NzQ0MDk=","name":"otus","full_name":"onyxim/otus","owner":{"login":"onyxim","id":29899080,"node_id":"MDQ6VXNlcjI5ODk5MDgw","avatar_url":"https://avatars3.githubusercontent.com/u/29899080?v=4","gravatar_id":"","url":"https://api.github.com/users/onyxim","html_url":"https://github.com/onyxim","followers_url":"https://api.github.com/users/onyxim/followers","following_url":"https://api.github.com/users/onyxim/following{/other_user}","gists_url":"https://api.github.com/users/onyxim/gists{/gist_id}","starred_url":"https://api.github.com/users/onyxim/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/onyxim/subscriptions","organizations_url":"https://api.github.com/users/onyxim/orgs","repos_url":"https://api.github.com/users/onyxim/repos","events_url":"https://api.github.com/users/onyxim/events{/privacy}","received_events_url":"https://api.github.com/users/onyxim/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/onyxim/otus","description":null,"fork":false,"url":"https://api.github.com/repos/onyxim/otus","forks_url":"https://api.github.com/repos/onyxim/otus/forks","keys_url":"https://api.github.com/repos/onyxim/otus/keys{/key_id}","collaborators_url":"https://api.github.com/repos/onyxim/otus/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/onyxim/otus/teams","hooks_url":"https://api.github.com/repos/onyxim/otus/hooks","issue_events_url":"https://api.github.com/repos/onyxim/otus/issues/events{/number}","events_url":"https://api.github.com/repos/onyxim/otus/events","assignees_url":"https://api.github.com/repos/onyxim/otus/assignees{/user}","branches_url":"https://api.github.com/repos/onyxim/otus/branches{/branch}","tags_url":"https://api.github.com/repos/onyxim/otus/tags","blobs_url":"https://api.github.com/repos/onyxim/otus/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/onyxim/otus/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/onyxim/otus/git/refs{/sha}","trees_url":"https://api.github.com/repos/onyxim/otus/git/trees{/sha}","statuses_url":"https://api.github.com/repos/onyxim/otus/statuses/{sha}","languages_url":"https://api.github.com/repos/onyxim/otus/languages","stargazers_url":"https://api.github.com/repos/onyxim/otus/stargazers","contributors_url":"https://api.github.com/repos/onyxim/otus/contributors","subscribers_url":"https://api.github.com/repos/onyxim/otus/subscribers","subscription_url":"https://api.github.com/repos/onyxim/otus/subscription","commits_url":"https://api.github.com/repos/onyxim/otus/commits{/sha}","git_commits_url":"https://api.github.com/repos/onyxim/otus/git/commits{/sha}","comments_url":"https://api.github.com/repos/onyxim/otus/comments{/number}","issue_comment_url":"https://api.github.com/repos/onyxim/otus/issues/comments{/number}","contents_url":"https://api.github.com/repos/onyxim/otus/contents/{+path}","compare_url":"https://api.github.com/repos/onyxim/otus/compare/{base}...{head}","merges_url":"https://api.github.com/repos/onyxim/otus/merges","archive_url":"https://api.github.com/repos/onyxim/otus/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/onyxim/otus/downloads","issues_url":"https://api.github.com/repos/onyxim/otus/issues{/number}","pulls_url":"https://api.github.com/repos/onyxim/otus/pulls{/number}","milestones_url":"https://api.github.com/repos/onyxim/otus/milestones{/number}","notifications_url":"https://api.github.com/repos/onyxim/otus/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/onyxim/otus/labels{/name}","releases_url":"https://api.github.com/repos/onyxim/otus/releases{/id}","deployments_url":"https://api.github.com/repos/onyxim/otus/deployments","created_at":"2018-08-22T07:42:29Z","updated_at":"2018-08-22T07:50:50Z","pushed_at":"2018-08-22T07:50:49Z","git_url":"git://github.com/onyxim/otus.git","ssh_url":"git@github.com:onyxim/otus.git","clone_url":"https://github.com/onyxim/otus.git","svn_url":"https://github.com/onyxim/otus","homepage":null,"size":0,"stargazers_count":0,"watchers_count":0,"language":"HTML","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":null,"forks":0,"open_issues":1,"watchers":0,"default_branch":"01_log"}},"_links":{"self":{"href":"https://api.github.com/repos/onyxim/otus/pulls/1"},"html":{"href":"https://github.com/onyxim/otus/pull/1"},"issue":{"href":"https://api.github.com/repos/onyxim/otus/issues/1"},"comments":{"href":"https://api.github.com/repos/onyxim/otus/issues/1/comments"},"review_comments":{"href":"https://api.github.com/repos/onyxim/otus/pulls/1/comments"},"review_comment":{"href":"https://api.github.com/repos/onyxim/otus/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/onyxim/otus/pulls/1/commits"},"statuses":{"href":"https://api.github.com/repos/onyxim/otus/statuses/c7dc3ba36d048c08afbedf93ee2f65d5a856cc10"}},"author_association":"OWNER"}}
{"actor":{"display_login":"VerusK"}}