{"action":"created","comment":{"url":"https://api.github.com/repos/apache/apex-malhar/pulls/comments/92760669","pull_request_review_id":12891262,"id":92760669,"diff_hunk":"@@ -0,0 +1,230 @@\n+/**\n+ * Licensed to the Apache Software Foundation (ASF) under one\n+ * or more contributor license agreements.  See the NOTICE file\n+ * distributed with this work for additional information\n+ * regarding copyright ownership.  The ASF licenses this file\n+ * to you under the Apache License, Version 2.0 (the\n+ * \"License\"); you may not use this file except in compliance\n+ * with the License.  You may obtain a copy of the License at\n+ *\n+ *   http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing,\n+ * software distributed under the License is distributed on an\n+ * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n+ * KIND, either express or implied.  See the License for the\n+ * specific language governing permissions and limitations\n+ * under the License.\n+ */\n+package com.datatorrent.contrib.parser;\n+\n+import org.codehaus.jettison.json.JSONArray;\n+import org.codehaus.jettison.json.JSONException;\n+import org.codehaus.jettison.json.JSONObject;\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+import java.io.IOException;\n+import java.util.LinkedList;\n+import java.util.List;\n+import java.util.regex.Matcher;\n+import java.util.regex.Pattern;\n+\n+/**\n+ * <p>\n+ * This is schema that defines fields and their regex\n+ * The operators use this information to validate the incoming tuples.\n+ * Information from JSON schema is saved in this object and is used by the\n+ * operators\n+ * <p>\n+ * <br>\n+ * <br>\n+ * Example schema <br>\n+ * <br>\n+ * {@code{ \"fields\": [{\"field\": \"host\",\"regex\": \"^([0-9.]+)\"},\n+ * {\"field\": \"userName\",\"regex\": \"(.*?)\"},\n+ * {\"field\": \"request\",\"regex\": \"\\\"((?:[^\\\"]|\\\")+)\\\"\"},\n+ * {\"field\": \"statusCode\",\"regex\": \"(\\\\d{3})\"},\n+ * {\"field\": \"bytes\",\"regex\": \"(\\\\d+|-)\"}]}\n+ */\n+public class LogSchemaDetails\n+{\n+  /**\n+   * This holds the list of field names in the same order as in the schema\n+   */\n+  private List<String> fieldNames = new LinkedList();\n+\n+  private List<Field> fields = new LinkedList();\n+\n+  /**\n+   * This holds regex pattern for the schema\n+   */\n+  private String pattern;\n+\n+  public LogSchemaDetails(String json)\n+  {\n+    try {\n+      initialize(json);\n+      createPattern();\n+    } catch (JSONException | IOException e) {\n+      logger.error(\"{}\", e);\n+      throw new IllegalArgumentException(e);\n+    }\n+  }\n+\n+  /**\n+   * For a given json string, this method sets the field members\n+   * @param json\n+   * @throws JSONException\n+   * @throws IOException\n+   */\n+  private void initialize(String json) throws JSONException, IOException\n+  {\n+    JSONObject jsonObject = new JSONObject(json);\n+    JSONArray fieldArray = jsonObject.getJSONArray(\"fields\");\n+\n+    for(int i = 0; i < fieldArray.length(); i++) {\n+      JSONObject obj = fieldArray.getJSONObject(i);\n+      Field field = new Field(obj.getString(\"field\"), obj.getString(\"regex\"));\n+      this.fields.add(field);\n+      this.fieldNames.add(field.name);\n+    }\n+  }\n+\n+  /**\n+   * creates regex group pattern from the regex given for each field\n+   */\n+  public void createPattern()\n+  {\n+    StringBuffer pattern = new StringBuffer();\n+    for(Field field: this.getFields()) {\n+      pattern.append(field.getRegex()).append(\" \");\n+    }\n+    logger.info(\"Created pattern for parsing the log {}\", pattern.toString().trim());\n+    this.setPattern(pattern.toString().trim());\n+  }\n+\n+  /**\n+   * creates json object by matching the log with given pattern\n+   * @param log\n+   * @return logObject\n+   * @throws Exception\n+   */\n+  public JSONObject createJsonFromLog(String log) throws JSONException\n+  {\n+    Pattern compile = Pattern.compile(this.pattern);\n+    Matcher m = compile.matcher(log);\n+    int count = m.groupCount();\n+    JSONObject logObject = null;\n+    if (m.find()) {\n+      int i = 1;\n+      logObject  = new JSONObject();\n+      for(String field: this.getFieldNames()) {\n+        if (i > count) {\n+          break;\n+        }\n+        logObject.put(field, m.group(i));\n+        i++;\n+      }\n+    }\n+    return logObject;\n+  }\n+\n+  /**\n+   * Get the list of fieldNames mentioned in schema\n+   * @return fieldNames\n+   */\n+  public List<String> getFieldNames()\n+  {\n+    return fieldNames;\n+  }\n+\n+  /**\n+   * Get the list of fields (field, regex) mentioned in schema\n+   * @return fields\n+   */\n+  public List<Field> getFields()\n+  {\n+    return fields;\n+  }\n+\n+  /**\n+   * Get the regex pattern for the schema\n+   * @return pattern\n+   */\n+  public String getPattern()\n+  {\n+    return pattern;\n+  }\n+\n+  /**\n+   * Set the regex pattern for schema\n+   * @param pattern\n+   */\n+  public void setPattern(String pattern)\n+  {\n+    this.pattern = pattern;\n+  }\n+\n+  public class Field\n+  {\n+    /**\n+     * name of the field\n+     */\n+    private String name;\n+    /**\n+     * regular expression for the field\n+     */\n+    private String regex;\n+\n+\n+    public Field(String name, String regex)\n+    {\n+      this.name = name;\n+      this.regex = regex;\n+    }\n+\n+    /**\n+     * Get the name of the field\n+     * @return name\n+     */\n+    public String getName()\n+    {\n+      return name;\n+    }\n+\n+    /**\n+     * Set the name of the field\n+     * @param name\n+     */\n+    public void setName(String name)\n+    {\n+      this.name = name;\n+    }\n+\n+    /**\n+     * Get the regular expression of the field\n+     * @return regex\n+     */\n+    public String getRegex()\n+    {\n+      return regex;\n+    }\n+\n+    /**\n+     * Set the regular expression of the field\n+     * @param regex\n+     */\n+    public void setRegex(String regex)\n+    {\n+      this.regex = regex;\n+    }\n+\n+    @Override\n+    public String toString()\n+    {\n+      return \"Fields [name=\" + name + \", regex=\" + regex +\"]\";\n+    }\n+  }\n+\n+  private static final Logger logger = LoggerFactory.getLogger(LogSchemaDetails.class);\n+}","path":"contrib/src/main/java/com/datatorrent/contrib/parser/LogSchemaDetails.java","position":230,"original_position":230,"commit_id":"25aaba3f05793df76737af7f00026463295ef1fa","original_commit_id":"25aaba3f05793df76737af7f00026463295ef1fa","user":{"login":"tushargosavi","id":183931,"avatar_url":"https://avatars.githubusercontent.com/u/183931?v=3","gravatar_id":"","url":"https://api.github.com/users/tushargosavi","html_url":"https://github.com/tushargosavi","followers_url":"https://api.github.com/users/tushargosavi/followers","following_url":"https://api.github.com/users/tushargosavi/following{/other_user}","gists_url":"https://api.github.com/users/tushargosavi/gists{/gist_id}","starred_url":"https://api.github.com/users/tushargosavi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tushargosavi/subscriptions","organizations_url":"https://api.github.com/users/tushargosavi/orgs","repos_url":"https://api.github.com/users/tushargosavi/repos","events_url":"https://api.github.com/users/tushargosavi/events{/privacy}","received_events_url":"https://api.github.com/users/tushargosavi/received_events","type":"User","site_admin":false},"body":"newline at the end.","created_at":"2016-12-16T06:51:23Z","updated_at":"2016-12-16T06:56:33Z","html_url":"https://github.com/apache/apex-malhar/pull/520#discussion_r92760669","pull_request_url":"https://api.github.com/repos/apache/apex-malhar/pulls/520","_links":{"self":{"href":"https://api.github.com/repos/apache/apex-malhar/pulls/comments/92760669"},"html":{"href":"https://github.com/apache/apex-malhar/pull/520#discussion_r92760669"},"pull_request":{"href":"https://api.github.com/repos/apache/apex-malhar/pulls/520"}}},"pull_request":{"url":"https://api.github.com/repos/apache/apex-malhar/pulls/520","id":96502539,"html_url":"https://github.com/apache/apex-malhar/pull/520","diff_url":"https://github.com/apache/apex-malhar/pull/520.diff","patch_url":"https://github.com/apache/apex-malhar/pull/520.patch","issue_url":"https://api.github.com/repos/apache/apex-malhar/issues/520","number":520,"state":"open","locked":false,"title":"APEXMALHAR-2365-Creation of generic log parser","user":{"login":"jogshraddha","id":5990011,"avatar_url":"https://avatars.githubusercontent.com/u/5990011?v=3","gravatar_id":"","url":"https://api.github.com/users/jogshraddha","html_url":"https://github.com/jogshraddha","followers_url":"https://api.github.com/users/jogshraddha/followers","following_url":"https://api.github.com/users/jogshraddha/following{/other_user}","gists_url":"https://api.github.com/users/jogshraddha/gists{/gist_id}","starred_url":"https://api.github.com/users/jogshraddha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jogshraddha/subscriptions","organizations_url":"https://api.github.com/users/jogshraddha/orgs","repos_url":"https://api.github.com/users/jogshraddha/repos","events_url":"https://api.github.com/users/jogshraddha/events{/privacy}","received_events_url":"https://api.github.com/users/jogshraddha/received_events","type":"User","site_admin":false},"body":"Jira : (https://issues.apache.org/jira/browse/APEXMALHAR-2365)\r\nAPEXMALHAR-2365-Creation of generic log parser\r\n\r\n","created_at":"2016-12-05T10:23:54Z","updated_at":"2016-12-16T06:56:33Z","closed_at":null,"merged_at":null,"merge_commit_sha":"0bd3e36e7073512a5a27ebd0edbb9d776cc9c145","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/apex-malhar/pulls/520/commits","review_comments_url":"https://api.github.com/repos/apache/apex-malhar/pulls/520/comments","review_comment_url":"https://api.github.com/repos/apache/apex-malhar/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/apex-malhar/issues/520/comments","statuses_url":"https://api.github.com/repos/apache/apex-malhar/statuses/25aaba3f05793df76737af7f00026463295ef1fa","head":{"label":"jogshraddha:APEXMALHAR-2365-LogParser","ref":"APEXMALHAR-2365-LogParser","sha":"25aaba3f05793df76737af7f00026463295ef1fa","user":{"login":"jogshraddha","id":5990011,"avatar_url":"https://avatars.githubusercontent.com/u/5990011?v=3","gravatar_id":"","url":"https://api.github.com/users/jogshraddha","html_url":"https://github.com/jogshraddha","followers_url":"https://api.github.com/users/jogshraddha/followers","following_url":"https://api.github.com/users/jogshraddha/following{/other_user}","gists_url":"https://api.github.com/users/jogshraddha/gists{/gist_id}","starred_url":"https://api.github.com/users/jogshraddha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jogshraddha/subscriptions","organizations_url":"https://api.github.com/users/jogshraddha/orgs","repos_url":"https://api.github.com/users/jogshraddha/repos","events_url":"https://api.github.com/users/jogshraddha/events{/privacy}","received_events_url":"https://api.github.com/users/jogshraddha/received_events","type":"User","site_admin":false},"repo":{"id":73695616,"name":"apex-malhar","full_name":"jogshraddha/apex-malhar","owner":{"login":"jogshraddha","id":5990011,"avatar_url":"https://avatars.githubusercontent.com/u/5990011?v=3","gravatar_id":"","url":"https://api.github.com/users/jogshraddha","html_url":"https://github.com/jogshraddha","followers_url":"https://api.github.com/users/jogshraddha/followers","following_url":"https://api.github.com/users/jogshraddha/following{/other_user}","gists_url":"https://api.github.com/users/jogshraddha/gists{/gist_id}","starred_url":"https://api.github.com/users/jogshraddha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jogshraddha/subscriptions","organizations_url":"https://api.github.com/users/jogshraddha/orgs","repos_url":"https://api.github.com/users/jogshraddha/repos","events_url":"https://api.github.com/users/jogshraddha/events{/privacy}","received_events_url":"https://api.github.com/users/jogshraddha/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jogshraddha/apex-malhar","description":"Mirror of Apache Apex malhar","fork":true,"url":"https://api.github.com/repos/jogshraddha/apex-malhar","forks_url":"https://api.github.com/repos/jogshraddha/apex-malhar/forks","keys_url":"https://api.github.com/repos/jogshraddha/apex-malhar/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jogshraddha/apex-malhar/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jogshraddha/apex-malhar/teams","hooks_url":"https://api.github.com/repos/jogshraddha/apex-malhar/hooks","issue_events_url":"https://api.github.com/repos/jogshraddha/apex-malhar/issues/events{/number}","events_url":"https://api.github.com/repos/jogshraddha/apex-malhar/events","assignees_url":"https://api.github.com/repos/jogshraddha/apex-malhar/assignees{/user}","branches_url":"https://api.github.com/repos/jogshraddha/apex-malhar/branches{/branch}","tags_url":"https://api.github.com/repos/jogshraddha/apex-malhar/tags","blobs_url":"https://api.github.com/repos/jogshraddha/apex-malhar/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jogshraddha/apex-malhar/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jogshraddha/apex-malhar/git/refs{/sha}","trees_url":"https://api.github.com/repos/jogshraddha/apex-malhar/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jogshraddha/apex-malhar/statuses/{sha}","languages_url":"https://api.github.com/repos/jogshraddha/apex-malhar/languages","stargazers_url":"https://api.github.com/repos/jogshraddha/apex-malhar/stargazers","contributors_url":"https://api.github.com/repos/jogshraddha/apex-malhar/contributors","subscribers_url":"https://api.github.com/repos/jogshraddha/apex-malhar/subscribers","subscription_url":"https://api.github.com/repos/jogshraddha/apex-malhar/subscription","commits_url":"https://api.github.com/repos/jogshraddha/apex-malhar/commits{/sha}","git_commits_url":"https://api.github.com/repos/jogshraddha/apex-malhar/git/commits{/sha}","comments_url":"https://api.github.com/repos/jogshraddha/apex-malhar/comments{/number}","issue_comment_url":"https://api.github.com/repos/jogshraddha/apex-malhar/issues/comments{/number}","contents_url":"https://api.github.com/repos/jogshraddha/apex-malhar/contents/{+path}","compare_url":"https://api.github.com/repos/jogshraddha/apex-malhar/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jogshraddha/apex-malhar/merges","archive_url":"https://api.github.com/repos/jogshraddha/apex-malhar/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jogshraddha/apex-malhar/downloads","issues_url":"https://api.github.com/repos/jogshraddha/apex-malhar/issues{/number}","pulls_url":"https://api.github.com/repos/jogshraddha/apex-malhar/pulls{/number}","milestones_url":"https://api.github.com/repos/jogshraddha/apex-malhar/milestones{/number}","notifications_url":"https://api.github.com/repos/jogshraddha/apex-malhar/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jogshraddha/apex-malhar/labels{/name}","releases_url":"https://api.github.com/repos/jogshraddha/apex-malhar/releases{/id}","deployments_url":"https://api.github.com/repos/jogshraddha/apex-malhar/deployments","created_at":"2016-11-14T10:53:16Z","updated_at":"2016-11-14T10:53:23Z","pushed_at":"2016-12-13T10:46:11Z","git_url":"git://github.com/jogshraddha/apex-malhar.git","ssh_url":"git@github.com:jogshraddha/apex-malhar.git","clone_url":"https://github.com/jogshraddha/apex-malhar.git","svn_url":"https://github.com/jogshraddha/apex-malhar","homepage":"","size":87920,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"f0a70fbbbd63c08fe90e0411d1dbc3562b82a05f","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":41348334,"name":"apex-malhar","full_name":"apache/apex-malhar","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/apex-malhar","description":"Mirror of Apache Apex malhar","fork":false,"url":"https://api.github.com/repos/apache/apex-malhar","forks_url":"https://api.github.com/repos/apache/apex-malhar/forks","keys_url":"https://api.github.com/repos/apache/apex-malhar/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/apex-malhar/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/apex-malhar/teams","hooks_url":"https://api.github.com/repos/apache/apex-malhar/hooks","issue_events_url":"https://api.github.com/repos/apache/apex-malhar/issues/events{/number}","events_url":"https://api.github.com/repos/apache/apex-malhar/events","assignees_url":"https://api.github.com/repos/apache/apex-malhar/assignees{/user}","branches_url":"https://api.github.com/repos/apache/apex-malhar/branches{/branch}","tags_url":"https://api.github.com/repos/apache/apex-malhar/tags","blobs_url":"https://api.github.com/repos/apache/apex-malhar/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/apex-malhar/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/apex-malhar/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/apex-malhar/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/apex-malhar/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/apex-malhar/languages","stargazers_url":"https://api.github.com/repos/apache/apex-malhar/stargazers","contributors_url":"https://api.github.com/repos/apache/apex-malhar/contributors","subscribers_url":"https://api.github.com/repos/apache/apex-malhar/subscribers","subscription_url":"https://api.github.com/repos/apache/apex-malhar/subscription","commits_url":"https://api.github.com/repos/apache/apex-malhar/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/apex-malhar/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/apex-malhar/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/apex-malhar/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/apex-malhar/contents/{+path}","compare_url":"https://api.github.com/repos/apache/apex-malhar/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/apex-malhar/merges","archive_url":"https://api.github.com/repos/apache/apex-malhar/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/apex-malhar/downloads","issues_url":"https://api.github.com/repos/apache/apex-malhar/issues{/number}","pulls_url":"https://api.github.com/repos/apache/apex-malhar/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/apex-malhar/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/apex-malhar/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/apex-malhar/labels{/name}","releases_url":"https://api.github.com/repos/apache/apex-malhar/releases{/id}","deployments_url":"https://api.github.com/repos/apache/apex-malhar/deployments","created_at":"2015-08-25T07:00:05Z","updated_at":"2016-12-09T01:46:32Z","pushed_at":"2016-12-14T11:11:43Z","git_url":"git://github.com/apache/apex-malhar.git","ssh_url":"git@github.com:apache/apex-malhar.git","clone_url":"https://github.com/apache/apex-malhar.git","svn_url":"https://github.com/apache/apex-malhar","homepage":"","size":99807,"stargazers_count":70,"watchers_count":70,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":111,"mirror_url":"git://git.apache.org/incubator-apex-malhar.git","open_issues_count":36,"forks":111,"open_issues":36,"watchers":70,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/apex-malhar/pulls/520"},"html":{"href":"https://github.com/apache/apex-malhar/pull/520"},"issue":{"href":"https://api.github.com/repos/apache/apex-malhar/issues/520"},"comments":{"href":"https://api.github.com/repos/apache/apex-malhar/issues/520/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/apex-malhar/pulls/520/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/apex-malhar/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/apex-malhar/pulls/520/commits"},"statuses":{"href":"https://api.github.com/repos/apache/apex-malhar/statuses/25aaba3f05793df76737af7f00026463295ef1fa"}}}}
{"actor":{"display_login":"tushargosavi"}}