{"action":"created","comment":{"url":"https://api.github.com/repos/deep-compute/procwrap/pulls/comments/94041343","pull_request_review_id":14571641,"id":94041343,"diff_hunk":"@@ -2,39 +2,157 @@\n \n import os\n import sys\n+import time\n+import requests\n import argparse\n import subprocess\n \n-def procwrap(command, pid_file=None):\n-    \"\"\"\n-    executes the process specified in args.command.\n-    returns the returncode of the command.\n-    \"\"\"\n-    if pid_file is not None:\n-        d = os.path.dirname(pid_file)\n-        if not os.path.exists(d):\n-            os.makedirs(d)\n+# what if this script fails somewhere.\n+# say the nsq doesn't connect, etc.\n+# we have to use local logging for this.\n+import logging\n+logging.basicConfig(\n+    stream=sys.stderr,\n+    datefmt=\"%Y-%m-%dT%H-%M-%S\",\n+    format=\"%(asctime)s %(name)s %(levelname)s %(message)s\",\n+)\n+logger = logging.getLogger()\n \n-    process = subprocess.Popen(command)\n-    if pid_file is not None:\n-        with open(pid_file, \"w\") as fp:\n-            fp.write(\"%s\" % process.pid)\n+class ProcessWrapper(object):\n+    _NSQD_HTTP_ADDRESS = \"http://127.0.0.1:4151\"\n+    _NSQ_LOG_TOPIC = \"log\"\n+    _DISABLE_NSQ = False\n+    _PID_FILE = None\n \n-    process.wait()\n-    return process.returncode\n+    def __init__(\n+            self,\n+            command,\n+            pid_file=_PID_FILE,\n+            disable_nsq=_DISABLE_NSQ,\n+            nsq_log_topic=_NSQ_LOG_TOPIC,\n+            nsqd_http_address=_NSQD_HTTP_ADDRESS):\n+\n+        self.command = command\n+        self.pid_file = pid_file\n+        self.disable_nsq = disable_nsq\n+        self.nsq_log_topic = nsq_log_topic\n+\n+        nsqd_http_address = nsqd_http_address.rstrip('/')\n+        if not nsqd_http_address.startswith(\"http://\"):\n+            nsqd_http_address = 'http://%s' % nsqd_http_address\n+\n+        self.nsqd_http_address = nsqd_http_address\n+\n+        self.process = None\n+        self.write_to_nsq = lambda msg: None\n+\n+        if not self.disable_nsq:\n+            self.init_nsq()\n+\n+    def init_nsq(self):\n+        session = requests.Session()\n+        # TODO use urljoin ?\n+        response = None\n+        try:\n+            response = session.get(\"%s/ping\" % self.nsqd_http_address)\n+            if response.status_code != 200:\n+                raise Exception(\"bad response %s\" % response)\n+        except:\n+            logger.exception(\n+                \"if you wish to disable nsqd, use --disable-nsq\"\n+            )\n+            sys.exit(-1)\n+\n+        # TODO nsq is a failure point for our logs. what do we do if nsq is down ?","path":"procwrap/procwrap.py","position":80,"original_position":80,"commit_id":"d22121925e185dd35e05537907466c478491bca4","original_commit_id":"d22121925e185dd35e05537907466c478491bca4","user":{"login":"prashanthellina","id":3989,"avatar_url":"https://avatars.githubusercontent.com/u/3989?v=3","gravatar_id":"","url":"https://api.github.com/users/prashanthellina","html_url":"https://github.com/prashanthellina","followers_url":"https://api.github.com/users/prashanthellina/followers","following_url":"https://api.github.com/users/prashanthellina/following{/other_user}","gists_url":"https://api.github.com/users/prashanthellina/gists{/gist_id}","starred_url":"https://api.github.com/users/prashanthellina/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/prashanthellina/subscriptions","organizations_url":"https://api.github.com/users/prashanthellina/orgs","repos_url":"https://api.github.com/users/prashanthellina/repos","events_url":"https://api.github.com/users/prashanthellina/events{/privacy}","received_events_url":"https://api.github.com/users/prashanthellina/received_events","type":"User","site_admin":false},"body":"@algrebe I think that is necessary if we want to buffer multiple logs lines and then multi publish to NSQ.","created_at":"2016-12-28T15:00:19Z","updated_at":"2016-12-28T15:00:19Z","html_url":"https://github.com/deep-compute/procwrap/pull/5#discussion_r94041343","pull_request_url":"https://api.github.com/repos/deep-compute/procwrap/pulls/5","_links":{"self":{"href":"https://api.github.com/repos/deep-compute/procwrap/pulls/comments/94041343"},"html":{"href":"https://github.com/deep-compute/procwrap/pull/5#discussion_r94041343"},"pull_request":{"href":"https://api.github.com/repos/deep-compute/procwrap/pulls/5"}}},"pull_request":{"url":"https://api.github.com/repos/deep-compute/procwrap/pulls/5","id":99553001,"html_url":"https://github.com/deep-compute/procwrap/pull/5","diff_url":"https://github.com/deep-compute/procwrap/pull/5.diff","patch_url":"https://github.com/deep-compute/procwrap/pull/5.patch","issue_url":"https://api.github.com/repos/deep-compute/procwrap/issues/5","number":5,"state":"open","locked":false,"title":"references #4, procwrap sends stderr json to nsq","user":{"login":"algrebe","id":1938871,"avatar_url":"https://avatars.githubusercontent.com/u/1938871?v=3","gravatar_id":"","url":"https://api.github.com/users/algrebe","html_url":"https://github.com/algrebe","followers_url":"https://api.github.com/users/algrebe/followers","following_url":"https://api.github.com/users/algrebe/following{/other_user}","gists_url":"https://api.github.com/users/algrebe/gists{/gist_id}","starred_url":"https://api.github.com/users/algrebe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/algrebe/subscriptions","organizations_url":"https://api.github.com/users/algrebe/orgs","repos_url":"https://api.github.com/users/algrebe/repos","events_url":"https://api.github.com/users/algrebe/events{/privacy}","received_events_url":"https://api.github.com/users/algrebe/received_events","type":"User","site_admin":false},"body":"- made procwrap into ProcessWrapper\r\nI imagine the complexity will be far more than a single function. With future addition of stats as well.\r\n- ProcessWrapper sends json data on stderr to nsq through http\r\nI tried using the tcp protocol, but felt it was overkill, with tornado or gevent clients. Apparently even bitly uses the http protocol most of the time. I'm hoping the overhead of http won't be that high, given the Session uses the same socket connection. Also, we can send multiple logs in the same nsq packet.\r\n\r\n> What happens if nsq is down ?","created_at":"2016-12-28T14:10:45Z","updated_at":"2016-12-28T15:00:19Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9871ce6924434a88ed224505715c01e9016a608c","assignee":{"login":"algrebe","id":1938871,"avatar_url":"https://avatars.githubusercontent.com/u/1938871?v=3","gravatar_id":"","url":"https://api.github.com/users/algrebe","html_url":"https://github.com/algrebe","followers_url":"https://api.github.com/users/algrebe/followers","following_url":"https://api.github.com/users/algrebe/following{/other_user}","gists_url":"https://api.github.com/users/algrebe/gists{/gist_id}","starred_url":"https://api.github.com/users/algrebe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/algrebe/subscriptions","organizations_url":"https://api.github.com/users/algrebe/orgs","repos_url":"https://api.github.com/users/algrebe/repos","events_url":"https://api.github.com/users/algrebe/events{/privacy}","received_events_url":"https://api.github.com/users/algrebe/received_events","type":"User","site_admin":false},"assignees":[{"login":"algrebe","id":1938871,"avatar_url":"https://avatars.githubusercontent.com/u/1938871?v=3","gravatar_id":"","url":"https://api.github.com/users/algrebe","html_url":"https://github.com/algrebe","followers_url":"https://api.github.com/users/algrebe/followers","following_url":"https://api.github.com/users/algrebe/following{/other_user}","gists_url":"https://api.github.com/users/algrebe/gists{/gist_id}","starred_url":"https://api.github.com/users/algrebe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/algrebe/subscriptions","organizations_url":"https://api.github.com/users/algrebe/orgs","repos_url":"https://api.github.com/users/algrebe/repos","events_url":"https://api.github.com/users/algrebe/events{/privacy}","received_events_url":"https://api.github.com/users/algrebe/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/deep-compute/procwrap/pulls/5/commits","review_comments_url":"https://api.github.com/repos/deep-compute/procwrap/pulls/5/comments","review_comment_url":"https://api.github.com/repos/deep-compute/procwrap/pulls/comments{/number}","comments_url":"https://api.github.com/repos/deep-compute/procwrap/issues/5/comments","statuses_url":"https://api.github.com/repos/deep-compute/procwrap/statuses/d22121925e185dd35e05537907466c478491bca4","head":{"label":"algrebe:logs-to-nsq","ref":"logs-to-nsq","sha":"d22121925e185dd35e05537907466c478491bca4","user":{"login":"algrebe","id":1938871,"avatar_url":"https://avatars.githubusercontent.com/u/1938871?v=3","gravatar_id":"","url":"https://api.github.com/users/algrebe","html_url":"https://github.com/algrebe","followers_url":"https://api.github.com/users/algrebe/followers","following_url":"https://api.github.com/users/algrebe/following{/other_user}","gists_url":"https://api.github.com/users/algrebe/gists{/gist_id}","starred_url":"https://api.github.com/users/algrebe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/algrebe/subscriptions","organizations_url":"https://api.github.com/users/algrebe/orgs","repos_url":"https://api.github.com/users/algrebe/repos","events_url":"https://api.github.com/users/algrebe/events{/privacy}","received_events_url":"https://api.github.com/users/algrebe/received_events","type":"User","site_admin":false},"repo":{"id":77510607,"name":"procwrap","full_name":"algrebe/procwrap","owner":{"login":"algrebe","id":1938871,"avatar_url":"https://avatars.githubusercontent.com/u/1938871?v=3","gravatar_id":"","url":"https://api.github.com/users/algrebe","html_url":"https://github.com/algrebe","followers_url":"https://api.github.com/users/algrebe/followers","following_url":"https://api.github.com/users/algrebe/following{/other_user}","gists_url":"https://api.github.com/users/algrebe/gists{/gist_id}","starred_url":"https://api.github.com/users/algrebe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/algrebe/subscriptions","organizations_url":"https://api.github.com/users/algrebe/orgs","repos_url":"https://api.github.com/users/algrebe/repos","events_url":"https://api.github.com/users/algrebe/events{/privacy}","received_events_url":"https://api.github.com/users/algrebe/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/algrebe/procwrap","description":"wrapper for basescripts' and funcservers'. ","fork":true,"url":"https://api.github.com/repos/algrebe/procwrap","forks_url":"https://api.github.com/repos/algrebe/procwrap/forks","keys_url":"https://api.github.com/repos/algrebe/procwrap/keys{/key_id}","collaborators_url":"https://api.github.com/repos/algrebe/procwrap/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/algrebe/procwrap/teams","hooks_url":"https://api.github.com/repos/algrebe/procwrap/hooks","issue_events_url":"https://api.github.com/repos/algrebe/procwrap/issues/events{/number}","events_url":"https://api.github.com/repos/algrebe/procwrap/events","assignees_url":"https://api.github.com/repos/algrebe/procwrap/assignees{/user}","branches_url":"https://api.github.com/repos/algrebe/procwrap/branches{/branch}","tags_url":"https://api.github.com/repos/algrebe/procwrap/tags","blobs_url":"https://api.github.com/repos/algrebe/procwrap/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/algrebe/procwrap/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/algrebe/procwrap/git/refs{/sha}","trees_url":"https://api.github.com/repos/algrebe/procwrap/git/trees{/sha}","statuses_url":"https://api.github.com/repos/algrebe/procwrap/statuses/{sha}","languages_url":"https://api.github.com/repos/algrebe/procwrap/languages","stargazers_url":"https://api.github.com/repos/algrebe/procwrap/stargazers","contributors_url":"https://api.github.com/repos/algrebe/procwrap/contributors","subscribers_url":"https://api.github.com/repos/algrebe/procwrap/subscribers","subscription_url":"https://api.github.com/repos/algrebe/procwrap/subscription","commits_url":"https://api.github.com/repos/algrebe/procwrap/commits{/sha}","git_commits_url":"https://api.github.com/repos/algrebe/procwrap/git/commits{/sha}","comments_url":"https://api.github.com/repos/algrebe/procwrap/comments{/number}","issue_comment_url":"https://api.github.com/repos/algrebe/procwrap/issues/comments{/number}","contents_url":"https://api.github.com/repos/algrebe/procwrap/contents/{+path}","compare_url":"https://api.github.com/repos/algrebe/procwrap/compare/{base}...{head}","merges_url":"https://api.github.com/repos/algrebe/procwrap/merges","archive_url":"https://api.github.com/repos/algrebe/procwrap/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/algrebe/procwrap/downloads","issues_url":"https://api.github.com/repos/algrebe/procwrap/issues{/number}","pulls_url":"https://api.github.com/repos/algrebe/procwrap/pulls{/number}","milestones_url":"https://api.github.com/repos/algrebe/procwrap/milestones{/number}","notifications_url":"https://api.github.com/repos/algrebe/procwrap/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/algrebe/procwrap/labels{/name}","releases_url":"https://api.github.com/repos/algrebe/procwrap/releases{/id}","deployments_url":"https://api.github.com/repos/algrebe/procwrap/deployments","created_at":"2016-12-28T06:55:56Z","updated_at":"2016-12-28T08:09:38Z","pushed_at":"2016-12-28T14:07:15Z","git_url":"git://github.com/algrebe/procwrap.git","ssh_url":"git@github.com:algrebe/procwrap.git","clone_url":"https://github.com/algrebe/procwrap.git","svn_url":"https://github.com/algrebe/procwrap","homepage":null,"size":3,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"deep-compute:master","ref":"master","sha":"d08350d9ea77a8e797970ac248742f417e5ff411","user":{"login":"deep-compute","id":18276853,"avatar_url":"https://avatars.githubusercontent.com/u/18276853?v=3","gravatar_id":"","url":"https://api.github.com/users/deep-compute","html_url":"https://github.com/deep-compute","followers_url":"https://api.github.com/users/deep-compute/followers","following_url":"https://api.github.com/users/deep-compute/following{/other_user}","gists_url":"https://api.github.com/users/deep-compute/gists{/gist_id}","starred_url":"https://api.github.com/users/deep-compute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/deep-compute/subscriptions","organizations_url":"https://api.github.com/users/deep-compute/orgs","repos_url":"https://api.github.com/users/deep-compute/repos","events_url":"https://api.github.com/users/deep-compute/events{/privacy}","received_events_url":"https://api.github.com/users/deep-compute/received_events","type":"Organization","site_admin":false},"repo":{"id":77510572,"name":"procwrap","full_name":"deep-compute/procwrap","owner":{"login":"deep-compute","id":18276853,"avatar_url":"https://avatars.githubusercontent.com/u/18276853?v=3","gravatar_id":"","url":"https://api.github.com/users/deep-compute","html_url":"https://github.com/deep-compute","followers_url":"https://api.github.com/users/deep-compute/followers","following_url":"https://api.github.com/users/deep-compute/following{/other_user}","gists_url":"https://api.github.com/users/deep-compute/gists{/gist_id}","starred_url":"https://api.github.com/users/deep-compute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/deep-compute/subscriptions","organizations_url":"https://api.github.com/users/deep-compute/orgs","repos_url":"https://api.github.com/users/deep-compute/repos","events_url":"https://api.github.com/users/deep-compute/events{/privacy}","received_events_url":"https://api.github.com/users/deep-compute/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/deep-compute/procwrap","description":"wrapper for basescripts' and funcservers'. ","fork":false,"url":"https://api.github.com/repos/deep-compute/procwrap","forks_url":"https://api.github.com/repos/deep-compute/procwrap/forks","keys_url":"https://api.github.com/repos/deep-compute/procwrap/keys{/key_id}","collaborators_url":"https://api.github.com/repos/deep-compute/procwrap/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/deep-compute/procwrap/teams","hooks_url":"https://api.github.com/repos/deep-compute/procwrap/hooks","issue_events_url":"https://api.github.com/repos/deep-compute/procwrap/issues/events{/number}","events_url":"https://api.github.com/repos/deep-compute/procwrap/events","assignees_url":"https://api.github.com/repos/deep-compute/procwrap/assignees{/user}","branches_url":"https://api.github.com/repos/deep-compute/procwrap/branches{/branch}","tags_url":"https://api.github.com/repos/deep-compute/procwrap/tags","blobs_url":"https://api.github.com/repos/deep-compute/procwrap/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/deep-compute/procwrap/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/deep-compute/procwrap/git/refs{/sha}","trees_url":"https://api.github.com/repos/deep-compute/procwrap/git/trees{/sha}","statuses_url":"https://api.github.com/repos/deep-compute/procwrap/statuses/{sha}","languages_url":"https://api.github.com/repos/deep-compute/procwrap/languages","stargazers_url":"https://api.github.com/repos/deep-compute/procwrap/stargazers","contributors_url":"https://api.github.com/repos/deep-compute/procwrap/contributors","subscribers_url":"https://api.github.com/repos/deep-compute/procwrap/subscribers","subscription_url":"https://api.github.com/repos/deep-compute/procwrap/subscription","commits_url":"https://api.github.com/repos/deep-compute/procwrap/commits{/sha}","git_commits_url":"https://api.github.com/repos/deep-compute/procwrap/git/commits{/sha}","comments_url":"https://api.github.com/repos/deep-compute/procwrap/comments{/number}","issue_comment_url":"https://api.github.com/repos/deep-compute/procwrap/issues/comments{/number}","contents_url":"https://api.github.com/repos/deep-compute/procwrap/contents/{+path}","compare_url":"https://api.github.com/repos/deep-compute/procwrap/compare/{base}...{head}","merges_url":"https://api.github.com/repos/deep-compute/procwrap/merges","archive_url":"https://api.github.com/repos/deep-compute/procwrap/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/deep-compute/procwrap/downloads","issues_url":"https://api.github.com/repos/deep-compute/procwrap/issues{/number}","pulls_url":"https://api.github.com/repos/deep-compute/procwrap/pulls{/number}","milestones_url":"https://api.github.com/repos/deep-compute/procwrap/milestones{/number}","notifications_url":"https://api.github.com/repos/deep-compute/procwrap/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/deep-compute/procwrap/labels{/name}","releases_url":"https://api.github.com/repos/deep-compute/procwrap/releases{/id}","deployments_url":"https://api.github.com/repos/deep-compute/procwrap/deployments","created_at":"2016-12-28T06:55:25Z","updated_at":"2016-12-28T08:08:54Z","pushed_at":"2016-12-28T14:10:45Z","git_url":"git://github.com/deep-compute/procwrap.git","ssh_url":"git@github.com:deep-compute/procwrap.git","clone_url":"https://github.com/deep-compute/procwrap.git","svn_url":"https://github.com/deep-compute/procwrap","homepage":null,"size":3,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":3,"forks":1,"open_issues":3,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/deep-compute/procwrap/pulls/5"},"html":{"href":"https://github.com/deep-compute/procwrap/pull/5"},"issue":{"href":"https://api.github.com/repos/deep-compute/procwrap/issues/5"},"comments":{"href":"https://api.github.com/repos/deep-compute/procwrap/issues/5/comments"},"review_comments":{"href":"https://api.github.com/repos/deep-compute/procwrap/pulls/5/comments"},"review_comment":{"href":"https://api.github.com/repos/deep-compute/procwrap/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/deep-compute/procwrap/pulls/5/commits"},"statuses":{"href":"https://api.github.com/repos/deep-compute/procwrap/statuses/d22121925e185dd35e05537907466c478491bca4"}}}}
{"actor":{"display_login":"prashanthellina"}}