{"action":"created","comment":{"url":"https://api.github.com/repos/greenplum-db/gpdb/pulls/comments/244262069","pull_request_review_id":188219248,"id":244262069,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDI0NDI2MjA2OQ==","diff_hunk":"@@ -1905,6 +1907,95 @@ def checkPGNamespace():\n             logger.error(formatErr(e[0], e[1], e[2]))\n \n \n+#############\n+def connect3run(qry):\n+    logger.debug('%s' % qry)\n+\n+    batch = []\n+    threads = []\n+    i = 1\n+\n+    # parallelise queries\n+    for dbid in GV.cfg:\n+        c = GV.cfg[dbid]\n+        db = connect2(c)\n+\n+        thread = execThread(c, db, qry)\n+        thread.start()\n+        logger.debug('launching query thread %s for dbid %i' %\n+                     (thread.getName(), dbid))\n+        threads.append(thread)\n+\n+        # we don't want too much going on at once\n+        if (i % GV.opt['-B']) == 0:\n+            # process this batch of threads\n+            batch.extend(processThread(threads))\n+            threads = []\n+\n+        i += 1\n+\n+    # process the rest of threads\n+    batch.extend(processThread(threads))\n+\n+    err = []\n+\n+    for [cfg, curs] in batch:\n+        relfilenodeInCatalogList = []\n+        relfilenodeInSystem = {}\n+        fileOnlyInSystem = []\n+        dbid = 1\n+        db2 = connect2(cfg)\n+        datidnames = db2.query('select oid, datname from pg_database').dictresult()\n+        for row in datidnames:\n+            if row['datname'] == GV.dbname:\n+                dbid = row['oid']\n+        for row in curs.dictresult():\n+            relfilenode = row['relfilenode']\n+            # If it is relmapped relation, need to fetch the mapping from relmap file\n+            if relfilenode == 0:\n+                relfilenode = db2.query('select pg_relation_filenode(%d)' % row['oid']).dictresult()[0]['pg_relation_filenode']\n+            relfilenodeInCatalogList.append(relfilenode)\n+\n+        filepath = cfg['datadir'] + '/base/%s' % dbid\n+        onlyfiles = [f for f in listdir(filepath) if isfile(join(filepath, f))]\n+        for f in onlyfiles:\n+            fl = f.lower()\n+            if fl == '16391.12345':","path":"gpMgmt/bin/gpcheckcat","position":66,"original_position":66,"commit_id":"ad1482ed8dfd99647d8717864511ecaa3d5958e2","original_commit_id":"ad1482ed8dfd99647d8717864511ecaa3d5958e2","user":{"login":"macroyuyang","id":9679273,"node_id":"MDQ6VXNlcjk2NzkyNzM=","avatar_url":"https://avatars2.githubusercontent.com/u/9679273?v=4","gravatar_id":"","url":"https://api.github.com/users/macroyuyang","html_url":"https://github.com/macroyuyang","followers_url":"https://api.github.com/users/macroyuyang/followers","following_url":"https://api.github.com/users/macroyuyang/following{/other_user}","gists_url":"https://api.github.com/users/macroyuyang/gists{/gist_id}","starred_url":"https://api.github.com/users/macroyuyang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/macroyuyang/subscriptions","organizations_url":"https://api.github.com/users/macroyuyang/orgs","repos_url":"https://api.github.com/users/macroyuyang/repos","events_url":"https://api.github.com/users/macroyuyang/events{/privacy}","received_events_url":"https://api.github.com/users/macroyuyang/received_events","type":"User","site_admin":false},"body":"What's this?","created_at":"2018-12-28T04:43:12Z","updated_at":"2018-12-28T04:43:12Z","html_url":"https://github.com/greenplum-db/gpdb/pull/6578#discussion_r244262069","pull_request_url":"https://api.github.com/repos/greenplum-db/gpdb/pulls/6578","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/greenplum-db/gpdb/pulls/comments/244262069"},"html":{"href":"https://github.com/greenplum-db/gpdb/pull/6578#discussion_r244262069"},"pull_request":{"href":"https://api.github.com/repos/greenplum-db/gpdb/pulls/6578"}}},"pull_request":{"url":"https://api.github.com/repos/greenplum-db/gpdb/pulls/6578","id":241186270,"node_id":"MDExOlB1bGxSZXF1ZXN0MjQxMTg2Mjcw","html_url":"https://github.com/greenplum-db/gpdb/pull/6578","diff_url":"https://github.com/greenplum-db/gpdb/pull/6578.diff","patch_url":"https://github.com/greenplum-db/gpdb/pull/6578.patch","issue_url":"https://api.github.com/repos/greenplum-db/gpdb/issues/6578","number":6578,"state":"open","locked":false,"title":"Modify gpcheckcat to perform check for filesystem objects in order to identify leaked files","user":{"login":"BaiShaoqi","id":5915448,"node_id":"MDQ6VXNlcjU5MTU0NDg=","avatar_url":"https://avatars0.githubusercontent.com/u/5915448?v=4","gravatar_id":"","url":"https://api.github.com/users/BaiShaoqi","html_url":"https://github.com/BaiShaoqi","followers_url":"https://api.github.com/users/BaiShaoqi/followers","following_url":"https://api.github.com/users/BaiShaoqi/following{/other_user}","gists_url":"https://api.github.com/users/BaiShaoqi/gists{/gist_id}","starred_url":"https://api.github.com/users/BaiShaoqi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BaiShaoqi/subscriptions","organizations_url":"https://api.github.com/users/BaiShaoqi/orgs","repos_url":"https://api.github.com/users/BaiShaoqi/repos","events_url":"https://api.github.com/users/BaiShaoqi/events{/privacy}","received_events_url":"https://api.github.com/users/BaiShaoqi/received_events","type":"User","site_admin":false},"body":"If delebrately `touch ./gpAux/gpdemo/datadirs/dbfast2/demoDataDir1/base/16384/16391.12345`\r\n\r\nRun `gpcheckcat -R relfilenode_filesystem_consistency  pivotal`(16384 is the dbid of database named pivotal)\r\n\r\nwill see output `extra file in filesystem': ['16391.12345']`\r\n```\r\nConnected as user 'sbai' to database 'pivotal', port '15432', gpdb version '6.0'\r\n-------------------------------------------------------------------\r\nBatch size: 4\r\n20181228:00:29:25:040968 gpcheckcat:ShaoqiBaisMBP2:sbai-[INFO]:------------------------------------\r\n20181228:00:29:25:040968 gpcheckcat:ShaoqiBaisMBP2:sbai-[INFO]:-Checking for leaked temporary schemas\r\n20181228:00:29:25:040968 gpcheckcat:ShaoqiBaisMBP2:sbai-[INFO]:-[OK] temporary schemas\r\nPerforming test 'relfilenode_filesystem_consistency'\r\n20181228:00:29:25:040968 gpcheckcat:ShaoqiBaisMBP2:sbai-[INFO]:------------------------------------\r\n20181228:00:29:25:040968 gpcheckcat:ShaoqiBaisMBP2:sbai-[INFO]:-Checking relfilenode filesystem consistency ...\r\n20181228:00:29:25:040968 gpcheckcat:ShaoqiBaisMBP2:sbai-[INFO]:-[FAIL] relfilenode filesystem inconsistency\r\n20181228:00:29:25:040968 gpcheckcat:ShaoqiBaisMBP2:sbai-[ERROR]:-found 1 relfilenode filesystem inconsistency\r\n20181228:00:29:25:040968 gpcheckcat:ShaoqiBaisMBP2:sbai-[ERROR]:-\r\n    SELECT relfilenode, oid\r\n    FROM pg_class\r\n\r\n20181228:00:29:25:040968 gpcheckcat:ShaoqiBaisMBP2:sbai-[ERROR]:-[{'definedprimary': 't', 'dbid': 3, 'hostname': 'ShaoqiBaisMBP2.lan', 'content': 1, 'datadir': '/Users/sbai/work/gpdb/gpAux/gpdemo/datadirs/dbfast2/demoDataDir1', 'address': 'ShaoqiBaisMBP2.lan', 'isprimary': 't', 'port': 25433}, {'extra file in filesystem': ['16391.12345']}]\r\nTotal runtime for test 'relfilenode_filesystem_consistency': 0:00:00.11\r\n\r\nSUMMARY REPORT\r\n===================================================================\r\nCompleted 1 test(s) on database 'pivotal' at 2018-12-28 00:29:25 with elapsed time 0:00:00\r\nFailed test(s) that are not reported here: relfilenode_filesystem_consistency\r\nSee /Users/sbai/gpAdminLogs/gpcheckcat_20181228.log for detail\r\n```","created_at":"2018-12-27T16:34:53Z","updated_at":"2018-12-28T04:43:12Z","closed_at":null,"merged_at":null,"merge_commit_sha":"123a970971bd798e84564389b237a0d02e2878ff","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/greenplum-db/gpdb/pulls/6578/commits","review_comments_url":"https://api.github.com/repos/greenplum-db/gpdb/pulls/6578/comments","review_comment_url":"https://api.github.com/repos/greenplum-db/gpdb/pulls/comments{/number}","comments_url":"https://api.github.com/repos/greenplum-db/gpdb/issues/6578/comments","statuses_url":"https://api.github.com/repos/greenplum-db/gpdb/statuses/ad1482ed8dfd99647d8717864511ecaa3d5958e2","head":{"label":"BaiShaoqi:check-extra-file","ref":"check-extra-file","sha":"ad1482ed8dfd99647d8717864511ecaa3d5958e2","user":{"login":"BaiShaoqi","id":5915448,"node_id":"MDQ6VXNlcjU5MTU0NDg=","avatar_url":"https://avatars0.githubusercontent.com/u/5915448?v=4","gravatar_id":"","url":"https://api.github.com/users/BaiShaoqi","html_url":"https://github.com/BaiShaoqi","followers_url":"https://api.github.com/users/BaiShaoqi/followers","following_url":"https://api.github.com/users/BaiShaoqi/following{/other_user}","gists_url":"https://api.github.com/users/BaiShaoqi/gists{/gist_id}","starred_url":"https://api.github.com/users/BaiShaoqi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BaiShaoqi/subscriptions","organizations_url":"https://api.github.com/users/BaiShaoqi/orgs","repos_url":"https://api.github.com/users/BaiShaoqi/repos","events_url":"https://api.github.com/users/BaiShaoqi/events{/privacy}","received_events_url":"https://api.github.com/users/BaiShaoqi/received_events","type":"User","site_admin":false},"repo":{"id":154122797,"node_id":"MDEwOlJlcG9zaXRvcnkxNTQxMjI3OTc=","name":"gpdb","full_name":"BaiShaoqi/gpdb","private":false,"owner":{"login":"BaiShaoqi","id":5915448,"node_id":"MDQ6VXNlcjU5MTU0NDg=","avatar_url":"https://avatars0.githubusercontent.com/u/5915448?v=4","gravatar_id":"","url":"https://api.github.com/users/BaiShaoqi","html_url":"https://github.com/BaiShaoqi","followers_url":"https://api.github.com/users/BaiShaoqi/followers","following_url":"https://api.github.com/users/BaiShaoqi/following{/other_user}","gists_url":"https://api.github.com/users/BaiShaoqi/gists{/gist_id}","starred_url":"https://api.github.com/users/BaiShaoqi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BaiShaoqi/subscriptions","organizations_url":"https://api.github.com/users/BaiShaoqi/orgs","repos_url":"https://api.github.com/users/BaiShaoqi/repos","events_url":"https://api.github.com/users/BaiShaoqi/events{/privacy}","received_events_url":"https://api.github.com/users/BaiShaoqi/received_events","type":"User","site_admin":false},"html_url":"https://github.com/BaiShaoqi/gpdb","description":"Greenplum Database","fork":true,"url":"https://api.github.com/repos/BaiShaoqi/gpdb","forks_url":"https://api.github.com/repos/BaiShaoqi/gpdb/forks","keys_url":"https://api.github.com/repos/BaiShaoqi/gpdb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/BaiShaoqi/gpdb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/BaiShaoqi/gpdb/teams","hooks_url":"https://api.github.com/repos/BaiShaoqi/gpdb/hooks","issue_events_url":"https://api.github.com/repos/BaiShaoqi/gpdb/issues/events{/number}","events_url":"https://api.github.com/repos/BaiShaoqi/gpdb/events","assignees_url":"https://api.github.com/repos/BaiShaoqi/gpdb/assignees{/user}","branches_url":"https://api.github.com/repos/BaiShaoqi/gpdb/branches{/branch}","tags_url":"https://api.github.com/repos/BaiShaoqi/gpdb/tags","blobs_url":"https://api.github.com/repos/BaiShaoqi/gpdb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/BaiShaoqi/gpdb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/BaiShaoqi/gpdb/git/refs{/sha}","trees_url":"https://api.github.com/repos/BaiShaoqi/gpdb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/BaiShaoqi/gpdb/statuses/{sha}","languages_url":"https://api.github.com/repos/BaiShaoqi/gpdb/languages","stargazers_url":"https://api.github.com/repos/BaiShaoqi/gpdb/stargazers","contributors_url":"https://api.github.com/repos/BaiShaoqi/gpdb/contributors","subscribers_url":"https://api.github.com/repos/BaiShaoqi/gpdb/subscribers","subscription_url":"https://api.github.com/repos/BaiShaoqi/gpdb/subscription","commits_url":"https://api.github.com/repos/BaiShaoqi/gpdb/commits{/sha}","git_commits_url":"https://api.github.com/repos/BaiShaoqi/gpdb/git/commits{/sha}","comments_url":"https://api.github.com/repos/BaiShaoqi/gpdb/comments{/number}","issue_comment_url":"https://api.github.com/repos/BaiShaoqi/gpdb/issues/comments{/number}","contents_url":"https://api.github.com/repos/BaiShaoqi/gpdb/contents/{+path}","compare_url":"https://api.github.com/repos/BaiShaoqi/gpdb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/BaiShaoqi/gpdb/merges","archive_url":"https://api.github.com/repos/BaiShaoqi/gpdb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/BaiShaoqi/gpdb/downloads","issues_url":"https://api.github.com/repos/BaiShaoqi/gpdb/issues{/number}","pulls_url":"https://api.github.com/repos/BaiShaoqi/gpdb/pulls{/number}","milestones_url":"https://api.github.com/repos/BaiShaoqi/gpdb/milestones{/number}","notifications_url":"https://api.github.com/repos/BaiShaoqi/gpdb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/BaiShaoqi/gpdb/labels{/name}","releases_url":"https://api.github.com/repos/BaiShaoqi/gpdb/releases{/id}","deployments_url":"https://api.github.com/repos/BaiShaoqi/gpdb/deployments","created_at":"2018-10-22T10:08:08Z","updated_at":"2018-10-25T07:26:13Z","pushed_at":"2018-12-27T17:55:36Z","git_url":"git://github.com/BaiShaoqi/gpdb.git","ssh_url":"git@github.com:BaiShaoqi/gpdb.git","clone_url":"https://github.com/BaiShaoqi/gpdb.git","svn_url":"https://github.com/BaiShaoqi/gpdb","homepage":"http://greenplum.org","size":333068,"stargazers_count":0,"watchers_count":0,"language":"C","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"greenplum-db:master","ref":"master","sha":"f1fb55788d75e0632ac509e83550216d11507f89","user":{"login":"greenplum-db","id":14097842,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE0MDk3ODQy","avatar_url":"https://avatars2.githubusercontent.com/u/14097842?v=4","gravatar_id":"","url":"https://api.github.com/users/greenplum-db","html_url":"https://github.com/greenplum-db","followers_url":"https://api.github.com/users/greenplum-db/followers","following_url":"https://api.github.com/users/greenplum-db/following{/other_user}","gists_url":"https://api.github.com/users/greenplum-db/gists{/gist_id}","starred_url":"https://api.github.com/users/greenplum-db/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/greenplum-db/subscriptions","organizations_url":"https://api.github.com/users/greenplum-db/orgs","repos_url":"https://api.github.com/users/greenplum-db/repos","events_url":"https://api.github.com/users/greenplum-db/events{/privacy}","received_events_url":"https://api.github.com/users/greenplum-db/received_events","type":"Organization","site_admin":false},"repo":{"id":44781140,"node_id":"MDEwOlJlcG9zaXRvcnk0NDc4MTE0MA==","name":"gpdb","full_name":"greenplum-db/gpdb","private":false,"owner":{"login":"greenplum-db","id":14097842,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE0MDk3ODQy","avatar_url":"https://avatars2.githubusercontent.com/u/14097842?v=4","gravatar_id":"","url":"https://api.github.com/users/greenplum-db","html_url":"https://github.com/greenplum-db","followers_url":"https://api.github.com/users/greenplum-db/followers","following_url":"https://api.github.com/users/greenplum-db/following{/other_user}","gists_url":"https://api.github.com/users/greenplum-db/gists{/gist_id}","starred_url":"https://api.github.com/users/greenplum-db/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/greenplum-db/subscriptions","organizations_url":"https://api.github.com/users/greenplum-db/orgs","repos_url":"https://api.github.com/users/greenplum-db/repos","events_url":"https://api.github.com/users/greenplum-db/events{/privacy}","received_events_url":"https://api.github.com/users/greenplum-db/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/greenplum-db/gpdb","description":"Greenplum Database","fork":false,"url":"https://api.github.com/repos/greenplum-db/gpdb","forks_url":"https://api.github.com/repos/greenplum-db/gpdb/forks","keys_url":"https://api.github.com/repos/greenplum-db/gpdb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/greenplum-db/gpdb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/greenplum-db/gpdb/teams","hooks_url":"https://api.github.com/repos/greenplum-db/gpdb/hooks","issue_events_url":"https://api.github.com/repos/greenplum-db/gpdb/issues/events{/number}","events_url":"https://api.github.com/repos/greenplum-db/gpdb/events","assignees_url":"https://api.github.com/repos/greenplum-db/gpdb/assignees{/user}","branches_url":"https://api.github.com/repos/greenplum-db/gpdb/branches{/branch}","tags_url":"https://api.github.com/repos/greenplum-db/gpdb/tags","blobs_url":"https://api.github.com/repos/greenplum-db/gpdb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/greenplum-db/gpdb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/greenplum-db/gpdb/git/refs{/sha}","trees_url":"https://api.github.com/repos/greenplum-db/gpdb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/greenplum-db/gpdb/statuses/{sha}","languages_url":"https://api.github.com/repos/greenplum-db/gpdb/languages","stargazers_url":"https://api.github.com/repos/greenplum-db/gpdb/stargazers","contributors_url":"https://api.github.com/repos/greenplum-db/gpdb/contributors","subscribers_url":"https://api.github.com/repos/greenplum-db/gpdb/subscribers","subscription_url":"https://api.github.com/repos/greenplum-db/gpdb/subscription","commits_url":"https://api.github.com/repos/greenplum-db/gpdb/commits{/sha}","git_commits_url":"https://api.github.com/repos/greenplum-db/gpdb/git/commits{/sha}","comments_url":"https://api.github.com/repos/greenplum-db/gpdb/comments{/number}","issue_comment_url":"https://api.github.com/repos/greenplum-db/gpdb/issues/comments{/number}","contents_url":"https://api.github.com/repos/greenplum-db/gpdb/contents/{+path}","compare_url":"https://api.github.com/repos/greenplum-db/gpdb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/greenplum-db/gpdb/merges","archive_url":"https://api.github.com/repos/greenplum-db/gpdb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/greenplum-db/gpdb/downloads","issues_url":"https://api.github.com/repos/greenplum-db/gpdb/issues{/number}","pulls_url":"https://api.github.com/repos/greenplum-db/gpdb/pulls{/number}","milestones_url":"https://api.github.com/repos/greenplum-db/gpdb/milestones{/number}","notifications_url":"https://api.github.com/repos/greenplum-db/gpdb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/greenplum-db/gpdb/labels{/name}","releases_url":"https://api.github.com/repos/greenplum-db/gpdb/releases{/id}","deployments_url":"https://api.github.com/repos/greenplum-db/gpdb/deployments","created_at":"2015-10-23T00:25:17Z","updated_at":"2018-12-28T03:18:49Z","pushed_at":"2018-12-28T03:26:35Z","git_url":"git://github.com/greenplum-db/gpdb.git","ssh_url":"git@github.com:greenplum-db/gpdb.git","clone_url":"https://github.com/greenplum-db/gpdb.git","svn_url":"https://github.com/greenplum-db/gpdb","homepage":"http://greenplum.org","size":335155,"stargazers_count":2876,"watchers_count":2876,"language":"C","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":871,"mirror_url":null,"archived":false,"open_issues_count":278,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":871,"open_issues":278,"watchers":2876,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/greenplum-db/gpdb/pulls/6578"},"html":{"href":"https://github.com/greenplum-db/gpdb/pull/6578"},"issue":{"href":"https://api.github.com/repos/greenplum-db/gpdb/issues/6578"},"comments":{"href":"https://api.github.com/repos/greenplum-db/gpdb/issues/6578/comments"},"review_comments":{"href":"https://api.github.com/repos/greenplum-db/gpdb/pulls/6578/comments"},"review_comment":{"href":"https://api.github.com/repos/greenplum-db/gpdb/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/greenplum-db/gpdb/pulls/6578/commits"},"statuses":{"href":"https://api.github.com/repos/greenplum-db/gpdb/statuses/ad1482ed8dfd99647d8717864511ecaa3d5958e2"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"macroyuyang"}}