{"action":"created","comment":{"url":"https://api.github.com/repos/kripken/emscripten/pulls/comments/52540126","id":52540126,"diff_hunk":"@@ -6191,28 +6193,46 @@ def process(filename):\n   )\n   src.close()\n '''\n+ \n+ #fontconfig = self.get_library('fontconfig', [os.path.join('src', '.libs', 'libfontconfig.a')]) # Used in file, but not needed, mostly\n \n-    #fontconfig = self.get_library('fontconfig', [os.path.join('src', '.libs', 'libfontconfig.a')]) # Used in file, but not needed, mostly\n+      freetype = self.get_freetype()\n \n-    freetype = self.get_freetype()\n+      poppler = self.get_library('poppler',\n+                                 [os.path.join('utils', 'pdftoppm.o'),\n+                                  os.path.join('utils', 'parseargs.o'),\n+                                  os.path.join('poppler', '.libs', 'libpoppler.a')],\n+                                 env_init={ 'FONTCONFIG_CFLAGS': ' ', 'FONTCONFIG_LIBS': ' ' },\n+                                 configure_args=['--disable-libjpeg', '--disable-libpng', '--disable-poppler-qt', '--disable-poppler-qt4', '--disable-cms', '--disable-cairo-output', '--disable-abiword-output', '--enable-shared=no'])\n \n-    poppler = self.get_library('poppler',\n-                               [os.path.join('utils', 'pdftoppm.o'),\n-                                os.path.join('utils', 'parseargs.o'),\n-                                os.path.join('poppler', '.libs', 'libpoppler.a')],\n-                               env_init={ 'FONTCONFIG_CFLAGS': ' ', 'FONTCONFIG_LIBS': ' ' },\n-                               configure_args=['--disable-libjpeg', '--disable-libpng', '--disable-poppler-qt', '--disable-poppler-qt4', '--disable-cms', '--disable-cairo-output', '--disable-abiword-output', '--enable-shared=no'])\n+      # Combine libraries\n \n-    # Combine libraries\n+      combined = os.path.join(self.get_dir(), 'poppler-combined.bc')\n+      Building.link(poppler + freetype, combined)\n \n-    combined = os.path.join(self.get_dir(), 'poppler-combined.bc')\n-    Building.link(poppler + freetype, combined)\n+      self.do_ll_run(combined,\n+                     map(ord, open(path_from_root('tests', 'poppler', 'ref.ppm'), 'r').read()).__str__().replace(' ', ''),\n+                     args='-scale-to 512 paper.pdf filename'.split(' '),\n+                     post_build=post)\n+                     #, build_ll_hook=self.do_autodebug)\n \n-    self.do_ll_run(combined,\n-                   map(ord, open(path_from_root('tests', 'poppler', 'ref.ppm'), 'r').read()).__str__().replace(' ', ''),\n-                   args='-scale-to 512 paper.pdf filename'.split(' '),\n-                   post_build=post)\n-                   #, build_ll_hook=self.do_autodebug)\n+    test()\n+    num_original_funcs = self.count_funcs('src.cpp.o.js')\n+    num_funcs_after_dfe = num_original_funcs\n+\n+    # Run with duplicate function elimination turned on\n+    dfe_supported_opt_levels = ['-O2', '-O3', '-Oz', '-Os']\n+\n+    for opt_level in dfe_supported_opt_levels:\n+      if opt_level in self.emcc_args:\n+        print >> sys.stderr, \"Testing poppler with ELIMINATE_DUPLICATE_FUNCTIONS set to 1\"\n+        Settings.ELIMINATE_DUPLICATE_FUNCTIONS = 1\n+        test()\n+        num_funcs_after_dfe = self.count_funcs('src.cpp.o.js')\n+        break\n+\n+    # Make sure that DFE ends up eliminating more than 250 functions\n+    assert((num_original_funcs - num_funcs_after_dfe) > 250)","path":"tests/test_core.py","position":94,"original_position":94,"commit_id":"7d9f604321e2c29e73d8947471adefd72df190d2","original_commit_id":"7d9f604321e2c29e73d8947471adefd72df190d2","user":{"login":"kripken","id":173661,"avatar_url":"https://avatars.githubusercontent.com/u/173661?v=3","gravatar_id":"","url":"https://api.github.com/users/kripken","html_url":"https://github.com/kripken","followers_url":"https://api.github.com/users/kripken/followers","following_url":"https://api.github.com/users/kripken/following{/other_user}","gists_url":"https://api.github.com/users/kripken/gists{/gist_id}","starred_url":"https://api.github.com/users/kripken/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kripken/subscriptions","organizations_url":"https://api.github.com/users/kripken/orgs","repos_url":"https://api.github.com/users/kripken/repos","events_url":"https://api.github.com/users/kripken/events{/privacy}","received_events_url":"https://api.github.com/users/kripken/received_events","type":"User","site_admin":false},"body":"outer parens are not needed, in python you can write `assert (num_original_funcs - num_funcs_after_dfe) > 250`\r\n\r\n(in fact the inner parens are not needed either due to order of operations, but if you feel they are clearer i'm fine with them)","created_at":"2016-02-10T22:54:49Z","updated_at":"2016-02-10T22:54:49Z","html_url":"https://github.com/kripken/emscripten/pull/4039#discussion_r52540126","pull_request_url":"https://api.github.com/repos/kripken/emscripten/pulls/4039","_links":{"self":{"href":"https://api.github.com/repos/kripken/emscripten/pulls/comments/52540126"},"html":{"href":"https://github.com/kripken/emscripten/pull/4039#discussion_r52540126"},"pull_request":{"href":"https://api.github.com/repos/kripken/emscripten/pulls/4039"}}},"pull_request":{"url":"https://api.github.com/repos/kripken/emscripten/pulls/4039","id":55932153,"html_url":"https://github.com/kripken/emscripten/pull/4039","diff_url":"https://github.com/kripken/emscripten/pull/4039.diff","patch_url":"https://github.com/kripken/emscripten/pull/4039.patch","issue_url":"https://api.github.com/repos/kripken/emscripten/issues/4039","number":4039,"state":"open","locked":false,"title":"Duplicate function elimination","user":{"login":"rcook","id":425396,"avatar_url":"https://avatars.githubusercontent.com/u/425396?v=3","gravatar_id":"","url":"https://api.github.com/users/rcook","html_url":"https://github.com/rcook","followers_url":"https://api.github.com/users/rcook/followers","following_url":"https://api.github.com/users/rcook/following{/other_user}","gists_url":"https://api.github.com/users/rcook/gists{/gist_id}","starred_url":"https://api.github.com/users/rcook/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rcook/subscriptions","organizations_url":"https://api.github.com/users/rcook/orgs","repos_url":"https://api.github.com/users/rcook/repos","events_url":"https://api.github.com/users/rcook/events{/privacy}","received_events_url":"https://api.github.com/users/rcook/received_events","type":"User","site_admin":false},"body":"This is a pull request containing duplicate function elimination work by Arnab Choudhury and other collaborators at Tableau.\r\n\r\n* Ran main test suite using `python tests/runner.py`.\r\n* No regressions were introduced\r\n\r\nCommit message follows:\r\n\r\n-----\r\n\r\nThis change adds support for duplicate function elimination to the\r\nJavaScript optimizer.  A new JS file has been added -\r\neliminate-duplicate-functions.js - which is used to postprocess the\r\noutput generated by Emscripten. We add a new file, rather than\r\naugmenting the existing JS optimizer file, for a variety of reasons -\r\npass independence, reduced coupling between Python scripts and the JS\r\noptimizer, etc.\r\n\r\nWe introduce a multipass algorithm in which each pass consists of the\r\nfollowing four phases:\r\n\r\nPhase 1 - identify duplicate functions using a hash of the function body\r\nPhase 2 - identify variable names that would conflict after renaming\r\nfunction calls\r\nPhase 3 - generate mapping from equivalent functions to their\r\nreplacement function - use the information from Phase 2 to ensure that\r\nthe replacement function is not a variable name\r\nPhase 4 - use the mapping generated in Phase 3 to perform the reduction\r\n\r\nNOTE: In some rare cases, we may actually not be able to move on from\r\nPhase 3 if we find that we cannot generate a mapping because of\r\nconflicts with variable names.\r\n\r\nOne pass can reveal new sets of identical functions which in turn can be\r\nreduced by further passes. Empirically, four or five passes are\r\nsufficient to eliminate all duplicate functions. Internally, therefore,\r\nthe elimination will perform 5 passes by default. This can be overridden\r\nby setting the EMCC_ELIMINATE_DUPE_FUNC_PASSES environment variable.\r\n\r\nGenerated asm.js is broken into several batches (at function boundaries)\r\nto enable parallelization of the elimination. This saves on memory and\r\nmakes use of more CPU cores to save on build time. A number of tests\r\nhave been introduced to test this functionality as well.\r\n\r\nThe change also introduces various tweaks to the amount of diagnostic\r\ninformation that is dumped out by the JavaScript optimizer. Verbose\r\nlogging is now only enabled in debug mode (via the EMCC_LOG_DEBUG\r\nenvironment variable). We also dump backtraces on encountering unhandled\r\nexceptions: this is useful when Emscripten runs as part of a large build\r\nprocess. In order to view detailed information about which functions\r\nwere merged, the EMCC_DUMP_EQUIVALENT_FUNCTIONS environment variable can\r\nbe set. This generates a log file in the same directory as the generated\r\nJavaScript listing the sets of merged functions. This can be decoded\r\nusing the symbol map generated by Emscripten. It is, therefore,\r\nrecommended that developers enable symbol map generation when attempting\r\nto modify or debug this feature.\r\n\r\nThe duplicate function removal is currently enabled if an optimization\r\nlevel greater than or equal to -O2 is specified. The function\r\nelimination process can be entirely bypassed by setting the\r\nEMCC_NO_DUPLICATE_FUNCTION_ELIMINATION environment variable.\r\n\r\nImprovements/future work\r\n\r\nIt has been observed that on average we experience a code size reduction\r\nof 25% when transpiling large C++ code bases.  Typically, C++ code that\r\nmakes heavy use of templates will experience the greatest reduction in\r\ncode size. There are several directions that future work might take:\r\n\r\n* Deduplication of code across templates: e.g. reduction of\r\nstd::vector<long> and std::vector<int> to single instantiations of\r\ntemplate code when appropriate\r\n\r\n* Histogram-based selection of candidates for replacement: improved code\r\nsize should be attainable by assigning the shortest identifiers to the\r\nmost frequently referenced functions (in the style of Huffman coding)\r\n\r\n* Convergence: the five-pass default chosen in this implementation is\r\nbased on empirical observations on a 150,000LOC C++ code base\r\n\r\n* Candidate selection: this will, most likely, influence both the\r\nconvergence time (i.e. number of passes) and the code size reduction;\r\ncurrently, when selecting candidates, we choose the shortest identifier\r\nfrom the list that is not also a variable name\r\n","created_at":"2016-01-13T17:54:52Z","updated_at":"2016-02-10T22:54:49Z","closed_at":null,"merged_at":null,"merge_commit_sha":"797f3da41eca9d739bb7e0215708a53b5b2ad0a7","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/kripken/emscripten/pulls/4039/commits","review_comments_url":"https://api.github.com/repos/kripken/emscripten/pulls/4039/comments","review_comment_url":"https://api.github.com/repos/kripken/emscripten/pulls/comments{/number}","comments_url":"https://api.github.com/repos/kripken/emscripten/issues/4039/comments","statuses_url":"https://api.github.com/repos/kripken/emscripten/statuses/7d9f604321e2c29e73d8947471adefd72df190d2","head":{"label":"rcook:p-rcook-duplicate-function-elimination","ref":"p-rcook-duplicate-function-elimination","sha":"7d9f604321e2c29e73d8947471adefd72df190d2","user":{"login":"rcook","id":425396,"avatar_url":"https://avatars.githubusercontent.com/u/425396?v=3","gravatar_id":"","url":"https://api.github.com/users/rcook","html_url":"https://github.com/rcook","followers_url":"https://api.github.com/users/rcook/followers","following_url":"https://api.github.com/users/rcook/following{/other_user}","gists_url":"https://api.github.com/users/rcook/gists{/gist_id}","starred_url":"https://api.github.com/users/rcook/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rcook/subscriptions","organizations_url":"https://api.github.com/users/rcook/orgs","repos_url":"https://api.github.com/users/rcook/repos","events_url":"https://api.github.com/users/rcook/events{/privacy}","received_events_url":"https://api.github.com/users/rcook/received_events","type":"User","site_admin":false},"repo":{"id":49528091,"name":"emscripten","full_name":"rcook/emscripten","owner":{"login":"rcook","id":425396,"avatar_url":"https://avatars.githubusercontent.com/u/425396?v=3","gravatar_id":"","url":"https://api.github.com/users/rcook","html_url":"https://github.com/rcook","followers_url":"https://api.github.com/users/rcook/followers","following_url":"https://api.github.com/users/rcook/following{/other_user}","gists_url":"https://api.github.com/users/rcook/gists{/gist_id}","starred_url":"https://api.github.com/users/rcook/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rcook/subscriptions","organizations_url":"https://api.github.com/users/rcook/orgs","repos_url":"https://api.github.com/users/rcook/repos","events_url":"https://api.github.com/users/rcook/events{/privacy}","received_events_url":"https://api.github.com/users/rcook/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/rcook/emscripten","description":"Emscripten: An LLVM-to-JavaScript Compiler","fork":true,"url":"https://api.github.com/repos/rcook/emscripten","forks_url":"https://api.github.com/repos/rcook/emscripten/forks","keys_url":"https://api.github.com/repos/rcook/emscripten/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rcook/emscripten/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rcook/emscripten/teams","hooks_url":"https://api.github.com/repos/rcook/emscripten/hooks","issue_events_url":"https://api.github.com/repos/rcook/emscripten/issues/events{/number}","events_url":"https://api.github.com/repos/rcook/emscripten/events","assignees_url":"https://api.github.com/repos/rcook/emscripten/assignees{/user}","branches_url":"https://api.github.com/repos/rcook/emscripten/branches{/branch}","tags_url":"https://api.github.com/repos/rcook/emscripten/tags","blobs_url":"https://api.github.com/repos/rcook/emscripten/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rcook/emscripten/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rcook/emscripten/git/refs{/sha}","trees_url":"https://api.github.com/repos/rcook/emscripten/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rcook/emscripten/statuses/{sha}","languages_url":"https://api.github.com/repos/rcook/emscripten/languages","stargazers_url":"https://api.github.com/repos/rcook/emscripten/stargazers","contributors_url":"https://api.github.com/repos/rcook/emscripten/contributors","subscribers_url":"https://api.github.com/repos/rcook/emscripten/subscribers","subscription_url":"https://api.github.com/repos/rcook/emscripten/subscription","commits_url":"https://api.github.com/repos/rcook/emscripten/commits{/sha}","git_commits_url":"https://api.github.com/repos/rcook/emscripten/git/commits{/sha}","comments_url":"https://api.github.com/repos/rcook/emscripten/comments{/number}","issue_comment_url":"https://api.github.com/repos/rcook/emscripten/issues/comments{/number}","contents_url":"https://api.github.com/repos/rcook/emscripten/contents/{+path}","compare_url":"https://api.github.com/repos/rcook/emscripten/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rcook/emscripten/merges","archive_url":"https://api.github.com/repos/rcook/emscripten/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rcook/emscripten/downloads","issues_url":"https://api.github.com/repos/rcook/emscripten/issues{/number}","pulls_url":"https://api.github.com/repos/rcook/emscripten/pulls{/number}","milestones_url":"https://api.github.com/repos/rcook/emscripten/milestones{/number}","notifications_url":"https://api.github.com/repos/rcook/emscripten/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rcook/emscripten/labels{/name}","releases_url":"https://api.github.com/repos/rcook/emscripten/releases{/id}","deployments_url":"https://api.github.com/repos/rcook/emscripten/deployments","created_at":"2016-01-12T20:51:13Z","updated_at":"2016-01-12T20:51:33Z","pushed_at":"2016-02-10T20:33:12Z","git_url":"git://github.com/rcook/emscripten.git","ssh_url":"git@github.com:rcook/emscripten.git","clone_url":"https://github.com/rcook/emscripten.git","svn_url":"https://github.com/rcook/emscripten","homepage":"","size":135012,"stargazers_count":0,"watchers_count":0,"language":"C","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"kripken:incoming","ref":"incoming","sha":"6e68704fea3b5cd0b0e43d3dcff2c90fc198c479","user":{"login":"kripken","id":173661,"avatar_url":"https://avatars.githubusercontent.com/u/173661?v=3","gravatar_id":"","url":"https://api.github.com/users/kripken","html_url":"https://github.com/kripken","followers_url":"https://api.github.com/users/kripken/followers","following_url":"https://api.github.com/users/kripken/following{/other_user}","gists_url":"https://api.github.com/users/kripken/gists{/gist_id}","starred_url":"https://api.github.com/users/kripken/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kripken/subscriptions","organizations_url":"https://api.github.com/users/kripken/orgs","repos_url":"https://api.github.com/users/kripken/repos","events_url":"https://api.github.com/users/kripken/events{/privacy}","received_events_url":"https://api.github.com/users/kripken/received_events","type":"User","site_admin":false},"repo":{"id":1357796,"name":"emscripten","full_name":"kripken/emscripten","owner":{"login":"kripken","id":173661,"avatar_url":"https://avatars.githubusercontent.com/u/173661?v=3","gravatar_id":"","url":"https://api.github.com/users/kripken","html_url":"https://github.com/kripken","followers_url":"https://api.github.com/users/kripken/followers","following_url":"https://api.github.com/users/kripken/following{/other_user}","gists_url":"https://api.github.com/users/kripken/gists{/gist_id}","starred_url":"https://api.github.com/users/kripken/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kripken/subscriptions","organizations_url":"https://api.github.com/users/kripken/orgs","repos_url":"https://api.github.com/users/kripken/repos","events_url":"https://api.github.com/users/kripken/events{/privacy}","received_events_url":"https://api.github.com/users/kripken/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/kripken/emscripten","description":"Emscripten: An LLVM-to-JavaScript Compiler","fork":false,"url":"https://api.github.com/repos/kripken/emscripten","forks_url":"https://api.github.com/repos/kripken/emscripten/forks","keys_url":"https://api.github.com/repos/kripken/emscripten/keys{/key_id}","collaborators_url":"https://api.github.com/repos/kripken/emscripten/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/kripken/emscripten/teams","hooks_url":"https://api.github.com/repos/kripken/emscripten/hooks","issue_events_url":"https://api.github.com/repos/kripken/emscripten/issues/events{/number}","events_url":"https://api.github.com/repos/kripken/emscripten/events","assignees_url":"https://api.github.com/repos/kripken/emscripten/assignees{/user}","branches_url":"https://api.github.com/repos/kripken/emscripten/branches{/branch}","tags_url":"https://api.github.com/repos/kripken/emscripten/tags","blobs_url":"https://api.github.com/repos/kripken/emscripten/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/kripken/emscripten/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/kripken/emscripten/git/refs{/sha}","trees_url":"https://api.github.com/repos/kripken/emscripten/git/trees{/sha}","statuses_url":"https://api.github.com/repos/kripken/emscripten/statuses/{sha}","languages_url":"https://api.github.com/repos/kripken/emscripten/languages","stargazers_url":"https://api.github.com/repos/kripken/emscripten/stargazers","contributors_url":"https://api.github.com/repos/kripken/emscripten/contributors","subscribers_url":"https://api.github.com/repos/kripken/emscripten/subscribers","subscription_url":"https://api.github.com/repos/kripken/emscripten/subscription","commits_url":"https://api.github.com/repos/kripken/emscripten/commits{/sha}","git_commits_url":"https://api.github.com/repos/kripken/emscripten/git/commits{/sha}","comments_url":"https://api.github.com/repos/kripken/emscripten/comments{/number}","issue_comment_url":"https://api.github.com/repos/kripken/emscripten/issues/comments{/number}","contents_url":"https://api.github.com/repos/kripken/emscripten/contents/{+path}","compare_url":"https://api.github.com/repos/kripken/emscripten/compare/{base}...{head}","merges_url":"https://api.github.com/repos/kripken/emscripten/merges","archive_url":"https://api.github.com/repos/kripken/emscripten/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/kripken/emscripten/downloads","issues_url":"https://api.github.com/repos/kripken/emscripten/issues{/number}","pulls_url":"https://api.github.com/repos/kripken/emscripten/pulls{/number}","milestones_url":"https://api.github.com/repos/kripken/emscripten/milestones{/number}","notifications_url":"https://api.github.com/repos/kripken/emscripten/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/kripken/emscripten/labels{/name}","releases_url":"https://api.github.com/repos/kripken/emscripten/releases{/id}","deployments_url":"https://api.github.com/repos/kripken/emscripten/deployments","created_at":"2011-02-12T05:23:30Z","updated_at":"2016-02-10T22:45:45Z","pushed_at":"2016-02-10T22:18:36Z","git_url":"git://github.com/kripken/emscripten.git","ssh_url":"git@github.com:kripken/emscripten.git","clone_url":"https://github.com/kripken/emscripten.git","svn_url":"https://github.com/kripken/emscripten","homepage":"","size":145368,"stargazers_count":9663,"watchers_count":9663,"language":"C","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1172,"mirror_url":null,"open_issues_count":730,"forks":1172,"open_issues":730,"watchers":9663,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/kripken/emscripten/pulls/4039"},"html":{"href":"https://github.com/kripken/emscripten/pull/4039"},"issue":{"href":"https://api.github.com/repos/kripken/emscripten/issues/4039"},"comments":{"href":"https://api.github.com/repos/kripken/emscripten/issues/4039/comments"},"review_comments":{"href":"https://api.github.com/repos/kripken/emscripten/pulls/4039/comments"},"review_comment":{"href":"https://api.github.com/repos/kripken/emscripten/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/kripken/emscripten/pulls/4039/commits"},"statuses":{"href":"https://api.github.com/repos/kripken/emscripten/statuses/7d9f604321e2c29e73d8947471adefd72df190d2"}}}}