{"action":"created","comment":{"url":"https://api.github.com/repos/TeamHG-Memex/html-text/pulls/comments/219571114","pull_request_review_id":157786571,"id":219571114,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIxOTU3MTExNA==","diff_hunk":"@@ -44,30 +53,108 @@ def parse_html(html):\n _whitespace = re.compile(r'\\s+')\n _has_trailing_whitespace = re.compile(r'\\s$').search\n _has_punct_after = re.compile(r'^[,:;.!?\"\\)]').search\n-_has_punct_before = re.compile(r'\\($').search\n+_has_open_bracket_before = re.compile(r'\\($').search\n \n \n-def selector_to_text(sel, guess_punct_space=True):\n-    \"\"\" Convert a cleaned selector to text.\n-    See html_text.extract_text docstring for description of the approach and options.\n+def _html_to_text(tree,\n+                  guess_punct_space=True,\n+                  guess_page_layout=False,\n+                  newline_tags=NEWLINE_TAGS,\n+                  double_newline_tags=DOUBLE_NEWLINE_TAGS):\n+    \"\"\"\n+    Convert a cleaned html tree to text.\n+    See html_text.extract_text docstring for description of the approach\n+    and options.\n     \"\"\"\n-    if guess_punct_space:\n-\n-        def fragments():\n-            prev = None\n-            for text in sel.xpath('.//text()').extract():\n-                if prev is not None and (_has_trailing_whitespace(prev)\n-                                         or (not _has_punct_after(text) and\n-                                             not _has_punct_before(prev))):\n-                    yield ' '\n-                yield text\n-                prev = text\n-\n-        return _whitespace.sub(' ', ''.join(fragments()).strip())\n \n+    def add_space(text, prev):\n+        if prev is None:\n+            return False\n+        if prev == '\\n' or prev == '\\n\\n':\n+            return False\n+        if not _has_trailing_whitespace(prev):\n+            if _has_punct_after(text) or _has_open_bracket_before(prev):\n+                return False\n+        return True\n+\n+    def add_newline(tag, prev):\n+        if prev is None or prev == '\\n\\n':\n+            return '', '\\n\\n'\n+        if tag in double_newline_tags:\n+            if prev == '\\n':\n+                return '\\n', '\\n\\n'\n+            return '\\n\\n', '\\n\\n'\n+        if tag in newline_tags:\n+            if prev == '\\n':\n+                return '', prev\n+            return '\\n', '\\n'\n+        return '', prev\n+\n+    def traverse_text_fragments(tree, prev, depth):\n+        space = ' '\n+        newline = ''\n+        text = ''\n+        if guess_page_layout:\n+            newline, prev[0] = add_newline(tree.tag, prev[0])\n+        if tree.text:\n+            text = _whitespace.sub(' ', tree.text.strip())\n+            if text and guess_punct_space and not add_space(text, prev[0]):\n+                space = ''\n+        if text:\n+            yield [newline, space, text]\n+            prev[0] = tree.text\n+            space = ' '\n+            newline = ''\n+        elif newline:\n+            yield [newline]\n+            newline = ''\n+\n+        for child in tree:\n+            for t in traverse_text_fragments(child, prev, depth + 1):\n+                yield t\n+\n+        if guess_page_layout:\n+            newline, prev[0] = add_newline(tree.tag, prev[0])\n+\n+        tail = ''\n+        if tree.tail and depth != 0:\n+            tail = _whitespace.sub(' ', tree.tail.strip())\n+            if tail:\n+                if guess_punct_space and not add_space(tail, prev[0]):\n+                    space = ''\n+        if tail:\n+            yield [newline, space, tail]\n+            prev[0] = tree.tail\n+        elif newline:\n+            yield [newline]\n+\n+    text = []\n+    for fragment in traverse_text_fragments(tree, [None], 0):\n+        text.extend(fragment)\n+    return ''.join(text).strip()\n+\n+\n+def selector_to_text(sel, guess_punct_space=True, guess_page_layout=False):\n+    \"\"\" Convert a cleaned selector to text.\n+    See html_text.extract_text docstring for description of the approach\n+    and options.\n+    \"\"\"\n+    if isinstance(sel, list):\n+        # if selecting a specific xpath\n+        text = []\n+        for t in sel:\n+            extracted = _html_to_text(\n+                t.root,\n+                guess_punct_space=guess_punct_space,\n+                guess_page_layout=guess_page_layout)\n+            if extracted:\n+                text.append(extracted)\n+        return ' '.join(text)","path":"html_text/html_text.py","position":133,"original_position":133,"commit_id":"05b979a6f43fa27d10eb29df1cdfae14b566f235","original_commit_id":"7aec8d2ae0c044996f8549d307a67c97ce71479e","user":{"login":"kmike","id":107893,"node_id":"MDQ6VXNlcjEwNzg5Mw==","avatar_url":"https://avatars3.githubusercontent.com/u/107893?v=4","gravatar_id":"","url":"https://api.github.com/users/kmike","html_url":"https://github.com/kmike","followers_url":"https://api.github.com/users/kmike/followers","following_url":"https://api.github.com/users/kmike/following{/other_user}","gists_url":"https://api.github.com/users/kmike/gists{/gist_id}","starred_url":"https://api.github.com/users/kmike/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kmike/subscriptions","organizations_url":"https://api.github.com/users/kmike/orgs","repos_url":"https://api.github.com/users/kmike/repos","events_url":"https://api.github.com/users/kmike/events{/privacy}","received_events_url":"https://api.github.com/users/kmike/received_events","type":"User","site_admin":false},"body":"I agree it is not clear which behavior is better; let's keep the current one, but have it in mind.","created_at":"2018-09-21T17:26:51Z","updated_at":"2018-09-21T17:26:51Z","html_url":"https://github.com/TeamHG-Memex/html-text/pull/9#discussion_r219571114","pull_request_url":"https://api.github.com/repos/TeamHG-Memex/html-text/pulls/9","author_association":"NONE","_links":{"self":{"href":"https://api.github.com/repos/TeamHG-Memex/html-text/pulls/comments/219571114"},"html":{"href":"https://github.com/TeamHG-Memex/html-text/pull/9#discussion_r219571114"},"pull_request":{"href":"https://api.github.com/repos/TeamHG-Memex/html-text/pulls/9"}},"in_reply_to_id":219002365},"pull_request":{"url":"https://api.github.com/repos/TeamHG-Memex/html-text/pulls/9","id":211539990,"node_id":"MDExOlB1bGxSZXF1ZXN0MjExNTM5OTkw","html_url":"https://github.com/TeamHG-Memex/html-text/pull/9","diff_url":"https://github.com/TeamHG-Memex/html-text/pull/9.diff","patch_url":"https://github.com/TeamHG-Memex/html-text/pull/9.patch","issue_url":"https://api.github.com/repos/TeamHG-Memex/html-text/issues/9","number":9,"state":"open","locked":false,"title":"Add guess page layout","user":{"login":"Kebniss","id":19157990,"node_id":"MDQ6VXNlcjE5MTU3OTkw","avatar_url":"https://avatars1.githubusercontent.com/u/19157990?v=4","gravatar_id":"","url":"https://api.github.com/users/Kebniss","html_url":"https://github.com/Kebniss","followers_url":"https://api.github.com/users/Kebniss/followers","following_url":"https://api.github.com/users/Kebniss/following{/other_user}","gists_url":"https://api.github.com/users/Kebniss/gists{/gist_id}","starred_url":"https://api.github.com/users/Kebniss/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Kebniss/subscriptions","organizations_url":"https://api.github.com/users/Kebniss/orgs","repos_url":"https://api.github.com/users/Kebniss/repos","events_url":"https://api.github.com/users/Kebniss/events{/privacy}","received_events_url":"https://api.github.com/users/Kebniss/received_events","type":"User","site_admin":false},"body":"I tested the new extraction algorithm on 1200 htmls and timed extraction performance:\r\n- `extract_text(html, guess_page_layout=True, guess_punct_space=True)`: \r\n    - avg extraction time per html: 0.00769ms\r\n    - fastest: 0.00006ms\r\n    - slowest: 0.07818ms\r\n- `html_text.selector_to_text(sel, guess_punct_space=True, guess_page_layout=True)`: \r\n    - avg extraction time per html: 0.01368ms\r\n    - fastest: 0.00012ms\r\n    - slowest: 0.12133ms\r\n\r\nDid the same on the old extraction algorithm:\r\n- `extract_text(html,  guess_punct_space=True)`:\r\n    - avg extraction time per html: 0.00941ms\r\n    - fastest: 0.00011ms\r\n    - slowest: 0.15546ms\r\n- `text = html_text.selector_to_text(sel, guess_punct_space=True)`:\r\n    - avg extraction time per html: 0.01445ms\r\n    - fastest:0.00016ms\r\n    - slowest: 0.12790ms\r\n\r\nhtml extraction speed has improved of ~20% while selector extraction improved of ~6%\r\n\r\nTODO:\r\n\r\n- [x] add multiple newlines\r\n- [x] add backward compatibility with selectors\r\n- [x] handle more tags if guess_page_layout=True\r\n- [x] add tests on real webpages with guess_page_layout=True\r\n- [x] handle nested tags without text when guess_page_layout=True\r\n- [x] update readme\r\n- [x] add performance evaluation\r\n","created_at":"2018-08-28T19:59:19Z","updated_at":"2018-09-21T17:26:51Z","closed_at":null,"merged_at":null,"merge_commit_sha":"81f83121a1b5208d36e9c8c39aed9b17e2b1e37e","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/TeamHG-Memex/html-text/pulls/9/commits","review_comments_url":"https://api.github.com/repos/TeamHG-Memex/html-text/pulls/9/comments","review_comment_url":"https://api.github.com/repos/TeamHG-Memex/html-text/pulls/comments{/number}","comments_url":"https://api.github.com/repos/TeamHG-Memex/html-text/issues/9/comments","statuses_url":"https://api.github.com/repos/TeamHG-Memex/html-text/statuses/05b979a6f43fa27d10eb29df1cdfae14b566f235","head":{"label":"Kebniss:add-guess-page-layout","ref":"add-guess-page-layout","sha":"05b979a6f43fa27d10eb29df1cdfae14b566f235","user":{"login":"Kebniss","id":19157990,"node_id":"MDQ6VXNlcjE5MTU3OTkw","avatar_url":"https://avatars1.githubusercontent.com/u/19157990?v=4","gravatar_id":"","url":"https://api.github.com/users/Kebniss","html_url":"https://github.com/Kebniss","followers_url":"https://api.github.com/users/Kebniss/followers","following_url":"https://api.github.com/users/Kebniss/following{/other_user}","gists_url":"https://api.github.com/users/Kebniss/gists{/gist_id}","starred_url":"https://api.github.com/users/Kebniss/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Kebniss/subscriptions","organizations_url":"https://api.github.com/users/Kebniss/orgs","repos_url":"https://api.github.com/users/Kebniss/repos","events_url":"https://api.github.com/users/Kebniss/events{/privacy}","received_events_url":"https://api.github.com/users/Kebniss/received_events","type":"User","site_admin":false},"repo":{"id":146042659,"node_id":"MDEwOlJlcG9zaXRvcnkxNDYwNDI2NTk=","name":"html-text","full_name":"Kebniss/html-text","private":false,"owner":{"login":"Kebniss","id":19157990,"node_id":"MDQ6VXNlcjE5MTU3OTkw","avatar_url":"https://avatars1.githubusercontent.com/u/19157990?v=4","gravatar_id":"","url":"https://api.github.com/users/Kebniss","html_url":"https://github.com/Kebniss","followers_url":"https://api.github.com/users/Kebniss/followers","following_url":"https://api.github.com/users/Kebniss/following{/other_user}","gists_url":"https://api.github.com/users/Kebniss/gists{/gist_id}","starred_url":"https://api.github.com/users/Kebniss/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Kebniss/subscriptions","organizations_url":"https://api.github.com/users/Kebniss/orgs","repos_url":"https://api.github.com/users/Kebniss/repos","events_url":"https://api.github.com/users/Kebniss/events{/privacy}","received_events_url":"https://api.github.com/users/Kebniss/received_events","type":"User","site_admin":false},"html_url":"https://github.com/Kebniss/html-text","description":"Extract text from HTML","fork":true,"url":"https://api.github.com/repos/Kebniss/html-text","forks_url":"https://api.github.com/repos/Kebniss/html-text/forks","keys_url":"https://api.github.com/repos/Kebniss/html-text/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Kebniss/html-text/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Kebniss/html-text/teams","hooks_url":"https://api.github.com/repos/Kebniss/html-text/hooks","issue_events_url":"https://api.github.com/repos/Kebniss/html-text/issues/events{/number}","events_url":"https://api.github.com/repos/Kebniss/html-text/events","assignees_url":"https://api.github.com/repos/Kebniss/html-text/assignees{/user}","branches_url":"https://api.github.com/repos/Kebniss/html-text/branches{/branch}","tags_url":"https://api.github.com/repos/Kebniss/html-text/tags","blobs_url":"https://api.github.com/repos/Kebniss/html-text/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Kebniss/html-text/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Kebniss/html-text/git/refs{/sha}","trees_url":"https://api.github.com/repos/Kebniss/html-text/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Kebniss/html-text/statuses/{sha}","languages_url":"https://api.github.com/repos/Kebniss/html-text/languages","stargazers_url":"https://api.github.com/repos/Kebniss/html-text/stargazers","contributors_url":"https://api.github.com/repos/Kebniss/html-text/contributors","subscribers_url":"https://api.github.com/repos/Kebniss/html-text/subscribers","subscription_url":"https://api.github.com/repos/Kebniss/html-text/subscription","commits_url":"https://api.github.com/repos/Kebniss/html-text/commits{/sha}","git_commits_url":"https://api.github.com/repos/Kebniss/html-text/git/commits{/sha}","comments_url":"https://api.github.com/repos/Kebniss/html-text/comments{/number}","issue_comment_url":"https://api.github.com/repos/Kebniss/html-text/issues/comments{/number}","contents_url":"https://api.github.com/repos/Kebniss/html-text/contents/{+path}","compare_url":"https://api.github.com/repos/Kebniss/html-text/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Kebniss/html-text/merges","archive_url":"https://api.github.com/repos/Kebniss/html-text/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Kebniss/html-text/downloads","issues_url":"https://api.github.com/repos/Kebniss/html-text/issues{/number}","pulls_url":"https://api.github.com/repos/Kebniss/html-text/pulls{/number}","milestones_url":"https://api.github.com/repos/Kebniss/html-text/milestones{/number}","notifications_url":"https://api.github.com/repos/Kebniss/html-text/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Kebniss/html-text/labels{/name}","releases_url":"https://api.github.com/repos/Kebniss/html-text/releases{/id}","deployments_url":"https://api.github.com/repos/Kebniss/html-text/deployments","created_at":"2018-08-24T21:44:12Z","updated_at":"2018-08-24T21:44:14Z","pushed_at":"2018-09-20T21:41:34Z","git_url":"git://github.com/Kebniss/html-text.git","ssh_url":"git@github.com:Kebniss/html-text.git","clone_url":"https://github.com/Kebniss/html-text.git","svn_url":"https://github.com/Kebniss/html-text","homepage":null,"size":107,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"TeamHG-Memex:master","ref":"master","sha":"483686543aaf7d13fb6ce69100d7d6e9b4ba226d","user":{"login":"TeamHG-Memex","id":9270052,"node_id":"MDEyOk9yZ2FuaXphdGlvbjkyNzAwNTI=","avatar_url":"https://avatars1.githubusercontent.com/u/9270052?v=4","gravatar_id":"","url":"https://api.github.com/users/TeamHG-Memex","html_url":"https://github.com/TeamHG-Memex","followers_url":"https://api.github.com/users/TeamHG-Memex/followers","following_url":"https://api.github.com/users/TeamHG-Memex/following{/other_user}","gists_url":"https://api.github.com/users/TeamHG-Memex/gists{/gist_id}","starred_url":"https://api.github.com/users/TeamHG-Memex/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TeamHG-Memex/subscriptions","organizations_url":"https://api.github.com/users/TeamHG-Memex/orgs","repos_url":"https://api.github.com/users/TeamHG-Memex/repos","events_url":"https://api.github.com/users/TeamHG-Memex/events{/privacy}","received_events_url":"https://api.github.com/users/TeamHG-Memex/received_events","type":"Organization","site_admin":false},"repo":{"id":69354297,"node_id":"MDEwOlJlcG9zaXRvcnk2OTM1NDI5Nw==","name":"html-text","full_name":"TeamHG-Memex/html-text","private":false,"owner":{"login":"TeamHG-Memex","id":9270052,"node_id":"MDEyOk9yZ2FuaXphdGlvbjkyNzAwNTI=","avatar_url":"https://avatars1.githubusercontent.com/u/9270052?v=4","gravatar_id":"","url":"https://api.github.com/users/TeamHG-Memex","html_url":"https://github.com/TeamHG-Memex","followers_url":"https://api.github.com/users/TeamHG-Memex/followers","following_url":"https://api.github.com/users/TeamHG-Memex/following{/other_user}","gists_url":"https://api.github.com/users/TeamHG-Memex/gists{/gist_id}","starred_url":"https://api.github.com/users/TeamHG-Memex/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TeamHG-Memex/subscriptions","organizations_url":"https://api.github.com/users/TeamHG-Memex/orgs","repos_url":"https://api.github.com/users/TeamHG-Memex/repos","events_url":"https://api.github.com/users/TeamHG-Memex/events{/privacy}","received_events_url":"https://api.github.com/users/TeamHG-Memex/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/TeamHG-Memex/html-text","description":"Extract text from HTML","fork":false,"url":"https://api.github.com/repos/TeamHG-Memex/html-text","forks_url":"https://api.github.com/repos/TeamHG-Memex/html-text/forks","keys_url":"https://api.github.com/repos/TeamHG-Memex/html-text/keys{/key_id}","collaborators_url":"https://api.github.com/repos/TeamHG-Memex/html-text/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/TeamHG-Memex/html-text/teams","hooks_url":"https://api.github.com/repos/TeamHG-Memex/html-text/hooks","issue_events_url":"https://api.github.com/repos/TeamHG-Memex/html-text/issues/events{/number}","events_url":"https://api.github.com/repos/TeamHG-Memex/html-text/events","assignees_url":"https://api.github.com/repos/TeamHG-Memex/html-text/assignees{/user}","branches_url":"https://api.github.com/repos/TeamHG-Memex/html-text/branches{/branch}","tags_url":"https://api.github.com/repos/TeamHG-Memex/html-text/tags","blobs_url":"https://api.github.com/repos/TeamHG-Memex/html-text/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/TeamHG-Memex/html-text/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/TeamHG-Memex/html-text/git/refs{/sha}","trees_url":"https://api.github.com/repos/TeamHG-Memex/html-text/git/trees{/sha}","statuses_url":"https://api.github.com/repos/TeamHG-Memex/html-text/statuses/{sha}","languages_url":"https://api.github.com/repos/TeamHG-Memex/html-text/languages","stargazers_url":"https://api.github.com/repos/TeamHG-Memex/html-text/stargazers","contributors_url":"https://api.github.com/repos/TeamHG-Memex/html-text/contributors","subscribers_url":"https://api.github.com/repos/TeamHG-Memex/html-text/subscribers","subscription_url":"https://api.github.com/repos/TeamHG-Memex/html-text/subscription","commits_url":"https://api.github.com/repos/TeamHG-Memex/html-text/commits{/sha}","git_commits_url":"https://api.github.com/repos/TeamHG-Memex/html-text/git/commits{/sha}","comments_url":"https://api.github.com/repos/TeamHG-Memex/html-text/comments{/number}","issue_comment_url":"https://api.github.com/repos/TeamHG-Memex/html-text/issues/comments{/number}","contents_url":"https://api.github.com/repos/TeamHG-Memex/html-text/contents/{+path}","compare_url":"https://api.github.com/repos/TeamHG-Memex/html-text/compare/{base}...{head}","merges_url":"https://api.github.com/repos/TeamHG-Memex/html-text/merges","archive_url":"https://api.github.com/repos/TeamHG-Memex/html-text/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/TeamHG-Memex/html-text/downloads","issues_url":"https://api.github.com/repos/TeamHG-Memex/html-text/issues{/number}","pulls_url":"https://api.github.com/repos/TeamHG-Memex/html-text/pulls{/number}","milestones_url":"https://api.github.com/repos/TeamHG-Memex/html-text/milestones{/number}","notifications_url":"https://api.github.com/repos/TeamHG-Memex/html-text/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/TeamHG-Memex/html-text/labels{/name}","releases_url":"https://api.github.com/repos/TeamHG-Memex/html-text/releases{/id}","deployments_url":"https://api.github.com/repos/TeamHG-Memex/html-text/deployments","created_at":"2016-09-27T12:28:53Z","updated_at":"2018-08-13T11:27:55Z","pushed_at":"2018-09-20T21:41:35Z","git_url":"git://github.com/TeamHG-Memex/html-text.git","ssh_url":"git@github.com:TeamHG-Memex/html-text.git","clone_url":"https://github.com/TeamHG-Memex/html-text.git","svn_url":"https://github.com/TeamHG-Memex/html-text","homepage":null,"size":21,"stargazers_count":24,"watchers_count":24,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":8,"mirror_url":null,"archived":false,"open_issues_count":5,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":8,"open_issues":5,"watchers":24,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/TeamHG-Memex/html-text/pulls/9"},"html":{"href":"https://github.com/TeamHG-Memex/html-text/pull/9"},"issue":{"href":"https://api.github.com/repos/TeamHG-Memex/html-text/issues/9"},"comments":{"href":"https://api.github.com/repos/TeamHG-Memex/html-text/issues/9/comments"},"review_comments":{"href":"https://api.github.com/repos/TeamHG-Memex/html-text/pulls/9/comments"},"review_comment":{"href":"https://api.github.com/repos/TeamHG-Memex/html-text/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/TeamHG-Memex/html-text/pulls/9/commits"},"statuses":{"href":"https://api.github.com/repos/TeamHG-Memex/html-text/statuses/05b979a6f43fa27d10eb29df1cdfae14b566f235"}},"author_association":"NONE"}}
{"actor":{"display_login":"kmike"}}