{"action":"created","comment":{"url":"https://api.github.com/repos/biolab/orange3/pulls/comments/74381020","id":74381020,"diff_hunk":"@@ -337,248 +349,171 @@ def open(filename, *args, **kwargs):\n         \"\"\"\n         return open_compressed(filename, *args, **kwargs)\n \n-    @staticmethod\n-    def parse_headers(data):\n-        \"\"\"Return (header rows, rest of data) as discerned from `data`\"\"\"\n+    def read_header(self):\n+        \"\"\"Subclasses may override this according to the details in FileFormat.\"\"\"\n+        raise NotImplementedError()\n+\n+    def read_contents(self, skiprows):\n+        \"\"\"Subclasses may override this according to the details in FileFormat.\"\"\"\n+        raise NotImplementedError()\n+\n+    def read(self):\n+        \"\"\"\n+        Subclasses may override this according to the details in FileFormat.\n+        If this is not overridden, it uses FileFormat.read_header and FileFormat.read_contents\n+        to construct a Table from headers and data.\n+        \"\"\"\n+        def tryparse_int_float(item):\n+            try:\n+                return int(item)\n+            except ValueError:\n+                try:\n+                    return float(item)\n+                except ValueError:\n+                    return None\n \n-        def is_number(item):\n-            try: float(item)\n-            except ValueError: return False\n-            return True\n         # Second row items are type identifiers\n         def header_test2(items):\n             return all(map(_RE_TYPES.match, items))\n+\n         # Third row items are flags and column attributes (attr=value)\n         def header_test3(items):\n             return all(map(_RE_FLAGS.match, items))\n \n-        data = iter(data)\n-        header_rows = []\n-\n-        # Try to parse a three-line header\n-        lines = []\n-        try:\n-            lines.append(list(next(data)))\n-            lines.append(list(next(data)))\n-            lines.append(list(next(data)))\n-        except StopIteration:\n-            lines, data = [], chain(lines, data)\n-        if lines:\n-            l1, l2, l3 = lines\n-            # Three-line header if line 2 & 3 match (1st line can be anything)\n-            if header_test2(l2) and header_test3(l3):\n-                header_rows = [l1, l2, l3]\n-            else:\n-                lines, data = [], chain((l1, l2, l3), data)\n-\n-        # Try to parse a single-line header\n-        if not header_rows:\n-            try: lines.append(list(next(data)))\n-            except StopIteration: pass\n-            if lines:\n-                # Header if none of the values in line 1 parses as a number\n-                if not all(is_number(i) for i in lines[0]):\n-                    header_rows = [lines[0]]\n-                else:\n-                    data = chain(lines, data)\n-\n-        return header_rows, data\n-\n-    @classmethod\n-    def data_table(self, data, headers=None):\n-        \"\"\"\n-        Return Orange.data.Table given rows of `headers` (iterable of iterable)\n-        and rows of `data` (iterable of iterable; if ``numpy.ndarray``, might\n-        as well **have it sorted column-major**, e.g. ``order='F'``).\n-\n-        Basically, the idea of subclasses is to produce those two iterables,\n-        however they might.\n-\n-        If `headers` is not provided, the header rows are extracted from `data`,\n-        assuming they precede it.\n-        \"\"\"\n-        if not headers:\n-            headers, data = self.parse_headers(data)\n-\n-        # Consider various header types (single-row, two-row, three-row, none)\n-        if 3 == len(headers):\n-            names, types, flags = map(list, headers)\n-        else:\n-            if 1 == len(headers):\n+        # read and parse the header before reading the rest of the file\n+        header_df = self.read_header()\n+        is_small = len(header_df) < 3\n+\n+        # the header can have 0, 1 or 3 columns\n+        # is a three-row header if the second and third columns match,\n+        # the first one can then be anything\n+        three_row_header = not is_small and header_test2([str(i) for i in header_df.iloc[1].fillna('')]) \\\n+                           and header_test3([str(i) for i in header_df.iloc[2].fillna('')])","path":"Orange/data/io.py","position":184,"original_position":184,"commit_id":"bb6a60e1b2671ffac42251f8a001b17cd3b37dd2","original_commit_id":"bb6a60e1b2671ffac42251f8a001b17cd3b37dd2","user":{"login":"sstanovnik","id":8071628,"avatar_url":"https://avatars.githubusercontent.com/u/8071628?v=3","gravatar_id":"","url":"https://api.github.com/users/sstanovnik","html_url":"https://github.com/sstanovnik","followers_url":"https://api.github.com/users/sstanovnik/followers","following_url":"https://api.github.com/users/sstanovnik/following{/other_user}","gists_url":"https://api.github.com/users/sstanovnik/gists{/gist_id}","starred_url":"https://api.github.com/users/sstanovnik/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sstanovnik/subscriptions","organizations_url":"https://api.github.com/users/sstanovnik/orgs","repos_url":"https://api.github.com/users/sstanovnik/repos","events_url":"https://api.github.com/users/sstanovnik/events{/privacy}","received_events_url":"https://api.github.com/users/sstanovnik/received_events","type":"User","site_admin":false},"body":"I don't understand the `map()` part. ","created_at":"2016-08-11T07:59:53Z","updated_at":"2016-08-11T07:59:53Z","html_url":"https://github.com/biolab/orange3/pull/1347#discussion_r74381020","pull_request_url":"https://api.github.com/repos/biolab/orange3/pulls/1347","_links":{"self":{"href":"https://api.github.com/repos/biolab/orange3/pulls/comments/74381020"},"html":{"href":"https://github.com/biolab/orange3/pull/1347#discussion_r74381020"},"pull_request":{"href":"https://api.github.com/repos/biolab/orange3/pulls/1347"}}},"pull_request":{"url":"https://api.github.com/repos/biolab/orange3/pulls/1347","id":74257124,"html_url":"https://github.com/biolab/orange3/pull/1347","diff_url":"https://github.com/biolab/orange3/pull/1347.diff","patch_url":"https://github.com/biolab/orange3/pull/1347.patch","issue_url":"https://api.github.com/repos/biolab/orange3/issues/1347","number":1347,"state":"open","locked":false,"title":"[WIP] pandas migration","user":{"login":"sstanovnik","id":8071628,"avatar_url":"https://avatars.githubusercontent.com/u/8071628?v=3","gravatar_id":"","url":"https://api.github.com/users/sstanovnik","html_url":"https://github.com/sstanovnik","followers_url":"https://api.github.com/users/sstanovnik/followers","following_url":"https://api.github.com/users/sstanovnik/following{/other_user}","gists_url":"https://api.github.com/users/sstanovnik/gists{/gist_id}","starred_url":"https://api.github.com/users/sstanovnik/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sstanovnik/subscriptions","organizations_url":"https://api.github.com/users/sstanovnik/orgs","repos_url":"https://api.github.com/users/sstanovnik/repos","events_url":"https://api.github.com/users/sstanovnik/events{/privacy}","received_events_url":"https://api.github.com/users/sstanovnik/received_events","type":"User","site_admin":false},"body":"This is the giant ~~(well, WIP)~~ pull request for converting the existing architecture to `pandas`. \r\nSee [this wiki page](https://github.com/biolab/orange3/wiki/pandas-migration) and [this Google Sheet](https://docs.google.com/spreadsheets/d/1yAGeUYut71Imd1c0YGgqp-QgmwtCQZfm9fat_mHvGcQ/edit?usp=sharing) for comments on more general architectural decisions. \r\n\r\n~~So far, the majority of the additional new API is done, but few functions actually work and none have been tested. This will break until I start writing tests and modifying existing code. This currently still does not pass tests, but the Table by itself is completely functional, as well as some other learners. Now in progress: making sure everything works.~~\r\n\r\nOver 95 % of all base widgets now work! ~~Most unit tests pass, the ones that don't are tied to sparse matrices and SQL support. Up next: (re)adding sparse Table and SQL Table support.~~ All unit tests now pass ~~(not including widget tests)~~!\r\n\r\nMulti-type sparse support depends on pandas 0.19.0 (unreleased). That release will fix a lot of sparse issues (some were even fixed by me =D).\r\n\r\nTodos and progress, along with code examples, is monitored in the Gsheet. \r\n\r\nLinked pull requests: biolab/orange3-text#97","created_at":"2016-06-17T15:03:49Z","updated_at":"2016-08-11T07:59:53Z","closed_at":null,"merged_at":null,"merge_commit_sha":"75fdc4752c0615e957e720b67ef4881a8ebc21c0","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/biolab/orange3/pulls/1347/commits","review_comments_url":"https://api.github.com/repos/biolab/orange3/pulls/1347/comments","review_comment_url":"https://api.github.com/repos/biolab/orange3/pulls/comments{/number}","comments_url":"https://api.github.com/repos/biolab/orange3/issues/1347/comments","statuses_url":"https://api.github.com/repos/biolab/orange3/statuses/bb6a60e1b2671ffac42251f8a001b17cd3b37dd2","head":{"label":"sstanovnik:koalas","ref":"koalas","sha":"bb6a60e1b2671ffac42251f8a001b17cd3b37dd2","user":{"login":"sstanovnik","id":8071628,"avatar_url":"https://avatars.githubusercontent.com/u/8071628?v=3","gravatar_id":"","url":"https://api.github.com/users/sstanovnik","html_url":"https://github.com/sstanovnik","followers_url":"https://api.github.com/users/sstanovnik/followers","following_url":"https://api.github.com/users/sstanovnik/following{/other_user}","gists_url":"https://api.github.com/users/sstanovnik/gists{/gist_id}","starred_url":"https://api.github.com/users/sstanovnik/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sstanovnik/subscriptions","organizations_url":"https://api.github.com/users/sstanovnik/orgs","repos_url":"https://api.github.com/users/sstanovnik/repos","events_url":"https://api.github.com/users/sstanovnik/events{/privacy}","received_events_url":"https://api.github.com/users/sstanovnik/received_events","type":"User","site_admin":false},"repo":{"id":58224040,"name":"orange3","full_name":"sstanovnik/orange3","owner":{"login":"sstanovnik","id":8071628,"avatar_url":"https://avatars.githubusercontent.com/u/8071628?v=3","gravatar_id":"","url":"https://api.github.com/users/sstanovnik","html_url":"https://github.com/sstanovnik","followers_url":"https://api.github.com/users/sstanovnik/followers","following_url":"https://api.github.com/users/sstanovnik/following{/other_user}","gists_url":"https://api.github.com/users/sstanovnik/gists{/gist_id}","starred_url":"https://api.github.com/users/sstanovnik/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sstanovnik/subscriptions","organizations_url":"https://api.github.com/users/sstanovnik/orgs","repos_url":"https://api.github.com/users/sstanovnik/repos","events_url":"https://api.github.com/users/sstanovnik/events{/privacy}","received_events_url":"https://api.github.com/users/sstanovnik/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/sstanovnik/orange3","description":"Orange 3 data mining suite: http://orange.biolab.si ","fork":true,"url":"https://api.github.com/repos/sstanovnik/orange3","forks_url":"https://api.github.com/repos/sstanovnik/orange3/forks","keys_url":"https://api.github.com/repos/sstanovnik/orange3/keys{/key_id}","collaborators_url":"https://api.github.com/repos/sstanovnik/orange3/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/sstanovnik/orange3/teams","hooks_url":"https://api.github.com/repos/sstanovnik/orange3/hooks","issue_events_url":"https://api.github.com/repos/sstanovnik/orange3/issues/events{/number}","events_url":"https://api.github.com/repos/sstanovnik/orange3/events","assignees_url":"https://api.github.com/repos/sstanovnik/orange3/assignees{/user}","branches_url":"https://api.github.com/repos/sstanovnik/orange3/branches{/branch}","tags_url":"https://api.github.com/repos/sstanovnik/orange3/tags","blobs_url":"https://api.github.com/repos/sstanovnik/orange3/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/sstanovnik/orange3/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/sstanovnik/orange3/git/refs{/sha}","trees_url":"https://api.github.com/repos/sstanovnik/orange3/git/trees{/sha}","statuses_url":"https://api.github.com/repos/sstanovnik/orange3/statuses/{sha}","languages_url":"https://api.github.com/repos/sstanovnik/orange3/languages","stargazers_url":"https://api.github.com/repos/sstanovnik/orange3/stargazers","contributors_url":"https://api.github.com/repos/sstanovnik/orange3/contributors","subscribers_url":"https://api.github.com/repos/sstanovnik/orange3/subscribers","subscription_url":"https://api.github.com/repos/sstanovnik/orange3/subscription","commits_url":"https://api.github.com/repos/sstanovnik/orange3/commits{/sha}","git_commits_url":"https://api.github.com/repos/sstanovnik/orange3/git/commits{/sha}","comments_url":"https://api.github.com/repos/sstanovnik/orange3/comments{/number}","issue_comment_url":"https://api.github.com/repos/sstanovnik/orange3/issues/comments{/number}","contents_url":"https://api.github.com/repos/sstanovnik/orange3/contents/{+path}","compare_url":"https://api.github.com/repos/sstanovnik/orange3/compare/{base}...{head}","merges_url":"https://api.github.com/repos/sstanovnik/orange3/merges","archive_url":"https://api.github.com/repos/sstanovnik/orange3/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/sstanovnik/orange3/downloads","issues_url":"https://api.github.com/repos/sstanovnik/orange3/issues{/number}","pulls_url":"https://api.github.com/repos/sstanovnik/orange3/pulls{/number}","milestones_url":"https://api.github.com/repos/sstanovnik/orange3/milestones{/number}","notifications_url":"https://api.github.com/repos/sstanovnik/orange3/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/sstanovnik/orange3/labels{/name}","releases_url":"https://api.github.com/repos/sstanovnik/orange3/releases{/id}","deployments_url":"https://api.github.com/repos/sstanovnik/orange3/deployments","created_at":"2016-05-06T17:35:01Z","updated_at":"2016-05-06T17:35:04Z","pushed_at":"2016-08-10T10:07:33Z","git_url":"git://github.com/sstanovnik/orange3.git","ssh_url":"git@github.com:sstanovnik/orange3.git","clone_url":"https://github.com/sstanovnik/orange3.git","svn_url":"https://github.com/sstanovnik/orange3","homepage":"","size":38985,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"biolab:master","ref":"master","sha":"1ee248ab3693066dd07a43a55ea6b2f25a6d759a","user":{"login":"biolab","id":555189,"avatar_url":"https://avatars.githubusercontent.com/u/555189?v=3","gravatar_id":"","url":"https://api.github.com/users/biolab","html_url":"https://github.com/biolab","followers_url":"https://api.github.com/users/biolab/followers","following_url":"https://api.github.com/users/biolab/following{/other_user}","gists_url":"https://api.github.com/users/biolab/gists{/gist_id}","starred_url":"https://api.github.com/users/biolab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/biolab/subscriptions","organizations_url":"https://api.github.com/users/biolab/orgs","repos_url":"https://api.github.com/users/biolab/repos","events_url":"https://api.github.com/users/biolab/events{/privacy}","received_events_url":"https://api.github.com/users/biolab/received_events","type":"Organization","site_admin":false},"repo":{"id":8357227,"name":"orange3","full_name":"biolab/orange3","owner":{"login":"biolab","id":555189,"avatar_url":"https://avatars.githubusercontent.com/u/555189?v=3","gravatar_id":"","url":"https://api.github.com/users/biolab","html_url":"https://github.com/biolab","followers_url":"https://api.github.com/users/biolab/followers","following_url":"https://api.github.com/users/biolab/following{/other_user}","gists_url":"https://api.github.com/users/biolab/gists{/gist_id}","starred_url":"https://api.github.com/users/biolab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/biolab/subscriptions","organizations_url":"https://api.github.com/users/biolab/orgs","repos_url":"https://api.github.com/users/biolab/repos","events_url":"https://api.github.com/users/biolab/events{/privacy}","received_events_url":"https://api.github.com/users/biolab/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/biolab/orange3","description":"Orange 3 data mining suite: http://orange.biolab.si ","fork":false,"url":"https://api.github.com/repos/biolab/orange3","forks_url":"https://api.github.com/repos/biolab/orange3/forks","keys_url":"https://api.github.com/repos/biolab/orange3/keys{/key_id}","collaborators_url":"https://api.github.com/repos/biolab/orange3/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/biolab/orange3/teams","hooks_url":"https://api.github.com/repos/biolab/orange3/hooks","issue_events_url":"https://api.github.com/repos/biolab/orange3/issues/events{/number}","events_url":"https://api.github.com/repos/biolab/orange3/events","assignees_url":"https://api.github.com/repos/biolab/orange3/assignees{/user}","branches_url":"https://api.github.com/repos/biolab/orange3/branches{/branch}","tags_url":"https://api.github.com/repos/biolab/orange3/tags","blobs_url":"https://api.github.com/repos/biolab/orange3/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/biolab/orange3/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/biolab/orange3/git/refs{/sha}","trees_url":"https://api.github.com/repos/biolab/orange3/git/trees{/sha}","statuses_url":"https://api.github.com/repos/biolab/orange3/statuses/{sha}","languages_url":"https://api.github.com/repos/biolab/orange3/languages","stargazers_url":"https://api.github.com/repos/biolab/orange3/stargazers","contributors_url":"https://api.github.com/repos/biolab/orange3/contributors","subscribers_url":"https://api.github.com/repos/biolab/orange3/subscribers","subscription_url":"https://api.github.com/repos/biolab/orange3/subscription","commits_url":"https://api.github.com/repos/biolab/orange3/commits{/sha}","git_commits_url":"https://api.github.com/repos/biolab/orange3/git/commits{/sha}","comments_url":"https://api.github.com/repos/biolab/orange3/comments{/number}","issue_comment_url":"https://api.github.com/repos/biolab/orange3/issues/comments{/number}","contents_url":"https://api.github.com/repos/biolab/orange3/contents/{+path}","compare_url":"https://api.github.com/repos/biolab/orange3/compare/{base}...{head}","merges_url":"https://api.github.com/repos/biolab/orange3/merges","archive_url":"https://api.github.com/repos/biolab/orange3/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/biolab/orange3/downloads","issues_url":"https://api.github.com/repos/biolab/orange3/issues{/number}","pulls_url":"https://api.github.com/repos/biolab/orange3/pulls{/number}","milestones_url":"https://api.github.com/repos/biolab/orange3/milestones{/number}","notifications_url":"https://api.github.com/repos/biolab/orange3/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/biolab/orange3/labels{/name}","releases_url":"https://api.github.com/repos/biolab/orange3/releases{/id}","deployments_url":"https://api.github.com/repos/biolab/orange3/deployments","created_at":"2013-02-22T12:52:56Z","updated_at":"2016-08-10T17:30:33Z","pushed_at":"2016-08-10T11:31:40Z","git_url":"git://github.com/biolab/orange3.git","ssh_url":"git@github.com:biolab/orange3.git","clone_url":"https://github.com/biolab/orange3.git","svn_url":"https://github.com/biolab/orange3","homepage":"","size":94121,"stargazers_count":452,"watchers_count":452,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":150,"mirror_url":null,"open_issues_count":49,"forks":150,"open_issues":49,"watchers":452,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/biolab/orange3/pulls/1347"},"html":{"href":"https://github.com/biolab/orange3/pull/1347"},"issue":{"href":"https://api.github.com/repos/biolab/orange3/issues/1347"},"comments":{"href":"https://api.github.com/repos/biolab/orange3/issues/1347/comments"},"review_comments":{"href":"https://api.github.com/repos/biolab/orange3/pulls/1347/comments"},"review_comment":{"href":"https://api.github.com/repos/biolab/orange3/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/biolab/orange3/pulls/1347/commits"},"statuses":{"href":"https://api.github.com/repos/biolab/orange3/statuses/bb6a60e1b2671ffac42251f8a001b17cd3b37dd2"}}}}
{"actor":{"display_login":"sstanovnik"}}