{"action":"created","comment":{"url":"https://api.github.com/repos/biolab/orange3/pulls/comments/57586665","id":57586665,"diff_hunk":"@@ -351,89 +381,17 @@ def __init__(self, data, learners, store_data=False, store_models=False,\n         super().__init__(data, len(learners), store_data=store_data,\n                          store_models=store_models, preprocessor=preprocessor,\n                          callback=callback)\n-        domain = data.domain\n-        X = data.X.copy()\n-        Y = data._Y.copy()\n-        metas = data.metas.copy()\n-\n-        teX, trX = X[:1], X[1:]\n-        teY, trY = Y[:1], Y[1:]\n-        te_metas, tr_metas = metas[:1], metas[1:]\n-        if data.has_weights():\n-            W = data.W.copy()\n-            teW, trW = W[:1], W[1:]\n-        else:\n-            W = teW = trW = None\n+        self.learners = learners\n \n-        self.row_indices = np.arange(len(data))\n-        if self.store_models:\n-            self.models = []\n-        self.actual = Y.flatten()\n-        nmethods = len(learners)\n-        n_callbacks = nmethods * len(data)\n-        for test_idx in self.row_indices:\n-            X[[0, test_idx]] = X[[test_idx, 0]]\n-            Y[[0, test_idx]] = Y[[test_idx, 0]]\n-            metas[[0, test_idx]] = metas[[test_idx, 0]]\n-            if W:\n-                W[[0, test_idx]] = W[[test_idx, 0]]\n-            test_data = Table.from_numpy(domain, teX, teY, te_metas, teW)\n-            train_data = Table.from_numpy(domain, trX, trY, tr_metas, trW)\n-            if self.preprocessor is not None:\n-                train_data = self.preprocessor(train_data)\n-            if self.store_models:\n-                fold_models = [None] * nmethods\n-                self.models.append(fold_models)\n-            for i, learner in enumerate(learners):\n-                model = self.train_if_succ(i, learner, train_data)\n-                self.call_callback((test_idx * nmethods + i) / n_callbacks)\n-                if not model:\n-                    continue\n-                if self.store_models:\n-                    fold_models[i] = model\n-                if data.domain.has_discrete_class:\n-                    values, probs = model(test_data, model.ValueProbs)\n-                    self.predicted[i][test_idx] = values\n-                    self.probabilities[i][test_idx, :] = probs\n-                elif data.domain.has_continuous_class:\n-                    values = model(test_data, model.Value)\n-                    self.predicted[i][test_idx] = values\n-        self.call_callback(1)\n+        self.indices = skl_cross_validation.LeaveOneOut(len(data))\n \n+        if self.preprocessor is not None:\n+            data = self.preprocessor(data)","path":"Orange/evaluation/testing.py","position":220,"original_position":220,"commit_id":"5271d19e3eafa5e3a8458918d5fd84d807fce2a4","original_commit_id":"5271d19e3eafa5e3a8458918d5fd84d807fce2a4","user":{"login":"s-alexey","id":9372243,"avatar_url":"https://avatars.githubusercontent.com/u/9372243?v=3","gravatar_id":"","url":"https://api.github.com/users/s-alexey","html_url":"https://github.com/s-alexey","followers_url":"https://api.github.com/users/s-alexey/followers","following_url":"https://api.github.com/users/s-alexey/following{/other_user}","gists_url":"https://api.github.com/users/s-alexey/gists{/gist_id}","starred_url":"https://api.github.com/users/s-alexey/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s-alexey/subscriptions","organizations_url":"https://api.github.com/users/s-alexey/orgs","repos_url":"https://api.github.com/users/s-alexey/repos","events_url":"https://api.github.com/users/s-alexey/events{/privacy}","received_events_url":"https://api.github.com/users/s-alexey/received_events","type":"User","site_admin":false},"body":"I can change it.\r\nBut, actually, what kind of preprocessing this function usually does? If it can do something like subtracting mean value then `preprocess(test)` just turns values into zeros. I was a little confused because of an absence of `fit` and `transform` methods.\r\n","created_at":"2016-03-28T15:45:57Z","updated_at":"2016-03-28T15:45:57Z","html_url":"https://github.com/biolab/orange3/pull/1114#discussion_r57586665","pull_request_url":"https://api.github.com/repos/biolab/orange3/pulls/1114","_links":{"self":{"href":"https://api.github.com/repos/biolab/orange3/pulls/comments/57586665"},"html":{"href":"https://github.com/biolab/orange3/pull/1114#discussion_r57586665"},"pull_request":{"href":"https://api.github.com/repos/biolab/orange3/pulls/1114"}}},"pull_request":{"url":"https://api.github.com/repos/biolab/orange3/pulls/1114","id":62743769,"html_url":"https://github.com/biolab/orange3/pull/1114","diff_url":"https://github.com/biolab/orange3/pull/1114.diff","patch_url":"https://github.com/biolab/orange3/pull/1114.patch","issue_url":"https://api.github.com/repos/biolab/orange3/issues/1114","number":1114,"state":"open","locked":false,"title":"[WIP] Refactor evaluation.testing module","user":{"login":"s-alexey","id":9372243,"avatar_url":"https://avatars.githubusercontent.com/u/9372243?v=3","gravatar_id":"","url":"https://api.github.com/users/s-alexey","html_url":"https://github.com/s-alexey","followers_url":"https://api.github.com/users/s-alexey/followers","following_url":"https://api.github.com/users/s-alexey/following{/other_user}","gists_url":"https://api.github.com/users/s-alexey/gists{/gist_id}","starred_url":"https://api.github.com/users/s-alexey/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s-alexey/subscriptions","organizations_url":"https://api.github.com/users/s-alexey/orgs","repos_url":"https://api.github.com/users/s-alexey/repos","events_url":"https://api.github.com/users/s-alexey/events{/privacy}","received_events_url":"https://api.github.com/users/s-alexey/received_events","type":"User","site_admin":false},"body":"Working on  #1102 I have found that there are something that can be improved:\r\n\r\n- inherit TestOnTrain from TestOnTest.\r\n- Make the main loop body in `__init__` method of each method as a separated method (`fit(data)`).\r\n- Some settings/data raises errors in sklearn functions (for example, when test set is too small). Possibly, we need to add 'check(data)' method.\r\n- Use LeaveOneOut from sklearn.\r\n- `sample` function is used only in tests.\r\n- remove preprocessor per sample calls.\r\n\r\nWork in progress, but I want your suggestions.","created_at":"2016-03-14T12:59:53Z","updated_at":"2016-03-28T15:45:57Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9555bdb1b5e6984adeb23efac84946bf2ce8985a","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/biolab/orange3/pulls/1114/commits","review_comments_url":"https://api.github.com/repos/biolab/orange3/pulls/1114/comments","review_comment_url":"https://api.github.com/repos/biolab/orange3/pulls/comments{/number}","comments_url":"https://api.github.com/repos/biolab/orange3/issues/1114/comments","statuses_url":"https://api.github.com/repos/biolab/orange3/statuses/5271d19e3eafa5e3a8458918d5fd84d807fce2a4","head":{"label":"s-alexey:testing-ref","ref":"testing-ref","sha":"5271d19e3eafa5e3a8458918d5fd84d807fce2a4","user":{"login":"s-alexey","id":9372243,"avatar_url":"https://avatars.githubusercontent.com/u/9372243?v=3","gravatar_id":"","url":"https://api.github.com/users/s-alexey","html_url":"https://github.com/s-alexey","followers_url":"https://api.github.com/users/s-alexey/followers","following_url":"https://api.github.com/users/s-alexey/following{/other_user}","gists_url":"https://api.github.com/users/s-alexey/gists{/gist_id}","starred_url":"https://api.github.com/users/s-alexey/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s-alexey/subscriptions","organizations_url":"https://api.github.com/users/s-alexey/orgs","repos_url":"https://api.github.com/users/s-alexey/repos","events_url":"https://api.github.com/users/s-alexey/events{/privacy}","received_events_url":"https://api.github.com/users/s-alexey/received_events","type":"User","site_admin":false},"repo":{"id":53073917,"name":"orange3","full_name":"s-alexey/orange3","owner":{"login":"s-alexey","id":9372243,"avatar_url":"https://avatars.githubusercontent.com/u/9372243?v=3","gravatar_id":"","url":"https://api.github.com/users/s-alexey","html_url":"https://github.com/s-alexey","followers_url":"https://api.github.com/users/s-alexey/followers","following_url":"https://api.github.com/users/s-alexey/following{/other_user}","gists_url":"https://api.github.com/users/s-alexey/gists{/gist_id}","starred_url":"https://api.github.com/users/s-alexey/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s-alexey/subscriptions","organizations_url":"https://api.github.com/users/s-alexey/orgs","repos_url":"https://api.github.com/users/s-alexey/repos","events_url":"https://api.github.com/users/s-alexey/events{/privacy}","received_events_url":"https://api.github.com/users/s-alexey/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/s-alexey/orange3","description":"Orange 3 data mining suite: http://orange.biolab.si ","fork":true,"url":"https://api.github.com/repos/s-alexey/orange3","forks_url":"https://api.github.com/repos/s-alexey/orange3/forks","keys_url":"https://api.github.com/repos/s-alexey/orange3/keys{/key_id}","collaborators_url":"https://api.github.com/repos/s-alexey/orange3/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/s-alexey/orange3/teams","hooks_url":"https://api.github.com/repos/s-alexey/orange3/hooks","issue_events_url":"https://api.github.com/repos/s-alexey/orange3/issues/events{/number}","events_url":"https://api.github.com/repos/s-alexey/orange3/events","assignees_url":"https://api.github.com/repos/s-alexey/orange3/assignees{/user}","branches_url":"https://api.github.com/repos/s-alexey/orange3/branches{/branch}","tags_url":"https://api.github.com/repos/s-alexey/orange3/tags","blobs_url":"https://api.github.com/repos/s-alexey/orange3/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/s-alexey/orange3/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/s-alexey/orange3/git/refs{/sha}","trees_url":"https://api.github.com/repos/s-alexey/orange3/git/trees{/sha}","statuses_url":"https://api.github.com/repos/s-alexey/orange3/statuses/{sha}","languages_url":"https://api.github.com/repos/s-alexey/orange3/languages","stargazers_url":"https://api.github.com/repos/s-alexey/orange3/stargazers","contributors_url":"https://api.github.com/repos/s-alexey/orange3/contributors","subscribers_url":"https://api.github.com/repos/s-alexey/orange3/subscribers","subscription_url":"https://api.github.com/repos/s-alexey/orange3/subscription","commits_url":"https://api.github.com/repos/s-alexey/orange3/commits{/sha}","git_commits_url":"https://api.github.com/repos/s-alexey/orange3/git/commits{/sha}","comments_url":"https://api.github.com/repos/s-alexey/orange3/comments{/number}","issue_comment_url":"https://api.github.com/repos/s-alexey/orange3/issues/comments{/number}","contents_url":"https://api.github.com/repos/s-alexey/orange3/contents/{+path}","compare_url":"https://api.github.com/repos/s-alexey/orange3/compare/{base}...{head}","merges_url":"https://api.github.com/repos/s-alexey/orange3/merges","archive_url":"https://api.github.com/repos/s-alexey/orange3/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/s-alexey/orange3/downloads","issues_url":"https://api.github.com/repos/s-alexey/orange3/issues{/number}","pulls_url":"https://api.github.com/repos/s-alexey/orange3/pulls{/number}","milestones_url":"https://api.github.com/repos/s-alexey/orange3/milestones{/number}","notifications_url":"https://api.github.com/repos/s-alexey/orange3/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/s-alexey/orange3/labels{/name}","releases_url":"https://api.github.com/repos/s-alexey/orange3/releases{/id}","deployments_url":"https://api.github.com/repos/s-alexey/orange3/deployments","created_at":"2016-03-03T18:36:52Z","updated_at":"2016-03-03T18:36:55Z","pushed_at":"2016-03-18T21:38:00Z","git_url":"git://github.com/s-alexey/orange3.git","ssh_url":"git@github.com:s-alexey/orange3.git","clone_url":"https://github.com/s-alexey/orange3.git","svn_url":"https://github.com/s-alexey/orange3","homepage":"","size":36559,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"biolab:master","ref":"master","sha":"d3c8ec9c27374f12096544b3f7ca3bc7d7149843","user":{"login":"biolab","id":555189,"avatar_url":"https://avatars.githubusercontent.com/u/555189?v=3","gravatar_id":"","url":"https://api.github.com/users/biolab","html_url":"https://github.com/biolab","followers_url":"https://api.github.com/users/biolab/followers","following_url":"https://api.github.com/users/biolab/following{/other_user}","gists_url":"https://api.github.com/users/biolab/gists{/gist_id}","starred_url":"https://api.github.com/users/biolab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/biolab/subscriptions","organizations_url":"https://api.github.com/users/biolab/orgs","repos_url":"https://api.github.com/users/biolab/repos","events_url":"https://api.github.com/users/biolab/events{/privacy}","received_events_url":"https://api.github.com/users/biolab/received_events","type":"Organization","site_admin":false},"repo":{"id":8357227,"name":"orange3","full_name":"biolab/orange3","owner":{"login":"biolab","id":555189,"avatar_url":"https://avatars.githubusercontent.com/u/555189?v=3","gravatar_id":"","url":"https://api.github.com/users/biolab","html_url":"https://github.com/biolab","followers_url":"https://api.github.com/users/biolab/followers","following_url":"https://api.github.com/users/biolab/following{/other_user}","gists_url":"https://api.github.com/users/biolab/gists{/gist_id}","starred_url":"https://api.github.com/users/biolab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/biolab/subscriptions","organizations_url":"https://api.github.com/users/biolab/orgs","repos_url":"https://api.github.com/users/biolab/repos","events_url":"https://api.github.com/users/biolab/events{/privacy}","received_events_url":"https://api.github.com/users/biolab/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/biolab/orange3","description":"Orange 3 data mining suite: http://orange.biolab.si ","fork":false,"url":"https://api.github.com/repos/biolab/orange3","forks_url":"https://api.github.com/repos/biolab/orange3/forks","keys_url":"https://api.github.com/repos/biolab/orange3/keys{/key_id}","collaborators_url":"https://api.github.com/repos/biolab/orange3/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/biolab/orange3/teams","hooks_url":"https://api.github.com/repos/biolab/orange3/hooks","issue_events_url":"https://api.github.com/repos/biolab/orange3/issues/events{/number}","events_url":"https://api.github.com/repos/biolab/orange3/events","assignees_url":"https://api.github.com/repos/biolab/orange3/assignees{/user}","branches_url":"https://api.github.com/repos/biolab/orange3/branches{/branch}","tags_url":"https://api.github.com/repos/biolab/orange3/tags","blobs_url":"https://api.github.com/repos/biolab/orange3/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/biolab/orange3/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/biolab/orange3/git/refs{/sha}","trees_url":"https://api.github.com/repos/biolab/orange3/git/trees{/sha}","statuses_url":"https://api.github.com/repos/biolab/orange3/statuses/{sha}","languages_url":"https://api.github.com/repos/biolab/orange3/languages","stargazers_url":"https://api.github.com/repos/biolab/orange3/stargazers","contributors_url":"https://api.github.com/repos/biolab/orange3/contributors","subscribers_url":"https://api.github.com/repos/biolab/orange3/subscribers","subscription_url":"https://api.github.com/repos/biolab/orange3/subscription","commits_url":"https://api.github.com/repos/biolab/orange3/commits{/sha}","git_commits_url":"https://api.github.com/repos/biolab/orange3/git/commits{/sha}","comments_url":"https://api.github.com/repos/biolab/orange3/comments{/number}","issue_comment_url":"https://api.github.com/repos/biolab/orange3/issues/comments{/number}","contents_url":"https://api.github.com/repos/biolab/orange3/contents/{+path}","compare_url":"https://api.github.com/repos/biolab/orange3/compare/{base}...{head}","merges_url":"https://api.github.com/repos/biolab/orange3/merges","archive_url":"https://api.github.com/repos/biolab/orange3/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/biolab/orange3/downloads","issues_url":"https://api.github.com/repos/biolab/orange3/issues{/number}","pulls_url":"https://api.github.com/repos/biolab/orange3/pulls{/number}","milestones_url":"https://api.github.com/repos/biolab/orange3/milestones{/number}","notifications_url":"https://api.github.com/repos/biolab/orange3/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/biolab/orange3/labels{/name}","releases_url":"https://api.github.com/repos/biolab/orange3/releases{/id}","deployments_url":"https://api.github.com/repos/biolab/orange3/deployments","created_at":"2013-02-22T12:52:56Z","updated_at":"2016-03-26T17:22:26Z","pushed_at":"2016-03-28T14:54:21Z","git_url":"git://github.com/biolab/orange3.git","ssh_url":"git@github.com:biolab/orange3.git","clone_url":"https://github.com/biolab/orange3.git","svn_url":"https://github.com/biolab/orange3","homepage":"","size":75817,"stargazers_count":237,"watchers_count":237,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":110,"mirror_url":null,"open_issues_count":68,"forks":110,"open_issues":68,"watchers":237,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/biolab/orange3/pulls/1114"},"html":{"href":"https://github.com/biolab/orange3/pull/1114"},"issue":{"href":"https://api.github.com/repos/biolab/orange3/issues/1114"},"comments":{"href":"https://api.github.com/repos/biolab/orange3/issues/1114/comments"},"review_comments":{"href":"https://api.github.com/repos/biolab/orange3/pulls/1114/comments"},"review_comment":{"href":"https://api.github.com/repos/biolab/orange3/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/biolab/orange3/pulls/1114/commits"},"statuses":{"href":"https://api.github.com/repos/biolab/orange3/statuses/5271d19e3eafa5e3a8458918d5fd84d807fce2a4"}}}}