{"action":"created","comment":{"url":"https://api.github.com/repos/hyperspy/hyperspy/pulls/comments/68261618","id":68261618,"diff_hunk":"@@ -0,0 +1,190 @@\n+# -*- coding: utf-8 -*-\n+# This file is a transcription of a MATLAB code obtained from the\n+# following research paper:\n+#   Jiashi Feng, Huan Xu and Shuicheng Yuan, \"Online Robust PCA via\n+#   Stochastic Optimization\", Advances in Neural Information Processing\n+#   Systems 26, (2013), pp. 404-412.\n+#\n+# Copyright 2013 Jiashi Feng\n+# Copyright 2016 The HyperSpy developers\n+#\n+# This file is part of  HyperSpy.\n+#\n+#  HyperSpy is free software: you can redistribute it and/or modify\n+# it under the terms of the GNU General Public License as published by\n+# the Free Software Foundation, either version 3 of the License, or\n+# (at your option) any later version.\n+#\n+#  HyperSpy is distributed in the hope that it will be useful,\n+# but WITHOUT ANY WARRANTY; without even the implied warranty of\n+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n+# GNU General Public License for more details.\n+#\n+# You should have received a copy of the GNU General Public License\n+# along with  HyperSpy.  If not, see <http://www.gnu.org/licenses/>.\n+\n+import logging\n+\n+import numpy as np\n+import scipy.linalg\n+\n+from hyperspy.misc.machine_learning.import_sklearn import (\n+    fast_svd, sklearn_installed)\n+\n+_logger = logging.getLogger(__name__)\n+\n+def _solveproj(z, X, I, lambda2):\n+    m, n = X.shape\n+    s = np.zeros(m)\n+    x = np.zeros(n)\n+    converged = False\n+    maxiter = 1e9\n+    iter = 0\n+\n+    ddt = np.dot(scipy.linalg.inv(np.dot(X.T, X) + I), X.T)\n+\n+    while converged is False:\n+        iter += 1\n+        xtmp = x\n+        x = np.dot(ddt, (z - s))\n+        stmp = s\n+        s = np.maximum(z - np.dot(X, x) - lambda2, 0.0)\n+        stopx = np.sqrt(np.dot(x - xtmp, (x - xtmp).conj()))\n+        stops = np.sqrt(np.dot(s - stmp, (s - stmp).conj()))\n+        stop = max(stopx, stops) / m\n+        if stop < 1e-5 or iter > maxiter:\n+            converged = True\n+\n+    return x, s\n+\n+def _updatecol(X, A, B, I):\n+    tmp, n = X.shape\n+    L = X\n+    A = A + I\n+\n+    for i in range(n):\n+        b = B[:, i]\n+        x = X[:, i]\n+        a = A[:, i]\n+        temp = (b - np.dot(X, a)) / A[i, i] + x\n+        L[:, i] = temp / max(np.sqrt(np.dot(temp, temp.conj())), 1)\n+\n+    return L\n+\n+def orpca(X, rank, fast=False, lambda1=None,\n+          lambda2=None, method=None,\n+          mask=None, seed=None):\n+    \"\"\"\n+    This function performs Online Robust PCA with\n+    with missing or corrupted data.\n+\n+    Parameters\n+    ----------\n+    X : numpy array\n+        is the [m x n] matrix of observations.\n+    rank : int\n+        The model dimensionality.\n+    lambda1 : float\n+        Nuclear norm regularization parameter.\n+    lambda2 : float","path":"hyperspy/learn/rpca.py","position":89,"original_position":89,"commit_id":"dafc098664cc3b1c9ee731a4a344039e35e2cf7b","original_commit_id":"dafc098664cc3b1c9ee731a4a344039e35e2cf7b","user":{"login":"francisco-dlp","id":989719,"avatar_url":"https://avatars.githubusercontent.com/u/989719?v=3","gravatar_id":"","url":"https://api.github.com/users/francisco-dlp","html_url":"https://github.com/francisco-dlp","followers_url":"https://api.github.com/users/francisco-dlp/followers","following_url":"https://api.github.com/users/francisco-dlp/following{/other_user}","gists_url":"https://api.github.com/users/francisco-dlp/gists{/gist_id}","starred_url":"https://api.github.com/users/francisco-dlp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/francisco-dlp/subscriptions","organizations_url":"https://api.github.com/users/francisco-dlp/orgs","repos_url":"https://api.github.com/users/francisco-dlp/repos","events_url":"https://api.github.com/users/francisco-dlp/events{/privacy}","received_events_url":"https://api.github.com/users/francisco-dlp/received_events","type":"User","site_admin":false},"body":"Same as above.","created_at":"2016-06-23T15:54:45Z","updated_at":"2016-06-23T15:54:45Z","html_url":"https://github.com/hyperspy/hyperspy/pull/1071#discussion_r68261618","pull_request_url":"https://api.github.com/repos/hyperspy/hyperspy/pulls/1071","_links":{"self":{"href":"https://api.github.com/repos/hyperspy/hyperspy/pulls/comments/68261618"},"html":{"href":"https://github.com/hyperspy/hyperspy/pull/1071#discussion_r68261618"},"pull_request":{"href":"https://api.github.com/repos/hyperspy/hyperspy/pulls/1071"}}},"pull_request":{"url":"https://api.github.com/repos/hyperspy/hyperspy/pulls/1071","id":73768861,"html_url":"https://github.com/hyperspy/hyperspy/pull/1071","diff_url":"https://github.com/hyperspy/hyperspy/pull/1071.diff","patch_url":"https://github.com/hyperspy/hyperspy/pull/1071.patch","issue_url":"https://api.github.com/repos/hyperspy/hyperspy/issues/1071","number":1071,"state":"open","locked":false,"title":"ENH Online Robust PCA","user":{"login":"tjof2","id":6626247,"avatar_url":"https://avatars.githubusercontent.com/u/6626247?v=3","gravatar_id":"","url":"https://api.github.com/users/tjof2","html_url":"https://github.com/tjof2","followers_url":"https://api.github.com/users/tjof2/followers","following_url":"https://api.github.com/users/tjof2/following{/other_user}","gists_url":"https://api.github.com/users/tjof2/gists{/gist_id}","starred_url":"https://api.github.com/users/tjof2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tjof2/subscriptions","organizations_url":"https://api.github.com/users/tjof2/orgs","repos_url":"https://api.github.com/users/tjof2/repos","events_url":"https://api.github.com/users/tjof2/events{/privacy}","received_events_url":"https://api.github.com/users/tjof2/received_events","type":"User","site_admin":false},"body":"Robust PCA seems to be making its way into scikit-learn, slowly... (see #972), but with one of the slow, iterative, memory-intensive SVT algorithms. However, I've been using this example of \"online\" robust PCA for a while now for outlier detection and working with missing data, which is much less memory-intensive and actually isn't too slow.\r\n\r\nSince OR-PCA actually returns a low-rank component **L** and a sparse error component **S** (where the observations **X** = **L** + **S**), to fit this into the current HyperSpy decomposition framework, I carry out a subsequent SVD on **L** to return factors and loadings.\r\n\r\nAlthough I've not currently implemented it in a way that allows for it, you can actually run OR-PCA in a truly online manner, i.e. update the estimated factors and loadings with each new sample that arrives, without having to re-run over the whole dataset. However, I think this quickly becomes complicated in the current HyperSpy machine learning framework, so consider this implementation more aimed at outlier detection.\r\n\r\nYou can see an example here: https://github.com/tjof2/hyperspy/blob/my-demos/examples/Online%20Robust%20PCA.ipynb\r\n\r\nFeedback welcome, this is my first \"big\" bit of Python!\r\n\r\n- [x] Documentation\r\n- [x] Write simple tests\r\n- [x] Option to feed in an initial mask for the sparse component **S**. \r\n\r\nThe last point is useful for sub-sampled STEM images, as it turns into a matrix completion problem.\r\n\r\n\r\n\r\n","created_at":"2016-06-14T16:09:24Z","updated_at":"2016-06-23T15:54:45Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6f91ddc4195cf641af01486333b9ed21c4c5fd98","assignee":null,"milestone":{"url":"https://api.github.com/repos/hyperspy/hyperspy/milestones/20","html_url":"https://github.com/hyperspy/hyperspy/milestones/RELEASE_next_minor","labels_url":"https://api.github.com/repos/hyperspy/hyperspy/milestones/20/labels","id":1827002,"number":20,"title":"RELEASE_next_minor","description":"Next release that adds functionality in a backwards-compatible manner.","creator":{"login":"francisco-dlp","id":989719,"avatar_url":"https://avatars.githubusercontent.com/u/989719?v=3","gravatar_id":"","url":"https://api.github.com/users/francisco-dlp","html_url":"https://github.com/francisco-dlp","followers_url":"https://api.github.com/users/francisco-dlp/followers","following_url":"https://api.github.com/users/francisco-dlp/following{/other_user}","gists_url":"https://api.github.com/users/francisco-dlp/gists{/gist_id}","starred_url":"https://api.github.com/users/francisco-dlp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/francisco-dlp/subscriptions","organizations_url":"https://api.github.com/users/francisco-dlp/orgs","repos_url":"https://api.github.com/users/francisco-dlp/repos","events_url":"https://api.github.com/users/francisco-dlp/events{/privacy}","received_events_url":"https://api.github.com/users/francisco-dlp/received_events","type":"User","site_admin":false},"open_issues":15,"closed_issues":1,"state":"open","created_at":"2016-06-14T16:25:26Z","updated_at":"2016-06-22T16:32:27Z","due_on":null,"closed_at":null},"commits_url":"https://api.github.com/repos/hyperspy/hyperspy/pulls/1071/commits","review_comments_url":"https://api.github.com/repos/hyperspy/hyperspy/pulls/1071/comments","review_comment_url":"https://api.github.com/repos/hyperspy/hyperspy/pulls/comments{/number}","comments_url":"https://api.github.com/repos/hyperspy/hyperspy/issues/1071/comments","statuses_url":"https://api.github.com/repos/hyperspy/hyperspy/statuses/dafc098664cc3b1c9ee731a4a344039e35e2cf7b","head":{"label":"tjof2:enh-robust-pca","ref":"enh-robust-pca","sha":"dafc098664cc3b1c9ee731a4a344039e35e2cf7b","user":{"login":"tjof2","id":6626247,"avatar_url":"https://avatars.githubusercontent.com/u/6626247?v=3","gravatar_id":"","url":"https://api.github.com/users/tjof2","html_url":"https://github.com/tjof2","followers_url":"https://api.github.com/users/tjof2/followers","following_url":"https://api.github.com/users/tjof2/following{/other_user}","gists_url":"https://api.github.com/users/tjof2/gists{/gist_id}","starred_url":"https://api.github.com/users/tjof2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tjof2/subscriptions","organizations_url":"https://api.github.com/users/tjof2/orgs","repos_url":"https://api.github.com/users/tjof2/repos","events_url":"https://api.github.com/users/tjof2/events{/privacy}","received_events_url":"https://api.github.com/users/tjof2/received_events","type":"User","site_admin":false},"repo":{"id":61048580,"name":"hyperspy","full_name":"tjof2/hyperspy","owner":{"login":"tjof2","id":6626247,"avatar_url":"https://avatars.githubusercontent.com/u/6626247?v=3","gravatar_id":"","url":"https://api.github.com/users/tjof2","html_url":"https://github.com/tjof2","followers_url":"https://api.github.com/users/tjof2/followers","following_url":"https://api.github.com/users/tjof2/following{/other_user}","gists_url":"https://api.github.com/users/tjof2/gists{/gist_id}","starred_url":"https://api.github.com/users/tjof2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tjof2/subscriptions","organizations_url":"https://api.github.com/users/tjof2/orgs","repos_url":"https://api.github.com/users/tjof2/repos","events_url":"https://api.github.com/users/tjof2/events{/privacy}","received_events_url":"https://api.github.com/users/tjof2/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/tjof2/hyperspy","description":"Multidimensional data analysis","fork":true,"url":"https://api.github.com/repos/tjof2/hyperspy","forks_url":"https://api.github.com/repos/tjof2/hyperspy/forks","keys_url":"https://api.github.com/repos/tjof2/hyperspy/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tjof2/hyperspy/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tjof2/hyperspy/teams","hooks_url":"https://api.github.com/repos/tjof2/hyperspy/hooks","issue_events_url":"https://api.github.com/repos/tjof2/hyperspy/issues/events{/number}","events_url":"https://api.github.com/repos/tjof2/hyperspy/events","assignees_url":"https://api.github.com/repos/tjof2/hyperspy/assignees{/user}","branches_url":"https://api.github.com/repos/tjof2/hyperspy/branches{/branch}","tags_url":"https://api.github.com/repos/tjof2/hyperspy/tags","blobs_url":"https://api.github.com/repos/tjof2/hyperspy/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tjof2/hyperspy/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tjof2/hyperspy/git/refs{/sha}","trees_url":"https://api.github.com/repos/tjof2/hyperspy/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tjof2/hyperspy/statuses/{sha}","languages_url":"https://api.github.com/repos/tjof2/hyperspy/languages","stargazers_url":"https://api.github.com/repos/tjof2/hyperspy/stargazers","contributors_url":"https://api.github.com/repos/tjof2/hyperspy/contributors","subscribers_url":"https://api.github.com/repos/tjof2/hyperspy/subscribers","subscription_url":"https://api.github.com/repos/tjof2/hyperspy/subscription","commits_url":"https://api.github.com/repos/tjof2/hyperspy/commits{/sha}","git_commits_url":"https://api.github.com/repos/tjof2/hyperspy/git/commits{/sha}","comments_url":"https://api.github.com/repos/tjof2/hyperspy/comments{/number}","issue_comment_url":"https://api.github.com/repos/tjof2/hyperspy/issues/comments{/number}","contents_url":"https://api.github.com/repos/tjof2/hyperspy/contents/{+path}","compare_url":"https://api.github.com/repos/tjof2/hyperspy/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tjof2/hyperspy/merges","archive_url":"https://api.github.com/repos/tjof2/hyperspy/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tjof2/hyperspy/downloads","issues_url":"https://api.github.com/repos/tjof2/hyperspy/issues{/number}","pulls_url":"https://api.github.com/repos/tjof2/hyperspy/pulls{/number}","milestones_url":"https://api.github.com/repos/tjof2/hyperspy/milestones{/number}","notifications_url":"https://api.github.com/repos/tjof2/hyperspy/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tjof2/hyperspy/labels{/name}","releases_url":"https://api.github.com/repos/tjof2/hyperspy/releases{/id}","deployments_url":"https://api.github.com/repos/tjof2/hyperspy/deployments","created_at":"2016-06-13T15:42:00Z","updated_at":"2016-06-13T15:42:04Z","pushed_at":"2016-06-23T07:14:14Z","git_url":"git://github.com/tjof2/hyperspy.git","ssh_url":"git@github.com:tjof2/hyperspy.git","clone_url":"https://github.com/tjof2/hyperspy.git","svn_url":"https://github.com/tjof2/hyperspy","homepage":"http://www.hyperspy.org","size":77458,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"hyperspy:master","ref":"master","sha":"e060c6a33da610d52473baeb5e58f45b320b3d6d","user":{"login":"hyperspy","id":989756,"avatar_url":"https://avatars.githubusercontent.com/u/989756?v=3","gravatar_id":"","url":"https://api.github.com/users/hyperspy","html_url":"https://github.com/hyperspy","followers_url":"https://api.github.com/users/hyperspy/followers","following_url":"https://api.github.com/users/hyperspy/following{/other_user}","gists_url":"https://api.github.com/users/hyperspy/gists{/gist_id}","starred_url":"https://api.github.com/users/hyperspy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hyperspy/subscriptions","organizations_url":"https://api.github.com/users/hyperspy/orgs","repos_url":"https://api.github.com/users/hyperspy/repos","events_url":"https://api.github.com/users/hyperspy/events{/privacy}","received_events_url":"https://api.github.com/users/hyperspy/received_events","type":"Organization","site_admin":false},"repo":{"id":2233998,"name":"hyperspy","full_name":"hyperspy/hyperspy","owner":{"login":"hyperspy","id":989756,"avatar_url":"https://avatars.githubusercontent.com/u/989756?v=3","gravatar_id":"","url":"https://api.github.com/users/hyperspy","html_url":"https://github.com/hyperspy","followers_url":"https://api.github.com/users/hyperspy/followers","following_url":"https://api.github.com/users/hyperspy/following{/other_user}","gists_url":"https://api.github.com/users/hyperspy/gists{/gist_id}","starred_url":"https://api.github.com/users/hyperspy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hyperspy/subscriptions","organizations_url":"https://api.github.com/users/hyperspy/orgs","repos_url":"https://api.github.com/users/hyperspy/repos","events_url":"https://api.github.com/users/hyperspy/events{/privacy}","received_events_url":"https://api.github.com/users/hyperspy/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/hyperspy/hyperspy","description":"Multidimensional data analysis","fork":false,"url":"https://api.github.com/repos/hyperspy/hyperspy","forks_url":"https://api.github.com/repos/hyperspy/hyperspy/forks","keys_url":"https://api.github.com/repos/hyperspy/hyperspy/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hyperspy/hyperspy/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hyperspy/hyperspy/teams","hooks_url":"https://api.github.com/repos/hyperspy/hyperspy/hooks","issue_events_url":"https://api.github.com/repos/hyperspy/hyperspy/issues/events{/number}","events_url":"https://api.github.com/repos/hyperspy/hyperspy/events","assignees_url":"https://api.github.com/repos/hyperspy/hyperspy/assignees{/user}","branches_url":"https://api.github.com/repos/hyperspy/hyperspy/branches{/branch}","tags_url":"https://api.github.com/repos/hyperspy/hyperspy/tags","blobs_url":"https://api.github.com/repos/hyperspy/hyperspy/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hyperspy/hyperspy/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hyperspy/hyperspy/git/refs{/sha}","trees_url":"https://api.github.com/repos/hyperspy/hyperspy/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hyperspy/hyperspy/statuses/{sha}","languages_url":"https://api.github.com/repos/hyperspy/hyperspy/languages","stargazers_url":"https://api.github.com/repos/hyperspy/hyperspy/stargazers","contributors_url":"https://api.github.com/repos/hyperspy/hyperspy/contributors","subscribers_url":"https://api.github.com/repos/hyperspy/hyperspy/subscribers","subscription_url":"https://api.github.com/repos/hyperspy/hyperspy/subscription","commits_url":"https://api.github.com/repos/hyperspy/hyperspy/commits{/sha}","git_commits_url":"https://api.github.com/repos/hyperspy/hyperspy/git/commits{/sha}","comments_url":"https://api.github.com/repos/hyperspy/hyperspy/comments{/number}","issue_comment_url":"https://api.github.com/repos/hyperspy/hyperspy/issues/comments{/number}","contents_url":"https://api.github.com/repos/hyperspy/hyperspy/contents/{+path}","compare_url":"https://api.github.com/repos/hyperspy/hyperspy/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hyperspy/hyperspy/merges","archive_url":"https://api.github.com/repos/hyperspy/hyperspy/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hyperspy/hyperspy/downloads","issues_url":"https://api.github.com/repos/hyperspy/hyperspy/issues{/number}","pulls_url":"https://api.github.com/repos/hyperspy/hyperspy/pulls{/number}","milestones_url":"https://api.github.com/repos/hyperspy/hyperspy/milestones{/number}","notifications_url":"https://api.github.com/repos/hyperspy/hyperspy/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hyperspy/hyperspy/labels{/name}","releases_url":"https://api.github.com/repos/hyperspy/hyperspy/releases{/id}","deployments_url":"https://api.github.com/repos/hyperspy/hyperspy/deployments","created_at":"2011-08-19T13:14:36Z","updated_at":"2016-06-19T10:37:00Z","pushed_at":"2016-06-23T15:41:35Z","git_url":"git://github.com/hyperspy/hyperspy.git","ssh_url":"git@github.com:hyperspy/hyperspy.git","clone_url":"https://github.com/hyperspy/hyperspy.git","svn_url":"https://github.com/hyperspy/hyperspy","homepage":"http://www.hyperspy.org","size":100600,"stargazers_count":59,"watchers_count":59,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":49,"mirror_url":null,"open_issues_count":170,"forks":49,"open_issues":170,"watchers":59,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/hyperspy/hyperspy/pulls/1071"},"html":{"href":"https://github.com/hyperspy/hyperspy/pull/1071"},"issue":{"href":"https://api.github.com/repos/hyperspy/hyperspy/issues/1071"},"comments":{"href":"https://api.github.com/repos/hyperspy/hyperspy/issues/1071/comments"},"review_comments":{"href":"https://api.github.com/repos/hyperspy/hyperspy/pulls/1071/comments"},"review_comment":{"href":"https://api.github.com/repos/hyperspy/hyperspy/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/hyperspy/hyperspy/pulls/1071/commits"},"statuses":{"href":"https://api.github.com/repos/hyperspy/hyperspy/statuses/dafc098664cc3b1c9ee731a4a344039e35e2cf7b"}}}}
{"actor":{"display_login":"francisco-dlp"}}