{"action":"created","comment":{"url":"https://api.github.com/repos/QuantConnect/Lean/pulls/comments/81424562","pull_request_review_id":2408468,"id":81424562,"diff_hunk":"@@ -44,21 +39,122 @@ public class GoogleDataDownloader : IDataDownloader\n         /// <returns>Enumerable of base data for this symbol</returns>\n         public IEnumerable<BaseData> Get(Symbol symbol, Resolution resolution, DateTime startUtc, DateTime endUtc)\n         {\n-            if (resolution != Resolution.Minute && resolution != Resolution.Hour)\n-                throw new NotSupportedException(\"Resolution not available: \" + resolution);\n-\n             if (symbol.ID.SecurityType != SecurityType.Equity)\n                 throw new NotSupportedException(\"SecurityType not available: \" + symbol.ID.SecurityType);\n \n             if (endUtc < startUtc)\n                 throw new ArgumentException(\"The end date must be greater or equal than the start date.\");\n \n+            switch (resolution)\n+            {\n+                case Resolution.Minute:\n+                case Resolution.Hour:\n+                    return GetMinuteOrHour(symbol, resolution, startUtc, endUtc);\n+\n+                case Resolution.Daily:\n+                    return GetDaily(symbol, resolution, startUtc, endUtc);\n+\n+                default:\n+                    throw new NotSupportedException(\"Resolution not available: \" + resolution);\n+            }\n+        }\n+\n+        // q = SYMBOL\n+        // startdate = Mon+D%2C+YYYY, e.g. Jan+1%2C+2005\n+        // enddate = Mon+D%2C+YYYY, e.g. Sep+29%2C+2016\n+        private const string UrlPrototypeDaily = @\"http://www.google.com/finance/historical?q={0}&startdate={1}&enddate={2}&output=csv\";\n+        private enum ConvertMonths\n+        {\n+            Jan = 1,\n+            Feb = 2,\n+            Mar = 3,\n+            Apr = 4,\n+            May = 5,\n+            Jun = 6,\n+            Jul = 7,\n+            Aug = 8,\n+            Sep = 9,\n+            Oct = 10,\n+            Nov = 11,\n+            Dec = 12\n+        };\n+\n+        private IEnumerable<BaseData> GetDaily(Symbol symbol, Resolution resolution, DateTime startUtc, DateTime endUtc)\n+        {\n+            var startdate = ((ConvertMonths)startUtc.Month).ToString()\n+                + @\"+\" + startUtc.Day.ToString()\n+                + @\"%2C+\" + startUtc.Year.ToString();\n+            var enddate = ((ConvertMonths)endUtc.Month).ToString()\n+                + @\"+\" + endUtc.Day.ToString()\n+                + @\"%2C+\" + endUtc.Year.ToString();\n+\n+            // Create the Google formatted URL.\n+            var url = string.Format(UrlPrototypeDaily, ConvertTicker(symbol), startdate, enddate);\n+\n+            // Download the data from Google.\n+            string[] lines;\n+            using (var client = new WebClient())\n+            {\n+                var data = client.DownloadString(url);\n+                lines = data.Split('\\n');\n+            }\n+\n+            // first line is header\n+            var currentLine = 1;\n+\n+            while (currentLine < lines.Length - 1)\n+            {\n+                // Format: Date,Open,High,Low,Close,Volume\n+                var columns = lines[currentLine].Split(',');\n+\n+                // date format: DD-Mon-YY, e.g. 27-Sep-16\n+                var DMY = columns[0].Split('-');\n+\n+                // date = 20160927\n+                var day = DMY[0].ToInt32();\n+                var month = (int)Enum.Parse(typeof(ConvertMonths), DMY[1]);\n+                var year = (DMY[2].ToInt32() > 70) ? 1900 + DMY[2].ToInt32() : 2000 + DMY[2].ToInt32();\n+                var time = new DateTime(year, month, day, 0, 0, 0);\n+\n+                // occasionally, the columns will have a '-' instead of a proper value\n+                List<Decimal?> ohlc = new List<Decimal?>()\n+                {\n+                    columns[1] != \"-\" ? (Decimal?)columns[1].ToDecimal() : null,\n+                    columns[2] != \"-\" ? (Decimal?)columns[2].ToDecimal() : null,\n+                    columns[3] != \"-\" ? (Decimal?)columns[3].ToDecimal() : null,\n+                    columns[4] != \"-\" ? (Decimal?)columns[4].ToDecimal() : null\n+                };\n+\n+                // let's try hard to fix any issues as good as we can\n+                // this code assumes that there is at least 1 good value\n+                if (ohlc[1] == null) ohlc[1] = ohlc.Where(val => val != null).Max();\n+                if (ohlc[2] == null) ohlc[2] = ohlc.Where(val => val != null).Min();\n+                if (ohlc[0] == null) ohlc[0] = ohlc.Where(val => val != null).Average();\n+                if (ohlc[3] == null) ohlc[3] = ohlc.Where(val => val != null).Average();","path":"ToolBox/GoogleDownloader/GoogleDataDownloader.cs","position":85,"original_position":123,"commit_id":"e5963e8b49da5bf14adfcb38e9e0e7c77fae304c","original_commit_id":"66f5d3982293fd2349f3963385ea67d8ab098920","user":{"login":"fbertram","id":16840687,"avatar_url":"https://avatars.githubusercontent.com/u/16840687?v=3","gravatar_id":"","url":"https://api.github.com/users/fbertram","html_url":"https://github.com/fbertram","followers_url":"https://api.github.com/users/fbertram/followers","following_url":"https://api.github.com/users/fbertram/following{/other_user}","gists_url":"https://api.github.com/users/fbertram/gists{/gist_id}","starred_url":"https://api.github.com/users/fbertram/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fbertram/subscriptions","organizations_url":"https://api.github.com/users/fbertram/orgs","repos_url":"https://api.github.com/users/fbertram/repos","events_url":"https://api.github.com/users/fbertram/events{/privacy}","received_events_url":"https://api.github.com/users/fbertram/received_events","type":"User","site_admin":false},"body":"I am not sure what Google does here, I see this about 20 times in 20 years. I agree with your concerns, and it makes me cringe too.\r\nI see the following options:\r\n(1) drop the bar, which will have some side effects.\r\n(2) fill missing values with zero. this might trigger quite erratic reactions in an algorithm.\r\n(3) try to repair. while this is not ideal, I feel that this creates the least amount of side effects\r\n\r\nLeft code as is, in the spirit of providing the best simulation we can. Basically, we have to admit that downloading data from Google is not ideal. The data from Yahoo are better, but they are not adjusted...","created_at":"2016-09-30T21:54:56Z","updated_at":"2016-09-30T21:54:56Z","html_url":"https://github.com/QuantConnect/Lean/pull/580#discussion_r81424562","pull_request_url":"https://api.github.com/repos/QuantConnect/Lean/pulls/580","_links":{"self":{"href":"https://api.github.com/repos/QuantConnect/Lean/pulls/comments/81424562"},"html":{"href":"https://github.com/QuantConnect/Lean/pull/580#discussion_r81424562"},"pull_request":{"href":"https://api.github.com/repos/QuantConnect/Lean/pulls/580"}}},"pull_request":{"url":"https://api.github.com/repos/QuantConnect/Lean/pulls/580","id":87543547,"html_url":"https://github.com/QuantConnect/Lean/pull/580","diff_url":"https://github.com/QuantConnect/Lean/pull/580.diff","patch_url":"https://github.com/QuantConnect/Lean/pull/580.patch","issue_url":"https://api.github.com/repos/QuantConnect/Lean/issues/580","number":580,"state":"open","locked":false,"title":"Feature/daily quotes","user":{"login":"fbertram","id":16840687,"avatar_url":"https://avatars.githubusercontent.com/u/16840687?v=3","gravatar_id":"","url":"https://api.github.com/users/fbertram","html_url":"https://github.com/fbertram","followers_url":"https://api.github.com/users/fbertram/followers","following_url":"https://api.github.com/users/fbertram/following{/other_user}","gists_url":"https://api.github.com/users/fbertram/gists{/gist_id}","starred_url":"https://api.github.com/users/fbertram/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fbertram/subscriptions","organizations_url":"https://api.github.com/users/fbertram/orgs","repos_url":"https://api.github.com/users/fbertram/repos","events_url":"https://api.github.com/users/fbertram/events{/privacy}","received_events_url":"https://api.github.com/users/fbertram/received_events","type":"User","site_admin":false},"body":"these changes implement download of daily quotes for the Google downloader. The code includes 2 workarounds worth mentioning:\r\n\r\n(1) Google seems to taint their data by replacing some quotes with '-'. The code tries its best to put reasonable values in, but of course cannot recover what was really lost here.\r\n\r\n(2) Google does not recognize SPY, but requires NYSEARCA:SPY instead. There is a function for translating this, I am sure there will be a dict requires at some point in the future.\r\n\r\nCheers, Felix","created_at":"2016-09-30T18:32:56Z","updated_at":"2016-09-30T21:54:56Z","closed_at":null,"merged_at":null,"merge_commit_sha":"0c4f6f5a9b5f17e13a5e851cbf80756cef4419c7","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/QuantConnect/Lean/pulls/580/commits","review_comments_url":"https://api.github.com/repos/QuantConnect/Lean/pulls/580/comments","review_comment_url":"https://api.github.com/repos/QuantConnect/Lean/pulls/comments{/number}","comments_url":"https://api.github.com/repos/QuantConnect/Lean/issues/580/comments","statuses_url":"https://api.github.com/repos/QuantConnect/Lean/statuses/e5963e8b49da5bf14adfcb38e9e0e7c77fae304c","head":{"label":"fbertram:feature/daily_quotes","ref":"feature/daily_quotes","sha":"e5963e8b49da5bf14adfcb38e9e0e7c77fae304c","user":{"login":"fbertram","id":16840687,"avatar_url":"https://avatars.githubusercontent.com/u/16840687?v=3","gravatar_id":"","url":"https://api.github.com/users/fbertram","html_url":"https://github.com/fbertram","followers_url":"https://api.github.com/users/fbertram/followers","following_url":"https://api.github.com/users/fbertram/following{/other_user}","gists_url":"https://api.github.com/users/fbertram/gists{/gist_id}","starred_url":"https://api.github.com/users/fbertram/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fbertram/subscriptions","organizations_url":"https://api.github.com/users/fbertram/orgs","repos_url":"https://api.github.com/users/fbertram/repos","events_url":"https://api.github.com/users/fbertram/events{/privacy}","received_events_url":"https://api.github.com/users/fbertram/received_events","type":"User","site_admin":false},"repo":{"id":67851362,"name":"Lean","full_name":"fbertram/Lean","owner":{"login":"fbertram","id":16840687,"avatar_url":"https://avatars.githubusercontent.com/u/16840687?v=3","gravatar_id":"","url":"https://api.github.com/users/fbertram","html_url":"https://github.com/fbertram","followers_url":"https://api.github.com/users/fbertram/followers","following_url":"https://api.github.com/users/fbertram/following{/other_user}","gists_url":"https://api.github.com/users/fbertram/gists{/gist_id}","starred_url":"https://api.github.com/users/fbertram/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fbertram/subscriptions","organizations_url":"https://api.github.com/users/fbertram/orgs","repos_url":"https://api.github.com/users/fbertram/repos","events_url":"https://api.github.com/users/fbertram/events{/privacy}","received_events_url":"https://api.github.com/users/fbertram/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/fbertram/Lean","description":"Lean Algorithmic Trading Engine by QuantConnect (C#, Python, F#, VB, Java)","fork":true,"url":"https://api.github.com/repos/fbertram/Lean","forks_url":"https://api.github.com/repos/fbertram/Lean/forks","keys_url":"https://api.github.com/repos/fbertram/Lean/keys{/key_id}","collaborators_url":"https://api.github.com/repos/fbertram/Lean/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/fbertram/Lean/teams","hooks_url":"https://api.github.com/repos/fbertram/Lean/hooks","issue_events_url":"https://api.github.com/repos/fbertram/Lean/issues/events{/number}","events_url":"https://api.github.com/repos/fbertram/Lean/events","assignees_url":"https://api.github.com/repos/fbertram/Lean/assignees{/user}","branches_url":"https://api.github.com/repos/fbertram/Lean/branches{/branch}","tags_url":"https://api.github.com/repos/fbertram/Lean/tags","blobs_url":"https://api.github.com/repos/fbertram/Lean/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/fbertram/Lean/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/fbertram/Lean/git/refs{/sha}","trees_url":"https://api.github.com/repos/fbertram/Lean/git/trees{/sha}","statuses_url":"https://api.github.com/repos/fbertram/Lean/statuses/{sha}","languages_url":"https://api.github.com/repos/fbertram/Lean/languages","stargazers_url":"https://api.github.com/repos/fbertram/Lean/stargazers","contributors_url":"https://api.github.com/repos/fbertram/Lean/contributors","subscribers_url":"https://api.github.com/repos/fbertram/Lean/subscribers","subscription_url":"https://api.github.com/repos/fbertram/Lean/subscription","commits_url":"https://api.github.com/repos/fbertram/Lean/commits{/sha}","git_commits_url":"https://api.github.com/repos/fbertram/Lean/git/commits{/sha}","comments_url":"https://api.github.com/repos/fbertram/Lean/comments{/number}","issue_comment_url":"https://api.github.com/repos/fbertram/Lean/issues/comments{/number}","contents_url":"https://api.github.com/repos/fbertram/Lean/contents/{+path}","compare_url":"https://api.github.com/repos/fbertram/Lean/compare/{base}...{head}","merges_url":"https://api.github.com/repos/fbertram/Lean/merges","archive_url":"https://api.github.com/repos/fbertram/Lean/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/fbertram/Lean/downloads","issues_url":"https://api.github.com/repos/fbertram/Lean/issues{/number}","pulls_url":"https://api.github.com/repos/fbertram/Lean/pulls{/number}","milestones_url":"https://api.github.com/repos/fbertram/Lean/milestones{/number}","notifications_url":"https://api.github.com/repos/fbertram/Lean/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/fbertram/Lean/labels{/name}","releases_url":"https://api.github.com/repos/fbertram/Lean/releases{/id}","deployments_url":"https://api.github.com/repos/fbertram/Lean/deployments","created_at":"2016-09-10T04:45:39Z","updated_at":"2016-09-10T04:45:43Z","pushed_at":"2016-09-30T21:52:45Z","git_url":"git://github.com/fbertram/Lean.git","ssh_url":"git@github.com:fbertram/Lean.git","clone_url":"https://github.com/fbertram/Lean.git","svn_url":"https://github.com/fbertram/Lean","homepage":"https://lean.quantconnect.com","size":89976,"stargazers_count":0,"watchers_count":0,"language":"C#","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"QuantConnect:master","ref":"master","sha":"96d9aeae4ea4402174c4c1e7827cf74274711c11","user":{"login":"QuantConnect","id":3912814,"avatar_url":"https://avatars.githubusercontent.com/u/3912814?v=3","gravatar_id":"","url":"https://api.github.com/users/QuantConnect","html_url":"https://github.com/QuantConnect","followers_url":"https://api.github.com/users/QuantConnect/followers","following_url":"https://api.github.com/users/QuantConnect/following{/other_user}","gists_url":"https://api.github.com/users/QuantConnect/gists{/gist_id}","starred_url":"https://api.github.com/users/QuantConnect/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/QuantConnect/subscriptions","organizations_url":"https://api.github.com/users/QuantConnect/orgs","repos_url":"https://api.github.com/users/QuantConnect/repos","events_url":"https://api.github.com/users/QuantConnect/events{/privacy}","received_events_url":"https://api.github.com/users/QuantConnect/received_events","type":"Organization","site_admin":false},"repo":{"id":27251463,"name":"Lean","full_name":"QuantConnect/Lean","owner":{"login":"QuantConnect","id":3912814,"avatar_url":"https://avatars.githubusercontent.com/u/3912814?v=3","gravatar_id":"","url":"https://api.github.com/users/QuantConnect","html_url":"https://github.com/QuantConnect","followers_url":"https://api.github.com/users/QuantConnect/followers","following_url":"https://api.github.com/users/QuantConnect/following{/other_user}","gists_url":"https://api.github.com/users/QuantConnect/gists{/gist_id}","starred_url":"https://api.github.com/users/QuantConnect/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/QuantConnect/subscriptions","organizations_url":"https://api.github.com/users/QuantConnect/orgs","repos_url":"https://api.github.com/users/QuantConnect/repos","events_url":"https://api.github.com/users/QuantConnect/events{/privacy}","received_events_url":"https://api.github.com/users/QuantConnect/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/QuantConnect/Lean","description":"Lean Algorithmic Trading Engine by QuantConnect (C#, Python, F#, VB, Java)","fork":false,"url":"https://api.github.com/repos/QuantConnect/Lean","forks_url":"https://api.github.com/repos/QuantConnect/Lean/forks","keys_url":"https://api.github.com/repos/QuantConnect/Lean/keys{/key_id}","collaborators_url":"https://api.github.com/repos/QuantConnect/Lean/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/QuantConnect/Lean/teams","hooks_url":"https://api.github.com/repos/QuantConnect/Lean/hooks","issue_events_url":"https://api.github.com/repos/QuantConnect/Lean/issues/events{/number}","events_url":"https://api.github.com/repos/QuantConnect/Lean/events","assignees_url":"https://api.github.com/repos/QuantConnect/Lean/assignees{/user}","branches_url":"https://api.github.com/repos/QuantConnect/Lean/branches{/branch}","tags_url":"https://api.github.com/repos/QuantConnect/Lean/tags","blobs_url":"https://api.github.com/repos/QuantConnect/Lean/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/QuantConnect/Lean/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/QuantConnect/Lean/git/refs{/sha}","trees_url":"https://api.github.com/repos/QuantConnect/Lean/git/trees{/sha}","statuses_url":"https://api.github.com/repos/QuantConnect/Lean/statuses/{sha}","languages_url":"https://api.github.com/repos/QuantConnect/Lean/languages","stargazers_url":"https://api.github.com/repos/QuantConnect/Lean/stargazers","contributors_url":"https://api.github.com/repos/QuantConnect/Lean/contributors","subscribers_url":"https://api.github.com/repos/QuantConnect/Lean/subscribers","subscription_url":"https://api.github.com/repos/QuantConnect/Lean/subscription","commits_url":"https://api.github.com/repos/QuantConnect/Lean/commits{/sha}","git_commits_url":"https://api.github.com/repos/QuantConnect/Lean/git/commits{/sha}","comments_url":"https://api.github.com/repos/QuantConnect/Lean/comments{/number}","issue_comment_url":"https://api.github.com/repos/QuantConnect/Lean/issues/comments{/number}","contents_url":"https://api.github.com/repos/QuantConnect/Lean/contents/{+path}","compare_url":"https://api.github.com/repos/QuantConnect/Lean/compare/{base}...{head}","merges_url":"https://api.github.com/repos/QuantConnect/Lean/merges","archive_url":"https://api.github.com/repos/QuantConnect/Lean/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/QuantConnect/Lean/downloads","issues_url":"https://api.github.com/repos/QuantConnect/Lean/issues{/number}","pulls_url":"https://api.github.com/repos/QuantConnect/Lean/pulls{/number}","milestones_url":"https://api.github.com/repos/QuantConnect/Lean/milestones{/number}","notifications_url":"https://api.github.com/repos/QuantConnect/Lean/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/QuantConnect/Lean/labels{/name}","releases_url":"https://api.github.com/repos/QuantConnect/Lean/releases{/id}","deployments_url":"https://api.github.com/repos/QuantConnect/Lean/deployments","created_at":"2014-11-28T04:20:19Z","updated_at":"2016-09-30T18:02:06Z","pushed_at":"2016-09-30T21:52:46Z","git_url":"git://github.com/QuantConnect/Lean.git","ssh_url":"git@github.com:QuantConnect/Lean.git","clone_url":"https://github.com/QuantConnect/Lean.git","svn_url":"https://github.com/QuantConnect/Lean","homepage":"https://lean.quantconnect.com","size":94582,"stargazers_count":582,"watchers_count":582,"language":"C#","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":424,"mirror_url":null,"open_issues_count":39,"forks":424,"open_issues":39,"watchers":582,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/QuantConnect/Lean/pulls/580"},"html":{"href":"https://github.com/QuantConnect/Lean/pull/580"},"issue":{"href":"https://api.github.com/repos/QuantConnect/Lean/issues/580"},"comments":{"href":"https://api.github.com/repos/QuantConnect/Lean/issues/580/comments"},"review_comments":{"href":"https://api.github.com/repos/QuantConnect/Lean/pulls/580/comments"},"review_comment":{"href":"https://api.github.com/repos/QuantConnect/Lean/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/QuantConnect/Lean/pulls/580/commits"},"statuses":{"href":"https://api.github.com/repos/QuantConnect/Lean/statuses/e5963e8b49da5bf14adfcb38e9e0e7c77fae304c"}}}}
{"actor":{"display_login":"fbertram"}}