{"action":"created","comment":{"url":"https://api.github.com/repos/apache/incubator-mxnet/pulls/comments/196640836","pull_request_review_id":130231256,"id":196640836,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDE5NjY0MDgzNg==","diff_hunk":"@@ -153,18 +153,57 @@ class PolyScheduler(LRScheduler):\n \n     \"\"\"\n \n-    def __init__(self, max_update, base_lr=0.01, pwr=2):\n-        super(PolyScheduler, self).__init__(base_lr)\n+    def __init__(self, max_update, pwr=2, **kwargs):\n+        super(PolyScheduler, self).__init__(**kwargs)\n         assert isinstance(max_update, int)\n         if max_update < 1:\n             raise ValueError(\"maximum number of updates must be strictly positive\")\n         self.base_lr_orig = self.base_lr\n         self.max_update = max_update\n         self.power = pwr\n-        self.base_lr = self.base_lr_orig\n \n     def __call__(self, num_update):\n         if num_update <= self.max_update:\n             self.base_lr = self.base_lr_orig * pow(1.0 - float(num_update) / float(self.max_update),\n                                                    self.power)\n         return self.base_lr\n+\n+class WarmupScheduler(LRScheduler):\n+    \"\"\"Implement linear warmup starting from lr_begin to given scheduler's base_lr.\n+\n+    Parameters\n+    ----------\n+    lr_begin: float\n+                  learning rate to start increasing from\n+    warmup_steps: int\n+                  number of warmup steps\n+    scheduler: LRScheduler\n+              scheduler following the warmup\n+    \"\"\"\n+    def __init__(self, lr_begin, warmup_steps, scheduler):\n+        super(WarmupScheduler, self).__init__()\n+        self.lr_begin = lr_begin\n+        self.scheduler = scheduler\n+        self.lr_final = self.scheduler.base_lr\n+        if self.lr_begin > self.lr_final:\n+            raise ValueError(\"Final lr has to be higher than beginning lr\")\n+        if warmup_steps <= 0:\n+            raise ValueError(\"Warmup steps has to be positive\")\n+        self.warmup_steps = warmup_steps\n+        self.lrs_updates = {}\n+        self.lr_difference = self.lr_final - self.lr_begin\n+\n+    def __call__(self, num_update):\n+        if num_update not in self.lrs_updates:\n+            if num_update < self.warmup_steps:\n+                increase = self.lr_difference * float(num_update)/float(self.warmup_steps)\n+                self.lrs_updates[num_update] = self.lr_begin + increase\n+            else:\n+                if isinstance(self.scheduler, PolyScheduler):\n+                    self.lrs_updates[num_update] = self.scheduler(num_update - self.warmup_steps)","path":"python/mxnet/lr_scheduler.py","position":76,"original_position":76,"commit_id":"3ad20d223bb1dbf9cc1f240d1eb80e303fa65f9c","original_commit_id":"3ad20d223bb1dbf9cc1f240d1eb80e303fa65f9c","user":{"login":"rahul003","id":3457240,"node_id":"MDQ6VXNlcjM0NTcyNDA=","avatar_url":"https://avatars1.githubusercontent.com/u/3457240?v=4","gravatar_id":"","url":"https://api.github.com/users/rahul003","html_url":"https://github.com/rahul003","followers_url":"https://api.github.com/users/rahul003/followers","following_url":"https://api.github.com/users/rahul003/following{/other_user}","gists_url":"https://api.github.com/users/rahul003/gists{/gist_id}","starred_url":"https://api.github.com/users/rahul003/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rahul003/subscriptions","organizations_url":"https://api.github.com/users/rahul003/orgs","repos_url":"https://api.github.com/users/rahul003/repos","events_url":"https://api.github.com/users/rahul003/events{/privacy}","received_events_url":"https://api.github.com/users/rahul003/received_events","type":"User","site_admin":false},"body":"Yeah for the above reason, but I'm updating the code to remove this special case, and pass a wamup_steps param to such schedulers so we can handle it cleanly","created_at":"2018-06-20T03:46:10Z","updated_at":"2018-06-20T03:46:10Z","html_url":"https://github.com/apache/incubator-mxnet/pull/11234#discussion_r196640836","pull_request_url":"https://api.github.com/repos/apache/incubator-mxnet/pulls/11234","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-mxnet/pulls/comments/196640836"},"html":{"href":"https://github.com/apache/incubator-mxnet/pull/11234#discussion_r196640836"},"pull_request":{"href":"https://api.github.com/repos/apache/incubator-mxnet/pulls/11234"}},"in_reply_to_id":196240711},"pull_request":{"url":"https://api.github.com/repos/apache/incubator-mxnet/pulls/11234","id":194133916,"node_id":"MDExOlB1bGxSZXF1ZXN0MTk0MTMzOTE2","html_url":"https://github.com/apache/incubator-mxnet/pull/11234","diff_url":"https://github.com/apache/incubator-mxnet/pull/11234.diff","patch_url":"https://github.com/apache/incubator-mxnet/pull/11234.patch","issue_url":"https://api.github.com/repos/apache/incubator-mxnet/issues/11234","number":11234,"state":"open","locked":false,"title":"[MXNET-535] Add Warmup Learning Rate Scheduler and fix bugs in LR Schedulers","user":{"login":"rahul003","id":3457240,"node_id":"MDQ6VXNlcjM0NTcyNDA=","avatar_url":"https://avatars1.githubusercontent.com/u/3457240?v=4","gravatar_id":"","url":"https://api.github.com/users/rahul003","html_url":"https://github.com/rahul003","followers_url":"https://api.github.com/users/rahul003/followers","following_url":"https://api.github.com/users/rahul003/following{/other_user}","gists_url":"https://api.github.com/users/rahul003/gists{/gist_id}","starred_url":"https://api.github.com/users/rahul003/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rahul003/subscriptions","organizations_url":"https://api.github.com/users/rahul003/orgs","repos_url":"https://api.github.com/users/rahul003/repos","events_url":"https://api.github.com/users/rahul003/events{/privacy}","received_events_url":"https://api.github.com/users/rahul003/received_events","type":"User","site_admin":false},"body":"## Description ##\r\nAdds warmup LR scheduler. Also fixes issues where base_lr is not taken by MultiFactorScheduler and FactorScheduler.\r\n\r\n## Checklist ##\r\n### Essentials ###\r\nPlease feel free to remove inapplicable items for your PR.\r\n- [x] The PR title starts with [MXNET-$JIRA_ID], where $JIRA_ID refers to the relevant [JIRA issue](https://issues.apache.org/jira/projects/MXNET/issues) created (except PRs with tiny changes)\r\n- [x] Changes are complete (i.e. I finished coding on this PR)\r\n- [x] All changes have test coverage:\r\n- Unit tests are added for small changes to verify correctness (e.g. adding a new operator)\r\n- Nightly tests are added for complicated/long-running ones (e.g. changing distributed kvstore)\r\n- Build tests will be added for build configuration changes (e.g. adding a new build option with NCCL)\r\n- [ ] Code is well-documented: \r\n- For user-facing API changes, API doc string has been updated. \r\n- For new C++ functions in header files, their functionalities and arguments are documented. \r\n- For new examples, README.md is added to explain the what the example does, the source of the dataset, expected performance on test set and reference to the original paper if applicable\r\n- Check the API doc at http://mxnet-ci-doc.s3-accelerate.dualstack.amazonaws.com/PR-$PR_ID/$BUILD_ID/index.html\r\n- [x] To the my best knowledge, examples are either not affected by this change, or have been fixed to be compatible with this change\r\n\r\n### Changes ###\r\n- [ ] Feature1, tests, (and when applicable, API doc)\r\n- [ ] Feature2, tests, (and when applicable, API doc)\r\n","created_at":"2018-06-11T23:59:54Z","updated_at":"2018-06-20T03:46:10Z","closed_at":null,"merged_at":null,"merge_commit_sha":"99c2c57cee70c1da6ed55090662b379c3935f48a","assignee":null,"assignees":[],"requested_reviewers":[{"login":"szha","id":2626883,"node_id":"MDQ6VXNlcjI2MjY4ODM=","avatar_url":"https://avatars2.githubusercontent.com/u/2626883?v=4","gravatar_id":"","url":"https://api.github.com/users/szha","html_url":"https://github.com/szha","followers_url":"https://api.github.com/users/szha/followers","following_url":"https://api.github.com/users/szha/following{/other_user}","gists_url":"https://api.github.com/users/szha/gists{/gist_id}","starred_url":"https://api.github.com/users/szha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/szha/subscriptions","organizations_url":"https://api.github.com/users/szha/orgs","repos_url":"https://api.github.com/users/szha/repos","events_url":"https://api.github.com/users/szha/events{/privacy}","received_events_url":"https://api.github.com/users/szha/received_events","type":"User","site_admin":false}],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/incubator-mxnet/pulls/11234/commits","review_comments_url":"https://api.github.com/repos/apache/incubator-mxnet/pulls/11234/comments","review_comment_url":"https://api.github.com/repos/apache/incubator-mxnet/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/incubator-mxnet/issues/11234/comments","statuses_url":"https://api.github.com/repos/apache/incubator-mxnet/statuses/3ad20d223bb1dbf9cc1f240d1eb80e303fa65f9c","head":{"label":"rahul003:warmup","ref":"warmup","sha":"3ad20d223bb1dbf9cc1f240d1eb80e303fa65f9c","user":{"login":"rahul003","id":3457240,"node_id":"MDQ6VXNlcjM0NTcyNDA=","avatar_url":"https://avatars1.githubusercontent.com/u/3457240?v=4","gravatar_id":"","url":"https://api.github.com/users/rahul003","html_url":"https://github.com/rahul003","followers_url":"https://api.github.com/users/rahul003/followers","following_url":"https://api.github.com/users/rahul003/following{/other_user}","gists_url":"https://api.github.com/users/rahul003/gists{/gist_id}","starred_url":"https://api.github.com/users/rahul003/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rahul003/subscriptions","organizations_url":"https://api.github.com/users/rahul003/orgs","repos_url":"https://api.github.com/users/rahul003/repos","events_url":"https://api.github.com/users/rahul003/events{/privacy}","received_events_url":"https://api.github.com/users/rahul003/received_events","type":"User","site_admin":false},"repo":{"id":96937059,"node_id":"MDEwOlJlcG9zaXRvcnk5NjkzNzA1OQ==","name":"mxnet","full_name":"rahul003/mxnet","owner":{"login":"rahul003","id":3457240,"node_id":"MDQ6VXNlcjM0NTcyNDA=","avatar_url":"https://avatars1.githubusercontent.com/u/3457240?v=4","gravatar_id":"","url":"https://api.github.com/users/rahul003","html_url":"https://github.com/rahul003","followers_url":"https://api.github.com/users/rahul003/followers","following_url":"https://api.github.com/users/rahul003/following{/other_user}","gists_url":"https://api.github.com/users/rahul003/gists{/gist_id}","starred_url":"https://api.github.com/users/rahul003/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rahul003/subscriptions","organizations_url":"https://api.github.com/users/rahul003/orgs","repos_url":"https://api.github.com/users/rahul003/repos","events_url":"https://api.github.com/users/rahul003/events{/privacy}","received_events_url":"https://api.github.com/users/rahul003/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/rahul003/mxnet","description":"Lightweight, Portable, Flexible Distributed/Mobile Deep Learning with Dynamic, Mutation-aware Dataflow Dep Scheduler; for Python, R, Julia, Scala, Go, Javascript and more","fork":true,"url":"https://api.github.com/repos/rahul003/mxnet","forks_url":"https://api.github.com/repos/rahul003/mxnet/forks","keys_url":"https://api.github.com/repos/rahul003/mxnet/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rahul003/mxnet/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rahul003/mxnet/teams","hooks_url":"https://api.github.com/repos/rahul003/mxnet/hooks","issue_events_url":"https://api.github.com/repos/rahul003/mxnet/issues/events{/number}","events_url":"https://api.github.com/repos/rahul003/mxnet/events","assignees_url":"https://api.github.com/repos/rahul003/mxnet/assignees{/user}","branches_url":"https://api.github.com/repos/rahul003/mxnet/branches{/branch}","tags_url":"https://api.github.com/repos/rahul003/mxnet/tags","blobs_url":"https://api.github.com/repos/rahul003/mxnet/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rahul003/mxnet/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rahul003/mxnet/git/refs{/sha}","trees_url":"https://api.github.com/repos/rahul003/mxnet/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rahul003/mxnet/statuses/{sha}","languages_url":"https://api.github.com/repos/rahul003/mxnet/languages","stargazers_url":"https://api.github.com/repos/rahul003/mxnet/stargazers","contributors_url":"https://api.github.com/repos/rahul003/mxnet/contributors","subscribers_url":"https://api.github.com/repos/rahul003/mxnet/subscribers","subscription_url":"https://api.github.com/repos/rahul003/mxnet/subscription","commits_url":"https://api.github.com/repos/rahul003/mxnet/commits{/sha}","git_commits_url":"https://api.github.com/repos/rahul003/mxnet/git/commits{/sha}","comments_url":"https://api.github.com/repos/rahul003/mxnet/comments{/number}","issue_comment_url":"https://api.github.com/repos/rahul003/mxnet/issues/comments{/number}","contents_url":"https://api.github.com/repos/rahul003/mxnet/contents/{+path}","compare_url":"https://api.github.com/repos/rahul003/mxnet/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rahul003/mxnet/merges","archive_url":"https://api.github.com/repos/rahul003/mxnet/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rahul003/mxnet/downloads","issues_url":"https://api.github.com/repos/rahul003/mxnet/issues{/number}","pulls_url":"https://api.github.com/repos/rahul003/mxnet/pulls{/number}","milestones_url":"https://api.github.com/repos/rahul003/mxnet/milestones{/number}","notifications_url":"https://api.github.com/repos/rahul003/mxnet/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rahul003/mxnet/labels{/name}","releases_url":"https://api.github.com/repos/rahul003/mxnet/releases{/id}","deployments_url":"https://api.github.com/repos/rahul003/mxnet/deployments","created_at":"2017-07-11T21:04:11Z","updated_at":"2018-06-18T21:38:09Z","pushed_at":"2018-06-19T02:06:06Z","git_url":"git://github.com/rahul003/mxnet.git","ssh_url":"git@github.com:rahul003/mxnet.git","clone_url":"https://github.com/rahul003/mxnet.git","svn_url":"https://github.com/rahul003/mxnet","homepage":"http://mxnet.io","size":40565,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"14275a5b5b271024ed7d3a1e3afa04edb4bbd86e","user":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":34864402,"node_id":"MDEwOlJlcG9zaXRvcnkzNDg2NDQwMg==","name":"incubator-mxnet","full_name":"apache/incubator-mxnet","owner":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/incubator-mxnet","description":"Lightweight, Portable, Flexible Distributed/Mobile Deep Learning with Dynamic, Mutation-aware Dataflow Dep Scheduler; for Python, R, Julia, Scala, Go, Javascript and more","fork":false,"url":"https://api.github.com/repos/apache/incubator-mxnet","forks_url":"https://api.github.com/repos/apache/incubator-mxnet/forks","keys_url":"https://api.github.com/repos/apache/incubator-mxnet/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/incubator-mxnet/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/incubator-mxnet/teams","hooks_url":"https://api.github.com/repos/apache/incubator-mxnet/hooks","issue_events_url":"https://api.github.com/repos/apache/incubator-mxnet/issues/events{/number}","events_url":"https://api.github.com/repos/apache/incubator-mxnet/events","assignees_url":"https://api.github.com/repos/apache/incubator-mxnet/assignees{/user}","branches_url":"https://api.github.com/repos/apache/incubator-mxnet/branches{/branch}","tags_url":"https://api.github.com/repos/apache/incubator-mxnet/tags","blobs_url":"https://api.github.com/repos/apache/incubator-mxnet/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/incubator-mxnet/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/incubator-mxnet/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/incubator-mxnet/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/incubator-mxnet/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/incubator-mxnet/languages","stargazers_url":"https://api.github.com/repos/apache/incubator-mxnet/stargazers","contributors_url":"https://api.github.com/repos/apache/incubator-mxnet/contributors","subscribers_url":"https://api.github.com/repos/apache/incubator-mxnet/subscribers","subscription_url":"https://api.github.com/repos/apache/incubator-mxnet/subscription","commits_url":"https://api.github.com/repos/apache/incubator-mxnet/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/incubator-mxnet/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/incubator-mxnet/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/incubator-mxnet/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/incubator-mxnet/contents/{+path}","compare_url":"https://api.github.com/repos/apache/incubator-mxnet/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/incubator-mxnet/merges","archive_url":"https://api.github.com/repos/apache/incubator-mxnet/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/incubator-mxnet/downloads","issues_url":"https://api.github.com/repos/apache/incubator-mxnet/issues{/number}","pulls_url":"https://api.github.com/repos/apache/incubator-mxnet/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/incubator-mxnet/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/incubator-mxnet/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/incubator-mxnet/labels{/name}","releases_url":"https://api.github.com/repos/apache/incubator-mxnet/releases{/id}","deployments_url":"https://api.github.com/repos/apache/incubator-mxnet/deployments","created_at":"2015-04-30T16:21:15Z","updated_at":"2018-06-20T01:04:35Z","pushed_at":"2018-06-20T03:22:44Z","git_url":"git://github.com/apache/incubator-mxnet.git","ssh_url":"git@github.com:apache/incubator-mxnet.git","clone_url":"https://github.com/apache/incubator-mxnet.git","svn_url":"https://github.com/apache/incubator-mxnet","homepage":"https://mxnet.apache.org","size":37556,"stargazers_count":14218,"watchers_count":14218,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":5247,"mirror_url":null,"archived":false,"open_issues_count":973,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":5247,"open_issues":973,"watchers":14218,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-mxnet/pulls/11234"},"html":{"href":"https://github.com/apache/incubator-mxnet/pull/11234"},"issue":{"href":"https://api.github.com/repos/apache/incubator-mxnet/issues/11234"},"comments":{"href":"https://api.github.com/repos/apache/incubator-mxnet/issues/11234/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/incubator-mxnet/pulls/11234/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/incubator-mxnet/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/incubator-mxnet/pulls/11234/commits"},"statuses":{"href":"https://api.github.com/repos/apache/incubator-mxnet/statuses/3ad20d223bb1dbf9cc1f240d1eb80e303fa65f9c"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"rahul003"}}