{"action":"created","comment":{"url":"https://api.github.com/repos/fle-internal/content-pack-maker/pulls/comments/83973848","pull_request_review_id":4782128,"id":83973848,"diff_hunk":"@@ -516,147 +575,147 @@ def download_and_clean_kalite_data(url, path, lang=\"en\") -> str:\n         ujson.dump(node_data, f)\n \n \n-topic_attributes = [\n-    'childData',\n-    'deleted',\n-    'description',\n-    'doNotPublish',\n-    'hide',\n-    'id',\n-    'kind',\n-    'slug',\n-    'title'\n-]\n-\n-exercise_attributes = [\n-    'allAssessmentItems',\n-    'curatedRelatedVideos',\n-    'description',\n-    'displayName',\n-    'fileName',\n-    'id',\n-    'kind',\n-    'name',\n-    'prerequisites',\n-    'slug',\n-    'title',\n-    'usesAssessmentItems'\n-]\n-\n-video_attributes = [\n-    'description',\n-    'downloadSize',\n-    'duration',\n-    'id',\n-    'imageUrl',\n-    'keywords',\n-    'kind',\n-    'licenseName',\n-    'readableId',\n-    'relatedExerciseUrl',\n-    'relativeUrl',\n-    'sha',\n-    'slug',\n-    'title',\n-    'translatedYoutubeLang',\n-    'youtubeId'\n-]\n-\n-en_lang_code = \"en\"\n-\n-\n-def retrieve_kalite_data(lang=en_lang_code, force=False, ka_domain=None, no_dubbed_videos=False) -> list:\n+def retrieve_kalite_data(lang=EN_LANG_CODE, force=False, ka_domain=KA_DOMAIN, no_dubbed_videos=False) -> list:\n     \"\"\"\n     Retrieve the KA content data direct from KA.\n+    Note: use the same language code in the video, topic and exercises node data to prevent issues.\n     \"\"\"\n \n-    if not ka_domain:\n-        ka_domain = \"www.khanacademy.org\"\n-\n-    lang_url = \"http://{ka_domain}/api/v2/topics/topictree?lang={lang}&projection={projection}\"\n-\n-    projection = OrderedDict([\n-        (\"topics\", [OrderedDict((key, 1) for key in topic_attributes)]),\n-        (\"exercises\", [OrderedDict((key, 1) for key in exercise_attributes)]),\n-        (\"videos\", [OrderedDict((key, 1) for key in video_attributes)])\n-    ])\n-\n-    url = lang_url.format(projection=json.dumps(projection), lang=lang, ka_domain=ka_domain)\n-\n-    node_data_path = download_and_clean_kalite_data(url, lang=lang, ignorecache=force, filename=\"nodes.json\")\n+    node_data = []\n+    topic_path_list = []\n+    exercise_ids = []\n+    youtube_ids = []\n \n-    with open(node_data_path, 'r') as f:\n-        node_data = ujson.load(f)\n+    \"\"\"\n+    Get all possible language codes for the language because one language may have multiple language codes or names.  \n \n-    if not lang == en_lang_code and not no_dubbed_videos:\n-        # Generate en_nodes.json json this will be used in dubbed video mappings.\n-        # This will cache en_nodes.json\n-        url = lang_url.format(projection=json.dumps(projection), lang=en_lang_code, ka_domain=ka_domain)\n-        download_and_clean_kalite_data(url, lang=en_lang_code, ignorecache=False, filename=\"en_nodes.json\")\n+    Example 1: Swahili language { \n+                \"sw\":{ \"name\":\"Swahili\", \"native_name\":\"Kiswahili\" },\n+                \"swa\":{ \"name\":\"Swahili\", \"native_name\":\"Kiswahili\" },\n+            }\n \n-        node_data = addin_dubbed_video_mappings(node_data, lang)\n+    Example 2: Somali language {\n+                \"som\":{ \"name\":\"Somali\", \"native_name\":\"Soomaaliga\" },\n+                \"so\":{ \"name\":\"Somali\", \"native_name\":\"Soomaaliga, af Soomaali\" },\n+            }\n+    \"\"\"\n \n+    # MUST: Get all possible language codes for the language.\n+    lang_codes = get_lang_code_list(lang)\n+\n+    # Loop-thru all lang codes and populate the topic, exercise, video lists while checking for duplicates.\n+    logging.info(\"Found %s language codes for the language %s.\" % (lang_codes, lang,))\n+    for lang_code in lang_codes:\n+        logging.info(\"  Processing language code %s...\" % lang_code)\n+        projection = json.dumps(PROJECTION_KEYS)\n+        url = API_URL.format(projection=projection, lang=lang_code, ka_domain=ka_domain)\n+        node_data_path = download_and_clean_kalite_data(url, lang=lang_code, ignorecache=force, filename=\"nodes.json\")\n+        with open(node_data_path, 'r') as f:\n+            node_data_temp = ujson.load(f)\n+        \"\"\"\n+        Some translated_youtube_lang value return from KHAN API did not match\n+            to the specify language code. We need to override it to use the same\n+            language code.\n+        Example: using pt-BR language code in the khan api will return pt translated_youtube_lang.\n+        \"\"\"\n+        for node_temp in node_data_temp:\n+            node_kind = node_temp.get(\"kind\")\n+            if (node_kind == NodeType.topic):\n+                if not node_temp[\"path\"] in topic_path_list:\n+                    topic_path_list.append(node_temp[\"path\"])\n+                    node_data.append(node_temp)\n+            if (node_kind == NodeType.exercise):\n+                if not node_temp[\"id\"] in exercise_ids:\n+                    exercise_ids.append(node_temp[\"id\"])\n+                    node_data.append(node_temp)    \n+            if (node_kind == NodeType.video):\n+                if not node_temp[\"youtube_id\"] in youtube_ids:\n+                    youtube_lang = node_temp[\"translated_youtube_lang\"]\n+                    if youtube_lang == lang:\n+                        youtube_ids.append(node_temp[\"youtube_id\"])\n+                        node_data.append(node_temp)\n+                    elif not youtube_lang != EN_LANG_CODE:","path":"contentpacks/khanacademy.py","position":271,"original_position":271,"commit_id":"7c4b9ac9ed1b3cb22c9d5ade841dda73425cf62e","original_commit_id":"7c4b9ac9ed1b3cb22c9d5ade841dda73425cf62e","user":{"login":"cpauya","id":175580,"avatar_url":"https://avatars.githubusercontent.com/u/175580?v=3","gravatar_id":"","url":"https://api.github.com/users/cpauya","html_url":"https://github.com/cpauya","followers_url":"https://api.github.com/users/cpauya/followers","following_url":"https://api.github.com/users/cpauya/following{/other_user}","gists_url":"https://api.github.com/users/cpauya/gists{/gist_id}","starred_url":"https://api.github.com/users/cpauya/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cpauya/subscriptions","organizations_url":"https://api.github.com/users/cpauya/orgs","repos_url":"https://api.github.com/users/cpauya/repos","events_url":"https://api.github.com/users/cpauya/events{/privacy}","received_events_url":"https://api.github.com/users/cpauya/received_events","type":"User","site_admin":false},"body":"To simplify, does this mean if `youtube_lang == EN_LANG_CODE`?","created_at":"2016-10-18T23:42:17Z","updated_at":"2016-10-18T23:42:17Z","html_url":"https://github.com/fle-internal/content-pack-maker/pull/39#discussion_r83973848","pull_request_url":"https://api.github.com/repos/fle-internal/content-pack-maker/pulls/39","_links":{"self":{"href":"https://api.github.com/repos/fle-internal/content-pack-maker/pulls/comments/83973848"},"html":{"href":"https://github.com/fle-internal/content-pack-maker/pull/39#discussion_r83973848"},"pull_request":{"href":"https://api.github.com/repos/fle-internal/content-pack-maker/pulls/39"}}},"pull_request":{"url":"https://api.github.com/repos/fle-internal/content-pack-maker/pulls/39","id":87043671,"html_url":"https://github.com/fle-internal/content-pack-maker/pull/39","diff_url":"https://github.com/fle-internal/content-pack-maker/pull/39.diff","patch_url":"https://github.com/fle-internal/content-pack-maker/pull/39.patch","issue_url":"https://api.github.com/repos/fle-internal/content-pack-maker/issues/39","number":39,"state":"open","locked":false,"title":"Used language name and native name for looking in dubbed video mappings.","user":{"login":"mrpau-richard","id":8663934,"avatar_url":"https://avatars.githubusercontent.com/u/8663934?v=3","gravatar_id":"","url":"https://api.github.com/users/mrpau-richard","html_url":"https://github.com/mrpau-richard","followers_url":"https://api.github.com/users/mrpau-richard/followers","following_url":"https://api.github.com/users/mrpau-richard/following{/other_user}","gists_url":"https://api.github.com/users/mrpau-richard/gists{/gist_id}","starred_url":"https://api.github.com/users/mrpau-richard/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrpau-richard/subscriptions","organizations_url":"https://api.github.com/users/mrpau-richard/orgs","repos_url":"https://api.github.com/users/mrpau-richard/repos","events_url":"https://api.github.com/users/mrpau-richard/events{/privacy}","received_events_url":"https://api.github.com/users/mrpau-richard/received_events","type":"User","site_admin":false},"body":"This fixes #38 & [#5294](https://github.com/learningequality/ka-lite/issues/5294)\r\n\r\nUsed language name and native name as reference when looking in [dubbed video mappings](https://docs.google.com/spreadsheets/d/1k5xh2UXV3EchRHnYzeP6-YGKrmba22vsltGSuU9bL88/edit#gid=0).\r\n\r\nIt may also solve other languages that use the native name in [dubbed video mappings](https://docs.google.com/spreadsheets/d/1k5xh2UXV3EchRHnYzeP6-YGKrmba22vsltGSuU9bL88/edit#gid=0).\r\n\r\nI uploaded a sample screenshot of `Swahili` when downloading videos.\r\n![screen shot 2016-09-28 at 9 07 44 am](https://cloud.githubusercontent.com/assets/8663934/18898304/562ead5c-8563-11e6-9739-90868cbd4d2f.png)\r\n\r\nI also uploaded the `Swahili` language pack for testing.\r\n[sw.zip](https://github.com/fle-internal/content-pack-maker/files/496997/sw.zip)\r\n\r\n\r\n/cc @aronasorman @jamalex @mrpau-eduard @benjaoming ","created_at":"2016-09-28T02:10:18Z","updated_at":"2016-10-18T23:42:17Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5041dae51d2540ef3ae46d2f6c2d443c0b75139a","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/fle-internal/content-pack-maker/pulls/39/commits","review_comments_url":"https://api.github.com/repos/fle-internal/content-pack-maker/pulls/39/comments","review_comment_url":"https://api.github.com/repos/fle-internal/content-pack-maker/pulls/comments{/number}","comments_url":"https://api.github.com/repos/fle-internal/content-pack-maker/issues/39/comments","statuses_url":"https://api.github.com/repos/fle-internal/content-pack-maker/statuses/7c4b9ac9ed1b3cb22c9d5ade841dda73425cf62e","head":{"label":"mrpau:feature/#38-some_video_nodes_incorrectly_flagged_having_no_files","ref":"feature/#38-some_video_nodes_incorrectly_flagged_having_no_files","sha":"7c4b9ac9ed1b3cb22c9d5ade841dda73425cf62e","user":{"login":"mrpau","id":8663898,"avatar_url":"https://avatars.githubusercontent.com/u/8663898?v=3","gravatar_id":"","url":"https://api.github.com/users/mrpau","html_url":"https://github.com/mrpau","followers_url":"https://api.github.com/users/mrpau/followers","following_url":"https://api.github.com/users/mrpau/following{/other_user}","gists_url":"https://api.github.com/users/mrpau/gists{/gist_id}","starred_url":"https://api.github.com/users/mrpau/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrpau/subscriptions","organizations_url":"https://api.github.com/users/mrpau/orgs","repos_url":"https://api.github.com/users/mrpau/repos","events_url":"https://api.github.com/users/mrpau/events{/privacy}","received_events_url":"https://api.github.com/users/mrpau/received_events","type":"Organization","site_admin":false},"repo":{"id":60687452,"name":"content-pack-maker","full_name":"mrpau/content-pack-maker","owner":{"login":"mrpau","id":8663898,"avatar_url":"https://avatars.githubusercontent.com/u/8663898?v=3","gravatar_id":"","url":"https://api.github.com/users/mrpau","html_url":"https://github.com/mrpau","followers_url":"https://api.github.com/users/mrpau/followers","following_url":"https://api.github.com/users/mrpau/following{/other_user}","gists_url":"https://api.github.com/users/mrpau/gists{/gist_id}","starred_url":"https://api.github.com/users/mrpau/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrpau/subscriptions","organizations_url":"https://api.github.com/users/mrpau/orgs","repos_url":"https://api.github.com/users/mrpau/repos","events_url":"https://api.github.com/users/mrpau/events{/privacy}","received_events_url":"https://api.github.com/users/mrpau/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mrpau/content-pack-maker","description":null,"fork":true,"url":"https://api.github.com/repos/mrpau/content-pack-maker","forks_url":"https://api.github.com/repos/mrpau/content-pack-maker/forks","keys_url":"https://api.github.com/repos/mrpau/content-pack-maker/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mrpau/content-pack-maker/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mrpau/content-pack-maker/teams","hooks_url":"https://api.github.com/repos/mrpau/content-pack-maker/hooks","issue_events_url":"https://api.github.com/repos/mrpau/content-pack-maker/issues/events{/number}","events_url":"https://api.github.com/repos/mrpau/content-pack-maker/events","assignees_url":"https://api.github.com/repos/mrpau/content-pack-maker/assignees{/user}","branches_url":"https://api.github.com/repos/mrpau/content-pack-maker/branches{/branch}","tags_url":"https://api.github.com/repos/mrpau/content-pack-maker/tags","blobs_url":"https://api.github.com/repos/mrpau/content-pack-maker/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mrpau/content-pack-maker/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mrpau/content-pack-maker/git/refs{/sha}","trees_url":"https://api.github.com/repos/mrpau/content-pack-maker/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mrpau/content-pack-maker/statuses/{sha}","languages_url":"https://api.github.com/repos/mrpau/content-pack-maker/languages","stargazers_url":"https://api.github.com/repos/mrpau/content-pack-maker/stargazers","contributors_url":"https://api.github.com/repos/mrpau/content-pack-maker/contributors","subscribers_url":"https://api.github.com/repos/mrpau/content-pack-maker/subscribers","subscription_url":"https://api.github.com/repos/mrpau/content-pack-maker/subscription","commits_url":"https://api.github.com/repos/mrpau/content-pack-maker/commits{/sha}","git_commits_url":"https://api.github.com/repos/mrpau/content-pack-maker/git/commits{/sha}","comments_url":"https://api.github.com/repos/mrpau/content-pack-maker/comments{/number}","issue_comment_url":"https://api.github.com/repos/mrpau/content-pack-maker/issues/comments{/number}","contents_url":"https://api.github.com/repos/mrpau/content-pack-maker/contents/{+path}","compare_url":"https://api.github.com/repos/mrpau/content-pack-maker/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mrpau/content-pack-maker/merges","archive_url":"https://api.github.com/repos/mrpau/content-pack-maker/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mrpau/content-pack-maker/downloads","issues_url":"https://api.github.com/repos/mrpau/content-pack-maker/issues{/number}","pulls_url":"https://api.github.com/repos/mrpau/content-pack-maker/pulls{/number}","milestones_url":"https://api.github.com/repos/mrpau/content-pack-maker/milestones{/number}","notifications_url":"https://api.github.com/repos/mrpau/content-pack-maker/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mrpau/content-pack-maker/labels{/name}","releases_url":"https://api.github.com/repos/mrpau/content-pack-maker/releases{/id}","deployments_url":"https://api.github.com/repos/mrpau/content-pack-maker/deployments","created_at":"2016-06-08T09:49:09Z","updated_at":"2016-06-08T09:49:10Z","pushed_at":"2016-10-18T08:33:39Z","git_url":"git://github.com/mrpau/content-pack-maker.git","ssh_url":"git@github.com:mrpau/content-pack-maker.git","clone_url":"https://github.com/mrpau/content-pack-maker.git","svn_url":"https://github.com/mrpau/content-pack-maker","homepage":null,"size":96014,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"fle-internal:master","ref":"master","sha":"68fc0071d053947d2fd64d6e068124201ca23aa2","user":{"login":"fle-internal","id":6564452,"avatar_url":"https://avatars.githubusercontent.com/u/6564452?v=3","gravatar_id":"","url":"https://api.github.com/users/fle-internal","html_url":"https://github.com/fle-internal","followers_url":"https://api.github.com/users/fle-internal/followers","following_url":"https://api.github.com/users/fle-internal/following{/other_user}","gists_url":"https://api.github.com/users/fle-internal/gists{/gist_id}","starred_url":"https://api.github.com/users/fle-internal/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fle-internal/subscriptions","organizations_url":"https://api.github.com/users/fle-internal/orgs","repos_url":"https://api.github.com/users/fle-internal/repos","events_url":"https://api.github.com/users/fle-internal/events{/privacy}","received_events_url":"https://api.github.com/users/fle-internal/received_events","type":"Organization","site_admin":false},"repo":{"id":43916699,"name":"content-pack-maker","full_name":"fle-internal/content-pack-maker","owner":{"login":"fle-internal","id":6564452,"avatar_url":"https://avatars.githubusercontent.com/u/6564452?v=3","gravatar_id":"","url":"https://api.github.com/users/fle-internal","html_url":"https://github.com/fle-internal","followers_url":"https://api.github.com/users/fle-internal/followers","following_url":"https://api.github.com/users/fle-internal/following{/other_user}","gists_url":"https://api.github.com/users/fle-internal/gists{/gist_id}","starred_url":"https://api.github.com/users/fle-internal/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fle-internal/subscriptions","organizations_url":"https://api.github.com/users/fle-internal/orgs","repos_url":"https://api.github.com/users/fle-internal/repos","events_url":"https://api.github.com/users/fle-internal/events{/privacy}","received_events_url":"https://api.github.com/users/fle-internal/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/fle-internal/content-pack-maker","description":null,"fork":false,"url":"https://api.github.com/repos/fle-internal/content-pack-maker","forks_url":"https://api.github.com/repos/fle-internal/content-pack-maker/forks","keys_url":"https://api.github.com/repos/fle-internal/content-pack-maker/keys{/key_id}","collaborators_url":"https://api.github.com/repos/fle-internal/content-pack-maker/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/fle-internal/content-pack-maker/teams","hooks_url":"https://api.github.com/repos/fle-internal/content-pack-maker/hooks","issue_events_url":"https://api.github.com/repos/fle-internal/content-pack-maker/issues/events{/number}","events_url":"https://api.github.com/repos/fle-internal/content-pack-maker/events","assignees_url":"https://api.github.com/repos/fle-internal/content-pack-maker/assignees{/user}","branches_url":"https://api.github.com/repos/fle-internal/content-pack-maker/branches{/branch}","tags_url":"https://api.github.com/repos/fle-internal/content-pack-maker/tags","blobs_url":"https://api.github.com/repos/fle-internal/content-pack-maker/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/fle-internal/content-pack-maker/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/fle-internal/content-pack-maker/git/refs{/sha}","trees_url":"https://api.github.com/repos/fle-internal/content-pack-maker/git/trees{/sha}","statuses_url":"https://api.github.com/repos/fle-internal/content-pack-maker/statuses/{sha}","languages_url":"https://api.github.com/repos/fle-internal/content-pack-maker/languages","stargazers_url":"https://api.github.com/repos/fle-internal/content-pack-maker/stargazers","contributors_url":"https://api.github.com/repos/fle-internal/content-pack-maker/contributors","subscribers_url":"https://api.github.com/repos/fle-internal/content-pack-maker/subscribers","subscription_url":"https://api.github.com/repos/fle-internal/content-pack-maker/subscription","commits_url":"https://api.github.com/repos/fle-internal/content-pack-maker/commits{/sha}","git_commits_url":"https://api.github.com/repos/fle-internal/content-pack-maker/git/commits{/sha}","comments_url":"https://api.github.com/repos/fle-internal/content-pack-maker/comments{/number}","issue_comment_url":"https://api.github.com/repos/fle-internal/content-pack-maker/issues/comments{/number}","contents_url":"https://api.github.com/repos/fle-internal/content-pack-maker/contents/{+path}","compare_url":"https://api.github.com/repos/fle-internal/content-pack-maker/compare/{base}...{head}","merges_url":"https://api.github.com/repos/fle-internal/content-pack-maker/merges","archive_url":"https://api.github.com/repos/fle-internal/content-pack-maker/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/fle-internal/content-pack-maker/downloads","issues_url":"https://api.github.com/repos/fle-internal/content-pack-maker/issues{/number}","pulls_url":"https://api.github.com/repos/fle-internal/content-pack-maker/pulls{/number}","milestones_url":"https://api.github.com/repos/fle-internal/content-pack-maker/milestones{/number}","notifications_url":"https://api.github.com/repos/fle-internal/content-pack-maker/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/fle-internal/content-pack-maker/labels{/name}","releases_url":"https://api.github.com/repos/fle-internal/content-pack-maker/releases{/id}","deployments_url":"https://api.github.com/repos/fle-internal/content-pack-maker/deployments","created_at":"2015-10-08T21:15:13Z","updated_at":"2016-06-15T20:18:34Z","pushed_at":"2016-10-18T08:33:39Z","git_url":"git://github.com/fle-internal/content-pack-maker.git","ssh_url":"git@github.com:fle-internal/content-pack-maker.git","clone_url":"https://github.com/fle-internal/content-pack-maker.git","svn_url":"https://github.com/fle-internal/content-pack-maker","homepage":null,"size":94607,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"open_issues_count":9,"forks":5,"open_issues":9,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/fle-internal/content-pack-maker/pulls/39"},"html":{"href":"https://github.com/fle-internal/content-pack-maker/pull/39"},"issue":{"href":"https://api.github.com/repos/fle-internal/content-pack-maker/issues/39"},"comments":{"href":"https://api.github.com/repos/fle-internal/content-pack-maker/issues/39/comments"},"review_comments":{"href":"https://api.github.com/repos/fle-internal/content-pack-maker/pulls/39/comments"},"review_comment":{"href":"https://api.github.com/repos/fle-internal/content-pack-maker/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/fle-internal/content-pack-maker/pulls/39/commits"},"statuses":{"href":"https://api.github.com/repos/fle-internal/content-pack-maker/statuses/7c4b9ac9ed1b3cb22c9d5ade841dda73425cf62e"}}}}
{"actor":{"display_login":"cpauya"}}