WikiCrawler/Tasks/Commons/CommonsCreatorFromWikidata.cs
/// <summary>
/// Creates creator templates as indicated by 'Category:Creator templates to be created by a bot'.
/// </summary>
Creates creator templates as indicated by 'Category:Creator templates to be created by a bot'.
Creates creator templates as indicated by ' Category : Creator templates to be created by a bot ' .
[BatchTask]
		public static void MakeCreatorsFromCat()
		{
			Api commonsApi = new Api(new Uri("https://commons.wikimedia.org"));
			Console.WriteLine("Logging in...");
			commonsApi.AutoLogIn();
			Wikidata.AutoLogIn();
			int successLimit = s_TestLimit;
			string normalCatLastPage = "";
			if (File.Exists("autocreators/stored.txt"))
			{
				using (StreamReader reader = new StreamReader(new FileStream("autocreators/stored.txt", FileMode.Open), Encoding.Default))
				{
					normalCatLastPage = reader.ReadLine();
				}
			}
			try
			{
				if (s_DoBotCat)
				{
					foreach (Article article in commonsApi.GetCategoryEntries("Category:Creator templates to be created by a bot", cmtype: CMType.subcat))
					{
						Console.WriteLine("Checking verified '" + article.title + "'...");
						if (!article.title.StartsWith("Category:"))
						{
							Console.WriteLine("Error: not a category.");
							continue;
						}
						Article articleContent = commonsApi.GetPage(article);
						string text = articleContent.revisions[0].text;
						string wikidataId = WikiUtils.GetTemplateParameter("wikidata", text);
						Console.WriteLine("Wikidata Id is " + wikidataId);
						if (ProcessCreatorCategory(commonsApi, articleContent, wikidataId))
						{
							--successLimit;
							if (successLimit <= 0)
							{
								return;
							}
						}
					}
				}
				if (s_DoNormalCat)
				{
					foreach (Article article in commonsApi.GetCategoryEntries("Category:People by name", cmtype: CMType.subcat, cmstartsortkeyprefix: normalCatLastPage))
					{
						Console.WriteLine("----- Checking '" + article.title + "'...");
						if (!article.title.StartsWith("Category:"))
						{
							Console.WriteLine("Error: not a category.");
							continue;
						}
						Article articleContent = commonsApi.GetPage(article);
						string text = articleContent.revisions[0].text;
						string name = article.title.Substring("Category:".Length);
						string wikidataId = "";
						if (string.IsNullOrEmpty(wikidataId))
						{
							string creatorPre = "{{Creator:";
							int creatorIndex = text.IndexOf(creatorPre);
							if (creatorIndex >= 0)
							{
								int creatorEnd = text.IndexOf("}}", creatorIndex);
								string creator = text.Substring(creatorIndex + 2, creatorEnd - (creatorIndex + 2));
								Article creatorArt = commonsApi.GetPage(creator);
								if (!MediaWiki.Article.IsNullOrEmpty(creatorArt))
								{
									Console.WriteLine("Checking Creator for wikidata id");
									wikidataId = MediaWiki.WikiUtils.GetTemplateParameter("wikidata", creatorArt.revisions[0].text);
								}
							}
						}
						if (string.IsNullOrEmpty(wikidataId))
						{
							string onWikidataTemplate;
							MediaWiki.WikiUtils.RemoveTemplate("On Wikidata", text, out onWikidataTemplate);
							if (!string.IsNullOrEmpty(onWikidataTemplate))
							{
								string[] split = onWikidataTemplate.Split('|');
								if (split.Length == 2)
								{
									Console.WriteLine("Matched ON WIKIDATA");
									wikidataId = split[1];
								}
							}
						}
						if (string.IsNullOrEmpty(wikidataId))
						{
							string yearOfBirth = "";
							string yearOfDeath = "";
							foreach (string cat in MediaWiki.WikiUtils.GetCategories(text))
							{
								if (cat.EndsWith(" births"))
								{
									yearOfBirth = cat.Substring("Category:".Length, 4);
								}
								else if (cat.EndsWith(" deaths"))
								{
									yearOfDeath = cat.Substring("Category:".Length, 4);
								}
							}
							if (string.IsNullOrEmpty(yearOfBirth) || string.IsNullOrEmpty(yearOfDeath))
							{
								Console.WriteLine("Insufficient information.");
							}
							else
							{
								wikidataId = GetWikidataId(article.GetTitle(), yearOfBirth, yearOfDeath);
								if (!string.IsNullOrEmpty(wikidataId))
								{
									Console.WriteLine("Got wikidata from Creator");
								}
							}
						}
						if (!string.IsNullOrEmpty(wikidataId))
						{
							Console.WriteLine("++ Matched " + wikidataId + ".");
							if (ProcessCreatorCategory(commonsApi, articleContent, wikidataId))
							{
								--successLimit;
								if (successLimit <= 0)
								{
									Console.WriteLine("Hit limit.");
									return;
								}
							}
						}
						else
						{
							Console.WriteLine("Couldn't determine Wikidata ID.");
						}
						normalCatLastPage = MediaWiki.WikiUtils.GetSortkey(article);
					}
				}
			}
			finally
			{
				using (StreamWriter writer = new StreamWriter(new FileStream("implicitcreators/stored.txt", FileMode.Create), Encoding.Default))
				{
					writer.WriteLine(normalCatLastPage);
				}
			}
		}
[ BatchTask ] public static void MakeCreatorsFromCat ( ) { Api commonsApi = new Api ( new Uri ( " " ) ) ; Console . WriteLine ( " " ) ; commonsApi . AutoLogIn ( ) ; Wikidata . AutoLogIn ( ) ; int successLimit = s_TestLimit ; string normalCatLastPage = " " ; if ( File . Exists ( " " ) ) { using ( StreamReader reader = new StreamReader ( new FileStream ( " " , FileMode . Open ) , Encoding . Default ) ) { normalCatLastPage = reader . ReadLine ( ) ; } } try { if ( s_DoBotCat ) { foreach ( Article article in commonsApi . GetCategoryEntries ( " " , cmtype : CMType . subcat ) ) { Console . WriteLine ( " " + article . title + " " ) ; if ( ! article . title . StartsWith ( " " ) ) { Console . WriteLine ( " " ) ; continue ; } Article articleContent = commonsApi . GetPage ( article ) ; string text = articleContent . revisions [ 0 ] . text ; string wikidataId = WikiUtils . GetTemplateParameter ( " " , text ) ; Console . WriteLine ( " " + wikidataId ) ; if ( ProcessCreatorCategory ( commonsApi , articleContent , wikidataId ) ) { -- successLimit ; if ( successLimit <= 0 ) { return ; } } } } if ( s_DoNormalCat ) { foreach ( Article article in commonsApi . GetCategoryEntries ( " " , cmtype : CMType . subcat , cmstartsortkeyprefix : normalCatLastPage ) ) { Console . WriteLine ( " " + article . title + " " ) ; if ( ! article . title . StartsWith ( " " ) ) { Console . WriteLine ( " " ) ; continue ; } Article articleContent = commonsApi . GetPage ( article ) ; string text = articleContent . revisions [ 0 ] . text ; string name = article . title . Substring ( " " . Length ) ; string wikidataId = " " ; if ( string . IsNullOrEmpty ( wikidataId ) ) { string creatorPre = " " ; int creatorIndex = text . IndexOf ( creatorPre ) ; if ( creatorIndex >= 0 ) { int creatorEnd = text . IndexOf ( " " , creatorIndex ) ; string creator = text . Substring ( creatorIndex + 2 , creatorEnd - ( creatorIndex + 2 ) ) ; Article creatorArt = commonsApi . GetPage ( creator ) ; if ( ! MediaWiki . Article . IsNullOrEmpty ( creatorArt ) ) { Console . WriteLine ( " " ) ; wikidataId = MediaWiki . WikiUtils . GetTemplateParameter ( " " , creatorArt . revisions [ 0 ] . text ) ; } } } if ( string . IsNullOrEmpty ( wikidataId ) ) { string onWikidataTemplate ; MediaWiki . WikiUtils . RemoveTemplate ( " " , text , out onWikidataTemplate ) ; if ( ! string . IsNullOrEmpty ( onWikidataTemplate ) ) { string [ ] split = onWikidataTemplate . Split ( ' ' ) ; if ( split . Length == 2 ) { Console . WriteLine ( " " ) ; wikidataId = split [ 1 ] ; } } } if ( string . IsNullOrEmpty ( wikidataId ) ) { string yearOfBirth = " " ; string yearOfDeath = " " ; foreach ( string cat in MediaWiki . WikiUtils . GetCategories ( text ) ) { if ( cat . EndsWith ( " " ) ) { yearOfBirth = cat . Substring ( " " . Length , 4 ) ; } else if ( cat . EndsWith ( " " ) ) { yearOfDeath = cat . Substring ( " " . Length , 4 ) ; } } if ( string . IsNullOrEmpty ( yearOfBirth ) || string . IsNullOrEmpty ( yearOfDeath ) ) { Console . WriteLine ( " " ) ; } else { wikidataId = GetWikidataId ( article . GetTitle ( ) , yearOfBirth , yearOfDeath ) ; if ( ! string . IsNullOrEmpty ( wikidataId ) ) { Console . WriteLine ( " " ) ; } } } if ( ! string . IsNullOrEmpty ( wikidataId ) ) { Console . WriteLine ( " " + wikidataId + " " ) ; if ( ProcessCreatorCategory ( commonsApi , articleContent , wikidataId ) ) { -- successLimit ; if ( successLimit <= 0 ) { Console . WriteLine ( " " ) ; return ; } } } else { Console . WriteLine ( " " ) ; } normalCatLastPage = MediaWiki . WikiUtils . GetSortkey ( article ) ; } } } finally { using ( StreamWriter writer = new StreamWriter ( new FileStream ( " " , FileMode . Create ) , Encoding . Default ) ) { writer . WriteLine ( normalCatLastPage ) ; } } }
Creates creator templates as indicated by 'Category:Creator templates to be created by a bot'.
Creates creator templates as indicated by ' Category : Creator templates to be created by a bot ' .
// find template parameter // check creator for a wikidata idea // check for On Wikidata // look for DOB and DOD categories //HACK: only works with 4-digit years