WikiCrawler/Tasks/Commons/NsrwFollowup.cs
/// <summary>
/// Updates page text for images extracted from NSRW after extracting them with CropTool.
/// </summary>
Updates page text for images extracted from NSRW after extracting them with CropTool.
Updates page text for images extracted from NSRW after extracting them with CropTool .
[BatchTask]
		public static void Do()
		{
			List<string> queue = new List<string>();
			using (StreamReader reader = new StreamReader(new FileStream("nsrwqueue.txt", FileMode.Open)))
			{
				while (!reader.EndOfStream)
				{
					queue.Add(reader.ReadLine());
				}
			}
			string nsrwDerivedInfobox;
			using (StreamReader reader = new StreamReader(new FileStream("nsrwreplace.txt", FileMode.Open)))
			{
				nsrwDerivedInfobox = reader.ReadToEnd();
			}
			Dictionary<string, List<string>> sourceMappedImages = new Dictionary<string, List<string>>();
			if (File.Exists("nsrwsource.txt"))
			{
				using (StreamReader reader = new StreamReader(new FileStream("nsrwsource.txt", FileMode.Open)))
				{
					while (!reader.EndOfStream)
					{
						string key = reader.ReadLine();
						int count = int.Parse(reader.ReadLine());
						List<string> destinations = new List<string>();
						for (int c = 0; c < count; c++)
						{
							destinations.Add(reader.ReadLine());
						}
						sourceMappedImages[key] = destinations;
					}
				}
			}
			try
			{
				Console.WriteLine("Logging in...");
				Api = new MediaWiki.Api(new Uri("https://commons.wikimedia.org/"));
				Api.AutoLogIn();
				for (int c = queue.Count - 1; c >= 0; c--)
				{
					if (string.IsNullOrEmpty(queue[c].Trim()))
					{
						queue.RemoveAt(c);
						continue;
					}
					string file = queue[c];
					if (!file.StartsWith("File:"))
						file = "File:" + file;
					Console.WriteLine(file);
					MediaWiki.Article article = Api.GetPage(file);
					string text = article.revisions.First().text;
					string extractedFrom = "{{Extracted from|";
					int extractedFromIndex = text.IndexOf(extractedFrom);
					if (extractedFromIndex < 0)
					{
						extractedFrom = "{{ExtractedFromNSRW|";
						extractedFromIndex = text.IndexOf(extractedFrom);
						if (extractedFromIndex < 0)
						{
							Console.WriteLine(file + ": no Extracted from template");
							continue;
						}
					}
					int st = extractedFromIndex + extractedFrom.Length;
					int d = st;
					for (; d < text.Length && text[d] != '}'; d++) ;
					string sourceFile = text.Substring(st, d - st);
					text = text.Replace("{{Extracted from|", "{{ExtractedFromNSRW|");
					text = text.Replace("{{ExtractImage}}", "");
					text = text.Replace("{{LA2-NSRW}}", nsrwDerivedInfobox);
					if (MediaWiki.WikiUtils.HasNoCategories(text))
					{
						string month = CultureInfo.CurrentCulture.DateTimeFormat.GetMonthName(DateTime.Now.Month);
						text = text.TrimEnd() + "\n{{Uncategorized|year=" + DateTime.Now.Year + 
							"|month=" + month + "|day=" + DateTime.Now.Day + "}}";
					}
					if (!sourceFile.StartsWith("File:"))
						sourceFile = "File:" + sourceFile;
					if (!sourceMappedImages.ContainsKey(sourceFile))
						sourceMappedImages[sourceFile] = new List<string>();
					sourceMappedImages[sourceFile].Add(file);
					text = text.Replace("\r\n", "\n");
					if (text != article.revisions.First().text)
					{
						article.revisions.First().text = text;
						Api.EditPage(article, "extracted image", bot: false);
					}
					queue.RemoveAt(c);
				}
				string[] keys = sourceMappedImages.Keys.ToArray();
				foreach (string s in keys)
				{
					UpdateSource(s, sourceMappedImages[s]);
					sourceMappedImages.Remove(s);
				}
			}
			finally
			{
				using (StreamWriter writer = new StreamWriter(new FileStream("nsrwqueue.txt", FileMode.Create)))
				{
					foreach (string file in queue)
					{
						writer.WriteLine(file);
					}
				}
				using (StreamWriter write = new StreamWriter(new FileStream("nsrwsource.txt", FileMode.Create)))
				{
					foreach (KeyValuePair<string, List<string>> kv in sourceMappedImages)
					{
						write.WriteLine(kv.Key);
						write.WriteLine(kv.Value.Count);
						foreach (string s in kv.Value)
						{
							write.WriteLine(s);
						}
					}
				}
			}
		}
[ BatchTask ] public static void Do ( ) { List < string > queue = new List < string > ( ) ; using ( StreamReader reader = new StreamReader ( new FileStream ( " " , FileMode . Open ) ) ) { while ( ! reader . EndOfStream ) { queue . Add ( reader . ReadLine ( ) ) ; } } string nsrwDerivedInfobox ; using ( StreamReader reader = new StreamReader ( new FileStream ( " " , FileMode . Open ) ) ) { nsrwDerivedInfobox = reader . ReadToEnd ( ) ; } Dictionary < string , List < string > > sourceMappedImages = new Dictionary < string , List < string > > ( ) ; if ( File . Exists ( " " ) ) { using ( StreamReader reader = new StreamReader ( new FileStream ( " " , FileMode . Open ) ) ) { while ( ! reader . EndOfStream ) { string key = reader . ReadLine ( ) ; int count = int . Parse ( reader . ReadLine ( ) ) ; List < string > destinations = new List < string > ( ) ; for ( int c = 0 ; c < count ; c ++ ) { destinations . Add ( reader . ReadLine ( ) ) ; } sourceMappedImages [ key ] = destinations ; } } } try { Console . WriteLine ( " " ) ; Api = new MediaWiki . Api ( new Uri ( " " ) ) ; Api . AutoLogIn ( ) ; for ( int c = queue . Count - 1 ; c >= 0 ; c -- ) { if ( string . IsNullOrEmpty ( queue [ c ] . Trim ( ) ) ) { queue . RemoveAt ( c ) ; continue ; } string file = queue [ c ] ; if ( ! file . StartsWith ( " " ) ) file = " " + file ; Console . WriteLine ( file ) ; MediaWiki . Article article = Api . GetPage ( file ) ; string text = article . revisions . First ( ) . text ; string extractedFrom = " " ; int extractedFromIndex = text . IndexOf ( extractedFrom ) ; if ( extractedFromIndex < 0 ) { extractedFrom = " " ; extractedFromIndex = text . IndexOf ( extractedFrom ) ; if ( extractedFromIndex < 0 ) { Console . WriteLine ( file + " " ) ; continue ; } } int st = extractedFromIndex + extractedFrom . Length ; int d = st ; for ( ; d < text . Length && text [ d ] != ' ' ; d ++ ) ; string sourceFile = text . Substring ( st , d - st ) ; text = text . Replace ( " " , " " ) ; text = text . Replace ( " " , " " ) ; text = text . Replace ( " " , nsrwDerivedInfobox ) ; if ( MediaWiki . WikiUtils . HasNoCategories ( text ) ) { string month = CultureInfo . CurrentCulture . DateTimeFormat . GetMonthName ( DateTime . Now . Month ) ; text = text . TrimEnd ( ) + " \n " + DateTime . Now . Year + " " + month + " " + DateTime . Now . Day + " " ; } if ( ! sourceFile . StartsWith ( " " ) ) sourceFile = " " + sourceFile ; if ( ! sourceMappedImages . ContainsKey ( sourceFile ) ) sourceMappedImages [ sourceFile ] = new List < string > ( ) ; sourceMappedImages [ sourceFile ] . Add ( file ) ; text = text . Replace ( " \r \n " , " \n " ) ; if ( text != article . revisions . First ( ) . text ) { article . revisions . First ( ) . text = text ; Api . EditPage ( article , " " , bot : false ) ; } queue . RemoveAt ( c ) ; } string [ ] keys = sourceMappedImages . Keys . ToArray ( ) ; foreach ( string s in keys ) { UpdateSource ( s , sourceMappedImages [ s ] ) ; sourceMappedImages . Remove ( s ) ; } } finally { using ( StreamWriter writer = new StreamWriter ( new FileStream ( " " , FileMode . Create ) ) ) { foreach ( string file in queue ) { writer . WriteLine ( file ) ; } } using ( StreamWriter write = new StreamWriter ( new FileStream ( " " , FileMode . Create ) ) ) { foreach ( KeyValuePair < string , List < string > > kv in sourceMappedImages ) { write . WriteLine ( kv . Key ) ; write . WriteLine ( kv . Value . Count ) ; foreach ( string s in kv . Value ) { write . WriteLine ( s ) ; } } } } }
Updates page text for images extracted from NSRW after extracting them with CropTool.
Updates page text for images extracted from NSRW after extracting them with CropTool .
//Read queue //Find source article //Read it //Replacements //Establish source //write out source files //save progress