WikiCrawler/Batch/BatchDownloader.cs
/// <summary>
/// Downloads the data for the specified key.
/// </summary>
/// <returns>The parsed data.</returns>
Downloads the data for the specified key.
Downloads the data for the specified key .
public Dictionary<string, string> Download(string key, bool cache = true)
	{
		if (!m_config.allowDataDownload)
		{
			throw new Exception("Data download disabled");
		}
		Console.WriteLine("Downloading metadata: " + key);
		Uri url = GetItemUri(key);
	redownload:
		string content = Download(url);
		try
		{
			Dictionary<string, string> metadata = ParseMetadata(content);
			if (metadata == null)
			{
				m_succeeded.Add(key);
			}
			else if (cache)
			{
				File.WriteAllText(
					GetMetadataCacheFilename(key),
					JsonConvert.SerializeObject(metadata, Formatting.Indented),
					Encoding.UTF8);
			}
			return metadata;
		}
		catch (RedownloadException)
		{
			goto redownload;
		}
		catch (Exception e)
		{
			Console.WriteLine(e.ToString());
			m_failMessages.Add(key.PadLeft(5) + "\t" + e.Message);
			return null;
		}
	}
public Dictionary < string , string > Download ( string key , bool cache = true ) { if ( ! m_config . allowDataDownload ) { throw new Exception ( " " ) ; } Console . WriteLine ( " " + key ) ; Uri url = GetItemUri ( key ) ; redownload : string content = Download ( url ) ; try { Dictionary < string , string > metadata = ParseMetadata ( content ) ; if ( metadata == null ) { m_succeeded . Add ( key ) ; } else if ( cache ) { File . WriteAllText ( GetMetadataCacheFilename ( key ) , JsonConvert . SerializeObject ( metadata , Formatting . Indented ) , Encoding . UTF8 ) ; } return metadata ; } catch ( RedownloadException ) { goto redownload ; } catch ( Exception e ) { Console . WriteLine ( e . ToString ( ) ) ; m_failMessages . Add ( key . PadLeft ( 5 ) + " \t " + e . Message ) ; return null ; } }
Downloads the data for the specified key.
Downloads the data for the specified key .
//TODO: crash when failing here
The parsed data.
The parsed data .
key
string
cache
bool