WikiCrawler/Batch/BatchDownloader.cs
/// <summary>
/// Downloads all metadata and images.
/// </summary>
Downloads all metadata and images.
Downloads all metadata and images .
public void DownloadAll()
	{
		string stopFile = Path.Combine(Configuration.DataDirectory, "STOP");
		try
		{
			HashSet<string> succeededMetadata = new HashSet<string>();
			foreach (string cachedData in Directory.GetFiles(MetadataCacheDirectory))
			{
				succeededMetadata.Add(Path.GetFileNameWithoutExtension(cachedData));
			}
			int saveOutInterval = 10;
			int saveOutTimer = saveOutInterval;
			m_heartbeatData["nTotal"] = GetKeys().Count();
			m_heartbeatData["nCompleted"] = m_succeeded.Count;
			m_heartbeatData["nDownloaded"] = succeededMetadata.Count;
			m_heartbeatData["nFailed"] = m_failMessages.Count;
			m_heartbeatData["nFailedLicense"] = 0;
			StartHeartbeat();
			foreach (string key in GetKeys())
			{
				if (!succeededMetadata.Contains(key) && !m_succeeded.Contains(key))
				{
					Dictionary<string, string> metadata = Download(key);
					if (metadata != null)
					{
						succeededMetadata.Add(key);
					}
					saveOutTimer--;
				}
				lock (m_heartbeatData)
				{
					m_heartbeatData["nCompleted"] = m_succeeded.Count;
					m_heartbeatData["nDownloaded"] = succeededMetadata.Count;
					m_heartbeatData["nFailed"] = m_failMessages.Count;
					m_heartbeatData["nFailedLicense"] = 0;
				}
				if (File.Exists(stopFile))
				{
					File.Delete(stopFile);
					Console.WriteLine("Received STOP signal.");
					return;
				}
				else if (Finished)
				{
					return;
				}
				if (saveOutTimer <= 0)
				{
					SaveOut();
					saveOutTimer = saveOutInterval;
				}
			}
		}
		finally
		{
			SaveOut();
			SendHeartbeat(true);
		}
	}
public void DownloadAll ( ) { string stopFile = Path . Combine ( Configuration . DataDirectory , " " ) ; try { HashSet < string > succeededMetadata = new HashSet < string > ( ) ; foreach ( string cachedData in Directory . GetFiles ( MetadataCacheDirectory ) ) { succeededMetadata . Add ( Path . GetFileNameWithoutExtension ( cachedData ) ) ; } int saveOutInterval = 10 ; int saveOutTimer = saveOutInterval ; m_heartbeatData [ " " ] = GetKeys ( ) . Count ( ) ; m_heartbeatData [ " " ] = m_succeeded . Count ; m_heartbeatData [ " " ] = succeededMetadata . Count ; m_heartbeatData [ " " ] = m_failMessages . Count ; m_heartbeatData [ " " ] = 0 ; StartHeartbeat ( ) ; foreach ( string key in GetKeys ( ) ) { if ( ! succeededMetadata . Contains ( key ) && ! m_succeeded . Contains ( key ) ) { Dictionary < string , string > metadata = Download ( key ) ; if ( metadata != null ) { succeededMetadata . Add ( key ) ; } saveOutTimer -- ; } lock ( m_heartbeatData ) { m_heartbeatData [ " " ] = m_succeeded . Count ; m_heartbeatData [ " " ] = succeededMetadata . Count ; m_heartbeatData [ " " ] = m_failMessages . Count ; m_heartbeatData [ " " ] = 0 ; } if ( File . Exists ( stopFile ) ) { File . Delete ( stopFile ) ; Console . WriteLine ( " " ) ; return ; } else if ( Finished ) { return ; } if ( saveOutTimer <= 0 ) { SaveOut ( ) ; saveOutTimer = saveOutInterval ; } } } finally { SaveOut ( ) ; SendHeartbeat ( true ) ; } }
Downloads all metadata and images.
Downloads all metadata and images .
// load the list of metadata that has already been downloaded // load metadata