WikiCrawler/MediaWiki/Api.cs
/// <summary>
/// Returns the Wiki entity data for all of the specified pages
/// </summary>
Returns the Wiki entity data for all of the specified pages
Returns the Wiki entity data for all of the specified pages
public Entity[] GetEntities(IList<string> ids = null,
			string sites = "",
			string titles = "",
			bool? redirects = null,
			string props = "info|sitelinks|aliases|labels|descriptions|claims|datatype",
			string languages = "",
			bool? languageFallback = null,
			bool? normalize = null,
			string sitefilter = "")
		{
			if (ids.Count == 0) return new Entity[0];
			const int maxEntities = 500;
			if (ids.Count > maxEntities)
			{
				int index = 0;
				List<string> idBuffer = new List<string>(maxEntities);
				List<Entity> results = new List<Entity>(maxEntities);
				while (true)
				{
					for (int i = 0; i < maxEntities; i++)
					{
						if (index + i >= ids.Count) break;
						idBuffer.Add(ids[index + i]);
					}
					Entity[] intermediateEntities = GetEntities(idBuffer, sites, titles, redirects, props, languages, languageFallback, normalize, sitefilter);
					results.AddRange(intermediateEntities);
					if (index >= ids.Count)
					{
						return results.ToArray();
					}
					index += maxEntities;
					idBuffer.Clear();
				}
			}
			string baseQuery = "format=json"
				+ "&action=wbgetentities";
			if (ids != null)
			{
				baseQuery += "&ids=" + UrlEncode(string.Join("|", ids));
			}
			if (!string.IsNullOrEmpty(sites))
			{
				baseQuery += "&sites=" + UrlEncode(sites);
			}
			if (!string.IsNullOrEmpty(titles))
			{
				baseQuery += "&titles=" + UrlEncode(titles);
			}
			if (redirects.HasValue)
			{
				baseQuery += "&redirects=" + (redirects.Value ? "yes" : "no");
			}
			if (!string.IsNullOrEmpty(props))
			{
				baseQuery += "&props=" + UrlEncode(props);
			}
			if (!string.IsNullOrEmpty(languages))
			{
				baseQuery += "&languages=" + UrlEncode(languages);
			}
			if (languageFallback.HasValue)
			{
				baseQuery += "&languageFallback=" + languageFallback.Value.ToString();
			}
			if (normalize.HasValue)
			{
				baseQuery += "&normalize=" + normalize.Value.ToString();
			}
			HttpWebRequest request = CreateApiRequest(baseQuery);
			string json;
			using (StreamReader read = new StreamReader(EasyWeb.GetResponseStream(request)))
			{
				json = read.ReadToEnd();
			}
			Dictionary<string, object> deser = (Dictionary<string, object>)s_jsonSerializer.DeserializeObject(json);
			if (!deser.ContainsKey("entities"))
			{
				return new Entity[0];
			}
			Dictionary<string, object> entities = (Dictionary<string, object>)deser["entities"];
			Entity[] ret = new Entity[entities.Count];
			int current = 0;
			foreach (KeyValuePair<string, object> page in entities)
			{
				ret[current++] = new Entity((Dictionary<string, object>)page.Value);
			}
			return ret;
		}
public Entity [ ] GetEntities ( IList < string > ids = null , string sites = " " , string titles = " " , bool ? redirects = null , string props = " " , string languages = " " , bool ? languageFallback = null , bool ? normalize = null , string sitefilter = " " ) { if ( ids . Count == 0 ) return new Entity [ 0 ] ; const int maxEntities = 500 ; if ( ids . Count > maxEntities ) { int index = 0 ; List < string > idBuffer = new List < string > ( maxEntities ) ; List < Entity > results = new List < Entity > ( maxEntities ) ; while ( true ) { for ( int i = 0 ; i < maxEntities ; i ++ ) { if ( index + i >= ids . Count ) break ; idBuffer . Add ( ids [ index + i ] ) ; } Entity [ ] intermediateEntities = GetEntities ( idBuffer , sites , titles , redirects , props , languages , languageFallback , normalize , sitefilter ) ; results . AddRange ( intermediateEntities ) ; if ( index >= ids . Count ) { return results . ToArray ( ) ; } index += maxEntities ; idBuffer . Clear ( ) ; } } string baseQuery = " " + " " ; if ( ids != null ) { baseQuery += " " + UrlEncode ( string . Join ( " " , ids ) ) ; } if ( ! string . IsNullOrEmpty ( sites ) ) { baseQuery += " " + UrlEncode ( sites ) ; } if ( ! string . IsNullOrEmpty ( titles ) ) { baseQuery += " " + UrlEncode ( titles ) ; } if ( redirects . HasValue ) { baseQuery += " " + ( redirects . Value ? " " : " " ) ; } if ( ! string . IsNullOrEmpty ( props ) ) { baseQuery += " " + UrlEncode ( props ) ; } if ( ! string . IsNullOrEmpty ( languages ) ) { baseQuery += " " + UrlEncode ( languages ) ; } if ( languageFallback . HasValue ) { baseQuery += " " + languageFallback . Value . ToString ( ) ; } if ( normalize . HasValue ) { baseQuery += " " + normalize . Value . ToString ( ) ; } HttpWebRequest request = CreateApiRequest ( baseQuery ) ; string json ; using ( StreamReader read = new StreamReader ( EasyWeb . GetResponseStream ( request ) ) ) { json = read . ReadToEnd ( ) ; } Dictionary < string , object > deser = ( Dictionary < string , object > ) s_jsonSerializer . DeserializeObject ( json ) ; if ( ! deser . ContainsKey ( " " ) ) { return new Entity [ 0 ] ; } Dictionary < string , object > entities = ( Dictionary < string , object > ) deser [ " " ] ; Entity [ ] ret = new Entity [ entities . Count ] ; int current = 0 ; foreach ( KeyValuePair < string , object > page in entities ) { ret [ current ++ ] = new Entity ( ( Dictionary < string , object > ) page . Value ) ; } return ret ; }
Returns the Wiki entity data for all of the specified pages
Returns the Wiki entity data for all of the specified pages
// maximum simultaneous request is 500 for bots on Wikidata // Read response //Parse and read //TODO: error handling?
ids
IList<string>
sites
string
titles
string
redirects
bool?
props
string
languages
string
languageFallback
bool?
normalize
bool?
sitefilter
string