WebCache/WebCache.cs
/// <summary>
/// Gets byte array from the url.
/// </summary>
/// <param name="url">URL.</param>
/// <param name="expiryDate">Expiry date. A resource older than this will be refetched.</param>
/// <returns>Byte array.</returns>
Gets byte array from the url.
Gets byte array from the url .
public async Task<byte[]> GetWebResourceAsync(string url, DateTimeOffset expiryDate)
        {
            logger?.LogInformation($"Getting resource for {url}");
            var result = GetResourceFromCache(url);
            var cacheDate = GetDateOfCachedResource(url);
            if (cacheDate != null && cacheDate < expiryDate)
            {
                logger?.LogDebug($"{url} has an expired cache.");
                DeleteResourceFromCache(url);
                result = null;
            }
            if (result == null)
            {
                int attempts = 0;
                while (attempts < DownloadAttempts)
                {
                    logger?.LogDebug($"Download attempt {attempts} for {url}.");
                    try
                    {
                        while (_concurrentDownloads > MaxConcurrentDownloads)
                        {
                            await Task.Delay(500);
                        }
                        if (RateLimit != null)
                        {
                            while (RateLimit.IsThrottled())
                            {
                                Thread.Sleep(1000);
                            }
                        }
                        long startTime = DateTimeOffset.Now.ToUnixTimeMilliseconds();
                        Interlocked.Increment(ref _concurrentDownloads);
                        using (var client = new CrawlerClient() { Timeout = 10 * 60 * 1000 })
                        {
                            client.Headers.Add("User-Agent", UserAgent);
                            var encodedUrl = new Uri(url).AbsoluteUri;
                            result = await client.DownloadDataTaskAsync(encodedUrl);
                            RateLimit?.AddRequest();
                        }
                        long endTime = DateTimeOffset.Now.ToUnixTimeMilliseconds();
                        long interval = endTime - startTime;
                        Interlocked.Add(ref _downloadTimeSpan, interval);
                        Interlocked.Add(ref _bytesDownloaded, result.LongLength);
                        logger?.LogDebug($"Downloaded resource for {url}.");
                        SaveResourceToCache(url, result);
                        Interlocked.Increment(ref _cacheMisses);
                        break;
                    }
                    catch (WebException ex)
                    {
                        if (ex.Status == WebExceptionStatus.NameResolutionFailure || ex.Status == WebExceptionStatus.Timeout)
                        {
                            attempts++;
                            if (attempts < DownloadAttempts)
                            {
                                await Task.Delay(1000);
                            }
                            else
                            {
                                logger?.LogWarning(ex, $"Exception downloading web page from url '{url}'");
                            }
                        }
                        else
                        {
                            throw ex;
                        }
                    }
                    finally
                    {
                        Interlocked.Decrement(ref _concurrentDownloads);
                    }
                }
            }
            else
            {
                Interlocked.Increment(ref _cacheHits);
            }
            return result;
        }
public async Task < byte [ ] > GetWebResourceAsync ( string url , DateTimeOffset expiryDate ) { logger ? . LogInformation ( $" Getting resource for  { url } " ) ; var result = GetResourceFromCache ( url ) ; var cacheDate = GetDateOfCachedResource ( url ) ; if ( cacheDate != null && cacheDate < expiryDate ) { logger ? . LogDebug ( $" { url }  has an expired cache. " ) ; DeleteResourceFromCache ( url ) ; result = null ; } if ( result == null ) { int attempts = 0 ; while ( attempts < DownloadAttempts ) { logger ? . LogDebug ( $" Download attempt  { attempts }  for  { url } . " ) ; try { while ( _concurrentDownloads > MaxConcurrentDownloads ) { await Task . Delay ( 500 ) ; } if ( RateLimit != null ) { while ( RateLimit . IsThrottled ( ) ) { Thread . Sleep ( 1000 ) ; } } long startTime = DateTimeOffset . Now . ToUnixTimeMilliseconds ( ) ; Interlocked . Increment ( ref _concurrentDownloads ) ; using ( var client = new CrawlerClient ( ) { Timeout = 10 * 60 * 1000 } ) { client . Headers . Add ( " " , UserAgent ) ; var encodedUrl = new Uri ( url ) . AbsoluteUri ; result = await client . DownloadDataTaskAsync ( encodedUrl ) ; RateLimit ? . AddRequest ( ) ; } long endTime = DateTimeOffset . Now . ToUnixTimeMilliseconds ( ) ; long interval = endTime - startTime ; Interlocked . Add ( ref _downloadTimeSpan , interval ) ; Interlocked . Add ( ref _bytesDownloaded , result . LongLength ) ; logger ? . LogDebug ( $" Downloaded resource for  { url } . " ) ; SaveResourceToCache ( url , result ) ; Interlocked . Increment ( ref _cacheMisses ) ; break ; } catch ( WebException ex ) { if ( ex . Status == WebExceptionStatus . NameResolutionFailure || ex . Status == WebExceptionStatus . Timeout ) { attempts ++ ; if ( attempts < DownloadAttempts ) { await Task . Delay ( 1000 ) ; } else { logger ? . LogWarning ( ex , $" Exception downloading web page from url ' { url } ' " ) ; } } else { throw ex ; } } finally { Interlocked . Decrement ( ref _concurrentDownloads ) ; } } } else { Interlocked . Increment ( ref _cacheHits ) ; } return result ; }
Gets byte array from the url.
Gets byte array from the url .
// purposefully tie up the thread so others don't continue to hammer the endpoint.
None
url
string
None
expiryDate
DateTimeOffset
Expiry date. A resource older than this will be refetched.
Expiry date . A resource older than this will be refetched .