protected override string DownloadJSON(string url) { var webClient = new CompressionWebClient(EnableCompression) { Encoding = Encoding.UTF8 }; foreach (var headerEntry in Headers) { webClient.Headers[headerEntry.Key] = headerEntry.Value; } LIMITER.RateLimit().Wait(); try { return(webClient.DownloadString(url)); } finally { LIMITER.RequestDone(); } }
protected override async Task <string> DownloadJSON(string url) { var webClient = new CompressionWebClient(EnableCompression) { Encoding = Encoding.UTF8 }; foreach (var headerEntry in Headers) { webClient.Headers[headerEntry.Key] = headerEntry.Value; } await LIMITER.RateLimit().ConfigureAwait(false); try { return(await webClient.DownloadStringTaskAsync(url).ConfigureAwait(false)); } finally { LIMITER.RequestDone(); } }
protected override async Task <string> DownloadJSON(string url) { bool retry = false; lock (_requestSync) { if (_denyRequestsUntilTime.HasValue) { if (RequestsDisabled) { return(null); } _failedRequests = 0; _denyRequestsUntilTime = null; } } var webClient = new CompressionWebClient(EnableCompression) { Encoding = Encoding.UTF8 }; foreach (var headerEntry in Headers) { webClient.Headers[headerEntry.Key] = headerEntry.Value; } await LIMITER.RateLimit().ConfigureAwait(false); try { string fullUrl = Mirrors[_currentMirror] + url; string json = await webClient.DownloadStringTaskAsync(fullUrl).ConfigureAwait(false); if (_failedRequests > 0) { _failedRequests--; } return(json); } catch (WebException ex) { if (ex.Status == WebExceptionStatus.Timeout || (ex.Response != null && ( (((HttpWebResponse)ex.Response).StatusCode == HttpStatusCode.ServiceUnavailable || ((HttpWebResponse)ex.Response).StatusCode == HttpStatusCode.RequestTimeout) ))) { //Rate limiting _failedRequests++; if (_failedRequests >= MAX_FAILED_REQUESTS) { DisableRequestsTemporarily(); return(null); } _currentMirror = (_currentMirror + 1) % Mirrors.Count; retry = true; } else { throw; } } finally { LIMITER.RequestDone(); } if (retry) { return(await DownloadJSON(url).ConfigureAwait(false)); } return(null); }