protected async Task <T> Get <T>(string route, JsonSerializerOptions jsonOptions = null)
{
using (var webClient = _webClientFactory.Create(
headers: new WebHeaderCollection {
{ HttpRequestHeader.Authorization, Credential },
{ HttpRequestHeader.ContentType, "application/json" },
}))
using (var stream = await webClient.OpenReadTaskAsync(_config.Url + "index.php?/api/v2/" + route))
{
return(await JsonSerializer.DeserializeAsync <T>(stream, jsonOptions));
}
}
public List <string> DownloadJournals(JournalTocsFetchMode action = JournalTocsFetchMode.Update)
{
_logger.Info("Downloading journals...");
var result = new List <string>();
using (var webClient = _webClientFactory.Create())
{
webClient.Encoding = _encoding;
var fetch = true;
do
{
_logger.Info($"\t...downloading batch #{_resumptionToken + 1}...");
var batch = webClient.DownloadString($"{_settings.AllJournalsRequestUrl}&action={action.ToString().ToLowerInvariant()}&resumptionToken={_resumptionToken}");
if (!batch.Contains(EndOfBatchesNotice) || !batch.Contains(Notice))
{
result.Add(batch.Replace("&", "&"));
_resumptionToken++;
}
else
{
fetch = false;
}
} while (fetch);
_logger.Info("Finised downloading journals.");
return(result);
}
}
public BlobRulesDriver(Uri url, IWebClientFactory webClientFactory, IScheduler scheduler = null)
{
_url = url;
_subject = new ReplaySubject <Dictionary <string, RuleDefinition> >(1);
scheduler = scheduler ?? TaskPoolScheduler.Default;
_subscription = Observable.Interval(TimeSpan.FromSeconds(30))
.StartWith(0)
.SubscribeOn(scheduler)
.Select((_) => Observable.FromAsync(async() =>
{
using (var client = webClientFactory.Create())
{
client.Encoding = Encoding.UTF8;
return(await client.DownloadStringTaskAsync(_url));
}
}))
.Switch()
.DistinctUntilChanged()
.Select(JsonConvert.DeserializeObject <Dictionary <string, RuleDefinition> >)
.DistinctUntilChanged(new DictionaryEqualityComparer <string, RuleDefinition>(new RuleDefinitionComparer()))
.Catch((Exception exception) =>
{
//Trace.TraceWarning($"Failed to update rules from {url}\r\n{exception}");
return(Observable.Empty <Dictionary <string, RuleDefinition> >().Delay(TimeSpan.FromMinutes(1)));
})
.Repeat()
.Do(_subject)
.Subscribe(x => OnRulesChange?.Invoke(x));
}
public void TestSetup()
{
_factory = Substitute.For <IWebClientFactory>();
_client = Substitute.For <IWebClient>();
_writer = Substitute.For <IDataWriter>();
_factory.Create().Returns(_client);
}
public InspectionOutcomesResponse GetInspectionsDetail(string firstLinkUrl)
{
InspectionOutcomesResponse inspectionOutcomesResponse;
try
{
_logger.Debug("Opening web client");
var webClient = _webClientFactory.Create();
using (var client = webClient)
{
_logger.Debug("Opening memory stream");
using (var stream =
new MemoryStream(client.DownloadData(new Uri(firstLinkUrl))))
{
_logger.Debug("Opened memory stream");
using (var package = new ExcelPackage(stream))
{
_logger.Debug("Opened excel package");
inspectionOutcomesResponse = _getOfstedDetailsFromExcelPackageService.ExtractOfstedInspections(package);
}
_logger.Debug("Closed excel package");
}
_logger.Debug("Closed memory stream");
}
_logger.Debug($"Closed web client");
}
catch (UriFormatException ex)
{
var message = $"Error whilst trying to read url: [{firstLinkUrl}]";
var exception = new UrlReadingException(message, ex);
_logger.Error(message, exception);
throw exception;
}
catch (COMException ex)
{
var message = $"Error whilst trying to read excel details from url: [{firstLinkUrl}], message: [{ex.Message}]";
var exception = new UrlReadingException(message, ex);
_logger.Error(message, exception);
throw exception;
}
catch (Exception ex)
{
var message = $"Error whilst trying to read excel details";
var exception = new UrlReadingException(message, ex);
_logger.Error(message, exception);
throw exception;
}
return(inspectionOutcomesResponse);
}
public void SetEnvironmentVariable(string variable, string value)
{
Logger.Debug("AppVeyor API Url: {0}", _appVeyorApiUrl);
var request = string.Format(SetEnvironmentVariableRequest, variable, EscapeStringValue(value.Replace("\r\n", "\n")));
Logger.Debug("Request body: {0}", request);
using (var webClient = _webClientFactory.Create(_appVeyorApiUrl))
{
webClient.UploadData("api/build/variables", "POST", Encoding.UTF8.GetBytes(request));
Logger.Info("Adding AppVeyor environment variable: {0}.", variable);
}
}
public void DownloadFile(FilePayload filePayload)
{
loggingService.LogVerbose($"Attempting to download file: {dateTimeService.DateTimeNow()}");
IWebClient webClient = null;
try
{
webClient = webClientFactory.Create();
SetUpFileDownload(filePayload, webClient);
DownloadFile(filePayload, webClient);
}
finally
{
webClientFactory.Release(webClient);
}
loggingService.LogVerbose($"File download complete: {dateTimeService.DateTimeNow()}");
}
public List <string> DownloadJournals(List <string> issns)
{
_logger.Info("Downloading journals...");
using (var webClient = _webClientFactory.Create())
{
webClient.Encoding = _encoding;
//_logger.Info($"\t...downloading batch #{_resumptionToken + 1}...");
//var resultString = webClient.DownloadString("http://www.journaltocs.ac.uk/API/RSS/GetJournalByIssn.php?sui=z7CsvQxb1udh849067j6&test=true&issns[]=1697-5200&issns[]=1502-4873");
var issnParams = issns.Aggregate(new StringBuilder(), (sb, a) => sb.Append($"&issns[]={a}"), sb => sb.ToString());
var resultString = webClient.DownloadString($"{_settings.ByIssnRequestUrl}{issnParams}");
return(new List <string> {
resultString
});
}
}
private Task <Exception> GetLink(Url link, SemaphoreSlim semaphore)
{
return(Task.Run(async() =>
{
await semaphore.WaitAsync();
try
{
using (var client = _client.Create())
{
var result = await client.Download(link).ConfigureAwait(false);
if (result.Data.IsEmpty() || !result.Data.ToLowerInvariant().Contains("<html"))
{
if (result.ErrorCode != 404)
{
throw new ApplicationException($"Empty data. Page {link}. Code {result.ErrorCode}");
}
return null;
}
result.Data = _simplifier.Simplify(result.Data);
_queue.AddRange(
HtmlHelpers.GetAllLinks(result.Data, result.Url)
.Select(x => x.Fix())
.Where(x => x != null && x.Domain == _domain.Domain)
);
_writer.Write(result.Data, result.Url.ToString());
return null;
}
}
catch (Exception ex)
{
return ex;
}
finally
{
semaphore.Release();
}
}));
}