public AircloakConnectionBuilder(
JsonApiClient apiClient,
IOptions <ConnectionOptions> options)
{
this.options = options;
this.apiClient = apiClient;
}
private static void InitializeService()
{
var apiClient = new JsonApiClient();
apiClient.HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd("MyAgent/1.0");
_service = new GitUserInfoService(apiClient);
}
public YtsClient(HttpClient client, ILogger <YtsClient> logger)
{
_client = client;
_ytsClient = new JsonApiClient(client, x =>
{
x.ContractResolver = new DefaultContractResolver
{
NamingStrategy = new SnakeCaseNamingStrategy()
};
x.DateTimeZoneHandling = DateTimeZoneHandling.Utc;
});
_clientPolicy = Policy.Handle <ApiRequestException>()
.WaitAndRetryAsync(
5,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, timeSpan, retryCount) => logger.LogWarning(
exception,
"delaying for {delay}ms, then retry #{retry}.",
timeSpan.TotalMilliseconds, retryCount));
}
public void Does_change_BasePath_JsonApiClient()
{
var client = new JsonApiClient("https://example.org");
Assert.That(client.SyncReplyBaseUri, Is.EqualTo("https://example.org/api/"));
Assert.That(client.AsyncOneWayBaseUri, Is.EqualTo("https://example.org/api/"));
client = new JsonApiClient("https://example.org")
{
UseBasePath = "/json/reply"
};
Assert.That(client.SyncReplyBaseUri, Is.EqualTo("https://example.org/json/reply/"));
Assert.That(client.AsyncOneWayBaseUri, Is.EqualTo("https://example.org/json/reply/"));
client.UseBasePath = "/api";
Assert.That(client.SyncReplyBaseUri, Is.EqualTo("https://example.org/api/"));
Assert.That(client.AsyncOneWayBaseUri, Is.EqualTo("https://example.org/api/"));
client = new JsonApiClient("https://example.org")
.Apply(c => c.UseBasePath = "/custom");
Assert.That(client.SyncReplyBaseUri, Is.EqualTo("https://example.org/custom/"));
Assert.That(client.AsyncOneWayBaseUri, Is.EqualTo("https://example.org/custom/"));
}
public AircloakConnectionBuilder(JsonApiClient apiClient, ExplorerConfig explorerConfig)
{
this.explorerConfig = explorerConfig;
this.apiClient = apiClient;
}
public Task <SynchronizationResult> SynchronizeNonFictionAsync(IProgress <object> progressHandler, CancellationToken cancellationToken)
{
return(Task.Run(async() =>
{
Logger.Debug("Synchronization has started.");
if (NonFictionBookCount == 0)
{
throw new Exception("Non-fiction table must not be empty.");
}
CheckAndCreateNonFictionIndexes(progressHandler, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("Synchronization has been cancelled.");
return SynchronizationResult.CANCELLED;
}
Logger.Debug("Loading last non-fiction book.");
NonFictionBook lastModifiedNonFictionBook = localDatabase.GetLastModifiedNonFictionBook();
Logger.Debug($"Last non-fiction book: Libgen ID = {lastModifiedNonFictionBook.LibgenId}, last modified date/time = {lastModifiedNonFictionBook.LastModifiedDateTime}.");
string jsonApiUrl = Mirrors[AppSettings.Mirrors.NonFictionSynchronizationMirrorName].NonFictionSynchronizationUrl;
if (jsonApiUrl == null)
{
throw new Exception("JSON API URL is null.");
}
JsonApiClient jsonApiClient = new JsonApiClient(HttpClient, jsonApiUrl, lastModifiedNonFictionBook.LastModifiedDateTime,
lastModifiedNonFictionBook.LibgenId);
progressHandler.Report(new ImportLoadLibgenIdsProgress());
BitArray existingLibgenIds = localDatabase.GetNonFictionLibgenIdsBitArray();
NonFictionImporter importer = new NonFictionImporter(localDatabase, existingLibgenIds, lastModifiedNonFictionBook);
progressHandler.Report(new SynchronizationProgress(0, 0, 0));
int downloadedBookCount = 0;
int totalAddedBookCount = 0;
int totalUpdatedBookCount = 0;
Importer.ImportProgressReporter importProgressReporter = (int objectsAdded, int objectsUpdated) =>
{
progressHandler.Report(new SynchronizationProgress(downloadedBookCount, totalAddedBookCount + objectsAdded,
totalUpdatedBookCount + objectsUpdated));
};
while (true)
{
List <NonFictionBook> currentBatch;
try
{
Logger.Debug("Downloading next batch from the server.");
currentBatch = await jsonApiClient.DownloadNextBatchAsync(cancellationToken);
}
catch (TaskCanceledException)
{
Logger.Debug("Synchronization has been cancelled.");
return SynchronizationResult.CANCELLED;
}
if (!currentBatch.Any())
{
Logger.Debug("Current batch is empty, download is complete.");
break;
}
downloadedBookCount += currentBatch.Count;
Logger.Debug($"Batch download is complete, {downloadedBookCount} books have been downloaded so far.");
progressHandler.Report(new SynchronizationProgress(downloadedBookCount, totalAddedBookCount, totalUpdatedBookCount));
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("Synchronization has been cancelled.");
return SynchronizationResult.CANCELLED;
}
Logger.Debug("Importing downloaded batch.");
Importer.ImportResult importResult =
importer.Import(currentBatch, importProgressReporter, SYNCHRONIZATION_PROGRESS_UPDATE_INTERVAL, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("Synchronization has been cancelled.");
return SynchronizationResult.CANCELLED;
}
totalAddedBookCount += importResult.AddedObjectCount;
totalUpdatedBookCount += importResult.UpdatedObjectCount;
Logger.Debug($"Batch has been imported, total added book count = {totalAddedBookCount}, total updated book count = {totalUpdatedBookCount}.");
}
Logger.Debug("Synchronization has been completed successfully.");
return SynchronizationResult.COMPLETED;
}));
}
public ContextBuilder(JsonApiClient apiClient)
{
this.apiClient = apiClient;
}
public Task <SynchronizationResult> SynchronizeNonFiction(IProgress <object> progressHandler, CancellationToken cancellationToken)
{
return(Task.Run(async() =>
{
Logger.Debug("Synchronization has started.");
if (NonFictionBookCount == 0)
{
throw new Exception("Non-fiction table must not be empty.");
}
CheckAndCreateNonFictionIndexes(progressHandler, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("Synchronization has been cancelled.");
return SynchronizationResult.CANCELLED;
}
Logger.Debug("Loading last non-fiction book.");
NonFictionBook lastModifiedNonFictionBook = localDatabase.GetLastModifiedNonFictionBook();
Logger.Debug($"Last non-fiction book: Libgen ID = {lastModifiedNonFictionBook.LibgenId}, last modified date/time = {lastModifiedNonFictionBook.LastModifiedDateTime}.");
string jsonApiUrl = Mirrors[AppSettings.Mirrors.NonFictionSynchronizationMirrorName].NonFictionSynchronizationUrl;
if (jsonApiUrl == null)
{
throw new Exception("JSON API URL is null.");
}
JsonApiClient jsonApiClient = new JsonApiClient(HttpClient, jsonApiUrl, lastModifiedNonFictionBook.LastModifiedDateTime,
lastModifiedNonFictionBook.LibgenId);
List <NonFictionBook> downloadedBooks = new List <NonFictionBook>();
progressHandler.Report(new JsonApiDownloadProgress(0));
while (true)
{
List <NonFictionBook> currentBatch;
try
{
Logger.Debug("Downloading next batch from the server.");
currentBatch = await jsonApiClient.DownloadNextBatchAsync(cancellationToken);
}
catch (TaskCanceledException)
{
Logger.Debug("Synchronization has been cancelled.");
return SynchronizationResult.CANCELLED;
}
if (!currentBatch.Any())
{
Logger.Debug("Current batch is empty, download is complete.");
break;
}
downloadedBooks.AddRange(currentBatch);
Logger.Debug($"{downloadedBooks.Count} books have been downloaded so far.");
progressHandler.Report(new JsonApiDownloadProgress(downloadedBooks.Count));
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("Synchronization has been cancelled.");
return SynchronizationResult.CANCELLED;
}
}
NonFictionImporter importer = new NonFictionImporter(localDatabase, lastModifiedNonFictionBook);
Logger.Debug("Importing data.");
importer.Import(downloadedBooks, progressHandler, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("Synchronization has been cancelled.");
return SynchronizationResult.CANCELLED;
}
Logger.Debug("Synchronization has been completed successfully.");
return SynchronizationResult.COMPLETED;
}));
}
