public Task <ImportSqlDumpResult> ImportSqlDumpAsync(string sqlDumpFilePath, IProgress <object> progressHandler, CancellationToken cancellationToken)
{
return(Task.Run(() =>
{
Logger.Debug("SQL dump import has started.");
using (SqlDumpReader sqlDumpReader = new SqlDumpReader(sqlDumpFilePath))
{
while (true)
{
bool tableFound = false;
while (sqlDumpReader.ReadLine())
{
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("SQL dump import has been cancelled.");
return ImportSqlDumpResult.CANCELLED;
}
if (sqlDumpReader.CurrentLineCommand == SqlDumpReader.LineCommand.CREATE_TABLE)
{
Logger.Debug("CREATE TABLE statement found.");
tableFound = true;
break;
}
progressHandler.Report(new ImportSearchTableDefinitionProgress(sqlDumpReader.CurrentFilePosition, sqlDumpReader.FileSize));
}
if (!tableFound)
{
Logger.Debug("CREATE TABLE statement was not found.");
return ImportSqlDumpResult.DATA_NOT_FOUND;
}
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("SQL dump import has been cancelled.");
return ImportSqlDumpResult.CANCELLED;
}
SqlDumpReader.ParsedTableDefinition parsedTableDefinition = sqlDumpReader.ParseTableDefinition();
TableType tableType = DetectImportTableType(parsedTableDefinition);
if (tableType == TableType.UNKNOWN)
{
continue;
}
progressHandler.Report(new ImportTableDefinitionFoundProgress(tableType));
bool insertFound = false;
while (sqlDumpReader.ReadLine())
{
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("SQL dump import has been cancelled.");
return ImportSqlDumpResult.CANCELLED;
}
if (sqlDumpReader.CurrentLineCommand == SqlDumpReader.LineCommand.INSERT)
{
Logger.Debug("INSERT statement found.");
insertFound = true;
break;
}
}
if (!insertFound)
{
Logger.Debug("INSERT statement was not found.");
return ImportSqlDumpResult.DATA_NOT_FOUND;
}
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("SQL dump import has been cancelled.");
return ImportSqlDumpResult.CANCELLED;
}
Logger.Debug($"Table type is {tableType}.");
Importer importer;
BitArray existingLibgenIds = null;
switch (tableType)
{
case TableType.NON_FICTION:
if (NonFictionBookCount != 0)
{
CheckAndCreateNonFictionIndexes(progressHandler, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return ImportSqlDumpResult.CANCELLED;
}
progressHandler.Report(new ImportLoadLibgenIdsProgress());
existingLibgenIds = localDatabase.GetNonFictionLibgenIdsBitArray();
}
importer = new NonFictionImporter(localDatabase, existingLibgenIds);
break;
case TableType.FICTION:
if (FictionBookCount != 0)
{
CheckAndCreateFictionIndexes(progressHandler, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return ImportSqlDumpResult.CANCELLED;
}
progressHandler.Report(new ImportLoadLibgenIdsProgress());
existingLibgenIds = localDatabase.GetFictionLibgenIdsBitArray();
}
importer = new FictionImporter(localDatabase, existingLibgenIds);
break;
case TableType.SCI_MAG:
if (SciMagArticleCount != 0)
{
CheckAndCreateSciMagIndexes(progressHandler, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return ImportSqlDumpResult.CANCELLED;
}
progressHandler.Report(new ImportLoadLibgenIdsProgress());
existingLibgenIds = localDatabase.GetSciMagLibgenIdsBitArray();
}
importer = new SciMagImporter(localDatabase, existingLibgenIds);
break;
default:
throw new Exception($"Unknown table type: {tableType}.");
}
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("SQL dump import has been cancelled.");
return ImportSqlDumpResult.CANCELLED;
}
Importer.ImportProgressReporter importProgressReporter = (int objectsAdded, int objectsUpdated) =>
{
progressHandler.Report(new ImportObjectsProgress(objectsAdded, objectsUpdated));
};
Logger.Debug("Importing data.");
importer.Import(sqlDumpReader, importProgressReporter, IMPORT_PROGRESS_UPDATE_INTERVAL, cancellationToken, parsedTableDefinition);
switch (tableType)
{
case TableType.NON_FICTION:
UpdateNonFictionBookCount();
break;
case TableType.FICTION:
UpdateFictionBookCount();
break;
case TableType.SCI_MAG:
UpdateSciMagArticleCount();
break;
}
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("SQL dump import has been cancelled.");
return ImportSqlDumpResult.CANCELLED;
}
switch (tableType)
{
case TableType.NON_FICTION:
DatabaseMetadata.NonFictionFirstImportComplete = true;
break;
case TableType.FICTION:
DatabaseMetadata.FictionFirstImportComplete = true;
break;
case TableType.SCI_MAG:
DatabaseMetadata.SciMagFirstImportComplete = true;
break;
}
localDatabase.UpdateMetadata(DatabaseMetadata);
Logger.Debug("SQL dump import has been completed successfully.");
return ImportSqlDumpResult.COMPLETED;
}
}
}));
}
public Task <SynchronizationResult> SynchronizeNonFictionAsync(IProgress <object> progressHandler, CancellationToken cancellationToken)
{
return(Task.Run(async() =>
{
Logger.Debug("Synchronization has started.");
if (NonFictionBookCount == 0)
{
throw new Exception("Non-fiction table must not be empty.");
}
CheckAndCreateNonFictionIndexes(progressHandler, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("Synchronization has been cancelled.");
return SynchronizationResult.CANCELLED;
}
Logger.Debug("Loading last non-fiction book.");
NonFictionBook lastModifiedNonFictionBook = localDatabase.GetLastModifiedNonFictionBook();
Logger.Debug($"Last non-fiction book: Libgen ID = {lastModifiedNonFictionBook.LibgenId}, last modified date/time = {lastModifiedNonFictionBook.LastModifiedDateTime}.");
string jsonApiUrl = Mirrors[AppSettings.Mirrors.NonFictionSynchronizationMirrorName].NonFictionSynchronizationUrl;
if (jsonApiUrl == null)
{
throw new Exception("JSON API URL is null.");
}
JsonApiClient jsonApiClient = new JsonApiClient(HttpClient, jsonApiUrl, lastModifiedNonFictionBook.LastModifiedDateTime,
lastModifiedNonFictionBook.LibgenId);
progressHandler.Report(new ImportLoadLibgenIdsProgress());
BitArray existingLibgenIds = localDatabase.GetNonFictionLibgenIdsBitArray();
NonFictionImporter importer = new NonFictionImporter(localDatabase, existingLibgenIds, lastModifiedNonFictionBook);
progressHandler.Report(new SynchronizationProgress(0, 0, 0));
int downloadedBookCount = 0;
int totalAddedBookCount = 0;
int totalUpdatedBookCount = 0;
Importer.ImportProgressReporter importProgressReporter = (int objectsAdded, int objectsUpdated) =>
{
progressHandler.Report(new SynchronizationProgress(downloadedBookCount, totalAddedBookCount + objectsAdded,
totalUpdatedBookCount + objectsUpdated));
};
while (true)
{
List <NonFictionBook> currentBatch;
try
{
Logger.Debug("Downloading next batch from the server.");
currentBatch = await jsonApiClient.DownloadNextBatchAsync(cancellationToken);
}
catch (TaskCanceledException)
{
Logger.Debug("Synchronization has been cancelled.");
return SynchronizationResult.CANCELLED;
}
if (!currentBatch.Any())
{
Logger.Debug("Current batch is empty, download is complete.");
break;
}
downloadedBookCount += currentBatch.Count;
Logger.Debug($"Batch download is complete, {downloadedBookCount} books have been downloaded so far.");
progressHandler.Report(new SynchronizationProgress(downloadedBookCount, totalAddedBookCount, totalUpdatedBookCount));
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("Synchronization has been cancelled.");
return SynchronizationResult.CANCELLED;
}
Logger.Debug("Importing downloaded batch.");
Importer.ImportResult importResult =
importer.Import(currentBatch, importProgressReporter, SYNCHRONIZATION_PROGRESS_UPDATE_INTERVAL, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("Synchronization has been cancelled.");
return SynchronizationResult.CANCELLED;
}
totalAddedBookCount += importResult.AddedObjectCount;
totalUpdatedBookCount += importResult.UpdatedObjectCount;
Logger.Debug($"Batch has been imported, total added book count = {totalAddedBookCount}, total updated book count = {totalUpdatedBookCount}.");
}
Logger.Debug("Synchronization has been completed successfully.");
return SynchronizationResult.COMPLETED;
}));
}
public int Import()
{
ServerConfiguration serverConfiguration = LoadServerConfiguration();
if (serverConfiguration == null)
{
return(1);
}
string databaseFilePath = serverConfiguration.DatabaseFilePath;
if (String.IsNullOrWhiteSpace(databaseFilePath))
{
Console.WriteLine("Database file path is not set in the server configuration file.");
return(2);
}
if (!File.Exists(databaseFilePath))
{
Console.WriteLine($"Couldn't find the database file at {databaseFilePath}");
return(3);
}
Console.WriteLine($"Opening database \"{databaseFilePath}\"...");
LocalDatabase localDatabase = LocalDatabase.OpenDatabase(databaseFilePath);
using (SqlDumpReader sqlDumpReader = new SqlDumpReader(importFilePath))
{
while (true)
{
bool tableFound = false;
while (sqlDumpReader.ReadLine())
{
if (sqlDumpReader.CurrentLineCommand == SqlDumpReader.LineCommand.CREATE_TABLE)
{
Logger.Debug("CREATE TABLE statement found.");
Console.WriteLine("CREATE TABLE statement found.");
tableFound = true;
break;
}
}
if (!tableFound)
{
Logger.Debug("CREATE TABLE statement was not found.");
Console.WriteLine("CREATE TABLE statement was not found.");
return(4);
}
SqlDumpReader.ParsedTableDefinition parsedTableDefinition = sqlDumpReader.ParseTableDefinition();
TableType tableType = DetectImportTableType(parsedTableDefinition);
if (tableType == TableType.UNKNOWN)
{
Console.WriteLine($"Table {parsedTableDefinition.TableName} doesn't match any known supported formats. It will be skipped.");
continue;
}
Logger.Debug($"Table type is {tableType}.");
ImportFormat?detectedImportFormat = ConvertTableTypeToImportFormat(tableType);
if (!detectedImportFormat.HasValue)
{
Console.WriteLine("Could not determine the format of the database dump.");
return(5);
}
if (detectedImportFormat.Value == importFormat)
{
Console.WriteLine($"Found a matching import format: {ImportFormatToString(detectedImportFormat.Value)}.");
}
else
{
Console.WriteLine($"Expected the import format {importFormat} but found {ImportFormatToString(detectedImportFormat.Value)}.");
return(6);
}
bool insertFound = false;
while (sqlDumpReader.ReadLine())
{
if (sqlDumpReader.CurrentLineCommand == SqlDumpReader.LineCommand.INSERT)
{
Logger.Debug("INSERT statement found.");
insertFound = true;
break;
}
}
if (!insertFound)
{
Logger.Debug("INSERT statement was not found.");
Console.WriteLine("Couldn't find any data to import.");
return(7);
}
Importer importer;
BitArray existingLibgenIds = null;
int existingBookCount = 0;
switch (tableType)
{
case TableType.NON_FICTION:
existingBookCount = localDatabase.CountNonFictionBooks();
if (existingBookCount != 0)
{
existingLibgenIds = localDatabase.GetNonFictionLibgenIdsBitArray();
}
importer = new NonFictionImporter(localDatabase, existingLibgenIds);
break;
case TableType.FICTION:
existingBookCount = localDatabase.CountFictionBooks();
if (existingBookCount != 0)
{
existingLibgenIds = localDatabase.GetFictionLibgenIdsBitArray();
}
importer = new FictionImporter(localDatabase, existingLibgenIds);
break;
case TableType.SCI_MAG:
existingBookCount = localDatabase.CountSciMagArticles();
if (existingBookCount != 0)
{
existingLibgenIds = localDatabase.GetSciMagLibgenIdsBitArray();
}
importer = new SciMagImporter(localDatabase, existingLibgenIds);
break;
default:
throw new Exception($"Unknown table type: {tableType}.");
}
bool updateProgressLine = false;
Importer.ImportProgressReporter importProgressReporter = (int objectsAdded, int objectsUpdated) =>
{
if (updateProgressLine)
{
Console.SetCursorPosition(0, Console.CursorTop - 1);
}
else
{
updateProgressLine = true;
}
if (objectsUpdated > 0)
{
Console.WriteLine($"Books added: {objectsAdded}, updated: {objectsUpdated}.");
}
else
{
Console.WriteLine($"Books added: {objectsAdded}.");
}
};
Logger.Debug("Importing data.");
Console.WriteLine("Importing data.");
importer.Import(sqlDumpReader, importProgressReporter, IMPORT_PROGRESS_UPDATE_INTERVAL, parsedTableDefinition);
DatabaseMetadata databaseMetadata = localDatabase.GetMetadata();
switch (tableType)
{
case TableType.NON_FICTION:
databaseMetadata.NonFictionFirstImportComplete = true;
break;
case TableType.FICTION:
databaseMetadata.FictionFirstImportComplete = true;
break;
case TableType.SCI_MAG:
databaseMetadata.SciMagFirstImportComplete = true;
break;
}
localDatabase.UpdateMetadata(databaseMetadata);
Logger.Debug("SQL dump import has been completed successfully.");
Console.WriteLine("SQL dump import has been completed successfully.");
return(0);
}
}
}
