private TableType DetectImportTableType(SqlDumpReader.ParsedTableDefinition parsedTableDefinition)
{
if (TableDefinitions.AllTables.TryGetValue(parsedTableDefinition.TableName, out TableDefinition tableDefinition))
{
foreach (SqlDumpReader.ParsedColumnDefinition parsedColumnDefinition in parsedTableDefinition.Columns)
{
if (tableDefinition.Columns.TryGetValue(parsedColumnDefinition.ColumnName.ToLower(), out ColumnDefinition columnDefinition))
{
if (columnDefinition.ColumnType == parsedColumnDefinition.ColumnType)
{
continue;
}
}
return(TableType.UNKNOWN);
}
return(tableDefinition.TableType);
}
return(TableType.UNKNOWN);
}
public abstract ImportResult Import(SqlDumpReader sqlDumpReader, ImportProgressReporter progressReporter, double progressUpdateInterval,
CancellationToken cancellationToken, SqlDumpReader.ParsedTableDefinition parsedTableDefinition, string databaseFullPath);
public Task <ImportSqlDumpResult> ImportSqlDumpAsync(string sqlDumpFilePath, IProgress <object> progressHandler, CancellationToken cancellationToken)
{
return(Task.Run(() =>
{
Logger.Debug("SQL dump import has started.");
using (SqlDumpReader sqlDumpReader = new SqlDumpReader(sqlDumpFilePath))
{
while (true)
{
bool tableFound = false;
while (sqlDumpReader.ReadLine())
{
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("SQL dump import has been cancelled.");
return ImportSqlDumpResult.CANCELLED;
}
if (sqlDumpReader.CurrentLineCommand == SqlDumpReader.LineCommand.CREATE_TABLE)
{
Logger.Debug("CREATE TABLE statement found.");
tableFound = true;
break;
}
progressHandler.Report(new ImportSearchTableDefinitionProgress(sqlDumpReader.CurrentFilePosition, sqlDumpReader.FileSize));
}
if (!tableFound)
{
Logger.Debug("CREATE TABLE statement was not found.");
return ImportSqlDumpResult.DATA_NOT_FOUND;
}
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("SQL dump import has been cancelled.");
return ImportSqlDumpResult.CANCELLED;
}
SqlDumpReader.ParsedTableDefinition parsedTableDefinition = sqlDumpReader.ParseTableDefinition();
TableType tableType = DetectImportTableType(parsedTableDefinition);
if (tableType == TableType.UNKNOWN)
{
continue;
}
progressHandler.Report(new ImportTableDefinitionFoundProgress(tableType));
bool insertFound = false;
while (sqlDumpReader.ReadLine())
{
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("SQL dump import has been cancelled.");
return ImportSqlDumpResult.CANCELLED;
}
if (sqlDumpReader.CurrentLineCommand == SqlDumpReader.LineCommand.INSERT)
{
Logger.Debug("INSERT statement found.");
insertFound = true;
break;
}
}
if (!insertFound)
{
Logger.Debug("INSERT statement was not found.");
return ImportSqlDumpResult.DATA_NOT_FOUND;
}
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("SQL dump import has been cancelled.");
return ImportSqlDumpResult.CANCELLED;
}
Logger.Debug($"Table type is {tableType}.");
Importer importer;
BitArray existingLibgenIds = null;
switch (tableType)
{
case TableType.NON_FICTION:
if (NonFictionBookCount != 0)
{
CheckAndCreateNonFictionIndexes(progressHandler, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return ImportSqlDumpResult.CANCELLED;
}
progressHandler.Report(new ImportLoadLibgenIdsProgress());
existingLibgenIds = localDatabase.GetNonFictionLibgenIdsBitArray();
}
importer = new NonFictionImporter(localDatabase, existingLibgenIds);
break;
case TableType.FICTION:
if (FictionBookCount != 0)
{
CheckAndCreateFictionIndexes(progressHandler, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return ImportSqlDumpResult.CANCELLED;
}
progressHandler.Report(new ImportLoadLibgenIdsProgress());
existingLibgenIds = localDatabase.GetFictionLibgenIdsBitArray();
}
importer = new FictionImporter(localDatabase, existingLibgenIds);
break;
case TableType.SCI_MAG:
if (SciMagArticleCount != 0)
{
CheckAndCreateSciMagIndexes(progressHandler, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return ImportSqlDumpResult.CANCELLED;
}
progressHandler.Report(new ImportLoadLibgenIdsProgress());
existingLibgenIds = localDatabase.GetSciMagLibgenIdsBitArray();
}
importer = new SciMagImporter(localDatabase, existingLibgenIds);
break;
default:
throw new Exception($"Unknown table type: {tableType}.");
}
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("SQL dump import has been cancelled.");
return ImportSqlDumpResult.CANCELLED;
}
Importer.ImportProgressReporter importProgressReporter = (int objectsAdded, int objectsUpdated) =>
{
progressHandler.Report(new ImportObjectsProgress(objectsAdded, objectsUpdated));
};
Logger.Debug("Importing data.");
importer.Import(sqlDumpReader, importProgressReporter, IMPORT_PROGRESS_UPDATE_INTERVAL, cancellationToken, parsedTableDefinition);
switch (tableType)
{
case TableType.NON_FICTION:
UpdateNonFictionBookCount();
break;
case TableType.FICTION:
UpdateFictionBookCount();
break;
case TableType.SCI_MAG:
UpdateSciMagArticleCount();
break;
}
if (cancellationToken.IsCancellationRequested)
{
Logger.Debug("SQL dump import has been cancelled.");
return ImportSqlDumpResult.CANCELLED;
}
switch (tableType)
{
case TableType.NON_FICTION:
DatabaseMetadata.NonFictionFirstImportComplete = true;
break;
case TableType.FICTION:
DatabaseMetadata.FictionFirstImportComplete = true;
break;
case TableType.SCI_MAG:
DatabaseMetadata.SciMagFirstImportComplete = true;
break;
}
localDatabase.UpdateMetadata(DatabaseMetadata);
Logger.Debug("SQL dump import has been completed successfully.");
return ImportSqlDumpResult.COMPLETED;
}
}
}));
}
public int Import()
{
ServerConfiguration serverConfiguration = LoadServerConfiguration();
if (serverConfiguration == null)
{
return(1);
}
string databaseFilePath = serverConfiguration.DatabaseFilePath;
if (String.IsNullOrWhiteSpace(databaseFilePath))
{
Console.WriteLine("Database file path is not set in the server configuration file.");
return(2);
}
if (!File.Exists(databaseFilePath))
{
Console.WriteLine($"Couldn't find the database file at {databaseFilePath}");
return(3);
}
Console.WriteLine($"Opening database \"{databaseFilePath}\"...");
LocalDatabase localDatabase = LocalDatabase.OpenDatabase(databaseFilePath);
using (SqlDumpReader sqlDumpReader = new SqlDumpReader(importFilePath))
{
while (true)
{
bool tableFound = false;
while (sqlDumpReader.ReadLine())
{
if (sqlDumpReader.CurrentLineCommand == SqlDumpReader.LineCommand.CREATE_TABLE)
{
Logger.Debug("CREATE TABLE statement found.");
Console.WriteLine("CREATE TABLE statement found.");
tableFound = true;
break;
}
}
if (!tableFound)
{
Logger.Debug("CREATE TABLE statement was not found.");
Console.WriteLine("CREATE TABLE statement was not found.");
return(4);
}
SqlDumpReader.ParsedTableDefinition parsedTableDefinition = sqlDumpReader.ParseTableDefinition();
TableType tableType = DetectImportTableType(parsedTableDefinition);
if (tableType == TableType.UNKNOWN)
{
Console.WriteLine($"Table {parsedTableDefinition.TableName} doesn't match any known supported formats. It will be skipped.");
continue;
}
Logger.Debug($"Table type is {tableType}.");
ImportFormat?detectedImportFormat = ConvertTableTypeToImportFormat(tableType);
if (!detectedImportFormat.HasValue)
{
Console.WriteLine("Could not determine the format of the database dump.");
return(5);
}
if (detectedImportFormat.Value == importFormat)
{
Console.WriteLine($"Found a matching import format: {ImportFormatToString(detectedImportFormat.Value)}.");
}
else
{
Console.WriteLine($"Expected the import format {importFormat} but found {ImportFormatToString(detectedImportFormat.Value)}.");
return(6);
}
bool insertFound = false;
while (sqlDumpReader.ReadLine())
{
if (sqlDumpReader.CurrentLineCommand == SqlDumpReader.LineCommand.INSERT)
{
Logger.Debug("INSERT statement found.");
insertFound = true;
break;
}
}
if (!insertFound)
{
Logger.Debug("INSERT statement was not found.");
Console.WriteLine("Couldn't find any data to import.");
return(7);
}
Importer importer;
BitArray existingLibgenIds = null;
int existingBookCount = 0;
switch (tableType)
{
case TableType.NON_FICTION:
existingBookCount = localDatabase.CountNonFictionBooks();
if (existingBookCount != 0)
{
existingLibgenIds = localDatabase.GetNonFictionLibgenIdsBitArray();
}
importer = new NonFictionImporter(localDatabase, existingLibgenIds);
break;
case TableType.FICTION:
existingBookCount = localDatabase.CountFictionBooks();
if (existingBookCount != 0)
{
existingLibgenIds = localDatabase.GetFictionLibgenIdsBitArray();
}
importer = new FictionImporter(localDatabase, existingLibgenIds);
break;
case TableType.SCI_MAG:
existingBookCount = localDatabase.CountSciMagArticles();
if (existingBookCount != 0)
{
existingLibgenIds = localDatabase.GetSciMagLibgenIdsBitArray();
}
importer = new SciMagImporter(localDatabase, existingLibgenIds);
break;
default:
throw new Exception($"Unknown table type: {tableType}.");
}
bool updateProgressLine = false;
Importer.ImportProgressReporter importProgressReporter = (int objectsAdded, int objectsUpdated) =>
{
if (updateProgressLine)
{
Console.SetCursorPosition(0, Console.CursorTop - 1);
}
else
{
updateProgressLine = true;
}
if (objectsUpdated > 0)
{
Console.WriteLine($"Books added: {objectsAdded}, updated: {objectsUpdated}.");
}
else
{
Console.WriteLine($"Books added: {objectsAdded}.");
}
};
Logger.Debug("Importing data.");
Console.WriteLine("Importing data.");
importer.Import(sqlDumpReader, importProgressReporter, IMPORT_PROGRESS_UPDATE_INTERVAL, parsedTableDefinition);
DatabaseMetadata databaseMetadata = localDatabase.GetMetadata();
switch (tableType)
{
case TableType.NON_FICTION:
databaseMetadata.NonFictionFirstImportComplete = true;
break;
case TableType.FICTION:
databaseMetadata.FictionFirstImportComplete = true;
break;
case TableType.SCI_MAG:
databaseMetadata.SciMagFirstImportComplete = true;
break;
}
localDatabase.UpdateMetadata(databaseMetadata);
Logger.Debug("SQL dump import has been completed successfully.");
Console.WriteLine("SQL dump import has been completed successfully.");
return(0);
}
}
}
public abstract ImportResult Import(SqlDumpReader sqlDumpReader, ImportProgressReporter progressReporter, double progressUpdateInterval,
SqlDumpReader.ParsedTableDefinition parsedTableDefinition);
public abstract void Import(SqlDumpReader sqlDumpReader, IProgress <object> progressHandler, CancellationToken cancellationToken,
SqlDumpReader.ParsedTableDefinition parsedTableDefinition);
