Esempio n. 1
0
        public Task <ImportSqlDumpResult> ImportSqlDumpAsync(string sqlDumpFilePath, IProgress <object> progressHandler, CancellationToken cancellationToken)
        {
            return(Task.Run(() =>
            {
                Logger.Debug("SQL dump import has started.");
                using (SqlDumpReader sqlDumpReader = new SqlDumpReader(sqlDumpFilePath))
                {
                    while (true)
                    {
                        bool tableFound = false;
                        while (sqlDumpReader.ReadLine())
                        {
                            if (cancellationToken.IsCancellationRequested)
                            {
                                Logger.Debug("SQL dump import has been cancelled.");
                                return ImportSqlDumpResult.CANCELLED;
                            }
                            if (sqlDumpReader.CurrentLineCommand == SqlDumpReader.LineCommand.CREATE_TABLE)
                            {
                                Logger.Debug("CREATE TABLE statement found.");
                                tableFound = true;
                                break;
                            }
                            progressHandler.Report(new ImportSearchTableDefinitionProgress(sqlDumpReader.CurrentFilePosition, sqlDumpReader.FileSize));
                        }
                        if (!tableFound)
                        {
                            Logger.Debug("CREATE TABLE statement was not found.");
                            return ImportSqlDumpResult.DATA_NOT_FOUND;
                        }
                        if (cancellationToken.IsCancellationRequested)
                        {
                            Logger.Debug("SQL dump import has been cancelled.");
                            return ImportSqlDumpResult.CANCELLED;
                        }
                        SqlDumpReader.ParsedTableDefinition parsedTableDefinition = sqlDumpReader.ParseTableDefinition();
                        TableType tableType = DetectImportTableType(parsedTableDefinition);
                        if (tableType == TableType.UNKNOWN)
                        {
                            continue;
                        }
                        progressHandler.Report(new ImportTableDefinitionFoundProgress(tableType));
                        bool insertFound = false;
                        while (sqlDumpReader.ReadLine())
                        {
                            if (cancellationToken.IsCancellationRequested)
                            {
                                Logger.Debug("SQL dump import has been cancelled.");
                                return ImportSqlDumpResult.CANCELLED;
                            }
                            if (sqlDumpReader.CurrentLineCommand == SqlDumpReader.LineCommand.INSERT)
                            {
                                Logger.Debug("INSERT statement found.");
                                insertFound = true;
                                break;
                            }
                        }
                        if (!insertFound)
                        {
                            Logger.Debug("INSERT statement was not found.");
                            return ImportSqlDumpResult.DATA_NOT_FOUND;
                        }
                        if (cancellationToken.IsCancellationRequested)
                        {
                            Logger.Debug("SQL dump import has been cancelled.");
                            return ImportSqlDumpResult.CANCELLED;
                        }
                        Logger.Debug($"Table type is {tableType}.");
                        Importer importer;
                        BitArray existingLibgenIds = null;
                        switch (tableType)
                        {
                        case TableType.NON_FICTION:
                            if (NonFictionBookCount != 0)
                            {
                                CheckAndCreateNonFictionIndexes(progressHandler, cancellationToken);
                                if (cancellationToken.IsCancellationRequested)
                                {
                                    return ImportSqlDumpResult.CANCELLED;
                                }
                                progressHandler.Report(new ImportLoadLibgenIdsProgress());
                                existingLibgenIds = localDatabase.GetNonFictionLibgenIdsBitArray();
                            }
                            importer = new NonFictionImporter(localDatabase, existingLibgenIds);
                            break;

                        case TableType.FICTION:
                            if (FictionBookCount != 0)
                            {
                                CheckAndCreateFictionIndexes(progressHandler, cancellationToken);
                                if (cancellationToken.IsCancellationRequested)
                                {
                                    return ImportSqlDumpResult.CANCELLED;
                                }
                                progressHandler.Report(new ImportLoadLibgenIdsProgress());
                                existingLibgenIds = localDatabase.GetFictionLibgenIdsBitArray();
                            }
                            importer = new FictionImporter(localDatabase, existingLibgenIds);
                            break;

                        case TableType.SCI_MAG:
                            if (SciMagArticleCount != 0)
                            {
                                CheckAndCreateSciMagIndexes(progressHandler, cancellationToken);
                                if (cancellationToken.IsCancellationRequested)
                                {
                                    return ImportSqlDumpResult.CANCELLED;
                                }
                                progressHandler.Report(new ImportLoadLibgenIdsProgress());
                                existingLibgenIds = localDatabase.GetSciMagLibgenIdsBitArray();
                            }
                            importer = new SciMagImporter(localDatabase, existingLibgenIds);
                            break;

                        default:
                            throw new Exception($"Unknown table type: {tableType}.");
                        }
                        if (cancellationToken.IsCancellationRequested)
                        {
                            Logger.Debug("SQL dump import has been cancelled.");
                            return ImportSqlDumpResult.CANCELLED;
                        }
                        Importer.ImportProgressReporter importProgressReporter = (int objectsAdded, int objectsUpdated) =>
                        {
                            progressHandler.Report(new ImportObjectsProgress(objectsAdded, objectsUpdated));
                        };
                        Logger.Debug("Importing data.");
                        importer.Import(sqlDumpReader, importProgressReporter, IMPORT_PROGRESS_UPDATE_INTERVAL, cancellationToken, parsedTableDefinition);
                        switch (tableType)
                        {
                        case TableType.NON_FICTION:
                            UpdateNonFictionBookCount();
                            break;

                        case TableType.FICTION:
                            UpdateFictionBookCount();
                            break;

                        case TableType.SCI_MAG:
                            UpdateSciMagArticleCount();
                            break;
                        }
                        if (cancellationToken.IsCancellationRequested)
                        {
                            Logger.Debug("SQL dump import has been cancelled.");
                            return ImportSqlDumpResult.CANCELLED;
                        }
                        switch (tableType)
                        {
                        case TableType.NON_FICTION:
                            DatabaseMetadata.NonFictionFirstImportComplete = true;
                            break;

                        case TableType.FICTION:
                            DatabaseMetadata.FictionFirstImportComplete = true;
                            break;

                        case TableType.SCI_MAG:
                            DatabaseMetadata.SciMagFirstImportComplete = true;
                            break;
                        }
                        localDatabase.UpdateMetadata(DatabaseMetadata);
                        Logger.Debug("SQL dump import has been completed successfully.");
                        return ImportSqlDumpResult.COMPLETED;
                    }
                }
            }));
        }
Esempio n. 2
0
        public int Import()
        {
            ServerConfiguration serverConfiguration = LoadServerConfiguration();

            if (serverConfiguration == null)
            {
                return(1);
            }
            string databaseFilePath = serverConfiguration.DatabaseFilePath;

            if (String.IsNullOrWhiteSpace(databaseFilePath))
            {
                Console.WriteLine("Database file path is not set in the server configuration file.");
                return(2);
            }
            if (!File.Exists(databaseFilePath))
            {
                Console.WriteLine($"Couldn't find the database file at {databaseFilePath}");
                return(3);
            }
            Console.WriteLine($"Opening database \"{databaseFilePath}\"...");
            LocalDatabase localDatabase = LocalDatabase.OpenDatabase(databaseFilePath);

            using (SqlDumpReader sqlDumpReader = new SqlDumpReader(importFilePath))
            {
                while (true)
                {
                    bool tableFound = false;
                    while (sqlDumpReader.ReadLine())
                    {
                        if (sqlDumpReader.CurrentLineCommand == SqlDumpReader.LineCommand.CREATE_TABLE)
                        {
                            Logger.Debug("CREATE TABLE statement found.");
                            Console.WriteLine("CREATE TABLE statement found.");
                            tableFound = true;
                            break;
                        }
                    }
                    if (!tableFound)
                    {
                        Logger.Debug("CREATE TABLE statement was not found.");
                        Console.WriteLine("CREATE TABLE statement was not found.");
                        return(4);
                    }
                    SqlDumpReader.ParsedTableDefinition parsedTableDefinition = sqlDumpReader.ParseTableDefinition();
                    TableType tableType = DetectImportTableType(parsedTableDefinition);
                    if (tableType == TableType.UNKNOWN)
                    {
                        Console.WriteLine($"Table {parsedTableDefinition.TableName} doesn't match any known supported formats. It will be skipped.");
                        continue;
                    }
                    Logger.Debug($"Table type is {tableType}.");
                    ImportFormat?detectedImportFormat = ConvertTableTypeToImportFormat(tableType);
                    if (!detectedImportFormat.HasValue)
                    {
                        Console.WriteLine("Could not determine the format of the database dump.");
                        return(5);
                    }
                    if (detectedImportFormat.Value == importFormat)
                    {
                        Console.WriteLine($"Found a matching import format: {ImportFormatToString(detectedImportFormat.Value)}.");
                    }
                    else
                    {
                        Console.WriteLine($"Expected the import format {importFormat} but found {ImportFormatToString(detectedImportFormat.Value)}.");
                        return(6);
                    }
                    bool insertFound = false;
                    while (sqlDumpReader.ReadLine())
                    {
                        if (sqlDumpReader.CurrentLineCommand == SqlDumpReader.LineCommand.INSERT)
                        {
                            Logger.Debug("INSERT statement found.");
                            insertFound = true;
                            break;
                        }
                    }
                    if (!insertFound)
                    {
                        Logger.Debug("INSERT statement was not found.");
                        Console.WriteLine("Couldn't find any data to import.");
                        return(7);
                    }
                    Importer importer;
                    BitArray existingLibgenIds = null;
                    int      existingBookCount = 0;
                    switch (tableType)
                    {
                    case TableType.NON_FICTION:
                        existingBookCount = localDatabase.CountNonFictionBooks();
                        if (existingBookCount != 0)
                        {
                            existingLibgenIds = localDatabase.GetNonFictionLibgenIdsBitArray();
                        }
                        importer = new NonFictionImporter(localDatabase, existingLibgenIds);
                        break;

                    case TableType.FICTION:
                        existingBookCount = localDatabase.CountFictionBooks();
                        if (existingBookCount != 0)
                        {
                            existingLibgenIds = localDatabase.GetFictionLibgenIdsBitArray();
                        }
                        importer = new FictionImporter(localDatabase, existingLibgenIds);
                        break;

                    case TableType.SCI_MAG:
                        existingBookCount = localDatabase.CountSciMagArticles();
                        if (existingBookCount != 0)
                        {
                            existingLibgenIds = localDatabase.GetSciMagLibgenIdsBitArray();
                        }
                        importer = new SciMagImporter(localDatabase, existingLibgenIds);
                        break;

                    default:
                        throw new Exception($"Unknown table type: {tableType}.");
                    }
                    bool updateProgressLine = false;
                    Importer.ImportProgressReporter importProgressReporter = (int objectsAdded, int objectsUpdated) =>
                    {
                        if (updateProgressLine)
                        {
                            Console.SetCursorPosition(0, Console.CursorTop - 1);
                        }
                        else
                        {
                            updateProgressLine = true;
                        }
                        if (objectsUpdated > 0)
                        {
                            Console.WriteLine($"Books added: {objectsAdded}, updated: {objectsUpdated}.");
                        }
                        else
                        {
                            Console.WriteLine($"Books added: {objectsAdded}.");
                        }
                    };
                    Logger.Debug("Importing data.");
                    Console.WriteLine("Importing data.");
                    importer.Import(sqlDumpReader, importProgressReporter, IMPORT_PROGRESS_UPDATE_INTERVAL, parsedTableDefinition);
                    DatabaseMetadata databaseMetadata = localDatabase.GetMetadata();
                    switch (tableType)
                    {
                    case TableType.NON_FICTION:
                        databaseMetadata.NonFictionFirstImportComplete = true;
                        break;

                    case TableType.FICTION:
                        databaseMetadata.FictionFirstImportComplete = true;
                        break;

                    case TableType.SCI_MAG:
                        databaseMetadata.SciMagFirstImportComplete = true;
                        break;
                    }
                    localDatabase.UpdateMetadata(databaseMetadata);
                    Logger.Debug("SQL dump import has been completed successfully.");
                    Console.WriteLine("SQL dump import has been completed successfully.");
                    return(0);
                }
            }
        }