Exemplo n.º 1
0
        public void OpenVerifyAndClean()
        {
            var tempFiles = Directory.GetFiles(Config.Path, "*.tmp");

            for (int i = 0; i < tempFiles.Length; i++)
            {
                File.Delete(tempFiles[i]);
            }

            ValidateReaderChecksumsMustBeLess(Config.WriterCheckpoint, Config.Checkpoints);

            var checkpoint = Config.WriterCheckpoint.Read();

            var expectedFiles = (int)((checkpoint + Config.ChunkSize - 1) / Config.ChunkSize);

            if (checkpoint == 0 && Config.FileNamingStrategy.GetAllVersionsFor(0).Length == 0)
            {
                Manager.AddNewChunk();
                expectedFiles = 1;
            }
            else
            {
                if (checkpoint == 0)
                {
                    expectedFiles = 1;
                }
                for (int i = 0; i < expectedFiles; ++i)
                {
                    var versions = Config.FileNamingStrategy.GetAllVersionsFor(i);
                    if (versions.Length == 0)
                    {
                        throw new CorruptDatabaseException(
                                  new ChunkNotFoundException(Config.FileNamingStrategy.GetFilenameFor(i)));
                    }

                    for (int j = 1; j < versions.Length; ++j)
                    {
                        File.Delete(versions[j]);
                    }

                    var chunkFileName = versions[0];
                    if (i == expectedFiles - 1)
                    {
                        var chunk = LoadLastChunk(chunkFileName);
                        Manager.AddChunk(chunk);

                        if (checkpoint % Config.ChunkSize == 0)
                        {
                            // there could be a new valid but empty chunk which could be corrupted
                            // so we can safely remove all its versions with no consequences
                            var files = Config.FileNamingStrategy.GetAllVersionsFor(expectedFiles);
                            for (int j = 0; j < files.Length; ++j)
                            {
                                File.Delete(files[j]);
                            }
                        }
                    }
                    else
                    {
                        var chunk = LoadChunk(chunkFileName);
                        Manager.AddChunk(chunk);
                    }
                }
            }

            EnsureNoOtherFiles(expectedFiles);

            Manager.EnableCaching();
        }
Exemplo n.º 2
0
        public void OpenVerifyAndClean(bool verifyHash = true)
        {
            var tempFiles = Config.FileNamingStrategy.GetAllTempFiles();

            for (int i = 0; i < tempFiles.Length; i++)
            {
                try
                {
                    File.Delete(tempFiles[i]);
                }
                catch (Exception exc)
                {
                    Log.ErrorException(exc, "Error while trying to delete remaining temp file: '{0}'.", tempFiles[i]);
                }
            }

            ValidateReaderChecksumsMustBeLess(Config.WriterCheckpoint, Config.Checkpoints);

            var checkpoint = Config.WriterCheckpoint.Read();

            var expectedFiles    = (int)((checkpoint + Config.ChunkSize - 1) / Config.ChunkSize);
            var correctFileCount = expectedFiles;
            var onChunkBoundary  = checkpoint > 0 && checkpoint % Config.ChunkSize == 0;

            if (checkpoint == 0 && Config.FileNamingStrategy.GetAllVersionsFor(0).Length == 0)
            {
                Manager.AddNewChunk();
                correctFileCount = expectedFiles = 1;
            }
            else
            {
                if (checkpoint == 0)
                {
                    correctFileCount = expectedFiles = 1;
                }
                for (int i = 0; i < expectedFiles; ++i)
                {
                    var versions = Config.FileNamingStrategy.GetAllVersionsFor(i);
                    if (versions.Length == 0)
                    {
                        throw new CorruptDatabaseException(
                                  new ChunkNotFoundException(Config.FileNamingStrategy.GetFilenameFor(i)));
                    }

                    for (int j = 1; j < versions.Length; ++j)
                    {
                        File.Delete(versions[j]);
                    }

                    var chunkFileName = versions[0];
                    if (i == expectedFiles - 1)
                    {
                        var chunk = LoadLastChunk(chunkFileName, verifyHash);
                        Manager.AddChunk(chunk);

                        if (onChunkBoundary)
                        {
                            if (!chunk.IsReadOnly)
                            {
                                chunk.Complete();
                            }

                            // there could be a new valid but empty chunk which could be corrupted
                            // so we can safely remove all its versions with no consequences
                            var files = Config.FileNamingStrategy.GetAllVersionsFor(expectedFiles);
                            for (int j = 0; j < files.Length; ++j)
                            {
                                File.Delete(files[j]);
                            }

                            Manager.AddNewChunk();
                            correctFileCount += 1;
                        }
                    }
                    else
                    {
                        var chunk = LoadChunk(chunkFileName, verifyHash);
                        Manager.AddChunk(chunk);
                    }
                }
            }

            EnsureNoOtherFiles(correctFileCount);

            Manager.EnableCaching();
        }
Exemplo n.º 3
0
        public void Open(bool verifyHash = true, bool readOnly = false)
        {
            ValidateReaderChecksumsMustBeLess(Config);
            var checkpoint = Config.WriterCheckpoint.Read();

            if (Config.InMemDb)
            {
                Manager.AddNewChunk();
                return;
            }

            var lastChunkNum      = (int)(checkpoint / Config.ChunkSize);
            var lastChunkVersions = Config.FileNamingStrategy.GetAllVersionsFor(lastChunkNum);

            for (int chunkNum = 0; chunkNum < lastChunkNum;)
            {
                var versions = Config.FileNamingStrategy.GetAllVersionsFor(chunkNum);
                if (versions.Length == 0)
                {
                    throw new CorruptDatabaseException(new ChunkNotFoundException(Config.FileNamingStrategy.GetFilenameFor(chunkNum, 0)));
                }

                TFChunk.TFChunk chunk;
                if (lastChunkVersions.Length == 0 && (chunkNum + 1) * (long)Config.ChunkSize == checkpoint)
                {
                    // The situation where the logical data size is exactly divisible by ChunkSize,
                    // so it might happen that we have checkpoint indicating one more chunk should exist,
                    // but the actual last chunk is (lastChunkNum-1) one and it could be not completed yet -- perfectly valid situation.
                    var footer = ReadChunkFooter(versions[0]);
                    if (footer.IsCompleted)
                    {
                        chunk = TFChunk.TFChunk.FromCompletedFile(versions[0], verifyHash: false, unbufferedRead: Config.Unbuffered, initialReaderCount: Config.InitialReaderCount);
                    }
                    else
                    {
                        chunk = TFChunk.TFChunk.FromOngoingFile(versions[0], Config.ChunkSize, checkSize: false, unbuffered: Config.Unbuffered, writethrough: Config.WriteThrough, initialReaderCount: Config.InitialReaderCount);
                        // chunk is full with data, we should complete it right here
                        if (!readOnly)
                        {
                            chunk.Complete();
                        }
                    }
                }
                else
                {
                    chunk = TFChunk.TFChunk.FromCompletedFile(versions[0], verifyHash: false, unbufferedRead: Config.Unbuffered, initialReaderCount: Config.InitialReaderCount);
                }
                Manager.AddChunk(chunk);
                chunkNum = chunk.ChunkHeader.ChunkEndNumber + 1;
            }

            if (lastChunkVersions.Length == 0)
            {
                var onBoundary = checkpoint == (Config.ChunkSize * (long)lastChunkNum);
                if (!onBoundary)
                {
                    throw new CorruptDatabaseException(new ChunkNotFoundException(Config.FileNamingStrategy.GetFilenameFor(lastChunkNum, 0)));
                }
                if (!readOnly)
                {
                    Manager.AddNewChunk();
                }
            }
            else
            {
                var chunkFileName = lastChunkVersions[0];
                var chunkHeader   = ReadChunkHeader(chunkFileName);
                var chunkLocalPos = chunkHeader.GetLocalLogPosition(checkpoint);
                if (chunkHeader.IsScavenged)
                {
                    var lastChunk = TFChunk.TFChunk.FromCompletedFile(chunkFileName, verifyHash: false, unbufferedRead: Config.Unbuffered, initialReaderCount: Config.InitialReaderCount);
                    if (lastChunk.ChunkFooter.LogicalDataSize != chunkLocalPos)
                    {
                        lastChunk.Dispose();
                        throw new CorruptDatabaseException(new BadChunkInDatabaseException(
                                                               string.Format("Chunk {0} is corrupted. Expected local chunk position: {1}, "
                                                                             + "but Chunk.LogicalDataSize is {2} (Chunk.PhysicalDataSize is {3}). Writer checkpoint: {4}.",
                                                                             chunkFileName, chunkLocalPos, lastChunk.LogicalDataSize, lastChunk.PhysicalDataSize, checkpoint)));
                    }
                    Manager.AddChunk(lastChunk);
                    if (!readOnly)
                    {
                        Log.Info("Moving WriterCheckpoint from {0} to {1}, as it points to the scavenged chunk. "
                                 + "If that was not caused by replication of scavenged chunks, that could be a bug.",
                                 checkpoint, lastChunk.ChunkHeader.ChunkEndPosition);
                        Config.WriterCheckpoint.Write(lastChunk.ChunkHeader.ChunkEndPosition);
                        Config.WriterCheckpoint.Flush();
                        Manager.AddNewChunk();
                    }
                }
                else
                {
                    var lastChunk = TFChunk.TFChunk.FromOngoingFile(chunkFileName, (int)chunkLocalPos, checkSize: false, unbuffered: Config.Unbuffered, writethrough: Config.WriteThrough, initialReaderCount: Config.InitialReaderCount);
                    Manager.AddChunk(lastChunk);
                }
            }

            EnsureNoExcessiveChunks(lastChunkNum);

            if (!readOnly)
            {
                RemoveOldChunksVersions(lastChunkNum);
                CleanUpTempFiles();
            }

            if (verifyHash && lastChunkNum > 0)
            {
                var preLastChunk   = Manager.GetChunk(lastChunkNum - 1);
                var lastBgChunkNum = preLastChunk.ChunkHeader.ChunkStartNumber - 1;
                ThreadPool.QueueUserWorkItem(_ =>
                {
                    for (int chunkNum = lastBgChunkNum; chunkNum >= 0;)
                    {
                        var chunk = Manager.GetChunk(chunkNum);
                        try
                        {
                            chunk.VerifyFileHash();
                        }
                        catch (FileBeingDeletedException exc)
                        {
                            Log.Trace("{0} exception was thrown while doing background validation of chunk {1}.",
                                      exc.GetType().Name, chunk);
                            Log.Trace("That's probably OK, especially if truncation was request at the same time: {0}.",
                                      exc.Message);
                        }
                        catch (Exception exc)
                        {
                            var msg = string.Format("Verification of chunk {0} failed, terminating server...", chunk);
                            Log.FatalException(exc, msg);
                            Application.Exit(ExitCode.Error, msg);
                            return;
                        }
                        chunkNum = chunk.ChunkHeader.ChunkStartNumber - 1;
                    }
                });
            }

            Manager.EnableCaching();
        }
Exemplo n.º 4
0
        public void Open(bool verifyHash = true, bool readOnly = false, int threads = 1)
        {
            Ensure.Positive(threads, "threads");

            ValidateReaderChecksumsMustBeLess(Config);
            var checkpoint = Config.WriterCheckpoint.Read();

            if (Config.InMemDb)
            {
                Manager.AddNewChunk();
                return;
            }

            var lastChunkNum      = (int)(checkpoint / Config.ChunkSize);
            var lastChunkVersions = Config.FileNamingStrategy.GetAllVersionsFor(lastChunkNum);
            var chunkEnumerator   = new TFChunkEnumerator(Config.FileNamingStrategy);

            try {
                Parallel.ForEach(GetAllLatestChunksExceptLast(chunkEnumerator, lastChunkNum),                 // the last chunk is dealt with separately
                                 new ParallelOptions {
                    MaxDegreeOfParallelism = threads
                },
                                 chunkInfo => {
                    TFChunk.TFChunk chunk;
                    if (lastChunkVersions.Length == 0 &&
                        (chunkInfo.ChunkStartNumber + 1) * (long)Config.ChunkSize == checkpoint)
                    {
                        // The situation where the logical data size is exactly divisible by ChunkSize,
                        // so it might happen that we have checkpoint indicating one more chunk should exist,
                        // but the actual last chunk is (lastChunkNum-1) one and it could be not completed yet -- perfectly valid situation.
                        var footer = ReadChunkFooter(chunkInfo.ChunkFileName);
                        if (footer.IsCompleted)
                        {
                            chunk = TFChunk.TFChunk.FromCompletedFile(chunkInfo.ChunkFileName, verifyHash: false,
                                                                      unbufferedRead: Config.Unbuffered,
                                                                      initialReaderCount: Config.InitialReaderCount,
                                                                      maxReaderCount: Config.MaxReaderCount,
                                                                      optimizeReadSideCache: Config.OptimizeReadSideCache,
                                                                      reduceFileCachePressure: Config.ReduceFileCachePressure);
                        }
                        else
                        {
                            chunk = TFChunk.TFChunk.FromOngoingFile(chunkInfo.ChunkFileName, Config.ChunkSize,
                                                                    checkSize: false,
                                                                    unbuffered: Config.Unbuffered,
                                                                    writethrough: Config.WriteThrough, initialReaderCount: Config.InitialReaderCount,
                                                                    maxReaderCount: Config.MaxReaderCount,
                                                                    reduceFileCachePressure: Config.ReduceFileCachePressure);
                            // chunk is full with data, we should complete it right here
                            if (!readOnly)
                            {
                                chunk.Complete();
                            }
                        }
                    }
                    else
                    {
                        chunk = TFChunk.TFChunk.FromCompletedFile(chunkInfo.ChunkFileName, verifyHash: false,
                                                                  unbufferedRead: Config.Unbuffered,
                                                                  initialReaderCount: Config.InitialReaderCount,
                                                                  maxReaderCount: Config.MaxReaderCount,
                                                                  optimizeReadSideCache: Config.OptimizeReadSideCache,
                                                                  reduceFileCachePressure: Config.ReduceFileCachePressure);
                    }

                    // This call is theadsafe.
                    Manager.AddChunk(chunk);
                });
            } catch (AggregateException aggEx) {
                // We only really care that *something* is wrong - throw the first inner exception.
                throw aggEx.InnerException;
            }

            if (lastChunkVersions.Length == 0)
            {
                var onBoundary = checkpoint == (Config.ChunkSize * (long)lastChunkNum);
                if (!onBoundary)
                {
                    throw new CorruptDatabaseException(
                              new ChunkNotFoundException(Config.FileNamingStrategy.GetFilenameFor(lastChunkNum, 0)));
                }
                if (!readOnly)
                {
                    Manager.AddNewChunk();
                }
            }
            else
            {
                var chunkFileName = lastChunkVersions[0];
                var chunkHeader   = ReadChunkHeader(chunkFileName);
                var chunkLocalPos = chunkHeader.GetLocalLogPosition(checkpoint);
                if (chunkHeader.IsScavenged)
                {
                    var lastChunk = TFChunk.TFChunk.FromCompletedFile(chunkFileName, verifyHash: false,
                                                                      unbufferedRead: Config.Unbuffered,
                                                                      initialReaderCount: Config.InitialReaderCount,
                                                                      maxReaderCount: Config.MaxReaderCount,
                                                                      optimizeReadSideCache: Config.OptimizeReadSideCache,
                                                                      reduceFileCachePressure: Config.ReduceFileCachePressure);
                    if (lastChunk.ChunkFooter.LogicalDataSize != chunkLocalPos)
                    {
                        lastChunk.Dispose();
                        throw new CorruptDatabaseException(new BadChunkInDatabaseException(
                                                               string.Format("Chunk {0} is corrupted. Expected local chunk position: {1}, "
                                                                             + "but Chunk.LogicalDataSize is {2} (Chunk.PhysicalDataSize is {3}). Writer checkpoint: {4}.",
                                                                             chunkFileName, chunkLocalPos, lastChunk.LogicalDataSize, lastChunk.PhysicalDataSize,
                                                                             checkpoint)));
                    }

                    Manager.AddChunk(lastChunk);
                    if (!readOnly)
                    {
                        _log.Information(
                            "Moving WriterCheckpoint from {checkpoint} to {chunkEndPosition}, as it points to the scavenged chunk. "
                            + "If that was not caused by replication of scavenged chunks, that could be a bug.",
                            checkpoint, lastChunk.ChunkHeader.ChunkEndPosition);
                        Config.WriterCheckpoint.Write(lastChunk.ChunkHeader.ChunkEndPosition);
                        Config.WriterCheckpoint.Flush();
                        Manager.AddNewChunk();
                    }
                }
                else
                {
                    var lastChunk = TFChunk.TFChunk.FromOngoingFile(chunkFileName, (int)chunkLocalPos, checkSize: false,
                                                                    unbuffered: Config.Unbuffered,
                                                                    writethrough: Config.WriteThrough, initialReaderCount: Config.InitialReaderCount,
                                                                    maxReaderCount: Config.MaxReaderCount,
                                                                    reduceFileCachePressure: Config.ReduceFileCachePressure);
                    Manager.AddChunk(lastChunk);
                }
            }

            _log.Information("Ensuring no excessive chunks...");
            EnsureNoExcessiveChunks(chunkEnumerator, lastChunkNum);
            _log.Information("Done ensuring no excessive chunks.");

            if (!readOnly)
            {
                _log.Information("Removing old chunk versions...");
                RemoveOldChunksVersions(chunkEnumerator, lastChunkNum);
                _log.Information("Done removing old chunk versions.");

                _log.Information("Cleaning up temp files...");
                CleanUpTempFiles();
                _log.Information("Done cleaning up temp files.");
            }

            if (verifyHash && lastChunkNum > 0)
            {
                var preLastChunk   = Manager.GetChunk(lastChunkNum - 1);
                var lastBgChunkNum = preLastChunk.ChunkHeader.ChunkStartNumber;
                ThreadPool.QueueUserWorkItem(_ => {
                    for (int chunkNum = lastBgChunkNum; chunkNum >= 0;)
                    {
                        var chunk = Manager.GetChunk(chunkNum);
                        try {
                            chunk.VerifyFileHash();
                        } catch (FileBeingDeletedException exc) {
                            _log.Debug(
                                "{exceptionType} exception was thrown while doing background validation of chunk {chunk}.",
                                exc.GetType().Name, chunk);
                            _log.Debug(
                                "That's probably OK, especially if truncation was request at the same time: {e}.",
                                exc.Message);
                        } catch (Exception exc) {
                            _log.Fatal(exc, "Verification of chunk {chunk} failed, terminating server...",
                                       chunk);
                            var msg = string.Format("Verification of chunk {0} failed, terminating server...", chunk);
                            Application.Exit(ExitCode.Error, msg);
                            return;
                        }

                        chunkNum = chunk.ChunkHeader.ChunkStartNumber - 1;
                    }
                });
            }

            Manager.EnableCaching();
        }