static void testSingleWrites(Dictionary <string, byte[]> extractedFiles, PersistentContainerCompressType compressType = 0) { var totalLengths = extractedFiles.Sum(p => p.Value.Length); Console.WriteLine(); Console.WriteLine("Single requests - write [Compress={0}]", compressType.ToString()); Console.WriteLine("Page\tms\tMB\tMB/sec\tLost(%)"); foreach (var pageSize in pageSizes) { var targetFileName = getFileNameByPageSize(pageSize); if (File.Exists(targetFileName)) { File.Delete(targetFileName); } var sw = Stopwatch.StartNew(); using (var container = new PersistentContainer(targetFileName, new PersistentContainerSettings(pageSize, 0, compressType))) foreach (var file in extractedFiles) { container.Put(file.Key, file.Value); } var targetFileLength = new FileInfo(targetFileName).Length; sw.Stop(); var lengthInMB = targetFileLength / 1048576.0; Console.WriteLine("{0,5}\t{1}\t{2:F1}\t{3:F1}\t{4}", pageSize, sw.ElapsedMilliseconds, lengthInMB, lengthInMB / (sw.ElapsedMilliseconds / 1000.0), compressType == PersistentContainerCompressType.None ? (((targetFileLength - totalLengths) / (double)targetFileLength) * 100).ToString("F2") : "-"); } }
internal PagedContainerHeader(PersistentContainerSettings settings) { PageSize = settings.PageSize; PageUserDataSize = settings.PageSize - 4; Flags = settings.Flags; CompressType = settings.CompressType; DataHandler = getDataHandler(settings.CompressType, settings.encryptorDecryptor); }
public PersistentContainerSettings(int pageSize = 4096, PersistentContainerFlags flags = 0, PersistentContainerCompressType compressType = PersistentContainerCompressType.None) { Extenders.ValidatePageSize(pageSize); PageSize = pageSize; Flags = flags; CompressType = compressType; }
internal PagedContainerHeader(Stream stm, IEncryptorDecryptor encryptorDecryptor) { if (stm.Length < HEADER_PART) { throw new InvalidDataException("PagedContainerHeader: File corrupted (too small)"); } stm.Position = 0; var buff = new byte[HEADER_PART]; stm.Read(buff, 0, buff.Length); int offset = 0; var sign = buff.GetInt(ref offset); // 4 byte if (sign != SIGN) { throw new InvalidDataException($"PagedContainerHeader: File corruped (signature {sign:X}h invalid, must be {SIGN:X}h)"); } PageSize = buff.GetInt(ref offset); // 4 byte PageUserDataSize = PageSize - 4; Extenders.ValidatePageSize(PageSize); DirectoryFirstPage = buff.GetInt(ref offset); // 4 byte if (DirectoryFirstPage < 0) { throw new InvalidDataException($"PagedContainerHeader: DirectoryFirstPage has invalid value ({DirectoryFirstPage})"); } Flags = (PersistentContainerFlags)(buff.GetUInt16(ref offset) & 0xFFFF); var flagsData = buff[offset]; CompressType = (PersistentContainerCompressType)flagsData; DataHandler = getDataHandler(CompressType, encryptorDecryptor); }
protected override void DoIt(Action <Func <PagedContainerAbstract> > action, PersistentContainerFlags flags = 0, PersistentContainerCompressType compressType = 0) => base.DoIt(action, PersistentContainerFlags.WriteDirImmediately, 0);
static void testBatchWrites(Dictionary <string, byte[]> extractedFiles, int batchSize, PersistentContainerCompressType compressType = 0) { Console.WriteLine(); Console.WriteLine("Batch requests [Compress={0}]: {1}", compressType.ToString(), batchSize); Console.WriteLine("Page ms MB/sec(w) MB/sec(r) Ratio"); var items = makeBatches(extractedFiles, batchSize); foreach (var pageSize in pageSizes) { var targetFileName = getFileNameByPageSize(pageSize); if (File.Exists(targetFileName)) { File.Delete(targetFileName); } var sw = Stopwatch.StartNew(); using (var container = new PersistentContainer(targetFileName, new PersistentContainerSettings(pageSize, 0, compressType))) foreach (var item in items) { container.Put(item); } var elapsedWrites = sw.ElapsedMilliseconds; var targetFileLength = new FileInfo(targetFileName).Length; sw.Restart(); var rawLength = 0; var compressedLength = 0; using (var container = new PersistentContainer(targetFileName, new PersistentContainerSettings(pageSize, 0, compressType))) { foreach (var entry in container.Find()) { compressedLength += entry.CompressedLength; rawLength += container.Get(entry.Name)?.Length ?? 0; } } var elapsedRead = sw.ElapsedMilliseconds; sw.Stop(); var lengthInMB = targetFileLength / 1048576.0; var rawLengthInMB = rawLength / 1048576.0; Console.WriteLine("{0,5} {1,8} {2,8:F1} {3,8:F1} {4,8:F2}x", pageSize, sw.ElapsedMilliseconds, lengthInMB / (elapsedWrites / 1000.0), rawLengthInMB / (elapsedRead / 1000.0), rawLength / (double)compressedLength); } }
IDataHandler getDataHandler(PersistentContainerCompressType compressType, IEncryptorDecryptor encryptorDecryptor) => compressType switch {
protected override void DoIt(Action <Func <PagedContainerAbstract> > action, PersistentContainerFlags flags = 0, PersistentContainerCompressType compressType = 0) => base.DoIt(action, flags, PersistentContainerCompressType.GZip);