static void testSingleWrites(Dictionary <string, byte[]> extractedFiles, PersistentContainerCompressType compressType = 0)
{
var totalLengths = extractedFiles.Sum(p => p.Value.Length);
Console.WriteLine();
Console.WriteLine("Single requests - write [Compress={0}]", compressType.ToString());
Console.WriteLine("Page\tms\tMB\tMB/sec\tLost(%)");
foreach (var pageSize in pageSizes)
{
var targetFileName = getFileNameByPageSize(pageSize);
if (File.Exists(targetFileName))
{
File.Delete(targetFileName);
}
var sw = Stopwatch.StartNew();
using (var container = new PersistentContainer(targetFileName, new PersistentContainerSettings(pageSize, 0, compressType)))
foreach (var file in extractedFiles)
{
container.Put(file.Key, file.Value);
}
var targetFileLength = new FileInfo(targetFileName).Length;
sw.Stop();
var lengthInMB = targetFileLength / 1048576.0;
Console.WriteLine("{0,5}\t{1}\t{2:F1}\t{3:F1}\t{4}",
pageSize, sw.ElapsedMilliseconds, lengthInMB,
lengthInMB / (sw.ElapsedMilliseconds / 1000.0),
compressType == PersistentContainerCompressType.None
? (((targetFileLength - totalLengths) / (double)targetFileLength) * 100).ToString("F2")
: "-");
}
}
internal PagedContainerHeader(PersistentContainerSettings settings)
{
PageSize = settings.PageSize;
PageUserDataSize = settings.PageSize - 4;
Flags = settings.Flags;
CompressType = settings.CompressType;
DataHandler = getDataHandler(settings.CompressType, settings.encryptorDecryptor);
}
public PersistentContainerSettings(int pageSize = 4096,
PersistentContainerFlags flags = 0,
PersistentContainerCompressType compressType = PersistentContainerCompressType.None)
{
Extenders.ValidatePageSize(pageSize);
PageSize = pageSize;
Flags = flags;
CompressType = compressType;
}
internal PagedContainerHeader(Stream stm, IEncryptorDecryptor encryptorDecryptor)
{
if (stm.Length < HEADER_PART)
{
throw new InvalidDataException("PagedContainerHeader: File corrupted (too small)");
}
stm.Position = 0;
var buff = new byte[HEADER_PART];
stm.Read(buff, 0, buff.Length);
int offset = 0;
var sign = buff.GetInt(ref offset); // 4 byte
if (sign != SIGN)
{
throw new InvalidDataException($"PagedContainerHeader: File corruped (signature {sign:X}h invalid, must be {SIGN:X}h)");
}
PageSize = buff.GetInt(ref offset); // 4 byte
PageUserDataSize = PageSize - 4;
Extenders.ValidatePageSize(PageSize);
DirectoryFirstPage = buff.GetInt(ref offset); // 4 byte
if (DirectoryFirstPage < 0)
{
throw new InvalidDataException($"PagedContainerHeader: DirectoryFirstPage has invalid value ({DirectoryFirstPage})");
}
Flags = (PersistentContainerFlags)(buff.GetUInt16(ref offset) & 0xFFFF);
var flagsData = buff[offset];
CompressType = (PersistentContainerCompressType)flagsData;
DataHandler = getDataHandler(CompressType, encryptorDecryptor);
}
protected override void DoIt(Action <Func <PagedContainerAbstract> > action, PersistentContainerFlags flags = 0, PersistentContainerCompressType compressType = 0) => base.DoIt(action, PersistentContainerFlags.WriteDirImmediately, 0);
static void testBatchWrites(Dictionary <string, byte[]> extractedFiles, int batchSize, PersistentContainerCompressType compressType = 0)
{
Console.WriteLine();
Console.WriteLine("Batch requests [Compress={0}]: {1}", compressType.ToString(), batchSize);
Console.WriteLine("Page ms MB/sec(w) MB/sec(r) Ratio");
var items = makeBatches(extractedFiles, batchSize);
foreach (var pageSize in pageSizes)
{
var targetFileName = getFileNameByPageSize(pageSize);
if (File.Exists(targetFileName))
{
File.Delete(targetFileName);
}
var sw = Stopwatch.StartNew();
using (var container = new PersistentContainer(targetFileName, new PersistentContainerSettings(pageSize, 0, compressType)))
foreach (var item in items)
{
container.Put(item);
}
var elapsedWrites = sw.ElapsedMilliseconds;
var targetFileLength = new FileInfo(targetFileName).Length;
sw.Restart();
var rawLength = 0;
var compressedLength = 0;
using (var container = new PersistentContainer(targetFileName, new PersistentContainerSettings(pageSize, 0, compressType)))
{
foreach (var entry in container.Find())
{
compressedLength += entry.CompressedLength;
rawLength += container.Get(entry.Name)?.Length ?? 0;
}
}
var elapsedRead = sw.ElapsedMilliseconds;
sw.Stop();
var lengthInMB = targetFileLength / 1048576.0;
var rawLengthInMB = rawLength / 1048576.0;
Console.WriteLine("{0,5} {1,8} {2,8:F1} {3,8:F1} {4,8:F2}x",
pageSize, sw.ElapsedMilliseconds,
lengthInMB / (elapsedWrites / 1000.0),
rawLengthInMB / (elapsedRead / 1000.0),
rawLength / (double)compressedLength);
}
}
IDataHandler getDataHandler(PersistentContainerCompressType compressType, IEncryptorDecryptor encryptorDecryptor) =>
compressType switch
{
protected override void DoIt(Action <Func <PagedContainerAbstract> > action, PersistentContainerFlags flags = 0, PersistentContainerCompressType compressType = 0) => base.DoIt(action, flags, PersistentContainerCompressType.GZip);
