private void LoaderForm_Shown(object sender, System.EventArgs e)
{
lblProgressMessage.Text = string.Empty;
Application.DoEvents(); // Make sure controls render before we do something.
var cacheFiles = RootEntry.GetCacheFileList(_cdeList);
var totalFiles = cacheFiles.Count();
var fileCounter = 0;
barLoading.Step = totalFiles;
barLoading.Maximum = totalFiles;
// Not doing a background worker.
_rootEntries = cacheFiles.Select(s =>
{
_timeIt.Start(s);
var re = RootEntry.LoadDirCache(s);
_timeIt.Stop();
++fileCounter;
lblProgressMessage.Text = $"Loading catalog {fileCounter} of {totalFiles}";
barLoading.Value = fileCounter;
Application.DoEvents(); // Get label to update.
return(re);
}).ToList();
Close();
}
public void LoadData(Func <int, bool> notifyCount)
{
var paths = new[] { _basePath };
var cacheFiles = RootEntry.GetCacheFileList(paths);
var cacheFileCount = cacheFiles.Count;
_loadMetric.Start("Loading catalogs");
_notifyCount(cacheFileCount);
_rootEntries = cacheFiles.Select(s =>
{
var re = RootEntry.LoadDirCache(s);
--cacheFileCount;
_notifyCount(cacheFileCount);
return(re);
}).ToList();
_loadMetric.Stop();
}
public void BeforeAllTests()
{
RootSmall = RootEntry.LoadDirCache(TestCatalog200K);
if (RootSmall == null)
{
Assert.Inconclusive("TestData not Found" + TestCatalog200K);
}
var e1 = CommonEntry.GetDirEntries(RootSmall);
RootSmallCount = e1.Count();
RootLarge = RootEntry.LoadDirCache(TestCatalog1_2M);
if (RootSmall == null)
{
Assert.Inconclusive("TestData not Found" + TestCatalog1_2M);
}
var e2 = CommonEntry.GetDirEntries(RootLarge);
RootLargeCount = e2.Count();
}
private static RootEntry MeasureLoad(string catalogName)
{
var sw = new Stopwatch();
sw.Start();
var reC = RootEntry.LoadDirCache(catalogName);
sw.Stop();
var loadTime = sw.ElapsedMilliseconds;
if (reC == null)
{
Console.WriteLine("Not Loaded!");
return(null);
}
Console.WriteLine("Loaded!");
Console.WriteLine("loadTime " + loadTime + " msecs");
return(reC);
}
public static void CreateCache(string path)
{
var re = new RootEntry();
try
{
re.SimpleScanCountEvent = ScanCountPrintDot;
re.SimpleScanEndEvent = ScanEndofEntries;
re.ExceptionEvent = PrintExceptions;
re.PopulateRoot(path);
if (Hack.BreakConsoleFlag)
{
Console.WriteLine(" * Break key detected incomplete scan will not be saved.");
return;
}
var oldRoot = RootEntry.LoadDirCache(re.DefaultFileName);
if (oldRoot != null)
{
Console.WriteLine($"Found cache \"{re.DefaultFileName}\"");
Console.WriteLine("Updating hashs on new scan from found cache file.");
oldRoot.TraverseTreesCopyHash(re);
}
re.SortAllChildrenByPath();
re.SaveRootEntry();
var scanTimeSpan = (re.ScanEndUTC - re.ScanStartUTC);
Console.WriteLine($"Scanned Path {re.Path}");
Console.WriteLine($"Scan time {scanTimeSpan.TotalMilliseconds:0.00} msecs");
Console.WriteLine($"Saved Scanned Path {re.DefaultFileName}");
Console.WriteLine(
$"Files {re.FileEntryCount:0,0} Dirs {re.DirEntryCount:0,0} Total Size of Files {re.Size:0,0}");
}
catch (ArgumentException aex)
{
Console.WriteLine($"Error: {aex.Message}");
}
}
public void describe_performance_test_compression_of_cde()
{
//wtf();
//specify = () => "gumby".should_be("gumby");
//describe["long perf test"] = () => {
//it["runs performance tests"] = () => {
// Console.WriteLine("performance testing.");
// specify = () => "gumby".should_be("gumby");
//};
RootEntry tiny = null;
tiny = RootEntry.LoadDirCache(TestCatalog32K);
RootEntry small = null;
small = RootEntry.LoadDirCache(TestCatalog200K);
RootEntry large = null;
large = RootEntry.LoadDirCache(TestCatalog1_2M);
scenario[] Scenarios =
{
new scenario {
desc = "Serialize tiny to MemStream", fqFileName = "tinyNone.cde",
isSerialising = true, compressCode = "None",
write = writeStream, root = tiny,
read = readStream
},
new scenario {
desc = "Serialize small to MemStream", fqFileName = "smallNone.cde",
isSerialising = true, compressCode = "None",
write = writeStream, root = small,
read = readStream
},
new scenario {
desc = "Serialize large to MemStream", fqFileName = "largeNone.cde",
isSerialising = true, compressCode = "None",
write = writeStream, root = large,
read = readStream
},
new scenario {
desc = "Serialize tiny to Gzip MemStream", fqFileName = "tinyGzip.cde",
isSerialising = true, compressCode = "Gzip",
write = writeGzipStream, root = tiny,
read = readGzipStream
},
new scenario {
desc = "Serialize small to Gzip MemStream", fqFileName = "smallGzip.cde",
isSerialising = true, compressCode = "Gzip",
write = writeGzipStream, root = small,
read = readGzipStream
},
new scenario {
desc = "Serialize large to Gzip MemStream", fqFileName = "largeGzip.cde",
isSerialising = true, compressCode = "Gzip",
write = writeGzipStream, root = large,
read = readGzipStream
},
// to similar to Gzip to bother testing
//new scenario {
// desc = "Serialize tiny to Deflate MemStream", fqFileName = "tinyDefl.cde",
// isSerialising = true, compressCode = "Defl",
// write = writeDeflateStream, root = tiny,
// read = readDeflateStream
// },
//new scenario {
// desc = "Serialize small to Deflate MemStream", fqFileName = "smallDefl.cde",
// isSerialising = true, compressCode = "Defl",
// write = writeDeflateStream, root = small,
// read = readDeflateStream
// },
//new scenario {
// desc = "Serialize large to Deflate MemStream", fqFileName = "largeDefl.cde",
// isSerialising = true, compressCode = "Defl",
// write = writeDeflateStream, root = large,
// read = readDeflateStream
// },
new scenario {
desc = "Serialize tiny to LZ4 MemStream",
isSerialising = true, compressCode = "lz4", fqFileName = "tinyLZ4.cde",
write = writeLZ4Stream, root = tiny,
read = readLZ4Stream
},
new scenario {
desc = "Serialize small to LZ4 MemStream",
isSerialising = true, compressCode = "lz4", fqFileName = "smallLZ4.cde",
write = writeLZ4Stream, root = small,
read = readLZ4Stream
},
new scenario {
desc = "Serialize large to LZ4 MemStream",
isSerialising = true, compressCode = "lz4", fqFileName = "largeLZ4.cde",
write = writeLZ4Stream, root = large,
read = readLZ4Stream
},
new scenario {
desc = "Serialize tiny to LZ4H MemStream",
isSerialising = true, compressCode = "lz4h", fqFileName = "tinyLZ4H.cde",
write = writeLZ4HStream, root = tiny,
read = readLZ4Stream
},
new scenario {
desc = "Serialize small to LZ4H MemStream",
isSerialising = true, compressCode = "lz4h", fqFileName = "smallLZ4H.cde",
write = writeLZ4HStream, root = small,
read = readLZ4Stream
},
new scenario {
desc = "Serialize large to LZ4H MemStream",
isSerialising = true, compressCode = "lz4h", fqFileName = "largeLZ4H.cde",
write = writeLZ4HStream, root = large,
read = readLZ4Stream
},
new scenario {
desc = "Serialize tiny to LZ4N MemStream",
isSerialising = true, compressCode = "lz4N", fqFileName = "tinyLZ4N.cde",
write = writeLZ4NStream, root = tiny,
read = readLZ4NStream
},
new scenario {
desc = "Serialize small to LZ4N MemStream",
isSerialising = true, compressCode = "lz4N", fqFileName = "smallLZ4N.cde",
write = writeLZ4NStream, root = small,
read = readLZ4NStream
},
new scenario {
desc = "Serialize large to LZ4N MemStream",
isSerialising = true, compressCode = "lz4N", fqFileName = "largeLZ4N.cde",
write = writeLZ4NStream, root = large,
read = readLZ4NStream
},
new scenario {
desc = "Serialize tiny to LZ4NH MemStream",
isSerialising = true, compressCode = "lz4NH", fqFileName = "tinyLZ4NH.cde",
write = writeLZ4NHStream, root = tiny,
read = readLZ4NStream
},
new scenario {
desc = "Serialize small to LZ4NH MemStream",
isSerialising = true, compressCode = "lz4NH", fqFileName = "smallLZ4NH.cde",
write = writeLZ4NHStream, root = small,
read = readLZ4NStream
},
new scenario {
desc = "Serialize large to LZ4NH MemStream",
isSerialising = true, compressCode = "lz4NH", fqFileName = "largeLZ4NH.cde",
write = writeLZ4NHStream, root = large,
read = readLZ4NStream
},
//new scenario
// {
// desc = "Blat small to MemStream",
// isSerialising = false,
// isCompressed = false,
// write = writeStream,
// root = small,
// },
};
for (var i = 0; i < Scenarios.Length; i++)
{
measureThis(ref Scenarios[i]);
}
Console.WriteLine("");
Console.WriteLine("");
for (var i = 0; i < Scenarios.Length; i++) // set originalSize on records from None records.
{
Scenarios[i].originalSize =
Scenarios.Where(s =>
s.count == Scenarios[i].count &&
s.compressCode == "None")
.First().writtenSize;
}
for (var i = 0; i < Scenarios.Length; i++)
{
Scenarios[i].printer();
}
}