public async Task Clusterization(string fileName)
{
var sp = new SP();
var ms = new MS();
var clist = new List <ER>();
using (var t_b = new TrainBag(1,
new BucketFactory(new ZipStoreFactory(sp)),
ms,
new BagSettings {
PerBucketMessages = 5000
}))
{
int index = 0;
var pushSW = Stopwatch.StartNew();
foreach (var l in ZipLogs.LoadLines(fileName))
{
var addr = await t_b.Push(l);
clist.Add(new ER(addr, l));
if ((++index) % 100000 == 0)
{
TestContext.Progress.WriteLine($"{index} messages");
}
}
pushSW.Stop();
TestContext.WriteLine($"Push time: {pushSW.ElapsedMilliseconds} ms, Speed: {(index * 1000)/(pushSW.ElapsedMilliseconds + 1)} msgs/sec");
}
using (var t_b = new TrainBag(1,
new BucketFactory(new ZipStoreFactory(sp)),
ms,
new BagSettings {
PerBucketMessages = 5000
}))
{
var index = 0;
TestContext.Progress.WriteLine("Reading ...");
foreach (var s in clist)
{
var entry = await t_b.ReadEntry(s.EntryAddress);
if (entry != s.Message)
{
throw new InvalidOperationException("Read write error");
}
if ((++index) % 100000 == 0)
{
TestContext.Progress.WriteLine($"{index} messages");
}
}
}
var compressedSize = sp.streams
.Select(q => q.Value).Select(q => q.Data.Length).Sum();
var sourceSize = new FileInfo(fileName).Length;
TestContext.WriteLine($"Compression: {compressedSize} vs {sourceSize} ({(compressedSize * 100) / sourceSize})");
}
public async Task PrimaryIndex(string fileName)
{
var random = new Random(0);
var sp = new SP();
var ms = new MS();
var clist = new List <ER>();
var startDateTime = DateTime.Now;
var dateTime = startDateTime;
var dt_start = DateTime.Now;
var st_ticks = new DateTime(2018, 1, 1).Ticks;
var dt_st_ticks = dt_start.Ticks;
var delta = dt_st_ticks - st_ticks;
var dateIndex = new SkipList <long, ulong>((a, b) => (float)Math.Abs(a - b));
using (var t_b = new TrainBag(1,
new BucketFactory(new ZipStoreFactory(sp)),
ms,
new BagSettings {
PerBucketMessages = 5000
}))
{
int index = 0;
for (int k = 0; k < 10; ++k)
{
var pushSW = Stopwatch.StartNew();
foreach (var l in ZipLogs.LoadLines(fileName))
{
var addr = await t_b.Push(l);
clist.Add(new ER(addr, l));
dateIndex.Add(dateTime.Ticks, addr);
dateTime += TimeSpan.FromSeconds((random.NextDouble() - 0.2) * 20.0);
if ((++index) % 100000 == 0)
{
TestContext.Progress.WriteLine($"{index} messages");
}
}
pushSW.Stop();
TestContext.Progress.WriteLine($"Iteration: {k}, time = {pushSW.ElapsedMilliseconds} ms");
}
var lrg_d = startDateTime + TimeSpan.FromSeconds((dateTime - startDateTime).TotalSeconds / 2);
var kv_s = dateIndex
.Larger(lrg_d.Ticks, false)
.Take(10).ToArray();
var rebT = Stopwatch.StartNew();
var dateIndex2 = new SkipList <long, ulong>((a, b) => (float)Math.Abs(a - b));
foreach (var a in dateIndex.KeyValues())
{
dateIndex2.Add(a.Key, a.Value);
}
rebT.Stop();
TestContext.Progress.WriteLine($"Rebuild time: {rebT.ElapsedMilliseconds} ms");
//TestContext.WriteLine($"Push time: {pushSW.ElapsedMilliseconds} ms, Speed: {(index * 1000) / (pushSW.ElapsedMilliseconds + 1)} msgs/sec");
}
}
