[Run("!arch-perf-read", "compress=null encrypt=null search=$(~@term)")]// -r args='term=abcd'
//[Run("!arch-perf-read", "compress=gzip encrypt=aes1 search=$(~@term)")]// -r args='term=abcd'
public void Read_LogMessages(string compress, string encrypt, string search)
{
search = search.Default("ABBA");
var msData = new FileStream("c:\\azos\\logging-{0}-{1}.lar".Args(compress.Default("none"), encrypt.Default("none")), FileMode.Open);
var msIdxId = new FileStream("c:\\azos\\logging-{0}-{1}.guid.lix".Args(compress.Default("none"), encrypt.Default("none")), FileMode.Open);
var volumeData = new DefaultVolume(CryptoMan, msData);
var volumeIdxId = new DefaultVolume(CryptoMan, msIdxId);
//volumeData.PageSizeBytes = 4 * 1024 * 1024;//<-------------------!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//volumeIdxId.PageSizeBytes = 4 * 128 * 1024;
var reader = new LogMessageArchiveReader(volumeData);
var idxReader = new StringIdxReader(volumeIdxId);
var time = Azos.Time.Timeter.StartNew();
var total = 0;
var wordCount = 0;
var found = 0;
//foreach (var idx in idxReader.All)
//{
// total++;
// if (idx.Value.EqualsOrdIgnoreCase(search))
// {
// found++;
// var data = reader.Entries(idx.Bookmark).FirstOrDefault();
// data.See();
// }
//}
////////Rent buffers from arena instead? who is going to release pages? page.Recycle() to release the buffer to the pool? What if they forget to call it?
////////var prealloc = new Page(0); //SIGNIFICANT SLOW-DOWN WHILE ALLOCATING PAGES. WHY MEMORY STREAM DOES ARRAY.CLEAR?
//////foreach (var page in reader.GetPagesStartingAt(0/*, preallocatedPage: prealloc*/).Take(200))
//////{
////// total++;
////// //if (total % 100 ==0)
////// "Page {0:X8} is {1:n0} --- Average size: {2:n0} Entries per page: {3:n0}".SeeArgs(page.PageId, page.Data.Array.Length, reader.AveragePageSizeBytes, page.Entries.Count());
////// reader.Recycle(page);
//////}
////////read all entries
//////foreach (var entry in reader.GetEntriesStartingAt(new Bookmark()/*, preallocatedPage: prealloc*/))//.Take(200));
//////{
////// if (!App.Active) break;
////// total++;
////// if (entry.ExceptionData!=null) wordCount++;
////// if ((total & 0xffff)==0) "Did {0:n0}".SeeArgs(total);
//////}
//reader.ParallelProcessRawEntryBatchesStartingAt(new Bookmark(), 50, (entries, loop, _) =>
//{
// var ec = 0;
// var wc = 0;
// foreach (var entry in entries)
// {
// if (!App.Active)
// {
// loop.Break();
// return;
// }
// if (entry.State == Entry.Status.Valid)
// {
// ec++;
// var msg = reader.Materialize(entry);
// try
// {
// // System.Threading.Thread.SpinWait(8_000);
// if (msg.Parameters.Length>0 && msg.Parameters.FirstOrDefault(c => !char.IsWhiteSpace(c))=='{')
////{
//// var map = msg.Parameters.JsonToDataObject() as JsonDataMap;
//// wc++;
////}
// }
// catch
// { }
// // if (msg.Guid.ToString().StartsWith("faca")) msg.See();
// }
// }
// if ((total & 0x0fff) == 0) "{0}".SeeArgs(total);
// System.Threading.Interlocked.Add(ref total, ec);//1);
// System.Threading.Interlocked.Add(ref wordCount, wc);//1);
//}, new ParallelOptions { MaxDegreeOfParallelism = 10 });
reader.ParallelProcessPageBatchesStartingAt(0, 5, (page, loop, _) =>
{
var ec = 0;
var wc = 0;
foreach (var entry in page.Entries)
{
if (!App.Active)
{
loop.Break();
return;
}
if (entry.State == Entry.Status.Valid)
{
ec++;
var msg = reader.Materialize(entry);
try
{
// System.Threading.Thread.SpinWait(8_000);
if (msg.Parameters.Length > 0 && msg.Parameters.FirstOrDefault(c => !char.IsWhiteSpace(c)) == '{')
{
var map = msg.Parameters.JsonToDataObject() as JsonDataMap;
wc++;
}
}
catch
{ }
// if (msg.Guid.ToString().StartsWith("faca")) msg.See();
}
}
if ((total & 0x0fff) == 0)
{
"{0}".SeeArgs(total);
}
System.Threading.Interlocked.Add(ref total, ec); //1);
System.Threading.Interlocked.Add(ref wordCount, wc); //1);
}, new ParallelOptions {
MaxDegreeOfParallelism = 10
});
time.Stop();
"Did {0:n0} found {1:n0}({2:n5}%) in {3:n1} sec at {4:n2} ops/sec WC = {5:n0}\n".SeeArgs(total, found, (double)found / total, time.ElapsedSec, total / time.ElapsedSec, wordCount);
volumeIdxId.Dispose();
volumeData.Dispose();
"CLOSED all volumes\n".See();
}
