public void Write_Read_Compare(string compress, string encrypt, int count, int pgsize)
{
var expected = FakeLogMessage.BuildRandomArr(count);
var ms = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("Log archive", LogMessageArchiveAppender.CONTENT_TYPE_LOG)
.SetVersion(1, 0)
.SetDescription("Testing log messages")
.SetChannel(Atom.Encode("dvop"))
.SetCompressionScheme(compress)
.SetEncryptionScheme(encrypt);
var volume = new DefaultVolume(CryptoMan, meta, ms)
{
PageSizeBytes = pgsize
};
using (var appender = new LogMessageArchiveAppender(volume,
NOPApplication.Instance.TimeSource,
NOPApplication.Instance.AppId, "dima@zhaba"))
{
for (var i = 0; i < count; i++)
{
appender.Append(expected[i]);
}
}
"The volume is {0:n0} bytes".SeeArgs(ms.Length);
var reader = new LogMessageArchiveReader(volume);
var got = reader.GetEntriesStartingAt(new Bookmark()).ToArray();
Aver.AreEqual(expected.Length, got.Length);
for (int i = 0; i < count; i++)
{
Aver.AreEqual(expected[i].App, got[i].App);
Aver.AreEqual(expected[i].ArchiveDimensions, got[i].ArchiveDimensions);
Aver.AreEqual(expected[i].Channel, got[i].Channel);
Aver.AreEqual(expected[i].Exception?.Message, got[i].Exception?.Message);
Aver.AreEqual(expected[i].From, got[i].From);
Aver.AreEqual(expected[i].Gdid, got[i].Gdid);
Aver.AreEqual(expected[i].Guid.ToString(), got[i].Guid.ToString());
Aver.AreEqual(expected[i].Host, got[i].Host);
Aver.AreEqual(expected[i].Parameters, got[i].Parameters);
Aver.AreEqual(expected[i].RelatedTo.ToString(), got[i].RelatedTo.ToString());
Aver.AreEqual(expected[i].Text, got[i].Text);
Aver.AreEqual(expected[i].Topic, got[i].Topic);
Aver.AreEqual((int)expected[i].Type, (int)got[i].Type);
Aver.AreEqual(expected[i].UTCTimeStamp, got[i].UTCTimeStamp);
}
volume.Dispose();
}
public void WriteRead(int cnt)
{
////var ms = new FileStream("c:\\azos\\archive.lar", FileMode.Create);// new MemoryStream();
var ms = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("Log archive", LogMessageArchiveAppender.CONTENT_TYPE_LOG)
.SetVersion(1, 0)
.SetDescription("Testing log messages")
.SetChannel(Atom.Encode("dvop"));
var volume = new DefaultVolume(CryptoMan, meta, ms);
using (var appender = new LogMessageArchiveAppender(volume,
NOPApplication.Instance.TimeSource,
NOPApplication.Instance.AppId, "dima@zhaba"))
{
for (var i = 0; i < cnt; i++)
{
var msg = new Message()
{
Text = "Message#" + i.ToString()
}.InitDefaultFields();
appender.Append(msg);
}
}
var reader = new LogMessageArchiveReader(volume);
Aver.AreEqual(cnt, reader.All.Count());
Aver.AreEqual(cnt, reader.GetEntriesStartingAt(new Bookmark()).Count());
Aver.AreEqual(cnt, reader.GetBookmarkedEntriesStartingAt(new Bookmark()).Count());
Aver.AreEqual(cnt, reader.GetEntriesAsObjectsStartingAt(new Bookmark()).Count());
Aver.AreEqual(cnt, reader.GetBookmarkedEntriesAsObjectsStartingAt(new Bookmark()).Count());
reader.All.ForEach((m, i) => Aver.AreEqual("Message#" + i.ToString(), m.Text));
reader.GetBookmarkedEntriesStartingAt(new Bookmark()).ForEach((t, i) =>
{
var pointed = reader.GetEntriesStartingAt(t.bm).First();
Aver.AreEqual("Message#" + i.ToString(), pointed.Text);
Aver.AreEqual(t.entry.Text, pointed.Text);
});
volume.Dispose();
}
[Run("!arch-perf-read", "compress=null encrypt=null search=$(~@term)")]// -r args='term=abcd'
//[Run("!arch-perf-read", "compress=gzip encrypt=aes1 search=$(~@term)")]// -r args='term=abcd'
public void Read_LogMessages(string compress, string encrypt, string search)
{
search = search.Default("ABBA");
var msData = new FileStream("c:\\azos\\logging-{0}-{1}.lar".Args(compress.Default("none"), encrypt.Default("none")), FileMode.Open);
var msIdxId = new FileStream("c:\\azos\\logging-{0}-{1}.guid.lix".Args(compress.Default("none"), encrypt.Default("none")), FileMode.Open);
var volumeData = new DefaultVolume(CryptoMan, msData);
var volumeIdxId = new DefaultVolume(CryptoMan, msIdxId);
//volumeData.PageSizeBytes = 4 * 1024 * 1024;//<-------------------!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//volumeIdxId.PageSizeBytes = 4 * 128 * 1024;
var reader = new LogMessageArchiveReader(volumeData);
var idxReader = new StringIdxReader(volumeIdxId);
var time = Azos.Time.Timeter.StartNew();
var total = 0;
var wordCount = 0;
var found = 0;
//foreach (var idx in idxReader.All)
//{
// total++;
// if (idx.Value.EqualsOrdIgnoreCase(search))
// {
// found++;
// var data = reader.Entries(idx.Bookmark).FirstOrDefault();
// data.See();
// }
//}
////////Rent buffers from arena instead? who is going to release pages? page.Recycle() to release the buffer to the pool? What if they forget to call it?
////////var prealloc = new Page(0); //SIGNIFICANT SLOW-DOWN WHILE ALLOCATING PAGES. WHY MEMORY STREAM DOES ARRAY.CLEAR?
//////foreach (var page in reader.GetPagesStartingAt(0/*, preallocatedPage: prealloc*/).Take(200))
//////{
////// total++;
////// //if (total % 100 ==0)
////// "Page {0:X8} is {1:n0} --- Average size: {2:n0} Entries per page: {3:n0}".SeeArgs(page.PageId, page.Data.Array.Length, reader.AveragePageSizeBytes, page.Entries.Count());
////// reader.Recycle(page);
//////}
////////read all entries
//////foreach (var entry in reader.GetEntriesStartingAt(new Bookmark()/*, preallocatedPage: prealloc*/))//.Take(200));
//////{
////// if (!App.Active) break;
////// total++;
////// if (entry.ExceptionData!=null) wordCount++;
////// if ((total & 0xffff)==0) "Did {0:n0}".SeeArgs(total);
//////}
//reader.ParallelProcessRawEntryBatchesStartingAt(new Bookmark(), 50, (entries, loop, _) =>
//{
// var ec = 0;
// var wc = 0;
// foreach (var entry in entries)
// {
// if (!App.Active)
// {
// loop.Break();
// return;
// }
// if (entry.State == Entry.Status.Valid)
// {
// ec++;
// var msg = reader.Materialize(entry);
// try
// {
// // System.Threading.Thread.SpinWait(8_000);
// if (msg.Parameters.Length>0 && msg.Parameters.FirstOrDefault(c => !char.IsWhiteSpace(c))=='{')
////{
//// var map = msg.Parameters.JsonToDataObject() as JsonDataMap;
//// wc++;
////}
// }
// catch
// { }
// // if (msg.Guid.ToString().StartsWith("faca")) msg.See();
// }
// }
// if ((total & 0x0fff) == 0) "{0}".SeeArgs(total);
// System.Threading.Interlocked.Add(ref total, ec);//1);
// System.Threading.Interlocked.Add(ref wordCount, wc);//1);
//}, new ParallelOptions { MaxDegreeOfParallelism = 10 });
reader.ParallelProcessPageBatchesStartingAt(0, 5, (page, loop, _) =>
{
var ec = 0;
var wc = 0;
foreach (var entry in page.Entries)
{
if (!App.Active)
{
loop.Break();
return;
}
if (entry.State == Entry.Status.Valid)
{
ec++;
var msg = reader.Materialize(entry);
try
{
// System.Threading.Thread.SpinWait(8_000);
if (msg.Parameters.Length > 0 && msg.Parameters.FirstOrDefault(c => !char.IsWhiteSpace(c)) == '{')
{
var map = msg.Parameters.JsonToDataObject() as JsonDataMap;
wc++;
}
}
catch
{ }
// if (msg.Guid.ToString().StartsWith("faca")) msg.See();
}
}
if ((total & 0x0fff) == 0)
{
"{0}".SeeArgs(total);
}
System.Threading.Interlocked.Add(ref total, ec); //1);
System.Threading.Interlocked.Add(ref wordCount, wc); //1);
}, new ParallelOptions {
MaxDegreeOfParallelism = 10
});
time.Stop();
"Did {0:n0} found {1:n0}({2:n5}%) in {3:n1} sec at {4:n2} ops/sec WC = {5:n0}\n".SeeArgs(total, found, (double)found / total, time.ElapsedSec, total / time.ElapsedSec, wordCount);
volumeIdxId.Dispose();
volumeData.Dispose();
"CLOSED all volumes\n".See();
}
