public void Metadata_AppSection()
{
var ms = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("V1", "none")
.SetApplicationSection(app => { app.AddAttributeNode("a", 1); })
.SetApplicationSection(app => { app.AddAttributeNode("b", -7); })
.SetApplicationSection(app => { app.AddChildNode("sub{ q=true b=-9 }".AsLaconicConfig()); });
var v1 = new DefaultVolume(CryptoMan, meta, ms);
var id = v1.Metadata.Id;
v1.Dispose();//it closes the stream
Aver.IsTrue(ms.Length > 0);
var v2 = new DefaultVolume(CryptoMan, ms);
v2.Metadata.See();
Aver.AreEqual(id, v2.Metadata.Id);
Aver.AreEqual("V1", v2.Metadata.Label);
Aver.IsTrue(v2.Metadata.SectionApplication.Exists);
Aver.AreEqual(1, v2.Metadata.SectionApplication.Of("a").ValueAsInt());
Aver.AreEqual(-7, v2.Metadata.SectionApplication.Of("b").ValueAsInt());
Aver.AreEqual(true, v2.Metadata.SectionApplication["sub"].Of("q").ValueAsBool());
Aver.AreEqual(-9, v2.Metadata.SectionApplication["sub"].Of("b").ValueAsInt());
}
public void Crypto_Bad_Keys(string compress)
{
var expected = Enumerable.Range(0, 10).Select(_ => new string (' ', 15)).ToArray();
var ms = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("String archive", StringArchiveAppender.CONTENT_TYPE_STRING)
.SetVersion(1, 0)
.SetDescription("Testing string messages")
.SetChannel(Atom.Encode("dvop"))
.SetCompressionScheme(compress)
.SetEncryptionScheme("aes1");
using (var volume = new DefaultVolume(CryptoMan, meta, ms))
using (var appender = new StringArchiveAppender(volume,
NOPApplication.Instance.TimeSource,
NOPApplication.Instance.AppId, "dima@zhaba"))
{
for (var i = 0; i < expected.Length; i++)
{
appender.Append(expected[i]);
}
}
using (var app = new AzosApplication(null, APP_BAD_CRYPTO_CONF.AsLaconicConfig()))
{
using (var volume = new DefaultVolume(app.SecurityManager.Cryptography, ms))//different key set
{
var page = new Page(0);
volume.ReadPage(0, page);//throws
}
}
}
public void Corrupt_Volume_Read(string compress, int pgsz, int cnt)
{
var ms = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("String archive", StringArchiveAppender.CONTENT_TYPE_STRING)
.SetVersion(1, 0)
.SetDescription("Testing string messages")
.SetChannel(Atom.Encode("dvop"))
.SetCompressionScheme(compress);
var volume = new DefaultVolume(NOPApplication.Instance.SecurityManager.Cryptography, meta, ms)
{
PageSizeBytes = pgsz
};
using (var appender = new StringArchiveAppender(volume, NOPApplication.Instance.TimeSource, new Atom(65), "*****@*****.**"))
{
for (var i = 0; i < cnt; i++)
{
appender.Append("string-data--------------------------------------------------------" + i.ToString());
}
}
"Volume size is {0:n0} bytes".SeeArgs(ms.Length);
var reader = new StringArchiveReader(volume);
var allCount = reader.All.Count();
Aver.AreEqual(cnt, allCount);
reader.All.ForEach((s, i) => Aver.IsTrue(s.EndsWith("---" + i.ToString())));
var pageCount = reader.GetPagesStartingAt(0).Count();
Aver.IsTrue(pageCount > 0);
"Before corruption: there are {0:n0} total records in {1:n0} pages".SeeArgs(allCount, pageCount);
var midPoint = ms.Length / 2;
ms.Position = midPoint;
for (var j = 0; j < pgsz * 2; j++)
{
ms.WriteByte(0x00); //corruption
}
"-------------- corrupted {0:n0} bytes of {1:n0} total at {2:n0} position ----------- ".SeeArgs(pgsz * 2, ms.Length, midPoint);
var allCount2 = reader.GetEntriesStartingAt(new Bookmark(), skipCorruptPages: true).Count();
Aver.IsTrue(allCount > allCount2);
var pageCount2 = reader.GetPagesStartingAt(0, skipCorruptPages: true).Count();
Aver.IsTrue(pageCount > pageCount2);
"After corruption: there are {0:n0} total records in {1:n0} pages".SeeArgs(allCount2, pageCount2);
volume.Dispose();
}
private VolumeMetadataBuilder getSpecificBuilder(string contentType, string compressionScheme, string encryptionScheme)
{
return(VolumeMetadataBuilder.Make("Primitive Idx Meta", contentType)
.SetVersion(1, 1)
.SetDescription("MumboJumbo testing")
.SetCompressionScheme(compressionScheme)
.SetEncryptionScheme(encryptionScheme));
}
//[Run("!arch-perf-write", "compress=gzip encrypt=aes1 cnt=16000000 para=16")]
public void Write_LogMessages(string compress, string encrypt, int CNT, int PARA)
{
var msData = new FileStream("c:\\azos\\logging-{0}-{1}.lar".Args(compress.Default("none"), encrypt.Default("none")), FileMode.Create);
var msIdxId = new FileStream("c:\\azos\\logging-{0}-{1}.guid.lix".Args(compress.Default("none"), encrypt.Default("none")), FileMode.Create);
var volumeData = new DefaultVolume(CryptoMan, VolumeMetadataBuilder.Make("Perf", LogMessageArchiveAppender.CONTENT_TYPE_LOG)
.SetVersion(1, 1)
.SetDescription("Perf testing")
.SetCompressionScheme(compress)
.SetEncryptionScheme(encrypt),
msData);
var volumeIdxId = new DefaultVolume(CryptoMan, VolumeMetadataBuilder.Make("Perf", StringIdxAppender.CONTENT_TYPE_IDX_STRING)
.SetVersion(1, 1)
.SetDescription("Perf testing")
.SetCompressionScheme(compress)
.SetEncryptionScheme(encrypt),
msIdxId);
volumeData.PageSizeBytes = 1024 * 1024;
volumeIdxId.PageSizeBytes = 128 * 1024;
var time = Azos.Time.Timeter.StartNew();
Parallel.For(0, PARA, _ => {
var app = Azos.Apps.ExecutionContext.Application;
using (var aIdxId = new StringIdxAppender(volumeIdxId,
NOPApplication.Instance.TimeSource,
NOPApplication.Instance.AppId, "dima@zhaba"))
{
using (var appender = new LogMessageArchiveAppender(volumeData,
NOPApplication.Instance.TimeSource,
NOPApplication.Instance.AppId,
"dima@zhaba",
onPageCommit: (e, b) => aIdxId.Append(new StringBookmark(e.Guid.ToString().TakeLastChars(4), b))))
{
for (var i = 0; app.Active && i < CNT / PARA; i++)
{
var msg = FakeLogMessage.BuildRandom();
appender.Append(msg);
}
}
}
});
time.Stop();
"Did {0:n0} in {1:n1} sec at {2:n2} ops/sec\n".SeeArgs(CNT, time.ElapsedSec, CNT / time.ElapsedSec);
volumeIdxId.Dispose();
volumeData.Dispose();
"CLOSED all volumes\n".See();
}
public void Write_Read_Compare(string compress, string encrypt, int count, int pgsize)
{
var expected = FakeLogMessage.BuildRandomArr(count);
var ms = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("Log archive", LogMessageArchiveAppender.CONTENT_TYPE_LOG)
.SetVersion(1, 0)
.SetDescription("Testing log messages")
.SetChannel(Atom.Encode("dvop"))
.SetCompressionScheme(compress)
.SetEncryptionScheme(encrypt);
var volume = new DefaultVolume(CryptoMan, meta, ms)
{
PageSizeBytes = pgsize
};
using (var appender = new LogMessageArchiveAppender(volume,
NOPApplication.Instance.TimeSource,
NOPApplication.Instance.AppId, "dima@zhaba"))
{
for (var i = 0; i < count; i++)
{
appender.Append(expected[i]);
}
}
"The volume is {0:n0} bytes".SeeArgs(ms.Length);
var reader = new LogMessageArchiveReader(volume);
var got = reader.GetEntriesStartingAt(new Bookmark()).ToArray();
Aver.AreEqual(expected.Length, got.Length);
for (int i = 0; i < count; i++)
{
Aver.AreEqual(expected[i].App, got[i].App);
Aver.AreEqual(expected[i].ArchiveDimensions, got[i].ArchiveDimensions);
Aver.AreEqual(expected[i].Channel, got[i].Channel);
Aver.AreEqual(expected[i].Exception?.Message, got[i].Exception?.Message);
Aver.AreEqual(expected[i].From, got[i].From);
Aver.AreEqual(expected[i].Gdid, got[i].Gdid);
Aver.AreEqual(expected[i].Guid.ToString(), got[i].Guid.ToString());
Aver.AreEqual(expected[i].Host, got[i].Host);
Aver.AreEqual(expected[i].Parameters, got[i].Parameters);
Aver.AreEqual(expected[i].RelatedTo.ToString(), got[i].RelatedTo.ToString());
Aver.AreEqual(expected[i].Text, got[i].Text);
Aver.AreEqual(expected[i].Topic, got[i].Topic);
Aver.AreEqual((int)expected[i].Type, (int)got[i].Type);
Aver.AreEqual(expected[i].UTCTimeStamp, got[i].UTCTimeStamp);
}
volume.Dispose();
}
public void Write_Read_Compare_PageSplit(int pc, int vsz, int pgsize, string compress, string encrypt, int count, int sz)
{
var expected = Enumerable.Range(0, count).Select(_ => new string(' ', sz)).ToArray();
var ms = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("String archive", StringArchiveAppender.CONTENT_TYPE_STRING)
.SetVersion(1, 0)
.SetDescription("Testing string messages")
.SetChannel(Atom.Encode("dvop"))
.SetCompressionScheme(compress)
.SetEncryptionScheme(encrypt);
var volume = new DefaultVolume(CryptoMan, meta, ms);
volume.PageSizeBytes = pgsize;
using (var appender = new StringArchiveAppender(volume,
NOPApplication.Instance.TimeSource,
NOPApplication.Instance.AppId, "dima@zhaba"))
{
for (var i = 0; i < count; i++)
{
appender.Append(expected[i]);
}
}
"Volume data stream is {0:n0} bytes".SeeArgs(ms.Length);
Aver.IsTrue(ms.Length < vsz);
// ms.GetBuffer().ToHexDump().See();
var reader = new StringArchiveReader(volume);
// reader.Pages(0).Select(p => (p.State, p.Entries.Count())).See("PAGES/Entries: ");
var pageCount = reader.GetPagesStartingAt(0).Count();
"Volume page count is: {0}".SeeArgs(pageCount);
Aver.AreEqual(pc, pageCount);
var got = reader.All.ToArray();
Aver.AreEqual(expected.Length, got.Length);
for (int i = 0; i < count; i++)
{
Aver.AreEqual(expected[i], got[i]);
}
volume.Dispose();
}
protected virtual VolumeMetadataBuilder DoBuildMetadata()
=> VolumeMetadataBuilder.Make(Name, LogMessageArchiveAppender.CONTENT_TYPE_LOG)
.SetVersion(ArchiveVersionMajor, ArchiveVersionMinor)
.SetChannel(ArchiveChannel)
.SetCompressionScheme(ArchiveCompressionScheme)
.SetEncryptionScheme(ArchiveEncryptionScheme)
.SetDescription(ArchiveDescription)
.SetApplicationSection(anode =>
{
var app = ArchiveApplicationSection;
if (app != null)
{
anode.MergeAttributes(app);
anode.MergeSections(app);
}
});
public void WriteRead(int cnt)
{
////var ms = new FileStream("c:\\azos\\archive.lar", FileMode.Create);// new MemoryStream();
var ms = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("Log archive", LogMessageArchiveAppender.CONTENT_TYPE_LOG)
.SetVersion(1, 0)
.SetDescription("Testing log messages")
.SetChannel(Atom.Encode("dvop"));
var volume = new DefaultVolume(CryptoMan, meta, ms);
using (var appender = new LogMessageArchiveAppender(volume,
NOPApplication.Instance.TimeSource,
NOPApplication.Instance.AppId, "dima@zhaba"))
{
for (var i = 0; i < cnt; i++)
{
var msg = new Message()
{
Text = "Message#" + i.ToString()
}.InitDefaultFields();
appender.Append(msg);
}
}
var reader = new LogMessageArchiveReader(volume);
Aver.AreEqual(cnt, reader.All.Count());
Aver.AreEqual(cnt, reader.GetEntriesStartingAt(new Bookmark()).Count());
Aver.AreEqual(cnt, reader.GetBookmarkedEntriesStartingAt(new Bookmark()).Count());
Aver.AreEqual(cnt, reader.GetEntriesAsObjectsStartingAt(new Bookmark()).Count());
Aver.AreEqual(cnt, reader.GetBookmarkedEntriesAsObjectsStartingAt(new Bookmark()).Count());
reader.All.ForEach((m, i) => Aver.AreEqual("Message#" + i.ToString(), m.Text));
reader.GetBookmarkedEntriesStartingAt(new Bookmark()).ForEach((t, i) =>
{
var pointed = reader.GetEntriesStartingAt(t.bm).First();
Aver.AreEqual("Message#" + i.ToString(), pointed.Text);
Aver.AreEqual(t.entry.Text, pointed.Text);
});
volume.Dispose();
}
public void Write_Read_Compare_Strings(bool ascii, int pgsize, string compress, int count, int min, int max)
{
var expected = ascii ? FakeLogMessage.BuildRandomASCIIStringArr(count, min, max) : FakeLogMessage.BuildRandomUnicodeStringArr(count, min, max);
var ms = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("String archive", StringArchiveAppender.CONTENT_TYPE_STRING)
.SetVersion(1, 0)
.SetDescription("Testing string messages")
.SetChannel(Atom.Encode("dvop"));
if (compress.IsNotNullOrWhiteSpace())
{
meta.SetCompressionScheme(compress);
}
var volume = new DefaultVolume(NOPApplication.Instance.SecurityManager.Cryptography, meta, ms);
volume.PageSizeBytes = pgsize;
using (var appender = new StringArchiveAppender(volume,
NOPApplication.Instance.TimeSource,
NOPApplication.Instance.AppId, "dima@zhaba"))
{
for (var i = 0; i < count; i++)
{
appender.Append(expected[i]);
}
}
"Volume data stream is {0:n0} bytes".SeeArgs(ms.Length);
var reader = new StringArchiveReader(volume);
var got = reader.GetEntriesStartingAt(new Bookmark()).ToArray();
Aver.AreEqual(expected.Length, got.Length);
for (int i = 0; i < count; i++)
{
Aver.AreEqual(expected[i], got[i]);
}
volume.Dispose();
}
public void Metadata_Basic()
{
var ms = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("Volume-1", "none")
.SetVersion(123, 456)
.SetDescription("My volume");
var v1 = new DefaultVolume(CryptoMan, meta, ms);
var id = v1.Metadata.Id;
v1.Dispose();//it closes the stream
Aver.IsTrue(ms.Length > 0);
var v2 = new DefaultVolume(CryptoMan, ms);
Aver.AreEqual(id, v2.Metadata.Id);
Aver.AreEqual("Volume-1", v2.Metadata.Label);
Aver.AreEqual("My volume", v2.Metadata.Description);
Aver.AreEqual(123, v2.Metadata.VersionMajor);
Aver.AreEqual(456, v2.Metadata.VersionMinor);
Aver.IsTrue(v2.Metadata.Channel.IsZero);
Aver.IsFalse(v2.Metadata.IsCompressed);
Aver.IsFalse(v2.Metadata.IsEncrypted);
}
public void Write_Read_Index_Primitives(bool usecache, string compress, string encrypt, int CNT, string idxCompress, string idxEncrypt, int pageLboundKb, int pageUboundKb, int idxLboundKb, int idxUboundKb)
{
var ctlMumbo = MumboJumbo.GetControl();
IPileImplementation pile = null;
ICacheImplementation cache = null;
IPageCache pageCache = null;
if (usecache)
{
pile = new DefaultPile(App);
cache = new LocalCache(App)
{
Pile = pile,
DefaultTableOptions = new TableOptions("*")
{
CollisionMode = CollisionMode.Durable
},
PileMaxMemoryLimit = 32L * 1024 * 1024 * 1024
};
pile.Start();
cache.Start();
pageCache = new PilePageCache(cache);
}
try
{
var msData = new MemoryStream();
var msIdxId = new MemoryStream();
var msIdxCid = new MemoryStream();
var msIdxDid = new MemoryStream();
var msIdxPn = new MemoryStream();
var msIdxLt = new MemoryStream();
var msIdxLn = new MemoryStream();
var msIdxAl = new MemoryStream();
var msIdxCd = new MemoryStream();
var msIdxNt = new MemoryStream();
var msIdxAmt = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("Primitive Idx", "mjumbo")
.SetVersion(1, 1)
.SetDescription("MumboJumbo testing")
.SetCompressionScheme(compress)
.SetEncryptionScheme(encrypt);
var volumeData = new DefaultVolume(CryptoMan, pageCache, meta, msData);
var IdxIdMetaBuilder = getSpecificBuilder(GdidIdxAppender.CONTENT_TYPE_IDX_GDID, idxCompress, idxEncrypt);
var IdxCidMetaBuilder = getSpecificBuilder(GuidIdxAppender.CONTENT_TYPE_IDX_GUID, idxCompress, idxEncrypt);
var IdxDidMetaBuilder = getSpecificBuilder(LongIdxAppender.CONTENT_TYPE_IDX_LONG, idxCompress, idxEncrypt);
var IdxPnMetaBuilder = getSpecificBuilder(IntIdxAppender.CONTENT_TYPE_IDX_INT, idxCompress, idxEncrypt);
var IdxLtMetaBuilder = getSpecificBuilder(DoubleIdxAppender.CONTENT_TYPE_IDX_DOUBLE, idxCompress, idxEncrypt);
var IdxLnMetaBuilder = getSpecificBuilder(DoubleIdxAppender.CONTENT_TYPE_IDX_DOUBLE, idxCompress, idxEncrypt);
var IdxAlMetaBuilder = getSpecificBuilder(DecimalIdxAppender.CONTENT_TYPE_IDX_DECIMAL, idxCompress, idxEncrypt);
var IdxCdMetaBuilder = getSpecificBuilder(DateTimeIdxAppender.CONTENT_TYPE_IDX_DATETIME, idxCompress, idxEncrypt);
var IdxNtMetaBuilder = getSpecificBuilder(StringIdxAppender.CONTENT_TYPE_IDX_STRING, idxCompress, idxEncrypt);
var IdxAmtMetaBuilder = getSpecificBuilder(AmountIdxAppender.CONTENT_TYPE_IDX_AMOUNT, idxCompress, idxEncrypt);
var volumeIdxId = new DefaultVolume(CryptoMan, IdxIdMetaBuilder, msIdxId);
var volumeIdxCid = new DefaultVolume(CryptoMan, IdxCidMetaBuilder, msIdxCid);
var volumeIdxDid = new DefaultVolume(CryptoMan, IdxDidMetaBuilder, msIdxDid);
var volumeIdxPn = new DefaultVolume(CryptoMan, IdxPnMetaBuilder, msIdxPn);
var volumeIdxLt = new DefaultVolume(CryptoMan, IdxLtMetaBuilder, msIdxLt);
var volumeIdxLn = new DefaultVolume(CryptoMan, IdxLnMetaBuilder, msIdxLn);
var volumeIdxAl = new DefaultVolume(CryptoMan, IdxAlMetaBuilder, msIdxAl);
var volumeIdxCd = new DefaultVolume(CryptoMan, IdxCdMetaBuilder, msIdxCd);
var volumeIdxNt = new DefaultVolume(CryptoMan, IdxNtMetaBuilder, msIdxNt);
var volumeIdxAmt = new DefaultVolume(CryptoMan, IdxAmtMetaBuilder, msIdxAmt);
volumeData.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(pageLboundKb, pageUboundKb) * 1024;
volumeIdxId.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxCid.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxDid.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxPn.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxLt.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxLn.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxAl.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxCd.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxNt.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxAmt.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
var reader = new MumboJumboArchiveReader(volumeData);
var idxIdReader = new GdidIdxReader(volumeIdxId);
var idxCidReader = new GuidIdxReader(volumeIdxCid);
var idxDidReader = new LongIdxReader(volumeIdxDid);
var idxPnReader = new IntIdxReader(volumeIdxPn);
var idxLtReader = new DoubleIdxReader(volumeIdxLt);
var idxLnReader = new DoubleIdxReader(volumeIdxLn);
var idxAlReader = new DecimalIdxReader(volumeIdxAl);
var idxCdReader = new DateTimeIdxReader(volumeIdxCd);
var idxNtReader = new StringIdxReader(volumeIdxNt);
var idxAmtReader = new AmountIdxReader(volumeIdxAmt);
var time = Azos.Time.Timeter.StartNew();
var app = Azos.Apps.ExecutionContext.Application;
using (var aIdxId = new GdidIdxAppender(volumeIdxId, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxCid = new GuidIdxAppender(volumeIdxCid, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxDid = new LongIdxAppender(volumeIdxDid, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxPn = new IntIdxAppender(volumeIdxPn, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxLt = new DoubleIdxAppender(volumeIdxLt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxLn = new DoubleIdxAppender(volumeIdxLn, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxAl = new DecimalIdxAppender(volumeIdxAl, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxCd = new DateTimeIdxAppender(volumeIdxCd, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxNt = new StringIdxAppender(volumeIdxNt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxAmt = new AmountIdxAppender(volumeIdxAmt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
{
using (var appender = new MumboJumboArchiveAppender(volumeData,
NOPApplication.Instance.TimeSource,
NOPApplication.Instance.AppId,
"dima@zhaba",
onPageCommit: (e, b) =>
{
aIdxId.Append(new GdidBookmark(e.ID, b));
aIdxCid.Append(new GuidBookmark(e.CorrelationId, b));
aIdxDid.Append(new LongBookmark(e.DeviceId, b));
aIdxPn.Append(new IntBookmark(e.PartNumber, b));
aIdxLt.Append(new DoubleBookmark(e.Latitude, b));
aIdxLn.Append(new DoubleBookmark(e.Longitude, b));
aIdxAl.Append(new DecimalBookmark(e.Altitude, b));
aIdxCd.Append(new DateTimeBookmark(e.CreateDate, b));
aIdxNt.Append(new StringBookmark(e.Note, b));
aIdxAmt.Append(new AmountBookmark(e.Amt, b));
}))
{
var messages = FakeRow.GenerateMany <MumboJumbo>(1, 1, (ulong)(CNT * .5) - 1);
foreach (var m in messages)
{
if (!App.Active)
{
break;
}
appender.Append(m);
}
///////////////////////////////////////////////
appender.Append(MumboJumbo.GetControl()); //we add 1 control message in the middle
///////////////////////////////////////////////
messages = FakeRow.GenerateMany <MumboJumbo>(1, 1, (ulong)(CNT * .5));
foreach (var m in messages)
{
if (!App.Active)
{
break;
}
appender.Append(m);
}
}
}
"Wrote {0} items, now reading".SeeArgs(CNT);
if (cache != null)
{
"Cache has {0} items".SeeArgs(cache.Count);
}
var gotOne = false;
// Find by GDID (ID)
foreach (var idx in idxIdReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.ID, idx.Value);
if (idx.Value == ctlMumbo.ID)
{
//data.See();
Aver.AreEqual(ctlMumbo.ID, data.ID);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find ID by {nameof(GdidIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(GdidIdxReader));
gotOne = false;
// Find by Guid (CorrelationId)
foreach (var idx in idxCidReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.CorrelationId, idx.Value);
if (idx.Value == ctlMumbo.CorrelationId)
{
//data.See();
Aver.AreEqual(ctlMumbo.CorrelationId, data.CorrelationId);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find CorrelationId by {nameof(GuidIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(GuidIdxReader));
gotOne = false;
// Find by Long (DeviceId)
foreach (var idx in idxDidReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.DeviceId, idx.Value);
if (idx.Value == ctlMumbo.DeviceId)
{
//data.See();
Aver.AreEqual(ctlMumbo.DeviceId, data.DeviceId);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find DeviceId by {nameof(LongIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(LongIdxReader));
gotOne = false;
// Find by Int (PartNumber)
foreach (var idx in idxPnReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.PartNumber, idx.Value);
if (idx.Value == ctlMumbo.PartNumber)
{
//data.See();
Aver.AreEqual(ctlMumbo.PartNumber, data.PartNumber);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find PartNumber by {nameof(IntIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(IntIdxReader));
gotOne = false;
// Find by Double (Latitude)
foreach (var idx in idxLtReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.Latitude, idx.Value);
if (idx.Value == ctlMumbo.Latitude)
{
//data.See();
Aver.AreEqual(ctlMumbo.Latitude, data.Latitude);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find Latitude by {nameof(DoubleIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(DoubleIdxReader));
gotOne = false;
// Find by Double (Longitude)
foreach (var idx in idxLnReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.Longitude, idx.Value);
if (idx.Value == ctlMumbo.Longitude)
{
//data.See();
Aver.AreEqual(ctlMumbo.Longitude, data.Longitude);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find Longitude by {nameof(DoubleIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(DoubleIdxReader));
gotOne = false;
// Find by Decimal (Altitude)
foreach (var idx in idxAlReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.Altitude, idx.Value);
if (idx.Value == ctlMumbo.Altitude)
{
//data.See();
Aver.AreEqual(ctlMumbo.Altitude, data.Altitude);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find Altitude by {nameof(DecimalIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(DecimalIdxReader));
gotOne = false;
// Find by DateTime (CreateDate)
foreach (var idx in idxCdReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.CreateDate, idx.Value);
if (idx.Value == ctlMumbo.CreateDate)
{
//data.See();
Aver.AreEqual(ctlMumbo.CreateDate, data.CreateDate);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find CreateDate by {nameof(DateTimeIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(DateTimeIdxReader));
gotOne = false;
// Find by String (Note)
foreach (var idx in idxNtReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.Note, idx.Value);
if (idx.Value == ctlMumbo.Note)
{
//data.See();
Aver.AreEqual(ctlMumbo.Note, data.Note);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find Note by {nameof(StringIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(StringIdxReader));
gotOne = false;
// Find by Amount (Amt)
foreach (var idx in idxAmtReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.Amt, idx.Value);
if (idx.Value == ctlMumbo.Amt)
{
//data.See();
Aver.AreEqual(ctlMumbo.Amt, data.Amt);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find Amt by {nameof(AmountIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(AmountIdxReader));
//gotOne = false;
time.Stop();
"Did {0:n0} in {1:n1} sec at {2:n2} ops/sec\n".SeeArgs(CNT, time.ElapsedSec, CNT / time.ElapsedSec);
volumeIdxId.Dispose();
volumeIdxCid.Dispose();
volumeIdxDid.Dispose();
volumeIdxPn.Dispose();
volumeIdxLt.Dispose();
volumeIdxLn.Dispose();
volumeIdxAl.Dispose();
volumeIdxCd.Dispose();
volumeIdxNt.Dispose();
volumeIdxAmt.Dispose();
volumeData.Dispose();
}
finally
{
DisposableObject.DisposeIfDisposableAndNull(ref pageCache);
DisposableObject.DisposeIfDisposableAndNull(ref cache);
DisposableObject.DisposeIfDisposableAndNull(ref pile);
"CLOSED all volumes\n".See();
}
}
public void PageInfo_Walk(int count, int psz, bool remount, int strvar)
{
string makestr(int i)
{
return(strvar > 0 ? (new string(' ', i % strvar) + "text-string-" + i.ToString()) : ("text-string-" + i.ToString()));
}
var ms = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("String archive", StringArchiveAppender.CONTENT_TYPE_STRING)
.SetVersion(1, 0)
.SetDescription("Testing string messages")
.SetChannel(Atom.Encode("dvop"));
var volume = new DefaultVolume(CryptoMan, meta, ms, ownsStream: false);
volume.PageSizeBytes = psz;
using (var appender = new StringArchiveAppender(volume,
NOPApplication.Instance.TimeSource,
NOPApplication.Instance.AppId, "dima@zhaba"))
{
for (var i = 0; i < count; i++)
{
appender.Append(makestr(i));
}
}
if (remount)
{
volume.Dispose();
volume = new DefaultVolume(CryptoMan, ms, ownsStream: false);
}
var pis = volume.ReadPageInfos(0);
"Page infos: {0}".SeeArgs(pis.Count());
foreach (var pi in pis)
{
var one = volume.ReadPageInfo(pi.PageId);
Aver.AreEqual(pi, one);
}
var total = 0;
var reader = new StringArchiveReader(volume);
foreach (var pi in pis)
{
var page = reader.GetOnePageAt(pi.PageId, true);
foreach (var entry in page.Entries)
{
if (entry.State == Entry.Status.Valid)
{
Aver.AreEqual(makestr(total), reader.Materialize(entry));
total++;
}
}
}
Aver.AreEqual(count, total);
volume.Dispose();
}
public void Page_Write_CorruptPage_Read(int count)
{
var ms = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("Volume-1", "raw");
var v1 = new DefaultVolume(CryptoMan, meta, ms);
var page = new Page(0);
Aver.IsTrue(page.State == Page.Status.Unset);
page.BeginWriting(new DateTime(1980, 7, 1, 15, 0, 0, DateTimeKind.Utc), Atom.Encode("app"), "*****@*****.**");
var data = new Dictionary <int, byte[]>();
for (var i = 0; i < count; i++)
{
//generate 1/2 empty arrays for best compression, another 1/2/ filled with random data
var buf = ((i & 1) == 0) ? new byte[1 + (i & 0x7f)] : Platform.RandomGenerator.Instance.NextRandomBytes(1 + (i & 0x7f));
var adr = page.Append(new ArraySegment <byte>(buf));
data[adr] = buf;
}
Aver.AreEqual(data.Count, data.Keys.Distinct().Count());//all addresses are unique
page.EndWriting();
var pid = v1.AppendPage(page); //append to volume
page = new Page(0); //we could have reused the existing page but we re-allocate for experiment cleanness
v1.ReadPage(pid, page);
//Aver that all are readable
foreach (var kvp in data)
{
var got = page[kvp.Key];
Aver.IsTrue(got.State == Entry.Status.Valid);
Aver.IsTrue(IOUtils.MemBufferEquals(kvp.Value, got.Raw.ToArray()));
}
//now corrupt First
var cadr = data.First().Key;
page.Data.Array[cadr] = 0xff; //corrupt underlying page memory
data[cadr] = null; //corrupted
//corrupt last
cadr = data.Last().Key;
page.Data.Array[cadr] = 0x00; //corrupt underlying page memory
data[cadr] = null; //corrupted
var keys = data.Keys.ToArray();
//corrupt a half of written
for (var i = 0; i < data.Count / 2; i++)
{
cadr = keys[Platform.RandomGenerator.Instance.NextScaledRandomInteger(2, data.Count - 2)];
page.Data.Array[cadr] = 0xff; //corrupt underlying page memory
data[cadr] = null; //corrupted
}
"\nStream size is: {0:n0} bytes".SeeArgs(ms.Length);
"{0:n0} total entries, {1:n0} are corrupt \n".SeeArgs(data.Count, data.Where(kvp => kvp.Value == null).Count());
//Aver that all which are SET are still readable, others are corrupt
foreach (var kvp in data)
{
var got = page[kvp.Key];
if (kvp.Value != null)//was not corrupted
{
Aver.IsTrue(got.State == Entry.Status.Valid);
Aver.IsTrue(IOUtils.MemBufferEquals(kvp.Value, got.Raw.ToArray()));
}
else
{
Aver.IsTrue(got.State == Entry.Status.BadHeader);
}
}
}
public void Page_Write_Read(string compress, string encrypt, int pad, bool remount)
{
var ms = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("Volume-1", "raw")
.SetCompressionScheme(compress)
.SetEncryptionScheme(encrypt);
var v1 = new DefaultVolume(CryptoMan, meta, ms);
var page = new Page(0);
Aver.IsTrue(page.State == Page.Status.Unset);
page.BeginWriting(new DateTime(1980, 7, 1, 15, 0, 0, DateTimeKind.Utc), Atom.Encode("app"), "*****@*****.**");
Aver.IsTrue(page.State == Page.Status.Writing);
var adr1 = page.Append(new ArraySegment <byte>(new byte[] { 1, 2, 3 }, 0, 3));
Aver.AreEqual(0, adr1);
var adr2 = page.Append(new ArraySegment <byte>(new byte[] { 4, 5 }, 0, 2));
Aver.IsTrue(adr2 > 0);
var adr3 = page.Append(new ArraySegment <byte>(new byte[pad]));
Aver.IsTrue(adr3 > adr2);
page.EndWriting();
Aver.IsTrue(page.State == Page.Status.Written);
var pid = v1.AppendPage(page); //append to volume
Aver.IsTrue(page.State == Page.Status.Written);
"Written volume {0} Stream size is {1} bytes".SeeArgs(v1.Metadata.Id, ms.Length);
if (remount)
{
v1.Dispose();
v1 = new DefaultVolume(CryptoMan, ms);//re-mount existing data from stream
"Re-mounted volume {0}".SeeArgs(v1.Metadata.Id);
}
page = new Page(0);//we could have reused the existing page but we re-allocate for experiment cleanness
Aver.IsTrue(page.State == Page.Status.Unset);
v1.ReadPage(pid, page);
Aver.IsTrue(page.State == Page.Status.Reading);
//page header read correctly
Aver.AreEqual(new DateTime(1980, 7, 1, 15, 0, 0, DateTimeKind.Utc), page.CreateUtc);
Aver.AreEqual(Atom.Encode("app"), page.CreateApp);
Aver.AreEqual("*****@*****.**", page.CreateHost);
var raw = page.Entries.ToArray();//all entry enumeration test
Aver.AreEqual(4, raw.Length);
Aver.IsTrue(raw[0].State == Entry.Status.Valid);
Aver.IsTrue(raw[1].State == Entry.Status.Valid);
Aver.IsTrue(raw[2].State == Entry.Status.Valid);
Aver.IsTrue(raw[3].State == Entry.Status.EOF);
Aver.AreEqual(0, raw[0].Address);
Aver.IsTrue(raw[1].Address > 0);
Aver.AreEqual(3, raw[0].Raw.Count);
Aver.AreEqual(1, raw[0].Raw.Array[raw[0].Raw.Offset + 0]);
Aver.AreEqual(2, raw[0].Raw.Array[raw[0].Raw.Offset + 1]);
Aver.AreEqual(3, raw[0].Raw.Array[raw[0].Raw.Offset + 2]);
Aver.AreEqual(2, raw[1].Raw.Count);
Aver.AreEqual(4, raw[1].Raw.Array[raw[1].Raw.Offset + 0]);
Aver.AreEqual(5, raw[1].Raw.Array[raw[1].Raw.Offset + 1]);
Aver.AreEqual(pad, raw[2].Raw.Count);
var one = page[adr1]; //indexer test
Aver.IsTrue(one.State == Entry.Status.Valid);
Aver.AreEqual(3, one.Raw.Count);
Aver.AreEqual(1, one.Raw.Array[one.Raw.Offset + 0]);
Aver.AreEqual(2, one.Raw.Array[one.Raw.Offset + 1]);
Aver.AreEqual(3, one.Raw.Array[one.Raw.Offset + 2]);
one = page[adr2];
Aver.IsTrue(one.State == Entry.Status.Valid);
Aver.AreEqual(2, one.Raw.Count);
Aver.AreEqual(4, one.Raw.Array[one.Raw.Offset + 0]);
Aver.AreEqual(5, one.Raw.Array[one.Raw.Offset + 1]);
one = page[adr3];
Aver.IsTrue(one.State == Entry.Status.Valid);
Aver.AreEqual(pad, one.Raw.Count);
}
public void Write_Read_Amount_Indexing_MemoryStream(string compress, string encrypt, int CNT, string idxCompress, string idxEncrypt)
{
var msData = new MemoryStream();
var msIdxAmt = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("Primitive Idx", "mjumbo")
.SetVersion(1, 1)
.SetDescription("MumboJumbo testing")
.SetCompressionScheme(compress)
.SetEncryptionScheme(encrypt);
var volumeData = new DefaultVolume(CryptoMan, meta, msData);
var metaIdx = VolumeMetadataBuilder.Make("Primitive Idx Meta", AmountIdxAppender.CONTENT_TYPE_IDX_AMOUNT)
.SetVersion(1, 1)
.SetDescription("MumboJumbo testing")
.SetCompressionScheme(idxCompress)
.SetEncryptionScheme(idxEncrypt);
var volumeIdxAmt = new DefaultVolume(CryptoMan, metaIdx, msIdxAmt);
volumeData.PageSizeBytes = 1024 * 1024;
volumeIdxAmt.PageSizeBytes = 128 * 1024;
var timeWrite = Azos.Time.Timeter.StartNew();
var app = Azos.Apps.ExecutionContext.Application;
using (var aIdxAmt = new AmountIdxAppender(volumeIdxAmt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
{
using (var appender = new MumboJumboArchiveAppender(volumeData,
NOPApplication.Instance.TimeSource,
NOPApplication.Instance.AppId,
"dima@zhaba",
onPageCommit: (e, b) =>
{
aIdxAmt.Append(new AmountBookmark(e.Amt, b));
}
))
{
var messages = FakeRow.GenerateMany <MumboJumbo>(1, 1, (ulong)(CNT * .5) - 1);
foreach (var m in messages)
{
appender.Append(m);
}
appender.Append(MumboJumbo.GetControl());
messages = FakeRow.GenerateMany <MumboJumbo>(1, 1, (ulong)(CNT * .5));
foreach (var m in messages)
{
appender.Append(m);
}
}
}
timeWrite.Stop();
"Did {0:n0} writes in {1:n1} sec at {2:n2} ops/sec\n".SeeArgs(CNT, timeWrite.ElapsedSec, CNT / timeWrite.ElapsedSec);
var ctlMumbo = MumboJumbo.GetControl();
var reader = new MumboJumboArchiveReader(volumeData);
var idxAmtReader = new AmountIdxReader(volumeIdxAmt);
var timeRead = Azos.Time.Timeter.StartNew();
var gotOne = false;
// Find by Amount (Amt)
foreach (var idx in idxAmtReader.All)
{
if (idx.Value == ctlMumbo.Amt)
{
var data = reader.GetEntriesStartingAt(idx.Bookmark).FirstOrDefault();
data.See();
Aver.AreEqual(ctlMumbo.Amt, data.Amt);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find Amt by {nameof(AmountIdxReader)}");
}
timeRead.Stop();
"Did {0:n0} reads in {1:n1} sec at {2:n2} ops/sec\n".SeeArgs(CNT, timeRead.ElapsedSec, CNT / timeRead.ElapsedSec);
volumeIdxAmt.Dispose();
volumeData.Dispose();
"CLOSED all volumes\n".See();
}