public void Crypto_Bad_Keys(string compress) { var expected = Enumerable.Range(0, 10).Select(_ => new string (' ', 15)).ToArray(); var ms = new MemoryStream(); var meta = VolumeMetadataBuilder.Make("String archive", StringArchiveAppender.CONTENT_TYPE_STRING) .SetVersion(1, 0) .SetDescription("Testing string messages") .SetChannel(Atom.Encode("dvop")) .SetCompressionScheme(compress) .SetEncryptionScheme("aes1"); using (var volume = new DefaultVolume(CryptoMan, meta, ms)) using (var appender = new StringArchiveAppender(volume, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) { for (var i = 0; i < expected.Length; i++) { appender.Append(expected[i]); } } using (var app = new AzosApplication(null, APP_BAD_CRYPTO_CONF.AsLaconicConfig())) { using (var volume = new DefaultVolume(app.SecurityManager.Cryptography, ms))//different key set { var page = new Page(0); volume.ReadPage(0, page);//throws } } }
public void Metadata_AppSection() { var ms = new MemoryStream(); var meta = VolumeMetadataBuilder.Make("V1", "none") .SetApplicationSection(app => { app.AddAttributeNode("a", 1); }) .SetApplicationSection(app => { app.AddAttributeNode("b", -7); }) .SetApplicationSection(app => { app.AddChildNode("sub{ q=true b=-9 }".AsLaconicConfig()); }); var v1 = new DefaultVolume(CryptoMan, meta, ms); var id = v1.Metadata.Id; v1.Dispose();//it closes the stream Aver.IsTrue(ms.Length > 0); var v2 = new DefaultVolume(CryptoMan, ms); v2.Metadata.See(); Aver.AreEqual(id, v2.Metadata.Id); Aver.AreEqual("V1", v2.Metadata.Label); Aver.IsTrue(v2.Metadata.SectionApplication.Exists); Aver.AreEqual(1, v2.Metadata.SectionApplication.Of("a").ValueAsInt()); Aver.AreEqual(-7, v2.Metadata.SectionApplication.Of("b").ValueAsInt()); Aver.AreEqual(true, v2.Metadata.SectionApplication["sub"].Of("q").ValueAsBool()); Aver.AreEqual(-9, v2.Metadata.SectionApplication["sub"].Of("b").ValueAsInt()); }
private void Init() { #if UNITY_EDITOR gameObject.hideFlags = HideFlags.HideAndDontSave; #else DontDestroyOnLoad(gameObject); #endif if (Instance != null && Instance != this) { if (Debug.isDebugBuild) { Debug.Log($"Extra Volume Manager cleaned up. GUID:{gameObject.GetInstanceID()}"); } StopAllCoroutines(); Utility.SafeDestroy(gameObject); } else { Instance = this; gameObject.name = "[DefaultVolume]"; if (Debug.isDebugBuild) { Debug.Log($"Default Volume is {gameObject.GetInstanceID()}"); } Utility.QualityLevelChange += UpdateVolume; UpdateVolume(0, Utility.GetTrueQualityLevel()); // First time set } }
private void Start() { if (!Instance) { Instance = this; if (Application.isPlaying) { DontDestroyOnLoad(gameObject); } _currentQualityLevel = QualitySettings.GetQualityLevel(); UpdateVolume(); } else if (Instance != this) { if (UniversalRenderPipeline.asset.debugLevel != PipelineDebugLevel.Disabled) { Debug.Log($"Extra Volume Manager cleaned up. GUID:{gameObject.GetInstanceID()}"); } #if UNITY_EDITOR DestroyImmediate(gameObject); return; #else Destroy(gameObject); return; #endif } }
public void Corrupt_Volume_Read(string compress, int pgsz, int cnt) { var ms = new MemoryStream(); var meta = VolumeMetadataBuilder.Make("String archive", StringArchiveAppender.CONTENT_TYPE_STRING) .SetVersion(1, 0) .SetDescription("Testing string messages") .SetChannel(Atom.Encode("dvop")) .SetCompressionScheme(compress); var volume = new DefaultVolume(NOPApplication.Instance.SecurityManager.Cryptography, meta, ms) { PageSizeBytes = pgsz }; using (var appender = new StringArchiveAppender(volume, NOPApplication.Instance.TimeSource, new Atom(65), "*****@*****.**")) { for (var i = 0; i < cnt; i++) { appender.Append("string-data--------------------------------------------------------" + i.ToString()); } } "Volume size is {0:n0} bytes".SeeArgs(ms.Length); var reader = new StringArchiveReader(volume); var allCount = reader.All.Count(); Aver.AreEqual(cnt, allCount); reader.All.ForEach((s, i) => Aver.IsTrue(s.EndsWith("---" + i.ToString()))); var pageCount = reader.GetPagesStartingAt(0).Count(); Aver.IsTrue(pageCount > 0); "Before corruption: there are {0:n0} total records in {1:n0} pages".SeeArgs(allCount, pageCount); var midPoint = ms.Length / 2; ms.Position = midPoint; for (var j = 0; j < pgsz * 2; j++) { ms.WriteByte(0x00); //corruption } "-------------- corrupted {0:n0} bytes of {1:n0} total at {2:n0} position ----------- ".SeeArgs(pgsz * 2, ms.Length, midPoint); var allCount2 = reader.GetEntriesStartingAt(new Bookmark(), skipCorruptPages: true).Count(); Aver.IsTrue(allCount > allCount2); var pageCount2 = reader.GetPagesStartingAt(0, skipCorruptPages: true).Count(); Aver.IsTrue(pageCount > pageCount2); "After corruption: there are {0:n0} total records in {1:n0} pages".SeeArgs(allCount2, pageCount2); volume.Dispose(); }
private void OnDestroy() { StopAllCoroutines(); if (Instance == this) { Instance = null; } Utility.QualityLevelChange -= UpdateVolume; }
//[Run("!arch-perf-write", "compress=gzip encrypt=aes1 cnt=16000000 para=16")] public void Write_LogMessages(string compress, string encrypt, int CNT, int PARA) { var msData = new FileStream("c:\\azos\\logging-{0}-{1}.lar".Args(compress.Default("none"), encrypt.Default("none")), FileMode.Create); var msIdxId = new FileStream("c:\\azos\\logging-{0}-{1}.guid.lix".Args(compress.Default("none"), encrypt.Default("none")), FileMode.Create); var volumeData = new DefaultVolume(CryptoMan, VolumeMetadataBuilder.Make("Perf", LogMessageArchiveAppender.CONTENT_TYPE_LOG) .SetVersion(1, 1) .SetDescription("Perf testing") .SetCompressionScheme(compress) .SetEncryptionScheme(encrypt), msData); var volumeIdxId = new DefaultVolume(CryptoMan, VolumeMetadataBuilder.Make("Perf", StringIdxAppender.CONTENT_TYPE_IDX_STRING) .SetVersion(1, 1) .SetDescription("Perf testing") .SetCompressionScheme(compress) .SetEncryptionScheme(encrypt), msIdxId); volumeData.PageSizeBytes = 1024 * 1024; volumeIdxId.PageSizeBytes = 128 * 1024; var time = Azos.Time.Timeter.StartNew(); Parallel.For(0, PARA, _ => { var app = Azos.Apps.ExecutionContext.Application; using (var aIdxId = new StringIdxAppender(volumeIdxId, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) { using (var appender = new LogMessageArchiveAppender(volumeData, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba", onPageCommit: (e, b) => aIdxId.Append(new StringBookmark(e.Guid.ToString().TakeLastChars(4), b)))) { for (var i = 0; app.Active && i < CNT / PARA; i++) { var msg = FakeLogMessage.BuildRandom(); appender.Append(msg); } } } }); time.Stop(); "Did {0:n0} in {1:n1} sec at {2:n2} ops/sec\n".SeeArgs(CNT, time.ElapsedSec, CNT / time.ElapsedSec); volumeIdxId.Dispose(); volumeData.Dispose(); "CLOSED all volumes\n".See(); }
public void Write_Read_Compare(string compress, string encrypt, int count, int pgsize) { var expected = FakeLogMessage.BuildRandomArr(count); var ms = new MemoryStream(); var meta = VolumeMetadataBuilder.Make("Log archive", LogMessageArchiveAppender.CONTENT_TYPE_LOG) .SetVersion(1, 0) .SetDescription("Testing log messages") .SetChannel(Atom.Encode("dvop")) .SetCompressionScheme(compress) .SetEncryptionScheme(encrypt); var volume = new DefaultVolume(CryptoMan, meta, ms) { PageSizeBytes = pgsize }; using (var appender = new LogMessageArchiveAppender(volume, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) { for (var i = 0; i < count; i++) { appender.Append(expected[i]); } } "The volume is {0:n0} bytes".SeeArgs(ms.Length); var reader = new LogMessageArchiveReader(volume); var got = reader.GetEntriesStartingAt(new Bookmark()).ToArray(); Aver.AreEqual(expected.Length, got.Length); for (int i = 0; i < count; i++) { Aver.AreEqual(expected[i].App, got[i].App); Aver.AreEqual(expected[i].ArchiveDimensions, got[i].ArchiveDimensions); Aver.AreEqual(expected[i].Channel, got[i].Channel); Aver.AreEqual(expected[i].Exception?.Message, got[i].Exception?.Message); Aver.AreEqual(expected[i].From, got[i].From); Aver.AreEqual(expected[i].Gdid, got[i].Gdid); Aver.AreEqual(expected[i].Guid.ToString(), got[i].Guid.ToString()); Aver.AreEqual(expected[i].Host, got[i].Host); Aver.AreEqual(expected[i].Parameters, got[i].Parameters); Aver.AreEqual(expected[i].RelatedTo.ToString(), got[i].RelatedTo.ToString()); Aver.AreEqual(expected[i].Text, got[i].Text); Aver.AreEqual(expected[i].Topic, got[i].Topic); Aver.AreEqual((int)expected[i].Type, (int)got[i].Type); Aver.AreEqual(expected[i].UTCTimeStamp, got[i].UTCTimeStamp); } volume.Dispose(); }
public void Write_Read_Compare_PageSplit(int pc, int vsz, int pgsize, string compress, string encrypt, int count, int sz) { var expected = Enumerable.Range(0, count).Select(_ => new string(' ', sz)).ToArray(); var ms = new MemoryStream(); var meta = VolumeMetadataBuilder.Make("String archive", StringArchiveAppender.CONTENT_TYPE_STRING) .SetVersion(1, 0) .SetDescription("Testing string messages") .SetChannel(Atom.Encode("dvop")) .SetCompressionScheme(compress) .SetEncryptionScheme(encrypt); var volume = new DefaultVolume(CryptoMan, meta, ms); volume.PageSizeBytes = pgsize; using (var appender = new StringArchiveAppender(volume, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) { for (var i = 0; i < count; i++) { appender.Append(expected[i]); } } "Volume data stream is {0:n0} bytes".SeeArgs(ms.Length); Aver.IsTrue(ms.Length < vsz); // ms.GetBuffer().ToHexDump().See(); var reader = new StringArchiveReader(volume); // reader.Pages(0).Select(p => (p.State, p.Entries.Count())).See("PAGES/Entries: "); var pageCount = reader.GetPagesStartingAt(0).Count(); "Volume page count is: {0}".SeeArgs(pageCount); Aver.AreEqual(pc, pageCount); var got = reader.All.ToArray(); Aver.AreEqual(expected.Length, got.Length); for (int i = 0; i < count; i++) { Aver.AreEqual(expected[i], got[i]); } volume.Dispose(); }
public void WriteRead(int cnt) { ////var ms = new FileStream("c:\\azos\\archive.lar", FileMode.Create);// new MemoryStream(); var ms = new MemoryStream(); var meta = VolumeMetadataBuilder.Make("Log archive", LogMessageArchiveAppender.CONTENT_TYPE_LOG) .SetVersion(1, 0) .SetDescription("Testing log messages") .SetChannel(Atom.Encode("dvop")); var volume = new DefaultVolume(CryptoMan, meta, ms); using (var appender = new LogMessageArchiveAppender(volume, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) { for (var i = 0; i < cnt; i++) { var msg = new Message() { Text = "Message#" + i.ToString() }.InitDefaultFields(); appender.Append(msg); } } var reader = new LogMessageArchiveReader(volume); Aver.AreEqual(cnt, reader.All.Count()); Aver.AreEqual(cnt, reader.GetEntriesStartingAt(new Bookmark()).Count()); Aver.AreEqual(cnt, reader.GetBookmarkedEntriesStartingAt(new Bookmark()).Count()); Aver.AreEqual(cnt, reader.GetEntriesAsObjectsStartingAt(new Bookmark()).Count()); Aver.AreEqual(cnt, reader.GetBookmarkedEntriesAsObjectsStartingAt(new Bookmark()).Count()); reader.All.ForEach((m, i) => Aver.AreEqual("Message#" + i.ToString(), m.Text)); reader.GetBookmarkedEntriesStartingAt(new Bookmark()).ForEach((t, i) => { var pointed = reader.GetEntriesStartingAt(t.bm).First(); Aver.AreEqual("Message#" + i.ToString(), pointed.Text); Aver.AreEqual(t.entry.Text, pointed.Text); }); volume.Dispose(); }
public void Write_Read_Compare_Strings(bool ascii, int pgsize, string compress, int count, int min, int max) { var expected = ascii ? FakeLogMessage.BuildRandomASCIIStringArr(count, min, max) : FakeLogMessage.BuildRandomUnicodeStringArr(count, min, max); var ms = new MemoryStream(); var meta = VolumeMetadataBuilder.Make("String archive", StringArchiveAppender.CONTENT_TYPE_STRING) .SetVersion(1, 0) .SetDescription("Testing string messages") .SetChannel(Atom.Encode("dvop")); if (compress.IsNotNullOrWhiteSpace()) { meta.SetCompressionScheme(compress); } var volume = new DefaultVolume(NOPApplication.Instance.SecurityManager.Cryptography, meta, ms); volume.PageSizeBytes = pgsize; using (var appender = new StringArchiveAppender(volume, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) { for (var i = 0; i < count; i++) { appender.Append(expected[i]); } } "Volume data stream is {0:n0} bytes".SeeArgs(ms.Length); var reader = new StringArchiveReader(volume); var got = reader.GetEntriesStartingAt(new Bookmark()).ToArray(); Aver.AreEqual(expected.Length, got.Length); for (int i = 0; i < count; i++) { Aver.AreEqual(expected[i], got[i]); } volume.Dispose(); }
public void Metadata_Basic() { var ms = new MemoryStream(); var meta = VolumeMetadataBuilder.Make("Volume-1", "none") .SetVersion(123, 456) .SetDescription("My volume"); var v1 = new DefaultVolume(CryptoMan, meta, ms); var id = v1.Metadata.Id; v1.Dispose();//it closes the stream Aver.IsTrue(ms.Length > 0); var v2 = new DefaultVolume(CryptoMan, ms); Aver.AreEqual(id, v2.Metadata.Id); Aver.AreEqual("Volume-1", v2.Metadata.Label); Aver.AreEqual("My volume", v2.Metadata.Description); Aver.AreEqual(123, v2.Metadata.VersionMajor); Aver.AreEqual(456, v2.Metadata.VersionMinor); Aver.IsTrue(v2.Metadata.Channel.IsZero); Aver.IsFalse(v2.Metadata.IsCompressed); Aver.IsFalse(v2.Metadata.IsEncrypted); }
private void OnEnable() { if (!Instance) { Instance = this; if (Application.isPlaying) { DontDestroyOnLoad(gameObject); } _currentQualityLevel = QualitySettings.GetQualityLevel(); UpdateVolume(); } else if (Instance != this) { Debug.Log($"Extra Volume Manager cleaned up. GUID:{gameObject.GetInstanceID()}"); #if UNITY_EDITOR DestroyImmediate(gameObject); return; #else Destroy(gameObject); return; #endif } }
public void PageInfo_Walk(int count, int psz, bool remount, int strvar) { string makestr(int i) { return(strvar > 0 ? (new string(' ', i % strvar) + "text-string-" + i.ToString()) : ("text-string-" + i.ToString())); } var ms = new MemoryStream(); var meta = VolumeMetadataBuilder.Make("String archive", StringArchiveAppender.CONTENT_TYPE_STRING) .SetVersion(1, 0) .SetDescription("Testing string messages") .SetChannel(Atom.Encode("dvop")); var volume = new DefaultVolume(CryptoMan, meta, ms, ownsStream: false); volume.PageSizeBytes = psz; using (var appender = new StringArchiveAppender(volume, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) { for (var i = 0; i < count; i++) { appender.Append(makestr(i)); } } if (remount) { volume.Dispose(); volume = new DefaultVolume(CryptoMan, ms, ownsStream: false); } var pis = volume.ReadPageInfos(0); "Page infos: {0}".SeeArgs(pis.Count()); foreach (var pi in pis) { var one = volume.ReadPageInfo(pi.PageId); Aver.AreEqual(pi, one); } var total = 0; var reader = new StringArchiveReader(volume); foreach (var pi in pis) { var page = reader.GetOnePageAt(pi.PageId, true); foreach (var entry in page.Entries) { if (entry.State == Entry.Status.Valid) { Aver.AreEqual(makestr(total), reader.Materialize(entry)); total++; } } } Aver.AreEqual(count, total); volume.Dispose(); }
public void Page_Write_Read(string compress, string encrypt, int pad, bool remount) { var ms = new MemoryStream(); var meta = VolumeMetadataBuilder.Make("Volume-1", "raw") .SetCompressionScheme(compress) .SetEncryptionScheme(encrypt); var v1 = new DefaultVolume(CryptoMan, meta, ms); var page = new Page(0); Aver.IsTrue(page.State == Page.Status.Unset); page.BeginWriting(new DateTime(1980, 7, 1, 15, 0, 0, DateTimeKind.Utc), Atom.Encode("app"), "*****@*****.**"); Aver.IsTrue(page.State == Page.Status.Writing); var adr1 = page.Append(new ArraySegment <byte>(new byte[] { 1, 2, 3 }, 0, 3)); Aver.AreEqual(0, adr1); var adr2 = page.Append(new ArraySegment <byte>(new byte[] { 4, 5 }, 0, 2)); Aver.IsTrue(adr2 > 0); var adr3 = page.Append(new ArraySegment <byte>(new byte[pad])); Aver.IsTrue(adr3 > adr2); page.EndWriting(); Aver.IsTrue(page.State == Page.Status.Written); var pid = v1.AppendPage(page); //append to volume Aver.IsTrue(page.State == Page.Status.Written); "Written volume {0} Stream size is {1} bytes".SeeArgs(v1.Metadata.Id, ms.Length); if (remount) { v1.Dispose(); v1 = new DefaultVolume(CryptoMan, ms);//re-mount existing data from stream "Re-mounted volume {0}".SeeArgs(v1.Metadata.Id); } page = new Page(0);//we could have reused the existing page but we re-allocate for experiment cleanness Aver.IsTrue(page.State == Page.Status.Unset); v1.ReadPage(pid, page); Aver.IsTrue(page.State == Page.Status.Reading); //page header read correctly Aver.AreEqual(new DateTime(1980, 7, 1, 15, 0, 0, DateTimeKind.Utc), page.CreateUtc); Aver.AreEqual(Atom.Encode("app"), page.CreateApp); Aver.AreEqual("*****@*****.**", page.CreateHost); var raw = page.Entries.ToArray();//all entry enumeration test Aver.AreEqual(4, raw.Length); Aver.IsTrue(raw[0].State == Entry.Status.Valid); Aver.IsTrue(raw[1].State == Entry.Status.Valid); Aver.IsTrue(raw[2].State == Entry.Status.Valid); Aver.IsTrue(raw[3].State == Entry.Status.EOF); Aver.AreEqual(0, raw[0].Address); Aver.IsTrue(raw[1].Address > 0); Aver.AreEqual(3, raw[0].Raw.Count); Aver.AreEqual(1, raw[0].Raw.Array[raw[0].Raw.Offset + 0]); Aver.AreEqual(2, raw[0].Raw.Array[raw[0].Raw.Offset + 1]); Aver.AreEqual(3, raw[0].Raw.Array[raw[0].Raw.Offset + 2]); Aver.AreEqual(2, raw[1].Raw.Count); Aver.AreEqual(4, raw[1].Raw.Array[raw[1].Raw.Offset + 0]); Aver.AreEqual(5, raw[1].Raw.Array[raw[1].Raw.Offset + 1]); Aver.AreEqual(pad, raw[2].Raw.Count); var one = page[adr1]; //indexer test Aver.IsTrue(one.State == Entry.Status.Valid); Aver.AreEqual(3, one.Raw.Count); Aver.AreEqual(1, one.Raw.Array[one.Raw.Offset + 0]); Aver.AreEqual(2, one.Raw.Array[one.Raw.Offset + 1]); Aver.AreEqual(3, one.Raw.Array[one.Raw.Offset + 2]); one = page[adr2]; Aver.IsTrue(one.State == Entry.Status.Valid); Aver.AreEqual(2, one.Raw.Count); Aver.AreEqual(4, one.Raw.Array[one.Raw.Offset + 0]); Aver.AreEqual(5, one.Raw.Array[one.Raw.Offset + 1]); one = page[adr3]; Aver.IsTrue(one.State == Entry.Status.Valid); Aver.AreEqual(pad, one.Raw.Count); }
public void Page_Write_CorruptPage_Read(int count) { var ms = new MemoryStream(); var meta = VolumeMetadataBuilder.Make("Volume-1", "raw"); var v1 = new DefaultVolume(CryptoMan, meta, ms); var page = new Page(0); Aver.IsTrue(page.State == Page.Status.Unset); page.BeginWriting(new DateTime(1980, 7, 1, 15, 0, 0, DateTimeKind.Utc), Atom.Encode("app"), "*****@*****.**"); var data = new Dictionary <int, byte[]>(); for (var i = 0; i < count; i++) { //generate 1/2 empty arrays for best compression, another 1/2/ filled with random data var buf = ((i & 1) == 0) ? new byte[1 + (i & 0x7f)] : Platform.RandomGenerator.Instance.NextRandomBytes(1 + (i & 0x7f)); var adr = page.Append(new ArraySegment <byte>(buf)); data[adr] = buf; } Aver.AreEqual(data.Count, data.Keys.Distinct().Count());//all addresses are unique page.EndWriting(); var pid = v1.AppendPage(page); //append to volume page = new Page(0); //we could have reused the existing page but we re-allocate for experiment cleanness v1.ReadPage(pid, page); //Aver that all are readable foreach (var kvp in data) { var got = page[kvp.Key]; Aver.IsTrue(got.State == Entry.Status.Valid); Aver.IsTrue(IOUtils.MemBufferEquals(kvp.Value, got.Raw.ToArray())); } //now corrupt First var cadr = data.First().Key; page.Data.Array[cadr] = 0xff; //corrupt underlying page memory data[cadr] = null; //corrupted //corrupt last cadr = data.Last().Key; page.Data.Array[cadr] = 0x00; //corrupt underlying page memory data[cadr] = null; //corrupted var keys = data.Keys.ToArray(); //corrupt a half of written for (var i = 0; i < data.Count / 2; i++) { cadr = keys[Platform.RandomGenerator.Instance.NextScaledRandomInteger(2, data.Count - 2)]; page.Data.Array[cadr] = 0xff; //corrupt underlying page memory data[cadr] = null; //corrupted } "\nStream size is: {0:n0} bytes".SeeArgs(ms.Length); "{0:n0} total entries, {1:n0} are corrupt \n".SeeArgs(data.Count, data.Where(kvp => kvp.Value == null).Count()); //Aver that all which are SET are still readable, others are corrupt foreach (var kvp in data) { var got = page[kvp.Key]; if (kvp.Value != null)//was not corrupted { Aver.IsTrue(got.State == Entry.Status.Valid); Aver.IsTrue(IOUtils.MemBufferEquals(kvp.Value, got.Raw.ToArray())); } else { Aver.IsTrue(got.State == Entry.Status.BadHeader); } } }
[Run("!arch-perf-read", "compress=null encrypt=null search=$(~@term)")]// -r args='term=abcd' //[Run("!arch-perf-read", "compress=gzip encrypt=aes1 search=$(~@term)")]// -r args='term=abcd' public void Read_LogMessages(string compress, string encrypt, string search) { search = search.Default("ABBA"); var msData = new FileStream("c:\\azos\\logging-{0}-{1}.lar".Args(compress.Default("none"), encrypt.Default("none")), FileMode.Open); var msIdxId = new FileStream("c:\\azos\\logging-{0}-{1}.guid.lix".Args(compress.Default("none"), encrypt.Default("none")), FileMode.Open); var volumeData = new DefaultVolume(CryptoMan, msData); var volumeIdxId = new DefaultVolume(CryptoMan, msIdxId); //volumeData.PageSizeBytes = 4 * 1024 * 1024;//<-------------------!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! //volumeIdxId.PageSizeBytes = 4 * 128 * 1024; var reader = new LogMessageArchiveReader(volumeData); var idxReader = new StringIdxReader(volumeIdxId); var time = Azos.Time.Timeter.StartNew(); var total = 0; var wordCount = 0; var found = 0; //foreach (var idx in idxReader.All) //{ // total++; // if (idx.Value.EqualsOrdIgnoreCase(search)) // { // found++; // var data = reader.Entries(idx.Bookmark).FirstOrDefault(); // data.See(); // } //} ////////Rent buffers from arena instead? who is going to release pages? page.Recycle() to release the buffer to the pool? What if they forget to call it? ////////var prealloc = new Page(0); //SIGNIFICANT SLOW-DOWN WHILE ALLOCATING PAGES. WHY MEMORY STREAM DOES ARRAY.CLEAR? //////foreach (var page in reader.GetPagesStartingAt(0/*, preallocatedPage: prealloc*/).Take(200)) //////{ ////// total++; ////// //if (total % 100 ==0) ////// "Page {0:X8} is {1:n0} --- Average size: {2:n0} Entries per page: {3:n0}".SeeArgs(page.PageId, page.Data.Array.Length, reader.AveragePageSizeBytes, page.Entries.Count()); ////// reader.Recycle(page); //////} ////////read all entries //////foreach (var entry in reader.GetEntriesStartingAt(new Bookmark()/*, preallocatedPage: prealloc*/))//.Take(200)); //////{ ////// if (!App.Active) break; ////// total++; ////// if (entry.ExceptionData!=null) wordCount++; ////// if ((total & 0xffff)==0) "Did {0:n0}".SeeArgs(total); //////} //reader.ParallelProcessRawEntryBatchesStartingAt(new Bookmark(), 50, (entries, loop, _) => //{ // var ec = 0; // var wc = 0; // foreach (var entry in entries) // { // if (!App.Active) // { // loop.Break(); // return; // } // if (entry.State == Entry.Status.Valid) // { // ec++; // var msg = reader.Materialize(entry); // try // { // // System.Threading.Thread.SpinWait(8_000); // if (msg.Parameters.Length>0 && msg.Parameters.FirstOrDefault(c => !char.IsWhiteSpace(c))=='{') ////{ //// var map = msg.Parameters.JsonToDataObject() as JsonDataMap; //// wc++; ////} // } // catch // { } // // if (msg.Guid.ToString().StartsWith("faca")) msg.See(); // } // } // if ((total & 0x0fff) == 0) "{0}".SeeArgs(total); // System.Threading.Interlocked.Add(ref total, ec);//1); // System.Threading.Interlocked.Add(ref wordCount, wc);//1); //}, new ParallelOptions { MaxDegreeOfParallelism = 10 }); reader.ParallelProcessPageBatchesStartingAt(0, 5, (page, loop, _) => { var ec = 0; var wc = 0; foreach (var entry in page.Entries) { if (!App.Active) { loop.Break(); return; } if (entry.State == Entry.Status.Valid) { ec++; var msg = reader.Materialize(entry); try { // System.Threading.Thread.SpinWait(8_000); if (msg.Parameters.Length > 0 && msg.Parameters.FirstOrDefault(c => !char.IsWhiteSpace(c)) == '{') { var map = msg.Parameters.JsonToDataObject() as JsonDataMap; wc++; } } catch { } // if (msg.Guid.ToString().StartsWith("faca")) msg.See(); } } if ((total & 0x0fff) == 0) { "{0}".SeeArgs(total); } System.Threading.Interlocked.Add(ref total, ec); //1); System.Threading.Interlocked.Add(ref wordCount, wc); //1); }, new ParallelOptions { MaxDegreeOfParallelism = 10 }); time.Stop(); "Did {0:n0} found {1:n0}({2:n5}%) in {3:n1} sec at {4:n2} ops/sec WC = {5:n0}\n".SeeArgs(total, found, (double)found / total, time.ElapsedSec, total / time.ElapsedSec, wordCount); volumeIdxId.Dispose(); volumeData.Dispose(); "CLOSED all volumes\n".See(); }
public void Write_Read_Amount_Indexing_MemoryStream(string compress, string encrypt, int CNT, string idxCompress, string idxEncrypt) { var msData = new MemoryStream(); var msIdxAmt = new MemoryStream(); var meta = VolumeMetadataBuilder.Make("Primitive Idx", "mjumbo") .SetVersion(1, 1) .SetDescription("MumboJumbo testing") .SetCompressionScheme(compress) .SetEncryptionScheme(encrypt); var volumeData = new DefaultVolume(CryptoMan, meta, msData); var metaIdx = VolumeMetadataBuilder.Make("Primitive Idx Meta", AmountIdxAppender.CONTENT_TYPE_IDX_AMOUNT) .SetVersion(1, 1) .SetDescription("MumboJumbo testing") .SetCompressionScheme(idxCompress) .SetEncryptionScheme(idxEncrypt); var volumeIdxAmt = new DefaultVolume(CryptoMan, metaIdx, msIdxAmt); volumeData.PageSizeBytes = 1024 * 1024; volumeIdxAmt.PageSizeBytes = 128 * 1024; var timeWrite = Azos.Time.Timeter.StartNew(); var app = Azos.Apps.ExecutionContext.Application; using (var aIdxAmt = new AmountIdxAppender(volumeIdxAmt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) { using (var appender = new MumboJumboArchiveAppender(volumeData, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba", onPageCommit: (e, b) => { aIdxAmt.Append(new AmountBookmark(e.Amt, b)); } )) { var messages = FakeRow.GenerateMany <MumboJumbo>(1, 1, (ulong)(CNT * .5) - 1); foreach (var m in messages) { appender.Append(m); } appender.Append(MumboJumbo.GetControl()); messages = FakeRow.GenerateMany <MumboJumbo>(1, 1, (ulong)(CNT * .5)); foreach (var m in messages) { appender.Append(m); } } } timeWrite.Stop(); "Did {0:n0} writes in {1:n1} sec at {2:n2} ops/sec\n".SeeArgs(CNT, timeWrite.ElapsedSec, CNT / timeWrite.ElapsedSec); var ctlMumbo = MumboJumbo.GetControl(); var reader = new MumboJumboArchiveReader(volumeData); var idxAmtReader = new AmountIdxReader(volumeIdxAmt); var timeRead = Azos.Time.Timeter.StartNew(); var gotOne = false; // Find by Amount (Amt) foreach (var idx in idxAmtReader.All) { if (idx.Value == ctlMumbo.Amt) { var data = reader.GetEntriesStartingAt(idx.Bookmark).FirstOrDefault(); data.See(); Aver.AreEqual(ctlMumbo.Amt, data.Amt); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Amt by {nameof(AmountIdxReader)}"); } timeRead.Stop(); "Did {0:n0} reads in {1:n1} sec at {2:n2} ops/sec\n".SeeArgs(CNT, timeRead.ElapsedSec, CNT / timeRead.ElapsedSec); volumeIdxAmt.Dispose(); volumeData.Dispose(); "CLOSED all volumes\n".See(); }
public void Write_Read_Index_Primitives(bool usecache, string compress, string encrypt, int CNT, string idxCompress, string idxEncrypt, int pageLboundKb, int pageUboundKb, int idxLboundKb, int idxUboundKb) { var ctlMumbo = MumboJumbo.GetControl(); IPileImplementation pile = null; ICacheImplementation cache = null; IPageCache pageCache = null; if (usecache) { pile = new DefaultPile(App); cache = new LocalCache(App) { Pile = pile, DefaultTableOptions = new TableOptions("*") { CollisionMode = CollisionMode.Durable }, PileMaxMemoryLimit = 32L * 1024 * 1024 * 1024 }; pile.Start(); cache.Start(); pageCache = new PilePageCache(cache); } try { var msData = new MemoryStream(); var msIdxId = new MemoryStream(); var msIdxCid = new MemoryStream(); var msIdxDid = new MemoryStream(); var msIdxPn = new MemoryStream(); var msIdxLt = new MemoryStream(); var msIdxLn = new MemoryStream(); var msIdxAl = new MemoryStream(); var msIdxCd = new MemoryStream(); var msIdxNt = new MemoryStream(); var msIdxAmt = new MemoryStream(); var meta = VolumeMetadataBuilder.Make("Primitive Idx", "mjumbo") .SetVersion(1, 1) .SetDescription("MumboJumbo testing") .SetCompressionScheme(compress) .SetEncryptionScheme(encrypt); var volumeData = new DefaultVolume(CryptoMan, pageCache, meta, msData); var IdxIdMetaBuilder = getSpecificBuilder(GdidIdxAppender.CONTENT_TYPE_IDX_GDID, idxCompress, idxEncrypt); var IdxCidMetaBuilder = getSpecificBuilder(GuidIdxAppender.CONTENT_TYPE_IDX_GUID, idxCompress, idxEncrypt); var IdxDidMetaBuilder = getSpecificBuilder(LongIdxAppender.CONTENT_TYPE_IDX_LONG, idxCompress, idxEncrypt); var IdxPnMetaBuilder = getSpecificBuilder(IntIdxAppender.CONTENT_TYPE_IDX_INT, idxCompress, idxEncrypt); var IdxLtMetaBuilder = getSpecificBuilder(DoubleIdxAppender.CONTENT_TYPE_IDX_DOUBLE, idxCompress, idxEncrypt); var IdxLnMetaBuilder = getSpecificBuilder(DoubleIdxAppender.CONTENT_TYPE_IDX_DOUBLE, idxCompress, idxEncrypt); var IdxAlMetaBuilder = getSpecificBuilder(DecimalIdxAppender.CONTENT_TYPE_IDX_DECIMAL, idxCompress, idxEncrypt); var IdxCdMetaBuilder = getSpecificBuilder(DateTimeIdxAppender.CONTENT_TYPE_IDX_DATETIME, idxCompress, idxEncrypt); var IdxNtMetaBuilder = getSpecificBuilder(StringIdxAppender.CONTENT_TYPE_IDX_STRING, idxCompress, idxEncrypt); var IdxAmtMetaBuilder = getSpecificBuilder(AmountIdxAppender.CONTENT_TYPE_IDX_AMOUNT, idxCompress, idxEncrypt); var volumeIdxId = new DefaultVolume(CryptoMan, IdxIdMetaBuilder, msIdxId); var volumeIdxCid = new DefaultVolume(CryptoMan, IdxCidMetaBuilder, msIdxCid); var volumeIdxDid = new DefaultVolume(CryptoMan, IdxDidMetaBuilder, msIdxDid); var volumeIdxPn = new DefaultVolume(CryptoMan, IdxPnMetaBuilder, msIdxPn); var volumeIdxLt = new DefaultVolume(CryptoMan, IdxLtMetaBuilder, msIdxLt); var volumeIdxLn = new DefaultVolume(CryptoMan, IdxLnMetaBuilder, msIdxLn); var volumeIdxAl = new DefaultVolume(CryptoMan, IdxAlMetaBuilder, msIdxAl); var volumeIdxCd = new DefaultVolume(CryptoMan, IdxCdMetaBuilder, msIdxCd); var volumeIdxNt = new DefaultVolume(CryptoMan, IdxNtMetaBuilder, msIdxNt); var volumeIdxAmt = new DefaultVolume(CryptoMan, IdxAmtMetaBuilder, msIdxAmt); volumeData.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(pageLboundKb, pageUboundKb) * 1024; volumeIdxId.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxCid.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxDid.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxPn.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxLt.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxLn.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxAl.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxCd.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxNt.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxAmt.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; var reader = new MumboJumboArchiveReader(volumeData); var idxIdReader = new GdidIdxReader(volumeIdxId); var idxCidReader = new GuidIdxReader(volumeIdxCid); var idxDidReader = new LongIdxReader(volumeIdxDid); var idxPnReader = new IntIdxReader(volumeIdxPn); var idxLtReader = new DoubleIdxReader(volumeIdxLt); var idxLnReader = new DoubleIdxReader(volumeIdxLn); var idxAlReader = new DecimalIdxReader(volumeIdxAl); var idxCdReader = new DateTimeIdxReader(volumeIdxCd); var idxNtReader = new StringIdxReader(volumeIdxNt); var idxAmtReader = new AmountIdxReader(volumeIdxAmt); var time = Azos.Time.Timeter.StartNew(); var app = Azos.Apps.ExecutionContext.Application; using (var aIdxId = new GdidIdxAppender(volumeIdxId, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxCid = new GuidIdxAppender(volumeIdxCid, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxDid = new LongIdxAppender(volumeIdxDid, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxPn = new IntIdxAppender(volumeIdxPn, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxLt = new DoubleIdxAppender(volumeIdxLt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxLn = new DoubleIdxAppender(volumeIdxLn, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxAl = new DecimalIdxAppender(volumeIdxAl, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxCd = new DateTimeIdxAppender(volumeIdxCd, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxNt = new StringIdxAppender(volumeIdxNt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxAmt = new AmountIdxAppender(volumeIdxAmt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) { using (var appender = new MumboJumboArchiveAppender(volumeData, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba", onPageCommit: (e, b) => { aIdxId.Append(new GdidBookmark(e.ID, b)); aIdxCid.Append(new GuidBookmark(e.CorrelationId, b)); aIdxDid.Append(new LongBookmark(e.DeviceId, b)); aIdxPn.Append(new IntBookmark(e.PartNumber, b)); aIdxLt.Append(new DoubleBookmark(e.Latitude, b)); aIdxLn.Append(new DoubleBookmark(e.Longitude, b)); aIdxAl.Append(new DecimalBookmark(e.Altitude, b)); aIdxCd.Append(new DateTimeBookmark(e.CreateDate, b)); aIdxNt.Append(new StringBookmark(e.Note, b)); aIdxAmt.Append(new AmountBookmark(e.Amt, b)); })) { var messages = FakeRow.GenerateMany <MumboJumbo>(1, 1, (ulong)(CNT * .5) - 1); foreach (var m in messages) { if (!App.Active) { break; } appender.Append(m); } /////////////////////////////////////////////// appender.Append(MumboJumbo.GetControl()); //we add 1 control message in the middle /////////////////////////////////////////////// messages = FakeRow.GenerateMany <MumboJumbo>(1, 1, (ulong)(CNT * .5)); foreach (var m in messages) { if (!App.Active) { break; } appender.Append(m); } } } "Wrote {0} items, now reading".SeeArgs(CNT); if (cache != null) { "Cache has {0} items".SeeArgs(cache.Count); } var gotOne = false; // Find by GDID (ID) foreach (var idx in idxIdReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.ID, idx.Value); if (idx.Value == ctlMumbo.ID) { //data.See(); Aver.AreEqual(ctlMumbo.ID, data.ID); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find ID by {nameof(GdidIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(GdidIdxReader)); gotOne = false; // Find by Guid (CorrelationId) foreach (var idx in idxCidReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.CorrelationId, idx.Value); if (idx.Value == ctlMumbo.CorrelationId) { //data.See(); Aver.AreEqual(ctlMumbo.CorrelationId, data.CorrelationId); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find CorrelationId by {nameof(GuidIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(GuidIdxReader)); gotOne = false; // Find by Long (DeviceId) foreach (var idx in idxDidReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.DeviceId, idx.Value); if (idx.Value == ctlMumbo.DeviceId) { //data.See(); Aver.AreEqual(ctlMumbo.DeviceId, data.DeviceId); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find DeviceId by {nameof(LongIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(LongIdxReader)); gotOne = false; // Find by Int (PartNumber) foreach (var idx in idxPnReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.PartNumber, idx.Value); if (idx.Value == ctlMumbo.PartNumber) { //data.See(); Aver.AreEqual(ctlMumbo.PartNumber, data.PartNumber); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find PartNumber by {nameof(IntIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(IntIdxReader)); gotOne = false; // Find by Double (Latitude) foreach (var idx in idxLtReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.Latitude, idx.Value); if (idx.Value == ctlMumbo.Latitude) { //data.See(); Aver.AreEqual(ctlMumbo.Latitude, data.Latitude); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Latitude by {nameof(DoubleIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(DoubleIdxReader)); gotOne = false; // Find by Double (Longitude) foreach (var idx in idxLnReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.Longitude, idx.Value); if (idx.Value == ctlMumbo.Longitude) { //data.See(); Aver.AreEqual(ctlMumbo.Longitude, data.Longitude); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Longitude by {nameof(DoubleIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(DoubleIdxReader)); gotOne = false; // Find by Decimal (Altitude) foreach (var idx in idxAlReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.Altitude, idx.Value); if (idx.Value == ctlMumbo.Altitude) { //data.See(); Aver.AreEqual(ctlMumbo.Altitude, data.Altitude); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Altitude by {nameof(DecimalIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(DecimalIdxReader)); gotOne = false; // Find by DateTime (CreateDate) foreach (var idx in idxCdReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.CreateDate, idx.Value); if (idx.Value == ctlMumbo.CreateDate) { //data.See(); Aver.AreEqual(ctlMumbo.CreateDate, data.CreateDate); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find CreateDate by {nameof(DateTimeIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(DateTimeIdxReader)); gotOne = false; // Find by String (Note) foreach (var idx in idxNtReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.Note, idx.Value); if (idx.Value == ctlMumbo.Note) { //data.See(); Aver.AreEqual(ctlMumbo.Note, data.Note); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Note by {nameof(StringIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(StringIdxReader)); gotOne = false; // Find by Amount (Amt) foreach (var idx in idxAmtReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.Amt, idx.Value); if (idx.Value == ctlMumbo.Amt) { //data.See(); Aver.AreEqual(ctlMumbo.Amt, data.Amt); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Amt by {nameof(AmountIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(AmountIdxReader)); //gotOne = false; time.Stop(); "Did {0:n0} in {1:n1} sec at {2:n2} ops/sec\n".SeeArgs(CNT, time.ElapsedSec, CNT / time.ElapsedSec); volumeIdxId.Dispose(); volumeIdxCid.Dispose(); volumeIdxDid.Dispose(); volumeIdxPn.Dispose(); volumeIdxLt.Dispose(); volumeIdxLn.Dispose(); volumeIdxAl.Dispose(); volumeIdxCd.Dispose(); volumeIdxNt.Dispose(); volumeIdxAmt.Dispose(); volumeData.Dispose(); } finally { DisposableObject.DisposeIfDisposableAndNull(ref pageCache); DisposableObject.DisposeIfDisposableAndNull(ref cache); DisposableObject.DisposeIfDisposableAndNull(ref pile); "CLOSED all volumes\n".See(); } }