[Run("!arch-perf-read", "compress=null encrypt=null search=$(~@term)")]// -r args='term=abcd' //[Run("!arch-perf-read", "compress=gzip encrypt=aes1 search=$(~@term)")]// -r args='term=abcd' public void Read_LogMessages(string compress, string encrypt, string search) { search = search.Default("ABBA"); var msData = new FileStream("c:\\azos\\logging-{0}-{1}.lar".Args(compress.Default("none"), encrypt.Default("none")), FileMode.Open); var msIdxId = new FileStream("c:\\azos\\logging-{0}-{1}.guid.lix".Args(compress.Default("none"), encrypt.Default("none")), FileMode.Open); var volumeData = new DefaultVolume(CryptoMan, msData); var volumeIdxId = new DefaultVolume(CryptoMan, msIdxId); //volumeData.PageSizeBytes = 4 * 1024 * 1024;//<-------------------!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! //volumeIdxId.PageSizeBytes = 4 * 128 * 1024; var reader = new LogMessageArchiveReader(volumeData); var idxReader = new StringIdxReader(volumeIdxId); var time = Azos.Time.Timeter.StartNew(); var total = 0; var wordCount = 0; var found = 0; //foreach (var idx in idxReader.All) //{ // total++; // if (idx.Value.EqualsOrdIgnoreCase(search)) // { // found++; // var data = reader.Entries(idx.Bookmark).FirstOrDefault(); // data.See(); // } //} ////////Rent buffers from arena instead? who is going to release pages? page.Recycle() to release the buffer to the pool? What if they forget to call it? ////////var prealloc = new Page(0); //SIGNIFICANT SLOW-DOWN WHILE ALLOCATING PAGES. WHY MEMORY STREAM DOES ARRAY.CLEAR? //////foreach (var page in reader.GetPagesStartingAt(0/*, preallocatedPage: prealloc*/).Take(200)) //////{ ////// total++; ////// //if (total % 100 ==0) ////// "Page {0:X8} is {1:n0} --- Average size: {2:n0} Entries per page: {3:n0}".SeeArgs(page.PageId, page.Data.Array.Length, reader.AveragePageSizeBytes, page.Entries.Count()); ////// reader.Recycle(page); //////} ////////read all entries //////foreach (var entry in reader.GetEntriesStartingAt(new Bookmark()/*, preallocatedPage: prealloc*/))//.Take(200)); //////{ ////// if (!App.Active) break; ////// total++; ////// if (entry.ExceptionData!=null) wordCount++; ////// if ((total & 0xffff)==0) "Did {0:n0}".SeeArgs(total); //////} //reader.ParallelProcessRawEntryBatchesStartingAt(new Bookmark(), 50, (entries, loop, _) => //{ // var ec = 0; // var wc = 0; // foreach (var entry in entries) // { // if (!App.Active) // { // loop.Break(); // return; // } // if (entry.State == Entry.Status.Valid) // { // ec++; // var msg = reader.Materialize(entry); // try // { // // System.Threading.Thread.SpinWait(8_000); // if (msg.Parameters.Length>0 && msg.Parameters.FirstOrDefault(c => !char.IsWhiteSpace(c))=='{') ////{ //// var map = msg.Parameters.JsonToDataObject() as JsonDataMap; //// wc++; ////} // } // catch // { } // // if (msg.Guid.ToString().StartsWith("faca")) msg.See(); // } // } // if ((total & 0x0fff) == 0) "{0}".SeeArgs(total); // System.Threading.Interlocked.Add(ref total, ec);//1); // System.Threading.Interlocked.Add(ref wordCount, wc);//1); //}, new ParallelOptions { MaxDegreeOfParallelism = 10 }); reader.ParallelProcessPageBatchesStartingAt(0, 5, (page, loop, _) => { var ec = 0; var wc = 0; foreach (var entry in page.Entries) { if (!App.Active) { loop.Break(); return; } if (entry.State == Entry.Status.Valid) { ec++; var msg = reader.Materialize(entry); try { // System.Threading.Thread.SpinWait(8_000); if (msg.Parameters.Length > 0 && msg.Parameters.FirstOrDefault(c => !char.IsWhiteSpace(c)) == '{') { var map = msg.Parameters.JsonToDataObject() as JsonDataMap; wc++; } } catch { } // if (msg.Guid.ToString().StartsWith("faca")) msg.See(); } } if ((total & 0x0fff) == 0) { "{0}".SeeArgs(total); } System.Threading.Interlocked.Add(ref total, ec); //1); System.Threading.Interlocked.Add(ref wordCount, wc); //1); }, new ParallelOptions { MaxDegreeOfParallelism = 10 }); time.Stop(); "Did {0:n0} found {1:n0}({2:n5}%) in {3:n1} sec at {4:n2} ops/sec WC = {5:n0}\n".SeeArgs(total, found, (double)found / total, time.ElapsedSec, total / time.ElapsedSec, wordCount); volumeIdxId.Dispose(); volumeData.Dispose(); "CLOSED all volumes\n".See(); }