protected override void DoConfigure(IConfigSectionNode node) { base.DoConfigure(node); if (node == null || !node.Exists) { return; } DisposeAndNull(ref m_Cache); DisposeAndNull(ref m_Pile); var ncache = node[CONFIG_CACHE_SECTION]; m_Cache = FactoryUtils.MakeAndConfigureDirectedComponent <ICacheImplementation>(this, ncache, typeof(LocalCache), new[] { "Cache::{0}::{1}".Args(nameof(CacheModule), Name) }); if (m_Cache is LocalCache lcache) { var npile = node[CONFIG_PILE_SECTION]; m_Pile = FactoryUtils.MakeAndConfigureDirectedComponent <IPileImplementation>(this, npile, typeof(DefaultPile), new[] { "Pile::{0}::{1}".Args(nameof(CacheModule), Name) }); lcache.Pile = m_Pile; } }
public PileCacheUserStore(IPileImplementation pile) { m_Pile = pile; m_Cache = new LocalCache(m_Pile, null, null); m_Cache.Configure(null); m_Cache.Start(); }
public PileUserGraph(IPileImplementation pile) { m_Pile = pile; m_Cache = new LocalCache(); m_Cache.Pile = pile; m_Cache.DefaultTableOptions = new TableOptions("*"); m_Cache.DefaultTableOptions.CollisionMode = CollisionMode.Durable; //keep all entries (slower, uses chaining) m_Cache.DefaultTableOptions.DefaultMaxAgeSec = 0; //never expire items m_Cache.Start(); }
public LocalCache(IPileImplementation pile, object director) : base(director) { m_Pile = pile; }
public LocalCache(IPileImplementation pile) : base() { m_Pile = pile; }
protected override void DoConfigure(IConfigSectionNode node) { base.DoConfigure(node); if (node == null) { return; } cleanup(); //build forests foreach (var nforest in node.ChildrenNamed(CONFIG_FOREST_SECTION)) { var idForest = nforest.Of(Configuration.CONFIG_NAME_ATTR, "id").ValueAsAtom(Atom.ZERO); if (idForest.IsZero || !idForest.IsValid) { throw new ConfigException($"{nameof(ForestDataSource)} config `forest` section is missing a valid atom `$id`"); } var trees = new Registry <IDataStoreImplementation>(); var forest = new _forest(idForest, trees); if (!m_Forests.Register(forest)) { throw new ConfigException($"{nameof(ForestDataSource)} config duplicate section: ./forest[name='{forest.Name}']"); } //build trees foreach (var ntree in nforest.ChildrenNamed(CONFIG_TREE_SECTION)) { var idTree = ntree.Of(Configuration.CONFIG_NAME_ATTR).ValueAsAtom(Atom.ZERO); if (idTree.IsZero || !idTree.IsValid) { throw new ConfigException($"{nameof(ForestDataSource)} config `tree` section is missing a valid atom `$id`"); } var tree = FactoryUtils.MakeAndConfigureDirectedComponent <IDataStoreImplementation>(this, ntree); if (!trees.Register(tree)) { throw new ConfigException($"{nameof(ForestDataSource)} config duplicate section: ./forest[name='{forest.Name}']/tree['{tree.Name}']"); } } } //Build CACHE var ncache = node[CONFIG_CACHE_SECTION]; m_Cache = FactoryUtils.MakeAndConfigureDirectedComponent <ICacheImplementation>(this, ncache, typeof(LocalCache), new[] { "Cache::{0}::{1}".Args(nameof(ForestDataSource), Name) }); if (m_Cache is LocalCache lcache) { var npile = node[CONFIG_PILE_SECTION]; m_Pile = FactoryUtils.MakeAndConfigureDirectedComponent <IPileImplementation>(this, npile, typeof(DefaultPile), new[] { "Pile::{0}::{1}".Args(nameof(ForestDataSource), Name) }); lcache.Pile = m_Pile; } }
public void Write_Read_Index_Primitives(bool usecache, string compress, string encrypt, int CNT, string idxCompress, string idxEncrypt, int pageLboundKb, int pageUboundKb, int idxLboundKb, int idxUboundKb) { var ctlMumbo = MumboJumbo.GetControl(); IPileImplementation pile = null; ICacheImplementation cache = null; IPageCache pageCache = null; if (usecache) { pile = new DefaultPile(App); cache = new LocalCache(App) { Pile = pile, DefaultTableOptions = new TableOptions("*") { CollisionMode = CollisionMode.Durable }, PileMaxMemoryLimit = 32L * 1024 * 1024 * 1024 }; pile.Start(); cache.Start(); pageCache = new PilePageCache(cache); } try { var msData = new MemoryStream(); var msIdxId = new MemoryStream(); var msIdxCid = new MemoryStream(); var msIdxDid = new MemoryStream(); var msIdxPn = new MemoryStream(); var msIdxLt = new MemoryStream(); var msIdxLn = new MemoryStream(); var msIdxAl = new MemoryStream(); var msIdxCd = new MemoryStream(); var msIdxNt = new MemoryStream(); var msIdxAmt = new MemoryStream(); var meta = VolumeMetadataBuilder.Make("Primitive Idx", "mjumbo") .SetVersion(1, 1) .SetDescription("MumboJumbo testing") .SetCompressionScheme(compress) .SetEncryptionScheme(encrypt); var volumeData = new DefaultVolume(CryptoMan, pageCache, meta, msData); var IdxIdMetaBuilder = getSpecificBuilder(GdidIdxAppender.CONTENT_TYPE_IDX_GDID, idxCompress, idxEncrypt); var IdxCidMetaBuilder = getSpecificBuilder(GuidIdxAppender.CONTENT_TYPE_IDX_GUID, idxCompress, idxEncrypt); var IdxDidMetaBuilder = getSpecificBuilder(LongIdxAppender.CONTENT_TYPE_IDX_LONG, idxCompress, idxEncrypt); var IdxPnMetaBuilder = getSpecificBuilder(IntIdxAppender.CONTENT_TYPE_IDX_INT, idxCompress, idxEncrypt); var IdxLtMetaBuilder = getSpecificBuilder(DoubleIdxAppender.CONTENT_TYPE_IDX_DOUBLE, idxCompress, idxEncrypt); var IdxLnMetaBuilder = getSpecificBuilder(DoubleIdxAppender.CONTENT_TYPE_IDX_DOUBLE, idxCompress, idxEncrypt); var IdxAlMetaBuilder = getSpecificBuilder(DecimalIdxAppender.CONTENT_TYPE_IDX_DECIMAL, idxCompress, idxEncrypt); var IdxCdMetaBuilder = getSpecificBuilder(DateTimeIdxAppender.CONTENT_TYPE_IDX_DATETIME, idxCompress, idxEncrypt); var IdxNtMetaBuilder = getSpecificBuilder(StringIdxAppender.CONTENT_TYPE_IDX_STRING, idxCompress, idxEncrypt); var IdxAmtMetaBuilder = getSpecificBuilder(AmountIdxAppender.CONTENT_TYPE_IDX_AMOUNT, idxCompress, idxEncrypt); var volumeIdxId = new DefaultVolume(CryptoMan, IdxIdMetaBuilder, msIdxId); var volumeIdxCid = new DefaultVolume(CryptoMan, IdxCidMetaBuilder, msIdxCid); var volumeIdxDid = new DefaultVolume(CryptoMan, IdxDidMetaBuilder, msIdxDid); var volumeIdxPn = new DefaultVolume(CryptoMan, IdxPnMetaBuilder, msIdxPn); var volumeIdxLt = new DefaultVolume(CryptoMan, IdxLtMetaBuilder, msIdxLt); var volumeIdxLn = new DefaultVolume(CryptoMan, IdxLnMetaBuilder, msIdxLn); var volumeIdxAl = new DefaultVolume(CryptoMan, IdxAlMetaBuilder, msIdxAl); var volumeIdxCd = new DefaultVolume(CryptoMan, IdxCdMetaBuilder, msIdxCd); var volumeIdxNt = new DefaultVolume(CryptoMan, IdxNtMetaBuilder, msIdxNt); var volumeIdxAmt = new DefaultVolume(CryptoMan, IdxAmtMetaBuilder, msIdxAmt); volumeData.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(pageLboundKb, pageUboundKb) * 1024; volumeIdxId.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxCid.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxDid.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxPn.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxLt.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxLn.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxAl.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxCd.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxNt.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; volumeIdxAmt.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024; var reader = new MumboJumboArchiveReader(volumeData); var idxIdReader = new GdidIdxReader(volumeIdxId); var idxCidReader = new GuidIdxReader(volumeIdxCid); var idxDidReader = new LongIdxReader(volumeIdxDid); var idxPnReader = new IntIdxReader(volumeIdxPn); var idxLtReader = new DoubleIdxReader(volumeIdxLt); var idxLnReader = new DoubleIdxReader(volumeIdxLn); var idxAlReader = new DecimalIdxReader(volumeIdxAl); var idxCdReader = new DateTimeIdxReader(volumeIdxCd); var idxNtReader = new StringIdxReader(volumeIdxNt); var idxAmtReader = new AmountIdxReader(volumeIdxAmt); var time = Azos.Time.Timeter.StartNew(); var app = Azos.Apps.ExecutionContext.Application; using (var aIdxId = new GdidIdxAppender(volumeIdxId, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxCid = new GuidIdxAppender(volumeIdxCid, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxDid = new LongIdxAppender(volumeIdxDid, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxPn = new IntIdxAppender(volumeIdxPn, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxLt = new DoubleIdxAppender(volumeIdxLt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxLn = new DoubleIdxAppender(volumeIdxLn, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxAl = new DecimalIdxAppender(volumeIdxAl, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxCd = new DateTimeIdxAppender(volumeIdxCd, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxNt = new StringIdxAppender(volumeIdxNt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) using (var aIdxAmt = new AmountIdxAppender(volumeIdxAmt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba")) { using (var appender = new MumboJumboArchiveAppender(volumeData, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba", onPageCommit: (e, b) => { aIdxId.Append(new GdidBookmark(e.ID, b)); aIdxCid.Append(new GuidBookmark(e.CorrelationId, b)); aIdxDid.Append(new LongBookmark(e.DeviceId, b)); aIdxPn.Append(new IntBookmark(e.PartNumber, b)); aIdxLt.Append(new DoubleBookmark(e.Latitude, b)); aIdxLn.Append(new DoubleBookmark(e.Longitude, b)); aIdxAl.Append(new DecimalBookmark(e.Altitude, b)); aIdxCd.Append(new DateTimeBookmark(e.CreateDate, b)); aIdxNt.Append(new StringBookmark(e.Note, b)); aIdxAmt.Append(new AmountBookmark(e.Amt, b)); })) { var messages = FakeRow.GenerateMany <MumboJumbo>(1, 1, (ulong)(CNT * .5) - 1); foreach (var m in messages) { if (!App.Active) { break; } appender.Append(m); } /////////////////////////////////////////////// appender.Append(MumboJumbo.GetControl()); //we add 1 control message in the middle /////////////////////////////////////////////// messages = FakeRow.GenerateMany <MumboJumbo>(1, 1, (ulong)(CNT * .5)); foreach (var m in messages) { if (!App.Active) { break; } appender.Append(m); } } } "Wrote {0} items, now reading".SeeArgs(CNT); if (cache != null) { "Cache has {0} items".SeeArgs(cache.Count); } var gotOne = false; // Find by GDID (ID) foreach (var idx in idxIdReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.ID, idx.Value); if (idx.Value == ctlMumbo.ID) { //data.See(); Aver.AreEqual(ctlMumbo.ID, data.ID); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find ID by {nameof(GdidIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(GdidIdxReader)); gotOne = false; // Find by Guid (CorrelationId) foreach (var idx in idxCidReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.CorrelationId, idx.Value); if (idx.Value == ctlMumbo.CorrelationId) { //data.See(); Aver.AreEqual(ctlMumbo.CorrelationId, data.CorrelationId); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find CorrelationId by {nameof(GuidIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(GuidIdxReader)); gotOne = false; // Find by Long (DeviceId) foreach (var idx in idxDidReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.DeviceId, idx.Value); if (idx.Value == ctlMumbo.DeviceId) { //data.See(); Aver.AreEqual(ctlMumbo.DeviceId, data.DeviceId); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find DeviceId by {nameof(LongIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(LongIdxReader)); gotOne = false; // Find by Int (PartNumber) foreach (var idx in idxPnReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.PartNumber, idx.Value); if (idx.Value == ctlMumbo.PartNumber) { //data.See(); Aver.AreEqual(ctlMumbo.PartNumber, data.PartNumber); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find PartNumber by {nameof(IntIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(IntIdxReader)); gotOne = false; // Find by Double (Latitude) foreach (var idx in idxLtReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.Latitude, idx.Value); if (idx.Value == ctlMumbo.Latitude) { //data.See(); Aver.AreEqual(ctlMumbo.Latitude, data.Latitude); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Latitude by {nameof(DoubleIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(DoubleIdxReader)); gotOne = false; // Find by Double (Longitude) foreach (var idx in idxLnReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.Longitude, idx.Value); if (idx.Value == ctlMumbo.Longitude) { //data.See(); Aver.AreEqual(ctlMumbo.Longitude, data.Longitude); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Longitude by {nameof(DoubleIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(DoubleIdxReader)); gotOne = false; // Find by Decimal (Altitude) foreach (var idx in idxAlReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.Altitude, idx.Value); if (idx.Value == ctlMumbo.Altitude) { //data.See(); Aver.AreEqual(ctlMumbo.Altitude, data.Altitude); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Altitude by {nameof(DecimalIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(DecimalIdxReader)); gotOne = false; // Find by DateTime (CreateDate) foreach (var idx in idxCdReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.CreateDate, idx.Value); if (idx.Value == ctlMumbo.CreateDate) { //data.See(); Aver.AreEqual(ctlMumbo.CreateDate, data.CreateDate); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find CreateDate by {nameof(DateTimeIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(DateTimeIdxReader)); gotOne = false; // Find by String (Note) foreach (var idx in idxNtReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.Note, idx.Value); if (idx.Value == ctlMumbo.Note) { //data.See(); Aver.AreEqual(ctlMumbo.Note, data.Note); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Note by {nameof(StringIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(StringIdxReader)); gotOne = false; // Find by Amount (Amt) foreach (var idx in idxAmtReader.All) { if (!App.Active) { break; } var data = reader.GetEntriesStartingAt(idx.Bookmark).First(); Aver.AreEqual(data.Amt, idx.Value); if (idx.Value == ctlMumbo.Amt) { //data.See(); Aver.AreEqual(ctlMumbo.Amt, data.Amt); gotOne = true; break; } } if (!gotOne) { Aver.Fail($"Failed to find Amt by {nameof(AmountIdxReader)}"); } "Finished reading {0}".SeeArgs(nameof(AmountIdxReader)); //gotOne = false; time.Stop(); "Did {0:n0} in {1:n1} sec at {2:n2} ops/sec\n".SeeArgs(CNT, time.ElapsedSec, CNT / time.ElapsedSec); volumeIdxId.Dispose(); volumeIdxCid.Dispose(); volumeIdxDid.Dispose(); volumeIdxPn.Dispose(); volumeIdxLt.Dispose(); volumeIdxLn.Dispose(); volumeIdxAl.Dispose(); volumeIdxCd.Dispose(); volumeIdxNt.Dispose(); volumeIdxAmt.Dispose(); volumeData.Dispose(); } finally { DisposableObject.DisposeIfDisposableAndNull(ref pageCache); DisposableObject.DisposeIfDisposableAndNull(ref cache); DisposableObject.DisposeIfDisposableAndNull(ref pile); "CLOSED all volumes\n".See(); } }
public LocalCache(IPileImplementation pile, object director, string name) : base(director) { m_Pile = pile; Name = name; }
public LocalCache(IApplicationComponent director, string name, IPileImplementation pile) : base(director) { m_Pile = pile; Name = name; }