protected override void DoConfigure(IConfigSectionNode node)
{
base.DoConfigure(node);
if (node == null || !node.Exists)
{
return;
}
DisposeAndNull(ref m_Cache);
DisposeAndNull(ref m_Pile);
var ncache = node[CONFIG_CACHE_SECTION];
m_Cache = FactoryUtils.MakeAndConfigureDirectedComponent <ICacheImplementation>(this,
ncache,
typeof(LocalCache),
new[] { "Cache::{0}::{1}".Args(nameof(CacheModule), Name) });
if (m_Cache is LocalCache lcache)
{
var npile = node[CONFIG_PILE_SECTION];
m_Pile = FactoryUtils.MakeAndConfigureDirectedComponent <IPileImplementation>(this,
npile,
typeof(DefaultPile),
new[] { "Pile::{0}::{1}".Args(nameof(CacheModule), Name) });
lcache.Pile = m_Pile;
}
}
public PileCacheUserStore(IPileImplementation pile)
{
m_Pile = pile;
m_Cache = new LocalCache(m_Pile, null, null);
m_Cache.Configure(null);
m_Cache.Start();
}
public PileUserGraph(IPileImplementation pile)
{
m_Pile = pile;
m_Cache = new LocalCache();
m_Cache.Pile = pile;
m_Cache.DefaultTableOptions = new TableOptions("*");
m_Cache.DefaultTableOptions.CollisionMode = CollisionMode.Durable; //keep all entries (slower, uses chaining)
m_Cache.DefaultTableOptions.DefaultMaxAgeSec = 0; //never expire items
m_Cache.Start();
}
public LocalCache(IPileImplementation pile, object director) : base(director)
{
m_Pile = pile;
}
public LocalCache(IPileImplementation pile) : base()
{
m_Pile = pile;
}
protected override void DoConfigure(IConfigSectionNode node)
{
base.DoConfigure(node);
if (node == null)
{
return;
}
cleanup();
//build forests
foreach (var nforest in node.ChildrenNamed(CONFIG_FOREST_SECTION))
{
var idForest = nforest.Of(Configuration.CONFIG_NAME_ATTR, "id").ValueAsAtom(Atom.ZERO);
if (idForest.IsZero || !idForest.IsValid)
{
throw new ConfigException($"{nameof(ForestDataSource)} config `forest` section is missing a valid atom `$id`");
}
var trees = new Registry <IDataStoreImplementation>();
var forest = new _forest(idForest, trees);
if (!m_Forests.Register(forest))
{
throw new ConfigException($"{nameof(ForestDataSource)} config duplicate section: ./forest[name='{forest.Name}']");
}
//build trees
foreach (var ntree in nforest.ChildrenNamed(CONFIG_TREE_SECTION))
{
var idTree = ntree.Of(Configuration.CONFIG_NAME_ATTR).ValueAsAtom(Atom.ZERO);
if (idTree.IsZero || !idTree.IsValid)
{
throw new ConfigException($"{nameof(ForestDataSource)} config `tree` section is missing a valid atom `$id`");
}
var tree = FactoryUtils.MakeAndConfigureDirectedComponent <IDataStoreImplementation>(this, ntree);
if (!trees.Register(tree))
{
throw new ConfigException($"{nameof(ForestDataSource)} config duplicate section: ./forest[name='{forest.Name}']/tree['{tree.Name}']");
}
}
}
//Build CACHE
var ncache = node[CONFIG_CACHE_SECTION];
m_Cache = FactoryUtils.MakeAndConfigureDirectedComponent <ICacheImplementation>(this,
ncache,
typeof(LocalCache),
new[] { "Cache::{0}::{1}".Args(nameof(ForestDataSource), Name) });
if (m_Cache is LocalCache lcache)
{
var npile = node[CONFIG_PILE_SECTION];
m_Pile = FactoryUtils.MakeAndConfigureDirectedComponent <IPileImplementation>(this,
npile,
typeof(DefaultPile),
new[] { "Pile::{0}::{1}".Args(nameof(ForestDataSource), Name) });
lcache.Pile = m_Pile;
}
}
public void Write_Read_Index_Primitives(bool usecache, string compress, string encrypt, int CNT, string idxCompress, string idxEncrypt, int pageLboundKb, int pageUboundKb, int idxLboundKb, int idxUboundKb)
{
var ctlMumbo = MumboJumbo.GetControl();
IPileImplementation pile = null;
ICacheImplementation cache = null;
IPageCache pageCache = null;
if (usecache)
{
pile = new DefaultPile(App);
cache = new LocalCache(App)
{
Pile = pile,
DefaultTableOptions = new TableOptions("*")
{
CollisionMode = CollisionMode.Durable
},
PileMaxMemoryLimit = 32L * 1024 * 1024 * 1024
};
pile.Start();
cache.Start();
pageCache = new PilePageCache(cache);
}
try
{
var msData = new MemoryStream();
var msIdxId = new MemoryStream();
var msIdxCid = new MemoryStream();
var msIdxDid = new MemoryStream();
var msIdxPn = new MemoryStream();
var msIdxLt = new MemoryStream();
var msIdxLn = new MemoryStream();
var msIdxAl = new MemoryStream();
var msIdxCd = new MemoryStream();
var msIdxNt = new MemoryStream();
var msIdxAmt = new MemoryStream();
var meta = VolumeMetadataBuilder.Make("Primitive Idx", "mjumbo")
.SetVersion(1, 1)
.SetDescription("MumboJumbo testing")
.SetCompressionScheme(compress)
.SetEncryptionScheme(encrypt);
var volumeData = new DefaultVolume(CryptoMan, pageCache, meta, msData);
var IdxIdMetaBuilder = getSpecificBuilder(GdidIdxAppender.CONTENT_TYPE_IDX_GDID, idxCompress, idxEncrypt);
var IdxCidMetaBuilder = getSpecificBuilder(GuidIdxAppender.CONTENT_TYPE_IDX_GUID, idxCompress, idxEncrypt);
var IdxDidMetaBuilder = getSpecificBuilder(LongIdxAppender.CONTENT_TYPE_IDX_LONG, idxCompress, idxEncrypt);
var IdxPnMetaBuilder = getSpecificBuilder(IntIdxAppender.CONTENT_TYPE_IDX_INT, idxCompress, idxEncrypt);
var IdxLtMetaBuilder = getSpecificBuilder(DoubleIdxAppender.CONTENT_TYPE_IDX_DOUBLE, idxCompress, idxEncrypt);
var IdxLnMetaBuilder = getSpecificBuilder(DoubleIdxAppender.CONTENT_TYPE_IDX_DOUBLE, idxCompress, idxEncrypt);
var IdxAlMetaBuilder = getSpecificBuilder(DecimalIdxAppender.CONTENT_TYPE_IDX_DECIMAL, idxCompress, idxEncrypt);
var IdxCdMetaBuilder = getSpecificBuilder(DateTimeIdxAppender.CONTENT_TYPE_IDX_DATETIME, idxCompress, idxEncrypt);
var IdxNtMetaBuilder = getSpecificBuilder(StringIdxAppender.CONTENT_TYPE_IDX_STRING, idxCompress, idxEncrypt);
var IdxAmtMetaBuilder = getSpecificBuilder(AmountIdxAppender.CONTENT_TYPE_IDX_AMOUNT, idxCompress, idxEncrypt);
var volumeIdxId = new DefaultVolume(CryptoMan, IdxIdMetaBuilder, msIdxId);
var volumeIdxCid = new DefaultVolume(CryptoMan, IdxCidMetaBuilder, msIdxCid);
var volumeIdxDid = new DefaultVolume(CryptoMan, IdxDidMetaBuilder, msIdxDid);
var volumeIdxPn = new DefaultVolume(CryptoMan, IdxPnMetaBuilder, msIdxPn);
var volumeIdxLt = new DefaultVolume(CryptoMan, IdxLtMetaBuilder, msIdxLt);
var volumeIdxLn = new DefaultVolume(CryptoMan, IdxLnMetaBuilder, msIdxLn);
var volumeIdxAl = new DefaultVolume(CryptoMan, IdxAlMetaBuilder, msIdxAl);
var volumeIdxCd = new DefaultVolume(CryptoMan, IdxCdMetaBuilder, msIdxCd);
var volumeIdxNt = new DefaultVolume(CryptoMan, IdxNtMetaBuilder, msIdxNt);
var volumeIdxAmt = new DefaultVolume(CryptoMan, IdxAmtMetaBuilder, msIdxAmt);
volumeData.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(pageLboundKb, pageUboundKb) * 1024;
volumeIdxId.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxCid.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxDid.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxPn.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxLt.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxLn.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxAl.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxCd.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxNt.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
volumeIdxAmt.PageSizeBytes = Ambient.Random.NextScaledRandomInteger(idxLboundKb, idxUboundKb) * 1024;
var reader = new MumboJumboArchiveReader(volumeData);
var idxIdReader = new GdidIdxReader(volumeIdxId);
var idxCidReader = new GuidIdxReader(volumeIdxCid);
var idxDidReader = new LongIdxReader(volumeIdxDid);
var idxPnReader = new IntIdxReader(volumeIdxPn);
var idxLtReader = new DoubleIdxReader(volumeIdxLt);
var idxLnReader = new DoubleIdxReader(volumeIdxLn);
var idxAlReader = new DecimalIdxReader(volumeIdxAl);
var idxCdReader = new DateTimeIdxReader(volumeIdxCd);
var idxNtReader = new StringIdxReader(volumeIdxNt);
var idxAmtReader = new AmountIdxReader(volumeIdxAmt);
var time = Azos.Time.Timeter.StartNew();
var app = Azos.Apps.ExecutionContext.Application;
using (var aIdxId = new GdidIdxAppender(volumeIdxId, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxCid = new GuidIdxAppender(volumeIdxCid, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxDid = new LongIdxAppender(volumeIdxDid, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxPn = new IntIdxAppender(volumeIdxPn, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxLt = new DoubleIdxAppender(volumeIdxLt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxLn = new DoubleIdxAppender(volumeIdxLn, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxAl = new DecimalIdxAppender(volumeIdxAl, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxCd = new DateTimeIdxAppender(volumeIdxCd, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxNt = new StringIdxAppender(volumeIdxNt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
using (var aIdxAmt = new AmountIdxAppender(volumeIdxAmt, NOPApplication.Instance.TimeSource, NOPApplication.Instance.AppId, "dima@zhaba"))
{
using (var appender = new MumboJumboArchiveAppender(volumeData,
NOPApplication.Instance.TimeSource,
NOPApplication.Instance.AppId,
"dima@zhaba",
onPageCommit: (e, b) =>
{
aIdxId.Append(new GdidBookmark(e.ID, b));
aIdxCid.Append(new GuidBookmark(e.CorrelationId, b));
aIdxDid.Append(new LongBookmark(e.DeviceId, b));
aIdxPn.Append(new IntBookmark(e.PartNumber, b));
aIdxLt.Append(new DoubleBookmark(e.Latitude, b));
aIdxLn.Append(new DoubleBookmark(e.Longitude, b));
aIdxAl.Append(new DecimalBookmark(e.Altitude, b));
aIdxCd.Append(new DateTimeBookmark(e.CreateDate, b));
aIdxNt.Append(new StringBookmark(e.Note, b));
aIdxAmt.Append(new AmountBookmark(e.Amt, b));
}))
{
var messages = FakeRow.GenerateMany <MumboJumbo>(1, 1, (ulong)(CNT * .5) - 1);
foreach (var m in messages)
{
if (!App.Active)
{
break;
}
appender.Append(m);
}
///////////////////////////////////////////////
appender.Append(MumboJumbo.GetControl()); //we add 1 control message in the middle
///////////////////////////////////////////////
messages = FakeRow.GenerateMany <MumboJumbo>(1, 1, (ulong)(CNT * .5));
foreach (var m in messages)
{
if (!App.Active)
{
break;
}
appender.Append(m);
}
}
}
"Wrote {0} items, now reading".SeeArgs(CNT);
if (cache != null)
{
"Cache has {0} items".SeeArgs(cache.Count);
}
var gotOne = false;
// Find by GDID (ID)
foreach (var idx in idxIdReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.ID, idx.Value);
if (idx.Value == ctlMumbo.ID)
{
//data.See();
Aver.AreEqual(ctlMumbo.ID, data.ID);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find ID by {nameof(GdidIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(GdidIdxReader));
gotOne = false;
// Find by Guid (CorrelationId)
foreach (var idx in idxCidReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.CorrelationId, idx.Value);
if (idx.Value == ctlMumbo.CorrelationId)
{
//data.See();
Aver.AreEqual(ctlMumbo.CorrelationId, data.CorrelationId);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find CorrelationId by {nameof(GuidIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(GuidIdxReader));
gotOne = false;
// Find by Long (DeviceId)
foreach (var idx in idxDidReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.DeviceId, idx.Value);
if (idx.Value == ctlMumbo.DeviceId)
{
//data.See();
Aver.AreEqual(ctlMumbo.DeviceId, data.DeviceId);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find DeviceId by {nameof(LongIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(LongIdxReader));
gotOne = false;
// Find by Int (PartNumber)
foreach (var idx in idxPnReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.PartNumber, idx.Value);
if (idx.Value == ctlMumbo.PartNumber)
{
//data.See();
Aver.AreEqual(ctlMumbo.PartNumber, data.PartNumber);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find PartNumber by {nameof(IntIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(IntIdxReader));
gotOne = false;
// Find by Double (Latitude)
foreach (var idx in idxLtReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.Latitude, idx.Value);
if (idx.Value == ctlMumbo.Latitude)
{
//data.See();
Aver.AreEqual(ctlMumbo.Latitude, data.Latitude);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find Latitude by {nameof(DoubleIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(DoubleIdxReader));
gotOne = false;
// Find by Double (Longitude)
foreach (var idx in idxLnReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.Longitude, idx.Value);
if (idx.Value == ctlMumbo.Longitude)
{
//data.See();
Aver.AreEqual(ctlMumbo.Longitude, data.Longitude);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find Longitude by {nameof(DoubleIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(DoubleIdxReader));
gotOne = false;
// Find by Decimal (Altitude)
foreach (var idx in idxAlReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.Altitude, idx.Value);
if (idx.Value == ctlMumbo.Altitude)
{
//data.See();
Aver.AreEqual(ctlMumbo.Altitude, data.Altitude);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find Altitude by {nameof(DecimalIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(DecimalIdxReader));
gotOne = false;
// Find by DateTime (CreateDate)
foreach (var idx in idxCdReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.CreateDate, idx.Value);
if (idx.Value == ctlMumbo.CreateDate)
{
//data.See();
Aver.AreEqual(ctlMumbo.CreateDate, data.CreateDate);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find CreateDate by {nameof(DateTimeIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(DateTimeIdxReader));
gotOne = false;
// Find by String (Note)
foreach (var idx in idxNtReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.Note, idx.Value);
if (idx.Value == ctlMumbo.Note)
{
//data.See();
Aver.AreEqual(ctlMumbo.Note, data.Note);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find Note by {nameof(StringIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(StringIdxReader));
gotOne = false;
// Find by Amount (Amt)
foreach (var idx in idxAmtReader.All)
{
if (!App.Active)
{
break;
}
var data = reader.GetEntriesStartingAt(idx.Bookmark).First();
Aver.AreEqual(data.Amt, idx.Value);
if (idx.Value == ctlMumbo.Amt)
{
//data.See();
Aver.AreEqual(ctlMumbo.Amt, data.Amt);
gotOne = true;
break;
}
}
if (!gotOne)
{
Aver.Fail($"Failed to find Amt by {nameof(AmountIdxReader)}");
}
"Finished reading {0}".SeeArgs(nameof(AmountIdxReader));
//gotOne = false;
time.Stop();
"Did {0:n0} in {1:n1} sec at {2:n2} ops/sec\n".SeeArgs(CNT, time.ElapsedSec, CNT / time.ElapsedSec);
volumeIdxId.Dispose();
volumeIdxCid.Dispose();
volumeIdxDid.Dispose();
volumeIdxPn.Dispose();
volumeIdxLt.Dispose();
volumeIdxLn.Dispose();
volumeIdxAl.Dispose();
volumeIdxCd.Dispose();
volumeIdxNt.Dispose();
volumeIdxAmt.Dispose();
volumeData.Dispose();
}
finally
{
DisposableObject.DisposeIfDisposableAndNull(ref pageCache);
DisposableObject.DisposeIfDisposableAndNull(ref cache);
DisposableObject.DisposeIfDisposableAndNull(ref pile);
"CLOSED all volumes\n".See();
}
}
public LocalCache(IPileImplementation pile, object director, string name) : base(director)
{
m_Pile = pile;
Name = name;
}
public LocalCache(IApplicationComponent director, string name, IPileImplementation pile) : base(director)
{
m_Pile = pile;
Name = name;
}
