static public QuotedFunction ApplyMacros(INameLookup names, QuotedFunction f)
{
CatExpr list = new CatExpr(f.GetSubFxns().ToArray());
MetaCat.ApplyMacros(names, list);
return(new QuotedFunction(list));
}
public StreamNameLookupMetastreamDecorator(
INameLookup <StreamId> wrapped,
IMetastreamLookup <StreamId> metastreams)
{
_wrapped = wrapped;
_metastreams = metastreams;
}
public LogV3SystemStreams(
IMetastreamLookup <StreamId> metastreams,
INameLookup <StreamId> streamNames)
{
_metastreams = metastreams;
_streamNames = streamNames;
}
public void Replace(INameLookup names, CatExpr fxns, List <AstMacroTerm> pattern)
{
CatExpr pNewFxns = PatternToFxns(names, pattern);
// For debugging purposes only
if (Config.gbShowRewritingRuleApplications)
{
string sFrom = "";
for (int i = mnFxnIndex; i < mnFxnIndex + mnFxnCount; ++i)
{
if (i > mnFxnIndex)
{
sFrom += " ";
}
sFrom += fxns[i].msName;
}
string sTo = "";
for (int i = 0; i < pNewFxns.Count; ++i)
{
if (i > 0)
{
sTo += " ";
}
sTo += pNewFxns[i].msName;
}
MainClass.WriteLine("Rewriting { " + sFrom + " } to { " + sTo + " }");
}
if (mnFxnIndex < fxns.Count)
{
fxns.RemoveRange(mnFxnIndex, mnFxnCount);
}
fxns.InsertRange(mnFxnIndex, pNewFxns);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() {
IndexDirectory = GetFilePathFor("index"),
});
_provider = _logFormat.StreamNamesProvider;
_publisher = new InMemoryBus("publisher");
_tfReader = new FakeInMemoryTfReader(RecordOffset);
_tableIndex = new FakeInMemoryTableIndex <TStreamId>();
_provider.SetTableIndex(_tableIndex);
_readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", 5, 100,
() => _tfReader);
_indexBackend = new IndexBackend <TStreamId>(_readerPool, 100000, 100000);
_streamIds = _logFormat.StreamIds;
_validator = _logFormat.StreamIdValidator;
var emptyStreamId = _logFormat.EmptyStreamId;
_sizer = _logFormat.StreamIdSizer;
_indexReader = new IndexReader <TStreamId>(_indexBackend, _tableIndex, _provider, _validator,
_logFormat.StreamExistenceFilterReader, new StreamMetadata(maxCount: 100000), 100, false);
_streamNames = _logFormat.StreamNames;
_systemStreams = _logFormat.SystemStreams;
_indexWriter = new IndexWriter <TStreamId>(_indexBackend, _indexReader, _streamIds, _streamNames,
_systemStreams, emptyStreamId, _sizer);
_indexCommitter = new IndexCommitter <TStreamId>(_publisher, _indexBackend, _indexReader, _tableIndex,
_logFormat.StreamNameIndexConfirmer, _streamNames, _logFormat.EventTypeIndexConfirmer, _logFormat.EventTypes,
_systemStreams, _logFormat.StreamExistenceFilter, _logFormat.StreamExistenceFilterInitializer, new InMemoryCheckpoint(-1), false);
WriteEvents();
}
public AllReader(IIndexBackend backend, IIndexCommitter indexCommitter, INameLookup <TStreamId> streamNames)
{
Ensure.NotNull(backend, "backend");
Ensure.NotNull(indexCommitter, "indexCommitter");
Ensure.NotNull(indexCommitter, nameof(streamNames));
_backend = backend;
_indexCommitter = indexCommitter;
_streamNames = streamNames;
}
public ReadIndex(IPublisher bus,
ObjectPool <ITransactionFileReader> readerPool,
ITableIndex <TStreamId> tableIndex,
INameIndexConfirmer <TStreamId> streamNameIndex,
IValueLookup <TStreamId> streamIds,
IStreamNamesProvider <TStreamId> streamNamesProvider,
TStreamId emptyStreamName,
IValidator <TStreamId> streamIdValidator,
ISizer <TStreamId> sizer,
INameExistenceFilter streamExistenceFilter,
IExistenceFilterReader <TStreamId> streamExistenceFilterReader,
INameIndexConfirmer <TStreamId> eventTypeIndex,
int streamInfoCacheCapacity,
bool additionalCommitChecks,
long metastreamMaxCount,
int hashCollisionReadLimit,
bool skipIndexScanOnReads,
IReadOnlyCheckpoint replicationCheckpoint,
ICheckpoint indexCheckpoint)
{
Ensure.NotNull(bus, "bus");
Ensure.NotNull(readerPool, "readerPool");
Ensure.NotNull(tableIndex, "tableIndex");
Ensure.NotNull(streamIds, nameof(streamIds));
Ensure.NotNull(streamNamesProvider, nameof(streamNamesProvider));
Ensure.NotNull(streamIdValidator, nameof(streamIdValidator));
Ensure.NotNull(sizer, nameof(sizer));
Ensure.NotNull(streamExistenceFilter, nameof(streamExistenceFilter));
Ensure.NotNull(streamExistenceFilterReader, nameof(streamExistenceFilterReader));
Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity");
Ensure.Positive(metastreamMaxCount, "metastreamMaxCount");
Ensure.NotNull(replicationCheckpoint, "replicationCheckpoint");
Ensure.NotNull(indexCheckpoint, "indexCheckpoint");
var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount);
var indexBackend = new IndexBackend <TStreamId>(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity);
_indexReader = new IndexReader <TStreamId>(indexBackend, tableIndex, streamNamesProvider, streamIdValidator,
streamExistenceFilterReader, metastreamMetadata, hashCollisionReadLimit, skipIndexScanOnReads);
_streamIds = streamIds;
_streamNames = streamNamesProvider.StreamNames;
var systemStreams = streamNamesProvider.SystemStreams;
var eventTypeNames = streamNamesProvider.EventTypes;
var streamExistenceFilterInitializer = streamNamesProvider.StreamExistenceFilterInitializer;
_indexWriter = new IndexWriter <TStreamId>(indexBackend, _indexReader, _streamIds, _streamNames, systemStreams, emptyStreamName, sizer);
_indexCommitter = new IndexCommitter <TStreamId>(bus, indexBackend, _indexReader, tableIndex, streamNameIndex,
_streamNames, eventTypeIndex, eventTypeNames, systemStreams, streamExistenceFilter,
streamExistenceFilterInitializer, indexCheckpoint, additionalCommitChecks);
_allReader = new AllReader <TStreamId>(indexBackend, _indexCommitter, _streamNames, eventTypeNames);
}
private static void ApplyMacrosInner(INameLookup names, CatExpr fxns)
{
// Recursively apply macros for all quotations.
for (int i = 0; i < fxns.Count; ++i)
{
if (fxns[i] is PushFunction)
{
PushFunction qf = fxns[i] as PushFunction;
CatExpr tmp = new CatExpr(qf.GetChildren());
ApplyMacros(names, tmp);
fxns[i] = new PushFunction(tmp);
}
}
// This could be done multiple time
List <MacroMatch> matches = new List <MacroMatch>();
// The peephole is the maximum size of the range of functions that we will consider
// for rewriting. This helps to reduces the overall complexity of the algorithm.
//int nPeephole = 20;
// This is the maximum size of the sub-expression that will be considered for matching.
int nMaxSubExpr = 10;
// Find matches
int nLastMatchPos = -1;
for (int nPos = 0; nPos < fxns.Count; ++nPos)
{
string s = fxns[nPos].msName;
if (mMacros.ContainsKey(s))
{
foreach (AstMacro m in mMacros[s])
{
MacroMatch match = MacroMatch.Create(m, fxns, nLastMatchPos, nPos, nMaxSubExpr);
if (match != null)
{
nLastMatchPos = nPos;
matches.Add(match);
}
}
}
}
// Replace matches
for (int i = matches.Count - 1; i >= 0; --i)
{
MacroMatch m = matches[i];
List <AstMacroTerm> pattern = m.mMacro.mDest.mPattern;
m.Replace(names, fxns, pattern);
}
}
/// <summary>
/// This is a simple yet effective combination of optimizations.
/// </summary>
static public QuotedFunction Optimize(INameLookup names, QuotedFunction qf)
{
//qf = ApplyMacros(qf);
//qf = ApplyMacros(qf);
//qf = Expand(qf);
//qf = ApplyMacros(qf);
//qf = PartialEval(qf);
//qf = Expand(qf);
//qf = ReplaceSimpleQuotations(qf);
qf = ApplyMacros(names, qf);
qf = ExpandInline(qf, 4);
qf = ApplyMacros(names, qf);
return(qf);
}
/// <summary>
/// This is a simple yet effective combination of optimizations.
/// </summary>
static public QuotedFunction Optimize(INameLookup names, QuotedFunction qf)
{
//qf = ApplyMacros(qf);
//qf = ApplyMacros(qf);
//qf = Expand(qf);
//qf = ApplyMacros(qf);
//qf = PartialEval(qf);
//qf = Expand(qf);
//qf = ReplaceSimpleQuotations(qf);
qf = ApplyMacros(names, qf);
qf = ExpandInline(qf, 4);
qf = ApplyMacros(names, qf);
return qf;
}
public static void ApplyMacros(INameLookup names, CatExpr fxns)
{
if (Config.gbShowRewritingRuleApplications)
{
Output.Write("Before rewriting: ");
Output.WriteLine(fxns);
}
// repeat until a fix-point is reached (the algorithm converges)
bool bDone = false;
while (!bDone)
{
CatExpr original = fxns.Clone();
ApplyMacrosInner(names, fxns);
bDone = fxns.Equals(original);
}
if (Config.gbShowRewritingRuleApplications)
{
Output.Write("After rewriting: ");
Output.WriteLine(fxns);
}
}
public static void ApplyMacros(INameLookup names, CatExpr fxns)
{
if (Config.gbShowRewritingRuleApplications)
{
Output.Write("Before rewriting: ");
Output.WriteLine(fxns);
}
// repeat until a fix-point is reached (the algorithm converges)
bool bDone = false;
while (!bDone)
{
CatExpr original = fxns.Clone();
ApplyMacrosInner(names, fxns);
bDone = fxns.Equals(original);
}
if (Config.gbShowRewritingRuleApplications)
{
Output.Write("After rewriting: ");
Output.WriteLine(fxns);
}
}
private static void ApplyMacrosInner(INameLookup names, CatExpr fxns)
{
// Recursively apply macros for all quotations.
for (int i=0; i < fxns.Count; ++i)
{
if (fxns[i] is PushFunction)
{
PushFunction qf = fxns[i] as PushFunction;
CatExpr tmp = new CatExpr(qf.GetChildren());
ApplyMacros(names, tmp);
fxns[i] = new PushFunction(tmp);
}
}
// This could be done multiple time
List<MacroMatch> matches = new List<MacroMatch>();
// The peephole is the maximum size of the range of functions that we will consider
// for rewriting. This helps to reduces the overall complexity of the algorithm.
//int nPeephole = 20;
// This is the maximum size of the sub-expression that will be considered for matching.
int nMaxSubExpr = 10;
// Find matches
int nLastMatchPos = -1;
for (int nPos = 0; nPos < fxns.Count; ++nPos)
{
string s = fxns[nPos].msName;
if (mMacros.ContainsKey(s))
{
foreach (AstMacro m in mMacros[s])
{
MacroMatch match = MacroMatch.Create(m, fxns, nLastMatchPos, nPos, nMaxSubExpr);
if (match != null)
{
nLastMatchPos = nPos;
matches.Add(match);
}
}
}
}
// Replace matches
for (int i = matches.Count - 1; i >= 0; --i)
{
MacroMatch m = matches[i];
List<AstMacroTerm> pattern = m.mMacro.mDest.mPattern;
m.Replace(names, fxns, pattern);
}
}
public void Replace(INameLookup names, CatExpr fxns, List<AstMacroTerm> pattern)
{
CatExpr pNewFxns = PatternToFxns(names, pattern);
// For debugging purposes only
if (Config.gbShowRewritingRuleApplications)
{
string sFrom = "";
for (int i=mnFxnIndex; i < mnFxnIndex + mnFxnCount; ++i)
{
if (i > mnFxnIndex) sFrom += " ";
sFrom += fxns[i].msName;
}
string sTo = "";
for (int i=0; i < pNewFxns.Count; ++i)
{
if (i > 0) sTo += " ";
sTo += pNewFxns[i].msName;
}
Output.WriteLine("Rewriting { " + sFrom + " } to { " + sTo + " }");
}
if (mnFxnIndex < fxns.Count)
fxns.RemoveRange(mnFxnIndex, mnFxnCount);
fxns.InsertRange(mnFxnIndex, pNewFxns);
}
public CatExpr PatternToFxns(INameLookup names, List<AstMacroTerm> pattern)
{
CatExpr ret = new CatExpr();
foreach (AstMacroTerm t in pattern)
{
if (t is AstMacroTypeVar)
{
string s = t.ToString();
if (!mCapturedVars.ContainsKey(s))
throw new Exception("macro variable " + s + " was not captured");
CatExpr expr = mCapturedVars[s];
ret.AddRange(expr);
}
else if (t is AstMacroStackVar)
{
string s = (t as AstMacroStackVar).msName;
if (!mCapturedVars.ContainsKey(s))
throw new Exception("macro variable " + s + " was not captured");
CatExpr expr = mCapturedVars[s];
ret.AddRange(expr);
}
else if (t is AstMacroName)
{
string s = t.ToString();
if (s.Length < 1)
throw new Exception("internal error: macro name is empty string");
Function f = names.ThrowingLookup(s);
if (f == null)
{
if (Char.IsDigit(s[0]))
{
f = new PushInt(int.Parse(s));
}
else
{
throw new Exception("Could not find function " + s);
}
}
ret.Add(f);
}
else if (t is AstMacroQuote)
{
// TODO: handle typed terms within a quotation.
AstMacroQuote macroQuote = t as AstMacroQuote;
List<AstMacroTerm> localPattern = macroQuote.mTerms;
PushFunction q = new PushFunction(PatternToFxns(names, localPattern));
ret.Add(q);
}
else
{
throw new Exception("unrecognized macro term " + t.ToString());
}
}
return ret;
}
public EventTypeLookupSystemTypesDecorator(INameLookup <uint> wrapped)
{
_wrapped = wrapped;
}
public void InitializeWithConfirmed(INameLookup <string> source)
{
}
public LogV3StreamExistenceFilterInitializer(INameLookup <StreamId> streamNames)
{
_streamNames = streamNames;
}
public Executor(INameLookup other)
{
otherNames = other;
}
public CatExpr PatternToFxns(INameLookup names, List <AstMacroTerm> pattern)
{
CatExpr ret = new CatExpr();
foreach (AstMacroTerm t in pattern)
{
if (t is AstMacroTypeVar)
{
string s = t.ToString();
if (!mCapturedVars.ContainsKey(s))
{
throw new Exception("macro variable " + s + " was not captured");
}
CatExpr expr = mCapturedVars[s];
ret.AddRange(expr);
}
else if (t is AstMacroStackVar)
{
string s = (t as AstMacroStackVar).msName;
if (!mCapturedVars.ContainsKey(s))
{
throw new Exception("macro variable " + s + " was not captured");
}
CatExpr expr = mCapturedVars[s];
ret.AddRange(expr);
}
else if (t is AstMacroName)
{
string s = t.ToString();
if (s.Length < 1)
{
throw new Exception("internal error: macro name is empty string");
}
Function f = names.ThrowingLookup(s);
if (f == null)
{
if (Char.IsDigit(s[0]))
{
f = new PushInt(int.Parse(s));
}
else
{
throw new Exception("Could not find function " + s);
}
}
ret.Add(f);
}
else if (t is AstMacroQuote)
{
// TODO: handle typed terms within a quotation.
AstMacroQuote macroQuote = t as AstMacroQuote;
List <AstMacroTerm> localPattern = macroQuote.mTerms;
PushFunction q = new PushFunction(PatternToFxns(names, localPattern));
ret.Add(q);
}
else
{
throw new Exception("unrecognized macro term " + t.ToString());
}
}
return(ret);
}
static public QuotedFunction ApplyMacros(INameLookup names, QuotedFunction f)
{
CatExpr list = new CatExpr(f.GetSubFxns().ToArray());
MetaCat.ApplyMacros(names, list);
return new QuotedFunction(list);
}
