void WriteEnumeratorTableFields(TextWriter writer)
{
var size = ElementSizeStrategy.Get(_config.Manager.ElementSize);
if (_config.Tables.Count > 1)
{
WriteTableFields(writer, 2, null, size, false, false);
if (!_config.Manager.CacheTables)
{
writer.WriteLine();
var table = _config.Tables[0];
WriteTableFields(writer, 2, table.Index.ToString(CultureInfo.InvariantCulture), size, true, false);
for (var i = 1; i < _config.Tables.Count; i++)
{
table = _config.Tables[i];
writer.WriteLine();
WriteTableFields(writer, 2, table.Index.ToString(CultureInfo.InvariantCulture), size, true, false);
}
}
}
else
{
WriteTableFields(writer, 2, null, size, true, false);
}
}
public void None([ValueSource(nameof(_sizeValues))] int size)
{
var sizex = ElementSizeStrategy.Get((TableElementSize)size);
var cb = CompressedBlob.Compress(Compression.None, sizex, _sampleBlob1);
Assert.That(cb.Method, Is.EqualTo(Compression.None));
Assert.That(cb.ElementSize, Is.EqualTo(sizex));
Assert.That(cb.BlobSize, Is.EqualTo(sizex));
Assert.That(cb, Is.EquivalentTo(_sampleBlob1));
}
static void WriteStatistics(TextWriter writer, Statistics statistics, ElementSizeStrategy elementSize)
{
var actionsNatural = statistics.States * (statistics.TerminalColumns + statistics.NonTerminalColumns);
writer.WriteLine(" // [Statistics]");
writer.Write(" // Reductions : ");
writer.WriteLine(statistics.Reductions);
writer.Write(" // Terminals : ");
writer.Write(statistics.Terminals);
writer.Write(" (");
writer.Write(statistics.TerminalColumns);
writer.WriteLine(statistics.TerminalColumns == 1 ? " column)" : " columns)");
writer.Write(" // NonTerminals : ");
writer.Write(statistics.NonTerminals);
writer.Write(" (");
writer.Write(statistics.NonTerminalColumns);
writer.WriteLine(statistics.NonTerminalColumns == 1 ? " column)" : " columns)");
writer.Write(" // States : ");
writer.WriteLine(statistics.States);
writer.Write(" // Short Circuited : ");
writer.WriteLine(statistics.StatesShortCircuited);
writer.Write(" // With Goto Entries : ");
writer.WriteLine(statistics.StatesWithGotos);
writer.Write(" // With SR Conflicts : ");
writer.WriteLine(statistics.StatesWithSRConflicts);
writer.Write(" // Other : ");
writer.WriteLine(statistics.StatesOther);
writer.Write(" // Transition Table : ");
writer.Write(statistics.ActionsRunTime);
writer.Write("/");
writer.Write(actionsNatural);
writer.Write("(");
writer.Write(((statistics.ActionsRunTime / (decimal)actionsNatural) * 100).ToString("0.00", CultureInfo.InvariantCulture));
writer.WriteLine("%)");
writer.Write(" // Primary Offsets : ");
writer.WriteLine(statistics.States);
writer.Write(" // Goto Offsets : ");
writer.WriteLine(statistics.GotoOffsetsLen);
writer.Write(" // Actions : ");
writer.WriteLine(statistics.ActionsRunTime - statistics.States - statistics.GotoOffsetsLen);
var actionsBytes = elementSize.Size(statistics.ActionsRunTime);
var assemblyBytes = statistics.ActionsAssemblyBytes;
writer.Write(" // Memory Footprint : ");
writer.Write(actionsBytes);
writer.WriteLine(" bytes");
writer.Write(" // Assembly Footprint : ");
writer.Write(assemblyBytes);
writer.Write(" bytes (");
writer.Write(((assemblyBytes / (decimal)actionsBytes) * 100).ToString("0.00", CultureInfo.InvariantCulture));
writer.WriteLine("%)");
}
void WriteFields(TextWriter writer, ElementSizeStrategy elementSize)
{
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.Write(" readonly ");
writer.Write(elementSize.Keyword);
writer.WriteLine("[] _transitionTable;");
if (_config.Manager.Trace)
{
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" StringBuilder _traceBuilder;");
}
}
public void CTB([ValueSource(nameof(_sizeValues))] int size)
{
var sizex = ElementSizeStrategy.Get((TableElementSize)size);
var cb1 = CompressedBlob.Compress(Compression.CTB, sizex, _sampleBlob1);
Assert.That(cb1.Method, Is.EqualTo(Compression.CTB));
Assert.That(cb1.ElementSize, Is.EqualTo(sizex));
Assert.That(cb1.BlobSize, Is.EqualTo(U8SizeStrategy.Instance));
Assert.That(cb1, Is.EquivalentTo(_sampleBlob1CTB));
var cb2 = CompressedBlob.Compress(Compression.CTB, sizex, _sampleBlob2);
Assert.That(cb2.Method, Is.EqualTo(Compression.CTB));
Assert.That(cb2.ElementSize, Is.EqualTo(sizex));
Assert.That(cb2.BlobSize, Is.EqualTo(U8SizeStrategy.Instance));
Assert.That(cb2, Is.EquivalentTo(_sampleBlob2CTB));
}
void WriteInitTableSet(TextWriter writer, TableData data, ConfigTable table, int indent, string suffix)
{
var size = ElementSizeStrategy.Get(_config.Manager.ElementSize);
WriteLargeCharArray(writer, indent, "_charClassificationBoundries", suffix, 16, data.CharClassificationBoundries);
WriteLargeIntArray(writer, indent, "_charClassification", suffix, 16, data.CharClassification, size);
if (string.IsNullOrEmpty(table.Name))
{
WriteLargeIntArray(writer, indent, "_transitionTable", suffix, 16, data.TransitionTable);
}
else
{
WriteLargeIntArray(writer, indent, "_transitionTable", suffix, data.TransitionTable.Method, table.Name);
}
WriteTokenTable(writer, indent, table, suffix);
}
public void Auto([ValueSource(nameof(_sizeValues))] int size)
{
var sizex = ElementSizeStrategy.Get((TableElementSize)size);
var cb1 = CompressedBlob.Compress(Compression.Auto, sizex, _sampleBlob1);
Assert.That(cb1.Method, Is.EqualTo(Compression.Simple));
Assert.That(cb1.ElementSize, Is.EqualTo(sizex));
Assert.That(cb1.BlobSize, Is.EqualTo(sizex));
Assert.That(cb1, Is.EquivalentTo(Replace(_sampleBlob1Simple, -1, sizex.MaxValue)));
var cb2 = CompressedBlob.Compress(Compression.Auto, sizex, _sampleBlob2);
Assert.That(cb2.Method, Is.EqualTo(Compression.None));
Assert.That(cb2.ElementSize, Is.EqualTo(sizex));
Assert.That(cb2.BlobSize, Is.EqualTo(sizex));
Assert.That(cb2, Is.EquivalentTo(_sampleBlob2));
}
IEnumerable <HelperMethod> ExtractDistinctCompressionMethods(ElementSizeStrategy sizeStrategy)
{
var result = new List <HelperMethod>();
foreach (var data in _stateData)
{
var resourceName = _config.Tables[data.TableID].Name;
foreach (var method in HelperMethod.GetDecompressionMethods(data.TransitionTable.Method, sizeStrategy, !string.IsNullOrEmpty(resourceName)))
{
if (!result.Contains(method))
{
result.Add(method);
}
}
}
return(result);
}
void WriteStatistics(TextWriter writer)
{
var size = ElementSizeStrategy.Get(_config.Manager.ElementSize);
if (_stateData.Length == 1)
{
var data = _stateData[0];
WriteStatistics(writer, data.Statistics, null, size);
}
else if (_stateData.Length > 1)
{
WriteStatistics(writer, _stateData[0].Statistics, _stateData[0].TableID.ToString(CultureInfo.InvariantCulture), size);
for (var i = 1; i < _stateData.Length; i++)
{
writer.WriteLine();
WriteStatistics(writer, _stateData[i].Statistics, _stateData[i].TableID.ToString(CultureInfo.InvariantCulture), size);
}
}
}
void WriteTokenType(TextWriter writer)
{
var size = ElementSizeStrategy.Get(_config.Manager.ElementSize);
writer.Write(" public enum TokenType : ");
writer.Write(size.Keyword);
writer.WriteLine();
writer.WriteLine(" {");
writer.WriteLine(" EOF = 0,");
var tokenTypes = _config.TokenTypes;
for (var i = 0; i < tokenTypes.Count; i++)
{
writer.Write(" ");
writer.Write(tokenTypes[i]);
writer.Write(" = ");
writer.Write(i + 1);
writer.WriteLine(",");
}
writer.WriteLine(" }");
}
public static TableData ExtractData(Config config, ConfigTable table)
{
var charSets = FATools.ExtractAlphabet(table.Graph);
var charClassMap = GetCharClassMap(table.Graph, charSets);
var stateMap = GetStateMap(table.Graph);
var stateCount = table.Graph.States.Count;
var statistics = new Statistics()
{
CharClassifications = charSets.Length,
States = stateCount,
};
var data = new TableData()
{
TableID = table.Index,
Statistics = statistics,
};
{
var combined = ExtractTransitionTable(table, charSets, charClassMap, stateMap, statistics);
var offsetsSectionLen = stateCount;
var transitionTable = new int[offsetsSectionLen + combined.Count];
combined.CopyTo(transitionTable, offsetsSectionLen);
for (var i = 0; i < stateCount; i++)
{
var offset = combined.GetOffset(i);
if (offset.HasValue)
{
transitionTable[i] = offset.Value + offsetsSectionLen;
}
}
var transitionsBlob = CompressedBlob.Compress(config.Manager.TableCompression, ElementSizeStrategy.Get(config.Manager.ElementSize), transitionTable);
statistics.TransitionsRunTime = transitionTable.Length;
statistics.TransitionsAssemblyBytes = transitionsBlob.Bytes;
data.TransitionTable = transitionsBlob;
}
{
var ranges = GetRanges(charSets, charClassMap);
ExtractClasificationTable(ranges, out var boundries, out var classifications);
statistics.CharRanges = ranges.Length;
data.CharClassificationBoundries = boundries;
data.CharClassification = classifications;
data.StateMap = stateMap;
}
return(data);
}
void WriteParserClass(TextWriter writer)
{
var size = ElementSizeStrategy.Get(_config.Manager.ElementSize);
writer.Write(" ");
writer.Write(_config.Manager.Visibility == ClassVisibility.Public ? "public" : "internal");
writer.Write(" abstract class ");
writer.WriteLine(_config.Manager.ParserName);
writer.WriteLine(" {");
if (_data.NeedsTerminalMask)
{
writer.Write(" const ");
writer.Write(_config.Manager.TypeName);
writer.Write(" ColumnMask = (");
writer.Write(_config.Manager.TypeName);
writer.Write(")");
writer.Write(_data.TerminalMask);
writer.WriteLine(";");
writer.WriteLine();
}
WriteStatistics(writer, _data.Statistics, size);
WriteParserConstructor(writer);
writer.WriteLine();
if (_config.Manager.Trace)
{
WriteTraceProperty(writer);
writer.WriteLine();
}
foreach (var entryPoint in _config.EntryPoints)
{
WriteParseMethod(writer, entryPoint, _config.EntryPoints.Count != 1);
writer.WriteLine();
}
WriteParseMethod(writer);
writer.WriteLine();
WriteReductionMethod(writer);
writer.WriteLine();
if (!_config.UseErrorRecovery)
{
WriteUnexpectedTokenMethod(writer);
}
else
{
WriteReduceErrorMethod(writer);
writer.WriteLine();
WriteCanBeFollowedBy(writer);
}
writer.WriteLine();
foreach (var method in HelperMethod.GetDecompressionMethods(_data.Actions.Method, size, !string.IsNullOrEmpty(_config.TableResourceName)))
{
method.Write(writer, 2);
writer.WriteLine();
}
writer.Write(" protected abstract ");
writer.Write(_config.Manager.TypeName);
writer.Write(" GetTokenType(Type_");
writer.Write(_config.TerminalType.Label.Text);
writer.WriteLine(" terminal);");
writer.WriteLine();
foreach (var method in GetAbstractMethods(_config))
{
WriteReductionMethod(writer, method);
}
if (_config.Manager.Trace)
{
writer.WriteLine();
WriteTraceMethod(writer);
writer.WriteLine();
WriteReductionStringMethod(writer);
}
writer.WriteLine();
WriteFields(writer, size);
writer.WriteLine();
WriteStateStruct(writer);
writer.WriteLine();
WriteTableCacheMembers(writer);
writer.WriteLine(" }");
}
static void WriteStatistics(TextWriter writer, Statistics statistics, string suffix, ElementSizeStrategy elementSize)
{
var transitionsNatural = statistics.States * statistics.CharClassifications;
writer.Write(" // [Statistics");
if (!string.IsNullOrEmpty(suffix))
{
writer.Write(" Table ");
writer.Write(suffix);
}
writer.WriteLine("]");
writer.Write(" // Char Classifications : ");
writer.WriteLine(statistics.CharClassifications);
writer.Write(" // Char Ranges : ");
writer.WriteLine(statistics.CharRanges);
writer.Write(" // States : ");
writer.WriteLine(statistics.States);
writer.Write(" // Terminal : ");
writer.WriteLine(statistics.StatesTerminal);
writer.Write(" // Transition Table : ");
writer.Write(statistics.TransitionsRunTime);
writer.Write("/");
writer.Write(transitionsNatural);
writer.Write(" (");
writer.Write(((statistics.TransitionsRunTime / (decimal)transitionsNatural) * 100).ToString("0.00", CultureInfo.InvariantCulture));
writer.WriteLine("%)");
writer.Write(" // Offsets : ");
writer.WriteLine(statistics.States);
writer.Write(" // Actions : ");
writer.WriteLine(statistics.TransitionsRunTime - statistics.States);
var boundryBytes = statistics.CharRanges << 1;
var classificationBytes = elementSize.Size(statistics.CharRanges);
var transitionBytes = elementSize.Size(statistics.TransitionsRunTime);
var tokenTypeBytes = elementSize.Size(statistics.States);
var memoryFootprint = boundryBytes + classificationBytes + transitionBytes + tokenTypeBytes;
var assemblyFootprint = boundryBytes + classificationBytes + statistics.TransitionsAssemblyBytes + tokenTypeBytes;
writer.Write(" // Memory Footprint : ");
writer.Write(memoryFootprint);
writer.WriteLine(" bytes");
writer.Write(" // Boundries : ");
writer.Write(boundryBytes);
writer.WriteLine(" bytes");
writer.Write(" // Classifications : ");
writer.Write(classificationBytes);
writer.WriteLine(" bytes");
writer.Write(" // Transitions : ");
writer.Write(transitionBytes);
writer.WriteLine(" bytes");
writer.Write(" // Token Types : ");
writer.Write(tokenTypeBytes);
writer.WriteLine(" bytes");
writer.Write(" // Assembly Footprint : ");
writer.Write(assemblyFootprint);
writer.Write(" bytes (");
writer.Write(((assemblyFootprint / (decimal)memoryFootprint) * 100).ToString("0.00", CultureInfo.InvariantCulture));
writer.WriteLine("%)");
}
static void WriteTableFields(TextWriter writer, int indent, string suffix, ElementSizeStrategy elementSize, bool readOnly, bool @public)
{
CodeGenHelper.WriteIndent(writer, indent);
writer.WriteLine("[DebuggerBrowsable(DebuggerBrowsableState.Never)]");
CodeGenHelper.WriteIndent(writer, indent);
if (@public)
{
writer.Write("public ");
}
if (readOnly)
{
writer.Write("readonly ");
}
writer.Write("char[] _charClassificationBoundries");
if (!string.IsNullOrEmpty(suffix))
{
writer.Write('_');
writer.Write(suffix);
}
writer.WriteLine(";");
CodeGenHelper.WriteIndent(writer, indent);
writer.WriteLine("[DebuggerBrowsable(DebuggerBrowsableState.Never)]");
CodeGenHelper.WriteIndent(writer, indent);
if (@public)
{
writer.Write("public ");
}
if (readOnly)
{
writer.Write("readonly ");
}
writer.Write(elementSize.Keyword);
writer.Write("[] _charClassification");
if (!string.IsNullOrEmpty(suffix))
{
writer.Write('_');
writer.Write(suffix);
}
writer.WriteLine(";");
CodeGenHelper.WriteIndent(writer, indent);
writer.WriteLine("[DebuggerBrowsable(DebuggerBrowsableState.Never)]");
CodeGenHelper.WriteIndent(writer, indent);
if (@public)
{
writer.Write("public ");
}
if (readOnly)
{
writer.Write("readonly ");
}
writer.Write(elementSize.Keyword);
writer.Write("[] _transitionTable");
if (!string.IsNullOrEmpty(suffix))
{
writer.Write('_');
writer.Write(suffix);
}
writer.WriteLine(";");
CodeGenHelper.WriteIndent(writer, indent);
writer.WriteLine("[DebuggerBrowsable(DebuggerBrowsableState.Never)]");
CodeGenHelper.WriteIndent(writer, indent);
if (@public)
{
writer.Write("public ");
}
if (readOnly)
{
writer.Write("readonly ");
}
writer.Write("TokenType[] _tokenTypes");
if (!string.IsNullOrEmpty(suffix))
{
writer.Write('_');
writer.Write(suffix);
}
writer.WriteLine(";");
}
void WriteScannerClass(TextWriter writer)
{
var size = ElementSizeStrategy.Get(_config.Manager.ElementSize);
writer.Write(" ");
writer.Write(_config.Manager.Visibility == ClassVisibility.Public ? "public" : "internal");
writer.Write(" abstract class ");
writer.Write(_config.Manager.ClassName);
writer.WriteLine("<TToken> : IEnumerable<TToken>, IEnumerator<TToken>");
writer.WriteLine(" where TToken : class");
writer.WriteLine(" {");
WriteTokenType(writer);
if (_config.States.Count > 1)
{
writer.WriteLine();
WriteScannerState(writer);
}
writer.WriteLine();
WriteStatistics(writer);
WriteConstructor(writer);
writer.WriteLine();
WriteCopyConstructor(writer);
writer.WriteLine();
WriteDestructor(writer);
writer.WriteLine();
writer.WriteLine(" protected abstract TToken NewToken(TokenType type, string expressionString, int startPosition, int length);");
writer.WriteLine();
WriteNewScanner(writer);
writer.WriteLine();
WriteStartOfLine(writer);
foreach (var method in ExtractDistinctCompressionMethods(size))
{
writer.WriteLine();
method.Write(writer, 2);
}
if (_config.States.Count > 1)
{
writer.WriteLine();
WriteStateManagementMethods(writer);
}
writer.WriteLine();
WriteClassifyChar(writer);
writer.WriteLine();
WriteEnumerableMembers(writer);
writer.WriteLine();
WriteEnumeratorMembers(writer);
writer.WriteLine();
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" TToken _currentToken;");
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" int _nextCharPosition;");
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" int _started;");
if (!_fixedStartState.HasValue)
{
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" int _startState;");
}
if (_config.States.Count > 1)
{
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" ScannerState _currentState;");
writer.WriteLine();
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" readonly Stack<ScannerState> _state = new Stack<ScannerState>();");
}
else
{
writer.WriteLine();
}
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" List<int> _solIndicies = new List<int>();");
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" readonly string _expressionString;");
writer.WriteLine();
WriteEnumeratorTableFields(writer);
if (_config.Manager.CacheTables)
{
writer.WriteLine();
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" readonly TableCache _cache;");
writer.WriteLine();
WriteTableCacheClass(writer);
}
writer.WriteLine(" }");
}
public void Get()
{
Assert.That(ElementSizeStrategy.Get(TableElementSize.Byte), Is.SameAs(U8SizeStrategy.Instance));
Assert.That(ElementSizeStrategy.Get(TableElementSize.Short), Is.SameAs(U16SizeStrategy.Instance));
Assert.That(ElementSizeStrategy.Get(TableElementSize.Int), Is.SameAs(U32SizeStrategy.Instance));
}
static CompressedBlob ExtractTransitionTableBlob(Config config, Statistics statistics, TableData data)
{
var graph = config.Graph.Graph;
var actionsList = new int[data.StateMap.Count];
var lastGotoStateIndex = data.StateMap.Count;
var fragments = new List <TableFragment>();
foreach (var state in graph.States)
{
var index = data.StateMap[state];
var transitionRow = ExtractTransitionRow(data, state);
var gotoRow = ExtractGotoRow(data, state);
if (transitionRow.Fragment == null)
{
actionsList[index] = transitionRow.ShortCircuitReduction;
statistics.StatesShortCircuited++;
if (gotoRow != null)
{
statistics.StatesWithGotos++;
}
}
else
{
fragments.Add(transitionRow.Fragment);
if (transitionRow.HasReduction && transitionRow.HasShift)
{
statistics.StatesWithSRConflicts++;
}
if (gotoRow != null)
{
statistics.StatesWithGotos++;
}
else if (!transitionRow.HasReduction || !transitionRow.HasShift)
{
statistics.StatesOther++;
}
}
if (gotoRow != null)
{
fragments.Add(gotoRow);
statistics.NonTerminalColumns = Math.Max(statistics.NonTerminalColumns, gotoRow.Count);
lastGotoStateIndex = data.StateMap[state] + 1;
}
}
var combined = TableFragment.Combine(fragments);
var offsetSectionLen = graph.States.Count + lastGotoStateIndex;
Array.Resize(ref actionsList, offsetSectionLen + combined.Count);
for (var i = 0; i < graph.States.Count << 1; i++)
{
var offset = combined.GetOffset(i);
if (offset.HasValue)
{
actionsList[i] = offset.Value + offsetSectionLen;
}
}
combined.CopyTo(actionsList, offsetSectionLen);
var actionsBlob = CompressedBlob.Compress(config.Manager.TableCompression, ElementSizeStrategy.Get(config.Manager.ElementSize), actionsList);
statistics.ActionsRunTime = actionsList.Length;
statistics.ActionsAssemblyBytes = actionsBlob.Bytes;
statistics.GotoOffsetsLen = lastGotoStateIndex;
return(actionsBlob);
}
static void WriteLargeIntArray(TextWriter writer, int indent, string name, string suffix, int wrap, IList <int> data, ElementSizeStrategy elementSize)
{
CodeGenHelper.WriteIndent(writer, indent);
writer.Write(name);
if (!string.IsNullOrEmpty(suffix))
{
writer.Write('_');
writer.Write(suffix);
}
writer.Write(" = ");
CodeGenHelper.WriteLargeIntArray(writer, indent, wrap, data, elementSize);
writer.WriteLine(";");
}