void WriteEnumeratorTableFields(TextWriter writer)
{
var size = ElementSizeStrategy.Get(_config.Manager.ElementSize);
if (_config.Tables.Count > 1)
{
WriteTableFields(writer, 2, null, size, false, false);
if (!_config.Manager.CacheTables)
{
writer.WriteLine();
var table = _config.Tables[0];
WriteTableFields(writer, 2, table.Index.ToString(CultureInfo.InvariantCulture), size, true, false);
for (var i = 1; i < _config.Tables.Count; i++)
{
table = _config.Tables[i];
writer.WriteLine();
WriteTableFields(writer, 2, table.Index.ToString(CultureInfo.InvariantCulture), size, true, false);
}
}
}
else
{
WriteTableFields(writer, 2, null, size, true, false);
}
}
public void None([ValueSource(nameof(_sizeValues))] int size)
{
var sizex = ElementSizeStrategy.Get((TableElementSize)size);
var cb = CompressedBlob.Compress(Compression.None, sizex, _sampleBlob1);
Assert.That(cb.Method, Is.EqualTo(Compression.None));
Assert.That(cb.ElementSize, Is.EqualTo(sizex));
Assert.That(cb.BlobSize, Is.EqualTo(sizex));
Assert.That(cb, Is.EquivalentTo(_sampleBlob1));
}
public void Auto([ValueSource(nameof(_sizeValues))] int size)
{
var sizex = ElementSizeStrategy.Get((TableElementSize)size);
var cb1 = CompressedBlob.Compress(Compression.Auto, sizex, _sampleBlob1);
Assert.That(cb1.Method, Is.EqualTo(Compression.Simple));
Assert.That(cb1.ElementSize, Is.EqualTo(sizex));
Assert.That(cb1.BlobSize, Is.EqualTo(sizex));
Assert.That(cb1, Is.EquivalentTo(Replace(_sampleBlob1Simple, -1, sizex.MaxValue)));
var cb2 = CompressedBlob.Compress(Compression.Auto, sizex, _sampleBlob2);
Assert.That(cb2.Method, Is.EqualTo(Compression.None));
Assert.That(cb2.ElementSize, Is.EqualTo(sizex));
Assert.That(cb2.BlobSize, Is.EqualTo(sizex));
Assert.That(cb2, Is.EquivalentTo(_sampleBlob2));
}
public void CTB([ValueSource(nameof(_sizeValues))] int size)
{
var sizex = ElementSizeStrategy.Get((TableElementSize)size);
var cb1 = CompressedBlob.Compress(Compression.CTB, sizex, _sampleBlob1);
Assert.That(cb1.Method, Is.EqualTo(Compression.CTB));
Assert.That(cb1.ElementSize, Is.EqualTo(sizex));
Assert.That(cb1.BlobSize, Is.EqualTo(U8SizeStrategy.Instance));
Assert.That(cb1, Is.EquivalentTo(_sampleBlob1CTB));
var cb2 = CompressedBlob.Compress(Compression.CTB, sizex, _sampleBlob2);
Assert.That(cb2.Method, Is.EqualTo(Compression.CTB));
Assert.That(cb2.ElementSize, Is.EqualTo(sizex));
Assert.That(cb2.BlobSize, Is.EqualTo(U8SizeStrategy.Instance));
Assert.That(cb2, Is.EquivalentTo(_sampleBlob2CTB));
}
void WriteInitTableSet(TextWriter writer, TableData data, ConfigTable table, int indent, string suffix)
{
var size = ElementSizeStrategy.Get(_config.Manager.ElementSize);
WriteLargeCharArray(writer, indent, "_charClassificationBoundries", suffix, 16, data.CharClassificationBoundries);
WriteLargeIntArray(writer, indent, "_charClassification", suffix, 16, data.CharClassification, size);
if (string.IsNullOrEmpty(table.Name))
{
WriteLargeIntArray(writer, indent, "_transitionTable", suffix, 16, data.TransitionTable);
}
else
{
WriteLargeIntArray(writer, indent, "_transitionTable", suffix, data.TransitionTable.Method, table.Name);
}
WriteTokenTable(writer, indent, table, suffix);
}
void WriteStatistics(TextWriter writer)
{
var size = ElementSizeStrategy.Get(_config.Manager.ElementSize);
if (_stateData.Length == 1)
{
var data = _stateData[0];
WriteStatistics(writer, data.Statistics, null, size);
}
else if (_stateData.Length > 1)
{
WriteStatistics(writer, _stateData[0].Statistics, _stateData[0].TableID.ToString(CultureInfo.InvariantCulture), size);
for (var i = 1; i < _stateData.Length; i++)
{
writer.WriteLine();
WriteStatistics(writer, _stateData[i].Statistics, _stateData[i].TableID.ToString(CultureInfo.InvariantCulture), size);
}
}
}
void WriteTokenType(TextWriter writer)
{
var size = ElementSizeStrategy.Get(_config.Manager.ElementSize);
writer.Write(" public enum TokenType : ");
writer.Write(size.Keyword);
writer.WriteLine();
writer.WriteLine(" {");
writer.WriteLine(" EOF = 0,");
var tokenTypes = _config.TokenTypes;
for (var i = 0; i < tokenTypes.Count; i++)
{
writer.Write(" ");
writer.Write(tokenTypes[i]);
writer.Write(" = ");
writer.Write(i + 1);
writer.WriteLine(",");
}
writer.WriteLine(" }");
}
static CompressedBlob ExtractTransitionTableBlob(Config config, Statistics statistics, TableData data)
{
var graph = config.Graph.Graph;
var actionsList = new int[data.StateMap.Count];
var lastGotoStateIndex = data.StateMap.Count;
var fragments = new List <TableFragment>();
foreach (var state in graph.States)
{
var index = data.StateMap[state];
var transitionRow = ExtractTransitionRow(data, state);
var gotoRow = ExtractGotoRow(data, state);
if (transitionRow.Fragment == null)
{
actionsList[index] = transitionRow.ShortCircuitReduction;
statistics.StatesShortCircuited++;
if (gotoRow != null)
{
statistics.StatesWithGotos++;
}
}
else
{
fragments.Add(transitionRow.Fragment);
if (transitionRow.HasReduction && transitionRow.HasShift)
{
statistics.StatesWithSRConflicts++;
}
if (gotoRow != null)
{
statistics.StatesWithGotos++;
}
else if (!transitionRow.HasReduction || !transitionRow.HasShift)
{
statistics.StatesOther++;
}
}
if (gotoRow != null)
{
fragments.Add(gotoRow);
statistics.NonTerminalColumns = Math.Max(statistics.NonTerminalColumns, gotoRow.Count);
lastGotoStateIndex = data.StateMap[state] + 1;
}
}
var combined = TableFragment.Combine(fragments);
var offsetSectionLen = graph.States.Count + lastGotoStateIndex;
Array.Resize(ref actionsList, offsetSectionLen + combined.Count);
for (var i = 0; i < graph.States.Count << 1; i++)
{
var offset = combined.GetOffset(i);
if (offset.HasValue)
{
actionsList[i] = offset.Value + offsetSectionLen;
}
}
combined.CopyTo(actionsList, offsetSectionLen);
var actionsBlob = CompressedBlob.Compress(config.Manager.TableCompression, ElementSizeStrategy.Get(config.Manager.ElementSize), actionsList);
statistics.ActionsRunTime = actionsList.Length;
statistics.ActionsAssemblyBytes = actionsBlob.Bytes;
statistics.GotoOffsetsLen = lastGotoStateIndex;
return(actionsBlob);
}
public static TableData ExtractData(Config config, ConfigTable table)
{
var charSets = FATools.ExtractAlphabet(table.Graph);
var charClassMap = GetCharClassMap(table.Graph, charSets);
var stateMap = GetStateMap(table.Graph);
var stateCount = table.Graph.States.Count;
var statistics = new Statistics()
{
CharClassifications = charSets.Length,
States = stateCount,
};
var data = new TableData()
{
TableID = table.Index,
Statistics = statistics,
};
{
var combined = ExtractTransitionTable(table, charSets, charClassMap, stateMap, statistics);
var offsetsSectionLen = stateCount;
var transitionTable = new int[offsetsSectionLen + combined.Count];
combined.CopyTo(transitionTable, offsetsSectionLen);
for (var i = 0; i < stateCount; i++)
{
var offset = combined.GetOffset(i);
if (offset.HasValue)
{
transitionTable[i] = offset.Value + offsetsSectionLen;
}
}
var transitionsBlob = CompressedBlob.Compress(config.Manager.TableCompression, ElementSizeStrategy.Get(config.Manager.ElementSize), transitionTable);
statistics.TransitionsRunTime = transitionTable.Length;
statistics.TransitionsAssemblyBytes = transitionsBlob.Bytes;
data.TransitionTable = transitionsBlob;
}
{
var ranges = GetRanges(charSets, charClassMap);
ExtractClasificationTable(ranges, out var boundries, out var classifications);
statistics.CharRanges = ranges.Length;
data.CharClassificationBoundries = boundries;
data.CharClassification = classifications;
data.StateMap = stateMap;
}
return(data);
}
void WriteParserClass(TextWriter writer)
{
var size = ElementSizeStrategy.Get(_config.Manager.ElementSize);
writer.Write(" ");
writer.Write(_config.Manager.Visibility == ClassVisibility.Public ? "public" : "internal");
writer.Write(" abstract class ");
writer.WriteLine(_config.Manager.ParserName);
writer.WriteLine(" {");
if (_data.NeedsTerminalMask)
{
writer.Write(" const ");
writer.Write(_config.Manager.TypeName);
writer.Write(" ColumnMask = (");
writer.Write(_config.Manager.TypeName);
writer.Write(")");
writer.Write(_data.TerminalMask);
writer.WriteLine(";");
writer.WriteLine();
}
WriteStatistics(writer, _data.Statistics, size);
WriteParserConstructor(writer);
writer.WriteLine();
if (_config.Manager.Trace)
{
WriteTraceProperty(writer);
writer.WriteLine();
}
foreach (var entryPoint in _config.EntryPoints)
{
WriteParseMethod(writer, entryPoint, _config.EntryPoints.Count != 1);
writer.WriteLine();
}
WriteParseMethod(writer);
writer.WriteLine();
WriteReductionMethod(writer);
writer.WriteLine();
if (!_config.UseErrorRecovery)
{
WriteUnexpectedTokenMethod(writer);
}
else
{
WriteReduceErrorMethod(writer);
writer.WriteLine();
WriteCanBeFollowedBy(writer);
}
writer.WriteLine();
foreach (var method in HelperMethod.GetDecompressionMethods(_data.Actions.Method, size, !string.IsNullOrEmpty(_config.TableResourceName)))
{
method.Write(writer, 2);
writer.WriteLine();
}
writer.Write(" protected abstract ");
writer.Write(_config.Manager.TypeName);
writer.Write(" GetTokenType(Type_");
writer.Write(_config.TerminalType.Label.Text);
writer.WriteLine(" terminal);");
writer.WriteLine();
foreach (var method in GetAbstractMethods(_config))
{
WriteReductionMethod(writer, method);
}
if (_config.Manager.Trace)
{
writer.WriteLine();
WriteTraceMethod(writer);
writer.WriteLine();
WriteReductionStringMethod(writer);
}
writer.WriteLine();
WriteFields(writer, size);
writer.WriteLine();
WriteStateStruct(writer);
writer.WriteLine();
WriteTableCacheMembers(writer);
writer.WriteLine(" }");
}
public void Get()
{
Assert.That(ElementSizeStrategy.Get(TableElementSize.Byte), Is.SameAs(U8SizeStrategy.Instance));
Assert.That(ElementSizeStrategy.Get(TableElementSize.Short), Is.SameAs(U16SizeStrategy.Instance));
Assert.That(ElementSizeStrategy.Get(TableElementSize.Int), Is.SameAs(U32SizeStrategy.Instance));
}
void WriteScannerClass(TextWriter writer)
{
var size = ElementSizeStrategy.Get(_config.Manager.ElementSize);
writer.Write(" ");
writer.Write(_config.Manager.Visibility == ClassVisibility.Public ? "public" : "internal");
writer.Write(" abstract class ");
writer.Write(_config.Manager.ClassName);
writer.WriteLine("<TToken> : IEnumerable<TToken>, IEnumerator<TToken>");
writer.WriteLine(" where TToken : class");
writer.WriteLine(" {");
WriteTokenType(writer);
if (_config.States.Count > 1)
{
writer.WriteLine();
WriteScannerState(writer);
}
writer.WriteLine();
WriteStatistics(writer);
WriteConstructor(writer);
writer.WriteLine();
WriteCopyConstructor(writer);
writer.WriteLine();
WriteDestructor(writer);
writer.WriteLine();
writer.WriteLine(" protected abstract TToken NewToken(TokenType type, string expressionString, int startPosition, int length);");
writer.WriteLine();
WriteNewScanner(writer);
writer.WriteLine();
WriteStartOfLine(writer);
foreach (var method in ExtractDistinctCompressionMethods(size))
{
writer.WriteLine();
method.Write(writer, 2);
}
if (_config.States.Count > 1)
{
writer.WriteLine();
WriteStateManagementMethods(writer);
}
writer.WriteLine();
WriteClassifyChar(writer);
writer.WriteLine();
WriteEnumerableMembers(writer);
writer.WriteLine();
WriteEnumeratorMembers(writer);
writer.WriteLine();
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" TToken _currentToken;");
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" int _nextCharPosition;");
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" int _started;");
if (!_fixedStartState.HasValue)
{
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" int _startState;");
}
if (_config.States.Count > 1)
{
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" ScannerState _currentState;");
writer.WriteLine();
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" readonly Stack<ScannerState> _state = new Stack<ScannerState>();");
}
else
{
writer.WriteLine();
}
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" List<int> _solIndicies = new List<int>();");
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" readonly string _expressionString;");
writer.WriteLine();
WriteEnumeratorTableFields(writer);
if (_config.Manager.CacheTables)
{
writer.WriteLine();
writer.WriteLine(" [DebuggerBrowsable(DebuggerBrowsableState.Never)]");
writer.WriteLine(" readonly TableCache _cache;");
writer.WriteLine();
WriteTableCacheClass(writer);
}
writer.WriteLine(" }");
}