private SimulinkXmlParser(XElement doc)
{
this.doc = doc;
creator = new InternalParser(doc);
openSymbol = '{';
closeSymbol = '}';
}
private unsafe static void InternalParserByteSpanToUInt32_VariableLength()
{
List <ReadOnlySpan <byte> > byteSpanList = new List <ReadOnlySpan <byte> >();
foreach (string text in s_UInt32TextArray)
{
byte[] utf8ByteArray = Encoding.UTF8.GetBytes(text);
ReadOnlySpan <byte> utf8ByteSpan = new ReadOnlySpan <byte>(utf8ByteArray);
byteSpanList.Add(utf8ByteSpan);
}
EncodingData fd = EncodingData.InvariantUtf8;
TextFormat nf = new TextFormat('N');
foreach (var iteration in Benchmark.Iterations)
{
int bytesConsumed;
using (iteration.StartMeasurement())
{
for (int i = 0; i < LoadIterations; i++)
{
ReadOnlySpan <byte> utf8ByteSpan = byteSpanList[i % 10];
uint value;
InternalParser.TryParseUInt32(utf8ByteSpan, fd, nf, out value, out bytesConsumed);
DoNotIgnore(value, bytesConsumed);
}
}
}
}
[InlineData("0")] // min value
private unsafe static void InternalParserByteStarToUInt64(string text)
{
int length = text.Length;
byte[] utf8ByteArray = Encoding.UTF8.GetBytes(text);
EncodingData fd = EncodingData.InvariantUtf8;
TextFormat nf = new TextFormat('N');
foreach (var iteration in Benchmark.Iterations)
{
int bytesConsumed;
fixed(byte *utf8ByteStar = utf8ByteArray)
{
using (iteration.StartMeasurement())
{
for (int i = 0; i < LoadIterations; i++)
{
ulong value;
InternalParser.TryParseUInt64(utf8ByteStar, 0, length, fd, nf, out value, out bytesConsumed);
DoNotIgnore(value, bytesConsumed);
}
}
}
}
}
public unsafe void ParseSubstringToUInt32(string text, int index, int count, uint expectedValue, int expectedConsumed)
{
uint parsedValue;
int charsConsumed;
bool result = InternalParser.TryParseUInt32(text, index, count, out parsedValue, out charsConsumed);
Assert.True(result);
Assert.Equal(expectedValue, parsedValue);
Assert.Equal(expectedConsumed, charsConsumed);
}
public unsafe void ParseSpanOfCharToUInt32(string text, uint expectedValue, int expectedConsumed)
{
var span = new ReadOnlySpan <char>(text.ToCharArray());
uint parsedValue;
int charsConsumed;
bool result = InternalParser.TryParseUInt32(span, out parsedValue, out charsConsumed);
Assert.True(result);
Assert.Equal(expectedValue, parsedValue);
Assert.Equal(expectedConsumed, charsConsumed);
}
public Result <DyCodeModel> Parse(SourceBuffer buffer)
{
var ip = new InternalParser(new Scanner(buffer));
ip.Parse();
var cd = new DyCodeModel(ip.Root, ip.Imports.ToArray(), buffer.FileName ?? MEMORY);
return(ip.Errors.Count == 0
? Result.Create(cd)
: Result.Create(cd, ip.Errors));
}
public unsafe void ParseUtf8StringToUInt32(string text, uint expectedValue, int expectedConsumed)
{
var utf8 = new Utf8String(text);
uint parsedValue;
int bytesConsumed;
bool result = InternalParser.TryParseUInt32(utf8, out parsedValue, out bytesConsumed);
Assert.True(result);
Assert.Equal(expectedValue, parsedValue);
Assert.Equal(expectedConsumed, bytesConsumed);
}
public override IEnumerable <IRow> Extract(IUnstructuredReader input, IUpdatableRow output)
{
var parser = new InternalParser(output.Schema, this.expressions);
foreach (Stream current in input.Split((byte)'\n'))
{
var row = parser.ParseEvent(output, current);
if (row != null)
{
yield return(row);
}
}
}
public unsafe void ParseUtf8SpanOfBytesToUInt32(string text, uint expectedValue, int expectedConsumed)
{
byte[] textBuffer = Encoding.UTF8.GetBytes(text);
var span = new ReadOnlySpan <byte>(textBuffer);
uint parsedValue;
int bytesConsumed;
bool result = InternalParser.TryParseUInt32(span, TextEncoding.Utf8, out parsedValue, out bytesConsumed);
Assert.True(result);
Assert.Equal(expectedValue, parsedValue);
Assert.Equal(expectedConsumed, bytesConsumed);
}
public DeveroomGherkinDocument Parse(TextReader featureFileReader, string sourceFilePath)
{
var tokenScanner = (ITokenScanner) new HotfixTokenScanner(featureFileReader);
var tokenMatcher = new TokenMatcher(DialectProvider);
_astBuilder = new DeveroomGherkinAstBuilder(sourceFilePath, () => tokenMatcher.CurrentDialect);
var parser = new InternalParser(_astBuilder, AstBuilder.RecordStateForLine, _monitoringService);
var gherkinDocument = parser.Parse(tokenScanner, tokenMatcher);
CheckSemanticErrors(gherkinDocument);
return(gherkinDocument);
}
public static TokenType[] GetExpectedTokens(int state, IMonitoringService monitoringService)
{
var parser = new InternalParser(new NullAstBuilder(), null, monitoringService)
{
StopAtFirstError = true
};
try
{
parser.NullMatchToken(state, new Token(null, new Location()));
}
catch (UnexpectedEOFException ex)
{
return(ex.ExpectedTokenTypes.Select(type => (TokenType)Enum.Parse(typeof(TokenType), type.TrimStart('#'))).ToArray());
}
return(new TokenType[0]);
}
private static void InternalParserByteSpanToBool(string text)
{
bool value;
int bytesConsumed;
byte[] utf8ByteArray = Encoding.UTF8.GetBytes(text);
ReadOnlySpan <byte> utf8ByteSpan = new ReadOnlySpan <byte>(utf8ByteArray);
EncodingData fd = EncodingData.InvariantUtf8;
TextFormat nf = new TextFormat('N');
foreach (var iteration in Benchmark.Iterations)
{
using (iteration.StartMeasurement())
{
for (int i = 0; i < LoadIterations; i++)
{
InternalParser.TryParseBoolean(utf8ByteSpan, nf, fd, out value, out bytesConsumed);
}
}
}
}
private unsafe static void InternalParserByteStarToBool(string text)
{
bool value;
int bytesConsumed;
byte[] utf8ByteArray = Encoding.UTF8.GetBytes(text);
EncodingData fd = EncodingData.InvariantUtf8;
TextFormat nf = new TextFormat('N');
foreach (var iteration in Benchmark.Iterations)
{
fixed(byte *utf8ByteStar = utf8ByteArray)
{
using (iteration.StartMeasurement())
{
for (int i = 0; i < LoadIterations; i++)
{
InternalParser.TryParseBoolean(utf8ByteStar, 0, utf8ByteArray.Length, fd, nf, out value, out bytesConsumed);
}
}
}
}
}
public override SyntaxTreeNode Parse(string text)
{
int counter = 0;
string currentInput = text;
var pResults = new List <SyntaxTreeNode>();
for (; counter < MaxTimes; ++counter)
{
var result = InternalParser.Parse(currentInput);
if (result is null)
{
break;
}
currentInput = result.Rest;
pResults.Add(result);
}
if (counter < MinTimes)
{
return(null);
}
var rest = pResults.Count != 0 ? pResults[^ 1].Rest : text;
[InlineData("0")] // min value
private unsafe static void InternalParserByteSpanToUInt32(string text)
{
byte[] utf8ByteArray = Encoding.UTF8.GetBytes(text);
ReadOnlySpan <byte> utf8ByteSpan = new ReadOnlySpan <byte>(utf8ByteArray);
EncodingData fd = EncodingData.InvariantUtf8;
TextFormat nf = new TextFormat('N');
foreach (var iteration in Benchmark.Iterations)
{
int bytesConsumed;
fixed(byte *utf8ByteStar = utf8ByteArray)
{
using (iteration.StartMeasurement())
{
for (int i = 0; i < LoadIterations; i++)
{
uint value;
InternalParser.TryParseUInt32(utf8ByteSpan, nf, fd, out value, out bytesConsumed);
DoNotIgnore(value, bytesConsumed);
}
}
}
}
}
public override IEnumerable <IRow> Extract(IUnstructuredReader input, IUpdatableRow output)
{
var parser = new InternalParser(output.Schema, this.expressions);
int idxParseError;
bool hasParseError = HasColumnOfType(output.Schema, "ParseError", typeof(string), out idxParseError);
foreach (Stream current in input.Split((byte)'\n'))
{
IRow row;
try
{
if (hasParseError)
{
output.Set <string>(idxParseError, null);
}
row = parser.ParseEvent(output, current);
}
catch (Exception e)
{
if (hasParseError)
{
output.Set <string>(idxParseError, $"ParseError: {e.Message}");
row = output.AsReadOnly();
}
else
{
row = null;
}
}
if (row != null)
{
yield return(row);
}
}
}
public override void InitGrammar(Grammar grammar, string ruleName)
{
Grammar = grammar;
RuleName = ruleName;
InternalParser.InitGrammar(grammar, ruleName);
}
public override SyntaxTreeNode Parse(string text)
{
var pResult = InternalParser.Parse(text);
return(pResult ?? new SyntaxTreeNode(DefaultValue, text, this, null));
}
public ParserService_Should()
{
_parserService = new InternalParser();
}
private void Build(DStringLiteral node, Hints hints, CompilerContext ctx)
{
if (node.Chunks == null && NoPush(node, hints))
{
return;
}
if (node.Chunks != null)
{
cw.Type(new TypeHandle(DyType.String, true));
cw.GetMember(GetMemberNameId("concat"));
cw.FunPrep(node.Chunks.Count);
for (var i = 0; i < node.Chunks.Count; i++)
{
var c = node.Chunks[i];
if (c.IsCode)
{
var p = new InternalParser(new Scanner(SourceBuffer.FromString(c.GetContent())));
p.Parse();
if (p.Errors.Count > 0)
{
foreach (var e in p.Errors)
{
AddError(CompilerError.CodeIslandInvalid, new Location(node.Location.Line, node.Location.Column + e.Column), e.Message);
}
}
else
{
if (p.Root.Nodes == null || p.Root.Nodes.Count == 0)
{
AddError(CompilerError.CodeIslandEmpty, node.Location);
}
else
{
corrections.Push(node.Location);
Build(p.Root.Nodes[0], hints.Append(Push), ctx);
corrections.Pop();
}
}
}
else
{
cw.Push(c.GetContent());
}
cw.FunArgIx(i);
}
AddLinePragma(node);
cw.FunCall(node.Chunks.Count);
}
else
{
AddLinePragma(node);
cw.Push(node.Value);
}
PopIf(hints);
}
