public virtual void Reset(IListSource <Token> tokens, ISourceFile file, IParsingOptions parsingOptions)
{
CheckParam.IsNotNull("tokens", tokens);
CheckParam.IsNotNull("file", file);
_tokensRoot = _tokens = tokens;
_sourceFile = file;
_literalParser = parsingOptions?.LiteralParser ?? EcsLiteralHandlers.Value;
F = new LNodeFactory(file);
InputPosition = 0; // reads LT(0)
_tentativeErrors = new TentativeState(false);
}
public IListSource <LNode> Parse(ILexer <Token> input, IMessageSink msgs, IParsingOptions options)
{
if (options.PreserveComments)
{
var saver = new TriviaSaver(input, (int)TokenType.Newline);
var results = Parse(saver.Buffered(), input.SourceFile, msgs, options);
var injector = new StandardTriviaInjector(saver.TriviaList, saver.SourceFile, (int)TokenType.Newline, "/*", "*/", "//", options.Mode != ParsingMode.Expressions);
return(injector.Run(results.GetEnumerator()).Buffered());
}
else
{
var lexer = new WhitespaceFilter(input);
return(Parse(lexer.Buffered(), input.SourceFile, msgs, options));
}
}
public IListSource <LNode> Parse(ILexer <Token> input, IMessageSink msgs, IParsingOptions options)
{
if (options.PreserveComments)
{
// Filter out whitespace, including some newlines (those directly inside square brackets or parentheses)
var saver = new TriviaSaver(input, (int)TokenType.Newline);
var results = Parse(saver.Buffered(), input.SourceFile, msgs, options);
var injector = new StandardTriviaInjector(saver.TriviaList, input.SourceFile, (int)TokenType.Newline, "/*", "*/", "//", options.Mode != ParsingMode.Expressions);
injector.SLCommentSuffix = @"\\";
return(injector.Run(results.GetEnumerator()).Buffered());
}
else
{
return(Parse(new WhitespaceFilter(input).Buffered(), input.SourceFile, msgs, options));
}
}
public IListSource <LNode> Parse(ILexer <Token> input, IMessageSink msgs, IParsingOptions options)
{
var preprocessed = new EcsPreprocessor(input, options.PreserveComments);
var treeified = new TokensToTree(preprocessed, false);
var results = Parse(treeified.Buffered(), input.SourceFile, msgs, options);
if (options.PreserveComments)
{
var injector = new EcsTriviaInjector(preprocessed.TriviaList, input.SourceFile,
(int)TokenType.Newline, "/*", "*/", "//",
!options.Mode.IsOneOf <Symbol>(ParsingMode.Expressions, ParsingMode.FormalArguments));
return(injector.Run(results.GetEnumerator()).Buffered());
}
else
{
return(results);
}
}
public ILexer <Token> Tokenize(ICharSource text, string fileName, IMessageSink msgs, IParsingOptions options)
{
return(new EcsLexer(text, fileName, msgs)
{
SpacesPerTab = options.SpacesPerTab
});
}
public IListSource <LNode> Parse(IListSource <Token> input, ISourceFile file, IMessageSink msgs, IParsingOptions options)
{
// For efficiency we'd prefer to re-use our _parser object, but
// when parsing lazily, we can't re-use it because another parsing
// operation could start before this one is finished. To force
// greedy parsing, we can call ParseStmtsGreedy(), but the caller may
// prefer lazy parsing, especially if the input is large. As a
// compromise I'll check if the source file is larger than a
// certain arbitrary size. Also, ParseExprs() is always greedy
// so we can always re-use _parser in that case.
var inputType = options.Mode;
if (file.Text.TryGet(255).HasValue || inputType == ParsingMode.FormalArguments ||
inputType == ParsingMode.Types || inputType == ParsingMode.Expressions)
{
EcsParser parser = _parser;
if (parser == null)
{
_parser = parser = new EcsParser(input, file, msgs, options);
}
else
{
parser.ErrorSink = msgs ?? MessageSink.Default;
parser.Reset(input, file, options);
}
if (inputType == ParsingMode.Expressions)
{
return(parser.ParseExprs(false, allowUnassignedVarDecl: false));
}
else if (inputType == ParsingMode.FormalArguments)
{
return(parser.ParseExprs(false, allowUnassignedVarDecl: true));
}
else if (inputType == ParsingMode.Types)
{
return(LNode.List(parser.DataType()));
}
else
{
return(parser.ParseStmtsGreedy());
}
}
else
{
var parser = new EcsParser(input, file, msgs, options);
return(parser.ParseStmtsLazy().Buffered());
}
}
private static void HandleNumberValue(ref Utf8JsonReader reader, ref DdbReadStack state, IParsingOptions callbacks)
{
if (state.IsLastFrame && callbacks.HasNumberCallback)
{
callbacks.OnNumber(ref reader, ref state);
}
ref var current = ref state.GetCurrent();
public static async ValueTask <ReadResult <Document> > ReadAsync(Stream utf8Json, IParsingOptions options, bool returnCrc, CancellationToken cancellationToken = default)
{
var readerState = new JsonReaderState();
var readStack = new DdbReadStack(DdbReadStack.DefaultStackLength, options.Metadata);
try
{
options.StartParsing(ref readStack);
var buffer = ArrayPool <byte> .Shared.Rent(DefaultBufferSize);
var clearMax = 0;
try
{
var bytesInBuffer = 0;
uint crc = 0;
while (true)
{
var isFinalBlock = false;
while (true)
{
var bytesRead = await utf8Json.ReadAsync(new Memory <byte>(buffer, bytesInBuffer, buffer.Length - bytesInBuffer), cancellationToken).ConfigureAwait(false);
if (bytesRead == 0)
{
isFinalBlock = true;
break;
}
if (returnCrc)
{
crc = Crc32Algorithm.Append(crc, buffer, bytesInBuffer, bytesRead);
}
bytesInBuffer += bytesRead;
if (bytesInBuffer == buffer.Length)
{
break;
}
}
if (bytesInBuffer > clearMax)
{
clearMax = bytesInBuffer;
}
ReadCore(ref readerState, isFinalBlock, new ReadOnlySpan <byte>(buffer, 0, bytesInBuffer), ref readStack, options);
var bytesConsumed = (int)readStack.BytesConsumed;
bytesInBuffer -= bytesConsumed;
if (isFinalBlock)
{
break;
}
// Check if we need to shift or expand the buffer because there wasn't enough data to complete deserialization.
if ((uint)bytesInBuffer > ((uint)buffer.Length / 2))
{
// We have less than half the buffer available, double the buffer size.
byte[] dest = ArrayPool <byte> .Shared.Rent((buffer.Length < (int.MaxValue / 2))?buffer.Length * 2 : int.MaxValue);
// Copy the unprocessed data to the new buffer while shifting the processed bytes.
Buffer.BlockCopy(buffer, bytesConsumed, dest, 0, bytesInBuffer);
new Span <byte>(buffer, 0, clearMax).Clear();
ArrayPool <byte> .Shared.Return(buffer);
clearMax = bytesInBuffer;
buffer = dest;
}
else if (bytesInBuffer != 0)
{
// Shift the processed bytes to the beginning of buffer to make more room.
Buffer.BlockCopy(buffer, bytesConsumed, buffer, 0, bytesInBuffer);
}
}
return(new ReadResult <Document>(readStack.GetCurrent().CreateDocumentFromBuffer(), crc));
}
finally
{
new Span <byte>(buffer, 0, clearMax).Clear();
ArrayPool <byte> .Shared.Return(buffer);
}
}
finally
{
readStack.Dispose();
}
}
/// <summary>Parses a string and expects exactly one output.</summary>
/// <exception cref="InvalidOperationException">The output list was empty or contained multiple nodes.</exception>
public static LNode ParseSingle(this IParsingService parser, UString expr, IMessageSink msgs, IParsingOptions options)
{
var e = Parse(parser, expr, msgs, options);
return(Single(e));
}
public Les3Parser(IList <Token> list, ISourceFile file, IMessageSink sink, IParsingOptions options, int startIndex = 0)
: base(list, prev => new Token((int)TokenType.EOF, prev.EndIndex, 0, null), (int)TokenType.EOF, file, startIndex)
{
_literalParser = options?.LiteralParser ?? StandardLiteralHandlers.Value;
ErrorSink = sink;
}
private static void ReadCore(ref Utf8JsonReader reader, ref DdbReadStack state, IParsingOptions options)
{
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonTokenType.String:
{
HandleStringValue(ref reader, ref state);
break;
}
case JsonTokenType.PropertyName:
{
HandlePropertyName(ref reader, ref state);
break;
}
case JsonTokenType.StartObject:
{
if (!state.IsLastFrame || state.GetCurrent().IsProcessingValue())
{
// Parse inner object start
HandleNestedStartObject(ref state);
}
break;
}
case JsonTokenType.EndObject:
{
HandleEndObject(ref state);
break;
}
case JsonTokenType.True:
{
HandleBoolValue(ref state, true);
break;
}
case JsonTokenType.False:
{
HandleBoolValue(ref state, false);
break;
}
case JsonTokenType.StartArray:
{
HandleStartArray(ref state);
break;
}
case JsonTokenType.EndArray:
{
HandleEndArray(ref state);
break;
}
case JsonTokenType.Number:
{
HandleNumberValue(ref reader, ref state, options);
break;
}
case JsonTokenType.Null:
{
HandleNullValue(ref state);
break;
}
}
}
state.BytesConsumed += reader.BytesConsumed;
}
public IListSource <LNode> Parse(IListSource <Token> input, ISourceFile file, IMessageSink msgs, IParsingOptions options)
{
// For efficiency we'd prefer to re-use our _parser object, but
// when parsing lazily, we can't re-use it because another parsing
// operation could start before this one is finished. To force
// greedy parsing, we can call ParseStmtsGreedy(), but the caller may
// prefer lazy parsing, especially if the input is large. As a
// compromise I'll check if the source file is larger than a
// certain arbitrary size. Also, ParseExprs() is always greedy
// so we can always re-use _parser in that case.
bool exprMode = options.Mode == ParsingMode.Expressions;
if (options.Mode == ParsingMode.Expressions || file.Text.TryGet(255).HasValue)
{
Les2Parser parser = _parser;
if (parser == null)
{
_parser = parser = new Les2Parser(input, file, msgs);
}
else
{
parser.ErrorSink = msgs;
parser.Reset(input.AsList(), file);
}
if (options.Mode == ParsingMode.Expressions)
{
return(parser.ExprList());
}
else
{
return(parser.Start(new Holder <TokenType>(TokenType.Semicolon)).Buffered());
}
}
else
{
var parser = new Les2Parser(input, file, msgs);
return(parser.Start(new Holder <TokenType>(TokenType.Semicolon)).Buffered());
}
}
public ILexer <Token> Tokenize(ICharSource text, string fileName, IMessageSink msgs, IParsingOptions options)
{
return(new Les2Lexer(text, fileName, msgs)
{
SkipValueParsing = options.SurfaceScanOnly,
SpacesPerTab = options.SpacesPerTab
});
}
/// <summary>Parses a string and expects exactly one output.</summary>
/// <exception cref="InvalidOperationException">The output list was empty or contained multiple nodes.</exception>
public static LNode ParseSingle(this IParsingService parser, ICharSource text, string fileName, IMessageSink msgs = null, IParsingOptions options = null)
{
var e = parser.Parse(text, fileName, msgs ?? MessageSink.Default, options ?? _fileWithComments);
return(Single(e));
}
public IListSource <LNode> Parse(ICharSource text, string fileName, IMessageSink msgs, IParsingOptions options)
{
var lexer = Tokenize(text, fileName, msgs, options);
return(Parse(lexer, msgs, options));
}
private static void ReadCore(ref JsonReaderState readerState, bool isFinalBlock, ReadOnlySpan <byte> buffer, ref DdbReadStack readStack, IParsingOptions options)
{
var reader = new Utf8JsonReader(buffer, isFinalBlock, readerState);
readStack.BytesConsumed = 0;
ReadCore(ref reader, ref readStack, options);
readerState = reader.CurrentState;
}
public EcsParser(IListSource <Token> tokens, ISourceFile file, IMessageSink messageSink, IParsingOptions parsingOptions) : base(file)
{
ErrorSink = messageSink;
Reset(tokens, file, parsingOptions);
_triviaWordAttribute = F.Id(S.TriviaWordAttribute);
_triviaUseOperatorKeyword = F.Id(S.TriviaUseOperatorKeyword);
_triviaForwardedProperty = F.Id(S.TriviaForwardedProperty);
_filePrivate = F.Id(S.FilePrivate);
}
public void Reset(IList <Token> list, ISourceFile file, IParsingOptions options, int startIndex = 0)
{
Reset(list, default(Token), file, startIndex);
_literalParser = options?.LiteralParser ?? StandardLiteralHandlers.Value;
}
/// <summary>Parses a string by invoking <see cref="IParsingService.Parse(ICharSource, string, IMessageSink, IParsingOptions)"/> using an empty string as the file name.</summary>
public static IListSource <LNode> Parse(this IParsingService parser, UString input, IMessageSink msgs, IParsingOptions options)
{
return(parser.Parse(input, "", msgs ?? MessageSink.Default, options ?? _fileWithComments));
}