public virtual void Reset(IListSource <Token> tokens, ISourceFile file, IParsingOptions parsingOptions) { CheckParam.IsNotNull("tokens", tokens); CheckParam.IsNotNull("file", file); _tokensRoot = _tokens = tokens; _sourceFile = file; _literalParser = parsingOptions?.LiteralParser ?? EcsLiteralHandlers.Value; F = new LNodeFactory(file); InputPosition = 0; // reads LT(0) _tentativeErrors = new TentativeState(false); }
public IListSource <LNode> Parse(ILexer <Token> input, IMessageSink msgs, IParsingOptions options) { if (options.PreserveComments) { var saver = new TriviaSaver(input, (int)TokenType.Newline); var results = Parse(saver.Buffered(), input.SourceFile, msgs, options); var injector = new StandardTriviaInjector(saver.TriviaList, saver.SourceFile, (int)TokenType.Newline, "/*", "*/", "//", options.Mode != ParsingMode.Expressions); return(injector.Run(results.GetEnumerator()).Buffered()); } else { var lexer = new WhitespaceFilter(input); return(Parse(lexer.Buffered(), input.SourceFile, msgs, options)); } }
public IListSource <LNode> Parse(ILexer <Token> input, IMessageSink msgs, IParsingOptions options) { if (options.PreserveComments) { // Filter out whitespace, including some newlines (those directly inside square brackets or parentheses) var saver = new TriviaSaver(input, (int)TokenType.Newline); var results = Parse(saver.Buffered(), input.SourceFile, msgs, options); var injector = new StandardTriviaInjector(saver.TriviaList, input.SourceFile, (int)TokenType.Newline, "/*", "*/", "//", options.Mode != ParsingMode.Expressions); injector.SLCommentSuffix = @"\\"; return(injector.Run(results.GetEnumerator()).Buffered()); } else { return(Parse(new WhitespaceFilter(input).Buffered(), input.SourceFile, msgs, options)); } }
public IListSource <LNode> Parse(ILexer <Token> input, IMessageSink msgs, IParsingOptions options) { var preprocessed = new EcsPreprocessor(input, options.PreserveComments); var treeified = new TokensToTree(preprocessed, false); var results = Parse(treeified.Buffered(), input.SourceFile, msgs, options); if (options.PreserveComments) { var injector = new EcsTriviaInjector(preprocessed.TriviaList, input.SourceFile, (int)TokenType.Newline, "/*", "*/", "//", !options.Mode.IsOneOf <Symbol>(ParsingMode.Expressions, ParsingMode.FormalArguments)); return(injector.Run(results.GetEnumerator()).Buffered()); } else { return(results); } }
public ILexer <Token> Tokenize(ICharSource text, string fileName, IMessageSink msgs, IParsingOptions options) { return(new EcsLexer(text, fileName, msgs) { SpacesPerTab = options.SpacesPerTab }); }
public IListSource <LNode> Parse(IListSource <Token> input, ISourceFile file, IMessageSink msgs, IParsingOptions options) { // For efficiency we'd prefer to re-use our _parser object, but // when parsing lazily, we can't re-use it because another parsing // operation could start before this one is finished. To force // greedy parsing, we can call ParseStmtsGreedy(), but the caller may // prefer lazy parsing, especially if the input is large. As a // compromise I'll check if the source file is larger than a // certain arbitrary size. Also, ParseExprs() is always greedy // so we can always re-use _parser in that case. var inputType = options.Mode; if (file.Text.TryGet(255).HasValue || inputType == ParsingMode.FormalArguments || inputType == ParsingMode.Types || inputType == ParsingMode.Expressions) { EcsParser parser = _parser; if (parser == null) { _parser = parser = new EcsParser(input, file, msgs, options); } else { parser.ErrorSink = msgs ?? MessageSink.Default; parser.Reset(input, file, options); } if (inputType == ParsingMode.Expressions) { return(parser.ParseExprs(false, allowUnassignedVarDecl: false)); } else if (inputType == ParsingMode.FormalArguments) { return(parser.ParseExprs(false, allowUnassignedVarDecl: true)); } else if (inputType == ParsingMode.Types) { return(LNode.List(parser.DataType())); } else { return(parser.ParseStmtsGreedy()); } } else { var parser = new EcsParser(input, file, msgs, options); return(parser.ParseStmtsLazy().Buffered()); } }
private static void HandleNumberValue(ref Utf8JsonReader reader, ref DdbReadStack state, IParsingOptions callbacks) { if (state.IsLastFrame && callbacks.HasNumberCallback) { callbacks.OnNumber(ref reader, ref state); } ref var current = ref state.GetCurrent();
public static async ValueTask <ReadResult <Document> > ReadAsync(Stream utf8Json, IParsingOptions options, bool returnCrc, CancellationToken cancellationToken = default) { var readerState = new JsonReaderState(); var readStack = new DdbReadStack(DdbReadStack.DefaultStackLength, options.Metadata); try { options.StartParsing(ref readStack); var buffer = ArrayPool <byte> .Shared.Rent(DefaultBufferSize); var clearMax = 0; try { var bytesInBuffer = 0; uint crc = 0; while (true) { var isFinalBlock = false; while (true) { var bytesRead = await utf8Json.ReadAsync(new Memory <byte>(buffer, bytesInBuffer, buffer.Length - bytesInBuffer), cancellationToken).ConfigureAwait(false); if (bytesRead == 0) { isFinalBlock = true; break; } if (returnCrc) { crc = Crc32Algorithm.Append(crc, buffer, bytesInBuffer, bytesRead); } bytesInBuffer += bytesRead; if (bytesInBuffer == buffer.Length) { break; } } if (bytesInBuffer > clearMax) { clearMax = bytesInBuffer; } ReadCore(ref readerState, isFinalBlock, new ReadOnlySpan <byte>(buffer, 0, bytesInBuffer), ref readStack, options); var bytesConsumed = (int)readStack.BytesConsumed; bytesInBuffer -= bytesConsumed; if (isFinalBlock) { break; } // Check if we need to shift or expand the buffer because there wasn't enough data to complete deserialization. if ((uint)bytesInBuffer > ((uint)buffer.Length / 2)) { // We have less than half the buffer available, double the buffer size. byte[] dest = ArrayPool <byte> .Shared.Rent((buffer.Length < (int.MaxValue / 2))?buffer.Length * 2 : int.MaxValue); // Copy the unprocessed data to the new buffer while shifting the processed bytes. Buffer.BlockCopy(buffer, bytesConsumed, dest, 0, bytesInBuffer); new Span <byte>(buffer, 0, clearMax).Clear(); ArrayPool <byte> .Shared.Return(buffer); clearMax = bytesInBuffer; buffer = dest; } else if (bytesInBuffer != 0) { // Shift the processed bytes to the beginning of buffer to make more room. Buffer.BlockCopy(buffer, bytesConsumed, buffer, 0, bytesInBuffer); } } return(new ReadResult <Document>(readStack.GetCurrent().CreateDocumentFromBuffer(), crc)); } finally { new Span <byte>(buffer, 0, clearMax).Clear(); ArrayPool <byte> .Shared.Return(buffer); } } finally { readStack.Dispose(); } }
/// <summary>Parses a string and expects exactly one output.</summary> /// <exception cref="InvalidOperationException">The output list was empty or contained multiple nodes.</exception> public static LNode ParseSingle(this IParsingService parser, UString expr, IMessageSink msgs, IParsingOptions options) { var e = Parse(parser, expr, msgs, options); return(Single(e)); }
public Les3Parser(IList <Token> list, ISourceFile file, IMessageSink sink, IParsingOptions options, int startIndex = 0) : base(list, prev => new Token((int)TokenType.EOF, prev.EndIndex, 0, null), (int)TokenType.EOF, file, startIndex) { _literalParser = options?.LiteralParser ?? StandardLiteralHandlers.Value; ErrorSink = sink; }
private static void ReadCore(ref Utf8JsonReader reader, ref DdbReadStack state, IParsingOptions options) { while (reader.Read()) { switch (reader.TokenType) { case JsonTokenType.String: { HandleStringValue(ref reader, ref state); break; } case JsonTokenType.PropertyName: { HandlePropertyName(ref reader, ref state); break; } case JsonTokenType.StartObject: { if (!state.IsLastFrame || state.GetCurrent().IsProcessingValue()) { // Parse inner object start HandleNestedStartObject(ref state); } break; } case JsonTokenType.EndObject: { HandleEndObject(ref state); break; } case JsonTokenType.True: { HandleBoolValue(ref state, true); break; } case JsonTokenType.False: { HandleBoolValue(ref state, false); break; } case JsonTokenType.StartArray: { HandleStartArray(ref state); break; } case JsonTokenType.EndArray: { HandleEndArray(ref state); break; } case JsonTokenType.Number: { HandleNumberValue(ref reader, ref state, options); break; } case JsonTokenType.Null: { HandleNullValue(ref state); break; } } } state.BytesConsumed += reader.BytesConsumed; }
public IListSource <LNode> Parse(IListSource <Token> input, ISourceFile file, IMessageSink msgs, IParsingOptions options) { // For efficiency we'd prefer to re-use our _parser object, but // when parsing lazily, we can't re-use it because another parsing // operation could start before this one is finished. To force // greedy parsing, we can call ParseStmtsGreedy(), but the caller may // prefer lazy parsing, especially if the input is large. As a // compromise I'll check if the source file is larger than a // certain arbitrary size. Also, ParseExprs() is always greedy // so we can always re-use _parser in that case. bool exprMode = options.Mode == ParsingMode.Expressions; if (options.Mode == ParsingMode.Expressions || file.Text.TryGet(255).HasValue) { Les2Parser parser = _parser; if (parser == null) { _parser = parser = new Les2Parser(input, file, msgs); } else { parser.ErrorSink = msgs; parser.Reset(input.AsList(), file); } if (options.Mode == ParsingMode.Expressions) { return(parser.ExprList()); } else { return(parser.Start(new Holder <TokenType>(TokenType.Semicolon)).Buffered()); } } else { var parser = new Les2Parser(input, file, msgs); return(parser.Start(new Holder <TokenType>(TokenType.Semicolon)).Buffered()); } }
public ILexer <Token> Tokenize(ICharSource text, string fileName, IMessageSink msgs, IParsingOptions options) { return(new Les2Lexer(text, fileName, msgs) { SkipValueParsing = options.SurfaceScanOnly, SpacesPerTab = options.SpacesPerTab }); }
/// <summary>Parses a string and expects exactly one output.</summary> /// <exception cref="InvalidOperationException">The output list was empty or contained multiple nodes.</exception> public static LNode ParseSingle(this IParsingService parser, ICharSource text, string fileName, IMessageSink msgs = null, IParsingOptions options = null) { var e = parser.Parse(text, fileName, msgs ?? MessageSink.Default, options ?? _fileWithComments); return(Single(e)); }
public IListSource <LNode> Parse(ICharSource text, string fileName, IMessageSink msgs, IParsingOptions options) { var lexer = Tokenize(text, fileName, msgs, options); return(Parse(lexer, msgs, options)); }
private static void ReadCore(ref JsonReaderState readerState, bool isFinalBlock, ReadOnlySpan <byte> buffer, ref DdbReadStack readStack, IParsingOptions options) { var reader = new Utf8JsonReader(buffer, isFinalBlock, readerState); readStack.BytesConsumed = 0; ReadCore(ref reader, ref readStack, options); readerState = reader.CurrentState; }
public EcsParser(IListSource <Token> tokens, ISourceFile file, IMessageSink messageSink, IParsingOptions parsingOptions) : base(file) { ErrorSink = messageSink; Reset(tokens, file, parsingOptions); _triviaWordAttribute = F.Id(S.TriviaWordAttribute); _triviaUseOperatorKeyword = F.Id(S.TriviaUseOperatorKeyword); _triviaForwardedProperty = F.Id(S.TriviaForwardedProperty); _filePrivate = F.Id(S.FilePrivate); }
public void Reset(IList <Token> list, ISourceFile file, IParsingOptions options, int startIndex = 0) { Reset(list, default(Token), file, startIndex); _literalParser = options?.LiteralParser ?? StandardLiteralHandlers.Value; }
/// <summary>Parses a string by invoking <see cref="IParsingService.Parse(ICharSource, string, IMessageSink, IParsingOptions)"/> using an empty string as the file name.</summary> public static IListSource <LNode> Parse(this IParsingService parser, UString input, IMessageSink msgs, IParsingOptions options) { return(parser.Parse(input, "", msgs ?? MessageSink.Default, options ?? _fileWithComments)); }