private static void SkipWhitespace(ITextStream scanner) { while (!scanner.IsCompleted && CharUtility.IsWhiteSpace(scanner.Peek())) { scanner.Pop(); } }
public TokenList Tokenize(ITextStream stream, ILexingContext context) { var watch = Stopwatch.StartNew(); var tokens = new TokenList(); tokens.Add(Token.CreateEmpty(TokenType.StartOfFile, stream.Position)); while (!context.IsCancellationRequested) { if (stream.Position >= stream.Length) break; if (ConsumeComment(stream, tokens)) continue; if (ConsumeNewLine(stream, tokens)) continue; if (ConsumeWhitespace(stream)) continue; if (ConsumeInterpolation(stream, tokens)) continue; Token token; if (TryCreateToken(stream, out token)) tokens.Add(token); } // close stream with end of file token tokens.Add(Token.CreateEmpty(TokenType.EndOfFile, stream.Length)); watch.Stop(); LastTokenizationDuration = watch.Elapsed; return tokens; }
void ConsumeNameCharacters(ITextStream stream) { while (IsValidNameCharacter(stream.Current)) { stream.Advance(); } }
private bool ConsumeInterpolation(ITextStream stream, TokenList tokens) { if (stream.Current == '#' && stream.Peek(1) == '{') { tokens.Add(Token.Create(TokenType.OpenInterpolation, stream.Position, 2)); stream.Advance(2); while (stream.Current != '}' && !IsNewLine(stream.Current)) { Token token; if (TryCreateToken(stream, out token)) { tokens.Add(token); } } if (stream.Current == '}') { tokens.Add(Token.Create(TokenType.CloseInterpolation, stream.Position, 1)); stream.Advance(); } return(true); } return(false); }
public void Present(ITextStream stream) { stream.Write(_title); foreach (var option in _options) { stream.Write($"{option.Name}: {option.To}"); } }
public JsonStreamBase(ITextStream textStream, bool disposeStream, TimeSpan requestTimeout) { this._textStream = textStream; this._disposeStream = disposeStream; this._requestTimeout = requestTimeout; this._textStream.OnError.Subscribe((IObserver <Exception>) this._onErrorSubject); this._textStream.Select <string, JObject>(new Func <string, JObject>(this.SafeConvert)).Where <JObject>((Func <JObject, bool>)(_ => _ != null)).Subscribe <JObject>((IObserver <JObject>) this._onData, this._disposeCancel.Token); }
private TokenType ConsumeUnicode(ITextStream stream) { if (ConsumeUnicodeRange(stream)) { return(TokenType.UnicodeRange); } return(TokenType.Unknown); }
private bool ConsumeDecimalPart(ITextStream stream) { if (stream.Current == '.' && char.IsDigit(stream.Peek(1))) { stream.Advance(); return(true); } return(false); }
public IEnumerable <char> GetSequence(ITextStream stream) { using (stream) { while (!stream.IsCompleted) { yield return(stream.Pop()); } } }
private TokenType ConsumePeriod(ITextStream stream) { if (ConsumeNumber(stream)) { return(TokenType.Number); } stream.Advance(); return(TokenType.Period); }
private TokenType ConsumeTilde(ITextStream stream) { stream.Advance(); if (stream.Current == '=') { stream.Advance(); return(TokenType.IncludeMatch); } return(TokenType.Tilde); }
/// <summary> /// Reply to the client. /// </summary> /// <param name="stream">The text stream to perform the operation on.</param> /// <param name="response">The response.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns>A task which performs the operation.</returns> public static async Task ReplyAsync(this ITextStream stream, SmtpResponse response, CancellationToken cancellationToken) { if (stream == null) { throw new ArgumentNullException(nameof(stream)); } await stream.WriteLineAsync($"{(int)response.ReplyCode} {response.Message}", cancellationToken); await stream.FlushAsync(cancellationToken); }
private TokenType ConsumeGreaterThanSign(ITextStream stream) { stream.Advance(); if (stream.Current == '=') { stream.Advance(); return(TokenType.GreaterThanEqual); } return(TokenType.GreaterThan); }
public static bool InUndoContext(this ITextStream stream, Func <int, bool> callback) { int start = stream.Position; bool result = callback(start); if (!result) { stream.SeekTo(start); } return(result); }
private TokenType ConsumePlus(ITextStream stream) { var type = TokenType.Plus; stream.Advance(); if (ConsumeNumber(stream)) { type = TokenType.Number; } return(type); }
/// <summary> /// Flush the write buffers to the stream. /// </summary> /// <param name="stream">The stream to perform the operation on.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns>A task that represents the asynchronous flush operation.</returns> public static async Task FlushAsync(this ITextStream stream, CancellationToken cancellationToken) { if (stream == null) { throw new ArgumentNullException(nameof(stream)); } try { await stream.FlushAsync().WithCancellation(cancellationToken); } catch (OperationCanceledException) { } }
private TokenType ConsumeAsterisk(ITextStream stream) { var type = TokenType.Asterisk; stream.Advance(); if (stream.Current == '=') { type = TokenType.SubstringMatch; stream.Advance(); } return(type); }
static TokenType DetermineIdentifierType(ITextStream stream) { // handle conditional statements if (stream.Current == ' ' && stream.CompareOrdinal(" if")) { stream.Advance(3); return TokenType.Identifier; } if (stream.Current == '(') return TokenType.Function; return TokenType.Identifier; }
private TokenType ConsumeDollar(ITextStream stream) { var type = TokenType.Dollar; stream.Advance(); if (stream.Current == '=') { type = TokenType.SuffixMatch; stream.Advance(); } return(type); }
TokenList Tokenize(ITextStream stream, IParsingExecutionContext context) { var watch = Stopwatch.StartNew(); var tokens = new TokenList(); tokens.Add(Token.CreateEmpty(TokenType.StartOfFile, stream.Position)); while (!context.IsCancellationRequested) { if (stream.Position >= stream.Length) { break; } if (ConsumeComment(stream, tokens)) { continue; } if (ConsumeNewLine(stream, tokens)) { continue; } if (ConsumeWhitespace(stream)) { continue; } if (ConsumeInterpolation(stream, tokens)) { continue; } Token token; if (TryCreateToken(stream, out token)) { tokens.Add(token); } } // close stream with end of file token tokens.Add(Token.CreateEmpty(TokenType.EndOfFile, stream.Length)); watch.Stop(); LastTokenizationDuration = watch.Elapsed; return(tokens); }
private bool ConsumeComment(ITextStream stream, TokenList tokens) { if (stream.Current == '/') { int start = stream.Position; var next = stream.Peek(1); if (next == '/' && stream.Peek(2) == '/') { stream.Advance(3); tokens.Add(Token.Create(TokenType.XmlDocumentationComment, start, 3)); return(true); } else if (next == '/') { stream.Advance(2); tokens.Add(Token.Create(TokenType.CppComment, start, 2)); if (!IsNewLine(stream.Current)) { ConsumeCommentText(stream, tokens, s => IsNewLine(s.Current)); } return(true); } else if (next == '*') { stream.Advance(2); tokens.Add(Token.Create(TokenType.OpenCssComment, start, 2)); start = stream.Position; ConsumeCommentText(stream, tokens, s => s.Current == '*' && s.Peek(1) == '/'); if (stream.Current == '*' && stream.Peek(1) == '/') { start = stream.Position; stream.Advance(2); tokens.Add(Token.Create(TokenType.CloseCssComment, start, 2)); } return(true); } return(false); } return(false); }
/// <summary> /// Reads a line of characters asynchronously from the current stream and returns the data as a string. /// </summary> /// <param name="stream">The stream to perform the operation on.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns>A task that represents the asynchronous read operation.</returns> public static async Task <string> ReadLineAsync(this ITextStream stream, CancellationToken cancellationToken) { if (stream == null) { throw new ArgumentNullException(nameof(stream)); } try { return(await stream.ReadLineAsync().WithCancellation(cancellationToken)); } catch (OperationCanceledException) { return(null); } }
private bool ConsumeNewLine(ITextStream stream, TokenList tokens) { if (IsNewLine(stream.Current)) { int start = stream.Position; while (IsNewLine(stream.Current)) { stream.Advance(); } tokens.Add(Token.Create(TokenType.NewLine, start, stream.Position - start)); return(true); } return(false); }
static TokenType DetermineIdentifierType(ITextStream stream) { // handle conditional statements if (stream.Current == ' ' && stream.CompareOrdinal(" if")) { stream.Advance(3); return(TokenType.Identifier); } if (stream.Current == '(') { return(TokenType.Function); } return(TokenType.Identifier); }
private TokenType ConsumeLessThanSign(ITextStream stream) { stream.Advance(); if (stream.Current == '=') { stream.Advance(); return(TokenType.LessThanEqual); } if (stream.Current == '!' && stream.Peek(1) == '-' && stream.Peek(2) == '-') { stream.Advance(3); return(TokenType.OpenHtmlComment); } return(TokenType.LessThan); }
public static bool CompareOrdinal(this ITextStream stream, string value) { if (value == null) { return(false); } for (int i = 0; i < value.Length; i++) { if (value[i] != stream.Peek(i)) { return(false); } } return(true); }
private TokenType ConsumePipe(ITextStream stream) { stream.Advance(); if (stream.Current == '=') { stream.Advance(); return(TokenType.DashMatch); } if (stream.Current == '|') { stream.Advance(); return(TokenType.Column); } return(TokenType.Pipe); }
private bool ConsumeWhitespace(ITextStream stream) { switch (stream.Current) { case ' ': case '\t': //case '\f': //case '\n': //case '\r': while (IsWhitespace(stream.Current)) { stream.Advance(); } return(true); } return(false); }
private bool ConsumeUnicodeRange(ITextStream stream) { return(stream.InUndoContext(start => { if (stream.Peek(1) == '+' && (IsHexDigit(stream.Peek(2)) || stream.Peek(2) == '?')) { stream.Advance(3); int hexPosition = 1; while (hexPosition < 6) { if (!IsHexDigit(stream.Current)) { break; } hexPosition++; stream.Advance(); } if (stream.Current == '-' && IsHexDigit(stream.Peek(1))) { stream.Advance(2); hexPosition = 1; while (hexPosition < 6) { if (!IsHexDigit(stream.Current)) { break; } } hexPosition++; stream.Advance(); } return start != stream.Position; } return false; })); }
private bool ConsumeIdentifier(ITextStream stream) { return(stream.InUndoContext(start => { if (IsNameStart(stream.Current)) { stream.Advance(); ConsumeNameCharacters(stream); return true; } if (stream.Current == '-') { if (IsNameStart(stream.Peek(1))) { stream.Advance(); ConsumeNameCharacters(stream); return true; } if (IsValidEscape(stream.Peek(1), stream.Peek(2))) { stream.Advance(2); ConsumeNameCharacters(stream); return true; } } if (IsValidEscape(stream.Current, stream.Peek(1))) { stream.Advance(2); ConsumeNameCharacters(stream); return true; } return false; })); }
private TokenType HandleHyphen(ITextStream stream) { stream.Advance(); if (ConsumeNumber(stream)) { return(TokenType.Number); } stream.Reverse(); if (ConsumeIdentifier(stream)) { return(TokenType.Identifier); } stream.Advance(); if (stream.Peek(1) == '-' && stream.Peek(2) == '>') { stream.Advance(2); return(TokenType.CloseHtmlComment); } return(TokenType.Minus); }
/// <summary> /// Reads a line of characters asynchronously from the current stream and returns the data as a string. /// </summary> /// <param name="stream">The stream to perform the operation on.</param> /// <param name="timeout">The timeout to apply when reading from the stream.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns>A task that represents the asynchronous read operation.</returns> public static async Task <string> ReadLineAsync(this ITextStream stream, TimeSpan timeout, CancellationToken cancellationToken) { if (stream == null) { throw new ArgumentNullException(nameof(stream)); } var cancellationTokenSource = new CancellationTokenSource(); cancellationToken.Register(cancellationTokenSource.Cancel); var task = stream.ReadLineAsync(); if (task == await Task.WhenAny(task, Task.Delay(timeout, cancellationTokenSource.Token))) { cancellationTokenSource.Cancel(); cancellationTokenSource.Dispose(); return(await task); } throw new TimeoutException(); }
private bool ConsumeDecimalPart(ITextStream stream) { if (stream.Current == '.' && char.IsDigit(stream.Peek(1))) { stream.Advance(); return true; } return false; }
private bool ConsumeString(ITextStream stream, out TokenType type) { type = TokenType.Unknown; switch (stream.Current) { case '\'': case '"': { var open = stream.Current; while (stream.Advance()) { var current = stream.Current; // check for valid escapes if (current == '\\') { // check to see if we are escaping the new line (and thus, continuing the string) stream.Advance(); if (IsNewLine(stream.Current)) { stream.Advance(); continue; } // if escaping open quote, consume and advance if (current == open) { stream.Advance(); continue; } if (IsValidEscape('\\', current)) { stream.Advance(); continue; } } // unescaped new line is bad news bears if (IsNewLine(current)) { // go back to right before the new line //stream.Reverse(1); type = TokenType.BadString; return true; } // happy days, we have properly quoted string if (stream.Current == open) break; } // consume closing quote if (stream.Current == open) stream.Advance(); type = TokenType.String; break; } } return type != TokenType.Unknown; }
public dynamic Parse(ITextStream stream, ErrorReporter errors, SourcePoint startLocation) { Parser parser = CreateParser(); return parser.Parse(stream, errors, startLocation); }
public dynamic Parse(ITextStream stream, ErrorReporter errors) { object result = this.parser.Parse(stream, errors); return this.NormalizeResult(result); }
private TokenType HandleHyphen(ITextStream stream) { stream.Advance(); if (ConsumeNumber(stream)) return TokenType.Number; stream.Reverse(); if (ConsumeIdentifier(stream)) return TokenType.Identifier; stream.Advance(); if (stream.Peek(1) == '-' && stream.Peek(2) == '>') { stream.Advance(2); return TokenType.CloseHtmlComment; } return TokenType.Minus; }
private bool TryCreateToken(ITextStream stream, out Token token) { token = default(Token); if (stream.Position >= stream.Length) return false; int start = stream.Position; TokenType type = TokenType.Unknown; switch (stream.Current) { case '\'': case '"': ConsumeString(stream, out type); break; case '#': type = TokenType.Hash; stream.Advance(); break; case '$': type = ConsumeDollar(stream); break; case '(': type = TokenType.OpenFunctionBrace; stream.Advance(); break; case ')': type = TokenType.CloseFunctionBrace; stream.Advance(); break; case '*': type = ConsumeAsterisk(stream); break; case '+': type = ConsumePlus(stream); break; case ',': type = TokenType.Comma; stream.Advance(); break; case '-': type = HandleHyphen(stream); break; case '.': type = ConsumePeriod(stream); break; case '/': type = TokenType.Slash; stream.Advance(); break; case ':': type = TokenType.Colon; stream.Advance(); if (stream.Current == ':') { type = TokenType.DoubleColon; stream.Advance(); } break; case ';': type = TokenType.Semicolon; stream.Advance(); break; case '@': type = TokenType.At; stream.Advance(); break; case '<': type = ConsumeLessThanSign(stream); break; case '[': type = TokenType.OpenBrace; stream.Advance(); break; case ']': type = TokenType.CloseBrace; stream.Advance(); break; case '{': type = TokenType.OpenCurlyBrace; stream.Advance(); break; case '}': type = TokenType.CloseCurlyBrace; stream.Advance(); break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': type = TokenType.Number; ConsumeNumber(stream); break; case 'u': case 'U': type = ConsumeUnicode(stream); if (type == TokenType.Unknown && ConsumeIdentifier(stream)) type = DetermineIdentifierType(stream); break; case '|': type = ConsumePipe(stream); break; case '~': type = ConsumeTilde(stream); break; case '!': type = TokenType.Bang; stream.Advance(); if (stream.Current == '=') { stream.Advance(); type = TokenType.NotEqual; } break; case '&': type = TokenType.Ampersand; stream.Advance(); break; case '>': type = ConsumeGreaterThanSign(stream); break; case '=': type = TokenType.Equal; stream.Advance(); if (stream.Current == '=') { stream.Advance(); type = TokenType.DoubleEqual; } break; case '^': type = TokenType.Caret; stream.Advance(); if (stream.Current == '=') { type = TokenType.PrefixMatch; stream.Advance(); } break; case '%': type = TokenType.PercentSign; stream.Advance(); break; default: { if (ConsumeIdentifier(stream)) { type = DetermineIdentifierType(stream); } else { // just add as unknown token type = TokenType.Unknown; stream.Advance(); } break; } } token = Token.Create(type, start, stream.Position - start); return true; }
private TokenType ConsumeLessThanSign(ITextStream stream) { stream.Advance(); if (stream.Current == '=') { stream.Advance(); return TokenType.LessThanEqual; } if (stream.Current == '!' && stream.Peek(1) == '-' && stream.Peek(2) == '-') { stream.Advance(3); return TokenType.OpenHtmlComment; } return TokenType.LessThan; }
private bool ConsumeUnicodeRange(ITextStream stream) { return stream.InUndoContext(start => { if (stream.Peek(1) == '+' && (IsHexDigit(stream.Peek(2)) || stream.Peek(2) == '?')) { stream.Advance(3); int hexPosition = 1; while (hexPosition < 6) { if (!IsHexDigit(stream.Current)) break; hexPosition++; stream.Advance(); } if (stream.Current == '-' && IsHexDigit(stream.Peek(1))) { stream.Advance(2); hexPosition = 1; while (hexPosition < 6) { if (!IsHexDigit(stream.Current)) break; } hexPosition++; stream.Advance(); } return start != stream.Position; } return false; }); }
private bool ConsumeComment(ITextStream stream, TokenList tokens) { if (stream.Current == '/') { int start = stream.Position; var next = stream.Peek(1); if (next == '/' && stream.Peek(2) == '/') { stream.Advance(3); tokens.Add(Token.Create(TokenType.XmlDocumentationComment, start, 3)); return true; } else if (next == '/') { stream.Advance(2); tokens.Add(Token.Create(TokenType.CppComment, start, 2)); if (!IsNewLine(stream.Current)) ConsumeCommentText(stream, tokens, s => IsNewLine(s.Current)); return true; } else if (next == '*') { stream.Advance(2); tokens.Add(Token.Create(TokenType.OpenCssComment, start, 2)); start = stream.Position; ConsumeCommentText(stream, tokens, s => s.Current == '*' && s.Peek(1) == '/'); if (stream.Current == '*' && stream.Peek(1) == '/') { start = stream.Position; stream.Advance(2); tokens.Add(Token.Create(TokenType.CloseCssComment, start, 2)); } return true; } return false; } return false; }
private bool ConsumeInterpolation(ITextStream stream, TokenList tokens) { if (stream.Current == '#' && stream.Peek(1) == '{') { tokens.Add(Token.Create(TokenType.OpenInterpolation, stream.Position, 2)); stream.Advance(2); while (stream.Current != '}' && !IsNewLine(stream.Current)) { Token token; if (TryCreateToken(stream, out token)) tokens.Add(token); } if (stream.Current == '}') { tokens.Add(Token.Create(TokenType.CloseInterpolation, stream.Position, 1)); stream.Advance(); } return true; } return false; }
private bool ConsumeIdentifier(ITextStream stream) { return stream.InUndoContext(start => { if (IsNameStart(stream.Current)) { stream.Advance(); ConsumeNameCharacters(stream); return true; } if (stream.Current == '-') { if (IsNameStart(stream.Peek(1))) { stream.Advance(); ConsumeNameCharacters(stream); return true; } if (IsValidEscape(stream.Peek(1), stream.Peek(2))) { stream.Advance(2); ConsumeNameCharacters(stream); return true; } } if (IsValidEscape(stream.Current, stream.Peek(1))) { stream.Advance(2); ConsumeNameCharacters(stream); return true; } return false; }); }
private TokenType ConsumeGreaterThanSign(ITextStream stream) { stream.Advance(); if (stream.Current == '=') { stream.Advance(); return TokenType.GreaterThanEqual; } return TokenType.GreaterThan; }
private TokenType ConsumeDollar(ITextStream stream) { var type = TokenType.Dollar; stream.Advance(); if (stream.Current == '=') { type = TokenType.SuffixMatch; stream.Advance(); } return type; }
private TokenType ConsumeTilde(ITextStream stream) { stream.Advance(); if (stream.Current == '=') { stream.Advance(); return TokenType.IncludeMatch; } return TokenType.Tilde; }
void ConsumeNameCharacters(ITextStream stream) { while (IsValidNameCharacter(stream.Current)) stream.Advance(); }
private TokenType ConsumeUnicode(ITextStream stream) { if (ConsumeUnicodeRange(stream)) return TokenType.UnicodeRange; return TokenType.Unknown; }
private bool ConsumeNewLine(ITextStream stream, TokenList tokens) { if (IsNewLine(stream.Current)) { int start = stream.Position; while (IsNewLine(stream.Current)) stream.Advance(); tokens.Add(Token.Create(TokenType.NewLine, start, stream.Position - start)); return true; } return false; }
private bool ConsumeWhitespace(ITextStream stream) { switch (stream.Current) { case ' ': case '\t': //case '\f': //case '\n': //case '\r': while (IsWhitespace(stream.Current)) stream.Advance(); return true; } return false; }
private bool ConsumeNumber(ITextStream stream) { bool checkDecimal = !ConsumeDecimalPart(stream); if (char.IsDigit(stream.Current)) { while (char.IsDigit(stream.Current)) stream.Advance(); if (stream.Current == '.' && ConsumeDecimalPart(stream)) { while (char.IsDigit(stream.Current)) stream.Advance(); } return true; } return false; }
public Task<TokenList> TokenizeAsync(ITextStream stream, IParsingExecutionContext context) { return Task.Run(() => Tokenize(stream, context)); }
private TokenType ConsumePeriod(ITextStream stream) { if (ConsumeNumber(stream)) return TokenType.Number; stream.Advance(); return TokenType.Period; }
private TokenType ConsumeAsterisk(ITextStream stream) { var type = TokenType.Asterisk; stream.Advance(); if (stream.Current == '=') { type = TokenType.SubstringMatch; stream.Advance(); } return type; }
private TokenType ConsumePipe(ITextStream stream) { stream.Advance(); if (stream.Current == '=') { stream.Advance(); return TokenType.DashMatch; } if (stream.Current == '|') { stream.Advance(); return TokenType.Column; } return TokenType.Pipe; }
public dynamic Parse(ITextStream stream, ErrorReporter errors) { Parser parser = CreateParser(); return parser.Parse(stream, errors); }
private TokenType ConsumePlus(ITextStream stream) { var type = TokenType.Plus; stream.Advance(); if (ConsumeNumber(stream)) type = TokenType.Number; return type; }
public dynamic Parse(ITextStream stream, ErrorReporter errors, SourcePoint startLocation) { object result = this.parser.Parse(stream, errors, startLocation); return this.NormalizeResult(result); }
private bool ConsumeCommentText(ITextStream stream, TokenList tokens, Func<ITextStream, bool> predicate) { int start = stream.Position; while (stream.Position < stream.Length) { if (predicate(stream)) break; stream.Advance(); } if (start != stream.Position) { stream.Reverse(1); stream.Advance(); tokens.Add(Token.Create(TokenType.CommentText, start, stream.Position - start)); return true; } return false; }