/// <summary> /// Handles string sequence with escapes /// </summary> /// <param name="openQuote"></param> public static void HandleString(char openQuote, CharacterStream cs, Action <int, int> addToken) { int start = cs.Position; cs.MoveToNextChar(); if (!cs.IsEndOfStream()) { while (true) { if (cs.CurrentChar == openQuote) { cs.MoveToNextChar(); break; } if (cs.CurrentChar == '\\') { cs.MoveToNextChar(); } if (!cs.MoveToNextChar()) { break; } } } int length = cs.Position - start; if (length > 0) { addToken(start, length); } }
public static int HandleImaginaryPart(CharacterStream cs) { int start = cs.Position; // Check if this is actually complex number NumberTokenizer.SkipWhitespace(cs); if (cs.CurrentChar == '+' || cs.CurrentChar == '-') { cs.MoveToNextChar(); if (cs.CurrentChar == '+' || cs.CurrentChar == '-') { cs.MoveToNextChar(); } int imaginaryLength = NumberTokenizer.HandleNumber(cs); if (imaginaryLength > 0) { if (cs.CurrentChar == 'i') { cs.MoveToNextChar(); return(cs.Position - start); } } } return(0); }
private static void HandleString(int separatorLength, CharacterStream cs, Func <CharacterStream, bool> terminatorCheck) { cs.Advance(separatorLength); if (!cs.IsEndOfStream()) { while (true) { if (terminatorCheck(cs)) { cs.Advance(separatorLength); break; } if (cs.CurrentChar == '\\') { cs.MoveToNextChar(); } if (!cs.MoveToNextChar()) { break; } } } }
internal static int HandleHex(CharacterStream cs, int start) { while (CharacterStream.IsHex(cs.CurrentChar)) { cs.MoveToNextChar(); } // TODO: handle C99 floating point hex syntax like 0x1.1p-2 if (cs.CurrentChar == 'L') { cs.MoveToNextChar(); } return(cs.Position - start); }
public static void SkipIdentifier(CharacterStream cs, Func <CharacterStream, bool> isIdentifierLeadCharacter, Func <CharacterStream, bool> isIdentifierCharacter) { if (!isIdentifierLeadCharacter(cs)) { return; } if (cs.IsEndOfStream()) { return; } while (!cs.IsWhiteSpace()) { if (!isIdentifierCharacter(cs)) { break; } if (!cs.MoveToNextChar()) { break; } } }
internal static void SkipWhitespace(CharacterStream cs) { while (!cs.IsEndOfStream() && cs.IsWhiteSpace()) { cs.MoveToNextChar(); } }
public static string NormalizeWhitespace(this string s) { if (s == null || s.Length == 0) { return(s); } var cs = new CharacterStream(new TextStream(s)); var sb = new StringBuilder(); while (!cs.IsEndOfStream()) { var current = cs.Position; cs.SkipWhitespace(); if (cs.Position - current > 0) { sb.Append(' '); } while (!cs.IsEndOfStream() && !cs.IsWhiteSpace()) { sb.Append(cs.CurrentChar); cs.MoveToNextChar(); } } return(sb.ToString().Trim()); }
/// <summary> /// Checks file whitespace (typically Lint-type or style type checkers. /// </summary> /// <returns>A collection of validation errors</returns> public IReadOnlyCollection <IValidationError> ValidateWhitespace(ITextProvider tp) { if (!_linterEnabled) { return(Enumerable.Empty <IValidationError>().ToList()); } var warnings = _whitespaceFileCheckers .SelectMany(c => c(tp, _settings.LintOptions, _projectedBuffer)) .ToList(); var cs = new CharacterStream(tp); while (!cs.IsEndOfStream()) { if (cs.IsWhiteSpace()) { // Unrolled since most return nulls. warnings.AddRange(_whitespaceCharCheckers .Select(c => c(cs, _settings.LintOptions)) .Where(result => result != null)); } cs.MoveToNextChar(); } return(warnings.ToList()); }
/// <summary> /// Checks file whitespace (typically Lint-type or style type checkers. /// </summary> /// <returns>A collection of validation errors</returns> public IReadOnlyCollection <IValidationError> ValidateWhitespace(ITextProvider tp) { if (!_settings.LintOptions.Enabled) { return(Enumerable.Empty <IValidationError>().ToList()); } var warnings = _whitespaceFileCheckers.SelectMany(c => c(tp, _settings.LintOptions)).ToList(); var cs = new CharacterStream(tp); while (!cs.IsEndOfStream()) { if (cs.IsWhiteSpace()) { // Unrolled since most return nulls. foreach (var c in _whitespaceCharCheckers) { var result = c(cs, _settings.LintOptions); if (result != null) { warnings.Add(result); } } } cs.MoveToNextChar(); } return(warnings.ToList()); }
internal static int HandleExponent(CharacterStream cs, int start) { Debug.Assert(cs.CurrentChar == 'E' || cs.CurrentChar == 'e'); bool hasSign = false; cs.MoveToNextChar(); if (cs.IsWhiteSpace() || cs.IsEndOfStream()) { // 0.1E or 1e return(0); } if (cs.CurrentChar == '-' || cs.CurrentChar == '+') { hasSign = true; cs.MoveToNextChar(); } int digitsStart = cs.Position; // collect decimals while (cs.IsDecimal()) { cs.MoveToNextChar(); } if (hasSign && digitsStart == cs.Position) { return(0); // NaN like 1.0E- } // Technically if letter or braces follows this is not // a number but we'll leave it alone for now. // TODO: This code is not language specific and yet it currently // handles complex 'i' as well as R-specific 'L' suffix. // Ideally this needs to be extended in a way so language-specific // tokenizer can specify options or control number format. if (char.IsLetter(cs.CurrentChar) && cs.CurrentChar != 'i' && cs.CurrentChar != 'L') { return(0); } return(cs.Position - start); }
private static int GetNCharOperatorLength(CharacterStream cs) { // R allows user-defined infix operators. These have the form of // a string of characters delimited by the ‘%’ character. The string // can contain any printable character except ‘%’. if (cs.CurrentChar == '%' && !char.IsWhiteSpace(cs.NextChar)) { // In case of broken or partially typed operators // make sure we terminate at whitespace or end of the line // so in 'x <- y % z' '% z' is not an operator. int start = cs.Position; int length; cs.MoveToNextChar(); while (!cs.IsEndOfStream() && !cs.IsWhiteSpace()) { if (cs.CurrentChar == '%') { cs.MoveToNextChar(); length = cs.Position - start; cs.Position = start; return(length); } if (cs.IsAtNewLine()) { // x <- y %abcd cs.Position = start; return(1); } cs.MoveToNextChar(); } } return(Get3CharOrShorterOperatorLength(cs)); }
public void SkipWhitespace() { if (_cs.IsEndOfStream()) { return; } while (_cs.IsWhiteSpace()) { if (!_cs.MoveToNextChar()) { break; } } }
/// <summary> /// Handle generic comment. Comment goes to the end of the line. /// </summary> public static void HandleEolComment(CharacterStream cs, Action <int, int> addToken) { int start = cs.Position; while (!cs.IsEndOfStream() && !cs.IsAtNewLine()) { cs.MoveToNextChar(); } int length = cs.Position - start; if (length > 0) { addToken(start, length); } }
public IReadOnlyTextRangeCollection <BraceToken> Tokenize(string text) { var cs = new CharacterStream(text); var tokens = new TextRangeCollection <BraceToken>(); while (!cs.IsEndOfStream()) { BraceTokenType?t = null; switch (cs.CurrentChar) { case '(': t = OpenBrace; break; case ')': t = CloseBrace; break; case '{': t = OpenCurly; break; case '}': t = CloseCurly; break; case '[': t = OpenBracket; break; case ']': t = CloseBracket; break; } if (t != null) { tokens.Add(new BraceToken(cs.Position, 1, t.Value)); } cs.MoveToNextChar(); } return(new ReadOnlyTextRangeCollection <BraceToken>(tokens)); }
// public static object CharacterSteam { get; private set; } public static int HandleNumber(CharacterStream cs) { int start = cs.Position; if (cs.CurrentChar == '-' || cs.CurrentChar == '+') { cs.MoveToNextChar(); } if (cs.CurrentChar == '0' && cs.NextChar == 'x') { cs.Advance(2); return(HandleHex(cs, start)); } if (cs.CurrentChar == 'x' && CharacterStream.IsHex(cs.NextChar)) { cs.MoveToNextChar(); return(HandleHex(cs, start)); } int integerPartStart = cs.Position; int integerPartLength = 0; int fractionPartLength = 0; bool isDouble = false; // collect decimals (there may be none like in .1e+20 while (cs.IsDecimal()) { cs.MoveToNextChar(); integerPartLength++; } if (cs.CurrentChar == '.') { isDouble = true; // float/double cs.MoveToNextChar(); // If we've seen don we need to collect factional part of any while (cs.IsDecimal()) { cs.MoveToNextChar(); fractionPartLength++; } } if (integerPartLength + fractionPartLength == 0) { return(0); // +e or +.e is not a number and neither is lonely + or - } int numberLength; if (cs.CurrentChar == 'e' || cs.CurrentChar == 'E') { isDouble = true; numberLength = HandleExponent(cs, start); } else { numberLength = cs.Position - start; } // Verify double format if (isDouble && !IsValidDouble(cs, start, cs.Position)) { numberLength = 0; } if (numberLength > 0) { // skip over trailing 'L' if any if (cs.CurrentChar == 'L') { cs.MoveToNextChar(); numberLength++; } } return(numberLength); }