public static INumberLiteral Scan(TextInputRange input) { var chr = input.EndChar; if (!IsDecimalDigit(chr)) { return(null); } if (chr == '0') { var next = input.PeekChar(); switch (next) { case 'x': case 'X': input.Extend(nChars: 2); return(ScanNumber(input, radix: 16, isDigit: IsHexDigit)); case 'o': case 'O': input.Extend(nChars: 2); return(ScanNumber(input, radix: 8, isDigit: IsOctalDigit)); case 'b': case 'B': input.Extend(nChars: 2); return(ScanNumber(input, radix: 2, isDigit: IsBinaryDigit)); } } return(ScanDecimalNumber(input)); }
static bool HandleEscape(TextInputRange input, StringBuilder result) { input.Extend(); var chr = input.EndChar; switch (chr) { case '\0': return(false); case 't': result.Append(value: '\t'); break; case 'r': result.Append(value: '\r'); break; case 'n': result.Append(value: '\n'); break; // TODO: add Unicode escape handling default: result.Append(chr); break; } input.Extend(); return(true); }
static TokenData ScanSingleChar(TextInputRange input, Token token) { input.Extend(); return(new TokenData { Range = input.Clone(), Type = token }); }
static TokenData ScanNewLine(TextInputRange input) { input.NewLine(); input.ExtendWhitespaces(); return(new TokenData { Range = input.Clone(), Type = Token.NewLineIndentation }); }
static TokenData ScanWhitespaces(TextInputRange input) { input.Extend(); input.ExtendWhitespaces(); return(new TokenData { Range = input.Clone(), Type = Token.WhiteSpaceSeperator }); }
static TokenData ScanNumberLiteral(TextInputRange input) { var literal = NumberLiteralScanner.Scan(input); Debug.Assert(literal != null); // TODO: Create error token return(new TokenData { Range = literal.Range, Type = Token.NumberLiteral, Data = literal }); }
static TokenData ScanComment(TextInputRange input) { var comment = CommentScanner.Scan(input); Debug.Assert(comment); return(new TokenData { Range = input.Clone(), Type = Token.Comment }); }
// Scan basic double quoted strings public static IStringLiteral Scan(TextInputRange input) { var chr = input.EndChar; if (!IsDoubleQuote(chr)) { return(null); } input.Extend(); var result = new StringBuilder(); while (true) { chr = input.EndChar; if (chr == '\0') { return(null); // file end or invalid input } if (IsDoubleQuote(chr)) { break; } if (input.IsEndNewline) { input.NewLine(); } else { if (!IsTab(chr) && char.IsControl(chr)) { HandleControl(input); } else { if (IsBackslash(chr)) { if (!HandleEscape(input, result)) { return(null); } } else { result.Append(chr); input.Extend(); } } } } input.Extend(); return(new StringLiteral { Content = result.ToString(), Range = input.Clone() }); }
public void PeekCharEndTest() { var input = new TextInputRange { File = new TextFile { Content = "A", Filename = "" } }; Assert.AreEqual(expected: '\0', actual: input.PeekChar()); }
public void EndStringTest() { var input = new TextInputRange { File = new TextFile { Content = "ABC", Filename = "" } }; var s = input.EndString(nChars: 4); Assert.AreEqual(expected: "ABC", actual: s); }
public void IsNotKeywordTest() { var input = new TextInputRange { File = new TextFile { Content = "ABCD", Filename = "" } }; var s = input.IsKeyword(word: "ABC"); Assert.AreEqual(expected: false, actual: s); }
public void SkipTest() { var input = new TextInputRange { File = new TextFile { Content = "ABC ", Filename = "" } }; input.Skip(nChars: 2); Assert.AreEqual(expected: 'C', actual: input.EndChar); }
public void ExtendTest() { var input = new TextInputRange { File = new TextFile { Content = "ABC ", Filename = "" } }; input.Extend(nChars: 2); Assert.AreEqual(expected: "AB", actual: input.Text); Assert.AreEqual(expected: 3, actual: input.End.Column); }
public void TestNegativeBlockComments(string content) { var input = new TextInputRange { File = new TextFile { Content = content, Filename = "" } }; var exception = Assert.Throws <Exception>(() => CommentScanner.Scan(input)); Assert.True(exception.Message.Equals(value: "Line Comment not Escaped.")); }
public void ScanFailures(string content) { var input = new TextInputRange { File = new TextFile { Content = content, Filename = "" } }; var result = NumberLiteralScanner.Scan(input); Assert.IsNull(result); }
public void PeekCharStartTest() { var input = new TextInputRange { File = new TextFile { Content = "AB", Filename = "" } }; var t = input.PeekChar(); Assert.AreEqual(expected: 'B', actual: t); }
[TestCase(arg: "(+ )", TestName = "space is hard separator")] // closing bracket is missing, no entry left public void ScanNewFailure(string content) { var input = new TextInputRange { File = new TextFile { Content = content, Filename = "" } }; var result = OperatorScanner.Scan(input); Assert.IsNull(result); }
public void TestNegativeLineComments(string content) { var input = new TextInputRange { File = new TextFile { Content = content, Filename = "" } }; var output = CommentScanner.Scan(input); Assert.IsFalse(output); Assert.AreEqual(content[index: 0], input.EndChar); }
public void TestPositiveLineComments(string content) { var input = new TextInputRange { File = new TextFile { Content = content, Filename = "" } }; var output = CommentScanner.Scan(input); Assert.IsTrue(output); Assert.IsFalse(input.IsEndValid); }
/** * # single line * * #<optional># * block comments * #<optional># */ public static bool Scan(TextInputRange range) { if (range.EndChar != '#') { return(false); } do { range.Extend(); } while (range.IsEndValid && !range.IsEndNewline && !range.IsEndWhitespace && range.EndChar != '#'); // single line comment if (range.IsEndNewline) { return(true); } // is block comment if (range.EndChar == '#') { range.Extend(); var commentMarker = range.Text; while (range.IsEndValid && range.EndString(commentMarker.Length) != commentMarker) { if (range.IsEndNewline) { range.NewLine(); } else { range.Extend(); } } if (range.EndString(commentMarker.Length) != commentMarker) { throw new Exception(message: "Line Comment not Escaped."); } range.Extend(commentMarker.Length); } else { do { range.Extend(); } while (range.IsEndValid && !range.IsEndNewline); } return(true); }
public void ScanSuccess(string content, string output) { var input = new TextInputRange { File = new TextFile { Content = content, Filename = "" } }; var result = StringLiteralScanner.Scan(input); Assert.IsNotNull(result); Assert.AreEqual(output, result.Content); }
public void CollapseWhitespacesTest() { var input = new TextInputRange { File = new TextFile { Content = "AB C", Filename = "" } }; input.Extend(nChars: 2); input.CollapseWhitespaces(); Assert.AreEqual(expected: 'C', actual: input.EndChar); Assert.AreEqual(expected: "", actual: input.Text); }
public void ScanNewSuccess(string content, string id) { var input = new TextInputRange { File = new TextFile { Content = content, Filename = "" } }; var result = OperatorScanner.Scan(input); Assert.IsNotNull(result); Assert.AreEqual(id, result.Content); }
static bool IsStart(TextInputRange input) { var chr = input.EndChar; if (chr == '.') { input.Extend(); // dot is allowed at start but not later chr = input.EndChar; } if ('0' <= chr && '9' >= chr) { return(false); // digits are not allowed at start } return(IsContinue(input)); // no more special rules }
public void TestNegativeLineEndLineComments(string content) { var input = new TextInputRange { File = new TextFile { Content = content, Filename = "" } }; var output = CommentScanner.Scan(input); Assert.IsTrue(output); Assert.IsTrue(input.IsEndValid); Assert.AreEqual(expected: '\n', actual: input.EndChar); }
public void TestPositiveBlockComments(string content, int line) { var input = new TextInputRange { File = new TextFile { Content = content, Filename = "" } }; var output = CommentScanner.Scan(input); Assert.IsTrue(output); Assert.IsFalse(input.IsEndValid); Assert.AreEqual(line, input.End.Line); }
// this will scan the longest valid regular entry public static IIdentifierLiteral Scan(TextInputRange input) { if (!IsStart(input)) { input.Backtrack(); // dot might have been skipped return(null); } do { input.Extend(); } while (IsContinue(input)); return(new IdentifierLiteral { Content = input.Text, Range = input.Clone() }); }
public void ScanSuccess(string content, int radix, string integerPart, string fractionalPart, string exponentPart) { var input = new TextInputRange { File = new TextFile { Content = content, Filename = "" } }; var result = NumberLiteralScanner.Scan(input); Assert.IsNotNull(result); Assert.AreEqual(radix, result.Radix); Assert.AreEqual(integerPart, result.IntegerPart); Assert.AreEqual(fractionalPart, result.FractionalPart); Assert.AreEqual(exponentPart, result.ExponentPart); }
static bool IsContinue(TextInputRange input) { var chr = input.EndChar; switch (CharUnicodeInfo.GetUnicodeCategory(chr)) { case UnicodeCategory.LowercaseLetter: // lower case UnicodeCategory.UppercaseLetter: // UPPER case UnicodeCategory.DecimalDigitNumber: // 0-9 case UnicodeCategory.ConnectorPunctuation: // various underscores _‿⁀⁔_ return(true); case UnicodeCategory.TitlecaseLetter: // Ligatures Dž, Lj, Nj and Dz case UnicodeCategory.OtherLetter: // Hebrew etc. א case UnicodeCategory.MathSymbol: // +=- case UnicodeCategory.OtherSymbol: // ©®⌛⌚ case UnicodeCategory.DashPunctuation: // all kinds of hyphens case UnicodeCategory.LetterNumber: // Roman Numbers (look like normal letters) case UnicodeCategory.OtherNumber: // ½² case UnicodeCategory.PrivateUse: // emojis etc. case UnicodeCategory.CurrencySymbol: // ¢¥$€ case UnicodeCategory.OtherPunctuation: // like !?#. case UnicodeCategory.OpenPunctuation: // [{( case UnicodeCategory.InitialQuotePunctuation: // “« case UnicodeCategory.ClosePunctuation: // ]}) case UnicodeCategory.FinalQuotePunctuation: // ”» case UnicodeCategory.ModifierLetter: case UnicodeCategory.NonSpacingMark: case UnicodeCategory.SpacingCombiningMark: case UnicodeCategory.EnclosingMark: case UnicodeCategory.SpaceSeparator: case UnicodeCategory.LineSeparator: case UnicodeCategory.ParagraphSeparator: case UnicodeCategory.Control: case UnicodeCategory.Format: case UnicodeCategory.Surrogate: case UnicodeCategory.ModifierSymbol: case UnicodeCategory.OtherNotAssigned: return(false); default: throw new ArgumentOutOfRangeException(); } }
static void HandleControl(TextInputRange input) { // do not add arbitrary control characters to internal strings. input.Extend(); }