private static Token BlockComment(Tokenizer t) { int i = 0, nest = 1; if (!t.IsReadable(i + 1) || t.Read(i, 2) != "/*") { return Token.Empty; } for (i = 2; t.IsReadable(i + 1); i++) { if (t.Read(i, 2) == "*/") { ++i; if (--nest == 0) { break; } } if (t.Read(i, 2) == "/*") { ++i; ++nest; } } return t.TakeToken(++i, TokenType.BlockComment); }
private static Token TriplePunctuator(Tokenizer t) { TokenType type = TokenType.Unknoun; string sub = t.Read(0, 3); switch (sub) { case ":=:": type = TokenType.Swap; break; case "=<>": type = TokenType.Incomparable; break; case "=><": type = TokenType.Incomparable; break; case "<=>": type = TokenType.Incomparable; break; case ">=<": type = TokenType.Incomparable; break; case "<>=": type = TokenType.Incomparable; break; case "><=": type = TokenType.Incomparable; break; case "<<=": type = TokenType.LeftCompose | TokenType.LeftPipeline; break; case "=<<": type = TokenType.LeftCompose | TokenType.RightPipeline; break; case ">>=": type = TokenType.RightCompose | TokenType.LeftPipeline; break; case "=>>": type = TokenType.RightCompose | TokenType.RightPipeline; break; default: return Token.Empty; } return t.TakeToken(3, type); }
private static bool StringLiteral(Tokenizer t, List<Token> tokenList, List<Token> errorToken) { if (!t.IsReadable(0) || !t.MatchAny(0, "\'\"`")) { return false; } string quote = t.Read(0, 1); bool escape = false; tokenList.Add(t.TakeToken(1, TokenType.QuoteSeparator)); int i; for (i = 0; t.IsReadable(i); i++) { if (!escape && t.MatchAny(i, quote)) { if (i > 0) { tokenList.Add(t.TakeToken(i, TokenType.PlainText)); } tokenList.Add(t.TakeToken(1, TokenType.QuoteSeparator)); return true; } if (!escape && t.MatchAny(i, "{")) { if (i > 0) { tokenList.Add(t.TakeToken(i, TokenType.PlainText)); } BuiltInExpression(t, tokenList, errorToken); i = -1; } else if (t.MatchAny(i, "\\")) { escape = !escape; continue; } escape = false; } tokenList.Add(t.TakeToken(i, TokenType.PlainText)); return true; }
private static Token SinglePunctuator(Tokenizer t) { TokenType type = TokenType.Unknoun; string sub = t.Read(0, 1); switch (sub) { case ";": type = TokenType.EndExpression; break; case ":": type = TokenType.Pair; break; case ",": type = TokenType.List; break; case ".": type = TokenType.Access; break; case "#": type = TokenType.Zone; break; case "@": type = TokenType.Attribute; break; case "$": type = TokenType.Lambda; break; case "?": type = TokenType.Reject; break; case "!": type = TokenType.Template; break; case "|": type = TokenType.Typeof; break; case "&": type = TokenType.Refer; break; case "=": type = TokenType.Equal; break; case "<": type = TokenType.LessThan; break; case ">": type = TokenType.GreaterThan; break; case "~": type = TokenType.Combine; break; case "+": type = TokenType.Add; break; case "-": type = TokenType.Subtract; break; case "*": type = TokenType.Multiply; break; case "/": type = TokenType.Divide; break; case "%": type = TokenType.Modulo; break; case "(": type = TokenType.LeftParenthesis; break; case ")": type = TokenType.RightParenthesis; break; case "[": type = TokenType.LeftBracket; break; case "]": type = TokenType.RightBracket; break; case "{": type = TokenType.LeftBrace; break; case "}": type = TokenType.RightBrace; break; default: return Token.Empty; } return t.TakeToken(1, type); }
private static Token LineCommnet(Tokenizer t) { int i = 0; if (!t.IsReadable(i + 1)) { return Token.Empty; } if (t.Read(i, 2) != "//" && t.Read(i, 2) != "#!") { return Token.Empty; } for (i = 2; t.IsReadable(i); i++) { if (t.MatchAny(i, "\x0A\x0D")) { break; } } return t.TakeToken(i, TokenType.LineCommnet); }
private static Token DoublePunctuator(Tokenizer t) { TokenType type = TokenType.Unknoun; string sub = t.Read(0, 2); switch (sub) { case "->": type = TokenType.ReturnArrow; break; case "::": type = TokenType.Separator; break; case "..": type = TokenType.Range; break; case "@@": type = TokenType.Pragma; break; case "##": type = TokenType.Macro; break; case "??": type = TokenType.Nullable; break; case "||": type = TokenType.Or; break; case "&&": type = TokenType.And; break; case "!!": type = TokenType.Not; break; case "++": type = TokenType.Plus; break; case "--": type = TokenType.Minus; break; case "==": type = TokenType.Equal; break; case "<>": type = TokenType.NotEqual; break; case "><": type = TokenType.NotEqual; break; case "<=": type = TokenType.LessThanOrEqual; break; case "=<": type = TokenType.LessThanOrEqual; break; case ">=": type = TokenType.GreaterThanOrEqual; break; case "=>": type = TokenType.GreaterThanOrEqual; break; case "<<": type = TokenType.LeftCompose; break; case ">>": type = TokenType.RightCompose; break; case ":=": type = TokenType.LeftPipeline; break; case "=:": type = TokenType.RightPipeline; break; case "+=": type = TokenType.Add | TokenType.LeftPipeline; break; case "=+": type = TokenType.Add | TokenType.RightPipeline; break; case "-=": type = TokenType.Subtract | TokenType.LeftPipeline; break; case "=-": type = TokenType.Subtract | TokenType.RightPipeline; break; case "*=": type = TokenType.Multiply | TokenType.LeftPipeline; break; case "=*": type = TokenType.Multiply | TokenType.RightPipeline; break; case "/=": type = TokenType.Divide | TokenType.LeftPipeline; break; case "=/": type = TokenType.Divide | TokenType.RightPipeline; break; case "%=": type = TokenType.Modulo | TokenType.LeftPipeline; break; case "=%": type = TokenType.Modulo | TokenType.RightPipeline; break; default: return Token.Empty; } return t.TakeToken(2, type); }
public void Read(string text, int index, int length, string expected) { Tokenizer t = new Tokenizer(text, string.Empty); Assert.That(t.Read(index, length), Is.EqualTo(expected)); }