Ejemplo n.º 1
0
 private static Token BlockComment(Tokenizer t)
 {
     int i = 0, nest = 1;
     if (!t.IsReadable(i + 1) || t.Read(i, 2) != "/*")
     {
         return Token.Empty;
     }
     for (i = 2; t.IsReadable(i + 1); i++)
     {
         if (t.Read(i, 2) == "*/")
         {
             ++i;
             if (--nest == 0)
             {
                 break;
             }
         }
         if (t.Read(i, 2) == "/*")
         {
             ++i;
             ++nest;
         }
     }
     return t.TakeToken(++i, TokenType.BlockComment);
 }
Ejemplo n.º 2
0
 public void DigitStartString(string text, string eText, TokenType eType)
 {
     var t = new Tokenizer(text, string.Empty);
     var token = (Token)typeof(Lexer).Invoke("DigitStartString", t);
     Assert.That(token.Text, Is.EqualTo(eText));
     Assert.That(token.TokenType, Is.EqualTo(eType));
 }
Ejemplo n.º 3
0
 public void BlockComment(string text, string eText, TokenType eType)
 {
     var t = new Tokenizer(text, string.Empty);
     var token = (Token)typeof(Lexer).Invoke("BlockComment", t);
     Assert.That(token.Text, Is.EqualTo(eText));
     Assert.That(token.TokenType, Is.EqualTo(eType));
 }
Ejemplo n.º 4
0
 public static TokenCollection Lex(string text, string fileName)
 {
     var tokenList = new List<Token>();
     var errorToken = new List<Token>();
     var t = new Tokenizer(text, fileName);
     while (t.IsReadable())
     {
         LexPartion(t, tokenList, errorToken);
     }
     return new TokenCollection(text, fileName, tokenList, errorToken, t.Position);
 }
Ejemplo n.º 5
0
 public void BuiltInExpression(string text, string[] eTexts, TokenType[] eTypes)
 {
     var t = new Tokenizer(text, string.Empty);
     var tokenList = new List<Token>();
     var errorToken = new List<Token>();
     typeof(Lexer).Invoke("BuiltInExpression", t, tokenList, errorToken);
     Assert.That(tokenList, Is.All.Not.Null);
     Assert.That(List.Map(tokenList).Property("Text"), Is.EqualTo(eTexts));
     Assert.That(List.Map(tokenList).Property("TokenType"), Is.EqualTo(eTypes));
     Assert.That(errorToken.Count, Is.EqualTo(0));
 }
Ejemplo n.º 6
0
 private static Token DigitStartString(Tokenizer t)
 {
     int i;
     for (i = 0; t.IsReadable(i); i++)
     {
         if (t.MatchRange(i, '0', '9'))
         {
             continue;
         }
         if (i > 0 && (t.MatchRange(i, 'a', 'z') || t.MatchRange(i, 'A', 'Z') || t.MatchAny(i, "_")))
         {
             continue;
         }
         break;
     }
     return t.TakeToken(i, TokenType.DigitStartString);
 }
Ejemplo n.º 7
0
 public void TakeToken(string text, int length, TokenType type, string eText, bool eLine)
 {
     Tokenizer t = new Tokenizer(text, "file");
     var token = t.TakeToken(length, type);
     if (!token)
     {
         Assert.That(token, Is.EqualTo(Token.Empty));
     }
     else
     {
         Assert.That(token.Text, Is.EqualTo(eText));
         Assert.That(token.TokenType, Is.EqualTo(type));
         Assert.That(token.Position.File, Is.EqualTo("file"));
         Assert.That(token.Position.Total, Is.EqualTo(0));
         Assert.That(token.Position.Row, Is.EqualTo(0));
         Assert.That(token.Position.Length, Is.EqualTo(length));
         Assert.That(token.Position.Line, Is.EqualTo(1));
         Assert.That(t.Position.Total, Is.EqualTo(length));
         Assert.That(t.Position.Row, Is.EqualTo(eLine ? 0 : length));
         Assert.That(t.Position.Length, Is.EqualTo(0));
         Assert.That(t.Position.Line, Is.EqualTo(eLine ? 2 : 1));
     }
 }
Ejemplo n.º 8
0
 private static void BuiltInExpression(Tokenizer t, List<Token> tokenList, List<Token> errorToken)
 {
     if (!t.IsReadable(0) || !t.MatchAny(0, "{"))
     {
         return;
     }
     var result = new List<Token>();
     int nest = 0;
     while (t.IsReadable())
     {
         var tt = LexPartion(t, tokenList, errorToken);
         if (tt == TokenType.LeftBrace)
         {
             ++nest;
         }
         if (tt == TokenType.RightBrace)
         {
             if (--nest == 0)
             {
                 break;
             }
         }
     }
 }
Ejemplo n.º 9
0
 private static bool StringLiteral(Tokenizer t, List<Token> tokenList, List<Token> errorToken)
 {
     if (!t.IsReadable(0) || !t.MatchAny(0, "\'\"`"))
     {
         return false;
     }
     string quote = t.Read(0, 1);
     bool escape = false;
     tokenList.Add(t.TakeToken(1, TokenType.QuoteSeparator));
     int i;
     for (i = 0; t.IsReadable(i); i++)
     {
         if (!escape && t.MatchAny(i, quote))
         {
             if (i > 0)
             {
                 tokenList.Add(t.TakeToken(i, TokenType.PlainText));
             }
             tokenList.Add(t.TakeToken(1, TokenType.QuoteSeparator));
             return true;
         }
         if (!escape && t.MatchAny(i, "{"))
         {
             if (i > 0)
             {
                 tokenList.Add(t.TakeToken(i, TokenType.PlainText));
             }
             BuiltInExpression(t, tokenList, errorToken);
             i = -1;
         }
         else if (t.MatchAny(i, "\\"))
         {
             escape = !escape;
             continue;
         }
         escape = false;
     }
     tokenList.Add(t.TakeToken(i, TokenType.PlainText));
     return true;
 }
Ejemplo n.º 10
0
 private static Token TriplePunctuator(Tokenizer t)
 {
     TokenType type = TokenType.Unknoun;
     string sub = t.Read(0, 3);
     switch (sub)
     {
         case ":=:": type = TokenType.Swap; break;
         case "=<>": type = TokenType.Incomparable; break;
         case "=><": type = TokenType.Incomparable; break;
         case "<=>": type = TokenType.Incomparable; break;
         case ">=<": type = TokenType.Incomparable; break;
         case "<>=": type = TokenType.Incomparable; break;
         case "><=": type = TokenType.Incomparable; break;
         case "<<=": type = TokenType.LeftCompose | TokenType.LeftPipeline; break;
         case "=<<": type = TokenType.LeftCompose | TokenType.RightPipeline; break;
         case ">>=": type = TokenType.RightCompose | TokenType.LeftPipeline; break;
         case "=>>": type = TokenType.RightCompose | TokenType.RightPipeline; break;
         default: return Token.Empty;
     }
     return t.TakeToken(3, type);
 }
Ejemplo n.º 11
0
 public void DisjunctionLexer1()
 {
     var tokenizer = new Tokenizer("abc", string.Empty);
     var token = (Token)typeof(Lexer).Invoke("DisjunctionLexer", tokenizer, new Lexer.LexerFunction[] { t => Token.Empty, t => t.TakeToken(2, TokenType.Unknoun) });
     Assert.That(token.Text, Is.EqualTo("ab"));
 }
Ejemplo n.º 12
0
 private static Token SinglePunctuator(Tokenizer t)
 {
     TokenType type = TokenType.Unknoun;
     string sub = t.Read(0, 1);
     switch (sub)
     {
         case ";": type = TokenType.EndExpression; break;
         case ":": type = TokenType.Pair; break;
         case ",": type = TokenType.List; break;
         case ".": type = TokenType.Access; break;
         case "#": type = TokenType.Zone; break;
         case "@": type = TokenType.Attribute; break;
         case "$": type = TokenType.Lambda; break;
         case "?": type = TokenType.Reject; break;
         case "!": type = TokenType.Template; break;
         case "|": type = TokenType.Typeof; break;
         case "&": type = TokenType.Refer; break;
         case "=": type = TokenType.Equal; break;
         case "<": type = TokenType.LessThan; break;
         case ">": type = TokenType.GreaterThan; break;
         case "~": type = TokenType.Combine; break;
         case "+": type = TokenType.Add; break;
         case "-": type = TokenType.Subtract; break;
         case "*": type = TokenType.Multiply; break;
         case "/": type = TokenType.Divide; break;
         case "%": type = TokenType.Modulo; break;
         case "(": type = TokenType.LeftParenthesis; break;
         case ")": type = TokenType.RightParenthesis; break;
         case "[": type = TokenType.LeftBracket; break;
         case "]": type = TokenType.RightBracket; break;
         case "{": type = TokenType.LeftBrace; break;
         case "}": type = TokenType.RightBrace; break;
         default: return Token.Empty;
     }
     return t.TakeToken(1, type);
 }
Ejemplo n.º 13
0
 private static Token LineTerminator(Tokenizer t)
 {
     int i = 0;
     if (t.IsReadable(i) && t.MatchAny(i, "\x0A"))
     {
         i++;
         if (t.IsReadable(i) && t.MatchAny(i, "\x0D"))
         {
             i++;
         }
     }
     else if (t.IsReadable(i) && t.MatchAny(i, "\x0D"))
     {
         i++;
         if (t.IsReadable(i) && t.MatchAny(i, "\x0A"))
         {
             i++;
         }
     }
     return t.TakeToken(i, TokenType.LineTerminator);
 }
Ejemplo n.º 14
0
 private static Token OtherString(Tokenizer t)
 {
     int i;
     for (i = 0; t.IsReadable(i); i++)
     {
         if (!t.MatchRange(i, '\x00', '\x7F'))
         {
             continue;
         }
         break;
     }
     return t.TakeToken(i, TokenType.OtherString);
 }
Ejemplo n.º 15
0
 private static TokenType LexPartion(Tokenizer t, List<Token> tokenList, List<Token> errorToken)
 {
     Token temp = LineTerminator(t);
     if(temp)
     {
         tokenList.Add(temp);
         return temp.TokenType;
     }
     temp = DisjunctionLexer
         (
         t,
         WhiteSpace,
         BlockComment,
         LineCommnet
         );
     if (temp)
     {
         return temp.TokenType;
     }
     if (StringLiteral(t, tokenList, errorToken))
     {
         return TokenType.PlainText;
     }
     temp = DisjunctionLexer
         (
         t,
         TriplePunctuator,
         DoublePunctuator,
         SinglePunctuator,
         LetterStartString,
         DigitStartString
         );
     if (temp)
     {
         tokenList.Add(temp);
         return temp.TokenType;
     }
     errorToken.Add(OtherString(t));
     return TokenType.OtherString;
 }
Ejemplo n.º 16
0
 private static Token LineCommnet(Tokenizer t)
 {
     int i = 0;
     if (!t.IsReadable(i + 1))
     {
         return Token.Empty;
     }
     if (t.Read(i, 2) != "//" && t.Read(i, 2) != "#!")
     {
         return Token.Empty;
     }
     for (i = 2; t.IsReadable(i); i++)
     {
         if (t.MatchAny(i, "\x0A\x0D"))
         {
             break;
         }
     }
     return t.TakeToken(i, TokenType.LineCommnet);
 }
Ejemplo n.º 17
0
 public void StringLiteral(string text, string[] eTexts, TokenType[] eTypes)
 {
     var t = new Tokenizer(text, string.Empty);
     var tokenList = new List<Token>();
     var errorToken = new List<Token>();
     var result = (bool)typeof(Lexer).Invoke("StringLiteral", t, tokenList, errorToken);
     Assert.That(result, Is.EqualTo(eTexts.Length > 0));
     Assert.That(tokenList, Is.All.Not.Null);
     Assert.That(List.Map(tokenList).Property("Text"), Is.EqualTo(eTexts));
     Assert.That(List.Map(tokenList).Property("TokenType"), Is.EqualTo(eTypes));
     Assert.That(errorToken.Count, Is.EqualTo(0));
 }
Ejemplo n.º 18
0
 public void DisjunctionLexer2()
 {
     var tokenizer = new Tokenizer("abc", string.Empty);
     var token = (Token)typeof(Lexer).Invoke("DisjunctionLexer", tokenizer, new Lexer.LexerFunction[] { t => Token.Empty, t => Token.Empty });
     Assert.That(token, Is.EqualTo(Token.Empty));
 }
Ejemplo n.º 19
0
 public void IsReadable(string text, int index, bool expected)
 {
     Tokenizer t = new Tokenizer(text, string.Empty);
     Assert.That(t.IsReadable(index), Is.EqualTo(expected));
 }
Ejemplo n.º 20
0
 private static Token WhiteSpace(Tokenizer t)
 {
     int i;
     for (i = 0; t.IsReadable(i); i++)
     {
         if (t.MatchRange(i, '\x00', '\x09') || t.MatchRange(i, '\x0B', '\x0C') || t.MatchRange(i, '\x0E', '\x20') || t.MatchAny(i, "\x7F"))
         {
             continue;
         }
         break;
     }
     return t.TakeToken(i, TokenType.WhiteSpace);
 }
Ejemplo n.º 21
0
 public void Read(string text, int index, int length, string expected)
 {
     Tokenizer t = new Tokenizer(text, string.Empty);
     Assert.That(t.Read(index, length), Is.EqualTo(expected));
 }
Ejemplo n.º 22
0
 public void MatchRange(string text, char start, char end, bool expected)
 {
     Tokenizer t = new Tokenizer(text, string.Empty);
     Assert.That(t.MatchRange(0, start, end), Is.EqualTo(expected));
 }
Ejemplo n.º 23
0
 public void MatchAny(string text, string list, bool expected)
 {
     Tokenizer t = new Tokenizer(text, string.Empty);
     Assert.That(t.MatchAny(0, list), Is.EqualTo(expected));
 }
Ejemplo n.º 24
0
 private static Token DoublePunctuator(Tokenizer t)
 {
     TokenType type = TokenType.Unknoun;
     string sub = t.Read(0, 2);
     switch (sub)
     {
         case "->": type = TokenType.ReturnArrow; break;
         case "::": type = TokenType.Separator; break;
         case "..": type = TokenType.Range; break;
         case "@@": type = TokenType.Pragma; break;
         case "##": type = TokenType.Macro; break;
         case "??": type = TokenType.Nullable; break;
         case "||": type = TokenType.Or; break;
         case "&&": type = TokenType.And; break;
         case "!!": type = TokenType.Not; break;
         case "++": type = TokenType.Plus; break;
         case "--": type = TokenType.Minus; break;
         case "==": type = TokenType.Equal; break;
         case "<>": type = TokenType.NotEqual; break;
         case "><": type = TokenType.NotEqual; break;
         case "<=": type = TokenType.LessThanOrEqual; break;
         case "=<": type = TokenType.LessThanOrEqual; break;
         case ">=": type = TokenType.GreaterThanOrEqual; break;
         case "=>": type = TokenType.GreaterThanOrEqual; break;
         case "<<": type = TokenType.LeftCompose; break;
         case ">>": type = TokenType.RightCompose; break;
         case ":=": type = TokenType.LeftPipeline; break;
         case "=:": type = TokenType.RightPipeline; break;
         case "+=": type = TokenType.Add | TokenType.LeftPipeline; break;
         case "=+": type = TokenType.Add | TokenType.RightPipeline; break;
         case "-=": type = TokenType.Subtract | TokenType.LeftPipeline; break;
         case "=-": type = TokenType.Subtract | TokenType.RightPipeline; break;
         case "*=": type = TokenType.Multiply | TokenType.LeftPipeline; break;
         case "=*": type = TokenType.Multiply | TokenType.RightPipeline; break;
         case "/=": type = TokenType.Divide | TokenType.LeftPipeline; break;
         case "=/": type = TokenType.Divide | TokenType.RightPipeline; break;
         case "%=": type = TokenType.Modulo | TokenType.LeftPipeline; break;
         case "=%": type = TokenType.Modulo | TokenType.RightPipeline; break;
         default: return Token.Empty;
     }
     return t.TakeToken(2, type);
 }
Ejemplo n.º 25
0
 private static Token LetterStartString(Tokenizer t)
 {
     int i;
     bool escape = false;
     for (i = 0; t.IsReadable(i); i++)
     {
         if (escape && t.MatchRange(i, '!', '~'))
         {
             escape = false;
             continue;
         }
         if (t.MatchRange(i, 'a', 'z') || t.MatchRange(i, 'A', 'Z') || t.MatchAny(i, "_"))
         {
             escape = false;
             continue;
         }
         if (i > 0 && t.MatchRange(i, '0', '9'))
         {
             escape = false;
             continue;
         }
         if (t.MatchAny(i, "\\"))
         {
             escape = !escape;
             continue;
         }
         break;
     }
     return t.TakeToken(i, TokenType.LetterStartString);
 }
Ejemplo n.º 26
0
 private static Token DisjunctionLexer(Tokenizer t, params LexerFunction[] func)
 {
     foreach (var f in func)
     {
         var token = f(t);
         if (token)
         {
             return token;
         }
     }
     return Token.Empty;
 }
Ejemplo n.º 27
0
 public void LineTerminator(string text, string eText, TokenType eType)
 {
     var t = new Tokenizer(text, string.Empty);
     var token = (Token)typeof(Lexer).Invoke("LineTerminator", t);
     Assert.That(token.Text, Is.EqualTo(eText));
     Assert.That(token.TokenType, Is.EqualTo(eType));
 }