public bool ReadToken(TokenList parentList, List <TokenKind> terminators) { Advance(); if (terminators.Contains(CurrentToken.Kind)) { return(false); } //else switch (CurrentToken.Kind) { case TokenKind.Name: parentList.Add(ReadComplexName(TokenKind.Name)); break; case TokenKind.Dollar: Advance(); parentList.Add(ReadComplexName(TokenKind.Variable)); break; case TokenKind.LeftRound: ReadTokenList(parentList, TokenKind.RoundList, RoundListTerminators); break; case TokenKind.LeftSquare: ReadTokenList(parentList, TokenKind.SquareList, SquareListTerminators); break; case TokenKind.LeftAngle: ReadTokenList(parentList, TokenKind.SquareList, AngleListTerminators); break; case TokenKind.LineEnd: if (NextToken.Kind == TokenKind.Indent) { Advance(); ReadTokenLists(parentList); return(false); } return(false); //break; case TokenKind.Dedent: throw new Escape(); default: parentList.Add(CurrentToken); break; } return(true); }
public TokenList Tokenize(Stream str) { StreamReader sr = new StreamReader(str); String line; TokenList tokenList = new TokenList(); try { TokenList tokList = null; while ((line = sr.ReadLine()) != null) { tokList = TokenizeLine(line); if (tokList != null) { tokenList.AddRange(tokList); } } } finally { str.Close(); } tokenList.Add(new Token(TokenKind.FileEnd, 0)); return(tokenList); }
void Dedent(int indentLevel, TokenList tokenList) { while (IndentStack.Peek() != indentLevel) { tokenList.Add(new Token(TokenKind.Dedent, IndentStack.Pop())); } }
public bool ReadTokenList(TokenList parentList, TokenKind listKind, List <TokenKind> terminators) { TokenList outList = new TokenList(); bool more = true; while (more) { more = ReadToken(outList, terminators); } if (listKind != TokenKind.RoundList && outList.Count == 0) { return(true); } //else parentList.Add(new Token(listKind, outList)); return(true); }
Token ReadComplexName(TokenKind targetKind) { if (NextToken.Kind != TokenKind.Dot) { if (targetKind == TokenKind.Name) { return(CurrentToken); } else { return(new Token(CurrentToken, targetKind)); } } //else //TODO:Need to test this more TokenList nameList = new TokenList(); do { TokenKind kind = CurrentToken.Kind; if (kind == TokenKind.Name) { nameList.Add(CurrentToken); if (NextToken.Kind != TokenKind.Dot) { break; } } else if (kind == TokenKind.Dot) { continue; } else { break; } } while (Advance()); return(new Token(targetKind, new Token(TokenKind.DottedName, nameList))); }
private TokenList TokenizeLine(string line) { int oldLength = line.Length; line = line.TrimStart(' '); if (line == "") { return(null); } //else TokenList tokenList = new TokenList(); // int indentLevel = oldLength - line.Length; indentLevel += ExtraIndent; if (indentLevel > IndentStack.Peek()) { Indent(indentLevel, tokenList); } else if (indentLevel < IndentStack.Peek()) { Dedent(indentLevel, tokenList); } // Regex regexPattern = new Regex(TokenInfo.PatternString); MatchCollection matches = regexPattern.Matches(line); foreach (Match match in matches) { TokenKind i = 0; TokenInfo reserved; TokenKind code = 0; foreach (Group group in match.Groups) { string matchValue = group.Value; bool success = group.Success; // ignore capture index 0 and 1 (general and WhiteSpace) if (success && i > 0) { code = i; switch (i) { case TokenKind.Name: matchValue = matchValue.Replace('-', '_'); if (TokenInfo.Keywords.TryGetValue(matchValue, out reserved)) { code = reserved.Kind; } tokenList.Add(CreateToken(code, matchValue, line)); break; case TokenKind.Directive: matchValue = matchValue.Replace('-', '_'); matchValue = matchValue.Replace("#", "pound"); if (TokenInfo.Directives.TryGetValue(matchValue, out reserved)) { code = reserved.Kind; } switch (code) { case TokenKind.PoundIndent: ExtraIndent += 4; return(null); case TokenKind.PoundDedent: ExtraIndent -= 4; return(null); } break; case TokenKind.Type: matchValue = matchValue.Replace('-', '_'); matchValue = matchValue.TrimEnd(':'); tokenList.Add(CreateToken(code, matchValue, line)); break; case TokenKind.Property: matchValue = matchValue.Replace('-', '_'); matchValue = matchValue.TrimStart(':'); tokenList.Add(CreateToken(code, matchValue, line)); break; default: tokenList.Add(CreateToken(code, matchValue, line)); break; } } i++; } } tokenList.Add(new Token(TokenKind.LineEnd, indentLevel)); return(tokenList); }
void Indent(int indentLevel, TokenList tokenList) { IndentStack.Push(indentLevel); tokenList.Add(new Token(TokenKind.Indent, indentLevel)); }