public static Scripting.Parser.TokenCollection JoinTokens(this List <Scripting.Parser.TokenCollection> tokens) { Scripting.Parser.TokenCollection collections = new Scripting.Parser.TokenCollection(); foreach (var item in tokens) { collections.AddRange(item); collections.Add(Scripting.Parser.Token.LineBreak); } return(collections); }
public TokenCollection Tokenize(string source) { int lineCount = 1; int columnCount = 1; TokenCollection collection = new TokenCollection(); string[] lines = source.Split('\n'); foreach (string line in lines) { foreach (char character in line) { this.Tokenize(collection, character, lineCount, columnCount); columnCount++; } if (collection.Count == 0) { collection.Add(new Token(Token.LineBreak.Value, new Span(lineCount, columnCount, lineCount, columnCount + 1), TokenType.Newline)); } else if (collection.Last().ContentEquals(Token.LineContinue)) { collection.RemoveLast(); } else { collection.Add(new Token(Token.LineBreak.Value, new Span(lineCount, columnCount, lineCount, columnCount + 1), TokenType.Newline)); } columnCount = 1; lineCount++; } collection.RemoveAll((token) => { return(token.Type == TokenType.Whitespace); }); return(collection); }
// the methods Split, Contains has a corresponding *Cascade version. this means the method // will check whether the specified symbol is in identical level of blocks as the baseline. // taking nested blocks into consideration. // for example, the following token collection: // data a ( int : c ) : b // applying Split(":"), the sections are: // [1] data a ( int // [2] c ) // [3] b // applying SplitCascade(":"), the sections are: // [1] data a ( int : c ) // [2] b public List <TokenCollection> Split(Token token) { List <TokenCollection> result = new List <TokenCollection>(); TokenCollection collection = new TokenCollection(); foreach (var item in this) { if (item.ContentEquals(token)) { result.Add(collection); collection = new TokenCollection(); } else { collection.Add(item); } } if (collection.Count > 0) { result.Add(collection); } return(result); }
public LexicalError TokenizeDigit(TokenCollection collection, char character, int line, int column) { if (!character.IsDigit() && character != '.') { return(LexicalError.RuleNotMatch); } if (collection.Count == 0) { if (character.IsDigit()) { collection.Add(new Token(character.ToString(), new Span(line, column, line, column), TokenType.IntegerLiteral)); return(LexicalError.Ok); } else { return(LexicalError.RuleNotMatch); } } var previous = collection.Last(); switch (previous.Type) { case TokenType.IntegerLiteral: if (character.IsDigit()) { previous.Value += character; previous.Location.End = new Position(line, column); } else if (character == '.') { previous.Value += character; previous.Type = TokenType.FloatingLiteral; previous.Location.End = new Position(line, column); } else { return(LexicalError.RuleNotMatch); } return(LexicalError.Ok); case TokenType.FloatingLiteral: if (character.IsDigit()) { previous.Value += character; previous.Location.End = new Position(line, column); } else if (character == '.') { return(LexicalError.UnexpectedNumeral); } else { return(LexicalError.RuleNotMatch); } return(LexicalError.Ok); case TokenType.Comment: case TokenType.StringLiteral: previous.Value += character; previous.Location.End = new Position(line, column); return(LexicalError.Ok); case TokenType.Identifer: if (character == '.') { return(LexicalError.RuleNotMatch); } previous.Value += character; previous.Location.End = new Position(line, column); return(LexicalError.Ok); default: if (character.IsDigit()) { collection.Add(new Token(character.ToString(), new Span(line, column, line, column), TokenType.IntegerLiteral)); return(LexicalError.Ok); } else { return(LexicalError.RuleNotMatch); } } }
public LexicalError TokenizePunctuator(TokenCollection collection, char character, int line, int column) { if (!character.IsSymbol()) { return(LexicalError.RuleNotMatch); } if ((collection.Count == 0 || (character == '(') || (character == ')') || (character == '[') || (character == ']') || (character == '{') || (character == '}'))) { if (collection.Count != 0) { if (collection.Last().Type == TokenType.Comment || collection.Last().Type == TokenType.StringLiteral) { } else { collection.Add(new Token(character.ToString(), new Span(line, column, line, column), TokenType.Punctuator)); return(LexicalError.Ok); } } } var previous = collection.Last(); if (previous.Type != TokenType.StringLiteral && previous.Type != TokenType.Comment) { if (character == '\"') { collection.Add(new Token("", new Span(line, column, line, column), TokenType.StringLiteral)); return(LexicalError.Ok); } } switch (previous.Type) { case TokenType.Comment: previous.Value += character; previous.Location.End = new Position(line, column); return(LexicalError.Ok); case TokenType.Punctuator: if ((previous.Value == "(") || (previous.Value == ")") || (previous.Value == "[") || (previous.Value == "]") || (previous.Value == "{") || (previous.Value == "}")) { collection.Add(new Token(character.ToString(), new Span(line, column, line, column), TokenType.Punctuator)); return(LexicalError.Ok); } else { previous.Value += character; previous.Location.End = new Position(line, column); return(LexicalError.Ok); } case TokenType.StringLiteral: if (character == '\"') { if (previous.Value.EndsWith("\\")) { } else { previous.Location.End = new Position(line, column); collection.Add(new Token(" ", new Span(line, column, line, column), TokenType.Whitespace)); return(LexicalError.Ok); } } previous.Value += character; previous.Location.End = new Position(line, column); return(LexicalError.Ok); default: if (character == '\'') { collection.Add(new Token("", new Span(line, column, line, column), TokenType.Comment)); return(LexicalError.Ok); } collection.Add(new Token(character.ToString(), new Span(line, column, line, column), TokenType.Punctuator)); return(LexicalError.Ok); } }