public static Scripting.Parser.TokenCollection JoinTokens(this List <Scripting.Parser.TokenCollection> tokens)
{
Scripting.Parser.TokenCollection collections = new Scripting.Parser.TokenCollection();
foreach (var item in tokens)
{
collections.AddRange(item);
collections.Add(Scripting.Parser.Token.LineBreak);
}
return(collections);
}
public TokenCollection Tokenize(string source)
{
int lineCount = 1;
int columnCount = 1;
TokenCollection collection = new TokenCollection();
string[] lines = source.Split('\n');
foreach (string line in lines)
{
foreach (char character in line)
{
this.Tokenize(collection, character, lineCount, columnCount);
columnCount++;
}
if (collection.Count == 0)
{
collection.Add(new Token(Token.LineBreak.Value, new Span(lineCount, columnCount, lineCount, columnCount + 1), TokenType.Newline));
}
else if (collection.Last().ContentEquals(Token.LineContinue))
{
collection.RemoveLast();
}
else
{
collection.Add(new Token(Token.LineBreak.Value, new Span(lineCount, columnCount, lineCount, columnCount + 1), TokenType.Newline));
}
columnCount = 1;
lineCount++;
}
collection.RemoveAll((token) => { return(token.Type == TokenType.Whitespace); });
return(collection);
}
// the methods Split, Contains has a corresponding *Cascade version. this means the method
// will check whether the specified symbol is in identical level of blocks as the baseline.
// taking nested blocks into consideration.
// for example, the following token collection:
// data a ( int : c ) : b
// applying Split(":"), the sections are:
// [1] data a ( int
// [2] c )
// [3] b
// applying SplitCascade(":"), the sections are:
// [1] data a ( int : c )
// [2] b
public List <TokenCollection> Split(Token token)
{
List <TokenCollection> result = new List <TokenCollection>();
TokenCollection collection = new TokenCollection();
foreach (var item in this)
{
if (item.ContentEquals(token))
{
result.Add(collection);
collection = new TokenCollection();
}
else
{
collection.Add(item);
}
}
if (collection.Count > 0)
{
result.Add(collection);
}
return(result);
}
public LexicalError TokenizeDigit(TokenCollection collection, char character, int line, int column)
{
if (!character.IsDigit() && character != '.')
{
return(LexicalError.RuleNotMatch);
}
if (collection.Count == 0)
{
if (character.IsDigit())
{
collection.Add(new Token(character.ToString(), new Span(line, column, line, column), TokenType.IntegerLiteral));
return(LexicalError.Ok);
}
else
{
return(LexicalError.RuleNotMatch);
}
}
var previous = collection.Last();
switch (previous.Type)
{
case TokenType.IntegerLiteral:
if (character.IsDigit())
{
previous.Value += character;
previous.Location.End = new Position(line, column);
}
else if (character == '.')
{
previous.Value += character;
previous.Type = TokenType.FloatingLiteral;
previous.Location.End = new Position(line, column);
}
else
{
return(LexicalError.RuleNotMatch);
}
return(LexicalError.Ok);
case TokenType.FloatingLiteral:
if (character.IsDigit())
{
previous.Value += character;
previous.Location.End = new Position(line, column);
}
else if (character == '.')
{
return(LexicalError.UnexpectedNumeral);
}
else
{
return(LexicalError.RuleNotMatch);
}
return(LexicalError.Ok);
case TokenType.Comment:
case TokenType.StringLiteral:
previous.Value += character;
previous.Location.End = new Position(line, column);
return(LexicalError.Ok);
case TokenType.Identifer:
if (character == '.')
{
return(LexicalError.RuleNotMatch);
}
previous.Value += character;
previous.Location.End = new Position(line, column);
return(LexicalError.Ok);
default:
if (character.IsDigit())
{
collection.Add(new Token(character.ToString(), new Span(line, column, line, column), TokenType.IntegerLiteral));
return(LexicalError.Ok);
}
else
{
return(LexicalError.RuleNotMatch);
}
}
}
public LexicalError TokenizePunctuator(TokenCollection collection, char character, int line, int column)
{
if (!character.IsSymbol())
{
return(LexicalError.RuleNotMatch);
}
if ((collection.Count == 0 ||
(character == '(') ||
(character == ')') ||
(character == '[') ||
(character == ']') ||
(character == '{') ||
(character == '}')))
{
if (collection.Count != 0)
{
if (collection.Last().Type == TokenType.Comment ||
collection.Last().Type == TokenType.StringLiteral)
{
}
else
{
collection.Add(new Token(character.ToString(), new Span(line, column, line, column), TokenType.Punctuator));
return(LexicalError.Ok);
}
}
}
var previous = collection.Last();
if (previous.Type != TokenType.StringLiteral &&
previous.Type != TokenType.Comment)
{
if (character == '\"')
{
collection.Add(new Token("", new Span(line, column, line, column), TokenType.StringLiteral));
return(LexicalError.Ok);
}
}
switch (previous.Type)
{
case TokenType.Comment:
previous.Value += character;
previous.Location.End = new Position(line, column);
return(LexicalError.Ok);
case TokenType.Punctuator:
if ((previous.Value == "(") ||
(previous.Value == ")") ||
(previous.Value == "[") ||
(previous.Value == "]") ||
(previous.Value == "{") ||
(previous.Value == "}"))
{
collection.Add(new Token(character.ToString(), new Span(line, column, line, column), TokenType.Punctuator));
return(LexicalError.Ok);
}
else
{
previous.Value += character;
previous.Location.End = new Position(line, column);
return(LexicalError.Ok);
}
case TokenType.StringLiteral:
if (character == '\"')
{
if (previous.Value.EndsWith("\\"))
{
}
else
{
previous.Location.End = new Position(line, column);
collection.Add(new Token(" ", new Span(line, column, line, column), TokenType.Whitespace));
return(LexicalError.Ok);
}
}
previous.Value += character;
previous.Location.End = new Position(line, column);
return(LexicalError.Ok);
default:
if (character == '\'')
{
collection.Add(new Token("", new Span(line, column, line, column), TokenType.Comment));
return(LexicalError.Ok);
}
collection.Add(new Token(character.ToString(), new Span(line, column, line, column), TokenType.Punctuator));
return(LexicalError.Ok);
}
}
