private IncludeToken(Token parent, Scope scope, Span span, PreprocessorToken prepToken, string fileName, bool searchFileDir)
: base(parent, scope, span)
{
_prepToken = prepToken;
_fileName = fileName;
_searchFileDir = searchFileDir;
}
public WordToken(Token parent, Scope scope, Span span, string text)
: base(parent, scope, span)
{
if (string.IsNullOrWhiteSpace(text)) throw new ArgumentException("Text cannot be empty.");
_text = text;
}
public static BracesToken Parse(Token parent, Scope scope)
{
var startPos = scope.File.Position;
scope.File.MoveNext();
var indentScope = scope.IndentNonRoot();
indentScope.Hint |= ScopeHint.SuppressFunctionDefinition;
var retToken = new BracesToken(parent, scope, new Span(startPos, startPos));
retToken._openToken = new BraceToken(retToken, scope, new Span(startPos, scope.File.Position), retToken, true);
retToken._tokens.Add(retToken._openToken);
var closeToken = null as Token;
var done = false;
while (!scope.File.EndOfFile && !done)
{
var token = scope.File.ParseToken(retToken, indentScope);
if (token == null) break;
if (token.GetType() == typeof(OperatorToken) && (token as OperatorToken).Text == "}")
{
retToken._closeToken = new BraceToken(retToken, scope, token.Span, retToken, false);
retToken._tokens.Add(retToken._closeToken);
done = true;
}
else
{
retToken._tokens.Add(token);
}
}
retToken.Span = new Span(startPos, scope.File.Position);
return retToken;
}
public ExternVariableToken(Token parent, Scope scope, IEnumerable<Token> tokens, Token dataTypeToken, IEnumerable<WordToken> nameTokens)
: base(parent, scope, new Span(tokens.First().Span.Start, tokens.Last().Span.End))
{
_tokens = tokens.ToArray();
_dataTypeToken = dataTypeToken;
_nameTokens = nameTokens.ToArray();
}
public ExternFunctionToken(Token parent, Scope scope, IEnumerable<Token> tokens, Token dataTypeToken, IdentifierToken nameToken, BracketsToken argsToken)
: base(parent, scope, new Span(tokens.First().Span.Start, tokens.Last().Span.End))
{
_tokens = tokens.ToArray();
_dataTypeToken = dataTypeToken;
_nameToken = nameToken;
_argsToken = argsToken;
}
public FunctionCallToken(Token parent, Scope scope, IdentifierToken nameToken, BracketsToken argsToken)
: base(parent, scope, new Span(nameToken.Span.Start, argsToken.Span.End))
{
_nameToken = nameToken;
_argsToken = argsToken;
_nameToken.Parent = this;
_argsToken.Parent = this;
}
public Token(Token parent, Scope scope, Span span)
{
#if DEBUG
if (parent == null && this.GetType() != typeof(CodeFile)) throw new ArgumentNullException("parent");
#endif
_parent = parent;
_scope = scope;
_span = span;
}
private VariableDeclarationToken(Token parent, Scope scope, IEnumerable<Token> tokens, Token dataTypeToken, List<IdentifierToken> nameTokens)
: base(parent, scope, new Span(tokens.First().Span.Start, tokens.Last().Span.End))
{
_dataTypeToken = dataTypeToken;
_dataTypeToken.Parent = this;
_nameTokens = nameTokens;
_tokens = tokens.ToArray();
foreach (Token tok in _tokens) tok.Parent = this;
}
public static IfStatementToken Parse(Token parent, Scope scope, KeywordToken ifToken)
{
var ret = new IfStatementToken(parent, scope, new Span());
ret._tokens.Add(ifToken);
ifToken.Parent = ret;
var scopeIndent = scope;
scopeIndent.Hint |= ScopeHint.NotOnRoot | ScopeHint.SuppressFunctionDefinition | ScopeHint.SuppressVarDecl;
var done = false;
while (!done && !scope.File.EndOfFile)
{
var token = scope.File.ParseToken(ret, scopeIndent);
if (token == null) break;
ret._tokens.Add(token);
if (token.GetType() == typeof(BracesToken))
{
ret._trueScope = token as BracesToken;
done = true;
}
else if (token.GetType() == typeof(StatementEndToken) ||
(token.GetType() == typeof(OperatorToken) && (token.Text == "}" || token.Text == ")")))
{
ret._condition.Add(token);
done = true;
}
else
{
ret._condition.Add(token);
}
}
var afterBracesPos = scope.File.Position;
var elseToken = scope.File.ParseSingleToken(ret, scopeIndent);
if (elseToken != null && elseToken.GetType() == typeof(KeywordToken) && elseToken.Text == "else")
{
var falseScope = scope.File.ParseSingleToken(ret, scopeIndent);
if (falseScope != null && falseScope.GetType() == typeof(BracesToken))
{
ret._elseToken = elseToken as KeywordToken;
ret._falseScope = falseScope as BracesToken;
ret._tokens.Add(ret._elseToken);
ret._tokens.Add(ret._falseScope);
}
else scope.File.Position = afterBracesPos;
}
else scope.File.Position = afterBracesPos;
ret.Span = new Span(ifToken.Span.Start, ret._tokens.Last().Span.End);
return ret;
}
public static ReplaceSetToken Parse(Token parent, Scope scope, PreprocessorToken replaceToken)
{
#if DEBUG
if (replaceToken == null) throw new ArgumentNullException("replaceToken");
#endif
var file = scope.File;
var startPos = replaceToken.Span.Start;
var scopeIndent = scope.Indent();
var ret = new ReplaceSetToken(parent, scope, new Span(startPos, startPos));
ret._replaceToken = new ReplaceToken(ret, scope, replaceToken.Span, ret);
ret._tokens.Add(ret._replaceToken);
var done = false;
while (!file.EndOfFile && !done)
{
var token = file.ParseToken(ret, scopeIndent);
if (token == null) break;
if (token.GetType() == typeof(PreprocessorToken))
{
if ((token as PreprocessorToken).Text == "#with" && ret._withToken == null)
{
ret._withToken = new ReplaceToken(ret, scope, token.Span, ret);
ret._tokens.Add(ret._withToken);
}
else if ((token as PreprocessorToken).Text == "#endreplace")
{
ret._endReplaceToken = new ReplaceToken(ret, scope, token.Span, ret);
ret._tokens.Add(ret._endReplaceToken);
done = true;
}
else
{
ret._tokens.Add(token);
if (ret._withToken == null) ret._oldTokens.Add(token);
else ret._newTokens.Add(token);
}
}
else
{
ret._tokens.Add(token);
if (ret._withToken == null) ret._oldTokens.Add(token);
else ret._newTokens.Add(token);
}
}
ret.Span = new Span(startPos, file.Position);
return ret;
}
private DefineToken(Token parent, Scope scope, IEnumerable<Token> tokens, IdentifierToken nameToken, BracketsToken argsToken, IEnumerable<Token> bodyTokens)
: base(parent, scope, new Span(tokens.First().Span.Start, tokens.Last().Span.End))
{
_tokens = tokens.ToArray();
foreach (var token in _tokens) token.Parent = this;
_nameToken = nameToken;
_argsToken = argsToken;
_bodyTokens = bodyTokens.ToArray();
if (_argsToken == null && _bodyTokens.Length == 1 &&
(_bodyTokens[0].GetType() == typeof(DataTypeToken) || _bodyTokens[0].GetType() == typeof(DataTypeKeywordToken)))
{
_isDataType = true;
//File.AddDefinedDataType(_nameToken.Text);
}
}
public FunctionToken(Token parent, Scope scope, IEnumerable<Token> tokens, Token dataTypeToken, IdentifierToken nameToken, BracketsToken argsToken, BracesToken scopeToken)
: base(parent, scope, new Span(tokens.First().Span.Start, tokens.Last().Span.End))
{
_tokens = tokens.ToArray();
_dataTypeToken = dataTypeToken;
if (_dataTypeToken != null) _dataTypeToken.Parent = this;
_nameToken = nameToken;
_nameToken.Parent = this;
_argsToken = argsToken;
_argsToken.Parent = this;
_scopeToken = scopeToken;
_scopeToken.Parent = this;
foreach (var tok in _tokens) tok.Parent = this;
}
public static ArrayBracesToken Parse(Token parent, Scope scope)
{
var startPos = scope.File.Position;
scope.File.MoveNext();
var ret = new ArrayBracesToken(parent, scope, new Span(startPos, startPos));
ret._openToken = new ArrayBraceToken(ret, scope, new Span(startPos, scope.File.Position), ret, true);
ret._tokens.Add(ret._openToken);
var closeToken = null as Token;
var scopeIndent = scope.IndentNonRoot();
scopeIndent.Hint |= ScopeHint.SuppressFunctionDefinition | ScopeHint.SuppressVarDecl;
var done = false;
while (!scope.File.EndOfFile && !done)
{
var token = scope.File.ParseToken(ret, scopeIndent);
if (token == null) break;
if (token.GetType() == typeof(OperatorToken) && (token as OperatorToken).Text == "]")
{
ret._closeToken = new ArrayBraceToken(ret, scope, token.Span, ret, false);
ret._tokens.Add(ret._closeToken);
done = true;
}
else if (token.BreaksStatement)
{
scope.File.Position = token.Span.Start;
done = true;
}
else
{
ret._tokens.Add(token);
}
}
ret.Span = new Span(startPos, scope.File.Position);
return ret;
}
public static IncludeToken Parse(Token parent, Scope scope, PreprocessorToken prepToken)
{
var startPos = scope.File.Position;
scope.File.SeekEndOfLine();
var lineText = scope.File.GetText(new Span(startPos, scope.File.Position)).Trim();
var fileName = "";
var searchFileDir = false;
var match = _rxAngleBrackets.Match(lineText);
if (match.Success)
{
fileName = match.Groups[1].Value.Trim();
}
else if ((match = _rxQuotes.Match(lineText)).Success)
{
fileName = match.Groups[1].Value.Trim();
searchFileDir = true;
}
return new IncludeToken(parent, scope, new Span(prepToken.Span.Start, scope.File.Position), prepToken, fileName, searchFileDir);
}
public static BracketsToken Parse(Token parent, Scope scope)
{
var file = scope.File;
var startPos = file.Position;
file.MoveNext();
var scopeIndent = scope.IndentNonRoot();
scopeIndent.Hint |= ScopeHint.SuppressVarDecl | ScopeHint.SuppressFunctionDefinition | ScopeHint.SuppressFunctionCall;
var retToken = new BracketsToken(parent, scope, new Span(startPos, startPos));
retToken._openToken = new BracketToken(retToken, scope, new Span(startPos, file.Position), retToken, true);
retToken._tokens.Add(retToken._openToken);
var closeToken = null as Token;
var done = false;
while (!file.EndOfFile && !done)
{
var token = file.ParseToken(retToken, scopeIndent);
if (token != null)
{
if (token.GetType() == typeof(OperatorToken) && token.Text == ")")
{
retToken._closeToken = new BracketToken(retToken, scope, token.Span, retToken, false);
retToken._tokens.Add(retToken._closeToken);
done = true;
}
else
{
retToken._tokens.Add(token);
}
}
}
retToken.Span = new Span(startPos, file.Position);
return retToken;
}
public static Token Parse(Token parent, Scope scope, PreprocessorToken insertToken)
{
var ret = new InsertToken(parent, scope);
var scopeIndent = scope.Indent();
var tokens = new List<Token>();
tokens.Add(ret._startToken = new InsertBoundaryToken(ret, scope, insertToken.Span, true));
while (true)
{
var token = scope.File.ParseToken(parent, scopeIndent);
if (token == null) break;
if (token.GetType() == typeof(PreprocessorToken) && token.Text == "#endinsert")
{
tokens.Add(ret._endToken = new InsertBoundaryToken(ret, scope, token.Span, false));
break;
}
else tokens.Add(token);
}
ret._tokens = tokens.ToArray();
ret.Span = new Span(ret._tokens.First().Span.Start, ret._tokens.Last().Span.End);
return ret;
}
public static VariableDeclarationToken TryParse(Token parent, Scope scope, Token dataTypeToken, IdentifierToken nameToken)
{
if (scope.Hint.HasFlag(ScopeHint.SuppressVarDecl)) return null;
var startPos = scope.File.Position;
var tokens = new List<Token>();
tokens.Add(dataTypeToken);
tokens.Add(nameToken);
var nameTokens = new List<IdentifierToken>();
nameTokens.Add(nameToken);
if (scope.Hint.HasFlag(ScopeHint.FunctionArgs))
{
var token = scope.File.ParseSingleToken(parent, scope);
if (token == null || (token.GetType() != typeof(StatementEndToken) && token.GetType() != typeof(DelimiterToken)))
{
scope.File.Position = startPos;
return null;
}
tokens.Add(token);
}
else
{
var done = false;
var needDelim = true;
while (!done)
{
var token = scope.File.ParseSingleToken(parent, scope);
if (token == null)
{
scope.File.Position = startPos;
return null;
}
if (token.GetType() == typeof(StatementEndToken))
{
done = true;
tokens.Add(token);
}
else if (needDelim && token.GetType() == typeof(DelimiterToken))
{
needDelim = false;
tokens.Add(token);
}
else if (!needDelim && token.GetType() == typeof(IdentifierToken))
{
needDelim = true;
tokens.Add(token);
nameTokens.Add(token as IdentifierToken);
}
else if (needDelim && token.GetType() == typeof(ArrayBracesToken))
{
// This is an array variable.
tokens.Add(token);
}
else
{
scope.File.Position = startPos;
return null;
}
}
}
return new VariableDeclarationToken(parent, scope, tokens, dataTypeToken, nameTokens);
}
private ReplaceSetToken(Token parent, Scope scope, Span span)
: base(parent, scope, span)
{
}
public ReplaceToken(Token parent, Scope scope, Span span, ReplaceSetToken replaceSet)
: base(parent, scope, span)
{
_replaceSet = replaceSet;
}
public DelimiterToken(Token parent, Scope scope, Span span)
: base(parent, scope, span)
{
}
public static FunctionToken TryParse(Token parent, Scope scope, Token dataTypeToken, IdentifierToken nameToken)
{
if ((scope.Hint & ScopeHint.SuppressFunctionDefinition) != 0) return null;
var startPos = scope.File.Position;
var tokens = new List<Token>();
if (dataTypeToken != null) tokens.Add(dataTypeToken);
tokens.Add(nameToken);
var argsScope = scope;
argsScope.Hint |= ScopeHint.FunctionArgs;
var token = scope.File.ParseSingleToken(parent, argsScope);
if (token == null || token.GetType() != typeof(BracketsToken))
{
scope.File.Position = startPos;
return null;
}
var argsToken = token as BracketsToken;
tokens.Add(argsToken);
var innerScope = scope;
innerScope.Hint |= ScopeHint.SuppressFunctionDefinition | ScopeHint.NotOnRoot;
var scopeToken = null as BracesToken;
var optionsTokens = new List<Token>();
var done = false;
while (!done)
{
token = scope.File.ParseSingleToken(parent, innerScope);
if (token == null)
{
scope.File.Position = startPos;
return null;
}
if (token.GetType() == typeof(BracesToken))
{
tokens.Add(token);
scopeToken = token as BracesToken;
done = true;
}
else if (token.BreaksStatement)
{
scope.File.Position = startPos;
return null;
}
else
{
tokens.Add(token);
optionsTokens.Add(token);
}
}
return new FunctionToken(parent, scope, tokens, dataTypeToken, nameToken, argsToken, scopeToken);
}
public static Token Parse(Token parent, Scope scope, DataTypeKeywordToken firstToken)
{
var startPos = scope.File.Position;
if (firstToken.Text == "enum")
{
var tokens = new List<Token>();
tokens.Add(firstToken);
while (true)
{
var token = scope.File.ParseSingleToken(parent, scope);
if (token == null) break;
else if (token.GetType() == typeof(DataTypeKeywordToken) && (token as DataTypeKeywordToken).Text == "proto") tokens.Add(token);
else if (token.GetType() == typeof(DataTypeKeywordToken) && (token as DataTypeKeywordToken).Text == "nowarn") tokens.Add(token);
else if (token.GetType() == typeof(BracesToken)) tokens.Add(token);
else
{
scope.File.Position = tokens.Last().Span.End;
break;
}
}
return new DataTypeToken(parent, scope, tokens);
}
else if (firstToken.Text == "numeric")
{
var tokens = new List<Token>();
tokens.Add(firstToken);
var token = scope.File.ParseSingleToken(parent, scope);
if (token == null) return firstToken;
if (token.GetType() == typeof(BracketsToken))
{
tokens.Add(token);
token = scope.File.ParseSingleToken(parent, scope);
if (token == null) return new DataTypeToken(parent, scope, tokens);
}
var done = false;
while (!done)
{
if (token.GetType() == typeof(DataTypeKeywordToken))
{
switch ((token as DataTypeKeywordToken).Text)
{
case "unsigned":
case "currency":
case "local_currency":
case "LEADINGZEROS":
case "PROBE":
tokens.Add(token);
break;
default:
scope.File.Position = tokens.Last().Span.End;
done = true;
break;
}
}
else if (token.GetType() == typeof(IdentifierToken))
{
switch ((token as IdentifierToken).Text)
{
case "PROBE":
tokens.Add(token);
break;
default:
scope.File.Position = tokens.Last().Span.End;
done = true;
break;
}
}
else if (token.GetType() == typeof(StringLiteralToken))
{
tokens.Add(token);
}
else
{
scope.File.Position = tokens.Last().Span.End;
done = true;
}
if (!done)
{
token = scope.File.ParseSingleToken(parent, scope);
if (token == null) return new DataTypeToken(parent, scope, tokens);
}
}
return new DataTypeToken(parent, scope, tokens);
}
else if (firstToken.Text == "char")
{
var tokens = new List<Token>();
tokens.Add(firstToken);
var token = scope.File.ParseSingleToken(parent, scope);
if (token == null) return firstToken;
if (token.GetType() == typeof(BracketsToken))
{
tokens.Add(token);
token = scope.File.ParseSingleToken(parent, scope);
if (token == null) return new DataTypeToken(parent, scope, tokens);
}
if (token.GetType() == typeof(StringLiteralToken))
{
tokens.Add(token);
}
else scope.File.Position = tokens.Last().Span.End;
return new DataTypeToken(parent, scope, tokens);
}
else if (firstToken.Text == "date")
{
var tokens = new List<Token>();
tokens.Add(firstToken);
var token = scope.File.ParseSingleToken(parent, scope);
if (token == null) return token;
if (token.GetType() == typeof(NumberToken))
{
tokens.Add(token);
token = scope.File.ParseSingleToken(parent, scope);
if (token == null) return new DataTypeToken(parent, scope, tokens);
}
var done = false;
while (!done)
{
if (token.GetType() == typeof(StringLiteralToken))
{
tokens.Add(token);
}
else if (token.GetType() == typeof(DataTypeKeywordToken))
{
switch ((token as DataTypeKeywordToken).Text)
{
case "shortform":
case "longform":
case "alternate":
case "PROBE":
tokens.Add(token);
break;
default:
scope.File.Position = tokens.Last().Span.End;
done = true;
break;
}
}
else
{
scope.File.Position = tokens.Last().Span.End;
done = true;
}
if (!done)
{
token = scope.File.ParseSingleToken(parent, scope);
if (token == null) break;
}
}
return new DataTypeToken(parent, scope, tokens);
}
else if (firstToken.Text == "like")
{
var tableNameToken = scope.File.ParseSingleToken(parent, scope);
if (tableNameToken.GetType() != typeof(IdentifierToken))
{
scope.File.Position = startPos;
return firstToken;
}
var dotToken = scope.File.ParseSingleToken(parent, scope);
if (dotToken.GetType() != typeof(DotToken))
{
scope.File.Position = startPos;
return firstToken;
}
var fieldNameToken = scope.File.ParseSingleToken(parent, scope);
if (fieldNameToken.GetType() != typeof(IdentifierToken))
{
scope.File.Position = startPos;
return firstToken;
}
return new DataTypeToken(parent, scope, new Token[] { firstToken, tableNameToken, dotToken, fieldNameToken });
}
return firstToken;
}
public IdentifierToken(Token parent, Scope scope, Span span, string text)
: base(parent, scope, span, text)
{
}
public NumberToken(Token parent, Scope scope, Span span, string text)
: base(parent, scope, span)
{
_text = text;
}
public DataTypeKeywordToken(Token parent, Scope scope, Span span, string text)
: base(parent, scope, span, text)
{
}
public WhiteSpaceToken(Token parent, Scope scope, Span span)
: base(parent, scope, span)
{
}
private IfStatementToken(Token parent, Scope scope, Span span)
: base(parent, scope, span)
{
}
public DataTypeToken(Token parent, Scope scope, IEnumerable<Token> tokens)
: base(parent, scope, new Span(tokens.First().Span.Start, tokens.Last().Span.End))
{
_tokens = tokens.ToArray();
}
public ArrayBraceToken(Token parent, Scope scope, Span span, ArrayBracesToken braces, bool open)
: base(parent, scope, span)
{
_braces = braces;
_open = open;
}
public static DataTypeToken CreateFromDefinedWord(Token parent, Scope scope, Span span, string text)
{
return new DataTypeToken(parent, scope, new Token[] { new IdentifierToken(parent, scope, span, text) });
}
