private IncludeToken(Token parent, Scope scope, Span span, PreprocessorToken prepToken, string fileName, bool searchFileDir) : base(parent, scope, span) { _prepToken = prepToken; _fileName = fileName; _searchFileDir = searchFileDir; }
public WordToken(Token parent, Scope scope, Span span, string text) : base(parent, scope, span) { if (string.IsNullOrWhiteSpace(text)) throw new ArgumentException("Text cannot be empty."); _text = text; }
public static BracesToken Parse(Token parent, Scope scope) { var startPos = scope.File.Position; scope.File.MoveNext(); var indentScope = scope.IndentNonRoot(); indentScope.Hint |= ScopeHint.SuppressFunctionDefinition; var retToken = new BracesToken(parent, scope, new Span(startPos, startPos)); retToken._openToken = new BraceToken(retToken, scope, new Span(startPos, scope.File.Position), retToken, true); retToken._tokens.Add(retToken._openToken); var closeToken = null as Token; var done = false; while (!scope.File.EndOfFile && !done) { var token = scope.File.ParseToken(retToken, indentScope); if (token == null) break; if (token.GetType() == typeof(OperatorToken) && (token as OperatorToken).Text == "}") { retToken._closeToken = new BraceToken(retToken, scope, token.Span, retToken, false); retToken._tokens.Add(retToken._closeToken); done = true; } else { retToken._tokens.Add(token); } } retToken.Span = new Span(startPos, scope.File.Position); return retToken; }
public ExternVariableToken(Token parent, Scope scope, IEnumerable<Token> tokens, Token dataTypeToken, IEnumerable<WordToken> nameTokens) : base(parent, scope, new Span(tokens.First().Span.Start, tokens.Last().Span.End)) { _tokens = tokens.ToArray(); _dataTypeToken = dataTypeToken; _nameTokens = nameTokens.ToArray(); }
public ExternFunctionToken(Token parent, Scope scope, IEnumerable<Token> tokens, Token dataTypeToken, IdentifierToken nameToken, BracketsToken argsToken) : base(parent, scope, new Span(tokens.First().Span.Start, tokens.Last().Span.End)) { _tokens = tokens.ToArray(); _dataTypeToken = dataTypeToken; _nameToken = nameToken; _argsToken = argsToken; }
public FunctionCallToken(Token parent, Scope scope, IdentifierToken nameToken, BracketsToken argsToken) : base(parent, scope, new Span(nameToken.Span.Start, argsToken.Span.End)) { _nameToken = nameToken; _argsToken = argsToken; _nameToken.Parent = this; _argsToken.Parent = this; }
public Token(Token parent, Scope scope, Span span) { #if DEBUG if (parent == null && this.GetType() != typeof(CodeFile)) throw new ArgumentNullException("parent"); #endif _parent = parent; _scope = scope; _span = span; }
private VariableDeclarationToken(Token parent, Scope scope, IEnumerable<Token> tokens, Token dataTypeToken, List<IdentifierToken> nameTokens) : base(parent, scope, new Span(tokens.First().Span.Start, tokens.Last().Span.End)) { _dataTypeToken = dataTypeToken; _dataTypeToken.Parent = this; _nameTokens = nameTokens; _tokens = tokens.ToArray(); foreach (Token tok in _tokens) tok.Parent = this; }
public static IfStatementToken Parse(Token parent, Scope scope, KeywordToken ifToken) { var ret = new IfStatementToken(parent, scope, new Span()); ret._tokens.Add(ifToken); ifToken.Parent = ret; var scopeIndent = scope; scopeIndent.Hint |= ScopeHint.NotOnRoot | ScopeHint.SuppressFunctionDefinition | ScopeHint.SuppressVarDecl; var done = false; while (!done && !scope.File.EndOfFile) { var token = scope.File.ParseToken(ret, scopeIndent); if (token == null) break; ret._tokens.Add(token); if (token.GetType() == typeof(BracesToken)) { ret._trueScope = token as BracesToken; done = true; } else if (token.GetType() == typeof(StatementEndToken) || (token.GetType() == typeof(OperatorToken) && (token.Text == "}" || token.Text == ")"))) { ret._condition.Add(token); done = true; } else { ret._condition.Add(token); } } var afterBracesPos = scope.File.Position; var elseToken = scope.File.ParseSingleToken(ret, scopeIndent); if (elseToken != null && elseToken.GetType() == typeof(KeywordToken) && elseToken.Text == "else") { var falseScope = scope.File.ParseSingleToken(ret, scopeIndent); if (falseScope != null && falseScope.GetType() == typeof(BracesToken)) { ret._elseToken = elseToken as KeywordToken; ret._falseScope = falseScope as BracesToken; ret._tokens.Add(ret._elseToken); ret._tokens.Add(ret._falseScope); } else scope.File.Position = afterBracesPos; } else scope.File.Position = afterBracesPos; ret.Span = new Span(ifToken.Span.Start, ret._tokens.Last().Span.End); return ret; }
public static ReplaceSetToken Parse(Token parent, Scope scope, PreprocessorToken replaceToken) { #if DEBUG if (replaceToken == null) throw new ArgumentNullException("replaceToken"); #endif var file = scope.File; var startPos = replaceToken.Span.Start; var scopeIndent = scope.Indent(); var ret = new ReplaceSetToken(parent, scope, new Span(startPos, startPos)); ret._replaceToken = new ReplaceToken(ret, scope, replaceToken.Span, ret); ret._tokens.Add(ret._replaceToken); var done = false; while (!file.EndOfFile && !done) { var token = file.ParseToken(ret, scopeIndent); if (token == null) break; if (token.GetType() == typeof(PreprocessorToken)) { if ((token as PreprocessorToken).Text == "#with" && ret._withToken == null) { ret._withToken = new ReplaceToken(ret, scope, token.Span, ret); ret._tokens.Add(ret._withToken); } else if ((token as PreprocessorToken).Text == "#endreplace") { ret._endReplaceToken = new ReplaceToken(ret, scope, token.Span, ret); ret._tokens.Add(ret._endReplaceToken); done = true; } else { ret._tokens.Add(token); if (ret._withToken == null) ret._oldTokens.Add(token); else ret._newTokens.Add(token); } } else { ret._tokens.Add(token); if (ret._withToken == null) ret._oldTokens.Add(token); else ret._newTokens.Add(token); } } ret.Span = new Span(startPos, file.Position); return ret; }
private DefineToken(Token parent, Scope scope, IEnumerable<Token> tokens, IdentifierToken nameToken, BracketsToken argsToken, IEnumerable<Token> bodyTokens) : base(parent, scope, new Span(tokens.First().Span.Start, tokens.Last().Span.End)) { _tokens = tokens.ToArray(); foreach (var token in _tokens) token.Parent = this; _nameToken = nameToken; _argsToken = argsToken; _bodyTokens = bodyTokens.ToArray(); if (_argsToken == null && _bodyTokens.Length == 1 && (_bodyTokens[0].GetType() == typeof(DataTypeToken) || _bodyTokens[0].GetType() == typeof(DataTypeKeywordToken))) { _isDataType = true; //File.AddDefinedDataType(_nameToken.Text); } }
public FunctionToken(Token parent, Scope scope, IEnumerable<Token> tokens, Token dataTypeToken, IdentifierToken nameToken, BracketsToken argsToken, BracesToken scopeToken) : base(parent, scope, new Span(tokens.First().Span.Start, tokens.Last().Span.End)) { _tokens = tokens.ToArray(); _dataTypeToken = dataTypeToken; if (_dataTypeToken != null) _dataTypeToken.Parent = this; _nameToken = nameToken; _nameToken.Parent = this; _argsToken = argsToken; _argsToken.Parent = this; _scopeToken = scopeToken; _scopeToken.Parent = this; foreach (var tok in _tokens) tok.Parent = this; }
public static ArrayBracesToken Parse(Token parent, Scope scope) { var startPos = scope.File.Position; scope.File.MoveNext(); var ret = new ArrayBracesToken(parent, scope, new Span(startPos, startPos)); ret._openToken = new ArrayBraceToken(ret, scope, new Span(startPos, scope.File.Position), ret, true); ret._tokens.Add(ret._openToken); var closeToken = null as Token; var scopeIndent = scope.IndentNonRoot(); scopeIndent.Hint |= ScopeHint.SuppressFunctionDefinition | ScopeHint.SuppressVarDecl; var done = false; while (!scope.File.EndOfFile && !done) { var token = scope.File.ParseToken(ret, scopeIndent); if (token == null) break; if (token.GetType() == typeof(OperatorToken) && (token as OperatorToken).Text == "]") { ret._closeToken = new ArrayBraceToken(ret, scope, token.Span, ret, false); ret._tokens.Add(ret._closeToken); done = true; } else if (token.BreaksStatement) { scope.File.Position = token.Span.Start; done = true; } else { ret._tokens.Add(token); } } ret.Span = new Span(startPos, scope.File.Position); return ret; }
public static IncludeToken Parse(Token parent, Scope scope, PreprocessorToken prepToken) { var startPos = scope.File.Position; scope.File.SeekEndOfLine(); var lineText = scope.File.GetText(new Span(startPos, scope.File.Position)).Trim(); var fileName = ""; var searchFileDir = false; var match = _rxAngleBrackets.Match(lineText); if (match.Success) { fileName = match.Groups[1].Value.Trim(); } else if ((match = _rxQuotes.Match(lineText)).Success) { fileName = match.Groups[1].Value.Trim(); searchFileDir = true; } return new IncludeToken(parent, scope, new Span(prepToken.Span.Start, scope.File.Position), prepToken, fileName, searchFileDir); }
public static BracketsToken Parse(Token parent, Scope scope) { var file = scope.File; var startPos = file.Position; file.MoveNext(); var scopeIndent = scope.IndentNonRoot(); scopeIndent.Hint |= ScopeHint.SuppressVarDecl | ScopeHint.SuppressFunctionDefinition | ScopeHint.SuppressFunctionCall; var retToken = new BracketsToken(parent, scope, new Span(startPos, startPos)); retToken._openToken = new BracketToken(retToken, scope, new Span(startPos, file.Position), retToken, true); retToken._tokens.Add(retToken._openToken); var closeToken = null as Token; var done = false; while (!file.EndOfFile && !done) { var token = file.ParseToken(retToken, scopeIndent); if (token != null) { if (token.GetType() == typeof(OperatorToken) && token.Text == ")") { retToken._closeToken = new BracketToken(retToken, scope, token.Span, retToken, false); retToken._tokens.Add(retToken._closeToken); done = true; } else { retToken._tokens.Add(token); } } } retToken.Span = new Span(startPos, file.Position); return retToken; }
public static Token Parse(Token parent, Scope scope, PreprocessorToken insertToken) { var ret = new InsertToken(parent, scope); var scopeIndent = scope.Indent(); var tokens = new List<Token>(); tokens.Add(ret._startToken = new InsertBoundaryToken(ret, scope, insertToken.Span, true)); while (true) { var token = scope.File.ParseToken(parent, scopeIndent); if (token == null) break; if (token.GetType() == typeof(PreprocessorToken) && token.Text == "#endinsert") { tokens.Add(ret._endToken = new InsertBoundaryToken(ret, scope, token.Span, false)); break; } else tokens.Add(token); } ret._tokens = tokens.ToArray(); ret.Span = new Span(ret._tokens.First().Span.Start, ret._tokens.Last().Span.End); return ret; }
public static VariableDeclarationToken TryParse(Token parent, Scope scope, Token dataTypeToken, IdentifierToken nameToken) { if (scope.Hint.HasFlag(ScopeHint.SuppressVarDecl)) return null; var startPos = scope.File.Position; var tokens = new List<Token>(); tokens.Add(dataTypeToken); tokens.Add(nameToken); var nameTokens = new List<IdentifierToken>(); nameTokens.Add(nameToken); if (scope.Hint.HasFlag(ScopeHint.FunctionArgs)) { var token = scope.File.ParseSingleToken(parent, scope); if (token == null || (token.GetType() != typeof(StatementEndToken) && token.GetType() != typeof(DelimiterToken))) { scope.File.Position = startPos; return null; } tokens.Add(token); } else { var done = false; var needDelim = true; while (!done) { var token = scope.File.ParseSingleToken(parent, scope); if (token == null) { scope.File.Position = startPos; return null; } if (token.GetType() == typeof(StatementEndToken)) { done = true; tokens.Add(token); } else if (needDelim && token.GetType() == typeof(DelimiterToken)) { needDelim = false; tokens.Add(token); } else if (!needDelim && token.GetType() == typeof(IdentifierToken)) { needDelim = true; tokens.Add(token); nameTokens.Add(token as IdentifierToken); } else if (needDelim && token.GetType() == typeof(ArrayBracesToken)) { // This is an array variable. tokens.Add(token); } else { scope.File.Position = startPos; return null; } } } return new VariableDeclarationToken(parent, scope, tokens, dataTypeToken, nameTokens); }
private ReplaceSetToken(Token parent, Scope scope, Span span) : base(parent, scope, span) { }
public ReplaceToken(Token parent, Scope scope, Span span, ReplaceSetToken replaceSet) : base(parent, scope, span) { _replaceSet = replaceSet; }
public DelimiterToken(Token parent, Scope scope, Span span) : base(parent, scope, span) { }
public static FunctionToken TryParse(Token parent, Scope scope, Token dataTypeToken, IdentifierToken nameToken) { if ((scope.Hint & ScopeHint.SuppressFunctionDefinition) != 0) return null; var startPos = scope.File.Position; var tokens = new List<Token>(); if (dataTypeToken != null) tokens.Add(dataTypeToken); tokens.Add(nameToken); var argsScope = scope; argsScope.Hint |= ScopeHint.FunctionArgs; var token = scope.File.ParseSingleToken(parent, argsScope); if (token == null || token.GetType() != typeof(BracketsToken)) { scope.File.Position = startPos; return null; } var argsToken = token as BracketsToken; tokens.Add(argsToken); var innerScope = scope; innerScope.Hint |= ScopeHint.SuppressFunctionDefinition | ScopeHint.NotOnRoot; var scopeToken = null as BracesToken; var optionsTokens = new List<Token>(); var done = false; while (!done) { token = scope.File.ParseSingleToken(parent, innerScope); if (token == null) { scope.File.Position = startPos; return null; } if (token.GetType() == typeof(BracesToken)) { tokens.Add(token); scopeToken = token as BracesToken; done = true; } else if (token.BreaksStatement) { scope.File.Position = startPos; return null; } else { tokens.Add(token); optionsTokens.Add(token); } } return new FunctionToken(parent, scope, tokens, dataTypeToken, nameToken, argsToken, scopeToken); }
public static Token Parse(Token parent, Scope scope, DataTypeKeywordToken firstToken) { var startPos = scope.File.Position; if (firstToken.Text == "enum") { var tokens = new List<Token>(); tokens.Add(firstToken); while (true) { var token = scope.File.ParseSingleToken(parent, scope); if (token == null) break; else if (token.GetType() == typeof(DataTypeKeywordToken) && (token as DataTypeKeywordToken).Text == "proto") tokens.Add(token); else if (token.GetType() == typeof(DataTypeKeywordToken) && (token as DataTypeKeywordToken).Text == "nowarn") tokens.Add(token); else if (token.GetType() == typeof(BracesToken)) tokens.Add(token); else { scope.File.Position = tokens.Last().Span.End; break; } } return new DataTypeToken(parent, scope, tokens); } else if (firstToken.Text == "numeric") { var tokens = new List<Token>(); tokens.Add(firstToken); var token = scope.File.ParseSingleToken(parent, scope); if (token == null) return firstToken; if (token.GetType() == typeof(BracketsToken)) { tokens.Add(token); token = scope.File.ParseSingleToken(parent, scope); if (token == null) return new DataTypeToken(parent, scope, tokens); } var done = false; while (!done) { if (token.GetType() == typeof(DataTypeKeywordToken)) { switch ((token as DataTypeKeywordToken).Text) { case "unsigned": case "currency": case "local_currency": case "LEADINGZEROS": case "PROBE": tokens.Add(token); break; default: scope.File.Position = tokens.Last().Span.End; done = true; break; } } else if (token.GetType() == typeof(IdentifierToken)) { switch ((token as IdentifierToken).Text) { case "PROBE": tokens.Add(token); break; default: scope.File.Position = tokens.Last().Span.End; done = true; break; } } else if (token.GetType() == typeof(StringLiteralToken)) { tokens.Add(token); } else { scope.File.Position = tokens.Last().Span.End; done = true; } if (!done) { token = scope.File.ParseSingleToken(parent, scope); if (token == null) return new DataTypeToken(parent, scope, tokens); } } return new DataTypeToken(parent, scope, tokens); } else if (firstToken.Text == "char") { var tokens = new List<Token>(); tokens.Add(firstToken); var token = scope.File.ParseSingleToken(parent, scope); if (token == null) return firstToken; if (token.GetType() == typeof(BracketsToken)) { tokens.Add(token); token = scope.File.ParseSingleToken(parent, scope); if (token == null) return new DataTypeToken(parent, scope, tokens); } if (token.GetType() == typeof(StringLiteralToken)) { tokens.Add(token); } else scope.File.Position = tokens.Last().Span.End; return new DataTypeToken(parent, scope, tokens); } else if (firstToken.Text == "date") { var tokens = new List<Token>(); tokens.Add(firstToken); var token = scope.File.ParseSingleToken(parent, scope); if (token == null) return token; if (token.GetType() == typeof(NumberToken)) { tokens.Add(token); token = scope.File.ParseSingleToken(parent, scope); if (token == null) return new DataTypeToken(parent, scope, tokens); } var done = false; while (!done) { if (token.GetType() == typeof(StringLiteralToken)) { tokens.Add(token); } else if (token.GetType() == typeof(DataTypeKeywordToken)) { switch ((token as DataTypeKeywordToken).Text) { case "shortform": case "longform": case "alternate": case "PROBE": tokens.Add(token); break; default: scope.File.Position = tokens.Last().Span.End; done = true; break; } } else { scope.File.Position = tokens.Last().Span.End; done = true; } if (!done) { token = scope.File.ParseSingleToken(parent, scope); if (token == null) break; } } return new DataTypeToken(parent, scope, tokens); } else if (firstToken.Text == "like") { var tableNameToken = scope.File.ParseSingleToken(parent, scope); if (tableNameToken.GetType() != typeof(IdentifierToken)) { scope.File.Position = startPos; return firstToken; } var dotToken = scope.File.ParseSingleToken(parent, scope); if (dotToken.GetType() != typeof(DotToken)) { scope.File.Position = startPos; return firstToken; } var fieldNameToken = scope.File.ParseSingleToken(parent, scope); if (fieldNameToken.GetType() != typeof(IdentifierToken)) { scope.File.Position = startPos; return firstToken; } return new DataTypeToken(parent, scope, new Token[] { firstToken, tableNameToken, dotToken, fieldNameToken }); } return firstToken; }
public IdentifierToken(Token parent, Scope scope, Span span, string text) : base(parent, scope, span, text) { }
public NumberToken(Token parent, Scope scope, Span span, string text) : base(parent, scope, span) { _text = text; }
public DataTypeKeywordToken(Token parent, Scope scope, Span span, string text) : base(parent, scope, span, text) { }
public WhiteSpaceToken(Token parent, Scope scope, Span span) : base(parent, scope, span) { }
private IfStatementToken(Token parent, Scope scope, Span span) : base(parent, scope, span) { }
public DataTypeToken(Token parent, Scope scope, IEnumerable<Token> tokens) : base(parent, scope, new Span(tokens.First().Span.Start, tokens.Last().Span.End)) { _tokens = tokens.ToArray(); }
public ArrayBraceToken(Token parent, Scope scope, Span span, ArrayBracesToken braces, bool open) : base(parent, scope, span) { _braces = braces; _open = open; }
public static DataTypeToken CreateFromDefinedWord(Token parent, Scope scope, Span span, string text) { return new DataTypeToken(parent, scope, new Token[] { new IdentifierToken(parent, scope, span, text) }); }