public static ITypeDeclaration ParseBasicType(string Code, out DToken OptionalToken)
{
OptionalToken = null;
var p = Create(new StringReader(Code));
p.Step();
// Exception: If we haven't got any basic types as our first token, return this token via OptionalToken
if (!p.IsBasicType() || p.LA(__LINE__) || p.LA(__FILE__))
{
p.Step();
p.Peek(1);
OptionalToken = p.t;
// Only if a dot follows a 'this' or 'super' token we go on parsing; Return otherwise
if (!((p.t.Kind == This || p.t.Kind == Super) && p.laKind == Dot))
{
return(null);
}
}
var bt = p.BasicType();
while (p.IsBasicType2())
{
var bt2 = p.BasicType2();
bt2.InnerMost = bt;
bt = bt2;
}
return(bt);
}
static DToken SearchBackward(TextDocument doc, int caretOffset, CodeLocation caret,out DToken lastToken)
{
var ttp = doc.GetText(0, caretOffset);
var sr = new StringReader(ttp);
var lexer = new Lexer(sr);
lexer.NextToken();
var stk=new Stack<DToken>();
while (lexer.LookAhead.Kind!=DTokens.EOF)
{
if (lexer.LookAhead.Kind == DTokens.OpenParenthesis || lexer.LookAhead.Kind==DTokens.OpenSquareBracket || lexer.LookAhead.Kind==DTokens.OpenCurlyBrace)
stk.Push(lexer.LookAhead);
else if (lexer.LookAhead.Kind == DTokens.CloseParenthesis || lexer.LookAhead.Kind == DTokens.CloseSquareBracket || lexer.LookAhead.Kind == DTokens.CloseCurlyBrace)
{
if (stk.Peek().Kind == getOppositeBracketToken( lexer.LookAhead.Kind))
stk.Pop();
}
lexer.NextToken();
}
lastToken = lexer.CurrentToken;
sr.Close();
lexer.Dispose();
if (stk.Count < 1)
return null;
return stk.Pop();
}
DToken Peek(int n)
{
Lexer.StartPeek();
DToken x = la;
while (n > 0)
{
x = Lexer.Peek();
n--;
}
return(x);
}
/// <summary>
/// Reads the next token and gives it back.
/// </summary>
/// <returns>An <see cref="CurrentToken"/> object.</returns>
public virtual DToken NextToken()
{
if (stopLexing)
{
if (lookaheadToken != null && lookaheadToken.next != null &&
(lookaheadToken.next.Kind == DTokens.EOF || lookaheadToken.next.Kind == DTokens.__EOF__))
{
curToken = lookaheadToken;
lookaheadToken = lookaheadToken.next;
}
return lookaheadToken;
}
if (lookaheadToken == null)
{
lookaheadToken = Next();
return lookaheadToken;
}
prevToken = curToken;
curToken = lookaheadToken;
if (lookaheadToken.next == null)
lookaheadToken.next = Next();
lookaheadToken = lookaheadToken.next;
StartPeek();
return lookaheadToken;
}
public virtual void Dispose()
{
reader.Close();
reader = null;
curToken = lookaheadToken = peekToken = null;
sb = originalValue = null;
}
/// <summary>
/// Reads the next token and gives it back.
/// </summary>
/// <returns>An <see cref="CurrentToken"/> object.</returns>
public virtual DToken NextToken()
{
if (lookaheadToken == null)
{
lookaheadToken = Next();
TokenTracker.InformToken(lookaheadToken.Kind);
return lookaheadToken;
}
prevToken = curToken;
curToken = lookaheadToken;
if (lookaheadToken.next == null)
{
lookaheadToken.next = Next();
if (lookaheadToken.next != null)
TokenTracker.InformToken(lookaheadToken.next.Kind);
}
lookaheadToken = lookaheadToken.next;
StartPeek();
return lookaheadToken;
}
DToken ReadDigit(char ch, int x)
{
if (!Char.IsDigit(ch) && ch != '.')
{
OnError(Line, x, "Digit literals can only start with a digit (0-9) or a dot ('.')!");
return null;
}
unchecked
{ // prevent exception when ReaderPeek() = -1 is cast to char
int y = Line;
sb.Length = 0;
sb.Append(ch);
string prefix = null;
string expSuffix = "";
string suffix = null;
int exponent = 1;
bool HasDot = false;
LiteralSubformat subFmt = 0;
int NumBase = 0; // Set it to 0 initially - it'll be set to another value later for sure
char peek = (char)ReaderPeek();
// At first, check pre-comma values
if (ch == '0')
{
if (peek == 'x' || peek == 'X') // Hex values
{
prefix = "0x";
ReaderRead(); // skip 'x'
sb.Length = 0; // Remove '0' from 0x prefix from the stringvalue
NumBase = 16;
peek = (char)ReaderPeek();
while (IsHex(peek) || peek == '_')
{
if (peek != '_')
sb.Append((char)ReaderRead());
else ReaderRead();
peek = (char)ReaderPeek();
}
}
else if (peek == 'b' || peek == 'B') // Bin values
{
prefix = "0b";
ReaderRead(); // skip 'b'
sb.Length = 0;
NumBase = 2;
peek = (char)ReaderPeek();
while (IsBin(peek) || peek == '_')
{
if (peek != '_')
sb.Append((char)ReaderRead());
else ReaderRead();
peek = (char)ReaderPeek();
}
}
// Oct values have been removed in dmd 2.053
/*else if (IsOct(peek) || peek == '_') // Oct values
{
NumBase = 8;
prefix = "0";
sb.Length = 0;
while (IsOct(peek) || peek == '_')
{
if (peek != '_')
sb.Append((char)ReaderRead());
else ReaderRead();
peek = (char)ReaderPeek();
}
}*/
else
NumBase = 10; // Enables pre-comma parsing .. in this case we'd 000 literals or something like that
}
if (NumBase == 10 || (ch != '.' && NumBase == 0)) // Only allow further digits for 10-based integers, not for binary or hex values
{
NumBase = 10;
while (Char.IsDigit(peek) || peek == '_')
{
if (peek != '_')
sb.Append((char)ReaderRead());
else ReaderRead();
peek = (char)ReaderPeek();
}
}
#region Read digits that occur after a comma
DToken nextToken = null; // if we accidently read a 'dot'
bool AllowSuffixes = true;
if ((NumBase == 0 && ch == '.') || peek == '.')
{
if (ch != '.') ReaderRead();
else
{
NumBase = 10;
sb.Length = 0;
sb.Append('0');
}
peek = (char)ReaderPeek();
if (!IsLegalDigit(peek, NumBase))
{
if (peek == '.')
{
ReaderRead();
nextToken = new DToken(DTokens.DoubleDot, Col - 1, Line, 2);
}
else if(IsIdentifierPart(peek))
nextToken = new DToken(DTokens.Dot, Col - 1, Line, 1);
AllowSuffixes = false;
}
else
{
HasDot = true;
sb.Append('.');
while (IsLegalDigit(peek, NumBase))
{
if (peek == '_')
ReaderRead();
else
sb.Append((char)ReaderRead());
peek = (char)ReaderPeek();
}
}
}
#endregion
#region Exponents
if ((NumBase == 16) ? (peek == 'p' || peek == 'P') : (peek == 'e' || peek == 'E'))
{ // read exponent
string suff = peek.ToString();
ReaderRead();
peek = (char)ReaderPeek();
if (peek == '-' || peek == '+')
expSuffix += (char)ReaderRead();
peek = (char)ReaderPeek();
while (Char.IsDigit(peek) || peek == '_')
{ // read exponent value
if (peek == '_')
ReaderRead();
else
expSuffix += (char)ReaderRead();
peek = (char)ReaderPeek();
}
// Exponents just can be decimal integers
exponent = int.Parse(expSuffix);
expSuffix = suff + expSuffix;
peek = (char)ReaderPeek();
}
#endregion
#region Suffixes
if (!HasDot)
{
unsigned:
if (peek == 'u' || peek == 'U')
{
ReaderRead();
suffix += "u";
subFmt |= LiteralSubformat.Unsigned;
peek = (char)ReaderPeek();
}
if (peek == 'L')
{
subFmt |= LiteralSubformat.Long;
ReaderRead();
suffix += "L";
//islong = true;
peek = (char)ReaderPeek();
if (!subFmt.HasFlag(LiteralSubformat.Unsigned) && (peek == 'u' || peek == 'U'))
goto unsigned;
}
}
if(HasDot || AllowSuffixes)
{
if (peek == 'f' || peek == 'F')
{ // float value
ReaderRead();
suffix += "f";
subFmt |= LiteralSubformat.Float;
peek = (char)ReaderPeek();
}
else if (peek == 'L')
{ // real value
ReaderRead();
suffix += 'L';
subFmt |= LiteralSubformat.Real;
peek = (char)ReaderPeek();
}
}
if (peek == 'i')
{ // imaginary value
ReaderRead();
suffix += "i";
subFmt |= LiteralSubformat.Imaginary;
}
#endregion
string digit = sb.ToString();
string stringValue = prefix + digit + expSuffix + suffix;
DToken token = null;
#region Parse the digit string
var num = ParseFloatValue(digit, NumBase);
if (exponent != 1)
num *= Math.Pow(NumBase == 16 ? 2 : 10, exponent);
object val = null;
if (HasDot)
{
if (subFmt.HasFlag(LiteralSubformat.Float))
val = (float)num;
else
val = (double)num;
}
else
{
if (subFmt.HasFlag(LiteralSubformat.Unsigned))
{
if (subFmt.HasFlag(LiteralSubformat.Long))
val = (ulong)num;
else
val = (uint)num;
}
else
{
if (subFmt.HasFlag(LiteralSubformat.Long))
val = (long)num;
else
val = (int)num;
}
}
#endregion
token = new DToken(DTokens.Literal, new CodeLocation(x, y), new CodeLocation(x + stringValue.Length, y), stringValue, val,
subFmt.HasFlag(LiteralSubformat.Float) || subFmt.HasFlag(LiteralSubformat.Imaginary) || HasDot ?
(LiteralFormat.FloatingPoint | LiteralFormat.Scalar) : LiteralFormat.Scalar,
subFmt);
if (token != null)
token.next = nextToken;
return token;
}
}
/// <summary>
/// Rawly skip the current code block
/// </summary>
public override void SkipCurrentBlock()
{
int braceCount = 0;
// Scan already parsed tokens
var tok = lookaheadToken;
while (tok != null)
{
if (tok.Kind == DTokens.OpenCurlyBrace)
braceCount++;
else if (tok.Kind == DTokens.CloseCurlyBrace)
{
braceCount--;
if (braceCount < 0)
{
lookaheadToken = tok;
return;
}
}
tok = tok.next;
}
// Scan/proceed tokens rawly (skip them only until braceCount<0)
prevToken = LookAhead;
int nextChar;
while ((nextChar = ReaderRead()) != -1)
{
switch (nextChar)
{
// Handle line ends
case '\r':
case '\n':
HandleLineEnd((char)nextChar);
break;
// Handle comments
case '/':
int peek = ReaderPeek();
if (peek == '/' || peek == '*' || peek == '+')
{
ReadComment();
continue;
}
break;
// handle string literals
case 'r':
int pk = ReaderPeek();
if (pk == '"')
{
ReaderRead();
ReadVerbatimString('"');
}
break;
case '`':
ReadVerbatimString(nextChar);
break;
case '"':
ReadString(nextChar);
break;
case '\'':
ReadChar();
break;
case '{':
braceCount++;
continue;
case '}':
braceCount--;
if (braceCount < 0)
{
lookaheadToken = new DToken(DTokens.CloseCurlyBrace, Col - 1, Line);
StartPeek();
Peek();
return;
}
break;
}
}
}
public void Dispose()
{
reader = null;
prevToken = curToken = lookaheadToken = peekToken = null;
//sb = originalValue = null;
escapeSequenceBuffer = null;
identBuffer = null;
LexerErrors = null;
Comments = null;
}
public DToken(DToken t)
: this(t.Kind, t.col, t.line, t.val, t.literalValue, t.LiteralFormat)
{
next = t.next;
}
bool IsBasicType(DToken tk)
{
switch (tk.Kind) {
case DTokens.Typeof:
case DTokens.__vector:
case DTokens.Identifier:
return true;
case DTokens.Dot:
return tk.Next != null && tk.Next.Kind == (Identifier);
case DTokens.This:
case DTokens.Super:
return tk.Next != null && tk.Next.Kind == DTokens.Dot;
default:
return IsBasicType (tk.Kind) || IsFunctionAttribute_(tk.Kind);
}
}
void TokenStringParsing_AppendLiteralSubFormat(DToken tk, StringBuilder sb)
{
switch (tk.Subformat)
{
case LiteralSubformat.Utf8:
break;
case LiteralSubformat.Utf16:
sb.Append('w');
break;
case LiteralSubformat.Utf32:
sb.Append('d');
break;
//TODO
}
}
void TokenStringParsing_AppendToken(DToken tk, StringBuilder tokenString)
{
switch (tk.Kind)
{
case DTokens.Identifier:
tokenString.Append(tk.LiteralValue);
break;
case DTokens.Literal:
switch (tk.LiteralFormat)
{
case LiteralFormat.CharLiteral:
tokenString.Append('\'').Append(tk.LiteralValue).Append('\'');
break;
case LiteralFormat.None:
case LiteralFormat.Scalar:
case LiteralFormat.FloatingPoint:
tokenString.Append(tk.LiteralValue);
break;
case LiteralFormat.StringLiteral:
tokenString.Append('\"').Append(tk.LiteralValue).Append('\"');
break;
case LiteralFormat.VerbatimStringLiteral:
//TODO: More specific distinguishment between verbatim string types
tokenString.Append('`').Append(tk.LiteralValue).Append('`');
break;
}
TokenStringParsing_AppendLiteralSubFormat(tk, tokenString);
break;
default:
tokenString.Append(DTokens.GetTokenString(tk.Kind));
break;
}
tokenString.Append(' ');
}
protected override DToken Next()
{
int nextChar;
char ch;
bool hadLineEnd = false;
if (Line == 1 && Col == 1) hadLineEnd = true; // beginning of document
while ((nextChar = ReaderRead()) != -1)
{
DToken token;
switch (nextChar)
{
case ' ':
case '\t':
continue;
case '\r':
case '\n':
if (hadLineEnd)
{
// second line end before getting to a token
// -> here was a blank line
//specialTracker.AddEndOfLine(new Location(Col, Line));
}
HandleLineEnd((char)nextChar);
hadLineEnd = true;
continue;
case '/':
int peek = ReaderPeek();
if (peek == '/' || peek == '*' || peek == '+')
{
ReadComment();
continue;
}
else
{
token = ReadOperator('/');
}
break;
case 'r':
peek = ReaderPeek();
if (peek == '"')
{
ReaderRead();
token = ReadVerbatimString(peek);
break;
}
else
goto default;
case '`':
token = ReadVerbatimString(nextChar);
break;
case '"':
token = ReadString(nextChar);
break;
case '\'':
token = ReadChar();
break;
case '@':
int next = ReaderRead();
if (next == -1)
{
OnError(Line, Col, String.Format("EOF after @"));
continue;
}
else
{
int x = Col - 1;
int y = Line;
ch = (char)next;
if (Char.IsLetterOrDigit(ch) || ch == '_')
{
bool canBeKeyword;
string ident = ReadIdent(ch, out canBeKeyword);
token = new DToken(DTokens.PropertyAttribute, x - 1, y, ident);
}
else
{
OnError(y, x, String.Format("Unexpected char in Lexer.Next() : {0}", ch));
continue;
}
}
break;
default:
ch = (char)nextChar;
if (ch == 'x')
{
peek = ReaderPeek();
if (peek == '"') // HexString
{
ReaderRead(); // Skip the "
string numString = "";
while ((next = ReaderRead()) != -1)
{
ch = (char)next;
if (IsHex(ch))
numString += ch;
else if (!Char.IsWhiteSpace(ch))
break;
}
return new DToken(DTokens.Literal, Col - 1, Line, numString, ParseFloatValue(numString, 16), LiteralFormat.Scalar);
}
}
else if (ch == 'q') // Token strings
{
peek = ReaderPeek();
if (peek == '{'/*q{ ... }*/ || peek == '"'/* q"{{ ...}} }}"*/)
{
int x = Col - 1;
int y = Line;
string initDelim = "";
string endDelim = "";
string tokenString = "";
initDelim += (char)ReaderRead();
bool IsQuoted = false;
int BracketLevel = 0; // Only needed if IsQuoted is false
// Read out initializer
if (initDelim == "\"")
{
IsQuoted = true;
initDelim = "";
int pk = ReaderPeek();
ch = (char)pk;
if (Char.IsLetterOrDigit(ch)) // q"EOS EOS"
while ((next = ReaderRead()) != -1)
{
ch = (char)next;
if (!Char.IsWhiteSpace(ch))
initDelim += ch;
else
break;
}
else if (ch == '(' || ch == '<' || ch == '[' || ch == '{')
{
var firstBracket = ch;
while ((next = ReaderRead()) != -1)
{
ch = (char)next;
if (ch == firstBracket)
initDelim += ch;
else
break;
}
}
}
else if (initDelim == "{")
BracketLevel = 1;
// Build end delimiter
endDelim = initDelim.Replace('{', '}').Replace('[', ']').Replace('(', ')').Replace('<', '>');
if (IsQuoted) endDelim += "\"";
// Read tokens
bool inSuperComment = false,
inNestedComment = false;
while ((next = ReaderRead()) != -1)
{
ch = (char)next;
tokenString += ch;
// comments are treated as part of the string inside of tokenized string. curly braces inside the comments are ignored. WEIRD!
if (!inSuperComment && tokenString.EndsWith("/+")) inSuperComment = true;
else if (inSuperComment && tokenString.EndsWith("+/")) inSuperComment = false;
if (!inSuperComment)
{
if (!inNestedComment && tokenString.EndsWith("/*")) inNestedComment = true;
else if (inNestedComment && tokenString.EndsWith("*/")) inNestedComment = false;
}
if (!inNestedComment && !inSuperComment)
{
if (!IsQuoted && ch == '{')
BracketLevel++;
if (!IsQuoted && ch == '}')
BracketLevel--;
}
if (tokenString.EndsWith(endDelim) && (IsQuoted || BracketLevel < 1))
{
tokenString = tokenString.Remove(tokenString.Length - endDelim.Length);
break;
}
}
return new DToken(DTokens.Literal, x, y, tokenString, tokenString, LiteralFormat.VerbatimStringLiteral);
}
}
if (Char.IsLetter(ch) || ch == '_' || ch == '\\')
{
int x = Col - 1; // Col was incremented above, but we want the start of the identifier
int y = Line;
bool canBeKeyword;
string s = ReadIdent(ch, out canBeKeyword);
if (canBeKeyword && DTokens.Keywords.ContainsValue(s))
{
foreach (var kv in DTokens.Keywords)
if (s == kv.Value)
return new DToken(kv.Key, x, y, s.Length);
}
return new DToken(DTokens.Identifier, x, y, s);
}
else if (Char.IsDigit(ch))
token = ReadDigit(ch, Col - 1);
else
token = ReadOperator(ch);
break;
}
// try error recovery (token = null -> continue with next char)
if (token != null)
{
//token.prev = base.curToken;
return token;
}
}
return new DToken(DTokens.EOF, Col, Line, String.Empty);
}
/// <summary>
/// Gives back the next token. A second call to Peek() gives the next token after the last call for Peek() and so on.
/// </summary>
/// <returns>An <see cref="CurrentToken"/> object.</returns>
public DToken Peek()
{
if (peekToken == null) StartPeek();
if (peekToken.next == null)
peekToken.next = Next();
peekToken = peekToken.next;
return peekToken;
}
/// <summary>
/// Reads the next token and gives it back.
/// </summary>
/// <returns>An <see cref="CurrentToken"/> object.</returns>
public void NextToken()
{
if (lookaheadToken == null){
lookaheadToken = Next();
laKind = lookaheadToken.Kind;
Peek();
}
else
{
recyclePrevToken();
prevToken = curToken;
curToken = lookaheadToken;
if (lookaheadToken.next == null)
lookaheadToken.next = Next();
lookaheadToken = lookaheadToken.next;
laKind = lookaheadToken.Kind;
StartPeek();
Peek();
}
}
/// <summary>
/// Must be called before a peek operation.
/// </summary>
public void StartPeek()
{
peekToken = lookaheadToken;
}
public void PopLookAheadBackup()
{
prevToken = null;
curToken = null;
laBackup.Pop();
/*
var bk = laBackup.Pop();
if(laBackup.Count == 0)
{
while(bk != lookaheadToken)
{
var n = bk.next;
bk.next = null;
bk.LiteralValue = null;
tokenPool.Push(bk);
if((bk = n) == null)
return;
}
}*/
}
protected override DToken Next()
{
int nextChar;
char ch;
bool hadLineEnd = false;
int x = Col - 1;
int y = Line;
if (Line == 1 && Col == 1) hadLineEnd = true; // beginning of document
while ((nextChar = ReaderRead()) != -1)
{
DToken token;
switch (nextChar)
{
case ' ':
case '\t':
continue;
case '\r':
case '\n':
if (hadLineEnd)
{
// second line end before getting to a token
// -> here was a blank line
//specialTracker.AddEndOfLine(new Location(Col, Line));
}
HandleLineEnd((char)nextChar);
hadLineEnd = true;
continue;
case '/':
int peek = ReaderPeek();
if (peek == '/' || peek == '*' || peek == '+')
{
ReadComment();
continue;
}
else
{
token = ReadOperator('/');
}
break;
case 'r':
peek = ReaderPeek();
if (peek == '"')
{
ReaderRead();
token = ReadVerbatimString(peek);
break;
}
else
goto default;
case '`':
token = ReadVerbatimString(nextChar);
break;
case '"':
token = ReadString(nextChar);
break;
case '\\':
// http://digitalmars.com/d/1.0/lex.html#EscapeSequence
// - It's actually deprecated, but parse such literals anyway
string surr = "";
x=Col-1;
y=Line;
var lit=ReadEscapeSequence(out ch, out surr);
token = new DToken(DTokens.Literal, x,y, lit, ch.ToString(), LiteralFormat.StringLiteral);
OnError(y, x, "Escape sequence strings are deprecated!");
break;
case '\'':
token = ReadChar();
break;
case '@':
int next = ReaderRead();
if (next == -1)
{
OnError(Line, Col, String.Format("EOF after @"));
continue;
}
else
{
x = Col - 1;
y = Line;
ch = (char)next;
if (Char.IsLetterOrDigit(ch) || ch == '_')
{
bool canBeKeyword;
string ident = ReadIdent(ch, out canBeKeyword);
token = new DToken(DTokens.PropertyAttribute, x - 1, y, ident);
}
else
{
OnError(y, x, String.Format("Unexpected char in Lexer.Next() : {0}", ch));
continue;
}
}
break;
default:
ch = (char)nextChar;
if (ch == 'x')
{
peek = ReaderPeek();
if (peek == '"') // HexString
{
ReaderRead(); // Skip the "
string numString = "";
while ((next = ReaderRead()) != -1)
{
ch = (char)next;
if (IsHex(ch))
numString += ch;
else if (!Char.IsWhiteSpace(ch))
break;
}
return new DToken(DTokens.Literal, Col - 1, Line, numString, ParseFloatValue(numString, 16), LiteralFormat.Scalar);
}
}
else if (ch == 'q') // Token strings
{
peek = ReaderPeek();
if (peek == '{'/*q{ ... }*/ || peek == '"'/* q"{{ ...}} }}"*/)
{
x = Col - 1;
y = Line;
string initDelim = "";
string endDelim = "";
string tokenString = "";
initDelim += (char)ReaderRead();
bool IsQuoted = false;
int BracketLevel = 0; // Only needed if IsQuoted is false
// Read out initializer
if (initDelim == "\"")
{
IsQuoted = true;
initDelim = "";
int pk = ReaderPeek();
ch = (char)pk;
if (Char.IsLetterOrDigit(ch)) // q"EOS EOS"
while ((next = ReaderRead()) != -1)
{
ch = (char)next;
if (!Char.IsWhiteSpace(ch))
initDelim += ch;
else
break;
}
else if (ch == '(' || ch == '<' || ch == '[' || ch == '{')
{
var firstBracket = ch;
while ((next = ReaderRead()) != -1)
{
ch = (char)next;
if (ch == firstBracket)
initDelim += ch;
else
break;
}
}
}
else if (initDelim == "{")
BracketLevel = 1;
// Build end delimiter
endDelim = initDelim.Replace('{', '}').Replace('[', ']').Replace('(', ')').Replace('<', '>');
if (IsQuoted) endDelim += "\"";
// Read tokens
bool inSuperComment = false,
inNestedComment = false;
while ((next = ReaderRead()) != -1)
{
ch = (char)next;
tokenString += ch;
// comments are treated as part of the string inside of tokenized string. curly braces inside the comments are ignored. WEIRD!
if (!inSuperComment && tokenString.EndsWith("/+")) inSuperComment = true;
else if (inSuperComment && tokenString.EndsWith("+/")) inSuperComment = false;
if (!inSuperComment)
{
if (!inNestedComment && tokenString.EndsWith("/*")) inNestedComment = true;
else if (inNestedComment && tokenString.EndsWith("*/")) inNestedComment = false;
}
if (!inNestedComment && !inSuperComment)
{
if (!IsQuoted && ch == '{')
BracketLevel++;
if (!IsQuoted && ch == '}')
BracketLevel--;
}
if (tokenString.EndsWith(endDelim) && (IsQuoted || BracketLevel < 1))
{
tokenString = tokenString.Remove(tokenString.Length - endDelim.Length);
break;
}
}
return new DToken(DTokens.Literal, x, y, tokenString, tokenString, LiteralFormat.VerbatimStringLiteral);
}
}
if (Char.IsLetter(ch) || ch == '_' || ch == '\\')
{
x = Col - 1; // Col was incremented above, but we want the start of the identifier
y = Line;
bool canBeKeyword;
string s = ReadIdent(ch, out canBeKeyword);
if (canBeKeyword)
{
// Fill in static string surrogates directly
if (s == "__DATE__")
return new DToken(DTokens.Literal, x, y, 8) {
literalFormat= LiteralFormat.StringLiteral,
literalValue=DateTime.Now.ToString("MMM dd yyyy",System.Globalization.CultureInfo.InvariantCulture.DateTimeFormat)
};
else if (s == "__TIME__")
return new DToken(DTokens.Literal, x, y, 8)
{
literalFormat = LiteralFormat.StringLiteral,
literalValue = DateTime.Now.ToString("HH:mm:ss", System.Globalization.CultureInfo.InvariantCulture.DateTimeFormat)
};
else if (s == "__TIMESTAMP__")
return new DToken(DTokens.Literal, x, y, 12)
{
literalFormat = LiteralFormat.StringLiteral,
literalValue = DateTime.Now.ToString("ddd MMM dd HH:mm:ss yyyy", System.Globalization.CultureInfo.InvariantCulture.DateTimeFormat)
};
else if (s == "__VENDOR__")
return new DToken(DTokens.Literal, x, y, 10)
{
literalFormat = LiteralFormat.StringLiteral,
literalValue = "D Parser v"+System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString(3)+" by Alexander Bothe"
};
else if (s == "__VERSION__")
{
var lexerVersion=System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;
return new DToken(DTokens.Literal, x, y, 11)
{
literalFormat = LiteralFormat.Scalar,
literalValue = lexerVersion.Major*1000+lexerVersion.Minor
};
}
foreach (var kv in DTokens.Keywords)
if (s == kv.Value)
return new DToken(kv.Key, x, y, s.Length);
}
return new DToken(DTokens.Identifier, x, y, s);
}
else if (Char.IsDigit(ch))
token = ReadDigit(ch, Col - 1);
else
token = ReadOperator(ch);
break;
}
// try error recovery (token = null -> continue with next char)
if (token != null)
{
//token.prev = base.curToken;
return token;
}
else
{
OnError(Line, Col, "Invalid character");
StopLexing();
break;
}
}
return new DToken(DTokens.EOF, Col, Line, String.Empty);
}
public void RestoreLookAheadBackup()
{
prevToken = null;
curToken = null;
lookaheadToken = laBackup.Pop();
laKind = lookaheadToken.Kind;
StartPeek();
Peek();
}
public static ITypeDeclaration ParseBasicType(string Code,out DToken OptionalToken)
{
OptionalToken = null;
var p = Create(new StringReader(Code));
p.Step();
// Exception: If we haven't got any basic types as our first token, return this token via OptionalToken
if (!p.IsBasicType() || p.LA(__LINE__) || p.LA(__FILE__))
{
p.Step();
p.Peek(1);
OptionalToken = p.t;
// Only if a dot follows a 'this' or 'super' token we go on parsing; Return otherwise
if (!((p.t.Kind == This || p.t.Kind == Super) && p.laKind == Dot))
return null;
}
var bt= p.BasicType();
while (p.IsBasicType2())
{
var bt2 = p.BasicType2();
bt2.InnerMost = bt;
bt = bt2;
}
return bt;
}
public DToken(DToken t)
: this(t.Kind, t.col, t.line, t.val, t.literalValue, t.LiteralFormat)
{
next = t.next;
}
