public idToken ExpectTokenType(TokenType type, TokenSubType subType)
{
idToken token;
if ((token = ReadToken()) == null)
{
Error("couldn't read expected token");
return(null);
}
else if (token.Type != type)
{
Error("expected a '{0}' but found '{1}'", type.ToString().ToLower(), token.ToString());
return(null);
}
else if (token.Type == TokenType.Number)
{
if ((token.SubType & subType) != subType)
{
Error("expected '{0}' but found '{1}'", subType.ToString().ToLower(), token.ToString());
return(null);
}
}
else if (token.Type == TokenType.Punctuation)
{
if (token.SubType != subType)
{
Error("expected '{0}' but found '{1}'", _scriptStack.Peek().GetPunctuationFromID(subType), token.ToString());
return(null);
}
}
return(token);
}
public Token(TokenType type, TokenSubType subType, string sourceString, uint line, uint position)
{
Type = type;
SubType = subType;
SourceString = sourceString;
Line = line;
Position = position;
}
public Token(string fileName, int line, int column, TokenType type, string contents, TokenSubType subType = TokenSubType.None, object tag = null)
{
FileName = fileName;
Line = line;
Column = column;
Type = type;
SubType = subType;
Contents = contents;
Tag = tag;
}
/// <summary>
/// Check if the next token matches the given subtype. If they match, take the token.
/// </summary>
public bool MatchAndTake(TokenSubType subType)
{
var isMatch = Match(subType);
if (isMatch)
{
Take();
}
return(isMatch);
}
/// <summary>
/// Take a token from the stream. Throws an exception if the given subtype does not match the token subtype.
/// </summary>
public Token Take(TokenSubType subType)
{
var token = Take();
if (token.SubType != subType)
{
throw new MondCompilerException(token, CompilerError.ExpectedButFound, subType, token);
}
return(token);
}
public Token
(
string value,
TokenType type, TokenSubType subType
)
: this(
0, value, 0, 0,
Tokenizer.State.Context.Global,
type, subType
)
{
}
private static void TestNumber(string code, TokenSubType targetType)
{
var tokenizer = new Tokenizer(code, new TokenizerOptions());
Assert.IsTrue(
TokenizationTestUtils.Match(
tokenizer.GetNextToken(),
new Token(
0, code, 0, 0,
Tokenizer.State.Context.Global,
TokenType.Number, targetType
)
),
$"Should tokenize number '{code}' and classify it as '{targetType.ToString()}'"
);
TokenizationTestUtils.TestEOF(tokenizer);
}
private void TestOperator(string op, TokenSubType st, TokenType t)
{
var tokenizer = new Tokenizer(op, new TokenizerOptions());
Assert.IsTrue(
TokenizationTestUtils.Match(
tokenizer.GetNextToken(),
new Token(
0, op, 0, 0,
Tokenizer.State.Context.Global,
t, st
)
),
$"Should classify '{op}' as '{st}'"
);
TokenizationTestUtils.TestEOF(tokenizer);
}
public static Token Parse(Tokenizer.State state)
{
TokenSubType st = GetTypeFor(state.CurrentCharacter);
if (st == TokenSubType.Unknown)
{
return(null);
}
Token t = new Token(
value: state.CurrentCharacter.ToString(),
type: IsMathOperator(state.CurrentCharacter)
? TokenType.MathOperator
: TokenType.SpecialOperator,
subType: st
);
state.Index += 1;
return(t);
}
public Token
(
int index,
string value,
int line, int position,
Tokenizer.State.Context context,
TokenType type, TokenSubType subType
)
{
this.Context = context;
this.Index = index;
this.Line = line;
this.Position = position;
this.Value = value;
this.Length = value.Length;
this.Type = type;
this.SubType = subType;
}
public Token
(
int index,
string value,
int line, int position,
Tokenizer.State.Context context,
TokenType type, TokenSubType subType
)
{
Context = context;
Index = index;
Line = line;
Position = position;
Value = value;
Length = value.Length;
Type = type;
SubType = subType;
}
/// <summary>
/// Check if the next token matches the given subtype.
/// </summary>
public bool Match(TokenSubType subType, int distance = 0)
{
return(Peek(distance).SubType == subType);
}
public PdfToken(TokenType tokenType, TokenSubType tokenSubType, byte[] rawData)
{
this.Type = tokenType;
this.SubType = tokenSubType;
this.RawData = rawData;
}
public PdfToken(TokenType tokenType, byte[] rawData, TokenSubType tokenSubType, double value)
: this(tokenType, tokenSubType, rawData)
{
this.DoubleValue = value;
}
private void TestSpecOp(char op, TokenSubType st) => TestOperator(op.ToString(), st, TokenType.SpecialOperator);
private void TestMathOp(char op, TokenSubType st) => TestOperator(op.ToString(), st, TokenType.MathOperator);
private static void AddToken(string value, TokenType type, TokenSubType subType=TokenSubType.None, int pos=0, int line=0)
{
if (type == TokenType.Operation)
Lexems.Add(new Token(value, type, Helper.operationType(value[0]),pos,line,1));
else
if (type==TokenType.Identifier)
Lexems.Add(Helper.parseIdentifier(value,pos,line));
else
Lexems.Add(new Token(value, type, subType,pos,line,1));
if (type == TokenType.Identifier)
buffer = "";
}
/// <summary> /// Checks if token corresponds to given subtype /// </summary> /// <param name="st">Target extended type</param> /// <returns>True if token matches needed type</returns> public bool Is(TokenSubType st) => SubType == st;
/// <summary> /// Checks if token corresponds to given subtype and matches given value(string representation) /// </summary> /// <param name="st">Target extended type</param> /// <param name="v">Target value(string representation)</param> /// <returns>True if token matches needed value</returns> public bool Is(TokenSubType st, string v) => SubType == st && Value == v;
public string GetPunctuationFromID(TokenSubType id)
{
foreach(LexerPunctuation p in _punctuation)
{
if((int) p.N == (int) id)
{
return p.P;
}
}
return "unknown punctuation";
}
public idToken ExpectTokenType(TokenType type, TokenSubType subType)
{
idToken token;
if((token = ReadToken()) == null)
{
Error("couldn't read expected token");
return null;
}
string tokenValue = token.ToString();
if(token.Type != type)
{
Error("expected a '{0}' but found '{1}'", type.ToString().ToLower(), tokenValue);
return null;
}
else if(token.Type == TokenType.Number)
{
if((token.SubType & subType) != subType)
{
Error("expected '{0}' but found '{1}'", subType.ToString().ToLower(), tokenValue);
return null;
}
}
else if(token.Type == TokenType.Punctuation)
{
if(token.SubType != subType)
{
Error("expected '{0}' but found '{1}'", GetPunctuationFromID(subType), tokenValue);
return null;
}
}
return token;
}
public Token(
TokenType type, TokenSubType subType, string sourceString,
uint line, uint position, object value) :
this(type, subType, sourceString, line, position) => Value = value;
public Token(string value, TokenType type, TokenSubType subtype, int pos = 0, int line=0, int length = 1)
{
Value = value;
Type = type;
SubType = subtype;
Position = pos;
Line = line;
Length = length;
}
private static void AddToken(char c, TokenType type, TokenSubType subType, int pos = 0, int line = 0)
{
AddToken(c.ToString(), type, subType, pos, line);
}
public PdfToken(TokenType tokenType, byte[] rawData, TokenSubType tokenSubType, int value)
: this(tokenType, tokenSubType, rawData)
{
this.IntValue = value;
}
/// <summary>
/// Reads the token when a token with the given type is available.
/// </summary>
/// <param name="type"></param>
/// <param name="subType"></param>
/// <returns></returns>
public idToken CheckTokenType(TokenType type, TokenSubType subType)
{
idToken token;
if((token = ReadToken()) == null)
{
return null;
}
// if the type matches
if((token.Type == type) && ((token.SubType & subType) == subType))
{
return token;
}
// unread token
_scriptPosition = _lastScriptPosition;
_line = _lastLine;
return null;
}
public Token(Token token, TokenType type, string contents, TokenSubType subType = TokenSubType.None, object tag = null)
: this(token.FileName, token.Line, token.Column, type, contents, subType, tag)
{
}