/// <summary>
/// Register method
/// </summary>
/// <author>ALVES Quentin</author>
/// <note>Register a token to token table</note>
/// <param name="span" >Token span</param>
/// <param name="type" >Token type</param>
protected void Register(string span, ETokenTypes type)
{
if (this.token_table != null && !this.token_table.ContainsKey(span))
{
this.token_table.Add(span, type);
}
}
public void Lexer_ParseSimple(string text, ETokenTypes target)
{
var tokens = this.Parse(text);
Assert.Single(tokens);
Assert.Equal(target, tokens[0].Type);
Assert.Equal(text, tokens[0].Meta.Value);
}
public Token(ETokenTypes tokenType, string lexeme)
{
_tokenType = tokenType;
_lexeme = lexeme;
_keyword = EKeywords.None;
_variableType = EVariableTypes.None;
_variableNumber = 0;
_constantType = EConstantTypes.None;
_operatorPrecedence = -1;
_symbol = Symbols.Noop;
switch (_tokenType)
{
case ETokenTypes.Identifier:
{
EKeywords keyword;
if (Keywords.TryGetValue(_lexeme, out keyword))
{
_keyword = keyword;
}
else
{
int variableNumber;
if (int.TryParse(_lexeme.Substring(1), out variableNumber))
{
EVariableTypes variableType;
if (VariableTypes.TryGetValue(_lexeme[0], out variableType))
{
_variableType = variableType;
_variableNumber = variableNumber;
}
}
}
if (_keyword == EKeywords.None && _variableType == EVariableTypes.None)
{
throw new TokenizerException(string.Format("\"{0}\" is not a valid identifier: It's neither a keyword, an input variable Xn or a working variable Yn.", _lexeme));
}
}
break;
case ETokenTypes.IntegerLiteral:
_constantType = EConstantTypes.IntegerConstant;
break;
case ETokenTypes.FloatLiteral:
_constantType = EConstantTypes.FloatConstant;
break;
}
Symbols operatorSymbol;
if (BoolOpTable.TryGetValue(_keyword, out operatorSymbol) || SymbolTable.TryGetValue(_tokenType, out operatorSymbol))
{
_symbol = operatorSymbol;
_operatorPrecedence = OperatorPrecendences[_symbol];
}
}
/// <summary>
/// Match function
/// </summary>
/// <author>ALVES Quentin</author>
/// <note>Verify if the node_id token match a specific token type</note>
/// <param name="query" >Query token type</param>
/// <returns>Token</returns>
protected Token Match(ETokenTypes query)
{
if (this.Current.Type != query)
{
this.EmitErrr($"Unexpected token <{this.Current.Type}>, expected <{query}>", this.Current.Meta);
return(new Token(query, 0, 0, ""));
}
return(this.Next( ));
}
/**
* Reads the next token from the stream and returns it.
*
* @param BinaryStream Stream used to serialize from
* @param InNetworkStream Network stream this token belongs to
* @return Token serialized
*/
public static TokenBase ReadNextToken(BinaryReader BinaryStream, NetworkStream InNetworkStream)
{
TokenBase SerializedToken = null;
ETokenTypes TokenType = (ETokenTypes)BinaryStream.ReadByte();
// Handle token specific serialization.
switch (TokenType)
{
case ETokenTypes.FrameMarker:
SerializedToken = new TokenFrameMarker(BinaryStream);
break;
case ETokenTypes.SocketSendTo:
SerializedToken = new TokenSocketSendTo(BinaryStream);
break;
case ETokenTypes.SendBunch:
SerializedToken = new TokenSendBunch(BinaryStream);
break;
case ETokenTypes.SendRPC:
SerializedToken = new TokenSendRPC(BinaryStream);
break;
case ETokenTypes.ReplicateActor:
SerializedToken = new TokenReplicateActor(BinaryStream);
break;
case ETokenTypes.ReplicateProperty:
SerializedToken = new TokenReplicateProperty(BinaryStream);
break;
case ETokenTypes.EndOfStreamMarker:
SerializedToken = new TokenEndOfStreamMarker();
break;
case ETokenTypes.Event:
SerializedToken = new TokenEvent(BinaryStream);
break;
case ETokenTypes.RawSocketData:
SerializedToken = new TokenRawSocketData(BinaryStream);
break;
default:
throw new InvalidDataException();
}
TokenTypeStats[(int)TokenType]++;
SerializedToken.NetworkStream = InNetworkStream;
SerializedToken.TokenType = TokenType;
return(SerializedToken);
}
/**
* Reads the next token from the stream and returns it.
*
* @param BinaryStream Stream used to serialize from
* @param InNetworkStream Network stream this token belongs to
* @return Token serialized
*/
public static TokenBase ReadNextToken(BinaryReader BinaryStream, NetworkStream InNetworkStream)
{
TokenBase SerializedToken = null;
ETokenTypes TokenType = (ETokenTypes)BinaryStream.ReadByte();
// Handle token specific serialization.
switch (TokenType)
{
case ETokenTypes.FrameMarker:
SerializedToken = new TokenFrameMarker(BinaryStream);
break;
case ETokenTypes.SocketSendTo:
SerializedToken = new TokenSocketSendTo(BinaryStream);
break;
case ETokenTypes.SendBunch:
SerializedToken = new TokenSendBunch(BinaryStream);
break;
case ETokenTypes.SendRPC:
SerializedToken = new TokenSendRPC(BinaryStream);
break;
case ETokenTypes.ReplicateActor:
SerializedToken = new TokenReplicateActor(BinaryStream);
break;
case ETokenTypes.ReplicateProperty:
SerializedToken = new TokenReplicateProperty(BinaryStream);
break;
case ETokenTypes.EndOfStreamMarker:
SerializedToken = new TokenEndOfStreamMarker();
break;
case ETokenTypes.Event:
SerializedToken = new TokenEvent(BinaryStream);
break;
case ETokenTypes.RawSocketData:
SerializedToken = new TokenRawSocketData(BinaryStream);
break;
case ETokenTypes.SendAck:
SerializedToken = new TokenSendAck(BinaryStream);
break;
case ETokenTypes.WritePropertyHeader:
SerializedToken = new TokenWritePropertyHeader(BinaryStream);
break;
case ETokenTypes.ExportBunch:
SerializedToken = new TokenExportBunch(BinaryStream);
break;
case ETokenTypes.MustBeMappedGuids:
SerializedToken = new TokenMustBeMappedGuids(BinaryStream);
break;
case ETokenTypes.BeginContentBlock:
SerializedToken = new TokenBeginContentBlock(BinaryStream);
break;
case ETokenTypes.EndContentBlock:
SerializedToken = new TokenEndContentBlock(BinaryStream);
break;
case ETokenTypes.WritePropertyHandle:
SerializedToken = new TokenWritePropertyHandle(BinaryStream);
break;
case ETokenTypes.NameReference:
SerializedToken = new TokenNameReference(BinaryStream);
break;
case ETokenTypes.ConnectionReference:
SerializedToken = new TokenConnectionReference(BinaryStream);
break;
case ETokenTypes.ConnectionChange:
SerializedToken = new TokenConnectionChanged(BinaryStream);
break;
default:
throw new InvalidDataException();
}
TokenTypeStats[(int)TokenType]++;
SerializedToken.NetworkStream = InNetworkStream;
SerializedToken.TokenType = TokenType;
SerializedToken.ConnectionIndex = InNetworkStream.CurrentConnectionIndex;
return(SerializedToken);
}
/// <summary>
/// Constructor
/// </summary>
/// <author>ALVES Quentin</author>
/// <param name="type" >Type of the new token</param>
/// <param name="line" >Line of the new token from file start</param>
/// <param name="position" >Position of the token from current line start</param>
/// <param name="text" >Text that represent the token on the source text</param>
public Token(ETokenTypes type, int line, int position, string text)
{
this.Type = type;
this.Meta = new Textmeta(line, position, text);
}