/// <summary>
/// Creates a token of a single character.
/// </summary>
private LexedToken SingularToken(LexedTokenType type, char c)
{
MarkPosition();
currentValue.Append(c);
return(new LexedToken(type, c.ToString()));
}
/// <summary>
///
/// </summary>
/// <param name="TokenType"></param>
//Captures whatever is in the buffer into a new token which is placed on the token stream
//reset the buffer;
protected void CaptureToken(LexedTokenType TokenType)
{
var tkn = new LexedToken(TokenType, buffer.ToString(), CurLinePos, CurLineNum);
resetBuffer();
returnlist.Add(tkn);
}
public static void AssertTokenTypeIs(this LexedToken token, LexedTokenType ExpectedTokenType)
{
if (token.TokenType != ExpectedTokenType)
{
throw new Exception(String.Format("Token type {0} Expected but {1} found", token.TokenType.ToString(), ExpectedTokenType.ToString()));
}
}
public LexedToken(LexedTokenType TokenType, string TokenText, int LineNumber, int Position)
{
this.TokenType = TokenType;
this.TokenText = TokenText;
this.LineNumber = LineNumber;
this.Position = Position;
}
public LexedToken(LexedTokenType TokenType)
{
this.TokenType = TokenType;
this.TokenText = "";
this.LineNumber = 0;
this.Position = 0;
}
private bool LexForTokenEndingChar(LexedTokenType RecordAsType)
{
// if its a new line char - send it to the new line capturer
if (Current == '\n' || Current == '\r')
{
CaptureToken(RecordAsType);
CaptureNewLine();
return(true);
}
// if its a space or tab then captur the current value
if (Current == ' ' || Current == '\t')
{
//capture the buffer but dont' skip the white space - allow it to go back into the loop for processing
CaptureToken(RecordAsType);
PersumedTokenType = LexedTokenType.UNKNOWN;
return(true);
}
//next we check for any of the special chars - if found then close shop and let the the
//lexSpecialChar function take care of it
if (LexedToken.charToToken_Dictionary.ContainsKey(Current))
{
CaptureToken(RecordAsType);
PersumedTokenType = LexedTokenType.UNKNOWN;
LexSpecialChars();
return(true);
}
//its not a token ending char - let teh caller deal with it
return(false);
}
private void LexDelimitedText()
{
// if its another delimiter then close the text
if (Current == '\'')
{
CaptureToken(LexedTokenType.DELIMITEDTEXT);
Skip();
PersumedTokenType = LexedTokenType.UNKNOWN;
return;
}
//if its an escape char then peek forward to see if its a ' and captur only that
if (Current == '\\' && Peek() == '\'')
{
//don't need the slash cause its only escaping
Skip();
}
if (Current == '\r' || Current == '\n')
{
throwParserException("Unterminated Delimited Text");
}
Select();
}
public static void AssertTokenIs(this LexedToken token, LexedTokenType ExpectedTokenType, object ExpectedTokenValue)
{
token.AssertTokenTypeIs(ExpectedTokenType);
if (token.TokenText != ExpectedTokenValue.ToString())
{
throw new Exception(String.Format("Token value {0} Expected but {1} found", ExpectedTokenValue.ToString(), token.TokenText.ToString()));
}
}
/// <summary>
/// Parses a lexer token.
/// </summary>
/// <param name="tokenType">A <see cref="LexedTokenType"/> value specifying the type of token produced by the lexer.</param>
/// <param name="tokenText">The text associated with the lexer token.</param>
/// <param name="sourceOffset">The offset of the first character in the source text that produced the token.</param>
/// <param name="sourceLength">The number of characters in the source text that produced the token.</param>
/// <param name="options">A set of <see cref="TextParserOptions"/> values that specify how the text should be parsed.</param>
/// <returns>The parsed token.</returns>
private static TextParserToken ParseLexerToken(LexedTokenType tokenType, StringSegment tokenText, Int32 sourceOffset, Int32 sourceLength, TextParserOptions options)
{
var isIgnoringCommandCodes = (options & TextParserOptions.IgnoreCommandCodes) == TextParserOptions.IgnoreCommandCodes;
if (tokenType == LexedTokenType.Command && !isIgnoringCommandCodes)
{
return(ParseCommandToken(tokenText, sourceOffset, sourceLength));
}
return(new TextParserToken(TextParserTokenType.Text, tokenText, sourceOffset, sourceLength,
tokenType == LexedTokenType.NonBreakingWhiteSpace));
}
/// <summary>
/// Captures a new line as a token , willincrment line number and handle \r\n pair
/// </summary>
void CaptureNewLine()
{
//select whatever char triggerd
Select();
//if the next is /n then its a pair and selet the /n
if (!IsEOF && Current == '\n')
{
Select();
}
CaptureToken(LexedTokenType.NEWLINE);
PersumedTokenType = LexedTokenType.UNKNOWN;
CurLineNum++;
CurLinePos = 1;
}
/// <summary>
/// Validates that the current token type is as specified. Then moves the enumerator forward.
/// </summary>
/// <param name="type"></param>
/// <returns>False if the current token is not expected.</returns>
private bool ConsumeToken(LexedTokenType type)
{
if (currentToken == null)
{
return(false);
}
if (currentToken.Type != type)
{
return(false);
}
NextToken();
return(true);
}
private bool LexSpecialChars()
{
// check if the special char mappings has this token if not reurhtn
if (!LexedToken.charToToken_Dictionary.ContainsKey(Current))
{
return(false);
}
//select and capture the token based onthe mapping and set it back to unknown
var b = Current;
Select();
CaptureToken(LexedToken.charToToken_Dictionary[b]);
PersumedTokenType = LexedTokenType.UNKNOWN;
return(true);
}
private void LexRegex()
{
//for regex capture all chars until we reach another /
// hoever if we find a escaping backslash , then check to see if it is followed by a forward slash
if (Current == '\\')
{
var nextchar = Peek();
if (nextchar == null)
{
// this is a error
//return and allow the loop to throw the exception
return;
}
if (nextchar == '/')
{
//this is an ecscape char for a forward slash
//skip the escape char
Skip();
//and select the forward slash
Select();
return;
}
}
// if the char is a forward lsash then we know that the reges is over
if (Current == '/')
{
//capture the buffer as a regex
CaptureToken(LexedTokenType.REGEX);
//we don't need to capture the slash
Skip();
PersumedTokenType = LexedTokenType.UNKNOWN;
return;
}
//you can have a new line char in a regex
if (Current == '\n')
{
throwParserException("Regex expression must be closed");
}
//else its just a char select it into the buffer
Select();
}
private bool CurrentTokenIs(LexedTokenType type)
{
return(currentToken != null && currentToken.Type == type);
}
/// <summary>
/// Parses a lexer token.
/// </summary>
/// <param name="tokenType">A <see cref="LexedTokenType"/> value specifying the type of token produced by the lexer.</param>
/// <param name="tokenText">The text associated with the lexer token.</param>
/// <param name="sourceOffset">The offset of the first character in the source text that produced the token.</param>
/// <param name="sourceLength">The number of characters in the source text that produced the token.</param>
/// <param name="options">A set of <see cref="TextParserOptions"/> values that specify how the text should be parsed.</param>
/// <returns>The parsed token.</returns>
private static TextParserToken ParseLexerToken(LexedTokenType tokenType, StringSegment tokenText, Int32 sourceOffset, Int32 sourceLength, TextParserOptions options)
{
var isIgnoringCommandCodes = (options & TextParserOptions.IgnoreCommandCodes) == TextParserOptions.IgnoreCommandCodes;
if (tokenType == LexedTokenType.Command && !isIgnoringCommandCodes)
return ParseCommandToken(tokenText, sourceOffset, sourceLength);
return new TextParserToken(TextParserTokenType.Text, tokenText, sourceOffset, sourceLength,
tokenType == LexedTokenType.NonBreakingWhiteSpace);
}
public LexedToken(LexedTokenType type, string value)
{
Type = type;
Value = value;
}
private void LexBOL()
{
//Line starts with a slash - its a regex
if (Current == '/')
{
//we don't need to capture the slash
Skip();
PersumedTokenType = LexedTokenType.REGEX;
return;
}
// if its a single qoute then its a delimieted text
if (Current == '\'')
{
//Line starts with a slash - its a regex
//we don't need to capture the single quote
Skip();
PersumedTokenType = LexedTokenType.DELIMITEDTEXT;
return;
}
//if its a newline then capture it as a toke and exit
if (Current == '\n' || Current == '\r')
{
CaptureNewLine();
return;
}
// if its a delimieted Text
if (Current == '\'')
{
Skip();//no need to capture the deimilater
PersumedTokenType = LexedTokenType.DELIMITEDTEXT;
return;
}
if (Current == '!')
{
var peekChar = Peek();
if (peekChar.HasValue)
{
if (peekChar.Value == '(')
{
Select();
Select();
CaptureToken(LexedTokenType.EXCLAMATIONOPENPAREN);
return;
}
Select();
CaptureToken(LexedTokenType.EXCLAMATION);
return;
}
throwParserException("Invalid token '!'");
}
//if its a number
if (Char.IsDigit(Current))
{
Select();
PersumedTokenType = LexedTokenType.NUMBER;
return;
}
//see if its one of the specila chars we have mappings for
if (!LexSpecialChars())
{
//else we will just assume its string
Select();
PersumedTokenType = LexedTokenType.TEXT;
}
}