public Lexer(Stellarmass.LPC.Scanner.Scanner scannedData)
{
int lineCount = 0;
int tokenCount = 0;
GreedyLexmePair greedy = new GreedyLexmePair();
List <Lexme> tempTokensList = new List <Lexme>();
foreach (Scanner.CodeLine line in scannedData.Lines)
{
foreach (Scanner.Token scanned in line.Tokens)
{
try{
Lexme lexme = new Lexme();
lexme.Data = scanned.Data;
lexme.Type = IdentifyType(scanned, lineCount + 1, greedy);
//this will automatically combine lexme with any previous lexmes if they are supposed to
//be grouped together as a string or comment
greedy.CombineGroupedTokens(lexme, tempTokensList, lineCount + 1);
}
catch (LexerException e)
{
throw new LexerException(e.Message, lineCount);
}
tokenCount++;
} //end foreach
//OLD POSITION
lineCount++;
tokenCount = 0;
}
//NEW POSITION
//just in case the line ended before we could finish the token list
if (greedy.FlushGroup() != null)
{
tempTokensList.AddRange(greedy.FlushGroup());
}
Lines.Add(new Line(tempTokensList, Lines.Count));
}
/// <summary>
///
/// </summary>
/// <param name="sToken"></param>
/// <param name="lineNumber"></param>
/// <param name="pairState">A mini state machine used to see how to process lexmes when
/// we are waiting for closing pairs of lexmes</param>
/// <returns></returns>
private LexmeType IdentifyType(Scanner.Token sToken, int lineNumber, GreedyLexmePair greedy) //LexmeType pairState)
{
LexmeType type = LexmeType.none;
string data = sToken.Data.ToString();
//switch statements... ruining good code since 1967. (good thing this isn't good code ;)
switch (sToken.Type)
{
case Stellarmass.LPC.Scanner.TokenType.Whitespace:
{
type = LexmeType.Whitespace;
break;
}
case Stellarmass.LPC.Scanner.TokenType.NewLine:
{
type = LexmeType.NewLine;
//converting from '\n' back to a standard type for this environment
//sToken.Data = new StringBuilder(Environment.NewLine);
break;
}
case Stellarmass.LPC.Scanner.TokenType.Number:
{
type = LexmeType.NumberLiteral;
break;
}
case Stellarmass.LPC.Scanner.TokenType.Word:
{
if (LPC.LPCKeywords.IsInstruction(data))
{
type = LexmeType.Instruction;
}
else if (LPC.LPCKeywords.IsDataType(data))
{
type = LexmeType.DataType;
}
else if (LPC.LPCKeywords.IsKeyword(data))
{
type = LexmeType.Keyword;
}
else
{
type = LexmeType.Identifier;
}
break;
}
case Stellarmass.LPC.Scanner.TokenType.Symbol:
{
if (LPC.LPCKeywords.IsOperator(data))
{
type = LexmeType.Operator;
if (data == "(")
{
type = LexmeType.OpenParen;
}
if (data == ")")
{
type = LexmeType.CloseParen;
}
if (data == "{")
{
type = LexmeType.OpenBrace;
}
if (data == "}")
{
type = LexmeType.CloseBrace;
}
if (data == "[")
{
type = LexmeType.OpenBracket;
}
if (data == "]")
{
type = LexmeType.CloseBracket;
}
if (data == "<")
{
type = LexmeType.OpenChevron;
}
if (data == ">")
{
type = LexmeType.CloseChevron;
}
}
else if (LPC.LPCKeywords.IsSymbol(data))
{
switch (data)
{
//special cases for apostrophes, slashes, and backslashes
case "\'": { type = LexmeType.SingleQuote; break; }
case "\\'": { type = LexmeType.SingleQuote; break; }
case "\\": { type = LexmeType.EscapeCharacter; break; }
case "\"": { type = LexmeType.Quote; break; }
case "#": { type = LexmeType.PreprocessLine; break; }
default: { type = LexmeType.Symbol; break; }
}
}
else if (LPC.LPCKeywords.IsCommentMark(data))
{
switch (data)
{
case "/*": { type = LexmeType.OpenComment; break; }
case "*/": { type = LexmeType.CloseComment; break; }
case "//": { type = LexmeType.CommentLine; break; }
default: {
if (greedy.Active)
{
type = LexmeType.Grouped;
}
else
{
throw new LexerException(string.Format(LexerException.UnknownTokenParsed, (string)data));
}
break;
}
}
}
else
{
switch (data)
{
//special cases for apostrophes, slashes, and backslashes
case "\'": { type = LexmeType.SingleQuote; break; }
case "\\'": { type = LexmeType.SingleQuote; break; }
case "\\": { type = LexmeType.EscapeCharacter; break; }
default:
{
if (greedy.Active)
{
type = LexmeType.Grouped;
}
else
{
throw new LexerException(string.Format(LexerException.UnknownTokenParsed, (string)data));
}
break;
}
}
}
break;
}
default:
{
if (greedy.Active)
{
type = LexmeType.Grouped;
}
else
{
throw new LexerException(string.Format(LexerException.UnknownTokenParsed, (string)data));
}
break;
}
}
return(type);
}
