public Lexer(Stellarmass.LPC.Scanner.Scanner scannedData) { int lineCount = 0; int tokenCount = 0; GreedyLexmePair greedy = new GreedyLexmePair(); List <Lexme> tempTokensList = new List <Lexme>(); foreach (Scanner.CodeLine line in scannedData.Lines) { foreach (Scanner.Token scanned in line.Tokens) { try{ Lexme lexme = new Lexme(); lexme.Data = scanned.Data; lexme.Type = IdentifyType(scanned, lineCount + 1, greedy); //this will automatically combine lexme with any previous lexmes if they are supposed to //be grouped together as a string or comment greedy.CombineGroupedTokens(lexme, tempTokensList, lineCount + 1); } catch (LexerException e) { throw new LexerException(e.Message, lineCount); } tokenCount++; } //end foreach //OLD POSITION lineCount++; tokenCount = 0; } //NEW POSITION //just in case the line ended before we could finish the token list if (greedy.FlushGroup() != null) { tempTokensList.AddRange(greedy.FlushGroup()); } Lines.Add(new Line(tempTokensList, Lines.Count)); }
/// <summary> /// /// </summary> /// <param name="sToken"></param> /// <param name="lineNumber"></param> /// <param name="pairState">A mini state machine used to see how to process lexmes when /// we are waiting for closing pairs of lexmes</param> /// <returns></returns> private LexmeType IdentifyType(Scanner.Token sToken, int lineNumber, GreedyLexmePair greedy) //LexmeType pairState) { LexmeType type = LexmeType.none; string data = sToken.Data.ToString(); //switch statements... ruining good code since 1967. (good thing this isn't good code ;) switch (sToken.Type) { case Stellarmass.LPC.Scanner.TokenType.Whitespace: { type = LexmeType.Whitespace; break; } case Stellarmass.LPC.Scanner.TokenType.NewLine: { type = LexmeType.NewLine; //converting from '\n' back to a standard type for this environment //sToken.Data = new StringBuilder(Environment.NewLine); break; } case Stellarmass.LPC.Scanner.TokenType.Number: { type = LexmeType.NumberLiteral; break; } case Stellarmass.LPC.Scanner.TokenType.Word: { if (LPC.LPCKeywords.IsInstruction(data)) { type = LexmeType.Instruction; } else if (LPC.LPCKeywords.IsDataType(data)) { type = LexmeType.DataType; } else if (LPC.LPCKeywords.IsKeyword(data)) { type = LexmeType.Keyword; } else { type = LexmeType.Identifier; } break; } case Stellarmass.LPC.Scanner.TokenType.Symbol: { if (LPC.LPCKeywords.IsOperator(data)) { type = LexmeType.Operator; if (data == "(") { type = LexmeType.OpenParen; } if (data == ")") { type = LexmeType.CloseParen; } if (data == "{") { type = LexmeType.OpenBrace; } if (data == "}") { type = LexmeType.CloseBrace; } if (data == "[") { type = LexmeType.OpenBracket; } if (data == "]") { type = LexmeType.CloseBracket; } if (data == "<") { type = LexmeType.OpenChevron; } if (data == ">") { type = LexmeType.CloseChevron; } } else if (LPC.LPCKeywords.IsSymbol(data)) { switch (data) { //special cases for apostrophes, slashes, and backslashes case "\'": { type = LexmeType.SingleQuote; break; } case "\\'": { type = LexmeType.SingleQuote; break; } case "\\": { type = LexmeType.EscapeCharacter; break; } case "\"": { type = LexmeType.Quote; break; } case "#": { type = LexmeType.PreprocessLine; break; } default: { type = LexmeType.Symbol; break; } } } else if (LPC.LPCKeywords.IsCommentMark(data)) { switch (data) { case "/*": { type = LexmeType.OpenComment; break; } case "*/": { type = LexmeType.CloseComment; break; } case "//": { type = LexmeType.CommentLine; break; } default: { if (greedy.Active) { type = LexmeType.Grouped; } else { throw new LexerException(string.Format(LexerException.UnknownTokenParsed, (string)data)); } break; } } } else { switch (data) { //special cases for apostrophes, slashes, and backslashes case "\'": { type = LexmeType.SingleQuote; break; } case "\\'": { type = LexmeType.SingleQuote; break; } case "\\": { type = LexmeType.EscapeCharacter; break; } default: { if (greedy.Active) { type = LexmeType.Grouped; } else { throw new LexerException(string.Format(LexerException.UnknownTokenParsed, (string)data)); } break; } } } break; } default: { if (greedy.Active) { type = LexmeType.Grouped; } else { throw new LexerException(string.Format(LexerException.UnknownTokenParsed, (string)data)); } break; } } return(type); }