public void Dispose() { TokenStreamReader.Dispose(); TokenStreamReader = null; LanguageTokens = null; CurrentTokenBuffer?.Clear(); }
public bool MoveNext() { InitlializeTokenBufferIfEmpty(); if (CurrentSymbol == LexiconSymbol.SkipMaterial || CurrentSymbol == LexiconSymbol.NA) { CurrentTokenBuffer.Clear(); } while (!TokenStreamReader.EndOfStream) { var rawCharacter = TokenStreamReader.Read(); var convertedCharacter = Convert.ToChar(rawCharacter); var singleToken = LanguageTokens.FindLexiconSymbol(convertedCharacter); CurrentTokenBuffer.Add(convertedCharacter); var validTokenBuffer = LanguageTokens.FindLexiconSymbol(CurrentTokenBuffer); var tempValidTokenBuffer = validTokenBuffer; var rulesToApply = validTokenBuffer; rulesToApply = Rules.Select(appliedRule => rulesToApply = appliedRule(singleToken, rulesToApply, CurrentSymbol)) .Where(ls => ls != LexiconSymbol.NA) .LastOrDefault(); if (rulesToApply != tempValidTokenBuffer) { validTokenBuffer = rulesToApply; } if (CurrentTokenBuffer.Count > 0 && validTokenBuffer == LexiconSymbol.Assign) { CurrentTokenBuffer.RemoveAt(CurrentTokenBuffer.Count - 1); } CurrentSymbol = validTokenBuffer; var mayStopSymbol = LanguageTokens.FindLexiconSymbol(new List <char>() { convertedCharacter }); if (mayStopSymbol == LexiconSymbol.SkipMaterial) { CurrentSymbol = mayStopSymbol; } if (CurrentSymbol != LexiconSymbol.NA) { return(true); } } return(false); }