private bool StringState(char c, out LexedToken token)
        {
            token = null;

            if (c == '\\')
            {
                // we gonna get the escaped character from the next state
                StateFunction = EscapeStringState;

                // consume this slash
                return(true);
            }
            else if (c == '"')
            {
                // return the built string
                token = new LexedToken(LexedTokenType.String, currentValue.ToString());

                // consume this closing quotes
                return(true);
            }

            // build up the string
            currentValue.Append(c);
            return(true);
        }
        private bool NumberState(char c, out LexedToken token)
        {
            token = null;

            if (char.IsDigit(c))
            {
                // consume this digit
                currentValue.Append(c);
                return(true);
            }
            else if (!hasDecimalPlace && c == '.')
            {
                hasDecimalPlace = true;

                // our first decimal point, consume it
                currentValue.Append(c);
                return(true);
            }
            else
            {
                // we end it here since we found some other non-digit
                token = new LexedToken(LexedTokenType.Number, currentValue.ToString());

                // don't consume the character
                return(false);
            }
        }
        private bool LetterState(char c, out LexedToken token)
        {
            token = null;

            if (char.IsLetterOrDigit(c) || c == '_')
            {
                // bumped into a letter/digit, so we build the string further
                currentValue.Append(c);

                // consume it
                return(true);
            }
            else
            {
                // we hit some other random character
                // so we end our processing here and determine what we have
                string value = currentValue.ToString();

                if (keywords.Contains(value))
                {
                    token = new LexedToken(LexedTokenType.Keyword, value);
                }
                else
                {
                    token = new LexedToken(LexedTokenType.Identifier, value);
                }

                // and we don't consume the character in case it is a start of a new token
                return(false);
            }
        }
        public LexedToken GetNextToken()
        {
            LexedToken token       = null;
            bool       consumeChar = false;

            while (!isEOF)
            {
                // line number calculation
                if (currentChar == '\n')
                {
                    lineNumber++;
                    positionNumber = 0;
                }

                consumeChar = StateFunction(currentChar, out token);

                // we consume character only if the last
                // call asked us to do so
                if (consumeChar)
                {
                    NextChar();
                }

                if (token != null)
                {
                    // we managed to grab a token, so we should clear our state and return
                    // the token
                    ClearState();
                    token.PositionNumber = currentLexedPosition;
                    token.LineNumber     = currentLineNumber;

                    return(token);
                }

                // we didn't get any token
                // perhaps it was a state change
            }

            // we reached EOF, we see if we can get anything out of our current state?
            if (currentValue.Length > 0)
            {
                StateFunction('\0', out token);
                if (token != null)
                {
                    ClearState();
                    token.PositionNumber = currentLexedPosition;
                    token.LineNumber     = currentLineNumber;
                }
                return(token);
            }
            else
            {
                // nah, nothing left
                return(null);
            }
        }
        private bool EscapeStringState(char c, out LexedToken token)
        {
            token = null;

            if (escapeLookup.ContainsKey(c))
            {
                // we support this escape character
                currentValue.Append(escapeLookup[c]);

                // set the state back to string builder
                StateFunction = StringState;

                // consume this character
                return(true);
            }
            else
            {
                // return an invalid token and consume this
                MarkPosition();
                token = new LexedToken(LexedTokenType.Invalid, "\\" + c.ToString());

                return(true);
            }
        }
        /// <summary>
        /// The initial state
        /// </summary>
        private bool RootState(char c, out LexedToken token)
        {
            token = null;
            if (char.IsWhiteSpace(c)) // skip whitespaces
            {
                return(true);
            }

            if (char.IsLetter(c) || c == '_')
            {
                MarkPosition();

                // we bumped into a letter
                currentValue.Append(c);

                // so we change to an appropriate state
                StateFunction = LetterState;

                // consume this character
                return(true);
            }
            else if (char.IsDigit(c))
            {
                MarkPosition();

                // we got a first digit, let's not consume it first and let the
                // state function do that
                StateFunction = NumberState;

                return(false);
            }
            else if (c == '"')
            {
                MarkPosition();

                // the start of a string
                StateFunction = StringState;

                // consume this character
                return(true);
            }
            else if (c == '+')
            {
                token = SingularToken(LexedTokenType.OperatorPlus, c);
                return(true);
            }
            else if (c == '-')
            {
                token = SingularToken(LexedTokenType.OperatorMinus, c);
                return(true);
            }
            else if (c == '*')
            {
                token = SingularToken(LexedTokenType.OperatorMultiply, c);
                return(true);
            }
            else if (c == '/')
            {
                token = SingularToken(LexedTokenType.OperatorDivide, c);
                return(true);
            }
            else if (c == '(')
            {
                token = SingularToken(LexedTokenType.OpenParen, c);
                return(true);
            }
            else if (c == ')')
            {
                token = SingularToken(LexedTokenType.CloseParen, c);
                return(true);
            }
            else if (c == '{')
            {
                token = SingularToken(LexedTokenType.OpenBraces, c);
                return(true);
            }
            else if (c == '}')
            {
                token = SingularToken(LexedTokenType.CloseBraces, c);
                return(true);
            }
            else if (c == ';')
            {
                token = SingularToken(LexedTokenType.Semicolon, c);
                return(true);
            }
            else if (c == '=')
            {
                MarkPosition();

                // TODO: Transition to a state to get the ==
                currentValue.Append(c);
                token = new LexedToken(LexedTokenType.OperatorEqual, currentValue.ToString());
                return(true);
            }

            MarkPosition();
            currentValue.Append(c);
            token = new LexedToken(LexedTokenType.Invalid, currentValue.ToString());
            return(true);
        }