Exemplo n.º 1
0
 public void UngetToken(TokenRec <T> Token)
 {
     if (PushedBackToken != null)
     {
         Debug.Assert(false);
         throw new InvalidOperationException();
     }
     PushedBackToken = Token;
 }
Exemplo n.º 2
0
        /* compile a special function.  a special function has no function header, but is */
        /* simply some code to be executed.  the parameters the code is expecting are provided */
        /* in the FuncArray[] and NumParams.  the first parameter is deepest beneath the */
        /* top of stack.  the TextData is NOT altered.  if an error occurrs, *FunctionOut */
        /* will NOT contain a valid object */
        public static CompileErrors CompileSpecialFunction(
            CodeCenterRec CodeCenter,
            FunctionParamRec[] FuncArray,
            out int ErrorLineNumber,
            out DataTypes ReturnTypeOut,
            string TextData,
            bool suppressCILEmission,
            out PcodeRec FunctionOut,
            out ASTExpression ASTOut)
        {
            CompileErrors Error;

            ErrorLineNumber = -1;
            ASTOut          = null;
            FunctionOut     = null;
            ReturnTypeOut   = DataTypes.eInvalidDataType;

            ScannerRec <KeywordsType> TheScanner     = new ScannerRec <KeywordsType>(TextData, KeywordTable);
            SymbolTableRec            TheSymbolTable = new SymbolTableRec();

            // reconstitute function prototypes
            for (int i = 0; i < CodeCenter.RetainedFunctionSignatures.Length; i++)
            {
                SymbolRec functionSignature = ArgListTypesToSymbol(
                    CodeCenter.RetainedFunctionSignatures[i].Key,
                    CodeCenter.RetainedFunctionSignatures[i].Value.ArgsTypes,
                    CodeCenter.RetainedFunctionSignatures[i].Value.ReturnType);
                bool f = TheSymbolTable.Add(functionSignature);
                Debug.Assert(f); // should never fail (due to duplicate) since CodeCenter.RetainedFunctionSignatures is unique-keyed
            }

            /* build parameters into symbol table */
            int StackDepth         = 0;
            int MaxStackDepth      = 0;
            int ReturnAddressIndex = StackDepth;

            for (int i = 0; i < FuncArray.Length; i += 1)
            {
                SymbolRec TheParameter = new SymbolRec(FuncArray[i].ParameterName);
                TheParameter.SymbolBecomeVariable(FuncArray[i].ParameterType);
                /* allocate stack slot */
                StackDepth++;
                MaxStackDepth = Math.Max(MaxStackDepth, StackDepth);
                TheParameter.SymbolVariableStackLocation = StackDepth;
                if (!TheSymbolTable.Add(TheParameter)) // our own code should never pass in a formal arg list with duplicates
                {
                    Debug.Assert(false);
                    throw new InvalidOperationException();
                }
            }
            /* fence them off */
            TheSymbolTable.IncrementSymbolTableLevel();

            /* reserve spot for fake return address (so we have uniform calling convention everywhere) */
            StackDepth++;
            MaxStackDepth = Math.Max(MaxStackDepth, StackDepth);
            if (StackDepth != FuncArray.Length + 1)
            {
                // stack depth error before evaluating function
                Debug.Assert(false);
                throw new InvalidOperationException();
            }

            ASTExpressionList ListOfExpressions;

            Error = ParseExprList(
                out ListOfExpressions,
                new ParserContext(
                    TheScanner,
                    TheSymbolTable),
                out ErrorLineNumber);
            /* compile the thing */
            if (Error != CompileErrors.eCompileNoError)
            {
                return(Error);
            }
            ASTExpression TheExpressionThang = new ASTExpression(
                ListOfExpressions,
                TheScanner.GetCurrentLineNumber());

            /* make sure there is nothing after it */
            TokenRec <KeywordsType> Token = TheScanner.GetNextToken();

            if (Token.GetTokenType() != TokenTypes.eTokenEndOfInput)
            {
                ErrorLineNumber = TheScanner.GetCurrentLineNumber();
                return(CompileErrors.eCompileInputBeyondEndOfFunction);
            }

            DataTypes ResultingType;

            Error = TheExpressionThang.TypeCheck(out ResultingType, out ErrorLineNumber);
            if (Error != CompileErrors.eCompileNoError)
            {
                return(Error);
            }

            OptimizeAST(ref TheExpressionThang);

            PcodeRec TheFunctionCode = new PcodeRec();

            TheExpressionThang.PcodeGen(
                TheFunctionCode,
                ref StackDepth,
                ref MaxStackDepth);
            Debug.Assert(StackDepth <= MaxStackDepth);

            ReturnTypeOut = TheExpressionThang.ResultType;


            /* 2 extra words for retaddr, resultofexpr */
            if (StackDepth != FuncArray.Length + 1 /*retaddr*/ + 1 /*result*/)
            {
                // stack depth error after evaluating function
                Debug.Assert(false);
                throw new InvalidOperationException();
            }
            /* now put the return instruction */
            int unused;

            TheFunctionCode.AddPcodeInstruction(Pcodes.epReturnFromSubroutine, out unused, TheScanner.GetCurrentLineNumber());
            // special function returns without popping args -- so that args can be have in/out behavior
            TheFunctionCode.AddPcodeOperandInteger(0);
            StackDepth -= 1; /* pop retaddr */
            Debug.Assert(StackDepth <= MaxStackDepth);
            if (StackDepth != 1 + FuncArray.Length)
            {
                // stack depth is wrong at end of function
                Debug.Assert(false);
                throw new InvalidOperationException();
            }

            TheFunctionCode.MaxStackDepth = MaxStackDepth;

            /* optimize stupid things away */
            TheFunctionCode.OptimizePcode();


            if (CILObject.EnableCIL && !suppressCILEmission)
            {
                DataTypes[] argsTypes = new DataTypes[FuncArray.Length];
                string[]    argsNames = new string[FuncArray.Length];
                for (int i = 0; i < argsTypes.Length; i++)
                {
                    argsTypes[i] = FuncArray[i].ParameterType;
                    argsNames[i] = FuncArray[i].ParameterName;
                }
                CILAssembly cilAssembly = new CILAssembly();
                CILObject   cilObject   = new CILObject(
                    CodeCenter.ManagedFunctionLinker,
                    argsTypes,
                    argsNames,
                    TheExpressionThang.ResultType,
                    TheExpressionThang,
                    cilAssembly,
                    true /*argsByRef*/); // args by ref true for special functions to permit multiple return values
                TheFunctionCode.cilObject = cilObject;
                cilAssembly.Finish();
            }


            /* it worked, so return the dang thing */
            FunctionOut = TheFunctionCode;
            ASTOut      = TheExpressionThang;
            return(CompileErrors.eCompileNoError);
        }
Exemplo n.º 3
0
        // Compile multiple modules. (eliminates the need to do prototyping or function signature inference.)
        // CodeCenter is cleared, and if compilation succeeds, the functions are added to CodeCenter.
        public static CompileErrors CompileWholeProgram(
            out int ErrorLineNumber,
            out int ErrorModuleIndex,
            string[] TextDatas,
            object[] Signatures,
            CodeCenterRec CodeCenter,
            string[] Filenames)
        {
            Debug.Assert(TextDatas.Length == Signatures.Length);
            Debug.Assert(TextDatas.Length == Filenames.Length);

            ErrorLineNumber  = -1;
            ErrorModuleIndex = -1;

            CodeCenter.FlushAllCompiledFunctions();
            CodeCenter.RetainedFunctionSignatures = new KeyValuePair <string, FunctionSignature> [0];

            // parse

            List <SymbolRec>     SymbolTableEntriesForForm = new List <SymbolRec>();
            List <ASTExpression> FunctionBodyRecords       = new List <ASTExpression>();
            List <int>           ModuleIndices             = new List <int>();
            Dictionary <string, List <ParserContext.FunctionSymbolRefInfo> > FunctionRefSymbolList = new Dictionary <string, List <ParserContext.FunctionSymbolRefInfo> >();
            List <int> InitialLineNumbersOfForm = new List <int>();

            for (int module = 0; module < TextDatas.Length; module++)
            {
                string TextData = TextDatas[module];

                ErrorModuleIndex = module;

                ScannerRec <KeywordsType> TheScanner     = new ScannerRec <KeywordsType>(TextData, KeywordTable);
                SymbolTableRec            TheSymbolTable = new SymbolTableRec();

                /* loop until there are no more things to parse */
                while (true)
                {
                    TokenRec <KeywordsType> Token = TheScanner.GetNextToken();
                    int InitialLineNumberOfForm   = TheScanner.GetCurrentLineNumber();
                    if (Token.GetTokenType() == TokenTypes.eTokenEndOfInput)
                    {
                        /* no more functions to parse, so stop */
                        break;
                    }

                    SymbolRec     SymbolTableEntryForForm;
                    ASTExpression FunctionBodyRecord;

                    /* parse the function */
                    TheScanner.UngetToken(Token);
                    CompileErrors Error = ParseForm(
                        out SymbolTableEntryForForm,
                        out FunctionBodyRecord,
                        new ParserContext(
                            TheScanner,
                            TheSymbolTable,
                            FunctionRefSymbolList),
                        out ErrorLineNumber);
                    if (Error != CompileErrors.eCompileNoError)
                    {
                        return(Error);
                    }

                    Debug.Assert(FunctionBodyRecord != null);
                    ModuleIndices.Add(module);
                    SymbolTableEntriesForForm.Add(SymbolTableEntryForForm);
                    FunctionBodyRecords.Add(FunctionBodyRecord);
                    InitialLineNumbersOfForm.Add(InitialLineNumberOfForm);
                }

                foreach (KeyValuePair <string, List <ParserContext.FunctionSymbolRefInfo> > name in FunctionRefSymbolList)
                {
                    foreach (ParserContext.FunctionSymbolRefInfo funcRef in name.Value)
                    {
                        funcRef.module = module;
                    }
                }
            }

            // push function type signatures into function call refs

            Dictionary <string, bool> functionNamesUsed = new Dictionary <string, bool>();

            for (int i = 0; i < SymbolTableEntriesForForm.Count; i++)
            {
                ErrorModuleIndex = ModuleIndices[i];

                SymbolRec FunctionDeclarationSymbol = SymbolTableEntriesForForm[i];
                if (functionNamesUsed.ContainsKey(FunctionDeclarationSymbol.SymbolName))
                {
                    ErrorLineNumber = FunctionBodyRecords[i].LineNumber;
                    return(CompileErrors.eCompileMultiplyDeclaredFunction);
                }
                functionNamesUsed.Add(FunctionDeclarationSymbol.SymbolName, false);

                List <ParserContext.FunctionSymbolRefInfo> symbols;
                if (FunctionRefSymbolList.TryGetValue(FunctionDeclarationSymbol.SymbolName, out symbols))
                {
                    foreach (ParserContext.FunctionSymbolRefInfo functionRef in symbols)
                    {
                        functionRef.symbol.SymbolBecomeFunction2(
                            FunctionDeclarationSymbol.FunctionArgList,
                            FunctionDeclarationSymbol.FunctionReturnType);
                    }
                    FunctionRefSymbolList.Remove(FunctionDeclarationSymbol.SymbolName);
                }
            }

            foreach (KeyValuePair <string, List <ParserContext.FunctionSymbolRefInfo> > name in FunctionRefSymbolList)
            {
                foreach (ParserContext.FunctionSymbolRefInfo funcRef in name.Value)
                {
                    ErrorModuleIndex = funcRef.module;
                    ErrorLineNumber  = funcRef.lineNumber;
                    return(CompileErrors.eCompileMultiplyDeclaredFunction);
                }
            }

            // type check and type inference

            for (int i = 0; i < FunctionBodyRecords.Count; i++)
            {
                int           module = ModuleIndices[i];
                SymbolRec     SymbolTableEntryForForm = SymbolTableEntriesForForm[i];
                ASTExpression FunctionBodyRecord      = FunctionBodyRecords[i];
                int           InitialLineNumberOfForm = InitialLineNumbersOfForm[i];

                ErrorModuleIndex = module;

                /* SymbolTableEntryForForm will be the symbol table entry that */
                /* was added to the symbol table.  FunctionBodyRecord is either */
                /* an expression for a function or NIL if it was a prototype */

                Debug.Assert(!CodeCenter.CodeCenterHaveThisFunction(SymbolTableEntryForForm.SymbolName));

                /* step 1:  do type checking */
                DataTypes     ResultingType;
                CompileErrors Error = FunctionBodyRecord.TypeCheck(
                    out ResultingType,
                    out ErrorLineNumber);
                if (Error != CompileErrors.eCompileNoError)
                {
                    return(Error);
                }
                /* check to see that resulting type matches declared type */
                if (!CanRightBeMadeToMatchLeft(SymbolTableEntryForForm.FunctionReturnType, ResultingType))
                {
                    ErrorLineNumber = InitialLineNumberOfForm;
                    return(CompileErrors.eCompileTypeMismatch);
                }
                /* if it has to be promoted, then promote it */
                if (MustRightBePromotedToLeft(SymbolTableEntryForForm.FunctionReturnType, ResultingType))
                {
                    /* insert promotion operator above expression */
                    ASTExpression ReplacementExpr = PromoteTheExpression(
                        ResultingType,
                        SymbolTableEntryForForm.FunctionReturnType,
                        FunctionBodyRecord,
                        InitialLineNumberOfForm);
                    FunctionBodyRecord = ReplacementExpr;
                    /* sanity check */
                    Error = FunctionBodyRecord.TypeCheck(
                        out ResultingType,
                        out ErrorLineNumber);
                    if (Error != CompileErrors.eCompileNoError)
                    {
                        // type promotion caused failure
                        Debug.Assert(false);
                        throw new InvalidOperationException();
                    }
                    if (ResultingType != SymbolTableEntryForForm.FunctionReturnType)
                    {
                        // after type promotion, types are no longer the same
                        Debug.Assert(false);
                        throw new InvalidOperationException();
                    }
                }
            }

            // code generation

            CILAssembly cilAssembly = null;

            if (CILObject.EnableCIL)
            {
                cilAssembly = new CILAssembly();
            }

            FuncCodeRec[] TheWholeFunctionThings = new FuncCodeRec[FunctionBodyRecords.Count];
            for (int i = 0; i < FunctionBodyRecords.Count; i++)
            {
                int           module = ModuleIndices[i];
                SymbolRec     SymbolTableEntryForForm = SymbolTableEntriesForForm[i];
                ASTExpression FunctionBodyRecord      = FunctionBodyRecords[i];
                int           InitialLineNumberOfForm = InitialLineNumbersOfForm[i];

                string Filename  = Filenames[module];
                object Signature = Signatures[module];

                ErrorModuleIndex = module;

                Debug.Assert(!CodeCenter.CodeCenterHaveThisFunction(SymbolTableEntryForForm.SymbolName));

                /* step 1.5:  optimize the AST */
                OptimizeAST(ref FunctionBodyRecord);

                /* step 2:  do code generation */
                /* calling conventions:  */
                /*  - push the arguments */
                /*  - funccall pushes the return address */
                /* thus, upon entry, Stack[0] will be the return address */
                /* and Stack[-1] will be the rightmost argument */
                /* on return, args and retaddr are popped and retval replaces them */
                int           StackDepth             = 0;
                int           MaxStackDepth          = 0;
                int           ReturnValueLocation    = StackDepth; /* remember return value location */
                int           ArgumentIndex          = 0;
                SymbolListRec FormalArgumentListScan = SymbolTableEntryForForm.FunctionArgList;
                while (FormalArgumentListScan != null)
                {
                    SymbolRec TheFormalArg = FormalArgumentListScan.First;
                    StackDepth++;                                          /* allocate first */
                    MaxStackDepth = Math.Max(MaxStackDepth, StackDepth);
                    TheFormalArg.SymbolVariableStackLocation = StackDepth; /* remember */
                    ArgumentIndex++;
                    FormalArgumentListScan = FormalArgumentListScan.Rest;
                }
                /* reserve return address spot */
                StackDepth++;
                MaxStackDepth = Math.Max(MaxStackDepth, StackDepth);
                /* allocate the function code */
                PcodeRec TheFunctionCode = new PcodeRec();
                FunctionBodyRecord.PcodeGen(
                    TheFunctionCode,
                    ref StackDepth,
                    ref MaxStackDepth);
                Debug.Assert(StackDepth <= MaxStackDepth);
                /* 2 extra words for retaddr and resultofexpr */
                if (StackDepth != ArgumentIndex + 1 + 1)
                {
                    // stack depth error after evaluating function
                    Debug.Assert(false);
                    throw new InvalidOperationException();
                }
                /* now put the return instruction (pops retaddr and args, leaving retval) */
                int ignored;
                TheFunctionCode.AddPcodeInstruction(Pcodes.epReturnFromSubroutine, out ignored, InitialLineNumberOfForm);
                TheFunctionCode.AddPcodeOperandInteger(ArgumentIndex);
                StackDepth = StackDepth - (1 + ArgumentIndex);
                Debug.Assert(StackDepth <= MaxStackDepth);
                if (StackDepth != 1)
                {
                    // stack depth is wrong at end of function
                    Debug.Assert(false);
                    throw new InvalidOperationException();
                }

                TheFunctionCode.MaxStackDepth = MaxStackDepth;

                /* step 2.5:  optimize the code */
                TheFunctionCode.OptimizePcode();

                /* step 3:  create the function and save it away */
                FuncCodeRec TheWholeFunctionThing = new FuncCodeRec(
                    SymbolTableEntryForForm.SymbolName,
                    SymbolTableEntryForForm.FunctionArgList,
                    TheFunctionCode,
                    SymbolTableEntryForForm.FunctionReturnType,
                    Filename);

                TheWholeFunctionThings[i] = TheWholeFunctionThing;

                if (CILObject.EnableCIL)
                {
                    DataTypes[] argsTypes;
                    string[]    argsNames;
                    SymbolicArgListToType(SymbolTableEntryForForm, out argsTypes, out argsNames);
                    CILObject cilObject = new CILObject(
                        CodeCenter.ManagedFunctionLinker,
                        argsTypes,
                        argsNames,
                        SymbolTableEntryForForm.FunctionReturnType,
                        FunctionBodyRecord,
                        cilAssembly,
                        false /*argsByRef*/);
                    TheWholeFunctionThing.CILObject = cilObject;
                }
            }

            // register after entire assembly is emitted
            if (CILObject.EnableCIL)
            {
                cilAssembly.Finish();
            }

            for (int i = 0; i < TheWholeFunctionThings.Length; i++)
            {
                FuncCodeRec TheWholeFunctionThing = TheWholeFunctionThings[i];
                object      Signature             = Signatures[ModuleIndices[i]];

                CodeCenter.AddFunctionToCodeCenter(TheWholeFunctionThing, Signature);
            }

            // retain signatures for compilation of special functions

            CodeCenter.RetainedFunctionSignatures = new KeyValuePair <string, FunctionSignature> [SymbolTableEntriesForForm.Count];
            for (int i = 0; i < CodeCenter.RetainedFunctionSignatures.Length; i++)
            {
                DataTypes[] argsTypes;
                string[]    argsNames;
                SymbolicArgListToType(SymbolTableEntriesForForm[i], out argsTypes, out argsNames);
                CodeCenter.RetainedFunctionSignatures[i] = new KeyValuePair <string, FunctionSignature>(
                    SymbolTableEntriesForForm[i].SymbolName,
                    new FunctionSignature(
                        argsTypes,
                        SymbolTableEntriesForForm[i].FunctionReturnType));
            }

            return(CompileErrors.eCompileNoError);
        }
Exemplo n.º 4
0
        public TokenRec <T> GetNextToken()
        {
            int          C;
            TokenRec <T> Token;

            /* check for pushback */
            if (PushedBackToken != null)
            {
                TokenRec <T> Temp;

                Temp            = PushedBackToken;
                PushedBackToken = null;
                return(Temp);
            }

            /* get a character */
            C = GetCharacter();

            /* strip while space */
            bool cr = false;

            while (((C >= 0) && (C <= 32)) || (C == '#'))
            {
                if ((C == 13) || (C == 10))
                {
                    bool crPrev = cr;
                    cr = (C == 13);
                    if (!crPrev)
                    {
                        LineNumber++;
                    }
                }
                if (C == '#')
                {
                    /* comment */
                    while ((C != 13) && (C != 10) && (C != ENDOFTEXT))
                    {
                        C = GetCharacter();
                    }
                }
                else
                {
                    C = GetCharacter();
                }
            }

RestartParse:
            /* handle the end of text character */
            if (C == ENDOFTEXT)
            {
                Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenEndOfInput);
            }

            /* handle a string literal */
            else if (C == '\x22')
            {
                StringBuilder String = new StringBuilder();

                cr = false;
                C  = GetCharacter();
                while (C != '\x22')
                {
                    if (C == ENDOFTEXT)
                    {
                        goto BreakStringReadPoint;
                    }
                    if (C == '\\')
                    {
                        C = GetCharacter();
                        if (C == 'n')
                        {
                            String.Append(Environment.NewLine); // originally was '\n'
                            goto DoAnotherCharPoint;
                        }
                        else if ((C == '\x22') || (C == '\\') || (C == 10) || (C == 13))
                        {
                            /* keep these */
                        }
                        else
                        {
                            /* others become strange character */
                            C = '.';
                        }
                    }
                    String.Append((char)C);
                    if ((C == 10) || (C == 13))
                    {
                        bool crPrev = cr;
                        cr = (C == 13);
                        if (!crPrev)
                        {
                            LineNumber++;
                        }
                    }
DoAnotherCharPoint:
                    C = GetCharacter();
                }
BreakStringReadPoint:
                ;

                Token = new StringTokenRec <T>(String.ToString());
            }

            /* handle an identifier:  [a-zA-Z_][a-zA-Z0-9_]*  */
            else if (((C >= 'a') && (C <= 'z')) || ((C >= 'A') && (C <= 'Z')) || (C == '_'))
            {
                StringBuilder String       = new StringBuilder();
                int           KeywordIndex = -1; /* -1 == not a keyword */
                string        StringCopy;

                /* read the entire token */
                while (((C >= 'a') && (C <= 'z')) || ((C >= 'A') && (C <= 'Z')) || (C == '_') || ((C >= '0') && (C <= '9')))
                {
                    String.Append((char)C);
                    C = GetCharacter();
                }
                /* unget the character that made us stop */
                UngetCharacter();
                /* get the string out of the line buffer */
                StringCopy = String.ToString();

                /* figure out if it is a keyword (binary search) */
                int  LowBound            = 0;
                int  HighBoundPlusOne    = KeywordList.Length;
                bool ContinueLoopingFlag = true;
                while (ContinueLoopingFlag)
                {
                    int MidPoint;
                    int CompareResult;

                    if (LowBound > HighBoundPlusOne)
                    {
                        Debug.Assert(false);
                        throw new InvalidOperationException();
                    }

                    MidPoint = (LowBound + HighBoundPlusOne) / 2;

                    CompareResult = StringCopy.CompareTo(KeywordList[MidPoint].KeywordName);
                    /* CompareResult == 0  -->  found the target */
                    /* CompareResult < 0  --> in the first half of the list */
                    /* CompareResult > 0  --> in the second half of the list */

                    if (CompareResult == 0)
                    {
                        /* found the one */
                        KeywordIndex        = MidPoint;
                        ContinueLoopingFlag = false;
                    }
                    else
                    {
                        if (CompareResult < 0)
                        {
                            /* select first half of list */
                            HighBoundPlusOne = MidPoint;
                        }
                        else /* if (CompareResult > 0) */
                        {
                            /* select second half of list */
                            LowBound = MidPoint + 1;
                        }
                        /* termination condition:  if range in array collapses to an */
                        /* empty array, then there is no entry in the array */
                        if (LowBound == HighBoundPlusOne)
                        {
                            KeywordIndex        = -1; /* indicate there is no keyword */
                            ContinueLoopingFlag = false;
                        }
                    }
                }

                /* create the token */
                if (KeywordIndex == -1)
                {
                    /* no keyword; make a string containing token */
                    Token = new IdentifierTokenRec <T>(StringCopy);
                }
                else
                {
                    Token = new KeywordTokenRec <T>(KeywordList[KeywordIndex].TagValue);
                }
            }

            /* integer or floating?  [0-9]+  [0-9]+"."[0-9]+([Ee][+-]?[0-9]+)?[sdf]?  */
            else if (((C >= '0') && (C <= '9'))
                     // TODO: C# 2.0 hack - convert to elegant lambda evaluation after upgrade
                     || ((C == '.') && Eval(delegate() { int CC = GetCharacter(); UngetCharacter(); return((CC >= '0') || (CC <= '9')); })))
            {
                NumFormType   SpecifiedNumType = NumFormType.eTypeNotSpecified;
                NumStateType  NumberState      = NumStateType.eIntegerPart;
                StringBuilder String           = new StringBuilder();
                string        StringData;

                Token = null;

                while (((C >= '0') && (C <= '9')) || (C == '.') || (C == '+') || (C == '-') ||
                       (C == 's') || (C == 'd') || (C == 'f') || (C == 'e') || (C == 'E'))
                {
                    /* do some state changes */
                    if (C == '.')
                    {
                        if (NumberState != NumStateType.eIntegerPart)
                        {
                            Token = new ErrorTokenRec <T>(ScannerErrors.eScannerMalformedFloat);
                            goto AbortNumberErrorPoint;
                        }
                        else
                        {
                            NumberState = NumStateType.eFractionalPart;
                        }
                    }
                    else if ((C == 'e') || (C == 'E'))
                    {
                        if ((NumberState != NumStateType.eIntegerPart) && (NumberState != NumStateType.eFractionalPart))
                        {
                            Token = new ErrorTokenRec <T>(ScannerErrors.eScannerMalformedFloat);
                            goto AbortNumberErrorPoint;
                        }
                        else
                        {
                            NumberState = NumStateType.eExponentialPart;
                        }
                    }
                    else if ((C == '+') || (C == '-'))
                    {
                        if (NumberState != NumStateType.eExponentialPart)
                        {
                            /* this is not an error, since it could be a unary operator */
                            /* coming later, so we stop, but don't abort */
                            goto FinishNumberPoint; /* character ungot at target */
                        }
                        else
                        {
                            NumberState = NumStateType.eExponNumberPart;
                        }
                    }
                    else if ((C == 's') || (C == 'f'))
                    {
                        if (NumberState == NumStateType.eNumberFinished)
                        {
                            Token = new ErrorTokenRec <T>(ScannerErrors.eScannerMalformedFloat);
                            goto AbortNumberErrorPoint;
                        }
                        else
                        {
                            NumberState      = NumStateType.eNumberFinished;
                            SpecifiedNumType = NumFormType.eTypeSingle;
                            C = (char)32; /* so adding it to the string doesn't do damage */
                        }
                    }
                    else if (C == 'd')
                    {
                        if (NumberState == NumStateType.eNumberFinished)
                        {
                            Token = new ErrorTokenRec <T>(ScannerErrors.eScannerMalformedFloat);
                            goto AbortNumberErrorPoint;
                        }
                        else
                        {
                            NumberState      = NumStateType.eNumberFinished;
                            SpecifiedNumType = NumFormType.eTypeDouble;
                            C = (char)32;
                        }
                    }

                    /* actually save the character */
                    String.Append((char)C);

                    C = GetCharacter();
                }
FinishNumberPoint:
                UngetCharacter();

                StringData = String.ToString();

                /* if the token type is not specified, then see what we can guess */
                if (SpecifiedNumType == NumFormType.eTypeNotSpecified)
                {
                    if (NumberState == NumStateType.eIntegerPart)
                    {
                        /* if we only got as far as the integer part, then it's an int */
                        SpecifiedNumType = NumFormType.eTypeInteger;
                    }
                    else
                    {
                        /* otherwise, assume the highest precision type */
                        SpecifiedNumType = NumFormType.eTypeDouble;
                    }
                }

                /* create the token */
                switch (SpecifiedNumType)
                {
                default:
                    Debug.Assert(false);
                    throw new InvalidOperationException();

                case NumFormType.eTypeSingle:
                {
                    float v;
                    if (!Single.TryParse(StringData, out v))
                    {
                        // Reasons it could fail:
                        // 1: our scanner is more permissive than they are - accepting things like "."
                        // 2: number could be syntactically valid but out of range for the type.
                        Token = new ErrorTokenRec <T>(ScannerErrors.eScannerMalformedFloat);
                        goto AbortNumberErrorPoint;
                    }
                    Token = new SingleTokenRec <T>(v);
                }
                break;

                case NumFormType.eTypeDouble:
                {
                    double v;
                    if (!Double.TryParse(StringData, out v))
                    {
                        // Reasons it could fail:
                        // 1: our scanner is more permissive than they are - accepting things like "."
                        // 2: number could be syntactically valid but out of range for the type.
                        Token = new ErrorTokenRec <T>(ScannerErrors.eScannerMalformedFloat);
                        goto AbortNumberErrorPoint;
                    }
                    Token = new DoubleTokenRec <T>(v);
                }
                break;

                case NumFormType.eTypeInteger:
                {
                    int v;
                    if (!Int32.TryParse(StringData, out v))
                    {
                        // Reasons it could fail:
                        // 1: our scanner is more permissive than they are - accepting things like "."
                        // 2: number could be syntactically valid but out of range for the type.
                        Token = new ErrorTokenRec <T>(ScannerErrors.eScannerMalformedInteger);
                        goto AbortNumberErrorPoint;
                    }
                    Token = new IntegerTokenRec <T>(v);
                }
                break;
                }

                /* this is the escape point for when a bad character is encountered. */
AbortNumberErrorPoint:
                ;
            }

            /* handle a symbol */
            else
            {
                Token = null;

                switch (C)
                {
                default:
                    Token = new ErrorTokenRec <T>(ScannerErrors.eScannerUnknownCharacter);
                    break;

                case '(':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenOpenParen);
                    break;

                case ')':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenCloseParen);
                    break;

                case '[':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenOpenBracket);
                    break;

                case ']':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenCloseBracket);
                    break;

                case '{':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenOpenBrace);
                    break;

                case '}':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenCloseBrace);
                    break;

                case ':':
                    C = GetCharacter();
                    if (C == '=')
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenColonEqual);
                    }
                    else
                    {
                        /* push the character back */
                        UngetCharacter();
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenColon);
                    }
                    break;

                case ';':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenSemicolon);
                    break;

                case ',':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenComma);
                    break;

                case '+':
                    C = GetCharacter();
                    if (C == '=')
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenPlusEqual);
                    }
                    else if (C == '+')
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenPlusPlus);
                    }
                    else
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenPlus);
                        UngetCharacter();
                    }
                    break;

                case '-':
                    C = GetCharacter();
                    if (C == '=')
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenMinusEqual);
                    }
                    else if (C == '-')
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenMinusMinus);
                    }
                    else if (C == '>')
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenMinusGreater);
                    }
                    else
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenMinus);
                        UngetCharacter();
                    }
                    break;

                case '*':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenStar);
                    break;

                case '/':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenSlash);
                    break;

                case '=':
                    C = GetCharacter();
                    if (C == '=')
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenEqualEqual);
                    }
                    else
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenEqual);
                        UngetCharacter();
                    }
                    break;

                case '<':
                    C = GetCharacter();
                    if (C == '>')
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenLessGreater);
                    }
                    else if (C == '=')
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenLessEqual);
                    }
                    else if (C == '<')
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenLeftLeft);
                    }
                    else
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenLess);
                        UngetCharacter();
                    }
                    break;

                case '>':
                    C = GetCharacter();
                    if (C == '=')
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenGreaterEqual);
                    }
                    else if (C == '>')
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenRightRight);
                    }
                    else
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenGreater);
                        UngetCharacter();
                    }
                    break;

                case '^':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenCircumflex);
                    break;

                case '!':
                    C = GetCharacter();
                    if (C == '=')
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenBangEqual);
                    }
                    else
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenBang);
                        UngetCharacter();
                    }
                    break;

                case '&':
                    C = GetCharacter();
                    if (C == '&')
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenAmpersandAmpersand);
                    }
                    else
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenAmpersand);
                        UngetCharacter();
                    }
                    break;

                case '|':
                    C = GetCharacter();
                    if (C == '|')
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenPipePipe);
                    }
                    else
                    {
                        Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenPipe);
                        UngetCharacter();
                    }
                    break;

                case '~':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenTilde);
                    break;

                case '$':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenDollar);
                    break;

                case '@':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenAt);
                    break;

                case '%':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenPercent);
                    break;

                case '\\':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenBackslash);
                    break;

                case '?':
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenQuestion);
                    break;

                case '.':
                {
                    int CC = GetCharacter();
                    if ((CC >= '0') && (CC <= '9'))
                    {
                        UngetCharacter();
                        goto RestartParse;         // parse number
                    }
                }
                    Token = new UnqualifiedTokenRec <T>(TokenTypes.eTokenDot);
                    break;
                }
            }

            return(Token);
        }