/// <summary> /// Processes the next token in the token list. /// </summary> private void ProcessToken() { // Process the next token. #region Token Processing if (_currentToken == null) { NextToken(); return; } switch (_currentToken.ID) { case TokenID.CharDirective: switch (NextToken().ID) { case TokenID.KeywordIf: ParseIf(); break; case TokenID.TypeIdentifier: switch (_currentToken.Ident) { case "define": NextToken(); // Get the identifier. string ident = _currentToken.Ident; NextToken(); // Get the value. string value = _currentToken.Ident; _defineList.Add(new Define(ident, value, _currentToken.ID)); break; case "undefine": NextToken(); // Get the identifier. foreach (Define def in _defineList) { if (def.Ident.ToLower() == _currentToken.Ident.ToLower()) { _defineList.Remove(def); break; } } break; case "warning": Warning(ErrorCode.DirectiveError, NextToken().Ident); break; case "error": Error(ErrorCode.DirectiveError, NextToken().Ident); break; case "message": Message(ErrorCode.DirectiveError, NextToken().Ident); break; case "include": ExpectToken(TokenID.TypeString); string filePath = _currentToken.Ident; if (File.Exists(filePath) == false) { // See if its in one of the include paths. if (_includePaths != null) { foreach (object obj in _includePaths) { string path = obj as string; string newFilePath = path + "\\" + filePath; if (File.Exists(newFilePath) == true) { filePath = newFilePath; break; } } } // Nope? Check the tokens file name and see if its relative to that. if (_currentToken.File != "" && File.Exists(filePath) == false) { string includePath = Path.GetDirectoryName(_currentToken.File); string newFilePath = includePath + "\\" + filePath; if (File.Exists(newFilePath) == true) { filePath = newFilePath; } } } // Already included? if (_includedFiles.Contains(filePath)) { Message(ErrorCode.DuplicateSymbol, "Ignoring include file \"" + _currentToken.Ident + "\", file has already been included."); break; } if (File.Exists(filePath) == true) { // Open a stream so we can read in the script. Stream stream = StreamFactory.RequestStream(filePath, StreamMode.Open); if (stream == null) { return; } StreamReader reader = new StreamReader(stream); // Load in script and setup variables. string scriptText = reader.ReadToEnd(); // Create a lexer and convert script into a list // of tokens. Lexer lexer = new Lexer(); if (lexer.Analyse(scriptText, _compileFlags, filePath) > 0) { foreach (CompileError error in lexer.ErrorList) { _errorList.Add(error); } } // Insert the newly lexed tokens into this token list. _tokenList.InsertRange(_tokenIndex, lexer.TokenList); // Free up the stream. stream.Close(); reader.Close(); _includedFiles.Add(filePath); //DebugLogger.WriteLog("Included \"" + filePath + "\" into script."); } else { Warning(ErrorCode.NonExistantIncludeFile, "Unable to process include file \"" + _currentToken.Ident + "\", file dosen't exist."); } break; } break; default: Error(ErrorCode.InvalidDirective, "Found invalid or unexpected directive while pre processing script."); break; } break; default: _newTokenList.Add(LookupDefineToken(_currentToken)); break; } #endregion }
/// <summary> /// Compiles a script from a string into byte code. /// </summary> /// <param name="data">Data of script to compile.</param> /// <param name="flags">Bitmask of flags defining how the script should be compiled.</param> /// <param name="defineList">A list of defines to use while preprocessing script.</param> /// <param name="includePaths">A list of directory paths to use when looking for include files.</param> /// <param name="fileUrl">Contains the url of the file this data comes from.</param> /// <returns>The number of errors or warning this script has generated duren compilation.</returns> public int CompileString(string data, CompileFlags flags, Define[] defineList, object[] includePaths, string fileUrl) { #if !DEBUG try { #endif // Reset all variables used to store compilation details. _errorList.Clear(); _compiledSymbolList.Clear(); _compiledInstructionList.Clear(); _compiledDebugFileList.Clear(); _compiledDefineList.Clear(); _loopTrackerStack.Clear(); _metaDataList.Clear(); _currentToken = null; _currentPass = 0; _currentScope = null; _globalScope = new FunctionSymbol("$global", null); // Initialize global scope here. _memberScope = new FunctionSymbol("$member", null); // Initialize member scope here. _compileFlags = flags; _tokenList = null; _tokenIndex = 0; _memorySize = 1; // Reserve space for 'special' globals like 'this'. _internalVariableIndex = 0; _defaultEngineState = _defaultEditorState = null; _errorsOccured = false; _overrideInstructionScope = null; // Create the 'this' variable. VariableSymbol thisSymbol = new VariableSymbol(_globalScope); thisSymbol.DataType = new DataTypeValue(DataType.Object, false, false); thisSymbol.Identifier = "this"; thisSymbol.IsConstant = true; thisSymbol.IsUsed = true; thisSymbol.MemoryIndex = 0; thisSymbol.VariableType = VariableType.Constant; // Create a lexer and convert script into a list // of tokens. DebugLogger.WriteLog("Preforming lexical analysis on script."); Lexer lexer = new Lexer(); if (lexer.Analyse(data, _compileFlags, fileUrl) > 0) { foreach (CompileError error in lexer.ErrorList) _errorList.Add(error); } _tokenList = lexer.TokenList; // Add the script directory into the include path array. string includePath = Path.GetDirectoryName(fileUrl); string[] newIncludePaths = new string[includePaths.Length + 1]; includePaths.CopyTo(newIncludePaths, 0); newIncludePaths[newIncludePaths.Length - 1] = includePath; includePaths = newIncludePaths; // Create a pre-processor and process the token list DebugLogger.WriteLog("Preforming preprocessing on script."); PreProcessor preProcessor = new PreProcessor(); if (preProcessor.Process(_tokenList, _compileFlags, defineList, includePaths) > 0) { foreach (CompileError error in preProcessor.ErrorList) _errorList.Add(error); } _compiledDefineList = preProcessor.DefineList; _tokenList = preProcessor.TokenList; try { // Pass 0: Collect infomation. // Pass 1: Generate byte code. DebugLogger.WriteLog("Compiling script in 2 passes."); // Go over the source code in 2 passes. for (int pass = 0; pass < 2; pass++) { _currentPass = pass; _currentToken = null; _tokenIndex = 0; _currentScope = _globalScope; _internalVariableIndex = 0; // This needs to be reset so statements using // an internal variables can find them on the second pass. while (!EndOfTokenStream()) { try { ParseStatement(); } catch (CompilePanicModeException) { DebugLogger.WriteLog("Panic mode initialized in script."); // Don't do anything here, just allow the error to be // forgoten and carry on as normal. } // If there are any errors quit compilation now. if (_errorsOccured == true) throw new CompileBreakException(); } } // Yell at user if no default state has been declared. if (_defaultEngineState == null && (_compileFlags & CompileFlags.Library) == 0) Error(ErrorCode.MissingDefaultState, "Default engine state is missing."); } catch (CompileBreakException) { DebugLogger.WriteLog("Script compilation broken via CompileBreakException."); } // Check what errors occured. int fatalErrorCount = 0; int errorCount = 0; int warningCount = 0; int messageCount = 0; foreach (CompileError error in _errorList) { switch (error.AlertLevel) { case ErrorAlertLevel.Error: errorCount++; break; case ErrorAlertLevel.FatalError: fatalErrorCount++; break; case ErrorAlertLevel.Message: messageCount++; break; case ErrorAlertLevel.Warning: warningCount++; break; } DebugLogger.WriteLog(error.ToString(), LogAlertLevel.Warning); } DebugLogger.WriteLog("Script compiled with "+fatalErrorCount+" fatal errors, "+errorCount+" errors, "+warningCount+" warnings and "+messageCount+" messages."); // If there are any errors quit compilation now. if (_errorsOccured == true) return _errorList.Count; // Append an exit symbol onto the global scopes instruction list. CreateInstruction(OpCode.EXIT, _globalScope, _currentToken); DebugLogger.WriteLog("Optimizing and tweeking symbols and instructions."); // Compile symbol list. CompileSymbol(_globalScope); CompileSymbol(_memberScope); // Go through instruction list and replace placeholders with their values. int symbolIndex = 0; foreach (Symbol symbol in _compiledSymbolList) { symbol.Index = symbolIndex; symbolIndex++; // Update the entry point if this is a function or event. switch (symbol.Type) { case SymbolType.Function: ((FunctionSymbol)symbol).EntryPoint = _compiledInstructionList.Count; break; case SymbolType.Variable: bool check = true; if (symbol.Scope.Type == SymbolType.Function) if (((FunctionSymbol)symbol.Scope).IsImport == true) check = false; // Should we remove it rather than warning? if (((VariableSymbol)symbol).IsUsed == false && ((VariableSymbol)symbol).IsConstant == false && check == true) Warning(ErrorCode.UnusedVariable, "Variable \"" + symbol.Identifier + "\" is declared but never used."); break; } foreach (Instruction instruction in symbol.Instructions) { // Create a new debug file entry if this instruction // was generated in a previously unknown file. if (instruction.File != null && instruction.File != "") { bool found = false; foreach (string file in _compiledDebugFileList) if (file == instruction.File) found = true; if (found == false) _compiledDebugFileList.Add(instruction.File); } // Go through each operand attach to this instruction // and check for any trackers. foreach (Operand operand in instruction.Operands) { if (operand == null) continue; // Update operand based on type. switch (operand.OpType) { case OperandType.JumpTarget: operand.OpType = OperandType.InstrIndex; switch (symbol.Type) { case SymbolType.Function: operand.InstrIndex = ((FunctionSymbol)symbol).EntryPoint + operand.JumpTarget.InstrIndex; break; } break; case OperandType.SymbolIndexTracker: operand.OpType = OperandType.SymbolIndex; operand.SymbolIndex = _compiledSymbolList.IndexOf(operand.SymbolIndexTracker); break; } } _compiledInstructionList.Add(instruction); } } // Optimize this symbols instructions. (Currently somewhat error prone) //int instructionCount = _compiledInstructionList.Count; //ScriptOptimizer optimizer = new ScriptOptimizer(); //optimizer.Optimize(_compiledInstructionList, _compiledSymbolList); //_compiledInstructionList = optimizer.OptimizedInstructions; //_compiledSymbolList = optimizer.OptimizedSymbols; // int index = 0; //foreach (Instruction instr in _compiledInstructionList) //{ // System.Console.WriteLine("\t"+instr.Decompile()); // index++; //} #if !DEBUG } catch (Exception) { _errorList.Add(new CompileError(ErrorCode.InternalError, "An internal compiler error occured.", ErrorAlertLevel.FatalError, 0, 0, "")); } #endif return _errorList.Count; }
/// <summary> /// Processes the next token in the token list. /// </summary> private void ProcessToken() { // Process the next token. #region Token Processing if (_currentToken == null) { NextToken(); return; } switch (_currentToken.ID) { case TokenID.CharDirective: switch (NextToken().ID) { case TokenID.KeywordIf: ParseIf(); break; case TokenID.TypeIdentifier: switch (_currentToken.Ident) { case "define": NextToken(); // Get the identifier. string ident = _currentToken.Ident; NextToken(); // Get the value. string value = _currentToken.Ident; _defineList.Add(new Define(ident, value, _currentToken.ID)); break; case "undefine": NextToken(); // Get the identifier. foreach(Define def in _defineList) if (def.Ident.ToLower() == _currentToken.Ident.ToLower()) { _defineList.Remove(def); break; } break; case "warning": Warning(ErrorCode.DirectiveError,NextToken().Ident); break; case "error": Error(ErrorCode.DirectiveError, NextToken().Ident); break; case "message": Message(ErrorCode.DirectiveError, NextToken().Ident); break; case "include": ExpectToken(TokenID.TypeString); string filePath = _currentToken.Ident; if (File.Exists(filePath) == false) { // See if its in one of the include paths. if (_includePaths != null) { foreach (object obj in _includePaths) { string path = obj as string; string newFilePath = path + "\\" + filePath; if (File.Exists(newFilePath) == true) { filePath = newFilePath; break; } } } // Nope? Check the tokens file name and see if its relative to that. if (_currentToken.File != "" && File.Exists(filePath) == false) { string includePath = Path.GetDirectoryName(_currentToken.File); string newFilePath = includePath + "\\" + filePath; if (File.Exists(newFilePath) == true) filePath = newFilePath; } } // Already included? if (_includedFiles.Contains(filePath)) { Message(ErrorCode.DuplicateSymbol, "Ignoring include file \"" + _currentToken.Ident + "\", file has already been included."); break; } if (File.Exists(filePath) == true) { // Open a stream so we can read in the script. Stream stream = StreamFactory.RequestStream(filePath, StreamMode.Open); if (stream == null) return; StreamReader reader = new StreamReader(stream); // Load in script and setup variables. string scriptText = reader.ReadToEnd(); // Create a lexer and convert script into a list // of tokens. Lexer lexer = new Lexer(); if (lexer.Analyse(scriptText, _compileFlags, filePath) > 0) { foreach (CompileError error in lexer.ErrorList) _errorList.Add(error); } // Insert the newly lexed tokens into this token list. _tokenList.InsertRange(_tokenIndex, lexer.TokenList); // Free up the stream. stream.Close(); reader.Close(); _includedFiles.Add(filePath); //DebugLogger.WriteLog("Included \"" + filePath + "\" into script."); } else { Warning(ErrorCode.NonExistantIncludeFile,"Unable to process include file \""+_currentToken.Ident+"\", file dosen't exist."); } break; } break; default: Error(ErrorCode.InvalidDirective, "Found invalid or unexpected directive while pre processing script."); break; } break; default: _newTokenList.Add(LookupDefineToken(_currentToken)); break; } #endregion }