public static Module Analyze(Modules modules, CompilerSink sink, string name, string text) { CompilerContext context = new CompilerContext(name, sink); Parser parser = Parser.FromString(state, context, text); Statement Statement = parser.ParseFileInput(); Analyzer analyzer = new Analyzer(); return analyzer.DoAnalyze(modules, name, Statement); }
private CodeGen CompileModuleInit(CompilerContext context, GlobalSuite gs, TypeGen tg, string moduleName) { CodeGen init; if (!AutoImportAll) { init = OutputGenerator.GenerateModuleInitialize(context, gs, tg); } else { // auto-import all compiled modules, useful for CodeDom scenarios. init = OutputGenerator.GenerateModuleInitialize(context, gs, tg, staticTypes, delegate(CodeGen cg) { Location dummyLocation = new Location(1, 1); for (int i = 0; i < sourceFiles.Count; i++) { string otherModName = GetModuleFromFilename(sourceFiles[i]); if (otherModName == moduleName) continue; FromImportStatement stmt = new FromImportStatement( new DottedName(new SymbolId[] { SymbolTable.StringToId(otherModName) }), FromImportStatement.Star, null); stmt.Start = dummyLocation; stmt.End = dummyLocation; stmt.Emit(cg); } // Import the first part of all namespaces in all referenced assemblies // First, determine the set of unique such prefixes Dictionary<string, object> nsPrefixes = new Dictionary<string, object>(); foreach (string name in ReferencedAssemblies) { Assembly a = LoadAssembly(name); foreach (Type t in a.GetTypes()) { // We only care about public types if (!t.IsPublic) continue; // Ignore types that don't have a namespace if (t.Namespace == null) continue; string nsPrefix = t.Namespace.Split('.')[0]; nsPrefixes[nsPrefix] = null; } } // Import all the uniquer prefixes we found foreach (string nsPrefix in nsPrefixes.Keys) { SymbolId symbolId = SymbolTable.StringToId(nsPrefix); cg.Names.CreateGlobalSlot(symbolId); DottedName dottedName = new DottedName(new SymbolId[] { symbolId }); ImportStatement importStmt = new ImportStatement( new DottedName[] { dottedName }, new SymbolId[] { SymbolTable.Empty }); importStmt.Start = dummyLocation; importStmt.End = dummyLocation; importStmt.Emit(cg); } }); } return init; }
CompilerContext ICallerContext.CreateCompilerContext() { CompilerContext context = new CompilerContext(); context.TrueDivision = ((ICallerContext)this).TrueDivision; return context; }
private Binder(CompilerContext context) { this.define = new DefineBinder(this); this.delete = new DeleteBinder(this); this.context = context; }
public static GlobalSuite Bind(Stmt root, CompilerContext context) { Binder binder = new Binder(context); return(binder.DoBind(root)); }
public static Parser FromFile(SystemState state, CompilerContext context) { return FromFile(state, context, false, false); }
public static ScopeNode GetScopesFromText(string text) { CompilerContext context = new CompilerContext("<input>", new QuietCompilerSink()); Parser parser = Parser.FromString(state, context, text); Statement Statement = parser.ParseFileInput(); ScopeWalker walker = new ScopeWalker(); return walker.WalkScopes(Statement); }
public Tokenizer(SystemState state, CompilerContext context, char[] data) : this(data, false, state, context) { }
public Tokenizer(char[] data, bool verbatim, SystemState state, CompilerContext context) { this.data = data; this.length = data.Length; this.verbatim = verbatim; this.current.Line = 1; this.startLoc.Line = 1; this.endLoc.Line = 1; this.context = context; this.systemState = state; if (Options.WarningOnIndentationInconsistency || Options.ErrorOnIndentationInconsistency) { indentFormat = new StringBuilder[MAX_INDENT]; } }
private void SearchForCodeBlocks(IVsTextLines buffer) { // We don't want any change in the buffer while we are parsing, // so we have to lock it. ErrorHandler.ThrowOnFailure(buffer.LockBufferEx((uint)BufferLockFlags.BLF_READ)); try { // Find the total number of lines in the buffer. int totalLines; ErrorHandler.ThrowOnFailure(buffer.GetLineCount(out totalLines)); // Set the initial values for the variables used during the parsing. SimpleParserState state = SimpleParserState.WaitForExternalSource; TextSpanAndCookie blockSpan = new TextSpanAndCookie(); // Parse all the lines in the buffer for (int line = 0; line < totalLines; ++line) { // Get the text of the current line. int lineLen; ErrorHandler.ThrowOnFailure(buffer.GetLengthOfLine(line, out lineLen)); if (0 == lineLen) { // The line is empty, no point in parsing it. continue; } string lineText; ErrorHandler.ThrowOnFailure(buffer.GetLineText(line, 0, line, lineLen, out lineText)); // Create the tokenizer. CompilerContext context = new CompilerContext("", new QuietCompilerSink()); using (SystemState systemState = new SystemState()) { tokenizer = new Tokenizer(lineText.ToCharArray(), true, systemState, context); Token token = null; string commentText; // Read all the token looking for the code blocks inside a Snippet Statements // nested in an External Source statement. Note that the standard IronPython // parser does not return such statements and this is the reason for this // parser. while (!tokenizer.IsEndOfFile) { token = tokenizer.Next(); // This parser is strange in that it is only interested in comments: // an external code statement is in the form // #ExternalSource("PathOfTheOriginalFile", originalLineNumber) // ... (some code) ... // #End ExternalSource // and a snippet statement is // # Snippet Statement // ... (some code) ... // #End Snippet Statement // So if we want to find the text region inside a snippet nested // inside an external source, we are only interested in the comment tokens. if (TokenKind.Comment != token.Kind) { continue; } // The comments are line comments, so the comment's text is everything that // is after the beginning of the comment. commentText = CleanCommentStart(lineText.Substring(tokenizer.StartLocation.Column)); if (string.IsNullOrEmpty(commentText)) { continue; } switch (state) { case SimpleParserState.WaitForExternalSource: // This function returns a non zero value only if the comment text // is a valid external source statment. blockSpan.ulHTMLCookie = ParseExternalSource(commentText); if (0 != blockSpan.ulHTMLCookie) { // The CodeDOM provider is adding 1 to the line number, but in this // case the number is actualy the HTML editor's cookie, so we have to // restore the original value. blockSpan.ulHTMLCookie -= 1; state = SimpleParserState.WaitForSnippet; } break; case SimpleParserState.WaitForSnippet: // Check if this comment is the beginning of a snippet block. if (IsBeginSnippet(commentText)) { // This is the beginning of a snippet block, so // the actual code will start at the beginning of the // next line. blockSpan.CodeSpan.iStartLine = line + 1; // Set the default for the start index. blockSpan.CodeSpan.iStartIndex = 0; // Now we have to find the end of the snippet section // to complete the span of the code. state = SimpleParserState.WaitForEndSnippet; } else if (IsEndExternalSource(commentText)) { // This was and external block not related to the HTML editor. // Reset the text span and wait for the next external source. blockSpan = new TextSpanAndCookie(); state = SimpleParserState.WaitForExternalSource; } break; case SimpleParserState.WaitForEndSnippet: if (IsEndSnippet(commentText)) { // The code block ends at the end of the line before // this token. // Update the data about the code span and add the // block to the list of the blocks found. blockSpan.CodeSpan.iEndLine = line - 1; ErrorHandler.ThrowOnFailure( buffer.GetLengthOfLine(line - 1, out blockSpan.CodeSpan.iEndIndex)); blocks.Add(blockSpan); blockSpan = new TextSpanAndCookie(); state = SimpleParserState.WaitForEndExternal; } break; case SimpleParserState.WaitForEndExternal: // We expect only one snippet block inside the external source // section, so here we skip everything between the end of the first // snippet block and the end of the external code section. if (IsEndExternalSource(commentText)) { state = SimpleParserState.WaitForExternalSource; } break; } } } } } finally { // Make sure that the buffer is always unlocked when we exit this function. buffer.UnlockBufferEx((uint)BufferLockFlags.BLF_READ); } }
public CompilerContext CopyWithNewSourceFile(string newSourceFile) { CompilerContext ret = new CompilerContext(newSourceFile, sink); ret.trueDivision = this.trueDivision; return ret; }
private Parser(CompilerContext context, Tokenizer tokenizer) { this.tokenizer = tokenizer; this.context = context; }
public static Parser FromString(SystemState state, CompilerContext context, string text) { return new Parser(context, new Tokenizer(state, context, text.ToCharArray())); }
public static Parser FromFile(SystemState state, CompilerContext context, bool skipLine, bool verbatim) { string data; string fileName = context.SourceFile; // we choose ASCII by default, if the file has a Unicode header though // we'll automatically get it as unicode. Encoding encType = System.Text.Encoding.ASCII; if (fileName == "<stdin>") { Stream s = Console.OpenStandardInput(); using (StreamReader sr = new StreamReader(s, encType)) { if (skipLine) { sr.ReadLine(); } data = sr.ReadToEnd(); } return new Parser(context, new Tokenizer(data.ToCharArray(), verbatim, state, context)); } byte[] bytes = File.ReadAllBytes(fileName); using (StreamReader sr = new StreamReader(new MemoryStream(bytes, false), System.Text.Encoding.ASCII)) { string line = sr.ReadLine(); bool gotEncoding = false; // magic encoding must be on line 1 or 2 if (line != null && !(gotEncoding = TryGetEncoding(state, line, ref encType))) { line = sr.ReadLine(); if (line != null) { gotEncoding = TryGetEncoding(state, line, ref encType); } } if (gotEncoding && sr.CurrentEncoding != Encoding.ASCII && encType != sr.CurrentEncoding) { // we have both a BOM & an encoding type, throw an error context.Sink.AddError(fileName, "file has both Unicode marker and PEP-263 file encoding", String.Empty, CodeSpan.Empty, 0, Severity.Error); } } if (encType == null) context.AddError("unknown encoding type", "<unknown>", 0, 0, 0, 0, Severity.Error); // re-read w/ the correct encoding type... using (StreamReader sr = new StreamReader(new MemoryStream(bytes), encType)) { if (skipLine) { sr.ReadLine(); } data = sr.ReadToEnd(); } return new Parser(context, new Tokenizer(data.ToCharArray(), verbatim, state, context)); }
private void CompilePythonModule(string fileName, PythonCompilerSink sink, bool createMain) { assemblyGen.SetPythonSourceFile(fileName); CompilerContext context = new CompilerContext(fileName, sink); Parser p = Parser.FromFile(state, context); Statement body = p.ParseFileInput(); if (sink.Errors > 0) return; GlobalSuite gs = Compiler.Ast.Binder.Bind(body, context); string moduleName = GetModuleFromFilename(fileName); TypeGen tg = OutputGenerator.GenerateModuleType(moduleName, assemblyGen); CodeGen init = CompileModuleInit(context, gs, tg, moduleName); if (createMain) { CodeGen main = OutputGenerator.GenerateModuleEntryPoint(tg, init, moduleName, referencedAssemblies); assemblyGen.SetEntryPoint(main.MethodInfo, targetKind); } assemblyGen.AddPythonModuleAttribute(tg, moduleName); tg.FinishType(); }
/// <summary> /// Creates a fake PythonAst object which is represenative of the on-disk script code. /// </summary> private static Ast.PythonAst MakeAstFromSourceUnit(SourceUnit sourceUnit) { var compCtx = new CompilerContext(sourceUnit, new PythonCompilerOptions(), ErrorSink.Null); return(new Ast.PythonAst(compCtx)); }
private static PythonModule LoadFromSource(SystemState state, string fullName, string fileName) { CompilerContext context = new CompilerContext(fileName); Parser parser = Parser.FromFile(state, context); Statement s = parser.ParseFileInput(); return OutputGenerator.GenerateModule(state, context, s, fullName); }
private static PythonModule GenerateAndInitializeModule(ICallerContext context, CompilerContext cc, Parser parser, string name, string filename) { Statement stmt = parser.ParseFileInput(); PythonModule module = OutputGenerator.GenerateModule(context.SystemState, cc, stmt, name); module.Filename = filename; module.ModuleName = name; return Importer.InitializeModule(filename, module); }