public static Module Analyze(Modules modules, CompilerSink sink, string name, string text)
{
CompilerContext context = new CompilerContext(name, sink);
Parser parser = Parser.FromString(state, context, text);
Statement Statement = parser.ParseFileInput();
Analyzer analyzer = new Analyzer();
return analyzer.DoAnalyze(modules, name, Statement);
}
private CodeGen CompileModuleInit(CompilerContext context, GlobalSuite gs, TypeGen tg, string moduleName)
{
CodeGen init;
if (!AutoImportAll) {
init = OutputGenerator.GenerateModuleInitialize(context, gs, tg);
} else {
// auto-import all compiled modules, useful for CodeDom scenarios.
init = OutputGenerator.GenerateModuleInitialize(context, gs, tg, staticTypes, delegate(CodeGen cg) {
Location dummyLocation = new Location(1, 1);
for (int i = 0; i < sourceFiles.Count; i++) {
string otherModName = GetModuleFromFilename(sourceFiles[i]);
if (otherModName == moduleName) continue;
FromImportStatement stmt = new FromImportStatement(
new DottedName(new SymbolId[] { SymbolTable.StringToId(otherModName) }),
FromImportStatement.Star, null);
stmt.Start = dummyLocation;
stmt.End = dummyLocation;
stmt.Emit(cg);
}
// Import the first part of all namespaces in all referenced assemblies
// First, determine the set of unique such prefixes
Dictionary<string, object> nsPrefixes = new Dictionary<string, object>();
foreach (string name in ReferencedAssemblies) {
Assembly a = LoadAssembly(name);
foreach (Type t in a.GetTypes()) {
// We only care about public types
if (!t.IsPublic) continue;
// Ignore types that don't have a namespace
if (t.Namespace == null) continue;
string nsPrefix = t.Namespace.Split('.')[0];
nsPrefixes[nsPrefix] = null;
}
}
// Import all the uniquer prefixes we found
foreach (string nsPrefix in nsPrefixes.Keys) {
SymbolId symbolId = SymbolTable.StringToId(nsPrefix);
cg.Names.CreateGlobalSlot(symbolId);
DottedName dottedName = new DottedName(new SymbolId[] { symbolId });
ImportStatement importStmt = new ImportStatement(
new DottedName[] { dottedName },
new SymbolId[] { SymbolTable.Empty });
importStmt.Start = dummyLocation;
importStmt.End = dummyLocation;
importStmt.Emit(cg);
}
});
}
return init;
}
CompilerContext ICallerContext.CreateCompilerContext()
{
CompilerContext context = new CompilerContext();
context.TrueDivision = ((ICallerContext)this).TrueDivision;
return context;
}
private Binder(CompilerContext context)
{
this.define = new DefineBinder(this);
this.delete = new DeleteBinder(this);
this.context = context;
}
public static GlobalSuite Bind(Stmt root, CompilerContext context)
{
Binder binder = new Binder(context);
return(binder.DoBind(root));
}
public static Parser FromFile(SystemState state, CompilerContext context)
{
return FromFile(state, context, false, false);
}
public static ScopeNode GetScopesFromText(string text)
{
CompilerContext context = new CompilerContext("<input>", new QuietCompilerSink());
Parser parser = Parser.FromString(state, context, text);
Statement Statement = parser.ParseFileInput();
ScopeWalker walker = new ScopeWalker();
return walker.WalkScopes(Statement);
}
public Tokenizer(SystemState state, CompilerContext context, char[] data)
: this(data, false, state, context)
{
}
public Tokenizer(char[] data, bool verbatim, SystemState state, CompilerContext context)
{
this.data = data;
this.length = data.Length;
this.verbatim = verbatim;
this.current.Line = 1;
this.startLoc.Line = 1;
this.endLoc.Line = 1;
this.context = context;
this.systemState = state;
if (Options.WarningOnIndentationInconsistency || Options.ErrorOnIndentationInconsistency) {
indentFormat = new StringBuilder[MAX_INDENT];
}
}
private void SearchForCodeBlocks(IVsTextLines buffer)
{
// We don't want any change in the buffer while we are parsing,
// so we have to lock it.
ErrorHandler.ThrowOnFailure(buffer.LockBufferEx((uint)BufferLockFlags.BLF_READ));
try {
// Find the total number of lines in the buffer.
int totalLines;
ErrorHandler.ThrowOnFailure(buffer.GetLineCount(out totalLines));
// Set the initial values for the variables used during the parsing.
SimpleParserState state = SimpleParserState.WaitForExternalSource;
TextSpanAndCookie blockSpan = new TextSpanAndCookie();
// Parse all the lines in the buffer
for (int line = 0; line < totalLines; ++line) {
// Get the text of the current line.
int lineLen;
ErrorHandler.ThrowOnFailure(buffer.GetLengthOfLine(line, out lineLen));
if (0 == lineLen) {
// The line is empty, no point in parsing it.
continue;
}
string lineText;
ErrorHandler.ThrowOnFailure(buffer.GetLineText(line, 0, line, lineLen, out lineText));
// Create the tokenizer.
CompilerContext context = new CompilerContext("", new QuietCompilerSink());
using (SystemState systemState = new SystemState()) {
tokenizer = new Tokenizer(lineText.ToCharArray(), true, systemState, context);
Token token = null;
string commentText;
// Read all the token looking for the code blocks inside a Snippet Statements
// nested in an External Source statement. Note that the standard IronPython
// parser does not return such statements and this is the reason for this
// parser.
while (!tokenizer.IsEndOfFile) {
token = tokenizer.Next();
// This parser is strange in that it is only interested in comments:
// an external code statement is in the form
// #ExternalSource("PathOfTheOriginalFile", originalLineNumber)
// ... (some code) ...
// #End ExternalSource
// and a snippet statement is
// # Snippet Statement
// ... (some code) ...
// #End Snippet Statement
// So if we want to find the text region inside a snippet nested
// inside an external source, we are only interested in the comment tokens.
if (TokenKind.Comment != token.Kind) {
continue;
}
// The comments are line comments, so the comment's text is everything that
// is after the beginning of the comment.
commentText = CleanCommentStart(lineText.Substring(tokenizer.StartLocation.Column));
if (string.IsNullOrEmpty(commentText)) {
continue;
}
switch (state) {
case SimpleParserState.WaitForExternalSource:
// This function returns a non zero value only if the comment text
// is a valid external source statment.
blockSpan.ulHTMLCookie = ParseExternalSource(commentText);
if (0 != blockSpan.ulHTMLCookie) {
// The CodeDOM provider is adding 1 to the line number, but in this
// case the number is actualy the HTML editor's cookie, so we have to
// restore the original value.
blockSpan.ulHTMLCookie -= 1;
state = SimpleParserState.WaitForSnippet;
}
break;
case SimpleParserState.WaitForSnippet:
// Check if this comment is the beginning of a snippet block.
if (IsBeginSnippet(commentText)) {
// This is the beginning of a snippet block, so
// the actual code will start at the beginning of the
// next line.
blockSpan.CodeSpan.iStartLine = line + 1;
// Set the default for the start index.
blockSpan.CodeSpan.iStartIndex = 0;
// Now we have to find the end of the snippet section
// to complete the span of the code.
state = SimpleParserState.WaitForEndSnippet;
} else if (IsEndExternalSource(commentText)) {
// This was and external block not related to the HTML editor.
// Reset the text span and wait for the next external source.
blockSpan = new TextSpanAndCookie();
state = SimpleParserState.WaitForExternalSource;
}
break;
case SimpleParserState.WaitForEndSnippet:
if (IsEndSnippet(commentText)) {
// The code block ends at the end of the line before
// this token.
// Update the data about the code span and add the
// block to the list of the blocks found.
blockSpan.CodeSpan.iEndLine = line - 1;
ErrorHandler.ThrowOnFailure(
buffer.GetLengthOfLine(line - 1, out blockSpan.CodeSpan.iEndIndex));
blocks.Add(blockSpan);
blockSpan = new TextSpanAndCookie();
state = SimpleParserState.WaitForEndExternal;
}
break;
case SimpleParserState.WaitForEndExternal:
// We expect only one snippet block inside the external source
// section, so here we skip everything between the end of the first
// snippet block and the end of the external code section.
if (IsEndExternalSource(commentText)) {
state = SimpleParserState.WaitForExternalSource;
}
break;
}
}
}
}
} finally {
// Make sure that the buffer is always unlocked when we exit this function.
buffer.UnlockBufferEx((uint)BufferLockFlags.BLF_READ);
}
}
public CompilerContext CopyWithNewSourceFile(string newSourceFile)
{
CompilerContext ret = new CompilerContext(newSourceFile, sink);
ret.trueDivision = this.trueDivision;
return ret;
}
private Parser(CompilerContext context, Tokenizer tokenizer)
{
this.tokenizer = tokenizer;
this.context = context;
}
public static Parser FromString(SystemState state, CompilerContext context, string text)
{
return new Parser(context, new Tokenizer(state, context, text.ToCharArray()));
}
public static Parser FromFile(SystemState state, CompilerContext context, bool skipLine, bool verbatim)
{
string data;
string fileName = context.SourceFile;
// we choose ASCII by default, if the file has a Unicode header though
// we'll automatically get it as unicode.
Encoding encType = System.Text.Encoding.ASCII;
if (fileName == "<stdin>") {
Stream s = Console.OpenStandardInput();
using (StreamReader sr = new StreamReader(s, encType)) {
if (skipLine) {
sr.ReadLine();
}
data = sr.ReadToEnd();
}
return new Parser(context, new Tokenizer(data.ToCharArray(), verbatim, state, context));
}
byte[] bytes = File.ReadAllBytes(fileName);
using (StreamReader sr = new StreamReader(new MemoryStream(bytes, false), System.Text.Encoding.ASCII)) {
string line = sr.ReadLine();
bool gotEncoding = false;
// magic encoding must be on line 1 or 2
if (line != null && !(gotEncoding = TryGetEncoding(state, line, ref encType))) {
line = sr.ReadLine();
if (line != null) {
gotEncoding = TryGetEncoding(state, line, ref encType);
}
}
if (gotEncoding &&
sr.CurrentEncoding != Encoding.ASCII &&
encType != sr.CurrentEncoding) {
// we have both a BOM & an encoding type, throw an error
context.Sink.AddError(fileName, "file has both Unicode marker and PEP-263 file encoding", String.Empty, CodeSpan.Empty, 0, Severity.Error);
}
}
if (encType == null) context.AddError("unknown encoding type", "<unknown>", 0, 0, 0, 0, Severity.Error);
// re-read w/ the correct encoding type...
using (StreamReader sr = new StreamReader(new MemoryStream(bytes), encType)) {
if (skipLine) {
sr.ReadLine();
}
data = sr.ReadToEnd();
}
return new Parser(context, new Tokenizer(data.ToCharArray(), verbatim, state, context));
}
private void CompilePythonModule(string fileName, PythonCompilerSink sink, bool createMain)
{
assemblyGen.SetPythonSourceFile(fileName);
CompilerContext context = new CompilerContext(fileName, sink);
Parser p = Parser.FromFile(state, context);
Statement body = p.ParseFileInput();
if (sink.Errors > 0) return;
GlobalSuite gs = Compiler.Ast.Binder.Bind(body, context);
string moduleName = GetModuleFromFilename(fileName);
TypeGen tg = OutputGenerator.GenerateModuleType(moduleName, assemblyGen);
CodeGen init = CompileModuleInit(context, gs, tg, moduleName);
if (createMain) {
CodeGen main = OutputGenerator.GenerateModuleEntryPoint(tg, init, moduleName, referencedAssemblies);
assemblyGen.SetEntryPoint(main.MethodInfo, targetKind);
}
assemblyGen.AddPythonModuleAttribute(tg, moduleName);
tg.FinishType();
}
/// <summary>
/// Creates a fake PythonAst object which is represenative of the on-disk script code.
/// </summary>
private static Ast.PythonAst MakeAstFromSourceUnit(SourceUnit sourceUnit)
{
var compCtx = new CompilerContext(sourceUnit, new PythonCompilerOptions(), ErrorSink.Null);
return(new Ast.PythonAst(compCtx));
}
private static PythonModule LoadFromSource(SystemState state, string fullName, string fileName)
{
CompilerContext context = new CompilerContext(fileName);
Parser parser = Parser.FromFile(state, context);
Statement s = parser.ParseFileInput();
return OutputGenerator.GenerateModule(state, context, s, fullName);
}
private static PythonModule GenerateAndInitializeModule(ICallerContext context, CompilerContext cc, Parser parser, string name, string filename)
{
Statement stmt = parser.ParseFileInput();
PythonModule module = OutputGenerator.GenerateModule(context.SystemState, cc, stmt, name);
module.Filename = filename;
module.ModuleName = name;
return Importer.InitializeModule(filename, module);
}
