public static JsAnalyzer Analyze(string directory, AnalysisLimits limits = null, Action parseCallback = null)
{
List <AnalysisFile> files = new List <AnalysisFile>();
foreach (var file in Directory.GetFiles(directory, "*", SearchOption.AllDirectories))
{
if (String.Equals(Path.GetExtension(file), ".js", StringComparison.OrdinalIgnoreCase))
{
var relativeFile = file.Substring(directory.Length);
if (!limits.IsPathExceedNestingLimit(relativeFile))
{
files.Add(new AnalysisFile(file, File.ReadAllText(file)));
}
}
else if (String.Equals(Path.GetFileName(file), "package.json", StringComparison.OrdinalIgnoreCase))
{
JavaScriptSerializer serializer = new JavaScriptSerializer();
Dictionary <string, object> json;
try {
json = serializer.Deserialize <Dictionary <string, object> >(File.ReadAllText(file));
} catch {
continue;
}
object mainFile;
if (json != null && json.TryGetValue("main", out mainFile) && mainFile is string)
{
files.Add(AnalysisFile.PackageJson(file, (string)mainFile));
}
}
}
return(Analyze(limits, parseCallback, files.ToArray()));
}
/// <summary>
/// Loads a new instance from the specified registry key.
/// </summary>
/// <param name="key">
/// The key to load settings from. Each setting is a DWORD value. If
/// null, all settings are assumed to be unspecified and the default
/// values are used.
/// </param>
/// <param name="defaultToStdLib">
/// If True, unspecified settings are taken from the defaults for
/// standard library analysis. Otherwise, they are taken from the
/// usual defaults.
/// </param>
public static AnalysisLimits LoadFromStorage(RegistryKey key, bool defaultToStdLib = false)
{
var limits = defaultToStdLib ? AnalysisLimits.GetStandardLibraryLimits() : new AnalysisLimits();
if (key != null)
{
limits.CrossModule = (key.GetValue(CrossModuleId) as int?) ?? limits.CrossModule;
limits.CallDepth = (key.GetValue(CallDepthId) as int?) ?? limits.CallDepth;
limits.DecreaseCallDepth = (key.GetValue(DecreaseCallDepthId) as int?) ?? limits.DecreaseCallDepth;
limits.NormalArgumentTypes = (key.GetValue(NormalArgumentTypesId) as int?) ?? limits.NormalArgumentTypes;
limits.ListArgumentTypes = (key.GetValue(ListArgumentTypesId) as int?) ?? limits.ListArgumentTypes;
limits.DictArgumentTypes = (key.GetValue(DictArgumentTypesId) as int?) ?? limits.DictArgumentTypes;
limits.ReturnTypes = (key.GetValue(ReturnTypesId) as int?) ?? limits.ReturnTypes;
limits.YieldTypes = (key.GetValue(YieldTypesId) as int?) ?? limits.YieldTypes;
limits.InstanceMembers = (key.GetValue(InstanceMembersId) as int?) ?? limits.InstanceMembers;
limits.DictKeyTypes = (key.GetValue(DictKeyTypesId) as int?) ?? limits.DictKeyTypes;
limits.DictValueTypes = (key.GetValue(DictValueTypesId) as int?) ?? limits.DictValueTypes;
limits.IndexTypes = (key.GetValue(IndexTypesId) as int?) ?? limits.IndexTypes;
limits.AssignedTypes = (key.GetValue(AssignedTypesId) as int?) ?? limits.AssignedTypes;
limits.UnifyCallsToNew = ((key.GetValue(UnifyCallsToNewId) as int?) ?? (limits.UnifyCallsToNew ? 1 : 0)) != 0;
limits.ProcessCustomDecorators = ((key.GetValue(ProcessCustomDecoratorsId) as int?) ?? (limits.ProcessCustomDecorators ? 1 : 0)) != 0;
limits.UseTypeStubPackages = ((key.GetValue(UseTypeStubPackagesId) as int?) ?? (limits.UseTypeStubPackages ? 1 : 0)) != 0;
limits.UseTypeStubPackagesExclusively = ((key.GetValue(UseTypeStubPackagesExclusivelyId) as int?) ?? (limits.UseTypeStubPackagesExclusively ? 1 : 0)) != 0;
}
return(limits);
}
public static JsAnalyzer Analyze(AnalysisLimits limits, Action parseCallback, params AnalysisFile[] files)
{
Dictionary <string, IJsProjectEntry> entries = new Dictionary <string, IJsProjectEntry>();
var analyzer = new JsAnalyzer(limits);
foreach (var file in files)
{
if (Path.GetFileName(file.Filename).Equals("package.json", StringComparison.OrdinalIgnoreCase))
{
analyzer.AddPackageJson(file.Filename, file.Content);
}
else
{
var projEntry = analyzer.AddModule(file.Filename);
entries[file.Filename] = projEntry;
}
}
foreach (var file in files)
{
if (!Path.GetFileName(file.Filename).Equals("package.json", StringComparison.OrdinalIgnoreCase))
{
var source = Analysis.GetSourceUnit(file.Content);
Analysis.Prepare(
entries[file.Filename],
source
);
}
}
if (parseCallback != null)
{
parseCallback();
}
foreach (var file in files)
{
if (!Path.GetFileName(file.Filename).Equals("package.json", StringComparison.OrdinalIgnoreCase))
{
((IGroupableAnalysisProjectEntry)entries[file.Filename]).Analyze(CancellationToken.None, true);
}
}
foreach (var file in files)
{
IJsProjectEntry projEntry;
if (entries.TryGetValue(file.Filename, out projEntry))
{
projEntry.AnalysisGroup.AnalyzeQueuedEntries(CancellationToken.None);
break;
}
}
return(analyzer);
}
public TestAnalyzer(
IPythonInterpreterFactory factory,
string containerFilePath,
string codeFileBasePath,
Uri executorUri
)
{
_analyzer = PythonAnalyzer.CreateAsync(factory).WaitAndUnwrapExceptions();
_analyzer.Limits = AnalysisLimits.GetStandardLibraryLimits();
_containerFilePath = containerFilePath;
_codeFileBasePath = codeFileBasePath;
_executorUri = executorUri;
_entries = new List <IPythonProjectEntry>();
}
protected async Task <IModuleAnalysis> GetStubBasedAnalysis(
Server server,
string code,
AnalysisLimits limits,
IEnumerable <string> searchPaths,
IEnumerable <string> stubPaths)
{
if (limits != null)
{
server.Analyzer.Limits = limits;
}
server.Analyzer.SetSearchPaths(searchPaths);
server.Analyzer.SetTypeStubPaths(stubPaths);
var uri = await server.OpenDefaultDocumentAndGetUriAsync(code);
return(await server.GetAnalysisAsync(uri));
}
/// <summary>
/// Returns a new set of limits, set to the default for analyzing a
/// standard library.
/// </summary>
public static AnalysisLimits GetStandardLibraryLimits()
{
var limits = new AnalysisLimits();
limits.CrossModule = 0;
limits.CallDepth = 2;
limits.DecreaseCallDepth = 20;
limits.NormalArgumentTypes = 10;
limits.ListArgumentTypes = 5;
limits.DictArgumentTypes = 5;
limits.ReturnTypes = 10;
limits.YieldTypes = 10;
limits.InstanceMembers = 5;
limits.DictKeyTypes = 5;
limits.DictValueTypes = 10;
limits.IndexTypes = 5;
limits.AssignedTypes = 10;
limits.UnifyCallsToNew = true;
limits.ProcessCustomDecorators = true;
return(limits);
}
public static Dictionary <string, int> ToDictionary(this AnalysisLimits analysisLimits)
{
return(new Dictionary <string, int> {
{ CrossModuleId, analysisLimits.CrossModule },
{ CallDepthId, analysisLimits.CallDepth },
{ DecreaseCallDepthId, analysisLimits.DecreaseCallDepth },
{ NormalArgumentTypesId, analysisLimits.NormalArgumentTypes },
{ ListArgumentTypesId, analysisLimits.ListArgumentTypes },
{ DictArgumentTypesId, analysisLimits.DictArgumentTypes },
{ ReturnTypesId, analysisLimits.ReturnTypes },
{ YieldTypesId, analysisLimits.YieldTypes },
{ InstanceMembersId, analysisLimits.InstanceMembers },
{ DictKeyTypesId, analysisLimits.DictKeyTypes },
{ DictValueTypesId, analysisLimits.DictValueTypes },
{ IndexTypesId, analysisLimits.IndexTypes },
{ AssignedTypesId, analysisLimits.AssignedTypes },
{ UnifyCallsToNewId, analysisLimits.UnifyCallsToNew ? 1 : 0 },
{ ProcessCustomDecoratorsId, analysisLimits.ProcessCustomDecorators ? 1 : 0 },
{ UseTypeStubPackagesId, analysisLimits.UseTypeStubPackages ? 1 : 0 },
{ UseTypeStubPackagesExclusivelyId, analysisLimits.UseTypeStubPackagesExclusively ? 1 : 0 }
});
}
internal PythonAnalyzer AnalyzeDir(string dir, PythonLanguageVersion version = PythonLanguageVersion.V27, IEnumerable <string> excludeDirectories = null, CancellationToken?cancel = null)
{
List <string> files = new List <string>();
try {
ISet <string> excluded = null;
if (excludeDirectories != null)
{
excluded = new HashSet <string>(excludeDirectories, StringComparer.InvariantCultureIgnoreCase);
}
CollectFiles(dir, files, excluded);
} catch (DirectoryNotFoundException) {
return(null);
}
List <FileStreamReader> sourceUnits = new List <FileStreamReader>();
foreach (string file in files)
{
sourceUnits.Add(
new FileStreamReader(file)
);
}
Stopwatch sw = new Stopwatch();
sw.Start();
long start0 = sw.ElapsedMilliseconds;
// Explicitly specify the builtins name because we are using a 2.7
// interpreter for all versions.
var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version.ToVersion());
var projectState = new PythonAnalyzer(fact, fact.CreateInterpreter(), "__builtin__");
projectState.Limits = AnalysisLimits.GetStandardLibraryLimits();
var modules = new List <IPythonProjectEntry>();
foreach (var sourceUnit in sourceUnits)
{
try {
modules.Add(projectState.AddModule(
ModulePath.FromFullPath(sourceUnit.Path).ModuleName,
sourceUnit.Path,
null
));
} catch (ArgumentException) {
// Invalid module name, so skip the module
}
}
long start1 = sw.ElapsedMilliseconds;
Trace.TraceInformation("AddSourceUnit: {0} ms", start1 - start0);
var nodes = new List <Microsoft.PythonTools.Parsing.Ast.PythonAst>();
for (int i = 0; i < modules.Count; i++)
{
PythonAst ast = null;
try {
var sourceUnit = sourceUnits[i];
ast = Parser.CreateParser(sourceUnit, version).ParseFile();
} catch (Exception) {
}
nodes.Add(ast);
}
long start2 = sw.ElapsedMilliseconds;
Trace.TraceInformation("Parse: {0} ms", start2 - start1);
for (int i = 0; i < modules.Count; i++)
{
var ast = nodes[i];
if (ast != null)
{
modules[i].UpdateTree(ast, null);
}
}
long start3 = sw.ElapsedMilliseconds;
for (int i = 0; i < modules.Count; i++)
{
Trace.TraceInformation("Analyzing {1}: {0} ms", sw.ElapsedMilliseconds - start3, sourceUnits[i].Path);
var ast = nodes[i];
if (ast != null)
{
modules[i].Analyze(cancel ?? CancellationToken.None, true);
}
}
if (modules.Count > 0)
{
Trace.TraceInformation("Analyzing queue");
modules[0].AnalysisGroup.AnalyzeQueuedEntries(cancel ?? CancellationToken.None);
}
long start4 = sw.ElapsedMilliseconds;
Trace.TraceInformation("Analyze: {0} ms", start4 - start3);
return(projectState);
}
protected virtual AnalysisLimits GetLimits() => AnalysisLimits.GetDefaultLimits();
public void RunDirectory(string testDir, string packageId, string[] packages, int runOrder)
{
Stopwatch sw = new Stopwatch();
for (int i = 0; i < 3; i++)
{
GC.Collect(GC.MaxGeneration);
GC.WaitForPendingFinalizers();
}
var startingGcMem = GC.GetTotalMemory(true);
var workingSet = Process.GetCurrentProcess().WorkingSet64;
var startTime = sw.ElapsedMilliseconds;
long parserGcMem = 0;
long parserWorkingSet = 0;
sw.Start();
var analyzer = Analysis.Analyze(
testDir,
_lowAnalysis ? AnalysisLimits.MakeLowAnalysisLimits() : null,
() => {
for (int i = 0; i < 3; i++)
{
GC.Collect(GC.MaxGeneration);
GC.WaitForPendingFinalizers();
}
parserGcMem = GC.GetTotalMemory(true);
parserWorkingSet = Process.GetCurrentProcess().WorkingSet64;
}
);
sw.Stop();
var endingGcMem = GC.GetTotalMemory(true);
var endWorkingSet = Process.GetCurrentProcess().WorkingSet64;
int completionCount = 0, secondLevelCompletionCount = 0;
foreach (var module in analyzer.AllModules)
{
if (String.Equals(Path.Combine(testDir, "app.js"), module.FilePath, StringComparison.OrdinalIgnoreCase) ||
String.Equals(Path.Combine(testDir, "index.js"), module.FilePath, StringComparison.OrdinalIgnoreCase) ||
String.Equals(Path.Combine(testDir, "server.js"), module.FilePath, StringComparison.OrdinalIgnoreCase))
{
foreach (var package in packages)
{
string packageName = package.Split('@')[0];
string requireCode = String.Format("require('{0}')", packageName);
var moduleMembers = module.Analysis.GetMembersByIndex(requireCode, 0);
completionCount += moduleMembers.Count();
foreach (var moduleMember in moduleMembers)
{
var memberCode = requireCode + "." + moduleMember.Completion;
secondLevelCompletionCount += module.Analysis.GetMembersByIndex(memberCode, 0).Count();
}
if (_dumpMembers != null)
{
DumpMembers(module.Analysis, requireCode, 0);
}
}
break;
}
}
var time = sw.ElapsedMilliseconds - startTime;
int fileCount = analyzer.AllModules.Count();
var totalWorkingSet = (endWorkingSet - workingSet) / (1024 * 1024);
var parserTotalWorkingSet = (parserWorkingSet - workingSet) / (1024 * 1024);
var stats = new RunStats(
completionCount,
secondLevelCompletionCount,
time,
fileCount,
parserTotalWorkingSet,
(parserGcMem - startingGcMem) / (1024 * 1024),
totalWorkingSet,
(endingGcMem - startingGcMem) / (1024 * 1024),
runOrder
);
Log(stats, packageId);
if (_wait)
{
Console.ReadLine();
}
GC.KeepAlive(analyzer);
#if DEBUG
//Console.WriteLine(analyzer.GetAnalysisStats());
#endif
if (_cleanup)
{
Directory.Delete(testDir, true);
}
else
{
Console.WriteLine(" Ran in {0}", testDir);
}
}
protected virtual AnalysisLimits GetLimits()
{
return(AnalysisLimits.GetDefaultLimits());
}
public void SetLimits(AnalysisLimits limits)
{
_analyzer.Limits = limits;
}
public static JsAnalyzer Analyze(AnalysisLimits limits, params AnalysisFile[] files)
{
return(Analyze(limits, null, files));
}
/// <summary>
/// Returns a new set of limits, set to the default for analyzing a
/// standard library.
/// </summary>
public static AnalysisLimits GetStandardLibraryLimits()
{
var limits = new AnalysisLimits();
limits.CallDepth = 1;
limits.NormalArgumentTypes = 10;
limits.ListArgumentTypes = 5;
limits.DictArgumentTypes = 5;
limits.ReturnTypes = 10;
limits.YieldTypes = 10;
limits.InstanceMembers = 5;
limits.UnifyCallsToNew = true;
return limits;
}
public static AnalysisLimits FromDictionary(Dictionary <string, int> limits)
{
var analysisLimits = new AnalysisLimits();
int i;
if (limits.TryGetValue(CrossModuleId, out i))
{
analysisLimits.CrossModule = i;
}
if (limits.TryGetValue(CallDepthId, out i))
{
analysisLimits.CallDepth = i;
}
if (limits.TryGetValue(DecreaseCallDepthId, out i))
{
analysisLimits.DecreaseCallDepth = i;
}
if (limits.TryGetValue(NormalArgumentTypesId, out i))
{
analysisLimits.NormalArgumentTypes = i;
}
if (limits.TryGetValue(ListArgumentTypesId, out i))
{
analysisLimits.ListArgumentTypes = i;
}
if (limits.TryGetValue(DictArgumentTypesId, out i))
{
analysisLimits.DictArgumentTypes = i;
}
if (limits.TryGetValue(ReturnTypesId, out i))
{
analysisLimits.ReturnTypes = i;
}
if (limits.TryGetValue(YieldTypesId, out i))
{
analysisLimits.YieldTypes = i;
}
if (limits.TryGetValue(InstanceMembersId, out i))
{
analysisLimits.InstanceMembers = i;
}
if (limits.TryGetValue(DictKeyTypesId, out i))
{
analysisLimits.DictKeyTypes = i;
}
if (limits.TryGetValue(DictValueTypesId, out i))
{
analysisLimits.DictValueTypes = i;
}
if (limits.TryGetValue(IndexTypesId, out i))
{
analysisLimits.IndexTypes = i;
}
if (limits.TryGetValue(AssignedTypesId, out i))
{
analysisLimits.AssignedTypes = i;
}
if (limits.TryGetValue(UnifyCallsToNewId, out i))
{
analysisLimits.UnifyCallsToNew = i != 0;
}
if (limits.TryGetValue(ProcessCustomDecoratorsId, out i))
{
analysisLimits.ProcessCustomDecorators = i != 0;
}
if (limits.TryGetValue(UseTypeStubPackagesId, out i))
{
analysisLimits.UseTypeStubPackages = i != 0;
}
if (limits.TryGetValue(UseTypeStubPackagesExclusivelyId, out i))
{
analysisLimits.UseTypeStubPackagesExclusively = i != 0;
}
return(analysisLimits);
}
