private static void WriteResultToSarifLog(string inputPath, StreamWriter outputStream, SarifLog log, MethodDefinition moveNextMethod, DependencyPTGDomain depAnalysisResult,
SongTaoDependencyAnalysis dependencyAnalysis, IDictionary <string, ClassDefinition> processorMap)
{
var results = new List <Result>();
var escapes = depAnalysisResult.Dependencies.A1_Escaping.Select(traceable => traceable.ToString());
var inputUses = new HashSet <Traceable>();
var outputModifies = new HashSet <Traceable>();
if (!depAnalysisResult.IsTop)
{
if (depAnalysisResult.Dependencies.A4_Ouput.Any())
{
var outColumnMap = new MapSet <TraceableColumn, Traceable>();
var outColumnControlMap = new MapSet <TraceableColumn, Traceable>();
foreach (var outColum in depAnalysisResult.Dependencies.A4_Ouput.Keys)
{
var outColumns = depAnalysisResult.GetTraceables(outColum).OfType <TraceableColumn>()
.Where(t => t.TableKind == ProtectedRowKind.Output);
foreach (var column in outColumns)
{
if (!outColumnMap.ContainsKey(column))
{
outColumnMap.AddRange(column, depAnalysisResult.Dependencies.A4_Ouput[outColum]);
}
else
{
outColumnMap.AddRange(column, outColumnMap[column].Union(depAnalysisResult.Dependencies.A4_Ouput[outColum]));
}
if (!outColumnControlMap.ContainsKey(column))
{
outColumnControlMap.AddRange(column, depAnalysisResult.Dependencies.A4_Ouput_Control[outColum]);
}
else
{
outColumnControlMap.AddRange(column, outColumnControlMap[column].Union(depAnalysisResult.Dependencies.A4_Ouput_Control[outColum]));
}
}
}
foreach (var entryOutput in outColumnMap)
{
var result = new Result();
result.Id = "SingleColumn";
var column = entryOutput.Key;
var columnString = column.ToString();
var dependsOn = entryOutput.Value;
var controlDepends = new HashSet <Traceable>();
result.SetProperty("column", columnString);
result.SetProperty("data depends", dependsOn.Select(traceable => traceable.ToString()));
if (outColumnControlMap.ContainsKey(column))
{
var controlDependsOn = outColumnControlMap[column];
result.SetProperty("control depends", controlDependsOn.Select(traceable => traceable.ToString()));
}
else
{
result.SetProperty("control depends", new string[] { });
}
result.SetProperty("escapes", escapes);
results.Add(result);
inputUses.AddRange(dependsOn.Where(t => t.TableKind == ProtectedRowKind.Input));
outputModifies.Add(column);
}
//foreach (var outColum in depAnalysisResult.Dependencies.A4_Ouput.Keys)
//{
// //var outColumns = depAnalysisResult.Dependencies.A2_Variables[outColum].OfType<TraceableColumn>()
// // .Where(t => t.TableKind == ProtectedRowKind.Output);
// var outColumns = depAnalysisResult.GetTraceables(outColum).OfType<TraceableColumn>()
// .Where(t => t.TableKind == ProtectedRowKind.Output);
// foreach (var column in outColumns)
// {
// var result = new Result();
// result.Id = "SingleColumn";
// var columnString = column.ToString();
// var dependsOn = depAnalysisResult.Dependencies.A4_Ouput[outColum];
// //dependsOn.AddRange(traceables);
// result.SetProperty("column", columnString);
// result.SetProperty("data depends", dependsOn.Select(traceable => traceable.ToString()));
// if (depAnalysisResult.Dependencies.A4_Ouput_Control.ContainsKey(outColum))
// {
// var controlDependsOn = depAnalysisResult.Dependencies.A4_Ouput_Control[outColum];
// result.SetProperty("control depends", controlDependsOn.Where(t => !(t is Other)).Select(traceable => traceable.ToString()));
// inputUses.AddRange(controlDependsOn.Where(t => t.TableKind == ProtectedRowKind.Input));
// }
// else
// {
// result.SetProperty("control depends", new string[] { });
// }
// result.SetProperty("escapes", escapes);
// results.Add(result);
// inputUses.AddRange(dependsOn.Where(t => t.TableKind == ProtectedRowKind.Input));
// outputModifies.Add(column);
// }
//}
}
else
{
var result = new Result();
result.Id = "SingleColumn";
result.SetProperty("column", "_EMPTY_");
result.SetProperty("escapes", escapes);
results.Add(result);
}
var resultSummary = new Result();
resultSummary.Id = "Summary";
//var inputsString = inputUses.OfType<TraceableColumn>().Select(t => t.ToString());
//var outputsString = outputModifies.OfType<TraceableColumn>().Select(t => t.ToString());
//result2.SetProperty("Inputs", inputsString);
//result2.SetProperty("Outputs", outputsString);
var inputsString = dependencyAnalysis.InputColumns.Select(t => t.ToString());
var outputsString = dependencyAnalysis.OutputColumns.Select(t => t.ToString());
resultSummary.SetProperty("Inputs", inputsString);
resultSummary.SetProperty("Outputs", outputsString);
var inputSchemaString = InputSchema.Columns.Select(t => t.ToString());
var outputSchemaString = OutputSchema.Columns.Select(t => t.ToString());
resultSummary.SetProperty("SchemaInputs", inputSchemaString);
resultSummary.SetProperty("SchemaOutputs", outputSchemaString);
results.Add(resultSummary);
if (outputStream != null)
{
outputStream.WriteLine("Class: [{0}] {1}", moveNextMethod.ContainingType.FullPathName(), moveNextMethod.ToSignatureString());
if (depAnalysisResult.IsTop)
{
outputStream.WriteLine("Analysis returns TOP");
}
outputStream.WriteLine("Inputs: {0}", String.Join(", ", inputsString));
outputStream.WriteLine("Outputs: {0}", String.Join(", ", outputsString));
}
}
else
{
var result = new Result();
result.Id = "SingleColumn";
result.SetProperty("column", "_TOP_");
result.SetProperty("depends", "_TOP_");
results.Add(result);
var resultEmpty = new Result();
resultEmpty.Id = "Summary";
resultEmpty.SetProperty("Inputs", "_TOP_");
resultEmpty.SetProperty("Outputs", "_TOP_");
resultEmpty.SetProperty("SchemaInputs", "_TOP_");
resultEmpty.SetProperty("SchemaOutputs", "_TOP_");
results.Add(resultEmpty);
}
var id = String.Format("[{0}] {1}", moveNextMethod.ContainingType.FullPathName(), moveNextMethod.ToSignatureString());
// Very clumsy way to find the process number and the processor name from the MoveNext method.
// But it is the process number and processor name that allow us to link these results to the information
// in the XML file that describes the job.
// Climb the containing type chain from the MoveNext method until we find the entry in the dictionary whose
// value matches one of the classes.
var done = false;
foreach (var kv in processorMap)
{
if (done)
{
break;
}
var c = moveNextMethod.ContainingType as ClassDefinition;
while (c != null)
{
if (kv.Value == c)
{
id = kv.Value.Name + "|" + kv.Key;
done = true;
break;
}
c = c.ContainingType as ClassDefinition;
}
}
AddRunToSarifOutput(log, inputPath, id, results);
}
/// This does the interprocedural analysis.
/// It (currently) does NOT support recursive method invocations
/// </summary>
/// <param name="instruction"></param>
/// <param name="resolvedCallee"></param>
/// <param name="calleeCFG"></param>
private InterProceduralReturnInfo InterproceduralAnalysis(InterProceduralCallInfo callInfo, ControlFlowGraph calleeCFG)
{
if (stackDepth > InterproceduralManager.MaxStackDepth)
{
return(new InterProceduralReturnInfo(callInfo.CallerState));
}
stackDepth++;
// I currently do not support recursive calls
// Will add support for this in the near future
if (callStack.Contains(callInfo.Callee))
{
callInfo.CallerState.Dependencies.IsTop = true;
AnalysisStats.AddAnalysisReason(new AnalysisReason(callInfo.Caller, callInfo.Instruction, String.Format(CultureInfo.InvariantCulture, "Recursive call to {0}", callInfo.Callee.Name)));
return(new InterProceduralReturnInfo(callInfo.CallerState));
}
this.callStack.Push(callInfo.Callee);
System.Console.WriteLine("Analyzing Method {0} Stack: {1}", new string(' ', stackDepth * 2) + callInfo.Callee.ToSignatureString(), stackDepth);
// 1) Bind PTG and create a Poinst-to Analysis for the callee. In pta.Result[node.Exit] is the PTG at exit of the callee
var calleePTG = PTABindCallerCallee(callInfo.CallerPTG, callInfo.CallArguments, callInfo.Callee);
IteratorPointsToAnalysis calleePTA = new IteratorPointsToAnalysis(calleeCFG, callInfo.Callee, calleePTG);
IDictionary <IVariable, IExpression> equalities = new Dictionary <IVariable, IExpression>();
SongTaoDependencyAnalysis.PropagateExpressions(calleeCFG, equalities);
var rangesAnalysis = new RangeAnalysis(calleeCFG);
rangesAnalysis.Analyze();
// 2) Bind Parameters of the dependency analysis and run
var calleeDomain = BindCallerCallee(callInfo);
calleeDomain.PTG = calleePTG;
var dependencyAnalysis = new IteratorDependencyAnalysis(callInfo.Callee, calleeCFG, calleePTA, callInfo.ProtectedNodes, equalities, this, rangesAnalysis, calleeDomain, callInfo.ScopeData);
// If we already did the dataflow analysis for this method we recover the dataflow state
// This should be adapted (or removed) if we want the analysis to be context sensitive
//if (dataflowCache.ContainsKey(callInfo.Callee))
//{
// var previosResult = dataflowCache[callInfo.Callee];
// //DataFlowAnalysisResult<DependencyPTGDomain>[] resultsCopy = new DataFlowAnalysisResult<DependencyPTGDomain>[previosResult.Length];
// //for(int i=0; i< previosResult.Length; i++)
// //{
// // resultsCopy[i] = new DataFlowAnalysisResult<DependencyPTGDomain>();
// // resultsCopy[i].Input = previosResult[i].Input!=null? previosResult[i].Input.Clone(): null;
// // resultsCopy[i].Output = previosResult[i].Output!=null? previosResult[i].Output.Clone(): null;
// //}
// //dependencyAnalysis.SetPreviousResult(resultsCopy);
// dependencyAnalysis.SetPreviousResult(previosResult);
//}
dependencyAnalysis.Analyze();
this.dataflowCache[callInfo.Callee] = dependencyAnalysis.Result;
stackDepth--;
this.callStack.Pop();
// 3) Bind callee with caller
// Should I need the PTG of caller and callee?
//var exitCalleePTG = calleePTA.Result[calleeCFG.Exit.Id].Output;
var exitCalleePTG = dependencyAnalysis.Result[calleeCFG.Exit.Id].Output.PTG;
var exitResult = BindCalleeCaller(callInfo, calleeCFG, dependencyAnalysis);
// Recover the frame of the original Ptg and bind ptg results
//PointsToGraph bindPtg = PTABindCaleeCalleer(callInfo.CallLHS, calleeCFG, calleePTA);
var bindPtg = PTABindCaleeCalleer(callInfo.CallLHS, calleeCFG, exitCalleePTG, calleePTA.ReturnVariable);
exitResult.PTG = bindPtg;
return(new InterProceduralReturnInfo(exitResult));
//return new InterProceduralReturnInfo(exitResult, bindPtg);
}
public static SarifLog AnalyzeDll(string inputPath, ScopeMethodKind kind,
bool useScopeFactory = true, bool interProc = false, StreamWriter outputStream = null)
{
// Determine whether to use Interproc analysis
AnalysisOptions.DoInterProcAnalysis = interProc;
AnalysisStats.TotalNumberFolders++;
var host = new MyHost();
PlatformTypes.Resolve(host);
var loader = new MyLoader(host);
host.Loader = loader;
var scopeGenAssembly = loader.LoadMainAssembly(inputPath);
AnalysisStats.TotalDllsFound++;
loader.LoadCoreAssembly();
var program = new ScopeProgramAnalysis(host, loader);
// program.interprocAnalysisManager = new InterproceduralManager(host);
program.ScopeGenAssembly = scopeGenAssembly;
//program.ReferenceFiles = referenceFiles;
program.ClassFilters = new HashSet <string>();
program.ClousureFilters = new HashSet <string>();
program.EntryMethods = new HashSet <string>();
if (kind == ScopeMethodKind.Reducer || kind == ScopeMethodKind.All)
{
program.ClassFilters.Add("Reducer");
program.ClousureFilters.Add("<Reduce>d__");
program.EntryMethods.Add("Reduce");
}
if (kind == ScopeMethodKind.Producer || kind == ScopeMethodKind.All)
{
program.ClassFilters.Add("Processor");
program.ClousureFilters.Add("<Process>d__");
program.EntryMethods.Add("Process");
//program.ClassFilter = "Producer";
//program.ClousureFilter = "<Produce>d__";
//program.EntryMethod = "Produce";
}
program.MethodUnderAnalysisName = "MoveNext";
IEnumerable <Tuple <MethodDefinition, MethodDefinition, MethodDefinition> > scopeMethodPairs;
if (useScopeFactory)
{
scopeMethodPairs = program.ObtainScopeMethodsToAnalyze();
if (!scopeMethodPairs.Any())
{
if (outputStream != null)
{
outputStream.WriteLine("Failed to obtain methods from the ScopeFactory. ");
}
System.Console.WriteLine("Failed to obtain methods from the ScopeFactory.");
//System.Console.WriteLine("Now trying to find methods in the the assembly");
//scopeMethodPairs = program.ObtainScopeMethodsToAnalyzeFromAssemblies();
}
}
else
{
scopeMethodPairs = program.ObtainScopeMethodsToAnalyzeFromAssemblies();
}
if (scopeMethodPairs.Any())
{
var log = CreateSarifOutput();
IReadOnlyDictionary <string, Tuple <Schema, Schema> > allSchemas;
if (useScopeFactory)
{
allSchemas = program.ReadSchemasFromXML(inputPath);
}
else
{
allSchemas = program.ReadSchemasFromXML2(inputPath);
}
foreach (var methodPair in scopeMethodPairs)
{
AnalysisStats.TotalMethods++;
var entryMethodDef = methodPair.Item1;
var moveNextMethod = methodPair.Item2;
var getEnumMethod = methodPair.Item3;
System.Console.WriteLine("Method {0} on class {1}", moveNextMethod.Name, moveNextMethod.ContainingType.FullPathName());
Schema inputSchema = null;
Schema outputSchema = null;
Tuple <Schema, Schema> schemas;
if (allSchemas.TryGetValue(moveNextMethod.ContainingType.ContainingType.Name, out schemas))
{
inputSchema = schemas.Item1;
outputSchema = schemas.Item2;
}
try
{
InputSchema = inputSchema;
OutputSchema = outputSchema;
var dependencyAnalysis = new SongTaoDependencyAnalysis(host, program.interprocAnalysisManager, moveNextMethod, entryMethodDef, getEnumMethod);
var depAnalysisResult = dependencyAnalysis.AnalyzeMoveNextMethod();
WriteResultToSarifLog(inputPath, outputStream, log, moveNextMethod, depAnalysisResult, dependencyAnalysis, program.factoryReducerMap);
InputSchema = null;
OutputSchema = null;
}
catch (Exception e)
{
System.Console.WriteLine("Could not analyze {0}", inputPath);
System.Console.WriteLine("Exception {0}\n{1}", e.Message, e.StackTrace);
AnalysisStats.TotalofDepAnalysisErrors++;
AnalysisStats.AddAnalysisReason(new AnalysisReason(moveNextMethod, moveNextMethod.Body.Instructions[0],
String.Format(CultureInfo.InvariantCulture, "Throw exception {0}\n{1}", e.Message, e.StackTrace.ToString())));
}
}
return(log);
}
else
{
System.Console.WriteLine("No method {0} of type {1} in {2}", program.MethodUnderAnalysisName, program.ClassFilters, inputPath);
return(null);
}
}