/// <summary>
/// Main method for performing Scope analysis on a method.
/// </summary>
/// <param name="methodDefinition"></param>
public override void TraverseChildren(IMethodDefinition methodDefinition)
{
// We analyze only (interesting) processor methods.
if (IsInterestingProcessor(methodDefinition))
{
Utils.WriteLine("\n--------------------------------------------------");
Utils.WriteLine("Found interesting method " + methodDefinition.FullName());
// Create CFG and run basic analyses, such as copy-propagation.
var methodResult = AnalyzeMethod(methodDefinition);
Utils.WriteLine("Method has useful result: " + (!methodResult.UsedColumnsSummary.IsBottom && !methodResult.UsedColumnsSummary.IsTop));
Utils.WriteLine("Method has unsupported features: " + methodResult.Unsupported);
Utils.WriteLine("\n--------------------------------------------------");
results.Add(methodResult);
}
else
{
var methodResult = new ScopeMethodAnalysisResult(methodDefinition, mhost);
methodResult.Interesting = false;
results.Add(methodResult);
}
return;
}
private ScopeMethodAnalysisResult AnalyzeMethod(IMethodDefinition methodDefinition)
{
var methodResult = new ScopeMethodAnalysisResult(methodDefinition, mhost);
try
{
var cfg = PrepareMethod(methodDefinition);
Utils.WriteLine("CFG size " + cfg.Nodes.Count);
// Run escape analysis.
var escAnalysis = DoEscapeAnalysis(cfg, methodDefinition, methodResult);
methodResult.Unsupported = escAnalysis.Unsupported;
// If some row has escaped or the method has unsupported features,
// there is no need to analyze the method any further.
if (escAnalysis.InterestingRowEscaped || escAnalysis.Unsupported)
{
if (escAnalysis.InterestingRowEscaped && !escAnalysis.Unsupported)
{
Utils.WriteLine("A rowish data structure has escaped, no dependency information available.");
}
methodResult.UsedColumnsSummary = ColumnsDomain.Top;
}
else
{
// Otherwise, do constant-set propagation (CSP) analysis.
var cspAnalysis = DoConstantPropagationAnalysis(cfg, methodDefinition, methodResult);
var cspInfo = new NaiveScopeConstantsProvider(cspAnalysis.PreResults, mhost);
//Finally, do the actual used-columns analysis using results of the previous CSP analysis.
var clsAnalysis = DoUsedColumnsAnalysis(cfg, cspInfo, methodResult);
methodResult.Unsupported = cspAnalysis.Unsupported | clsAnalysis.Unsupported;
}
}
catch (Exception e)
{
Utils.WriteLine(String.Format("{0} METHOD FAILURE: {1}", methodDefinition.FullName(), e.Message));
Utils.WriteLine(e.StackTrace);
methodResult.Failed = true;
}
return(methodResult);
}
/// <summary>
/// Entry point method for performing used-columns analysis. The analysis expects results of
/// constant-set propagation analysis.
/// </summary>
/// <param name="cfg"></param>
/// <param name="cspInfo"></param>
/// <param name="results"></param>
/// <returns></returns>
private UsedColumnsAnalysis DoUsedColumnsAnalysis(ControlFlowGraph cfg, ConstantsInfoProvider cspInfo, ScopeMethodAnalysisResult results)
{
Utils.WriteLine("Running used columns analysis...");
var clsAnalysis = new UsedColumnsAnalysis(mhost, cfg, cspInfo, rowType, columnType);
var outcome = clsAnalysis.Analyze();
results.UsedColumnsSummary = outcome;
// We only save statistics about column accesses for methods with useful results.
if (!results.UsedColumnsSummary.IsTop && !results.UsedColumnsSummary.IsBottom)
{
results.ColumnStringAccesses += clsAnalysis.ColumnStringAccesses;
results.ColumnIndexAccesses += clsAnalysis.ColumnIndexAccesses;
}
Utils.WriteLine(results.UsedColumnsSummary.ToString());
Utils.WriteLine("Done with used columns analysis");
return(clsAnalysis);
}
/// <summary>
/// Entry point method for performing constant-set propagation analysis.
/// </summary>
/// <param name="cfg"></param>
/// <param name="method"></param>
/// <param name="results"></param>
/// <returns></returns>
private ConstantPropagationSetAnalysis DoConstantPropagationAnalysis(ControlFlowGraph cfg, IMethodDefinition method, ScopeMethodAnalysisResult results)
{
Utils.WriteLine("Running constant propagation set analysis...");
var cpsAnalysis = new ConstantPropagationSetAnalysis(cfg, method, mhost, schemaType);
results.CPropagationSummary = cpsAnalysis.Analyze()[cfg.Exit.Id].Output;
//Utils.WriteLine(results.CPropagationSummary.ToString());
Utils.WriteLine("Done with constant propagation set analysis");
return(cpsAnalysis);
}
/// <summary>
/// Entry point method for performing escape analysis.
/// </summary>
/// <param name="cfg"></param>
/// <param name="method"></param>
/// <param name="results"></param>
/// <returns></returns>
private NaiveScopeMayEscapeAnalysis DoEscapeAnalysis(ControlFlowGraph cfg, IMethodDefinition method, ScopeMethodAnalysisResult results)
{
Utils.WriteLine("Running escape analysis...");
var escAnalysis = new NaiveScopeMayEscapeAnalysis(cfg, method, mhost, rowType, rowSetType);
results.EscapeSummary = escAnalysis.Analyze()[cfg.Exit.Id].Output;
//Utils.WriteLine(results.EscapeSummary.ToString());
Utils.WriteLine("Something escaped: " + escAnalysis.InterestingRowEscaped);
Utils.WriteLine("Done with escape analysis");
return(escAnalysis);
}
//static Dictionary<string, int> TYPE_SIZES = new Dictionary<string, int>() { {"int", 4}, {"int?", 8}, {"float", 4}, {"float?", 8}, {"double", 8}, {"double?", 12},
// {"long", 8}, {"long?", 12}, {"DateTime", 8}, {"DateTime?", 12}, {"char", 2}, {"char?", 4}, {"string", 20}, {"string?", 20}, {"binary", 20 }, {"binary?", 20},
// {"Guid", 16}, {"Guid?", 20} };
//static int DEFAULT_TYPE_SIZE = 20;
private static void ComputeImprovementStats(ScopeMethodAnalysisResult result, ref ScopeAnalysisStats stats,
XElement vDef, Dictionary <string, string> pIdMapping)
{
stats.ColumnIndexAccesses += result.ColumnIndexAccesses;
stats.ColumnStringAccesses += result.ColumnStringAccesses;
if (vDef == null || pIdMapping == null)
{
return;
}
var column = result.UsedColumnsSummary;
if (column.IsBottom || column.IsTop)
{
return;
}
var pTypeFullName = result.ProcessorType.FullName();
Utils.WriteLine("Checking column usage for " + pTypeFullName);
if (!pIdMapping.ContainsKey(pTypeFullName))
{
Utils.WriteLine("WARNING: could not match processor mapping: " + pTypeFullName);
return;
}
stats.Mapped += 1;
try
{
var id = pIdMapping[pTypeFullName];
var operators = vDef.Descendants("operator");
// Id can appear several times in the xml file since the same reducer can be used multiple times
// and contained within different Scope vertices.
var process = operators.Where(op => op.Attribute("id") != null && op.Attribute("id").Value.Equals(id)).ToList().First();
// TODO: make parsing take into account commas in generics. Current approach
// does not invalidate results, but is not clean.
var input_schema = process.Descendants("input").Single().Attribute("schema").Value.Split(',');
var inputSchema = new Dictionary <string, string>();
foreach (var input in input_schema)
{
if (!input.Contains(":"))
{
continue;
}
var parts = input.Split(':');
var name = parts[0].Trim();
var type = parts[1].Trim();
inputSchema[name] = type;
}
var inputColumns = inputSchema.Keys.ToList();
// TODO: make parsing take into account commas in generics. Current approach
// does not invalidate results, but is not clean.
var output_schema = process.Descendants("output").Single().Attribute("schema").Value.Split(',');
var outputSchema = new Dictionary <string, string>();
foreach (var output in output_schema)
{
if (!output.Contains(":"))
{
continue;
}
var parts = output.Split(':');
var name = parts[0].Trim();
var type = parts[1].Trim();
outputSchema[name] = type;
}
var outputColumns = outputSchema.Keys.ToList();
var usedColumns = new HashSet <string>();
foreach (var c in column.Elements)
{
var val = c.Value;
if (val is string)
{
usedColumns.Add(val as string);
}
else if (val is int)
{
int index = Int32.Parse(val.ToString());
if (index >= 0 && index < inputColumns.Count)
{
usedColumns.Add(inputColumns[index]);
}
if (index >= 0 && index < outputColumns.Count)
{
usedColumns.Add(outputColumns[index]);
}
if ((index >= inputColumns.Count && index >= outputColumns.Count) || index < 0)
{
Utils.WriteLine("WARNING: some index was out of schema range: " + index);
}
}
else
{
Utils.WriteLine("WARNING: other value type used for indexing besides string and int: " + val);
return;
}
}
// Compute stats for schema input-output union.
var allSchemaColumns = new HashSet <string>(inputColumns.Union(outputColumns));
var redundants = allSchemaColumns.Except(usedColumns);
if (redundants.Any())
{
stats.UnionColumnsUnused += 1;
var savings = redundants.Count();
stats.UnionColumnsSavings += savings;
stats.UnionColumnsSavingsPercentages += savings / (double)allSchemaColumns.Count;
Utils.WriteLine(String.Format("SAVINGS (union) ({0}): used union columns subset of defined columns: {1}", result.Method.FullName(), savings));
}
else
{
stats.UnionColumnsAllUsed += 1;
Utils.WriteLine("ALL USED (union): all union columns used.");
}
if (allSchemaColumns.IsProperSubsetOf(usedColumns))
{
Utils.WriteLine("OVERAPPROXIMATION: redundant used columns: " + String.Join(" ", usedColumns.Except(allSchemaColumns)));
stats.Warnings += 1;
}
// Compute stats for input schema.
redundants = inputColumns.Except(usedColumns);
if (redundants.Any())
{
stats.InputColumnsUnused += 1;
var savings = redundants.Count();
stats.InputColumnsSavings += savings;
stats.InputColumnsSavingsPercentages += savings / (double)inputColumns.Count;
//var redundantInputByteSize = ComputeColumnsSize(redundants.Except(outputColumns), inputSchema);
//var inputByteSize = ComputeColumnsSize(inputColumns, inputSchema);
//stats.InputColumnsByteSavings += redundantInputByteSize;
//stats.InputColumnsByteSavingsPercentages += redundantInputByteSize / (double)inputByteSize;
Utils.WriteLine(String.Format("SAVINGS (input) ({0}): used input columns subset of defined columns: {1}", result.Method.FullName(), savings));
}
else
{
stats.InputColumnsAllUsed += 1;
Utils.WriteLine("All USED (input): all input columns used.");
}
// Compute stats for input schema.
redundants = outputColumns.Except(usedColumns);
if (redundants.Any())
{
stats.OutputColumnsUnused += 1;
var savings = redundants.Count();
stats.OutputColumnsSavings += savings;
stats.OutputColumnsSavingsPercentages += savings / (double)outputColumns.Count;
Utils.WriteLine(String.Format("SAVINGS (output) ({0}): used output columns subset of defined columns: {1}", result.Method.FullName(), savings));
}
else
{
stats.OutputColumnsAllUsed += 1;
Utils.WriteLine("All USED (output): all output columns used.");
}
}
catch (Exception e)
{
Utils.WriteLine(String.Format("ERROR: failed to compute column usage for {0} {1}", pTypeFullName, e.Message));
}
}
/// <summary>
/// Main method for performing Scope analysis on a method.
/// </summary>
/// <param name="methodDefinition"></param>
public override void TraverseChildren(IMethodDefinition methodDefinition)
{
//if (!methodDefinition.FullName().Contains("MMRV2.IndexSelection.DPGenDomainKeyProcessor"))
// return;
//if (!methodDefinition.FullName().Contains("ScopeML.Prediction.CompactModelBuilderReducer") || !methodDefinition.FullName().Contains("MoveNext"))
// return;
var methodResult = new ScopeMethodAnalysisResult(methodDefinition, mhost);
try
{
// We analyze only (interesting) processor methods.
if (IsInterestingProcessor(methodDefinition))
{
Utils.WriteLine("\n--------------------------------------------------\n");
Utils.WriteLine("Found interesting method " + methodDefinition.FullName());
// Create CFG and run basic analyses, such as copy-propagation.
var cfg = PrepareMethod(methodDefinition);
Utils.WriteLine("CFG size " + cfg.Nodes.Count);
//System.IO.File.WriteAllText(@"mbody-zvonimir.txt", _code);
// Run escape analysis.
var escAnalysis = DoEscapeAnalysis(cfg, methodDefinition, methodResult);
methodResult.Unsupported = escAnalysis.Unsupported;
// If some row has escaped or the method has unsupported features,
// there is no need to analyze the method any further.
if (escAnalysis.InterestingRowEscaped || escAnalysis.Unsupported)
{
if (escAnalysis.InterestingRowEscaped && !escAnalysis.Unsupported)
{
Utils.WriteLine("A rowish data structure has escaped, no dependency information available.");
}
methodResult.UsedColumnsSummary = ColumnsDomain.Top;
}
else
{
// Otherwise, do constant-set propagation (CSP) analysis.
var cspAnalysis = DoConstantPropagationAnalysis(cfg, methodDefinition, methodResult);
var cspInfo = new NaiveScopeConstantsProvider(cspAnalysis.PreResults, mhost);
//Finally, do the actual used-columns analysis using results of the previous CSP analysis.
var clsAnalysis = DoUsedColumnsAnalysis(cfg, cspInfo, methodResult);
methodResult.Unsupported = cspAnalysis.Unsupported | clsAnalysis.Unsupported;
}
Utils.WriteLine("Method has useful result: " + (!methodResult.UsedColumnsSummary.IsBottom && !methodResult.UsedColumnsSummary.IsTop));
Utils.WriteLine("Method has unsupported features: " + methodResult.Unsupported);
Utils.WriteLine("\n--------------------------------------------------\n");
}
else
{
methodResult.Interesting = false;
}
}
catch (Exception e)
{
Utils.WriteLine(String.Format("{0} METHOD FAILURE: {1}", methodDefinition.FullName(), e.Message));
Utils.WriteLine(e.StackTrace);
methodResult.Failed = true;
}
results.Add(methodResult);
return;
}
