public DependencyVisitor(string filename, Program program, List <Tuple <string, string, int> > changeLog, int timeOut, bool prune = true, bool dataOnly = false) { this.filename = filename; this.program = program; this.callGraph = CallGraphHelper.ComputeCallGraph(program); this.nodeToImpl = Utils.ComputeNodeToImpl(program); this.ProcDependencies = new Dictionary <Procedure, Dependencies>(); this.dominatedBy = new Dictionary <Block, HashSet <Block> >(); this.branchCondVars = new Dictionary <Block, VarSet>(); this.worklist = new WorkList <Dependencies>(); this.procEntryTDTaint = new Dictionary <Procedure, Dependencies>(); this.procExitTDTaint = new Dictionary <Procedure, Dependencies>(); // populate changedProcs,changedBlock from changedLines this.changedBlocks = Utils.ComputeChangedBlocks(program, changeLog); this.changedProcs = Utils.ComputeChangedProcs(program, changeLog); this.prune = prune; this.dataOnly = dataOnly; this.taintOnly = false; this.timeOut = timeOut * 1000; // change to ms }
public override Program VisitProgram(Program program) { var callGraph = CallGraphHelper.ComputeCallGraph(program); var worklist = new List <Procedure>(); program.TopLevelDeclarations.Iter(d => { if (d is Procedure) { worklist.Add(d as Procedure); } }); int numVisited = 0; while (worklist.Count > 0) { currentProc = worklist.First(); worklist.Remove(currentProc); if (!ProcReadSet.ContainsKey(currentProc)) { ProcReadSet[currentProc] = new VarSet(); } var initialRS = new VarSet(ProcReadSet[currentProc]); var impl = program.Implementations.SingleOrDefault(i => i.Name == currentProc.Name); if (impl == null) {// a stub ProcReadSet[currentProc].Add(Utils.VariableUtils.NonDetVar); ProcReadSet[currentProc].UnionWith(currentProc.InParams); ProcReadSet[currentProc].UnionWith(currentProc.OutParams); currentProc.Modifies.Iter(m => ProcReadSet[currentProc].UnionWith(Utils.VariableUtils.ExtractVars(m))); } else {// a procedure with a body Console.WriteLine("Visiting: {0} ({1}/{2})", impl.Name, ++numVisited, program.Implementations.Count()); Visit(impl); } if (!initialRS.SetEquals(ProcReadSet[currentProc]) && callGraph.Nodes.Contains(currentProc)) { // the read set changed worklist.AddRange(callGraph.Predecessors(currentProc)); // add all callers to WL } //GC.Collect(); } ProcReadSet.Iter(prs => Utils.VariableUtils.FixFormals(program.Implementations.SingleOrDefault(i => i.Name == prs.Key.Name), prs.Value)); return(program); }
public Program PruneProgram() { this.callGraph = CallGraphHelper.ComputeCallGraph(program); foreach (var proc in changedProcedures) { this.ComputeTransitiveCallers(proc); } foreach (var proc in callers) { this.ComputeTransitiveCallees(proc); } return(this.Prune()); }
public void Prune() { var cg = CallGraphHelper.ComputeCallGraph(this.program); foreach (var proc in cg.Nodes) { this.IsTransitivelyFree(proc, cg); } foreach (var proc in this.transitivelyFreeProceduresCache.Keys.Where(x => this.transitivelyFreeProceduresCache[x])) { var impl = Util.getImplByName(this.program, proc.Name); if (impl != null) { Console.WriteLine("Pruning impls with no non-free contract transitively: " + proc.Name); this.program.RemoveTopLevelDeclaration(impl); } } }
static void Main(string[] args) { if (args.Count() != 4) { Console.WriteLine("The program is to be run as: SymDiffPostProcess.exe tainted.csv tainted.dac.csv v1.bpl_changed_lines.txt _v2.bpl"); return; } if (args.ToList().Any(x => x == "-break")) { Debugger.Launch(); } var taintDependency = args[0]; var taintDac = args[1]; var changedFile = args[2]; var bplFile = args[3]; Dictionary <string, HashSet <int> > procToDacTaint = getProcToLinesMap(taintDac); Dictionary <string, HashSet <int> > procToDepTaint = getProcToLinesMap(taintDependency); Dictionary <string, HashSet <int> > procToChange = getProcToLinesMap(changedFile); FilterChangedFromTainted(procToDacTaint, procToChange); FilterChangedFromTainted(procToDepTaint, procToChange); BoogieUtils.InitializeBoogie(""); Program program = BoogieUtils.ParseProgram(bplFile); program.Resolve(); program.Typecheck(); var callGraph = CallGraphHelper.ComputeCallGraph(program); var lengths = BoogieUtils.FindProcsAtDistance(callGraph, procToChange.Keys.ToList()); Dictionary <int, int> distanceToLinesDac = ComputeDistanceToLines(lengths, procToDacTaint); PrintDistanceBuckets(distanceToLinesDac, "dacTaintBuckets.csv"); Dictionary <int, int> distanceToLinesDep = ComputeDistanceToLines(lengths, procToDepTaint); PrintDistanceBuckets(distanceToLinesDep, "depTaintBuckets.csv"); }
public RefineDependencyWL(string filename, Program program, Dictionary <Procedure, Dependencies> lowerBoundProcDependencies, Dictionary <Procedure, Dependencies> upperBoundProcDependencies, int stackBound) { this.filename = filename; this.program = program; this.lowerBoundProcDependencies = lowerBoundProcDependencies; this.upperBoundProcDependencies = upperBoundProcDependencies; this.stackBound = stackBound; this.callGraph = CallGraphHelper.ComputeCallGraph(program); currDependencies = null; program.TopLevelDeclarations.OfType <Procedure>().Iter(proc => { if (upperBoundProcDependencies.ContainsKey(proc)) //some unreachable procs such as havoc_Assert are not present in the tables { Debug.Assert(upperBoundProcDependencies[proc].Keys.All(v => proc.InParams.Contains(v) || proc.OutParams.Contains(v) || v is GlobalVariable)); Debug.Assert(upperBoundProcDependencies[proc].Values.All(d => d.All(v => proc.InParams.Contains(v) || proc.OutParams.Contains(v) || v is GlobalVariable))); Debug.Assert(lowerBoundProcDependencies[proc].Keys.All(v => proc.InParams.Contains(v) || proc.OutParams.Contains(v) || v is GlobalVariable)); Debug.Assert(lowerBoundProcDependencies[proc].Values.All(d => d.All(v => proc.InParams.Contains(v) || proc.OutParams.Contains(v) || v is GlobalVariable))); } } ); }
public void RunFixedPoint(Stopwatch sw) { var worklist = new List <Procedure>(); bool acyclic; List <Procedure> topSortedProcedures; Graph <Procedure> callGraph = CallGraphHelper.ComputeCallGraph(program); //constant //HACK: lets try with self-recursion (includes loops) removed Graph <Procedure> selfLessCg = new Graph <Procedure>(); callGraph.Edges.Where(e => e.Item1 != e.Item2).Iter(e => selfLessCg.AddEdge(e.Item1, e.Item2)); selfLessCg.TarjanTopSort(out acyclic, out topSortedProcedures, true); if (acyclic) { worklist.AddRange(topSortedProcedures); } else { Console.WriteLine("---------\nMutual recursion exists: the work lists are not optimized\n--------"); //TODO: Compute SCCs and topSort over SCCs worklist.AddRange(CallGraphHelper.BottomUp(callGraph)); //does this help? // worklist.AddRange(program.TopLevelDeclarations.Where(d => d is Procedure).Select(p => p as Procedure)); } //least fixed point, starting with lower-bound currDependencies = new Dictionary <Procedure, Dependencies>(lowerBoundProcDependencies); //important note about * currDependencies.Iter(kv => { Debug.Assert(!IsStub(kv.Key), string.Format("Stubs not {0} expected...did you run dependency.exe?", kv.Key)); currDependencies[kv.Key].Iter (v => currDependencies[kv.Key][v.Key].Remove(Utils.VariableUtils.NonDetVar)); } ); while (worklist.Count > 0) { //the insertions into the Worklist also has to be sorted topologically //TODO: handle the case for general recursion if (topSortedProcedures.Count() > 0) //HACK!: we had a sorted list { worklist = topSortedProcedures.Where(x => worklist.Contains(x)).ToList(); } var proc = worklist.Last(); //why last? worklist.Remove(proc); var impl = program.Implementations.SingleOrDefault(i => i.Name == proc.Name); if (impl == null) { continue; } Utils.LogStopwatch(sw, string.Format("Analyzing Procedure {0}", proc.Name), Analysis.Timeout); var newProg = Utils.CrossProgramUtils.ReplicateProgram(program, filename); // resolve dependencies, callGraph, impl, etc from program -> newProg RefineDependencyPerImplementation rdpi = new RefineDependencyPerImplementation(newProg, (Implementation)Utils.CrossProgramUtils.ResolveTopLevelDeclsAcrossPrograms(impl, program, newProg), Utils.CrossProgramUtils.ResolveDependenciesAcrossPrograms(upperBoundProcDependencies, program, newProg), Utils.CrossProgramUtils.ResolveDependenciesAcrossPrograms(currDependencies, program, newProg), stackBound, CallGraphHelper.ComputeCallGraph(newProg)); var tmp = Utils.CrossProgramUtils.ResolveDependenciesAcrossPrograms(rdpi.Run(), newProg, program)[proc]; Debug.Assert(tmp.Keys.All(v => proc.InParams.Contains(v) || proc.OutParams.Contains(v) || v is GlobalVariable)); Debug.Assert(tmp.Values.All(d => d.All(v => proc.InParams.Contains(v) || proc.OutParams.Contains(v) || v is GlobalVariable))); // the lower bound has changed, update the dependencies and add callers to WL if (!(tmp.Equals(currDependencies[proc]))) { worklist.AddRange(callGraph.Predecessors(proc)); currDependencies[proc] = tmp; } } }
public override Program VisitProgram(Program node) { //Console.WriteLine("Starting..."); //Console.ReadLine(); var orderedSCCs = CallGraphHelper.ComputeOrderedSCCs(callGraph); orderedSCCs.Reverse(); int numVisited = 0; ProcReadSetVisitor rsv = new ProcReadSetVisitor(); foreach (var scc in orderedSCCs) { foreach (var proc in scc) { var impl = node.Implementations.FirstOrDefault(i => i.Proc == proc); if (impl == null) { continue; } //Console.Write("Visiting: {0} ({1}/{2}) [{3} cmds, {4} vars]", impl.Name, ++numVisited, program.Implementations.Count(), impl.Blocks.Sum(b => b.Cmds.Count + 1), impl.LocVars.Count); Stopwatch s = Stopwatch.StartNew(); ManualResetEvent wait = new ManualResetEvent(false); Thread work = new Thread(new ThreadStart(() => { Visit(impl); wait.Set(); })); work.Start(); Boolean signal = wait.WaitOne(timeOut); s.Stop(); if (!signal) { work.Abort(); Console.WriteLine("Aborted due to timeout. Reverting to readSet"); // the worklist alg was interrupted, clear the worklist to be safe worklist.workList.Clear(); worklist.stateSpace.Clear(); GC.Collect(); // compute the read set instead rsv.currentProc = impl.Proc; rsv.Visit(impl); // turn it to dependencies (\forall r \in ReadSet: r <- ReadSet) ProcDependencies[impl.Proc] = new Dependencies(); rsv.ProcReadSet[impl.Proc].Iter(r => ProcDependencies[impl.Proc][r] = rsv.ProcReadSet[impl.Proc]); //ProcDependencies[impl.Proc].FixFormals(impl); } else { //Console.WriteLine(" {0} s", s.ElapsedMilliseconds / 1000.0); // maintain the readSet (for cases where the analysis is too long and we revert to readset) rsv.ProcReadSet[impl.Proc] = new VarSet(); rsv.ProcReadSet[impl.Proc].UnionWith(ProcDependencies[impl.Proc].Keys); ProcDependencies[impl.Proc].Values.Iter(vs => rsv.ProcReadSet[impl.Proc].UnionWith(vs)); } } foreach (var proc in scc) { var impl = node.Implementations.FirstOrDefault(i => i.Proc == proc); if (impl == null) { continue; } Analysis.PopulateTaintLog(impl, Utils.ExtractTaint(this)); } worklist.stateSpace.Clear(); if (numVisited % 25 == 0) { GC.Collect(); } } // Removed. Printing directly to screen can be too huge. //ProcDependencies.Iter(pd => Console.Out.WriteLine(pd.Key + " : " + pd.Value)); //node.Implementations.Iter(impl => Visit(impl)); // compute top down taint orderedSCCs.Reverse(); foreach (var scc in orderedSCCs) { //foreach (var proc in procEntryTDTaint.Keys) foreach (var proc in scc) { if (!procEntryTDTaint.ContainsKey(proc)) { continue; } var impl = program.Implementations.Single(i => i.Proc == proc); var entry = Utils.GetImplEntry(impl); if (!worklist.stateSpace.ContainsKey(entry)) { worklist.stateSpace[entry] = new Dependencies(); } if (worklist.stateSpace[entry].JoinWith(procEntryTDTaint[impl.Proc])) { worklist.Propagate(entry); Visit(impl); } } } // the top down taint was removed from ProcDependencies so it won't flow up, so add it back in now procExitTDTaint.Iter(pd => ProcDependencies[pd.Key].JoinWith(pd.Value)); // gathering the tainted scalar outputs for each procedure var taintedProcScalarOutputs = new Dictionary <Procedure, VarSet>(); ProcDependencies.Iter(pd => { var procedure = pd.Key; var impl = node.Implementations.FirstOrDefault(i => i.Proc == procedure); var dependencies = pd.Value; taintedProcScalarOutputs[procedure] = new VarSet(); foreach (var r in impl.OutParams) { if (r.TypedIdent.Type.IsInt && dependencies.ContainsKey(r) && (dependencies[r].Contains(Utils.VariableUtils.BottomUpTaintVar) || dependencies[r].Contains(Utils.VariableUtils.TopDownTaintVar))) { taintedProcScalarOutputs[procedure].Add(r); } } }); taintedProcScalarOutputs.Iter(t => Console.WriteLine("Tainted outputs for " + t.Key + " : " + t.Value)); procEntryTDTaint.Iter(pd => Console.WriteLine("Tainted inputs/globals for " + pd.Key + " : " + pd.Value)); // for now, before we finish running, we replace all implementation outputs with procedure outputs // TODO: in the future we should only use implementation inputs\outputs and lose the procedures overall ProcDependencies.Iter(pd => { var impl = node.Implementations.FirstOrDefault(i => i.Proc == pd.Key); pd.Value.FixFormals(impl); foreach (var o in impl.OutParams) { pd.Value.Remove(o); } }); return(node); }
static int Main(string[] args) { if (args.Length < 1) { Usage(); return(-1); } sw = new Stopwatch(); sw.Start(); CommandLineOptions.Install(new CommandLineOptions()); CommandLineOptions.Clo.RunningBoogieFromCommandLine = true; var boogieOptions = "/typeEncoding:m -timeLimit:20 -removeEmptyBlocks:0 /printModel:1 /printModelToFile:model.dmp /printInstrumented "; // /errorLimit:1"; //IMPORTANT: need the next to avoid crash while creating prover CommandLineOptions.Clo.Parse(boogieOptions.Split(' ')); //IMPORTANT: need these three to make use of UNSAT cores!! CommandLineOptions.Clo.UseUnsatCoreForContractInfer = true; //ROHIT CommandLineOptions.Clo.ContractInfer = true; //ROHIT CommandLineOptions.Clo.ExplainHoudini = true; #region Command line parsing statsFile = args[0] + ".csv"; string changeList = null; args.Where(x => x.StartsWith(CmdLineOptsNames.taint + ":")) .Iter(s => changeList = s.Split(':')[1]); args.Where(x => x.StartsWith(CmdLineOptsNames.dacMerged + ":")) .Iter(s => DacMerged = s.Split(':')[1]); args.Where(x => x.StartsWith(CmdLineOptsNames.depTainted + ":")) .Iter(s => DependencyTaint = s.Split(':')[1]); ConservativeTaint = changeList == null; DumpTaint = args.Any(x => x.ToLower() == CmdLineOptsNames.dumpTaint.ToLower()); DataOnly = args.Any(x => x.ToLower() == CmdLineOptsNames.dataOnly.ToLower()); BothDependencies = args.Any(x => x.ToLower() == CmdLineOptsNames.both.ToLower()) && !DataOnly; PrintStats = args.Any(x => x.ToLower() == CmdLineOptsNames.stats.ToLower() || x.StartsWith(CmdLineOptsNames.stats + ":")); args.Where(x => x.StartsWith(CmdLineOptsNames.stats + ":")) .Iter(s => statsFile = s.Split(':')[1]); SemanticDep = args.Any(x => x.Contains(CmdLineOptsNames.semanticDep)); Refine = args.Any(x => x == CmdLineOptsNames.refine || x.StartsWith(CmdLineOptsNames.refine + ":")); args.Where(x => x.StartsWith(CmdLineOptsNames.refine + ":")) .Iter(s => StackBound = int.Parse(s.Split(':')[1])); args.Where(x => x.StartsWith(CmdLineOptsNames.timeout + ":")) .Iter(s => Timeout = int.Parse(s.Split(':')[1])); args.Where(x => x.StartsWith(CmdLineOptsNames.inlineDepthDependency + ":")) .Iter(s => InlineDepth = int.Parse(s.Split(':')[1])); CoarseDiff = args.Any(x => x.Contains(CmdLineOptsNames.coarseDiff)); UseSummariesOnly = args.Any(x => x.Contains(CmdLineOptsNames.summaries)); SplitMapsWithAliasAnalysis = args.Any(x => x.StartsWith(CmdLineOptsNames.splitMapsWithAliasAnalysis)); // refined must have pruned dependencies Prune = Refine || args.Any(x => x.ToLower() == CmdLineOptsNames.prune.ToLower()); if (StackBound < 2) { throw new Exception("Argument k to /refine:k has to be > 1"); } ReadSet = args.Any(x => x.ToLower() == CmdLineOptsNames.readSet.ToLower()); noMinUnsatCore = args.Any(x => x.ToLower() == CmdLineOptsNames.noMinUnsatCore.ToLower()); AbstractNonTainted = args.Any(x => x.ToLower() == CmdLineOptsNames.abstractNonTainted.ToLower()); AnnotateDependencies = args.Any(x => x.ToLower() == CmdLineOptsNames.annotateDependencies.ToLower()); StripValueIs = args.Any(x => x.ToLower() == CmdLineOptsNames.stripValueIs.ToLower()); RefinedStmtTaintAnalysis = args.Any(x => x.ToLower() == CmdLineOptsNames.refinedStmtTaintAnalysis.ToLower()); if (args.Any(x => x.ToLower() == CmdLineOptsNames.debug.ToLower())) { Debugger.Launch(); } #endregion var filename = args[0]; if (!Utils.ParseProgram(filename, out program)) { Usage(); return(-1); } #region Cleanups Debug.Assert(program.Resolve() == 0 && program.Typecheck() == 0, "Initial program has errors."); //first thing is to prune based on callgraph, if receiving a change list if (changeList != null) { changeLog = PopulateChangeLog(changeList, program); HashSet <Procedure> changedProcs = new HashSet <Procedure>(); foreach (var tuple in changeLog) { changedProcs.Add(program.FindProcedure(tuple.Item2)); } //add an attribute to identify the changedProcs for later analysis changedProcs.Iter(p => p.AddAttribute("syntacticChangedProc", new string[] {})); program = new CallGraphBasedPruning(program, changedProcs).PruneProgram(); Debug.Assert(program.Resolve() == 0 && program.Typecheck() == 0, "After Callgraph pruning the program has errors."); } //second thing is to remove Stubs program = new source.ProcessStubs(program).EliminateStubs(); //cleanup assume value_is, as we are not printing a trace now //Dont cleanup value_is without this flag, SymDiff gets confused as it expects value_is for models if (StripValueIs) { (new Utils.RemoveValueIsAssumes()).Visit(program); } // create explicit variables for conditionals if (changeList != null) //only do this for taint analysis { if (!CoarseDiff) //only do this if we're not using coarse diff, since otherwise it does not matter. { (new Utils.AddExplicitConditionalVars()).Visit(program); } } //remove some HAVOC generated methods (should be in preprocess) //program.RemoveTopLevelDeclarations(x => (x is Procedure && ((Procedure)x).Name.Contains("HAVOC_memset"))); //program.RemoveTopLevelDeclarations(x => (x is Implementation && ((Implementation)x).Name.Contains("HAVOC_memset"))); //remove any forall axiom if (SplitMapsWithAliasAnalysis) { Console.WriteLine("Cleanup:Removing all background quantified axioms"); program.RemoveTopLevelDeclarations(x => (x is Axiom && ((Axiom)x).Expr is ForallExpr)); //rewrite map update M := M [x := y] --> M[x] := y (new Utils.RewriteSingletonMapUdates()).Visit(program); } #endregion //inline before proceeding if (InlineDepth > 0) { new DependencyInliner(program, InlineDepth).InlineImplementations(); Utils.PrintProgram(program, "__after_inlining.bpl"); Utils.ParseProgram("__after_inlining.bpl", out program); } if (SplitMapsWithAliasAnalysis) { var s = new SplitHeapUsingAliasAnalysis(program, filename); program = s.Run(); //the program is overwritten, no more use of return(0); } CallGraphHelper.WriteCallGraph(filename + ".cg", CallGraphHelper.ComputeCallGraph(program)); Dictionary <string, HashSet <int> > sourceLines = new Dictionary <string, HashSet <int> >(); program.Implementations.Iter(i => i.Blocks.Iter(b => { var sourceFile = Utils.AttributeUtils.GetSourceFile(b); if (sourceFile != null) { if (!sourceLines.ContainsKey(sourceFile)) { sourceLines[sourceFile] = new HashSet <int>(); } Utils.AttributeUtils.GetSourceLines(b).Iter(x => sourceLines[sourceFile].Add(x)); } })); // remove redundant cmds //program.Implementations.Iter(i => i.Blocks.Iter(b => b.Cmds.RemoveAll(c => c is AssertCmd))); Analysis.DacSimplifier = new DacBasedSimplification(program, null); if (DacMerged != null) { Program mergedProgram; Utils.ParseProgram(DacMerged, out mergedProgram); Analysis.DacSimplifier = new DacBasedSimplification(program, mergedProgram); if (mergedProgram == null) { Console.WriteLine("[Error] Merged file not found: {0}", DacMerged); Environment.Exit(-1); } mergedProgram.Resolve(); mergedProgram.Typecheck(); DacSimplifier.Start(); } RunAnalysis(filename, program); #region Display and Log DumpTaintedLinesToCsv(taintLog, string.IsNullOrEmpty(DacMerged) ? "tainted.csv" : "tainted.dac.csv"); Utils.DisplayHtmlHelper displayHtml; if (DependencyTaint != null) { List <Tuple <string, string, int> > dependencyTaintedLines = Utils.StatisticsHelper.ReadCSVOutput(DependencyTaint); displayHtml = new Utils.DisplayHtmlHelper(changeLog, taintLog, dependenciesLog, taintedModSetLog, dependencyTaintedLines); } else { displayHtml = new Utils.DisplayHtmlHelper(changeLog, taintLog, dependenciesLog, taintedModSetLog); } displayHtml.GenerateHtmlOutput(); if (PrintStats) { Utils.StatisticsHelper.GenerateCSVOutput(statsFile, statsLog); Console.WriteLine("Statistics generated in " + statsFile); } if (DumpTaint) { Utils.StatisticsHelper.GenerateCSVOutput(args[0] + ".taint", taintLog); } Dictionary <string, HashSet <int> > oldSourceLines = new Dictionary <string, HashSet <int> >(sourceLines); sourceLines.Clear(); program.Implementations.Iter(i => i.Blocks.Iter(b => { var sourceFile = Utils.AttributeUtils.GetSourceFile(b); if (sourceFile != null) { if (!sourceLines.ContainsKey(sourceFile)) { sourceLines[sourceFile] = new HashSet <int>(); } //sourceLines[sourceFile].Add(Utils.AttributeUtils.GetSourceLine(b)); Utils.AttributeUtils.GetSourceLines(b).Iter(x => sourceLines[sourceFile].Add(x)); } })); if (changeList != null) { // print number of tainted lines var taintedLines = new HashSet <Tuple <string, int> >(taintLog.Select(x => new Tuple <string, int>(x.Item1, x.Item3))); Console.WriteLine("#Orig lines, #Tainted lines, #Lines after abstractNonTainted: {0}, {1}, {2}", oldSourceLines.Sum(fl => fl.Value.Count), taintedLines.Count(), sourceLines.Sum(fl => fl.Value.Count)); if (taintedLines.Count() == 0) { Utils.PrintError("WARNING: Result may be inaccurate as the #tainted lines is 0"); } } #endregion sw.Stop(); return(0); }