public SlicerWorker(
GeneralOptions options,
ISimpleLineWriter outputLineWriter,
ISlicerEnvironment <Local, Parameter, Method, Field, Property, Event, Type, Attribute, Assembly> environment
)
{
this.options = options;
this.environment = environment;
this.mdDecoder = environment.MetaDataDecoder;
this.contractDecoder = environment.ContractDecoder;
this.fieldsDB = new FieldsDB <Local, Parameter, Method, Field, Property, Event, Type, Attribute, Assembly>(this.mdDecoder);
this.output = new TextWriterOutputWithPrefix(outputLineWriter.AsTextWriter(), options, "[SlicerWorker] "); // TODO
var basicDriver = new BasicAnalysisDriver <Local, Parameter, Method, Field, Property, Event, Type, Attribute, Assembly, ILogOptions>(this.mdDecoder, this.contractDecoder, this.output, options);
this.driver = new OldAnalysisDriver <Local, Parameter, Method, Field, Property, Event, Type, Attribute, Assembly, ILogOptions, IMethodResult <SymbolicValue> >(basicDriver);
this.OptionsData = OptionsHashing.GetOptionsData(environment.MethodAnalyzers, environment.ClassAnalyzers, options);
EGraphStats.IncrementalJoin = options.incrementalEgraphJoin; // used in OptimisticHeapAnalysis
// TODO: initialize analysis needed by the slicer, but not more
// TODO: is that it?
}
public ClousotWorker(CallAnyClousotMainFunc callClousotMain, IWorkerId id, FList <string> baseArgs, IScheduler scheduler, ISimpleLineWriter output)
: base(id, baseArgs, scheduler)
{
Contract.Requires(id != null);
this.callClousotMain = callClousotMain;
this.output = output;
}
public static ISlicerWorker <Method, Field, Type, Assembly> Worker <Local, Parameter, Method, Field, Property, Event, Type, Attribute, Assembly>(
GeneralOptions options,
ISimpleLineWriter output,
ISlicerEnvironment <Local, Parameter, Method, Field, Property, Event, Type, Attribute, Assembly> environment
) where Type : IEquatable <Type>
where Method : IEquatable <Method>
{
return(new SlicerWorker <Local, Parameter, Method, Field, Property, Event, Type, Attribute, Assembly>(options, output, environment));
}
public static int Main(string[] args, ISimpleLineWriter lineWriter)
{
Contract.Requires(args != null);
var textWriter = lineWriter.AsTextWriter();
var outputFactory = new FullTextWriterOutputFactory <System.Compiler.Method, System.Compiler.AssemblyNode>(textWriter);
return(Main(args, outputFactory, null));
}
public static int Main(string[] args, ISimpleLineWriter lineWriter)
{
Contract.Requires(args != null);
var textWriter = lineWriter.AsTextWriter();
var outputFactory = new FullTextWriterOutputFactory<System.Compiler.Method, System.Compiler.AssemblyNode>(textWriter);
return Main(args, outputFactory, null);
}
public static int Main(string[] args, ISimpleLineWriter lineWriter)
{
Contract.Requires(args != null);
var textWriter = lineWriter.AsTextWriter();
var outputFactory = new FullTextWriterOutputFactory<MethodReferenceAdaptor, IAssemblyReference>(textWriter);
return Main(args, outputFactory, null);
}
public static int Main(string[] args, ISimpleLineWriter lineWriter)
{
Contract.Requires(args != null);
var textWriter = lineWriter.AsTextWriter();
var outputFactory = new FullTextWriterOutputFactory <MethodReferenceAdaptor, IAssemblyReference>(textWriter);
return(Main(args, outputFactory, null));
}
public static TextWriter AsTextWriter(this ISimpleLineWriter writer)
{
var writerAsFromTextWriter = writer as IFromTextWriter;
if (writerAsFromTextWriter != null)
{
return(writerAsFromTextWriter.OriginalTextWriter);
}
return(new SimpleLineWriter.ToTextWriter(writer));
}
public ClousotWorker(CallLocalClousotMainFunc <Method, Assembly> callClousotMain, IWorkerId id, FList <string> baseArgs, IScheduler scheduler, ISimpleLineWriter output, IDB db)
: base(id, baseArgs, scheduler)
{
Contract.Requires(id != null);
this.callClousotMain = callClousotMain;
this.output = output;
this.db = db;
}
private NewCCI2Driver(string[] args, ISimpleLineWriter output)
{
/* TODO: argsForWorker
* We currently keep all the arguments except the general arguments
* But we should also remove arguments like -select, -namespaceSelect, -typenameSelect, -memberNameSelect -analyzeFrom, -analyzeTo (or translate them)
*/
this.options = FilteringGeneralOptions.ParseCommandLineArguments(args, this.methodAnalyzers, this.classAnalyzers, out this.argsForWorker);
this.output = output;
}
public ClousotSlicer2(GeneralOptions options, ISimpleLineWriter output, Dictionary <string, IMethodAnalysis> methodAnalyzers, Dictionary <string, IClassAnalysis> classAnalyzers)
{
var env = Environment.For(options, methodAnalyzers, classAnalyzers);
this.disposableObjectAllocatedByThisHost.Add(env);
this.options = options;
this.methodAnalyzers = methodAnalyzers;
this.classAnalyzers = classAnalyzers;
this.worker = SlicerWorkerFactory.Worker(options, output, env);
}
public static void WriteLine(this ISimpleLineWriter writer, object value)
{
writer.WriteLine("{0}", value);
}
public static void WriteLine(this ISimpleLineWriter writer, string value)
{
writer.WriteLine("{0}", value);
}
public static void WriteLine(this ISimpleLineWriter writer)
{
writer.WriteLine("");
}
public ToMultithreadedTextWriter(ISimpleLineWriter lineWriter)
{
this.lineWriter = lineWriter;
}
public ToTextWriter(ISimpleLineWriter lineWriter)
{
this.lineWriter = lineWriter;
}
public RemoteClousotWorkerFactory(IScheduler scheduler, FList <string> args, ISimpleLineWriter output)
: base(scheduler, args)
{
this.output = output;
}
public ToMultithreadedTextWriter(ISimpleLineWriter lineWriter)
{
this.lineWriter = lineWriter;
}
public ToTextWriter(ISimpleLineWriter lineWriter)
{
this.lineWriter = lineWriter;
}
public static TextWriter AsMultithreadedTextWriter(this ISimpleLineWriter writer)
{
return(new SimpleLineWriter.ToMultithreadedTextWriter(writer));
}
public LocalClousotWorkerFactory(IScheduler scheduler, FList <string> args, ISimpleLineWriter output, IDB db)
: base(scheduler, args)
{
this.db = db;
this.output = output;
}
public Clousot2WorkerFactory(IScheduler scheduler, FList <string> args, ISimpleLineWriter output, IDB db)
: base(scheduler, args, output, db)
{
}
// Entry point
public static int Main(string[] args, ISimpleLineWriter output)
{
var start = DateTime.Now;
var slicingOptions = new SlicingOptions(args);
// we use it to parse the clousot options, and to split the work
var clousot = new NewCCI2Driver(slicingOptions.remainingArgs.ToArray(), output);
if (!clousot.CheckOptions())
{
return -1;
}
var errorCode = 0;
WorkerPool workerPool = null;
IQueue queue = null;
IWorkerFactory localWorkerFactory = null;
List<IWorkerFactory> remoteWorkerFactories = null;
Func<SliceDefinition, ISliceWorkResult> workOnSlice = null;
Dictionary<ISliceId, int> failedRegressions = null;
var localWorkers = slicingOptions.LocalWorkers;
var remoteWorkers = slicingOptions.RemoteWorkers;
var workers = localWorkers + remoteWorkers;
// If we have workers, we create a Worker factory
if (workers > 0)
{
IDB db;
if (remoteWorkers > 0 || clousot.options.useCache)
{
// Use Clousot Database
db = new StdDB(clousot.options);
}
else
{
// In-memory database
db = new MemorySingletonDB();
}
workerPool = new WorkerPool("Worker", canCancel: false);
queue = new FIFOQueue(workerPool, db);
// When a job is done, it choses which jobs to put in the queue (e.g., analyze the dependencies)
var scheduler =
clousot.options.InferObjectInvariantsOnlyForReadonlyFields?
new LazySchedulerForObjectInvariants(queue, db) : // use LazySchedulerForObjectInvariants for global fixpoint computation including object invariants
new LazyScheduler(queue, db); // use LazyScheduler for the global fixpoint computation
//var scheduler = new NoloopScheduler(queue);
// Usual cache
var clousotDB = slicingOptions.useDB ? db : null;
var argsForWorker = clousot.argsForWorker.ToFList();
if (localWorkers > 0)
{
if (slicingOptions.cci1)
localWorkerFactory = new Clousot1WorkerFactory(scheduler, argsForWorker, output, clousotDB);
else
localWorkerFactory = new Clousot2WorkerFactory(scheduler, argsForWorker, output, clousotDB);
// TODO: use a lighter version of ClousotMain since options are already checked here
}
if (remoteWorkers > 0) // so far 1 factory per remote worker but we can do better
{
// TODO: specifiy, for each address the number of workers
// We have a list, because we can have several addresses
remoteWorkerFactories = slicingOptions.serviceAddress.Select(addr => new Clousot2SWorkerFactory(scheduler, argsForWorker, output, addr)).ToList<IWorkerFactory>();
}
if (clousot.options.IsRegression)
{
failedRegressions = new Dictionary<ISliceId, int>();
}
// fail if any work fails
scheduler.OnWorkDone += (sliceId, returnCode) =>
{
if (errorCode >= 0)
{
if (returnCode < 0) // special error code, keep only one
{
errorCode = returnCode;
}
else
{
int prevValue;
if (clousot.options.IsRegression)
{
lock (failedRegressions)
{
Contract.Assume(failedRegressions != null);
if (failedRegressions.TryGetValue(sliceId, out prevValue))
{
output.WriteLine("[Regression] We already analyzed {0} with outcome {1}. Now we update the outcome to {2}", sliceId.Dll, prevValue, returnCode);
}
failedRegressions[sliceId] = returnCode;
}
}
errorCode += returnCode; // regression error count, additive
}
}
};
// What we do for each slice. Two things:
// 1. Register the slice in the db
// 2. Add to the queue (via the scheduler, who decides how to do it)
workOnSlice = sliceDef =>
{
var sliceId = db.RegisterSlice(sliceDef);
scheduler.FeedQueue(new ISliceId[] { sliceId });
return null;
};
}
ISlicerResult slicerResult = null;
if (slicingOptions.sliceFirst)
{
slicerResult = clousot.SplitWork(workOnSlice);
output.WriteLine("Slicing time: {0}", DateTime.Now - start);
}
if (workerPool != null)
{
if (localWorkerFactory != null)
for (var i = 0; i < localWorkers; i++)
workerPool.CreateWorker(localWorkerFactory);
if (remoteWorkerFactories != null)
foreach (var factory in remoteWorkerFactories)
workerPool.CreateWorker(factory);
}
if (!slicingOptions.sliceFirst)
{
slicerResult = clousot.SplitWork(workOnSlice);
output.WriteLine("Slicing time and thread creation time : {0}", DateTime.Now - start);
}
if (workerPool != null)
{
// workerPool != null ==> queue != null
Contract.Assume(queue != null);
workerPool.WaitAllAnd(queue.EmptyQueueWaitHandle);
// Something else can arrive at the queue, so we want to stop all of them
workerPool.StopAll();
}
if (slicerResult != null)
{
var errors = slicerResult.GetErrors();
if (errors.Any())
{
foreach (var errMessage in errors)
output.WriteLine(errMessage);
errorCode = errors.Count();
}
}
output.WriteLine("Total analysis time: {0}", DateTime.Now - start);
var returnValue = errorCode;
if (failedRegressions != null && clousot.options.IsRegression && errorCode >= 0)
{
returnValue = failedRegressions.Where(pair => pair.Value != 0).Select(pair => Math.Abs(pair.Value)).Sum();
}
#if DEBUG
if (clousot.options.IsRegression)
{
Console.WriteLine("[Regression] Returned value {0}", returnValue);
}
#endif
return returnValue;
}
private NewCCI2Driver(string[] args, ISimpleLineWriter output)
{
/* TODO: argsForWorker
* We currently keep all the arguments except the general arguments
* But we should also remove arguments like -select, -namespaceSelect, -typenameSelect, -memberNameSelect -analyzeFrom, -analyzeTo (or translate them)
*/
this.options = FilteringGeneralOptions.ParseCommandLineArguments(args, this.methodAnalyzers, this.classAnalyzers, out this.argsForWorker);
this.output = output;
}
// Entry point
public static int Main(string[] args, ISimpleLineWriter output)
{
var start = DateTime.Now;
var slicingOptions = new SlicingOptions(args);
// we use it to parse the clousot options, and to split the work
var clousot = new NewCCI2Driver(slicingOptions.remainingArgs.ToArray(), output);
if (!clousot.CheckOptions())
{
return(-1);
}
var errorCode = 0;
WorkerPool workerPool = null;
IQueue queue = null;
IWorkerFactory localWorkerFactory = null;
List <IWorkerFactory> remoteWorkerFactories = null;
Func <SliceDefinition, ISliceWorkResult> workOnSlice = null;
Dictionary <ISliceId, int> failedRegressions = null;
var localWorkers = slicingOptions.LocalWorkers;
var remoteWorkers = slicingOptions.RemoteWorkers;
var workers = localWorkers + remoteWorkers;
// If we have workers, we create a Worker factory
if (workers > 0)
{
IDB db;
if (remoteWorkers > 0 || clousot.options.useCache)
{
// Use Clousot Database
db = new StdDB(clousot.options);
}
else
{
// In-memory database
db = new MemorySingletonDB();
}
workerPool = new WorkerPool("Worker", canCancel: false);
queue = new FIFOQueue(workerPool, db);
// When a job is done, it choses which jobs to put in the queue (e.g., analyze the dependencies)
var scheduler =
clousot.options.InferObjectInvariantsOnlyForReadonlyFields?
new LazySchedulerForObjectInvariants(queue, db) : // use LazySchedulerForObjectInvariants for global fixpoint computation including object invariants
new LazyScheduler(queue, db); // use LazyScheduler for the global fixpoint computation
//var scheduler = new NoloopScheduler(queue);
// Usual cache
var clousotDB = slicingOptions.useDB ? db : null;
var argsForWorker = clousot.argsForWorker.ToFList();
if (localWorkers > 0)
{
if (slicingOptions.cci1)
{
localWorkerFactory = new Clousot1WorkerFactory(scheduler, argsForWorker, output, clousotDB);
}
else
{
localWorkerFactory = new Clousot2WorkerFactory(scheduler, argsForWorker, output, clousotDB);
}
// TODO: use a lighter version of ClousotMain since options are already checked here
}
if (remoteWorkers > 0) // so far 1 factory per remote worker but we can do better
{
// TODO: specifiy, for each address the number of workers
// We have a list, because we can have several addresses
remoteWorkerFactories = slicingOptions.serviceAddress.Select(addr => new Clousot2SWorkerFactory(scheduler, argsForWorker, output, addr)).ToList <IWorkerFactory>();
}
if (clousot.options.IsRegression)
{
failedRegressions = new Dictionary <ISliceId, int>();
}
// fail if any work fails
scheduler.OnWorkDone += (sliceId, returnCode) =>
{
if (errorCode >= 0)
{
if (returnCode < 0) // special error code, keep only one
{
errorCode = returnCode;
}
else
{
int prevValue;
if (clousot.options.IsRegression)
{
lock (failedRegressions)
{
Contract.Assume(failedRegressions != null);
if (failedRegressions.TryGetValue(sliceId, out prevValue))
{
output.WriteLine("[Regression] We already analyzed {0} with outcome {1}. Now we update the outcome to {2}", sliceId.Dll, prevValue, returnCode);
}
failedRegressions[sliceId] = returnCode;
}
}
errorCode += returnCode; // regression error count, additive
}
}
};
// What we do for each slice. Two things:
// 1. Register the slice in the db
// 2. Add to the queue (via the scheduler, who decides how to do it)
workOnSlice = sliceDef =>
{
var sliceId = db.RegisterSlice(sliceDef);
scheduler.FeedQueue(new ISliceId[] { sliceId });
return(null);
};
}
ISlicerResult slicerResult = null;
if (slicingOptions.sliceFirst)
{
slicerResult = clousot.SplitWork(workOnSlice);
output.WriteLine("Slicing time: {0}", DateTime.Now - start);
}
if (workerPool != null)
{
if (localWorkerFactory != null)
{
for (var i = 0; i < localWorkers; i++)
{
workerPool.CreateWorker(localWorkerFactory);
}
}
if (remoteWorkerFactories != null)
{
foreach (var factory in remoteWorkerFactories)
{
workerPool.CreateWorker(factory);
}
}
}
if (!slicingOptions.sliceFirst)
{
slicerResult = clousot.SplitWork(workOnSlice);
output.WriteLine("Slicing time and thread creation time : {0}", DateTime.Now - start);
}
if (workerPool != null)
{
// workerPool != null ==> queue != null
Contract.Assume(queue != null);
workerPool.WaitAllAnd(queue.EmptyQueueWaitHandle);
// Something else can arrive at the queue, so we want to stop all of them
workerPool.StopAll();
}
if (slicerResult != null)
{
var errors = slicerResult.GetErrors();
if (errors.Any())
{
foreach (var errMessage in errors)
{
output.WriteLine(errMessage);
}
errorCode = errors.Count();
}
}
output.WriteLine("Total analysis time: {0}", DateTime.Now - start);
var returnValue = errorCode;
if (failedRegressions != null && clousot.options.IsRegression && errorCode >= 0)
{
returnValue = failedRegressions.Where(pair => pair.Value != 0).Select(pair => Math.Abs(pair.Value)).Sum();
}
#if DEBUG
if (clousot.options.IsRegression)
{
Console.WriteLine("[Regression] Returned value {0}", returnValue);
}
#endif
return(returnValue);
}
public Clousot2SWorkerFactory(IScheduler scheduler, FList <string> args, ISimpleLineWriter output, string serviceAddress)
: base(scheduler, AddExtraArgs(args, serviceAddress), output)
{
}
