private int MainInternal(string name, string[] args, TextWriterAsync textWriter)
{
Contract.Requires(name != null);
Contract.Requires(args != null);
Contract.Requires(textWriter != null);
try
{
CloudotLogging.LogAnalysiRequest(name, args);
#if INTERNALCALL
return(CallClousotInternally(name, args, textWriter));
#else
return(CallClousotViaProcess(name, args, textWriter));
#endif
}
catch (ExitRequestedException e)
{
return(e.ExitCode);
}
catch (Exception e)
{
CloudotLogging.WriteLine("Got an exception. Reporting it to the client and aborting the session.");
// We need to catch all exception, otherwise the service is turned into an faulted state and remains unusable
textWriter.WriteLine("Cloudot: caught exception: {0}".PrefixWithCurrentTime(), e);
if (e.InnerException != null)
{
textWriter.WriteLine("Inner exception: {0}", e.InnerException);
}
return(-4444);
}
finally
{
CloudotLogging.WriteLine("Analysis of request #{0} done in {1}", this.myId, DateTime.Now - this.start);
textWriter.Close(); // make sure all pending output is flushed
}
}
[Pure] // Even if we change the file system...
public static bool TryRestoreAnalysisPackage(string packageFileName, string outputDir, out string[] clousotNormalizedParameters)
{
Contract.Requires(packageFileName != null);
Contract.Requires(outputDir != null);
Contract.Ensures(!Contract.Result <bool>() || Contract.ValueAtReturn(out clousotNormalizedParameters) != null);
clousotNormalizedParameters = null;
try
{
// Step 0: make sure the directory exists
DirectoryInfo outputDirInfo;
if (Directory.Exists(outputDir))
{
outputDirInfo = new DirectoryInfo(outputDir);
}
else
{
outputDirInfo = Directory.CreateDirectory(outputDir);
}
if (TryRestorePackage(packageFileName, outputDir))
{
return(true);
}
return(false);
}
catch (Exception e)
{
CloudotLogging.WriteLine("Something went wrong in restoring the package (exception of type {0}). Aborting.", e.GetType());
return(false);
}
}
private static string[] ForgeClousotCommandLineFromOutputFile(string outputDir)
{
try
{
var lines = File.ReadAllLines(Path.Combine(outputDir, CloudotInstructionsFileName));
var result = new List <string>();
var optionsWithPaths = GeneralOptions.GetClousotOptionsWithPaths().ToArray();
var optionsBounds = FindOptions(lines);
for (var i = optionsBounds.Item1; i < optionsBounds.Item2; i++)
{
var line = lines[i];
if (line.Length > 0)
{
var subLine = line.Substring(1);
if (optionsWithPaths.Any(optionName => subLine.StartsWith(optionName)))
{
line = ReplaceVolumesWithLocalRootPath(outputDir, line);
}
else if (IsPath(line))
{
line = ReplaceVolumesWithLocalRootPath(outputDir, line);
}
}
result.Add(line);
}
return(result.ToArray());
}
catch (Exception)
{
CloudotLogging.WriteLine("Something went wrong while getting the cmd line from dir {0}", outputDir);
return(null);
}
}
private void CreateProcessesAndPopulateTheQueue(int initialCount = INITIALPOOLSIZE)
{
Contract.Requires(initialCount >= 0);
lock (this.availableWorkers)
{
CloudotLogging.WriteLine("Creating {0} workers", initialCount);
for (var i = 0; i < initialCount; i++)
{
var processInfo = SetupProcessInfoToWait();
try
{
var exe = Process.Start(processInfo);
if (exe != null)
{
this.availableWorkers.Enqueue(exe);
}
else
{
CloudotLogging.WriteLine("Failed to create the Clousot process");
}
}
catch
{
CloudotLogging.WriteLine("Error in starting the Clousot/cccheck process for the pool");
}
}
}
}
private void Dispose(bool Disposing)
{
// We want to make sure there are no zombie processes left
CloudotLogging.WriteLine("Killing all the workers");
foreach (var exe in this.availableWorkers)
{
try
{
if (!exe.HasExited)
{
exe.Kill();
}
}
catch (Exception e)
{
CloudotLogging.WriteLine("[Debug] Exception raised while trying to kill the idle process {0} -- We just continue as it means that the OS has already removed the process", exe.Id);
CloudotLogging.WriteLine("[Debug] Exception {0}", e);
}
}
foreach (var exe in this.scheduledWorkers)
{
try
{
if (!exe.HasExited)
{
exe.Kill();
}
}
catch (Exception e)
{
CloudotLogging.WriteLine("Exception {0} raised while trying to kill an active process -- We just continue as it means that the process was already terminated", e.GetType());
}
}
}
public int AnalyzeUsingAnalysisPackage(string name, string packageLocation, string encodingWebName)
{
// Step 0: initialize the server
var textWriter = InitializeServiceInstance(new string[] { packageLocation }, encodingWebName);
// Step 1: Compute the location in the Semantic index
var semanticIndexLocation = ComputeSemanticIndexLocation(name);
// Step 1: restore the package
string[] clousotOptions;
if (FileTransfer.TryRestoreAnalysisPackage(packageLocation, semanticIndexLocation, out clousotOptions))
{
CloudotLogging.WriteLine("Package restored, command line on the server {0}", String.Join(" ", clousotOptions));
return(MainInternal(name, clousotOptions, textWriter));
}
return(-1);
}
private static bool TryRestorePackage(string fileName, string outputDir)
{
var start = DateTime.Now;
try
{
CloudotLogging.WriteLine("Restoring the analysis package {0} in {1}", fileName, outputDir);
using (var zipArchive = ZipFile.OpenRead(fileName))
{
foreach (var entry in zipArchive.Entries)
{
var destinationFileName = MakeOutputFileName(outputDir, entry.FullName);
// If the file was already unzipped, skip it
if (File.Exists(destinationFileName))
{
continue; // Skip the file
}
// Make sure the destination directory exists
var dir = Path.GetDirectoryName(destinationFileName);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
entry.ExtractToFile(destinationFileName);
}
}
}
catch (Exception e)
{
CloudotLogging.WriteLine("Something went wrong in restoring the file {0} (got exception of type {1}). Aborting.", fileName, e.GetType());
CloudotLogging.WriteLine("Exception: {0}", e);
return(false);
}
CloudotLogging.WriteLine("Done Restoring the analysis package ({0} elapsed)", DateTime.Now - start);
return(true);
}
// This Clousot as Service host started from the command line
// Usage: run it, and wait for readline to kill everything
// We need a service per box
public static void Main()
{
CloudotLogging.WriteLine("Starting Cloudot service...");
CloudotLogging.WriteLine();
try
{
using (var host = new ClousotServiceHost())
{
host.AddLogger(Console.Out.AsLineWriter()); // TODO: Replace the Console.Out with the equivalent in CloudotLogging
host.Open();
CloudotLogging.WriteLine(string.Format("The service is ready and listening at {0}", String.Join(", ", host.BaseAddresses)));
CloudotLogging.WriteLine("Press <ENTER> to terminate service.");
Console.ReadLine();
host.Close();
}
}
catch (TimeoutException timeProblem)
{
CloudotLogging.WriteLine(timeProblem.Message);
if (timeProblem.InnerException != null)
{
CloudotLogging.WriteLine(timeProblem.InnerException.Message);
}
Console.ReadLine();
}
catch (CommunicationException commProblem)
{
CloudotLogging.WriteLine(commProblem.Message);
if (commProblem.InnerException != null)
{
CloudotLogging.WriteLine(commProblem.InnerException.Message);
}
Console.ReadLine();
}
}
/// <summary>
/// Common initialization for the service.
/// Essentially writes down some logging string and getting the callback on which the service will respond
/// </summary>
/// <returns>The text writer object the service will write on</returns>
private TextWriterAsync InitializeServiceInstance(string[] args, string encodingWebName)
{
Contract.Requires(args != null);
// Update the request, and save some statistics
this.myId = RequestId++;
this.start = DateTime.Now;
if (this.OnMain != null)
{
this.OnMain(args);
}
// here is the magic: the callback object is recovered in the line below
var callbackLineWriter = OperationContext.Current.GetCallbackChannel <IVerySimpleLineWriter>();
// And now we create the output factory for Clousot
var textWriter = new TextWriterAsync(callbackLineWriter.AsTextWriter(encodingWebName));
CloudotLogging.WriteLine("Starting processing request #" + myId);
return(textWriter);
}
private void SendWorkToProcess(Process exeProcess, string[] args)
{
Contract.Requires(exeProcess != null);
Contract.Requires(args != null);
CloudotLogging.WriteLine("Dispatching the analysis to worker process {0}", exeProcess.Id);
var pipeName = CommonNames.GetPipeNameForCloudotWorker(exeProcess.Id);
var namedPipeClient = new NamedPipeClientStream(pipeName);
try
{
// Try to connect to the process via the pipe
namedPipeClient.Connect(200);
// Send the arguments -- our protocol is that once the clousot process received the arguments, it will proceed with the analysis, as usual
new BinaryFormatter().Serialize(namedPipeClient, args);
}
catch (Exception e)
{
CloudotLogging.WriteLine("Something went wrong while sending the arguments to process {0}{1}{2}", exeProcess.Id, Environment.NewLine, e.Message);
}
}
protected override void OnClosing()
{
CloudotLogging.WriteLine("Closing the service");
base.OnClosing();
}
protected override void OnAbort()
{
CloudotLogging.WriteLine("Aborting the service");
base.OnAbort();
}
protected override void OnClosed()
{
CloudotLogging.WriteLine("The service is now closed");
base.OnClosed();
}
public ClousotService()
{
CloudotLogging.WriteLine("[Debug] Inside Clousot Service constructor");
}
private int CallClousotViaProcess(string name, string[] args, TextWriter textwriter)
{
Contract.Requires(name != null);
Contract.Requires(args != null);
Contract.Requires(textwriter != null);
try
{
string outputFileName;
using (var outputFileOnServer = CloudotLogging.GetAFreshOutputFileName(name, out outputFileName))
{
CloudotLogging.WriteLine("Saving the output at {0}", outputFileName);
// Start the process with the info we specified
using (var exe = workersPool.GetAProcessFromTheQueue())
{
SendWorkToProcess(exe, args);
// Reads the standard error and pushes it to the
Task.Factory.StartNew(
() =>
{
while (!exe.HasExited)
{
var lineErr = exe.StandardError.ReadLine();
textwriter.WriteLine(lineErr);
outputFileOnServer.WriteLine(lineErr);
}
// var lineOut = exe.StandardOutput.ReadToEnd();
var lineOut = exe.StandardError.ReadToEnd();
textwriter.Write(lineOut);
outputFileOnServer.Write(lineOut);
});
// Reads the standard output
string line;
while (!exe.HasExited)
{
line = exe.StandardOutput.ReadLine();
textwriter.WriteLine(line);
outputFileOnServer.WriteLine(line);
}
line = exe.StandardOutput.ReadToEnd();
textwriter.WriteLine(line);
outputFileOnServer.WriteLine(line);
// line = exe.StandardError.ReadToEnd();
// textwriter.Write(line);
// outputFileOnServer.WriteLineAsync(line);
// Before returning, let's flush everything
textwriter.Flush();
outputFileOnServer.Flush();
//exe.WaitForExit(); // Why we need it?
return(exe.ExitCode);
}
}
}
catch (Exception e)
{
CloudotLogging.WriteLine("Some exception (of type {0}) happened during the analysis {1}", e.GetType(), this.myId);
return(-1);
}
finally
{
// do nothing
}
}