private void BeginTestInternal(Options options)
{
try
{
if (this.testAsyncResults == null)
this.testAsyncResults = new Dictionary<string, IAsyncResult>();
if (this.waitHandles == null)
this.waitHandles = new WaitHandle[this.maxWaitHandles];
var index = nbWaitHandles;
if (index == waitHandles.Length)
{
index = WaitHandle.WaitAny(waitHandles, waitHandles.Length * SingleTestMaxWait);
Assert.AreNotEqual(index, WaitHandle.WaitTimeout, "Previous tests timed out");
this.nbWaitHandles--;
}
Exception dummyOutException;
string dummyOutString;
var asyncResult = this.actionDelegate.BeginInvoke(options, out dummyOutException, out dummyOutString, null, null);
this.testAsyncResults.Add(options.TestName, asyncResult);
this.waitHandles[index] = asyncResult.AsyncWaitHandle;
this.nbWaitHandles++;
Console.WriteLine(this.BeginMessage);
}
catch (Exception e)
{
Console.WriteLine("EXCEPTION: {0}", e.Message);
Assert.Fail("Exception caught");
}
}
private void BeginTestInternal(Options options)
{
try
{
if (testAsyncResults == null)
testAsyncResults = new Dictionary<string, IAsyncResult>();
if (waitHandles == null)
waitHandles = new WaitHandle[maxWaitHandles];
var index = nbWaitHandles;
if (index == waitHandles.Length)
{
index = WaitHandle.WaitAny(waitHandles, waitHandles.Length * SingleTestMaxWait);
Assert.NotEqual(index, WaitHandle.WaitTimeout);
nbWaitHandles--;
}
Exception dummyOutException;
string dummyOutString;
var asyncResult = actionDelegate.BeginInvoke(options, out dummyOutException, out dummyOutString, null, null);
testAsyncResults.Add(options.TestName, asyncResult);
waitHandles[index] = asyncResult.AsyncWaitHandle;
nbWaitHandles++;
testOutputHelper.WriteLine(this.BeginMessage);
}
catch (Exception e)
{
testOutputHelper.WriteLine("EXCEPTION: {0}", e.Message);
Assert.True(false, "Exception caught");
}
}
internal static string Rewrite(string absoluteSourceDir, string absoluteBinary, Options options)
{
var referencedir = options.MakeAbsolute(Path.Combine(ReferenceDirRoot, options.BuildFramework));
var contractreferencedir = options.MakeAbsolute(Path.Combine(ContractReferenceDirRoot, options.ContractFramework));
var absoluteSourcedir = Path.GetDirectoryName(absoluteBinary);
var absoluteSource = absoluteBinary;
var libPathsString = FormLibPaths(absoluteSourceDir, contractreferencedir, options);
var targetName = options.TestName + ".rw" + Path.GetExtension(absoluteBinary);
var binDir = options.UseBinDir ? Path.Combine(Path.Combine(absoluteSourcedir, "bin"), options.BuildFramework) : absoluteSourcedir;
var targetfile = Path.Combine(binDir, targetName);
Console.WriteLine("Rewriting '{0}' to '{1}'", absoluteBinary, targetfile);
if (!Directory.Exists(binDir))
{
Directory.CreateDirectory(binDir);
}
var exitCode = RunProcess(binDir, options.MakeAbsolute(FoxtrotExe),
String.Format("/out:{0} -nobox -libpaths:{3} -libpaths:{4} {5} {1} {2}", targetfile, options.FoxtrotOptions, absoluteSource, referencedir, contractreferencedir, libPathsString),
options.TestName);
if (exitCode != 0)
{
return null;
}
else
{
if (options.UseBinDir)
{
File.Copy(targetfile, Path.Combine(absoluteSourcedir, targetName), true);
return Path.Combine(absoluteSourcedir, targetName);
}
}
return targetfile;
}
private static void WriteRSPFile(string dir, Options options, string args)
{
using (var file = new StreamWriter(Path.Combine(dir, options.TestName + ".rsp")))
{
file.WriteLine(args);
file.Close();
}
}
public void ExtractorFailures()
{
var options = new Options(this.TestContext);
options.ContractFramework = @".NetFramework\v4.0";
options.BuildFramework = @".NetFramework\v4.0";
options.FoxtrotOptions = options.FoxtrotOptions + " /nologo /verbose:4 /iw:-";
TestDriver.BuildExtractExpectFailureOrWarnings(options);
}
public void AsyncTestsWithDotNet45()
{
var options = new Options(this.TestContext);
options.FoxtrotOptions = options.FoxtrotOptions + String.Format(" /throwonfailure /rw:{0}.exe,TestInfrastructure.RewriterMethods", Path.GetFileNameWithoutExtension(options.TestName));
options.BuildFramework = @".NETFramework\v4.5";
options.ContractFramework = @".NETFramework\v4.0";
options.UseTestHarness = true;
TestDriver.BuildRewriteRun(options);
}
internal static object Rewrite(string absoluteSourceDir, string absoluteBinary, Options options, bool alwaysCapture = false)
{
string referencedir;
if (Path.IsPathRooted(options.ReferencesFramework))
{
referencedir = options.ReferencesFramework;
}
else
{
referencedir = options.MakeAbsolute(Path.Combine(ReferenceDirRoot, options.ReferencesFramework));
}
var contractreferencedir = options.MakeAbsolute(Path.Combine(ContractReferenceDirRoot, options.ContractFramework));
var absoluteSourcedir = Path.GetDirectoryName(absoluteBinary);
var absoluteSource = absoluteBinary;
var libPathsString = FormLibPaths(absoluteSourceDir, contractreferencedir, options);
var targetName = options.TestName + ".rw" + Path.GetExtension(absoluteBinary);
var binDir = options.UseBinDir ? Path.Combine(Path.Combine(absoluteSourcedir, "bin"), options.BuildFramework) : absoluteSourcedir;
var targetfile = Path.Combine(binDir, targetName);
Console.WriteLine("Rewriting '{0}' to '{1}'", absoluteBinary, targetfile);
if (!Directory.Exists(binDir))
{
Directory.CreateDirectory(binDir);
}
var optionString = String.Format("/out:{0} -nobox -libpaths:{3} {4} {1} {2}", targetfile, options.FoxtrotOptions, absoluteSource, referencedir, libPathsString);
if (absoluteBinary.EndsWith("mscorlib.dll"))
optionString = String.Format("/platform:{0} {1}", absoluteBinary, optionString);
var capture = RunProcessAndCapture(binDir, options.GetFullExecutablePath(FoxtrotExe), optionString, options.TestName);
if (capture.ExitCode != 0)
{
if (options.MustSucceed)
{
Assert.AreEqual(0, capture.ExitCode, "{0} returned an errorcode of {1}.", FoxtrotExe, capture.ExitCode);
}
return capture;
}
else
{
if (alwaysCapture) return capture;
if (options.UseBinDir)
{
var fileName = Path.Combine(absoluteSourcedir, targetName);
if (File.Exists(fileName))
{
try
{
File.SetAttributes(fileName, FileAttributes.Normal);
}
catch { }
}
File.Copy(targetfile, fileName, true);
return fileName;
}
}
return targetfile;
}
internal static string RewriteAndVerify(string sourceDir, string binary, Options options)
{
if (!Path.IsPathRooted(sourceDir)) { sourceDir = options.MakeAbsolute(sourceDir); }
if (!Path.IsPathRooted(binary)) { binary = options.MakeAbsolute(binary); }
string target = Rewrite(sourceDir, binary, options);
if (target != null)
{
PEVerify(target, options);
}
return target;
}
internal static int PEVerify(string assemblyFile, Options options)
{
var peVerifyPath = options.MakeAbsolute(Path.Combine(ToolsRoot, String.Format(@"{0}\peverify.exe", options.BuildFramework)));
assemblyFile = options.MakeAbsolute(assemblyFile);
var path = Path.GetDirectoryName(assemblyFile);
var file = Path.GetFileName(assemblyFile);
if (file == "mscorlib.dll") return -1; // peverify returns 0 for mscorlib without verifying.
var exitCode = RunProcess(path, peVerifyPath, "/unique \"" + file + "\"", "repro");
return exitCode;
}
// We have no control on the order of the tests, so we make sure
// to always call Begin before End
public void BeginTest(Options options)
{
if (this.skipTest(options))
return;
this.beginTestsProcessed = true;
if (this.orderReversed)
this.EndTestInternal(options);
else
this.BeginTestInternal(options);
}
public static void BuildAndAnalyze1Slicing(ITestOutputHelper testOutputHelper, Options options)
{
var output = Output.ConsoleOutputFor(testOutputHelper, options.TestName);
try
{
BuildAndAnalyze1Slicing(options, output.Item1);
}
finally
{
testOutputHelper.WriteLine(output.Item2.ToString());
}
}
internal static void Clousot1Slicing(string absoluteSourceDir, string absoluteBinary, Options options, Output output)
{
var referencedir = options.MakeAbsolute(Path.Combine(ReferenceDirRoot, options.BuildFramework));
var contractreferencedir = options.MakeAbsolute(Path.Combine(ContractReferenceDirRoot, options.ContractFramework));
var absoluteBinaryDir = Path.GetDirectoryName(absoluteBinary);
var absoluteSource = absoluteBinary;
var libPathsString = FormLibPaths(contractreferencedir, options) + " /libpaths:.";
var args = String.Format("{0} -cci1 /regression /define:cci1only;clousot1 -framework:{4} -libpaths:{2} {3} {1}", options.ClousotOptions, absoluteSource, referencedir, libPathsString, options.Framework);
if (options.Fast || System.Diagnostics.Debugger.IsAttached)
{
output.WriteLine("Calling NewCCI2Driver.Main with: {0}", args);
// Use output to avoid Clousot from closing the Console
Assert.AreEqual(0, Microsoft.Research.CodeAnalysis.NewCCI2Driver.Main(args.Split(' '), output));
}
else
RunProcess(absoluteBinaryDir, options.GetFullExecutablePath(Clousot2SlicingExe), args, output);
}
public static void BuildAndAnalyze(ITestOutputHelper testOutputHelper, Options options)
{
var output = Output.ConsoleOutputFor(testOutputHelper, options.TestName);
try
{
string absoluteSourceDir;
var target = Build(options, "/d:CLOUSOT1", output.Item1, out absoluteSourceDir);
if (target != null)
{
Clousot(absoluteSourceDir, target, options, output.Item1);
}
}
finally
{
testOutputHelper.WriteLine(output.Item2.ToString());
}
}
internal static void Clousot(string absoluteSourceDir, string absoluteBinary, Options options, string testName, int testindex, Output output)
{
var referencedir = options.MakeAbsolute(Path.Combine(ReferenceDirRoot, options.BuildFramework));
var contractreferencedir = options.MakeAbsolute(Path.Combine(ContractReferenceDirRoot, options.ContractFramework));
var absoluteBinaryDir = Path.GetDirectoryName(absoluteBinary);
var absoluteSource = absoluteBinary;
var libPathsString = FormLibPaths(contractreferencedir, options);
var args = string.Format("{0} /regression /define:cci1only;clousot1 -framework:{4} -libpaths:{2} {3} {1}", options.ClousotOptions, absoluteSource, referencedir, libPathsString, options.Framework);
WriteRSPFile(absoluteBinaryDir, testName, args);
if (options.Fast || Debugger.IsAttached)
{
output.WriteLine("Calling CCI1Driver.Main with: {0}", args);
// Use output to avoid Clousot from closing the Console
Assert.Equal(0, Microsoft.Research.CodeAnalysis.CCI1Driver.Main(args.Split(' '), output));
}
else
{
RunProcess(absoluteBinaryDir, options.GetFullExecutablePath(ClousotExe), args, output, testName);
}
}
[Ignore()] // Old Roslyn bits are not compatible with CCRewrite. Test (and old binaries) should be removed in the next iteration.
public void BuildRewriteRunFromSourcesRoslynV45()
{
var options = new Options(this.TestContext);
options.IsLegacyRoslyn = true;
options.FoxtrotOptions = options.FoxtrotOptions + String.Format(" /throwonfailure /rw:{0}.exe,TestInfrastructure.RewriterMethods", Path.GetFileNameWithoutExtension(options.TestName));
options.BuildFramework = @"Roslyn\v4.5";
options.ReferencesFramework = @".NetFramework\v4.5";
options.ContractFramework = @".NETFramework\v4.0";
options.UseTestHarness = true;
TestDriver.BuildRewriteRun(options);
}
private void EndTestInternal(Options options)
{
Assert.IsNotNull(this.testAsyncResults, "Begin part of the test not selected");
IAsyncResult asyncResult;
if (!this.testAsyncResults.TryGetValue(options.TestName, out asyncResult))
Assert.Fail("Begin part of the test not run");
this.testAsyncResults.Remove(options.TestName);
Assert.IsTrue(asyncResult.AsyncWaitHandle.WaitOne(SingleTestMaxWait), "Test timed out");
Exception exceptionThrown;
string dataReceived;
this.actionDelegate.EndInvoke(out exceptionThrown, out dataReceived, asyncResult);
Console.WriteLine();
Console.WriteLine("This test case was performed {0}synchronously", asyncResult.CompletedSynchronously ? "" : "a");
Console.WriteLine();
Console.Write(dataReceived);
if (exceptionThrown != null)
throw exceptionThrown;
}
private void ActionAsIsolated(Options options, out Exception exceptionThrown, out string dataReceived)
{
using (var stringWriter = new StringWriter())
{
var output = new Output(String.Format("Isolated::{0}", options.TestName), stringWriter);
exceptionThrown = null;
try
{
this.action(options, output);
}
catch (Exception e)
{
exceptionThrown = e;
}
dataReceived = stringWriter.ToString();
}
}
private Options GrabTestOptions(string testGroupName)
{
var options = new Options(testGroupName, TestContext);
CurrentGroupInfo = options.Group;
return options;
}
private static bool SkipForCCI2(Options options) { return options.SkipForCCI2; }
public void RewriteFrameworkDlls40()
{
var options = new Options(this.TestContext);
var framework = @".NetFramework\v4.0";
options.ContractFramework = framework;
var dllpath = FrameworkBinariesToRewritePath + framework;
options.BuildFramework = options.MakeAbsolute(dllpath);
var extrareferencedir = options.MakeAbsolute(TestDriver.ReferenceDirRoot);
options.LibPaths.Add(extrareferencedir);
options.FoxtrotOptions = options.FoxtrotOptions + " /verbose:4";
options.UseContractReferenceAssemblies = false;
if (File.Exists(options.MakeAbsolute(Path.Combine(dllpath, options.SourceFile))))
{
TestDriver.RewriteBinary(options, dllpath);
}
}
private Options GrabTestOptions(string testGroupName)
{
var dataRow = TestContext.DataRow;
var sourceFile = Options.LoadString(dataRow, "Name");
var clousotOptions = Options.LoadString(dataRow, "Options");
var useContractReferenceAssemblies = Options.LoadBool(dataRow, "ContractReferenceAssemblies", false);
var useExe = Options.LoadBool(dataRow, "Exe", false);
var compilerOptions = Options.LoadString(dataRow, "CompilerOptions");
var references = Options.LoadList(dataRow, "References");
var libPaths = Options.LoadList(dataRow, "LibPaths");
var compilerCode = Options.LoadString(dataRow, "Compiler", "CS");
var skipForCCI2 = Options.LoadBool(dataRow, "SkipCCI2", false);
var skipSlicing = Options.LoadBool(dataRow, "SkipSlicing", false);
var options = new Options(
sourceFile: sourceFile,
clousotOptions: clousotOptions,
useContractReferenceAssemblies: useContractReferenceAssemblies,
useExe: useExe,
compilerOptions: compilerOptions,
references: references.ToArray(),
libPaths: libPaths.ToArray(),
compilerCode: compilerCode,
skipForCCI2: skipForCCI2,
skipSlicing: skipSlicing,
skipForNet35: false);
options.TestGroupName = testGroupName;
CurrentGroupInfo = options.Group;
return options;
}
public void RewriteQuickGraphData()
{
var options = new Options("QuickGraphData", (string)TestContext.DataRow["FoxtrotOptions"], TestContext);
options.FoxtrotOptions = options.FoxtrotOptions + " /verbose:4";
options.Delete(@"Foxtrot\Tests\QuickGraph\QuickGraphBinaries\QuickGraph.Contracts.dll");
TestDriver.RewriteAndVerify("", @"Foxtrot\Tests\Quickgraph\QuickGraphBinaries\Quickgraph.Data.dll", options);
}
public void RewriteQuickGraphDataOOB()
{
var options = new Options("QuickGraphDataOOB", (string)TestContext.DataRow["FoxtrotOptions"], TestContext);
options.LibPaths.Add(@"Foxtrot\Tests\QuickGraph\QuickGraphBinaries\Contracts"); // subdirectory containing contracts.
options.FoxtrotOptions = options.FoxtrotOptions + " /verbose:4";
TestDriver.RewriteAndVerify("", @"Foxtrot\Tests\Quickgraph\QuickGraphBinaries\Quickgraph.Data.dll", options);
}
public void RewriteQuickGraph()
{
var options = new Options("QuickGraph", (string)TestContext.DataRow["FoxtrotOptions"], TestContext);
TestDriver.RewriteAndVerify("", @"Foxtrot\Tests\Quickgraph\QuickGraphBinaries\Quickgraph.dll", options);
}
public void BuildRewriteFromSources45WithPublicSurfaceOnly()
{
var options = new Options(this.TestContext);
// For testing purposes you can remove /publicsurface and see what happen. Part of the tests should fail!
//options.FoxtrotOptions = options.FoxtrotOptions + String.Format("/throwonfailure /rw:{0}.exe,TestInfrastructure.RewriterMethods", Path.GetFileNameWithoutExtension(options.TestName));
options.FoxtrotOptions = options.FoxtrotOptions + String.Format(" /publicsurface /throwonfailure /rw:{0}.exe,TestInfrastructure.RewriterMethods", Path.GetFileNameWithoutExtension(options.TestName));
options.BuildFramework = @".NETFramework\v4.5";
options.ContractFramework = @".NETFramework\v4.0";
options.UseTestHarness = true;
TestDriver.BuildRewriteRun(options);
}
private static void EnsureService(Options options)
{
lock (serviceProcessLock) // prevent the service to be run twice at the same time
{
if (serviceProcess == null)
StartService(options);
Assert.IsFalse(serviceProcess.HasExited, "Service needed but service process already exited");
}
}
// [TestMethod] no longer needed as we don't use the 2008 solution anymore.
public void CheckOOBCProjectSources()
{
var v9Sources = new[]{
@"Microsoft.Research\Contracts\Microsoft.VisualBasic.Compatibility\Microsoft.VisualBasic.Compatibility.csproj",
@"Microsoft.Research\Contracts\Microsoft.VisualBasic.Contracts\Microsoft.VisualBasic.Contracts.csproj",
@"Microsoft.Research\Contracts\mscorlib\mscorlib.Contracts.csproj",
@"Microsoft.Research\Contracts\System\System.Contracts.csproj",
@"Microsoft.Research\Contracts\System.Configuration\System.Configuration.csproj",
@"Microsoft.Research\Contracts\System.Configuration.Install\System.Configuration.Install.csproj",
@"Microsoft.Research\Contracts\System.Core.Contracts\System.Core.Contracts.csproj",
@"Microsoft.Research\Contracts\System.Data\System.Data\System.Data.csproj",
@"Microsoft.Research\Contracts\System.Drawing\System.Drawing.csproj",
@"Microsoft.Research\Contracts\System.Security\System.Security.csproj",
@"Microsoft.Research\Contracts\System.Web\System.Web.csproj",
@"Microsoft.Research\Contracts\System.Windows\System.Windows.csproj",
@"Microsoft.Research\Contracts\System.Windows.Browser\System.Windows.Browser.csproj",
@"Microsoft.Research\Contracts\System.Windows.Forms\Windows.Forms\System.Windows.Forms.csproj",
@"Microsoft.Research\Contracts\System.Xml\System.Xml.csproj",
@"Microsoft.Research\Contracts\System.Xml.Linq\System.Xml.Linq.csproj",
@"Microsoft.Research\Contracts\WindowsBase\WindowsBase.csproj",
};
var v10Sources = new[]{
@"Microsoft.Research\Contracts\Microsoft.VisualBasic.Compatibility\Microsoft.VisualBasic.Compatibility10.csproj",
@"Microsoft.Research\Contracts\Microsoft.VisualBasic.Contracts\Microsoft.VisualBasic.Contracts10.csproj",
@"Microsoft.Research\Contracts\mscorlib\mscorlib.Contracts10.csproj",
@"Microsoft.Research\Contracts\System\System.Contracts10.csproj",
@"Microsoft.Research\Contracts\System.Configuration\System.Configuration10.csproj",
@"Microsoft.Research\Contracts\System.Configuration.Install\System.Configuration.Install10.csproj",
@"Microsoft.Research\Contracts\System.Core.Contracts\System.Core.Contracts10.csproj",
@"Microsoft.Research\Contracts\System.Data\System.Data\System.Data10.csproj",
@"Microsoft.Research\Contracts\System.Drawing\System.Drawing10.csproj",
@"Microsoft.Research\Contracts\System.Security\System.Security10.csproj",
@"Microsoft.Research\Contracts\System.Web\System.Web10.csproj",
@"Microsoft.Research\Contracts\System.Windows\System.Windows10.csproj",
@"Microsoft.Research\Contracts\System.Windows.Browser\System.Windows.Browser10.csproj",
@"Microsoft.Research\Contracts\System.Windows.Forms\Windows.Forms\System.Windows.Forms10.csproj",
@"Microsoft.Research\Contracts\System.Xml\System.Xml10.csproj",
@"Microsoft.Research\Contracts\System.Xml.Linq\System.Xml.Linq10.csproj",
@"Microsoft.Research\Contracts\WindowsBase\WindowsBase10.csproj",
@"Microsoft.Research\Contracts\System.Numerics\System.Numerics.csproj"
};
for (int i = 0; i < v9Sources.Length; i++)
{
var opt = new Options(TestContext);
CheckSourceFiles(opt.MakeAbsolute(v9Sources[i]), opt.MakeAbsolute(v10Sources[i]));
}
}
private static void StartService(Options options)
{
if (serviceProcess != null)
StopService();
// First make sure another instance is not already running (because we don't know which version is running)
foreach (var process in Process.GetProcessesByName(Path.GetFileNameWithoutExtension(ClousotServiceHostExe)))
{
process.CloseMainWindow();
if (!process.WaitForExit(1000))
process.Kill();
}
var serviceHostDir = options.MakeAbsolute(Path.GetDirectoryName(ClousotServiceHostExe));
// note: we do not want to use ClousotServiceHostExe from the deployment directory because the app.config will be missing
serviceProcess = StartServiceProcess(serviceHostDir, options.MakeAbsolute(ClousotServiceHostExe), "", Output.Ignore);
}
/// <summary>
/// Creates <see cref="Options"/> instance with default values suitable for testing roslyn-based compiler.
/// </summary>
/// <param name="compilerVersion">Should be the same as a folder name in the /Imported/Roslyn folder.</param>
private void CreateRoslynOptions(Options options, string compilerVersion)
{
// For VS14RC+ version compiler name is the same Csc.exe, and behavior from async/iterator perspective is similar
// to old compiler as well. That's why IsLegacyRoslyn should be false in this test case.
options.IsLegacyRoslyn = false;
options.FoxtrotOptions = options.FoxtrotOptions + String.Format(" /throwonfailure /rw:{0}.exe,TestInfrastructure.RewriterMethods", Path.GetFileNameWithoutExtension(options.TestName));
options.CompilerPath = string.Format(@"Roslyn\{0}", compilerVersion);
options.BuildFramework = @".NetFramework\v4.5";
options.ReferencesFramework = @".NetFramework\v4.5";
options.ContractFramework = @".NETFramework\v4.5";
options.UseTestHarness = true;
}
public void BuildRewriteRunFromSourcesRoslynVS14RC()
{
var options = new Options(this.TestContext);
// For VS14RC+ version compiler name is the same Csc.exe, and behavior from async/iterator perspective is similar
// to old compiler as well. That's why IsLegacyRoslyn should be false in this test case.
options.IsLegacyRoslyn = false;
options.FoxtrotOptions = options.FoxtrotOptions + String.Format(" /throwonfailure /rw:{0}.exe,TestInfrastructure.RewriterMethods", Path.GetFileNameWithoutExtension(options.TestName));
options.BuildFramework = @"Roslyn\VS14RC";
options.ReferencesFramework = @".NetFramework\v4.5";
options.ContractFramework = @".NETFramework\v4.0";
options.UseTestHarness = true;
TestDriver.BuildRewriteRun(options);
}