private NUnit.Framework.Internal.TestMethod BuildTestMethod(
NUnit.Framework.Internal.Test parentSuite,
TestCaseInformation testCase)
{
var testMethod = new TestMethod(
executeMethod, parentSuite, testCase.Method.DeclaringType.FullName, testCase.Method.Name)
{
Seed = _randomizer.Next()
};
var parms = new NUnit.Framework.Internal.TestCaseParameters(new object[] { testCase });
//CheckTestMethodAttributes(testMethod);
// DIRTY: NUnit's NUnitTestCaseBuilder class does collusion with the TestMethod class sharing internal field for the 'parms'.
// It private method sets the parms.
//CheckTestMethodSignature(testMethod, parms);
var t = typeof(NUnit.Framework.Internal.TestMethod);
var f = t.GetField("parms", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly);
f.SetValue(testMethod, parms);
parms.ApplyToTest(testMethod);
var name = string.Format(
"{0}({1})",
testCase.Name,
string.Join(",", testCase.Arguments.Select(Utilities.GetCSharpLanguageExpression)));
testMethod.Name = name;
testMethod.FullName = string.Format(
"{0}.{1}.{2}",
testCase.CategoryName,
testCase.Id,
name);
return(testMethod);
}
public static async Task ExecuteTestAsync(TestCaseInformation caseInfo)
{
Assert.IsTrue(caseInfo.Method.IsPublic && caseInfo.Method.IsStatic);
// Split current thread context.
await Task.Yield();
///////////////////////////////////////////////
// Step 1-1: Create translation context.
var translateContext = new TranslateContext(caseInfo.Method.DeclaringType.Assembly.Location, false);
// Step 1-2: Prepare target methods.
var targetTypes = new HashSet <string>(
new[] { caseInfo.Method }.
Concat(caseInfo.AdditionalMethods).
Select(method => method.DeclaringType).
Concat(caseInfo.AdditionalTypes).
Select(type => type.FullName).
Distinct());
var targetMethods = new[] { caseInfo.Method }.
Concat(caseInfo.AdditionalMethods).
GroupBy(method => method.Name).
ToDictionary(g => g.Key, g => g.ToArray());
var prepared = AssemblyPreparer.Prepare(
translateContext,
t => targetTypes.Contains(t.FriendlyName),
m => targetMethods.ContainsKey(m.Name));
// Step 1-3: Extract prepared target type/caseInfo.Method informations.
var targetType =
translateContext.Assembly.Modules
.First().Types
.First(t => t.FriendlyName == caseInfo.Method.DeclaringType.FullName);
var targetMethod =
targetType.DeclaredMethods
.First(m => m.Name == caseInfo.Method.Name);
var targetAdditionalMethods =
targetType.DeclaredMethods
.Where(m => (caseInfo.AdditionalMethods.FirstOrDefault(am => m.Name == am.Name) != null))
.ToArray();
// Step 1-4: Translate caseInfo.Method bodies.
var header = CodeTextWriter.Create(new StringWriter(), " ");
AssemblyWriter.InternalWriteHeader(
header, translateContext, prepared, false);
header.Flush();
var body = CodeTextWriter.Create(new StringWriter(), " ");
AssemblyWriter.InternalWriteSourceCode(
body, translateContext, prepared, DebugInformationOptions.Full, false);
body.Flush();
// Step 1-5: Write Visual C++ project file and Visual Studio Code launch config from template.
// Note: It's only debugging purpose. The test doesn't use.
var translatedPath = Path.GetFullPath(
Path.Combine(
Path.GetDirectoryName(caseInfo.Method.DeclaringType.Assembly.Location),
caseInfo.CategoryName,
caseInfo.Id,
caseInfo.Name));
var vcxprojTemplatePath = Path.Combine(translatedPath, "test.vcxproj");
await TestUtilities.CopyResourceToTextFileAsync(vcxprojTemplatePath, "test.vcxproj");
var launchTemplatePath = Path.Combine(translatedPath, ".vscode", "launch.json");
await TestUtilities.CopyResourceToTextFileAsync(launchTemplatePath, "launch.json");
// Step 1-6: Write source code into a file from template.
var expectedType = targetMethod.ReturnType;
var sourceCodeStream = new MemoryStream();
await TestUtilities.CopyResourceToStreamAsync(
sourceCodeStream,
// If the target caseInfo.Method result is void-type, we have to use void-type tolerant template.
(expectedType.IsVoidType || (caseInfo.Assert == TestCaseAsserts.CauseBreak))? "test_void.c" : "test.c");
sourceCodeStream.Position = 0;
var sourceCode = new StreamReader(sourceCodeStream);
var constants = caseInfo.Arguments.
Zip(targetMethod.Parameters, (arg, p) => new Constant(
p.ParameterName,
p.TargetType,
arg?.GetType(),
Utilities.GetCLanguageExpression(arg))
).
ToArray();
var argumentList = constants.
Select(constant => new Constant(
"_" + constant.SymbolName,
constant.TargetType,
null,
GetCLangaugeSafeConversionExpression(constant.TargetType, constant.ExpressionType, constant.SymbolName))).
ToArray();
var arguments = argumentList;
var locals = argumentList.
Select(argument =>
new Constant(
argument.SymbolName,
argument.TargetType,
argument.ExpressionType,
Utilities.GetCLanguageExpression(expectedType.InternalStaticEmptyValue))).
ToArray();
if (!(expectedType.IsVoidType || (caseInfo.Assert == TestCaseAsserts.CauseBreak)))
{
// VERY DIRTY:
constants = constants.Concat(new Constant[]
{
new Constant(
"expected",
expectedType,
caseInfo.Expected?.GetType(),
Utilities.GetCLanguageExpression(caseInfo.Expected)),
}).
ToArray();
arguments = constants.
Select(constant => new Constant(
"_" + constant.SymbolName,
constant.TargetType,
null,
GetCLangaugeSafeConversionExpression(constant.TargetType, constant.ExpressionType, constant.SymbolName))).
ToArray();
locals = arguments.
Select(argument =>
new Constant(
argument.SymbolName,
argument.TargetType,
argument.ExpressionType,
Utilities.GetCLanguageExpression(argument.TargetType.InternalStaticEmptyValue))).
Concat(new Constant[]
{
new Constant(
"_actual",
expectedType,
caseInfo.Expected?.GetType(),
Utilities.GetCLanguageExpression(expectedType.InternalStaticEmptyValue)),
}).
ToArray();
}
// Construct test definitions.
var expectedSymbolName = locals.
Any(entry => (entry.SymbolName == "_expected") && entry.TargetType.IsReferenceType) ?
"frame__._expected" :
"_expected";
var actualSymbolName = locals.
Any(entry => (entry.SymbolName == "_actual") && entry.TargetType.IsReferenceType) ?
"frame__._actual" :
"_actual";
var replaceValues = new Dictionary <string, object>
{
{ "testName", targetMethod.FriendlyName },
{ "type", targetMethod.ReturnType.CLanguageTypeName },
{ "body", body.Parent.ToString() },
{ "constants", string.Join(" ", constants.
Select(entry => string.Format("{0} {1} = {2};",
(entry.ExpressionType != null) ? Utilities.GetCLanguageTypeName(entry.ExpressionType) : entry.TargetType.CLanguageTypeName,
entry.SymbolName,
entry.Expression))) },
{ "locals", string.Join(" ", locals.
Where(entry => !entry.TargetType.IsReferenceType).
Select(entry => string.Format("{0} {1} = {2};",
entry.TargetType.CLanguageTypeName,
entry.SymbolName,
entry.Expression))) },
{ "frames", string.Join(" ", locals.
Where(entry => entry.TargetType.IsReferenceType).
Select(entry => string.Format("{0} {1};",
entry.TargetType.CLanguageTypeName,
entry.SymbolName))) },
{ "frameCount", locals.
Where(entry => entry.TargetType.IsClass).Count() },
{ "arguments", string.Join(" ", arguments.
Select(entry => string.Format("{0}{1} = {2};",
entry.TargetType.IsReferenceType ? "frame__." : string.Empty,
entry.SymbolName,
entry.Expression))) },
{ "actual", actualSymbolName },
{ "function", targetMethod.CLanguageFunctionName },
{ "argumentList", string.Join(", ", argumentList.
Select(arg => string.Format(
"{0}{1}",
arg.TargetType.IsReferenceType ? "frame__." : string.Empty,
arg.SymbolName))) },
{ "equality", GetCLanguageCompareExpression(expectedType, expectedSymbolName, actualSymbolName) },
{ "format", GetCLanguagePrintFormatFromType(targetMethod.ReturnType) },
{ "expectedExpression", GetCLanguagePrintArgumentExpression(expectedType, expectedSymbolName) },
{ "actualExpression", GetCLanguagePrintArgumentExpression(expectedType, actualSymbolName) },
};
var sourcePath = Path.Combine(translatedPath, "test.c");
var headerPath = Path.Combine(translatedPath, "test.h");
await Task.WhenAll(
TestUtilities.WriteTextFileAsync(sourcePath, sourceCode, replaceValues),
TestUtilities.WriteTextFileAsync(headerPath, header.Parent.ToString()));
///////////////////////////////////////////////
// Step 2: Test and verify result by real IL code at this runtime.
object rawResult;
switch (caseInfo.Assert)
{
case TestCaseAsserts.IgnoreValidateInvokeResult:
try
{
rawResult = caseInfo.Method.Invoke(null, caseInfo.Arguments);
}
catch
{
// ignore.
rawResult = null;
}
break;
case TestCaseAsserts.CauseBreak:
rawResult = null;
break;
default:
rawResult = caseInfo.Method.Invoke(null, caseInfo.Arguments);
Assert.AreEqual(caseInfo.Expected, rawResult);
break;
}
///////////////////////////////////////////////
// Step 3: Test compiled C source code and execute.
string sanitized = null;
try
{
var il2cRuntimeSourcePaths = new[] {
// Use combined runtime source. 5 times faster!
Path.Combine(il2cRuntimePath, "il2c_combined.c")
};
#if DEBUG
var executedResult = await GccDriver.CompileAndRunAsync(false, sourcePath, il2cRuntimePath);
#else
var executedResult = await GccDriver.CompileAndRunAsync(true, sourcePath, il2cRuntimePath);
#endif
sanitized = executedResult.Trim(' ', '\r', '\n');
}
catch (Exception ex)
{
if ((caseInfo.Assert == TestCaseAsserts.CauseBreak) && ex.Message.Contains("ExitCode=-2147483645"))
{
return;
}
throw;
}
Assert.IsFalse(caseInfo.Assert == TestCaseAsserts.CauseBreak, "Code didn't break.");
///////////////////////////////////////////////
// Step 4: Verify result.
Assert.AreEqual("Success", sanitized);
}
public static Task Brfalse([ValueSource(nameof(_Brfalse))] TestCaseInformation caseInfo) => TestFramework.ExecuteTestAsync(caseInfo);
public static Task Call([ValueSource(nameof(_Call))] TestCaseInformation caseInfo) => TestFramework.ExecuteTestAsync(caseInfo);
public static Task Stloc_s([ValueSource(nameof(_Stloc_s))] TestCaseInformation caseInfo) => TestFramework.ExecuteTestAsync(caseInfo);
public static Task Unbox_any([ValueSource(nameof(_Unbox_any))] TestCaseInformation caseInfo) => TestFramework.ExecuteTestAsync(caseInfo);
public static Task TypeRelations([ValueSource(nameof(_TypeRelations))] TestCaseInformation caseInfo) => TestFramework.ExecuteTestAsync(caseInfo);
public static Task Ldc_i4_m1([ValueSource(nameof(_Ldc_i4_m1))] TestCaseInformation caseInfo) => TestFramework.ExecuteTestAsync(caseInfo);
public static Task ExceptionThrownByCLI([ValueSource(nameof(_ExceptionThrownByCLI))] TestCaseInformation caseInfo) => TestFramework.ExecuteTestAsync(caseInfo);
public static Task ExceptionHandling([ValueSource(nameof(_ExceptionHandling))] TestCaseInformation caseInfo) => TestFramework.ExecuteTestAsync(caseInfo);
public static Task ArrayTypes([ValueSource(nameof(_ArrayTypes))] TestCaseInformation caseInfo) => TestFramework.ExecuteTestAsync(caseInfo);
public static async Task ExecuteTestAsync(TestCaseInformation caseInfo)
{
Assert.IsTrue(caseInfo.Method.IsPublic && caseInfo.Method.IsStatic);
// Split current thread context.
await Task.Delay(1);
#if DEBUG
const string configuration = "Debug";
#else
const string configuration = "Release";
#endif
///////////////////////////////////////////////
// Step 1-1: Create translation context.
var translateContext = new TranslateContext(
caseInfo.Method.DeclaringType.Assembly.Location,
true,
TargetPlatforms.Generic);
// Step 1-2: Prepare target methods.
var targetTypes = new HashSet <string>(
new[] { caseInfo.Method }.
Concat(caseInfo.AdditionalMethods).
Select(method => method.DeclaringType).
Concat(caseInfo.AdditionalTypes).
Select(type => type.FullName).
Distinct());
var targetMethods = new[] { caseInfo.Method }.
Concat(caseInfo.AdditionalMethods).
GroupBy(method => method.Name).
ToDictionary(g => g.Key, g => g.ToArray());
var prepared = AssemblyPreparer.Prepare(
translateContext,
t => targetTypes.Contains(t.FriendlyName),
m => targetMethods.ContainsKey(m.Name));
// Step 1-3: Extract prepared target type/caseInfo.Method informations.
var targetType =
translateContext.Assembly.Modules
.First().Types
.First(t => t.FriendlyName == caseInfo.Method.DeclaringType.FullName);
var targetMethod =
targetType.DeclaredMethods
.First(m => m.Name == caseInfo.Method.Name);
var targetAdditionalMethods =
targetType.DeclaredMethods
.Where(m => (caseInfo.AdditionalMethods.FirstOrDefault(am => m.Name == am.Name) != null))
.ToArray();
// Step 1-4: Create storing director for test artifacts
// HACK: It's reliability for Windows MAX_PATH limitation.
var basePath =
Path.GetFullPath(
Path.Combine(
Path.GetDirectoryName(caseInfo.Method.DeclaringType.Assembly.Location),
"..", "..", "..", "..", "..",
"test-artifacts", configuration));
while (true)
{
try
{
if (!Directory.Exists(basePath))
{
Directory.CreateDirectory(basePath);
}
break;
}
catch
{
}
}
// Step 1-4: Translate caseInfo.Method bodies.
var translatedPath =
Path.Combine(
basePath,
caseInfo.CategoryName,
caseInfo.Id,
caseInfo.UniqueName);
var logw = new StringWriter();
var storage = new CodeTextStorage(logw, translatedPath, false, " ");
AssemblyWriter.WriteHeader(
storage,
translateContext,
prepared);
var sourceFiles = AssemblyWriter.WriteSourceCode(
storage,
translateContext,
prepared,
true,
DebugInformationOptions.CommentOnly);
// Step 1-5: Write source code into a file from template.
var expectedType = targetMethod.ReturnType;
var sourceCodeStream = new MemoryStream();
await TestUtilities.CopyResourceToStreamAsync(
sourceCodeStream,
// If the target caseInfo.Method result is void-type, we have to use void-type tolerant template.
(expectedType.IsVoidType || (caseInfo.Assert == TestCaseAsserts.CauseBreak))? "test_void.c" : "test.c");
sourceCodeStream.Position = 0;
var sourceCode = new StreamReader(sourceCodeStream);
var constants = caseInfo.Arguments.
Zip(targetMethod.Parameters, (arg, p) => new Constant(
p.ParameterName,
p.TargetType,
arg?.GetType(),
Utilities.GetCLanguageExpression(arg))
).
ToArray();
var argumentList = constants.
Select(constant => new Constant(
"_" + constant.SymbolName,
constant.TargetType,
null,
GetCLangaugeSafeConversionExpression(constant.TargetType, constant.ExpressionType, constant.SymbolName))).
ToArray();
var arguments = argumentList;
var locals = argumentList.
Select(argument =>
new Constant(
argument.SymbolName,
argument.TargetType,
argument.ExpressionType,
Utilities.GetCLanguageExpression(expectedType.InternalStaticEmptyValue))).
ToArray();
if (!(expectedType.IsVoidType || (caseInfo.Assert == TestCaseAsserts.CauseBreak)))
{
// VERY DIRTY:
constants = constants.Concat(new Constant[]
{
new Constant(
"expected",
expectedType,
caseInfo.Expected?.GetType(),
Utilities.GetCLanguageExpression(caseInfo.Expected)),
}).
ToArray();
arguments = constants.
Select(constant => new Constant(
"_" + constant.SymbolName,
constant.TargetType,
null,
GetCLangaugeSafeConversionExpression(constant.TargetType, constant.ExpressionType, constant.SymbolName))).
ToArray();
locals = arguments.
Select(argument =>
new Constant(
argument.SymbolName,
argument.TargetType,
argument.ExpressionType,
Utilities.GetCLanguageExpression(argument.TargetType.InternalStaticEmptyValue))).
Concat(new Constant[]
{
new Constant(
"_actual",
expectedType,
caseInfo.Expected?.GetType(),
Utilities.GetCLanguageExpression(expectedType.InternalStaticEmptyValue)),
}).
ToArray();
}
// Construct test definitions.
var expectedSymbolName = locals.
Any(entry => (entry.SymbolName == "_expected") && entry.TargetType.IsReferenceType) ?
"frame__._expected" :
"_expected";
var actualSymbolName = locals.
Any(entry => (entry.SymbolName == "_actual") && entry.TargetType.IsReferenceType) ?
"frame__._actual" :
"_actual";
var replaceValues = new Dictionary <string, object>
{
{ "assemblyName", targetMethod.DeclaringType.DeclaringModule.DeclaringAssembly.Name },
{ "testName", targetMethod.FriendlyName },
{ "type", targetMethod.ReturnType.CLanguageTypeName },
{ "constants", string.Join(" ", constants.
Select(entry => string.Format("{0} {1} = {2};",
(entry.ExpressionType != null) ? Utilities.GetCLanguageTypeName(entry.ExpressionType) : entry.TargetType.CLanguageTypeName,
entry.SymbolName,
entry.Expression))) },
{ "locals", string.Join(" ", locals.
Where(entry => !entry.TargetType.IsReferenceType).
Select(entry => string.Format("{0} {1} = {2};",
entry.TargetType.CLanguageTypeName,
entry.SymbolName,
entry.Expression))) },
{ "frames", string.Join(" ", locals.
Where(entry => entry.TargetType.IsReferenceType).
Select(entry => string.Format("{0} {1};",
entry.TargetType.CLanguageTypeName,
entry.SymbolName))) },
{ "frameCount", locals.
Where(entry => entry.TargetType.IsClass).Count() },
{ "arguments", string.Join(" ", arguments.
Select(entry => string.Format("{0}{1} = {2};",
entry.TargetType.IsReferenceType ? "frame__." : string.Empty,
entry.SymbolName,
entry.Expression))) },
{ "actual", actualSymbolName },
{ "function", targetMethod.CLanguageFunctionFullName },
{ "argumentList", string.Join(", ", argumentList.
Select(arg => string.Format(
"{0}{1}",
arg.TargetType.IsReferenceType ? "frame__." : string.Empty,
arg.SymbolName))) },
{ "equality", GetCLanguageCompareExpression(expectedType, expectedSymbolName, actualSymbolName) },
{ "format", GetCLanguagePrintFormatFromType(targetMethod.ReturnType) },
{ "expectedExpression", GetCLanguagePrintArgumentExpression(expectedType, expectedSymbolName) },
{ "actualExpression", GetCLanguagePrintArgumentExpression(expectedType, actualSymbolName) },
{ "sourcePath", translateContext.Assembly.Name + "_bundle.c" }
};
// Step 1-6: Write Visual C++ project file and Visual Studio Code launch config from template.
// Note: It's only debugging purpose. The test doesn't use.
var vcxprojTemplatePath = Path.Combine(translatedPath, "test.vcxproj");
await TestUtilities.CopyResourceToTextFileAsync(vcxprojTemplatePath, "test.vcxproj", replaceValues);
TestContext.AddTestAttachment(vcxprojTemplatePath, "Generated VC++ project");
var launchTemplatePath = Path.Combine(translatedPath, ".vscode", "launch.json");
await TestUtilities.CopyResourceToTextFileAsync(launchTemplatePath, "launch.json", replaceValues);
var sourcePath = Path.Combine(translatedPath, "test.c");
await TestUtilities.WriteTextFileAsync(sourcePath, sourceCode, replaceValues);
///////////////////////////////////////////////
// Step 2: Test and verify result by real IL code at this runtime.
object rawResult;
switch (caseInfo.Assert)
{
case TestCaseAsserts.IgnoreValidateInvokeResult:
break;
case TestCaseAsserts.CauseBreak:
rawResult = null;
break;
default:
CultureInfo.CurrentCulture = CultureInfo.InvariantCulture;
CultureInfo.CurrentUICulture = CultureInfo.InvariantCulture;
rawResult = caseInfo.Method.Invoke(null, caseInfo.Arguments);
Assert.AreEqual(caseInfo.Expected, rawResult);
break;
}
///////////////////////////////////////////////
// Step 3: Test compiled C source code and execute.
string sanitized = null;
try
{
var executedResult = await CMakeDriver.BuildDirectlyAsync(
binPath,
configuration,
sourcePath,
il2cRuntimePath);
sanitized = executedResult.Trim(' ', '\r', '\n');
}
catch (Exception ex)
{
if ((caseInfo.Assert == TestCaseAsserts.CauseBreak) && ex.Message.Contains("ExitCode=-2147483645"))
{
return;
}
throw;
}
Assert.IsFalse(caseInfo.Assert == TestCaseAsserts.CauseBreak, "Code didn't break.");
///////////////////////////////////////////////
// Step 4: Verify result.
Assert.AreEqual("Success", sanitized);
}
public static Task System_Delegate([ValueSource(nameof(_System_Delegate))] TestCaseInformation caseInfo) => TestFramework.ExecuteTestAsync(caseInfo);