public void SetTestContainers(List <string> testContainerFiles)
{
// @todo maybe check if something changed at all before firing events and so?!
this._cachedContainers.Clear();
foreach (var containerFile in testContainerFiles)
{
var index = this._cachedContainers
.FindIndex(x => x.Source.Equals(containerFile, StringComparison.OrdinalIgnoreCase));
if (index != -1)
{
return;
}
if (!TestContainerHelper.IsTestContainerFile(containerFile))
{
return;
}
var container = new CTestContainer(this, containerFile);
this._cachedContainers.Add(container);
}
if (null == this.TestContainersUpdated)
{
return;
}
this.TestContainersUpdated(this, EventArgs.Empty);
}
public void RunTests(IEnumerable <string> sources, IRunContext runContext,
IFrameworkHandle frameworkHandle)
{
this._log = frameworkHandle;
this.Log(TestMessageLevel.Informational, "running tests (src) ...");
var sourcesList = sources as IList <string> ?? sources.ToList();
if (!this.SetupEnvironment(sourcesList.First()))
{
return;
}
this.Log(TestMessageLevel.Informational, "using configuration: "
+ this._config.ActiveConfiguration);
this._runningFromSources = true;
var logFileDir = this._config.CacheDir + "\\Testing\\Temporary";
this.Log(TestMessageLevel.Informational,
"logs are written to (" + TestContainerHelper.ToLinkPath(logFileDir) + ")");
foreach (var source in sourcesList)
{
var cases = TestContainerHelper.ParseTestContainerFile(
source, frameworkHandle, null, this._config.ActiveConfiguration);
this.RunTests(cases.Values, runContext, frameworkHandle);
}
this._runningFromSources = false;
this.Log(TestMessageLevel.Informational, "running tests (src) done");
}
/**
* @brief recursively collects all test container files that can be found
*/
public static IEnumerable <string> CollectCTestTestfiles(string currentDir)
{
var file = new FileInfo(Path.Combine(currentDir, TestContainerHelper.TestFileName + TestContainerHelper.TestFileExtension));
if (!file.Exists)
{
return(Enumerable.Empty <string>());
}
var res = new List <string>();
var content = file.OpenText().ReadToEnd();
var matches = Regex.Matches(content, @".*[sS][uB][bB][dD][iI][rR][sS]\s*\((?<subdir>.*)\)");
var subdirs = (from Match match in matches select match.Groups["subdir"].Value).ToList();
if (content.Contains("add_test"))
{
res.Add(file.FullName);
}
foreach (var dir in subdirs)
{
var subpath = dir.Trim('\"');
subpath = Path.Combine(currentDir, subpath);
res.AddRange(TestContainerHelper.CollectCTestTestfiles(subpath));
}
return(res);
}
private bool SetupEnvironment(string source)
{
var cacheDir = TestContainerHelper.FindCMakeCacheDirectory(source);
this._config = CTestAdapterConfig.ReadFromDisk(Path.Combine(cacheDir, Constants.CTestAdapterConfigFileName)) ??
CTestAdapterConfig.ReadFromCache(cacheDir);
if (this._config == null)
{
this.Log(TestMessageLevel.Error, "could not initialize environment");
return(false);
}
return(true);
}
private void OnTestContainerRemoved(object sender, Events.TestContainerEventArgs e)
{
if (e == null)
{
return;
}
if (!this._testContainers.Contains(e.File))
{
return;
}
if (!TestContainerHelper.IsTestContainerFile(e.File))
{
return;
}
this._testContainerWatcher.RemoveWatch(e.File);
this._testContainers.Remove(e.File);
if (null != this.TestContainersChanged)
{
this.TestContainersChanged(this, new TestContainerListArgs(this._testContainers));
}
}
public static string FindCTestExe(string basePath)
{
var file = new FileInfo(Path.Combine(basePath, Constants.CTestExecutableName));
if (file.Exists)
{
return(file.FullName);
}
var cdir = new DirectoryInfo(basePath);
var subdirs = cdir.GetDirectories();
foreach (var dir in subdirs)
{
var res = TestContainerHelper.FindCTestExe(dir.FullName);
if (res != string.Empty)
{
return(res);
}
}
return(string.Empty);
}
public void FindTestContainers()
{
this.Log(LogLevel.Debug, "FindTestContainers (clear)");
this._testContainerWatcher.Clear();
this._testContainers.Clear();
if (!this.CTestAdapterEnabled)
{
return;
}
this.Log(LogLevel.Debug, "FindTestContainers (search)");
// @todo verify something changed actually?!?
var files = TestContainerHelper.CollectCTestTestfiles(this._package.CMakeCacheDirectory).ToList();
foreach (var file in files)
{
this._testContainerWatcher.AddWatch(file);
this._testContainers.Add(file);
}
if (null != this.TestContainersChanged)
{
this.TestContainersChanged(this, new TestContainerListArgs(this._testContainers));
}
}
public void DiscoverTests(IEnumerable <string> sources,
IDiscoveryContext discoveryContext,
IMessageLogger logger,
ITestCaseDiscoverySink discoverySink)
{
this._log = logger;
this.Log(TestMessageLevel.Informational, "discovering ...");
var v = sources as IList <string> ?? sources.ToList();
// verify we have a CMakeCache.txt directory
var cacheDir = TestContainerHelper.FindCMakeCacheDirectory(v.First());
if (!cacheDir.Any())
{
this.Log(TestMessageLevel.Informational, "cmake cache not found");
return;
}
// read parameters
var cfg = CTestAdapterConfig.ReadFromDisk(Path.Combine(cacheDir, Constants.CTestAdapterConfigFileName)) ??
CTestAdapterConfig.ReadFromCache(cacheDir);
if (null == cfg)
{
this.Log(TestMessageLevel.Error, "could not create CTestAdapterConfig");
return;
}
// make sure a configuration is set
if (!cfg.ActiveConfiguration.Any())
{
if (cfg.TrySetActiveConfigFromConfigTypes())
{
this.Log(TestMessageLevel.Warning,
"Configuration fallback to: " + cfg.ActiveConfiguration);
}
else
{
this.Log(TestMessageLevel.Error, "could not set Configuration");
return;
}
}
this.Log(TestMessageLevel.Informational, "using configuration: " + cfg.ActiveConfiguration);
// make sure we have a ctest executable
if (!File.Exists(cfg.CTestExecutable))
{
cfg.CTestExecutable = TestContainerHelper.FindCTestExe(cfg.CacheDir);
}
if (!File.Exists(cfg.CTestExecutable))
{
this.Log(TestMessageLevel.Error,
"ctest not found, tried: \"" + cfg.CTestExecutable + "\"");
return;
}
this.Log(TestMessageLevel.Informational, "using ctest binary: " + cfg.CTestExecutable);
// collect all existing tests by executing ctest
var collection = TestContainerHelper.FindAllTestsWithCtest(cfg);
foreach (var source in v)
{
var cases = TestContainerHelper.ParseTestContainerFile(source, this._log, collection, cfg.ActiveConfiguration);
foreach (var c in cases)
{
discoverySink.SendTestCase(c.Value);
}
}
this.Log(TestMessageLevel.Informational, "discovering done");
}
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext,
IFrameworkHandle frameworkHandle)
{
this._log = frameworkHandle;
this.Log(TestMessageLevel.Informational, "running tests ...");
var testCases = tests as IList <TestCase> ?? tests.ToList();
if (!testCases.Any())
{
return;
}
if (!this._runningFromSources)
{
if (!this.SetupEnvironment(testCases.First().Source))
{
return;
}
}
// make sure a configuration is set
if (!this._config.ActiveConfiguration.Any())
{
if (this._config.TrySetActiveConfigFromConfigTypes())
{
this.Log(TestMessageLevel.Warning,
"Configuration fallback to: " + this._config.ActiveConfiguration);
}
else
{
this.Log(TestMessageLevel.Error, "could not set Configuration");
return;
}
}
// make sure we have a ctest executable
if (!File.Exists(this._config.CTestExecutable))
{
this._config.CTestExecutable = TestContainerHelper.FindCTestExe(this._config.CacheDir);
}
if (!File.Exists(this._config.CTestExecutable))
{
this.Log(TestMessageLevel.Error,
"ctest not found, tried: \"" + this._config.CTestExecutable + "\"");
return;
}
if (!Directory.Exists(this._config.CacheDir))
{
this.Log(TestMessageLevel.Error,
"working directory not found: " + TestContainerHelper.ToLinkPath(this._config.CacheDir));
return;
}
this.Log(TestMessageLevel.Informational,
"working directory is " + TestContainerHelper.ToLinkPath(this._config.CacheDir));
var logFileDir = this._config.CacheDir + "\\Testing\\Temporary";
if (!this._runningFromSources)
{
this.Log(TestMessageLevel.Informational,
"ctest (" + this._config.CTestExecutable + ")");
this.Log(TestMessageLevel.Informational,
"logs are written to (" + TestContainerHelper.ToLinkPath(logFileDir) + ")");
}
this._parentProcess = Process.GetCurrentProcess();
this._ctestProcess = new Process();
if (this._procParam == null)
{
this._procParam = new ProcessStartInfo
{
CreateNoWindow = true,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
WindowStyle = ProcessWindowStyle.Hidden
};
}
// run test cases
foreach (var test in testCases)
{
var testResult = new TestResult(test)
{
ComputerName = Environment.MachineName,
Outcome = TestOutcome.Skipped
};
// verify we have a run directory and a ctest executable
var args = "-R \"^" + test.FullyQualifiedName + "$\"";
if (this._config.ActiveConfiguration.Any())
{
args += " -C \"" + this._config.ActiveConfiguration + "\"";
}
this._procParam.Arguments = args;
this._procParam.FileName = this._config.CTestExecutable;
this._procParam.WorkingDirectory = this._config.CacheDir;
this._ctestProcess.StartInfo = this._procParam;
var logFileName = logFileDir + "\\LastTest.log";
if (File.Exists(logFileName))
{
File.Delete(logFileName);
}
var logMsg = "ctest " + test.FullyQualifiedName;
if (this._config.ActiveConfiguration.Any())
{
logMsg += " -C " + this._config.ActiveConfiguration;
}
this.Log(TestMessageLevel.Informational, logMsg);
if (this._cancelled)
{
break;
}
if (runContext.IsBeingDebugged)
{
/// @todo check if child process debugging is available?!?
this._ctestProcess.Start();
}
else
{
this._ctestProcess.Start();
}
this._ctestProcess.WaitForExit();
if (this._cancelled)
{
break;
}
var output = this._ctestProcess.StandardOutput.ReadToEnd();
if (!File.Exists(logFileName))
{
this.Log(TestMessageLevel.Warning, "logfile not found: "
+ TestContainerHelper.ToLinkPath(logFileName));
}
var content = File.ReadAllText(logFileName);
var logFileBackup = test.FullyQualifiedName + ".log";
var invalidChars = new string(Path.GetInvalidFileNameChars()) +
new string(Path.GetInvalidPathChars());
foreach (char c in invalidChars)
{
logFileBackup = logFileBackup.Replace(c.ToString(), "_");
}
logFileBackup = logFileDir + "\\" + logFileBackup;
File.Copy(logFileName, logFileBackup, true);
var matchesDuration = CTestExecutor.RegexDuration.Match(content);
var timeSpan = new TimeSpan();
if (matchesDuration.Success)
{
timeSpan = TimeSpan.FromSeconds(
double.Parse(matchesDuration.Groups[CTestExecutor.RegexFieldDuration].Value,
System.Globalization.CultureInfo.InvariantCulture.NumberFormat));
}
else
{
this.Log(TestMessageLevel.Warning,
"could not get runtime of test " + test.FullyQualifiedName);
}
testResult.Duration = timeSpan;
testResult.Outcome = this._ctestProcess.ExitCode == 0 ? TestOutcome.Passed : TestOutcome.Failed;
if (this._ctestProcess.ExitCode != 0)
{
var matchesOutput = CTestExecutor.RegexOutput.Match(content);
testResult.ErrorMessage = matchesOutput.Groups[CTestExecutor.RegexFieldOutput].Value;
this.Log(TestMessageLevel.Error,
"ERROR IN TEST " + test.FullyQualifiedName + ":");
this.Log(TestMessageLevel.Error, output);
this.Log(TestMessageLevel.Error,
"END OF TEST OUTPUT FROM " + test.FullyQualifiedName);
}
this.Log(TestMessageLevel.Informational,
"Log saved to " + TestContainerHelper.ToLinkPath(logFileBackup));
frameworkHandle.RecordResult(testResult);
}
this._ctestProcess.Dispose();
this._ctestProcess = null;
this._parentProcess = null;
this.Log(TestMessageLevel.Informational, "running tests done");
}
public static Dictionary <string, TestCase> ParseTestContainerFile(string source, IMessageLogger log,
CTestTestCollection collection, string activeConfiguration)
{
log.SendMessage(TestMessageLevel.Informational, "Parsing CTest file: " + TestContainerHelper.ToLinkPath(source));
var cases = new Dictionary <string, TestCase>();
var content = File.ReadLines(source);
var skipFoundTests = false;
var lineNumber = 0;
foreach (var line in content)
{
lineNumber++;
var ifcheck = TestContainerHelper.IfLevelRegex.Match(line);
if (ifcheck.Success)
{
var cfgRegexString = ifcheck.Groups[FieldNameCfgRegex].Value;
var cfgRegex = new Regex(cfgRegexString);
skipFoundTests = !cfgRegex.IsMatch(activeConfiguration);
continue;
}
else if (line == "else()")
{
skipFoundTests = true;
continue;
}
else if (line == "endif()")
{
skipFoundTests = false;
continue;
}
var test = TestContainerHelper.AddTestRegex.Match(line);
if (test.Success)
{
var testname = test.Groups[FieldNameTestname].Value;
if (skipFoundTests)
{
//log.SendMessage(TestMessageLevel.Warning,
//"skipping test because of Configuration mismatch: line " + lineNumber + ", test " + testname);
continue;
}
if (null != collection)
{
if (!collection.TestExists(testname))
{
log.SendMessage(TestMessageLevel.Warning,
"test not listed by ctest -N : " + testname);
}
}
if (cases.ContainsKey(testname))
{
continue;
}
var testcase = new TestCase(testname, CTestExecutor.ExecutorUri, source)
{
CodeFilePath = source,
DisplayName = testname,
LineNumber = lineNumber,
};
if (null != collection)
{
if (collection.TestExists(testname))
{
testcase.DisplayName = collection[testname].Number.ToString().PadLeft(3, '0') + ": " + testname;
}
}
cases.Add(testname, testcase);
}
}
return(cases);
}
