public async Task TestDownloadArtifactAsyncWithMinimatchPattern()
{
byte[] sourceContent1 = GenerateRandomData();
byte[] sourceContent2 = GenerateRandomData();
TestFile sourceFile1 = new TestFile(sourceContent1);
TestFile sourceFile2 = new TestFile(sourceContent2);
sourceFile1.PlaceItem(Path.Combine(Directory.GetCurrentDirectory(), Path.Combine(TestDownloadSourceFolder, "drop/test2.txt")));
sourceFile2.PlaceItem(Path.Combine(Directory.GetCurrentDirectory(), Path.Combine(TestDownloadSourceFolder, "drop/test3.txt")));
using (var hostContext = new TestHostContext(this))
{
var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace());
var provider = new FileShareProvider(context, null, context.CreateArtifactsTracer(), new MockDedupManifestArtifactClientFactory());
string sourcePath = Path.Combine(Directory.GetCurrentDirectory(), TestDownloadSourceFolder);
string destPath = Path.Combine(Directory.GetCurrentDirectory(), TestDestFolder);
ArtifactDownloadParameters downloadParameters = new ArtifactDownloadParameters();
downloadParameters.TargetDirectory = destPath;
downloadParameters.MinimatchFilters = new string[] { "drop/test2.txt" };
BuildArtifact buildArtifact = new BuildArtifact();
buildArtifact.Name = "drop";
buildArtifact.Resource = new ArtifactResource();
buildArtifact.Resource.Data = sourcePath;
await provider.DownloadMultipleArtifactsAsync(downloadParameters, new List <BuildArtifact> {
buildArtifact
}, CancellationToken.None, context);
var sourceFiles = Directory.GetFiles(sourcePath);
var destFiles = Directory.GetFiles(Path.Combine(destPath, buildArtifact.Name));
Assert.Equal(1, destFiles.Length);
foreach (var file in sourceFiles)
{
string destFile = destFiles.FirstOrDefault(f => Path.GetFileName(f).Equals(Path.GetFileName(file)));
Assert.True(StructuralComparisons.StructuralEqualityComparer.Equals(ComputeHash(file), ComputeHash(destFile)));
}
TestCleanup();
}
}
public async Task DefaultsToCurrentSystemOemEncoding()
{
// This test verifies that the additional code pages encoding provider is registered.
// By default, only Unicode encodings, ASCII, and code page 28591 are supported. An
// additional provider must be registered to support the full set of encodings that
// were included in Full .NET prior to 4.6.
//
// For example, on an en-US box, this is required for loading the encoding for the
// default console output code page '437'. Without loading the correct encoding for
// code page IBM437, some characters cannot be translated correctly, e.g. write 'ç'
// from powershell.exe.
using (TestHostContext hc = new TestHostContext(this))
{
Tracing trace = hc.GetTrace();
var processInvoker = new ProcessInvokerWrapper();
processInvoker.Initialize(hc);
var stdout = new List <string>();
var stderr = new List <string>();
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
stdout.Add(e.Data);
};
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
stderr.Add(e.Data);
};
await processInvoker.ExecuteAsync(
workingDirectory : "",
fileName : "powershell.exe",
arguments : $@"-NoLogo -Sta -NoProfile -NonInteractive -ExecutionPolicy Unrestricted -Command ""Write-Host 'From STDOUT ''ç''' ; Write-Error 'From STDERR ''ç'''""",
environment : null,
requireExitCodeZero : false,
cancellationToken : CancellationToken.None);
Assert.Equal(1, stdout.Count);
Assert.Equal("From STDOUT 'ç'", stdout[0]);
Assert.True(stderr.Count > 0);
Assert.True(stderr[0].Contains("From STDERR 'ç'"));
}
}
public async Task OomScoreAdjIsWriten_FromEnv()
{
// We are on a system that supports oom_score_adj in procfs as assumed by ProcessInvoker
string testProcPath = $"/proc/{Process.GetCurrentProcess().Id}/oom_score_adj";
if (File.Exists(testProcPath))
{
using (TestHostContext hc = new TestHostContext(this))
using (var tokenSource = new CancellationTokenSource())
{
Tracing trace = hc.GetTrace();
using (var processInvoker = new ProcessInvokerWrapper())
{
processInvoker.Initialize(hc);
int oomScoreAdj = -9999;
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
oomScoreAdj = int.Parse(e.Data);
tokenSource.Cancel();
};
try
{
var proc = await processInvoker.ExecuteAsync("", "bash", "-c \"cat /proc/$$/oom_score_adj\"",
new Dictionary <string, string> {
{ "PIPELINE_JOB_OOMSCOREADJ", "1234" }
},
false, null, false, null, false, false,
highPriorityProcess : false,
cancellationToken : tokenSource.Token);
Assert.Equal(oomScoreAdj, 1234);
}
catch (OperationCanceledException)
{
trace.Info("Caught expected OperationCanceledException");
}
}
}
}
}
public async Task TestCancel()
{
const int SecondsToRun = 20;
using (TestHostContext hc = new TestHostContext(this))
using (var tokenSource = new CancellationTokenSource())
{
Tracing trace = hc.GetTrace();
using (var processInvoker = new ProcessInvokerWrapper())
{
processInvoker.Initialize(hc);
Stopwatch watch = Stopwatch.StartNew();
Task execTask = processInvoker.ExecuteAsync("", "bash", $"-c \"sleep {SecondsToRun}s\"", null, tokenSource.Token);
await Task.Delay(500);
tokenSource.Cancel();
try
{
await execTask;
}
catch (OperationCanceledException)
{
trace.Info("Get expected OperationCanceledException.");
}
Assert.True(execTask.IsCompleted);
Assert.True(!execTask.IsFaulted);
Assert.True(execTask.IsCanceled);
watch.Stop();
long elapsedSeconds = watch.ElapsedMilliseconds / 1000;
// if cancellation fails, then execution time is more than 15 seconds
long expectedSeconds = (SecondsToRun * 3) / 4;
Assert.True(elapsedSeconds <= expectedSeconds, $"cancellation failed, because task took too long to run. {elapsedSeconds}");
}
}
}
private void ValidateLocStrings(TestHostContext hc, string project)
{
using (hc)
{
Tracing trace = hc.GetTrace();
var keys = new List <string>();
var badLines = new List <BadLineInfo>();
// Search for source files within the project.
trace.Verbose("Searching source files:");
string[] sourceFiles = Directory.GetFiles(
TestUtil.GetProjectPath(project),
"*.cs",
SearchOption.AllDirectories);
foreach (string sourceFile in sourceFiles)
{
// Skip files in the obj directory.
if (sourceFile.Contains(StringUtil.Format("{0}obj{0}", Path.DirectorySeparatorChar)))
{
continue;
}
trace.Verbose($" {sourceFile}");
foreach (string line in File.ReadAllLines(sourceFile))
{
// Search for calls to the StringUtil.Loc method within the line.
const string Pattern = "StringUtil.Loc(";
int searchIndex = 0;
int patternIndex;
while (searchIndex < line.Length &&
(patternIndex = line.IndexOf(Pattern, searchIndex)) >= 0)
{
// Bump the search index in preparation for the for the next iteration within the same line.
searchIndex = patternIndex + Pattern.Length;
// Extract the resource key.
int keyStartIndex = patternIndex + Pattern.Length;
int keyEndIndex;
if (keyStartIndex + 2 < line.Length && // Key should start with a ", be followed by at least
line[keyStartIndex] == '"' && // one character, and end with a ".
(keyEndIndex = line.IndexOf('"', keyStartIndex + 1)) > 0)
{
// Remove the first and last double quotes.
keyStartIndex++;
keyEndIndex--;
string key = line.Substring(
startIndex: keyStartIndex,
length: keyEndIndex - keyStartIndex + 1);
if (ValidKeyRegex.IsMatch(key))
{
// A valid key was extracted.
keys.Add(key);
continue;
}
}
// Something went wrong. The pattern was found, but the resource key could not be determined.
badLines.Add(new BadLineInfo {
File = sourceFile, Line = line
});
}
}
}
// Load the strings.
string stringsFile = Path.Combine(TestUtil.GetSrcPath(), "Misc", "layoutbin", "en-US", "strings.json");
Assert.True(File.Exists(stringsFile), $"File does not exist: {stringsFile}");
var resourceDictionary = IOUtil.LoadObject <Dictionary <string, object> >(stringsFile);
// Find missing keys.
string[] missingKeys =
keys
.Where(x => !resourceDictionary.ContainsKey(x))
.OrderBy(x => x)
.ToArray();
if (missingKeys.Length > 0)
{
trace.Error("One or more resource keys missing from resources file:");
foreach (string missingKey in missingKeys)
{
trace.Error($" {missingKey}");
}
}
// Validate whether resource keys couldn't be interpreted.
if (badLines.Count > 0)
{
trace.Error("Bad lines detected. Unable to interpret resource key(s).");
IEnumerable <IGrouping <string, BadLineInfo> > badLineGroupings =
badLines
.GroupBy(x => x.File)
.OrderBy(x => x.Key)
.ToArray();
foreach (IGrouping <string, BadLineInfo> badLineGrouping in badLineGroupings)
{
trace.Error($"File: {badLineGrouping.First().File}");
foreach (BadLineInfo badLine in badLineGrouping)
{
trace.Error($" Line: {badLine.Line}");
}
}
}
Assert.True(missingKeys.Length == 0, $"One or more resource keys missing from resources files. Consult the trace log: {hc.TraceFileName}");
Assert.True(badLines.Count == 0, $"Unable to determine one or more resource keys. Consult the trace log: {hc.TraceFileName}");
}
}
public void TestGettingArtifactItemsWithMinimatchPattern(string[] pttrn, int count, string[] expectedPaths)
{
using (TestHostContext hostContext = new TestHostContext(this))
{
AgentTaskPluginExecutionContext context = new AgentTaskPluginExecutionContext(hostContext.GetTrace());
ArtifactItemFilters filters = new ArtifactItemFilters(null, context.CreateArtifactsTracer());
List <FileContainerItem> items = new List <FileContainerItem>
{
new FileContainerItem()
{
ItemType = ContainerItemType.Folder, Path = "ArtifactForTest"
},
new FileContainerItem()
{
ItemType = ContainerItemType.File, Path = "ArtifactForTest/File1.txt"
},
new FileContainerItem()
{
ItemType = ContainerItemType.Folder, Path = "ArtifactForTest/Folder1"
},
new FileContainerItem()
{
ItemType = ContainerItemType.File, Path = "ArtifactForTest/Folder1/File2.txt"
},
new FileContainerItem()
{
ItemType = ContainerItemType.File, Path = "ArtifactForTest/Folder1/File21.txt"
},
new FileContainerItem()
{
ItemType = ContainerItemType.Folder, Path = "ArtifactForTest/Folder1/Folder2"
},
new FileContainerItem()
{
ItemType = ContainerItemType.File, Path = "ArtifactForTest/Folder1/Folder2/File3.txt"
}
};
List <string> paths = new List <string>();
foreach (FileContainerItem item in items)
{
paths.Add(item.Path);
}
string[] minimatchPatterns = pttrn;
Options customMinimatchOptions = new Options()
{
Dot = true,
NoBrace = true,
AllowWindowsPaths = PlatformUtil.RunningOnWindows
};
Hashtable map = filters.GetMapToFilterItems(paths, minimatchPatterns, customMinimatchOptions);
List <FileContainerItem> resultItems = filters.ApplyPatternsMapToContainerItems(items, map);
Assert.Equal(count, resultItems.Count);
string listPaths = string.Join(", ", expectedPaths);
List <string> resultPathsList = new List <string>();
foreach (FileContainerItem item in resultItems)
{
resultPathsList.Add(item.Path);
}
string resultPaths = string.Join(", ", resultPathsList);
Assert.Equal(listPaths, resultPaths);
}
}
public async Task SuccessReadProcessEnv()
{
using (TestHostContext hc = new TestHostContext(this))
{
Tracing trace = hc.GetTrace();
string envName = Guid.NewGuid().ToString();
string envValue = Guid.NewGuid().ToString();
Process sleep = null;
try
{
// TODO: this was formerly skipped on Windows - why?
hc.EnqueueInstance <IProcessInvoker>(new ProcessInvokerWrapper());
// sleep 15 seconds
string sleepCmd = (TestUtil.IsWindows())
? "powershell"
: "sleep";
string sleepArgs = (TestUtil.IsWindows())
? "-Command \"Start-Sleep -s 15\""
: "15s";
var startInfo = new ProcessStartInfo(sleepCmd, sleepArgs);
startInfo.Environment[envName] = envValue;
sleep = Process.Start(startInfo);
var timeout = Process.GetProcessById(sleep.Id);
while (timeout == null)
{
await Task.Delay(1500);
timeout = Process.GetProcessById(sleep.Id);
}
try
{
await Task.Delay(2000);
trace.Info($"Read env from {timeout.Id}");
var value = timeout.GetEnvironmentVariable(hc, envName);
if (string.Equals(value, envValue, StringComparison.OrdinalIgnoreCase))
{
trace.Info($"Find the env.");
return;
}
else
{
Assert.True(false, "Expected environment '" + envValue + "' did not match actual '" + value + "'");
}
}
catch (Exception ex)
{
trace.Error(ex);
Assert.True(false, "Fail to retrive process environment variable due to exception: " + ex.Message + "\n" + ex.StackTrace);
}
Assert.True(false, "Fail to retrive process environment variable.");
}
finally
{
try
{
sleep?.Kill();
}
catch { }
}
}
}