private ActionContainer PrepareRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction)
{
var repositoryReference = repositoryAction.Reference as Pipelines.RepositoryPathReference;
if (string.Equals(repositoryReference.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase))
{
Trace.Info($"Repository action is in 'self' repository.");
return(null);
}
var setupInfo = new ActionContainer();
string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref);
string actionEntryDirectory = destDirectory;
string dockerFileRelativePath = repositoryReference.Name;
ArgUtil.NotNull(repositoryReference, nameof(repositoryReference));
if (!string.IsNullOrEmpty(repositoryReference.Path))
{
actionEntryDirectory = Path.Combine(destDirectory, repositoryReference.Path);
dockerFileRelativePath = $"{dockerFileRelativePath}/{repositoryReference.Path}";
setupInfo.ActionRepository = $"{repositoryReference.Name}/{repositoryReference.Path}@{repositoryReference.Ref}";
}
else
{
setupInfo.ActionRepository = $"{repositoryReference.Name}@{repositoryReference.Ref}";
}
// find the docker file or action.yml file
var dockerFile = Path.Combine(actionEntryDirectory, "Dockerfile");
var dockerFileLowerCase = Path.Combine(actionEntryDirectory, "dockerfile");
var actionManifest = Path.Combine(actionEntryDirectory, Constants.Path.ActionManifestYmlFile);
var actionManifestYaml = Path.Combine(actionEntryDirectory, Constants.Path.ActionManifestYamlFile);
if (File.Exists(actionManifest) || File.Exists(actionManifestYaml))
{
executionContext.Debug($"action.yml for action: '{actionManifest}'.");
var manifestManager = HostContext.GetService <IActionManifestManager>();
ActionDefinitionData actionDefinitionData = null;
if (File.Exists(actionManifest))
{
actionDefinitionData = manifestManager.Load(executionContext, actionManifest);
}
else
{
actionDefinitionData = manifestManager.Load(executionContext, actionManifestYaml);
}
if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Container)
{
var containerAction = actionDefinitionData.Execution as ContainerActionExecutionData;
if (containerAction.Image.EndsWith("Dockerfile") || containerAction.Image.EndsWith("dockerfile"))
{
var dockerFileFullPath = Path.Combine(actionEntryDirectory, containerAction.Image);
executionContext.Debug($"Dockerfile for action: '{dockerFileFullPath}'.");
setupInfo.Dockerfile = dockerFileFullPath;
setupInfo.WorkingDirectory = destDirectory;
return(setupInfo);
}
else if (containerAction.Image.StartsWith("docker://", StringComparison.OrdinalIgnoreCase))
{
var actionImage = containerAction.Image.Substring("docker://".Length);
executionContext.Debug($"Container image for action: '{actionImage}'.");
setupInfo.Image = actionImage;
return(setupInfo);
}
else
{
throw new NotSupportedException($"'{containerAction.Image}' should be either '[path]/Dockerfile' or 'docker://image[:tag]'.");
}
}
else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.NodeJS)
{
Trace.Info($"Action node.js file: {(actionDefinitionData.Execution as NodeJSActionExecutionData).Script}, no more preparation.");
return(null);
}
else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Plugin)
{
Trace.Info($"Action plugin: {(actionDefinitionData.Execution as PluginActionExecutionData).Plugin}, no more preparation.");
return(null);
}
else
{
throw new NotSupportedException(actionDefinitionData.Execution.ExecutionType.ToString());
}
}
else if (File.Exists(dockerFile))
{
executionContext.Debug($"Dockerfile for action: '{dockerFile}'.");
setupInfo.Dockerfile = dockerFile;
setupInfo.WorkingDirectory = destDirectory;
return(setupInfo);
}
else if (File.Exists(dockerFileLowerCase))
{
executionContext.Debug($"Dockerfile for action: '{dockerFileLowerCase}'.");
setupInfo.Dockerfile = dockerFileLowerCase;
setupInfo.WorkingDirectory = destDirectory;
return(setupInfo);
}
else
{
var fullPath = IOUtil.ResolvePath(actionEntryDirectory, "."); // resolve full path without access filesystem.
throw new InvalidOperationException($"Can't find 'action.yml', 'action.yaml' or 'Dockerfile' under '{fullPath}'. Did you forget to run actions/checkout before running your local action?");
}
}
/// <summary>
/// _work
/// \_update
/// \bin
/// \externals
/// \run.sh
/// \run.cmd
/// \package.zip //temp download .zip/.tar.gz
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
private async Task DownloadLatestAgent(CancellationToken token)
{
string latestAgentDirectory = IOUtil.GetUpdatePath(HostContext);
IOUtil.DeleteDirectory(latestAgentDirectory, token);
Directory.CreateDirectory(latestAgentDirectory);
string archiveFile;
if (_targetPackage.Platform.StartsWith("win"))
{
archiveFile = Path.Combine(latestAgentDirectory, "agent.zip");
}
else
{
archiveFile = Path.Combine(latestAgentDirectory, "agent.tar.gz");
}
Trace.Info($"Save latest agent into {archiveFile}.");
try
{
using (var httpClient = new HttpClient(HostContext.CreateHttpClientHandler()))
{
//open zip stream in async mode
using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true))
{
using (Stream result = await httpClient.GetStreamAsync(_targetPackage.DownloadUrl))
{
//81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k).
await result.CopyToAsync(fs, 81920, token);
await fs.FlushAsync(token);
}
}
}
if (archiveFile.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
{
ZipFile.ExtractToDirectory(archiveFile, latestAgentDirectory);
}
else if (archiveFile.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase))
{
var whichUtil = HostContext.GetService <IWhichUtil>();
string tar = whichUtil.Which("tar");
if (string.IsNullOrEmpty(tar))
{
throw new NotSupportedException($"tar -xzf");
}
// tar -xzf
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Info(args.Data);
}
});
processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Error(args.Data);
}
});
int exitCode = await processInvoker.ExecuteAsync(latestAgentDirectory, tar, $"-xzf \"{archiveFile}\"", null, token);
if (exitCode != 0)
{
throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}.");
}
}
}
else
{
throw new NotSupportedException($"{archiveFile}");
}
Trace.Info($"Finished getting latest agent package at: {latestAgentDirectory}.");
}
finally
{
try
{
// delete .zip file
if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile))
{
Trace.Verbose("Deleting latest agent package zip: {0}", archiveFile);
IOUtil.DeleteFile(archiveFile);
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the temp folder
Trace.Warning("Failed to delete agent package zip '{0}'. Exception: {1}", archiveFile, ex);
}
}
// copy latest agent into agent root folder
// copy bin from _work/_update -> bin.version under root
string binVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.BinDirectory}.{_targetPackage.Version}");
Directory.CreateDirectory(binVersionDir);
Trace.Info($"Copy {Path.Combine(latestAgentDirectory, Constants.Path.BinDirectory)} to {binVersionDir}.");
IOUtil.CopyDirectory(Path.Combine(latestAgentDirectory, Constants.Path.BinDirectory), binVersionDir, token);
// copy externals from _work/_update -> externals.version under root
string externalsVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.ExternalsDirectory}.{_targetPackage.Version}");
Directory.CreateDirectory(externalsVersionDir);
Trace.Info($"Copy {Path.Combine(latestAgentDirectory, Constants.Path.ExternalsDirectory)} to {externalsVersionDir}.");
IOUtil.CopyDirectory(Path.Combine(latestAgentDirectory, Constants.Path.ExternalsDirectory), externalsVersionDir, token);
// copy and replace all .sh/.cmd files
Trace.Info($"Copy any remaining .sh/.cmd files into agent root.");
foreach (FileInfo file in new DirectoryInfo(latestAgentDirectory).GetFiles() ?? new FileInfo[0])
{
// Copy and replace the file.
file.CopyTo(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), file.Name), true);
}
// for windows service back compat with old windows agent, we need make sure the servicehost.exe is still the old name
// if the current bin folder has VsoAgentService.exe, then the new agent bin folder needs VsoAgentService.exe as well
#if OS_WINDOWS
if (File.Exists(Path.Combine(IOUtil.GetBinPath(), "VsoAgentService.exe")))
{
Trace.Info($"Make a copy of AgentService.exe, name it VsoAgentService.exe");
File.Copy(Path.Combine(binVersionDir, "AgentService.exe"), Path.Combine(binVersionDir, "VsoAgentService.exe"), true);
File.Copy(Path.Combine(binVersionDir, "AgentService.exe.config"), Path.Combine(binVersionDir, "VsoAgentService.exe.config"), true);
Trace.Info($"Make a copy of Agent.Listener.exe, name it VsoAgent.exe");
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.exe"), Path.Combine(binVersionDir, "VsoAgent.exe"), true);
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.dll"), Path.Combine(binVersionDir, "VsoAgent.dll"), true);
// in case of we remove all pdb file from agent package.
if (File.Exists(Path.Combine(binVersionDir, "AgentService.pdb")))
{
File.Copy(Path.Combine(binVersionDir, "AgentService.pdb"), Path.Combine(binVersionDir, "VsoAgentService.pdb"), true);
}
if (File.Exists(Path.Combine(binVersionDir, "Agent.Listener.pdb")))
{
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.pdb"), Path.Combine(binVersionDir, "VsoAgent.pdb"), true);
}
}
#endif
}
public void MergeKey2(string tablename, string indexname, BigEntityTableMeta meta)
{
ProcessTraceUtil.StartTrace();
IndexMergeInfo mergeinfo = null;
lock (meta)
{
mergeinfo = meta.IndexMergeInfos.Find(p => indexname.Equals(p.IndexName));
if (mergeinfo == null)
{
mergeinfo = new IndexMergeInfo();
mergeinfo.IndexName = indexname;
meta.IndexMergeInfos.Add(mergeinfo);
}
if (mergeinfo.IsMergin)
{
return;
}
meta.NewAddCount = 0;
mergeinfo.IsMergin = true;
}
DateTime timestart = DateTime.Now;
string newindexfile = string.Empty;
byte[] bigbuffer = new byte[1024 * 1024];
byte[] smallbuffer = new byte[2048];
try
{
ProcessTraceUtil.TraceMem("开始整理key:" + indexname, "m");
long lasmargepos = 0;
long newIndexMergePos = mergeinfo.IndexMergePos;
string indexfile = indexname.Equals(meta.KeyName) ? GetKeyFile(tablename) : GetIndexFile(tablename, indexname);
var tablelocker = GetKeyLocker(tablename, string.Empty);
//新的磁盘索引
List <BigEntityTableIndexItem> newdiskindexlist = new List <BigEntityTableIndexItem>();
var loadFactor = (int)Math.Max(4, new FileInfo(indexfile).Length / MAX_KEYBUFFER);
using (var reader = ObjTextReader.CreateReader(indexfile))
{
long readoldstartpostion = reader.ReadedPostion();
try
{
tablelocker.EnterWriteLock();
keyindexmemtemplist[tablename] = keyindexmemlist[tablename];
keyindexmemlist[tablename] = new SortArrayList <BigEntityTableIndexItem>();
}
finally
{
tablelocker.ExitWriteLock();
}
var listtemp = keyindexmemtemplist[tablename].GetList().ToList();
listtemp = listtemp.Select(p => new BigEntityTableIndexItem
{
Del = p.Del,
Key = p.Key,
KeyOffset = p.KeyOffset,
len = p.len,
Offset = p.Offset,
Index = p.Index,
RangeIndex = p.RangeIndex
}).ToList();
int readcount = listtemp.Count;
ProcessTraceUtil.TraceMem("从内存中读取新增数据，共" + readcount + "条", "m");
if (readcount == 0)
{
return;
}
lasmargepos = listtemp.Max(p => p.KeyOffset);
reader.SetPostion(lasmargepos);
reader.ReadObject <BigEntityTableIndexItem>();
lasmargepos = reader.ReadedPostion();
//优化确定哪些部分是不需要一个个读入的
long copybefore = 0, copylast = 0;
long lastrankindex = 0;
var indexarray = keyindexdisklist[tablename];
using (var sortarray = new Collections.SorteArray <BigEntityTableIndexItem>(indexarray))
{
int mid = -1;
int pos = sortarray.Find(listtemp.First(), ref mid);
ProcessTraceUtil.Trace("查找新数据在老数中插入的开始位置:mid=" + mid + ",pos=" + pos);
if (pos == -1 && mid != -1)
{
//小于最小的
copybefore = indexarray[mid].KeyOffset;
lastrankindex = indexarray[mid].RangeIndex;
ProcessTraceUtil.Trace("老数据可以直接copy的部分:0->" + copybefore);
}
else if (pos != -1)
{
copybefore = indexarray[pos].KeyOffset;
lastrankindex = indexarray[pos].RangeIndex;
ProcessTraceUtil.Trace("老数据可以直接copy的部分:0->" + copybefore);
}
//优化确定后面读到哪
mid = -1;
pos = sortarray.Find(listtemp.Last(), ref mid);
ProcessTraceUtil.Trace("查找新数据在老数据中插入的结束位置:mid=" + mid + ",pos=" + pos);
if (pos == -1 && mid != -1 && mid < indexarray.Length - 1)
{
//小于最小的
copylast = indexarray[mid + 1].KeyOffset;
ProcessTraceUtil.Trace("老数据可以直接copy的部分:" + copylast + "->" + mergeinfo.IndexMergePos);
}
else if (pos != -1)
{
copylast = indexarray[pos].KeyOffset;
ProcessTraceUtil.Trace("老数据可以直接copy的部分:" + copylast + "->" + mergeinfo.IndexMergePos);
}
}
newindexfile = (indexname.Equals(meta.KeyName) ? GetKeyFile(tablename) : GetIndexFile(tablename, indexname)) + ".temp";
if (File.Exists(newindexfile))
{
File.Delete(newindexfile);
}
//快速copy
if (copybefore > 0)
{
ProcessTraceUtil.TraceMem("直接copy前面不在排序范围的数据:0->" + copybefore, "m");
IOUtil.CopyFile(indexfile, newindexfile, FileMode.Create, 0, copybefore - 1);
readoldstartpostion = copybefore;
ProcessTraceUtil.TraceMem("copy数据完成", "m");
newdiskindexlist.AddRange(keyindexdisklist[tablename].Where(p => p.KeyOffset < copybefore));
}
bool isall = false;
ProcessTraceUtil.TraceMem("开始读取在排序范围内的数据", "m");
while (true)
{
ProcessTraceUtil.Trace("读取老数据,开始位置:" + readoldstartpostion);
reader.SetPostion(readoldstartpostion);
var listordered = new List <BigEntityTableIndexItem>();
var loadcount = 0;
long keyoffset = 0;
foreach (var item in reader.ReadObjectsWating <BigEntityTableIndexItem>(1, p => keyoffset = p, bigbuffer))
{
item.KeyOffset = keyoffset;
item.SetIndex(meta.KeyIndexInfo);
if (item.KeyOffset >= mergeinfo.IndexMergePos)
{
break;
}
if (copylast > 0 && item.KeyOffset >= copylast)
{
break;
}
listordered.Add(item);
if (++loadcount >= MERGE_TRIGGER_NEW_COUNT)
{
break;
}
}
readoldstartpostion = reader.ReadedPostion();
bool isonlyoldlist = false;
if (listordered.Count == 0)
{
ProcessTraceUtil.TraceMem("老数据没有了，全部是新数据:" + listtemp.Count, "m");
listordered = MergeAndSort2(listordered, listtemp).ToList();
foreach (var item in listordered)
{
item.RangeIndex = lastrankindex++;
}
isall = true;
}
else if (listtemp.Count == 0 && listordered.Count > 10000)
{
ProcessTraceUtil.TraceMem("新数据没有了，全部是老数据:" + listordered.Count, "m");
//copy
IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, listordered.First().KeyOffset, listordered.Last().KeyOffset - 1, false);
var listorderedlast = listordered.Last();
long copyoffset = 0;
using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
{
var item = new BigEntityTableIndexItem {
Del = listorderedlast.Del, Key = listorderedlast.Key, len = listorderedlast.len, Offset = listorderedlast.Offset, Index = listorderedlast.Index
};
item.KeyOffset = nw.GetWritePosition();
copyoffset = nw.GetWritePosition() - listorderedlast.KeyOffset;
nw.AppendObject(item);
newIndexMergePos = nw.GetWritePosition();
}
isonlyoldlist = true;
//更新索引
foreach (var item in listordered)
{
item.RangeIndex = lastrankindex++;
item.KeyOffset += copyoffset;
}
ProcessTraceUtil.TraceMem("直接copy数据完成", "m");
}
else
{
ProcessTraceUtil.TraceMem("老数据条数为:" + listordered.Count + "，新数据条数为:" + listtemp.Count, "m");
var listordermax = listordered.Last();
int idx = 0;
foreach (var item in listtemp)
{
if (item.CompareTo(listordermax) > 0)
{
break;
}
else
{
idx++;
}
}
List <BigEntityTableIndexItem> smalllist = listtemp.Take(idx).ToList();
listtemp = listtemp.Skip(idx).ToList();
listordered = MergeAndSort2(listordered, smalllist).ToList();
foreach (var item in listordered)
{
item.RangeIndex = lastrankindex++;
//ProcessTraceUtil.Trace("rangeindex:" + item.Key[0] + "->" + (item.RangeIndex));
}
ProcessTraceUtil.TraceMem("排序完成:" + listordered.Count + "条", "m");
}
if (listordered.Count > 0)
{
if (!isonlyoldlist)
{
ProcessTraceUtil.TraceMem("把排好的数据写入到新索引文件:" + listordered.Count + "条", "m");
using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
{
foreach (var item in listordered)
{
item.KeyOffset = nw.GetWritePosition();
nw.AppendObject(item);
}
newIndexMergePos = nw.GetWritePosition();
}
ProcessTraceUtil.TraceMem("写入到新索引文件完成", "m");
}
if (listordered.Count <= 2 || loadFactor == 1)
{
newdiskindexlist.AddRange(listordered);
}
else
{
newdiskindexlist.Add(listordered.First());
int idx = 0;
foreach (var item in listordered)
{
if ((++idx) % loadFactor == 0)
{
newdiskindexlist.Add(item);
}
}
//newdiskindexlist.AddRange(listordered.Where(p => (++idx) % loadFactor == 0));
if (idx % loadFactor != 0)
{
newdiskindexlist.Add(listordered.Last());
}
}
}
ProcessTraceUtil.TraceMem("写入到新索引文件后整理索引完成", "m");
if (isall)
{
if (copylast > 0 && copylast < mergeinfo.IndexMergePos)
{
ProcessTraceUtil.TraceMem("copy已排序的大于新增最大数部分" + copylast + "->" + mergeinfo.IndexMergePos, "m");
var offset = 0L;
using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
{
offset = nw.GetWritePosition() - copylast;
}
//copy
long newindexpos = IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, copylast, mergeinfo.IndexMergePos - 1, false);
foreach (var p in keyindexdisklist[tablename])
{
if (p.KeyOffset >= copylast && p.KeyOffset < mergeinfo.IndexMergePos)
{
newdiskindexlist.Add(new BigEntityTableIndexItem
{
Del = p.Del,
Key = p.Key,
KeyOffset = p.KeyOffset + offset,
len = p.len,
Offset = p.Offset,
Index = p.Index,
RangeIndex = p.RangeIndex + readcount
});
}
}
ProcessTraceUtil.TraceMem("copy数据完成->" + offset, "m");
}
break;
}
else
{
if (listtemp.Count > 0)
{
long newcopybefore = 0;
using (var sortarray = new Collections.SorteArray <BigEntityTableIndexItem>(indexarray))
{
int mid = -1;
int pos = sortarray.Find(listtemp.First(), ref mid);
ProcessTraceUtil.Trace("查找已经排序的小于新增最小数据部分:mid=" + mid + ",pos=" + pos);
if (pos == -1 && mid != -1)
{
//小于最小的
newcopybefore = indexarray[mid].KeyOffset;
lastrankindex = indexarray[mid].RangeIndex + readcount - listtemp.Count;
}
else if (pos != -1)
{
newcopybefore = indexarray[pos].KeyOffset;
lastrankindex = indexarray[pos].RangeIndex + readcount - listtemp.Count;
}
}
if (newcopybefore > readoldstartpostion)
{
ProcessTraceUtil.Trace("中间copy");
var offset = 0L;
using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
{
offset = nw.GetWritePosition() - readoldstartpostion;
}
IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, readoldstartpostion, newcopybefore - 1, false);
foreach (var p in keyindexdisklist[tablename])
{
if (p.KeyOffset >= readoldstartpostion && p.KeyOffset < newcopybefore)
{
newdiskindexlist.Add(new BigEntityTableIndexItem
{
Del = p.Del,
Key = p.Key,
KeyOffset = p.KeyOffset + offset,
len = p.len,
Offset = p.Offset,
Index = p.Index,
RangeIndex = p.RangeIndex + readcount - listtemp.Count
});
}
}
readoldstartpostion = newcopybefore;
ProcessTraceUtil.Trace("中间copy完成");
}
else if (newcopybefore < readoldstartpostion)
{
ProcessTraceUtil.Trace("补中间");
reader.SetPostion(newcopybefore);
//补充数目
foreach (var item in reader.ReadObjectsWating <BigEntityTableIndexItem>(1, p => keyoffset = p, smallbuffer))
{
item.KeyOffset = keyoffset;
item.SetIndex(meta.KeyIndexInfo);
//ProcessTraceUtil.Trace(item.Key[0].ToString());
if (item.KeyOffset >= mergeinfo.IndexMergePos)
{
break;
}
if (copylast > 0 && item.KeyOffset >= copylast)
{
break;
}
if (item.KeyOffset == readoldstartpostion)
{
break;
}
if (item.Del)
{
continue;
}
//ProcessTraceUtil.Trace("补中间+1");
lastrankindex++;
}
}
}
else
{
ProcessTraceUtil.Trace("中间copy2");
var offset = 0L;
using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
{
offset = nw.GetWritePosition() - readoldstartpostion;
}
IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, readoldstartpostion, copylast - 1, false);
foreach (var p in keyindexdisklist[tablename])
{
if (p.KeyOffset >= readoldstartpostion && p.KeyOffset < copylast)
{
newdiskindexlist.Add(new BigEntityTableIndexItem
{
Del = p.Del,
Key = p.Key,
KeyOffset = p.KeyOffset + offset,
len = p.len,
Offset = p.Offset,
Index = p.Index,
RangeIndex = p.RangeIndex + readcount
});
}
}
readoldstartpostion = copylast;
ProcessTraceUtil.Trace("中间copy2完成");
}
}
}
}
//后面copy
string tablefile = GetTableFile(tablename);
var idxreader = ObjTextReader.CreateReader(indexfile);
var newwriter = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2);
try
{
long nextcopypos = 0;
ProcessTraceUtil.TraceMem("读取后面的数据->" + lasmargepos, "m");
idxreader.SetPostion(lasmargepos);
bool hasitem = false;
bool isfirst = true;
long keyoffset = 0;
foreach (var item in idxreader.ReadObjectsWating <BigEntityTableIndexItem>(1, p => keyoffset = p, smallbuffer))
{
hasitem = true;
item.SetIndex(meta.KeyIndexInfo);
item.KeyOffset = keyoffset;
if (item.KeyOffset > newwriter.GetWritePosition())
{
var spacelen = item.KeyOffset - newwriter.GetWritePosition();
ProcessTraceUtil.Trace("没有对齐，尝试对齐，spacelen->" + spacelen);
if (spacelen % 3 == 0)
{
newwriter.FillSpace(spacelen / 3);
}
else
{
ProcessTraceUtil.Trace("对齐失败");
var ex = new Exception("无法对齐");
ex.Data.Add("老索引文件当前位置", item.KeyOffset);
ex.Data.Add("新索引文件写入位置", newwriter.GetWritePosition());
throw ex;
}
}
if (item.KeyOffset == newwriter.GetWritePosition())
{
nextcopypos = newwriter.GetWritePosition();
if (isfirst)
{
newIndexMergePos = nextcopypos;
}
ProcessTraceUtil.Trace("新老索引文件已经对齐:" + item.KeyOffset);
break;
}
isfirst = false;
item.KeyOffset = newwriter.GetWritePosition();
newwriter.AppendObject(item);
}
if (nextcopypos > 0)
{
newwriter.Dispose();
ProcessTraceUtil.Trace("copy后面的数据->" + nextcopypos);
nextcopypos = IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, nextcopypos, -512);
}
else if (!hasitem)
{
var idxpos = idxreader.ReadedPostion();
if (idxpos == newwriter.GetWritePosition())
{
nextcopypos = idxpos;
newwriter.Dispose();
}
else
{
ProcessTraceUtil.Trace(idxpos + " vs " + newwriter.GetWritePosition());
}
}
ProcessTraceUtil.TraceMem("读取后面的数据完成", "m");
if (nextcopypos == 0)
{
throw new Exception("更新索引出错");
}
try
{
BigEntityTableIndexItem[] oldindexarray = null;
BigEntityTableIndexItem[] newdiskindexarray = newdiskindexlist.ToArray();
tablelocker.EnterWriteLock();
ProcessTraceUtil.TraceMem("读取后面的数据->" + lasmargepos, "m");
if (nextcopypos <= 0)
{
using (newwriter)
{
using (idxreader)
{
foreach (var item in idxreader.ReadObjectsWating <BigEntityTableIndexItem>(1, bytes: bigbuffer))
{
item.SetIndex(meta.KeyIndexInfo);
item.KeyOffset = newwriter.GetWritePosition();
newwriter.AppendObject(item);
}
}
}
}
else
{
nextcopypos = IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, nextcopypos, long.MaxValue);
ProcessTraceUtil.TraceMem("继续copy后面的数据->" + nextcopypos, "m");
idxreader.Dispose();
}
ProcessTraceUtil.TraceMem("读取后面的数据完成", "m");
//更新索引
mergeinfo.LoadFactor = loadFactor;
keyindexdisklist.TryRemove(tablename, out oldindexarray);
keyindexdisklist.TryAdd(tablename, newdiskindexarray);
keyindexmemtemplist[tablename] = new SortArrayList <BigEntityTableIndexItem>();
int trycount = 0;
while (true)
{
try
{
File.Delete(indexfile);
ProcessTraceUtil.Trace("删除旧文件完成");
break;
}
catch (System.IO.IOException ex)
{
Thread.Sleep(1);
trycount++;
if (trycount > 1000)
{
throw ex;
}
}
}
File.Move(newindexfile, indexfile);
ProcessTraceUtil.TraceMem("删除旧文件，重命名新文件完成", "m");
idxreader = null;
}
finally
{
tablelocker.ExitWriteLock();
}
string metafile = GetMetaFile(tablename);
mergeinfo.IndexMergePos = lasmargepos;
SerializerHelper.SerializerToXML(meta, metafile, true);
ProcessTraceUtil.Trace("更新元文件，更新索引完成");
}
catch (Exception ex)
{
Console.WriteLine("整理索引后面部分出错" + ex.ToString());
//LogManager.LogHelper.Instance.Error("整理索引后面部分出错", ex);
}
finally
{
if (idxreader != null)
{
idxreader.Dispose();
}
if (newwriter != null && !newwriter.Isdispose)
{
newwriter.Dispose();
}
if (File.Exists(newindexfile))
{
File.Delete(newindexfile);
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
//LogManager.LogHelper.Instance.Error("整理索引出错", ex);
}
finally
{
//GC.Collect();
//ProcessTraceUtil.TraceMem("回收内存","m");
mergeinfo.IsMergin = false;
var info = ProcessTraceUtil.PrintTrace();
Console.WriteLine(info);
//LogManager.LogHelper.Instance.Debug("整理索引过程:" + info);
}
}
public async Task <List <JobExtensionRunner> > PrepareActionsAsync(IExecutionContext executionContext, IEnumerable <Pipelines.JobStep> steps)
{
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(steps, nameof(steps));
executionContext.Output("Prepare all required actions");
Dictionary <string, List <Guid> > imagesToPull = new Dictionary <string, List <Guid> >(StringComparer.OrdinalIgnoreCase);
Dictionary <string, List <Guid> > imagesToBuild = new Dictionary <string, List <Guid> >(StringComparer.OrdinalIgnoreCase);
Dictionary <string, ActionContainer> imagesToBuildInfo = new Dictionary <string, ActionContainer>(StringComparer.OrdinalIgnoreCase);
List <JobExtensionRunner> containerSetupSteps = new List <JobExtensionRunner>();
IEnumerable <Pipelines.ActionStep> actions = steps.OfType <Pipelines.ActionStep>();
// TODO: Deprecate the PREVIEW_ACTION_TOKEN
// Log even if we aren't using it to ensure users know.
if (!string.IsNullOrEmpty(executionContext.Variables.Get("PREVIEW_ACTION_TOKEN")))
{
executionContext.Warning("The 'PREVIEW_ACTION_TOKEN' secret is deprecated. Please remove it from the repository's secrets");
}
// Clear the cache (for self-hosted runners)
// Note, temporarily avoid this step for the on-premises product, to avoid rate limiting.
var configurationStore = HostContext.GetService <IConfigurationStore>();
var isHostedServer = configurationStore.GetSettings().IsHostedServer;
if (isHostedServer)
{
IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken);
}
foreach (var action in actions)
{
if (action.Reference.Type == Pipelines.ActionSourceType.ContainerRegistry)
{
ArgUtil.NotNull(action, nameof(action));
var containerReference = action.Reference as Pipelines.ContainerRegistryReference;
ArgUtil.NotNull(containerReference, nameof(containerReference));
ArgUtil.NotNullOrEmpty(containerReference.Image, nameof(containerReference.Image));
if (!imagesToPull.ContainsKey(containerReference.Image))
{
imagesToPull[containerReference.Image] = new List <Guid>();
}
Trace.Info($"Action {action.Name} ({action.Id}) needs to pull image '{containerReference.Image}'");
imagesToPull[containerReference.Image].Add(action.Id);
}
else if (action.Reference.Type == Pipelines.ActionSourceType.Repository)
{
// only download the repository archive
await DownloadRepositoryActionAsync(executionContext, action);
// more preparation base on content in the repository (action.yml)
var setupInfo = PrepareRepositoryActionAsync(executionContext, action);
if (setupInfo != null)
{
if (!string.IsNullOrEmpty(setupInfo.Image))
{
if (!imagesToPull.ContainsKey(setupInfo.Image))
{
imagesToPull[setupInfo.Image] = new List <Guid>();
}
Trace.Info($"Action {action.Name} ({action.Id}) from repository '{setupInfo.ActionRepository}' needs to pull image '{setupInfo.Image}'");
imagesToPull[setupInfo.Image].Add(action.Id);
}
else
{
ArgUtil.NotNullOrEmpty(setupInfo.ActionRepository, nameof(setupInfo.ActionRepository));
if (!imagesToBuild.ContainsKey(setupInfo.ActionRepository))
{
imagesToBuild[setupInfo.ActionRepository] = new List <Guid>();
}
Trace.Info($"Action {action.Name} ({action.Id}) from repository '{setupInfo.ActionRepository}' needs to build image '{setupInfo.Dockerfile}'");
imagesToBuild[setupInfo.ActionRepository].Add(action.Id);
imagesToBuildInfo[setupInfo.ActionRepository] = setupInfo;
}
}
}
}
if (imagesToPull.Count > 0)
{
foreach (var imageToPull in imagesToPull)
{
Trace.Info($"{imageToPull.Value.Count} steps need to pull image '{imageToPull.Key}'");
containerSetupSteps.Add(new JobExtensionRunner(runAsync: this.PullActionContainerAsync,
condition: $"{PipelineTemplateConstants.Success}()",
displayName: $"Pull {imageToPull.Key}",
data: new ContainerSetupInfo(imageToPull.Value, imageToPull.Key)));
}
}
if (imagesToBuild.Count > 0)
{
foreach (var imageToBuild in imagesToBuild)
{
var setupInfo = imagesToBuildInfo[imageToBuild.Key];
Trace.Info($"{imageToBuild.Value.Count} steps need to build image from '{setupInfo.Dockerfile}'");
containerSetupSteps.Add(new JobExtensionRunner(runAsync: this.BuildActionContainerAsync,
condition: $"{PipelineTemplateConstants.Success}()",
displayName: $"Build {setupInfo.ActionRepository}",
data: new ContainerSetupInfo(imageToBuild.Value, setupInfo.Dockerfile, setupInfo.WorkingDirectory)));
}
}
#if !OS_LINUX
if (containerSetupSteps.Count > 0)
{
executionContext.Output("Container action is only supported on Linux, skip pull and build docker images.");
containerSetupSteps.Clear();
}
#endif
return(containerSetupSteps);
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(Inputs, nameof(Inputs));
ArgUtil.Directory(TaskDirectory, nameof(TaskDirectory));
// Warn about legacy handler.
ExecutionContext.Warning($"Task '{this.Task.Name}' ({this.Task.Version}) is using deprecated task execution handler. The task should use the supported task-lib: https://aka.ms/tasklib");
// Resolve the target script.
string target = GetTarget();
ArgUtil.NotNullOrEmpty(target, nameof(target));
string scriptFile = Path.Combine(TaskDirectory, target);
ArgUtil.File(scriptFile, nameof(scriptFile));
// Determine the working directory.
string workingDirectory = GetWorkingDirectory();
if (String.IsNullOrEmpty(workingDirectory))
{
workingDirectory = Path.GetDirectoryName(scriptFile);
}
else
{
if (!Directory.Exists(workingDirectory))
{
Directory.CreateDirectory(workingDirectory);
}
}
// Copy the OM binaries into the legacy host folder.
ExecutionContext.Output(StringUtil.Loc("PrepareTaskExecutionHandler"));
IOUtil.CopyDirectory(
source: HostContext.GetDirectory(WellKnownDirectory.ServerOM),
target: HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost),
cancellationToken: ExecutionContext.CancellationToken);
Trace.Info("Finished copying files.");
// Add the legacy ps host environment variables.
AddLegacyHostEnvironmentVariables(scriptFile: scriptFile, workingDirectory: workingDirectory);
AddPrependPathToEnvironment();
// Add proxy setting to LegacyVSTSPowerShellHost.exe.config
var agentProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentProxy.ProxyAddress))
{
AddProxySetting(agentProxy);
}
// Invoke the process.
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += OnDataReceived;
processInvoker.ErrorDataReceived += OnDataReceived;
try
{
String vstsPSHostExe = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost), "LegacyVSTSPowerShellHost.exe");
Int32 exitCode = await processInvoker.ExecuteAsync(workingDirectory : workingDirectory,
fileName : vstsPSHostExe,
arguments : "",
environment : Environment,
requireExitCodeZero : false,
outputEncoding : null,
killProcessOnCancel : false,
redirectStandardIn : null,
inheritConsoleHandler : !ExecutionContext.Variables.Retain_Default_Encoding,
cancellationToken : ExecutionContext.CancellationToken);
// the exit code from vstsPSHost.exe indicate how many error record we get during execution
// -1 exit code means infrastructure failure of Host itself.
// this is to match current handler's logic.
if (exitCode > 0)
{
if (ExecutionContext.Result != null)
{
ExecutionContext.Debug($"Task result already set. Not failing due to error count ({exitCode}).");
}
else
{
// We fail task and add issue.
ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Error(StringUtil.Loc("PSScriptError", exitCode));
}
}
else if (exitCode < 0)
{
// We fail task and add issue.
ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Error(StringUtil.Loc("VSTSHostNonZeroReturn", exitCode));
}
}
finally
{
processInvoker.OutputDataReceived -= OnDataReceived;
processInvoker.ErrorDataReceived -= OnDataReceived;
}
}
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(Data, nameof(Data));
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(Inputs, nameof(Inputs));
ArgUtil.Directory(TaskDirectory, nameof(TaskDirectory));
// Resolve the target script.
ArgUtil.NotNullOrEmpty(Data.Target, nameof(Data.Target));
string scriptFile = Path.Combine(TaskDirectory, Data.Target);
ArgUtil.File(scriptFile, nameof(scriptFile));
// Determine the working directory.
string workingDirectory = Data.WorkingDirectory;
if (String.IsNullOrEmpty(workingDirectory))
{
workingDirectory = Path.GetDirectoryName(scriptFile);
}
else
{
if (!Directory.Exists(workingDirectory))
{
Directory.CreateDirectory(workingDirectory);
}
}
// scriptName
AddEnvironmentVariable("VSTSPSHOSTSCRIPTNAME", scriptFile);
// workingFolder
AddEnvironmentVariable("VSTSPSHOSTWORKINGFOLDER", workingDirectory);
// outputPreference
AddEnvironmentVariable("VSTSPSHOSTOUTPUTPREFER", ExecutionContext.WriteDebug ? "Continue" : "SilentlyContinue");
// inputParameters
if (Inputs.Count > 0)
{
AddEnvironmentVariable("VSTSPSHOSTINPUTPARAMETER", JsonUtility.ToString(Inputs));
}
List <String> arguments = new List <string>();
Dictionary <String, String> argumentParameters = new Dictionary <String, String>();
if (string.IsNullOrEmpty(Data.ArgumentFormat))
{
// treatInputsAsArguments
AddEnvironmentVariable("VSTSPSHOSTINPUTISARG", "True");
}
else
{
MatchCollection matches = _argumentMatching.Matches(Data.ArgumentFormat);
if (matches[0].Value.StartsWith("-"))
{
String currentKey = String.Empty;
foreach (Match match in matches)
{
if (match.Value.StartsWith("-"))
{
currentKey = match.Value.Trim('-');
argumentParameters.Add(currentKey, String.Empty);
}
else if (!match.Value.StartsWith("-") && !String.IsNullOrEmpty(currentKey))
{
argumentParameters[currentKey] = match.Value;
currentKey = String.Empty;
}
else
{
throw new Exception($"Found value {match.Value} with no corresponding named parameter");
}
}
}
else
{
foreach (Match match in matches)
{
arguments.Add(match.Value);
}
}
// arguments
if (arguments.Count > 0)
{
AddEnvironmentVariable("VSTSPSHOSTARGS", JsonUtility.ToString(arguments));
}
// argumentParameters
if (argumentParameters.Count > 0)
{
AddEnvironmentVariable("VSTSPSHOSTARGPARAMETER", JsonUtility.ToString(argumentParameters));
}
}
// additionalStatement
List <Tuple <String, List <Tuple <String, String> > > > additionalStatement = GetAdditionalCommandsForAzurePowerShell(Inputs);
if (additionalStatement.Count > 0)
{
AddEnvironmentVariable("VSTSPSHOSTSTATEMENTS", JsonUtility.ToString(additionalStatement));
}
// push all variable.
foreach (var variable in ExecutionContext.Variables.Public.Concat(ExecutionContext.Variables.Private))
{
AddEnvironmentVariable("VSTSPSHOSTVAR_" + variable.Key, variable.Value);
}
// push all public variable.
foreach (var variable in ExecutionContext.Variables.Public)
{
AddEnvironmentVariable("VSTSPSHOSTPUBVAR_" + variable.Key, variable.Value);
}
// push all endpoints
List <String> ids = new List <string>();
foreach (ServiceEndpoint endpoint in ExecutionContext.Endpoints)
{
string partialKey = null;
if (string.Equals(endpoint.Name, ServiceEndpoints.SystemVssConnection, StringComparison.OrdinalIgnoreCase))
{
partialKey = ServiceEndpoints.SystemVssConnection.ToUpperInvariant();
AddEnvironmentVariable("VSTSPSHOSTSYSTEMENDPOINT_URL", endpoint.Url.ToString());
AddEnvironmentVariable("VSTSPSHOSTSYSTEMENDPOINT_AUTH", JsonUtility.ToString(endpoint.Authorization));
}
else
{
partialKey = endpoint.Id.ToString("D").ToUpperInvariant();
ids.Add(partialKey);
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_URL_" + partialKey, endpoint.Url.ToString());
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_NAME_" + partialKey, endpoint.Name);
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_TYPE_" + partialKey, endpoint.Type);
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_AUTH_" + partialKey, JsonUtility.ToString(endpoint.Authorization));
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_DATA_" + partialKey, JsonUtility.ToString(endpoint.Data));
}
}
if (ids.Count > 0)
{
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_IDS", JsonUtility.ToString(ids));
}
// Invoke the process.
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += OnDataReceived;
processInvoker.ErrorDataReceived += OnDataReceived;
try
{
String vstsPSHostExe = Path.Combine(IOUtil.GetExternalsPath(), "vstshost", "LegacyVSTSPowerShellHost.exe");
Int32 exitCode = await processInvoker.ExecuteAsync(workingDirectory : workingDirectory,
fileName : vstsPSHostExe,
arguments : "",
environment : Environment,
cancellationToken : ExecutionContext.CancellationToken);
// the exit code from vstsPSHost.exe indicate how many error record we get during execution
// -1 exit code means infrastructure failure of Host itself.
// this is to match current handler's logic.
if (exitCode > 0)
{
if (ExecutionContext.Result != null)
{
Trace.Info($"Task result already set. Not failing due to error count ({exitCode}).");
}
else
{
// We fail task and add issue.
ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Error(StringUtil.Loc("PSScriptError", exitCode));
}
}
else if (exitCode < 0)
{
// We fail task and add issue.
ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Error(StringUtil.Loc("VSTSHostNonZeroReturn", exitCode));
}
}
finally
{
processInvoker.OutputDataReceived -= OnDataReceived;
processInvoker.ErrorDataReceived -= OnDataReceived;
}
}
}
private void button_Import_Click(object sender, EventArgs e)
{
OpenFileDialog dialog = new OpenFileDialog();
dialog.Filter = "bin files (*.txt)|*.txt";
dialog.FileName = "";
dialog.Title = "导入文本";
DialogResult dr = dialog.ShowDialog();
bool bImport = (dr == DialogResult.OK);
dialog.Dispose();
FileStream fs = null;
if (bImport)
{
String fileName = dialog.FileName;
try
{
if (File.Exists(fileName))
{
fs = File.Open(fileName, FileMode.Open);
}
else
{
return;
}
ArrayList texts = IOUtil.readTextLinesGBK(fs);
for (int i = 0; i < textsManager.getElementCount(); i++)
{
if (i >= texts.Count)
{
break;
}
String text = (String)texts[i];
TextElement element = textsManager.getElement(i);
element.setValue(text);
}
textsManager.refreshUI();
textsManager.refreshUIAide();
}
catch (Exception ex1)
{
Console.WriteLine(ex1.StackTrace);
}
finally
{
if (fs != null)
{
try
{
fs.Close();
fs = null;
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
}
}
textsManager.refreshUI(listBox_Texts);
}
public ConfigurationManagerL0()
{
_agentServer = new Mock <IAgentServer>();
_credMgr = new Mock <ICredentialManager>();
_promptManager = new Mock <IPromptManager>();
_store = new Mock <IConfigurationStore>();
_extnMgr = new Mock <IExtensionManager>();
_rsaKeyManager = new Mock <IRSAKeyManager>();
#if OS_WINDOWS
_serviceControlManager = new Mock <IWindowsServiceControlManager>();
#endif
#if !OS_WINDOWS
_serviceControlManager = new Mock <ILinuxServiceControlManager>();
#endif
#if !OS_WINDOWS
string eulaFile = Path.Combine(IOUtil.GetExternalsPath(), Constants.Path.TeeDirectory, "license.html");
Directory.CreateDirectory(IOUtil.GetExternalsPath());
Directory.CreateDirectory(Path.Combine(IOUtil.GetExternalsPath(), Constants.Path.TeeDirectory));
File.WriteAllText(eulaFile, "testeulafile");
#endif
_capabilitiesManager = new CapabilitiesManager();
_agentServer.Setup(x => x.ConnectAsync(It.IsAny <VssConnection>())).Returns(Task.FromResult <object>(null));
_store.Setup(x => x.IsConfigured()).Returns(false);
_store.Setup(x => x.HasCredentials()).Returns(false);
_store.Setup(x => x.GetSettings()).Returns(
() => _configMgrAgentSettings
);
_store.Setup(x => x.SaveSettings(It.IsAny <AgentSettings>())).Callback((AgentSettings settings) =>
{
_configMgrAgentSettings = settings;
});
_credMgr.Setup(x => x.GetCredentialProvider(It.IsAny <string>())).Returns(new TestAgentCredential());
#if !OS_WINDOWS
_serviceControlManager.Setup(x => x.GenerateScripts(It.IsAny <AgentSettings>()));
#endif
var expectedPools = new List <TaskAgentPool>()
{
new TaskAgentPool(_expectedPoolName)
{
Id = _expectedPoolId
}
};
_agentServer.Setup(x => x.GetAgentPoolsAsync(It.IsAny <string>())).Returns(Task.FromResult(expectedPools));
var expectedAgents = new List <TaskAgent>();
_agentServer.Setup(x => x.GetAgentsAsync(It.IsAny <int>(), It.IsAny <string>())).Returns(Task.FromResult(expectedAgents));
var expectedAgent = new TaskAgent(_expectedAgentName)
{
Id = 1
};
_agentServer.Setup(x => x.AddAgentAsync(It.IsAny <int>(), It.IsAny <TaskAgent>())).Returns(Task.FromResult(expectedAgent));
_agentServer.Setup(x => x.UpdateAgentAsync(It.IsAny <int>(), It.IsAny <TaskAgent>())).Returns(Task.FromResult(expectedAgent));
rsa = RSA.Create();
rsa.KeySize = 2048;
_rsaKeyManager.Setup(x => x.CreateKey()).Returns(rsa);
}
/// <summary>
/// Reads and parses data from the DDS file to be imported.
/// </summary>
/// <param name="br">The currently active BinaryReader.</param>
/// <param name="xivTex">The Texture data.</param>
/// <param name="newWidth">The width of the DDS texture to be imported.</param>
/// <param name="newHeight">The height of the DDS texture to be imported.</param>
/// <param name="newMipCount">The number of mipmaps the DDS texture to be imported contains.</param>
/// <returns>A tuple containing the compressed DDS data, a list of offsets to the mipmap parts, a list with the number of parts per mipmap.</returns>
public static (List <byte> compressedDDS, List <short> mipPartOffsets, List <short> mipPartCounts) ReadDDS(BinaryReader br, XivTex xivTex, int newWidth, int newHeight, int newMipCount)
{
var compressedDDS = new List <byte>();
var mipPartOffsets = new List <short>();
var mipPartCount = new List <short>();
int mipLength;
switch (xivTex.TextureFormat)
{
case XivTexFormat.DXT1:
mipLength = (newWidth * newHeight) / 2;
break;
case XivTexFormat.DXT5:
case XivTexFormat.A8:
mipLength = newWidth * newHeight;
break;
case XivTexFormat.A1R5G5B5:
case XivTexFormat.A4R4G4B4:
mipLength = (newWidth * newHeight) * 2;
break;
case XivTexFormat.L8:
case XivTexFormat.A8R8G8B8:
case XivTexFormat.X8R8G8B8:
case XivTexFormat.R32F:
case XivTexFormat.G16R16F:
case XivTexFormat.G32R32F:
case XivTexFormat.A16B16G16R16F:
case XivTexFormat.A32B32G32R32F:
case XivTexFormat.DXT3:
case XivTexFormat.D16:
default:
mipLength = (newWidth * newHeight) * 4;
break;
}
br.BaseStream.Seek(128, SeekOrigin.Begin);
for (var i = 0; i < newMipCount; i++)
{
var mipParts = (int)Math.Ceiling(mipLength / 16000f);
mipPartCount.Add((short)mipParts);
if (mipParts > 1)
{
for (var j = 0; j < mipParts; j++)
{
int uncompLength;
var comp = true;
if (j == mipParts - 1)
{
uncompLength = mipLength % 16000;
}
else
{
uncompLength = 16000;
}
var uncompBytes = br.ReadBytes(uncompLength);
var compressed = IOUtil.Compressor(uncompBytes);
if (compressed.Length > uncompLength)
{
compressed = uncompBytes;
comp = false;
}
compressedDDS.AddRange(BitConverter.GetBytes(16));
compressedDDS.AddRange(BitConverter.GetBytes(0));
compressedDDS.AddRange(!comp
? BitConverter.GetBytes(32000)
: BitConverter.GetBytes(compressed.Length));
compressedDDS.AddRange(BitConverter.GetBytes(uncompLength));
compressedDDS.AddRange(compressed);
var padding = 128 - (compressed.Length % 128);
compressedDDS.AddRange(new byte[padding]);
mipPartOffsets.Add((short)(compressed.Length + padding + 16));
}
}
else
{
int uncompLength;
var comp = true;
if (mipLength != 16000)
{
uncompLength = mipLength % 16000;
}
else
{
uncompLength = 16000;
}
var uncompBytes = br.ReadBytes(uncompLength);
var compressed = IOUtil.Compressor(uncompBytes);
if (compressed.Length > uncompLength)
{
compressed = uncompBytes;
comp = false;
}
compressedDDS.AddRange(BitConverter.GetBytes(16));
compressedDDS.AddRange(BitConverter.GetBytes(0));
compressedDDS.AddRange(!comp
? BitConverter.GetBytes(32000)
: BitConverter.GetBytes(compressed.Length));
compressedDDS.AddRange(BitConverter.GetBytes(uncompLength));
compressedDDS.AddRange(compressed);
var padding = 128 - (compressed.Length % 128);
compressedDDS.AddRange(new byte[padding]);
mipPartOffsets.Add((short)(compressed.Length + padding + 16));
}
if (mipLength > 32)
{
mipLength = mipLength / 4;
}
else
{
mipLength = 8;
}
}
return(compressedDDS, mipPartOffsets, mipPartCount);
}
public async Task <TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Steps, nameof(message.Steps));
Trace.Info("Job ID {0}", message.JobId);
DateTime jobStartTimeUtc = DateTime.UtcNow;
ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
Uri jobServerUrl = systemConnection.Url;
Trace.Info($"Creating job server with URL: {jobServerUrl}");
// jobServerQueue is the throttling reporter.
_jobServerQueue = HostContext.GetService <IJobServerQueue>();
VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
await jobServer.ConnectAsync(jobConnection);
_jobServerQueue.Start(message);
HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}");
IExecutionContext jobContext = null;
CancellationTokenRegistration?runnerShutdownRegistration = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
jobContext.Debug($"Starting: {message.JobDisplayName}");
runnerShutdownRegistration = HostContext.RunnerShutdownToken.Register(() =>
{
// log an issue, then runner get shutdown by Ctrl-C or Ctrl-Break.
// the server will use Ctrl-Break to tells the runner that operating system is shutting down.
string errorMessage;
switch (HostContext.RunnerShutdownReason)
{
case ShutdownReason.UserCancelled:
errorMessage = "The runner has received a shutdown signal. This can happen when the runner service is stopped, or a manually started runner is canceled.";
break;
case ShutdownReason.OperatingSystemShutdown:
errorMessage = $"Operating system is shutting down for computer '{Environment.MachineName}'";
break;
default:
throw new ArgumentException(HostContext.RunnerShutdownReason.ToString(), nameof(HostContext.RunnerShutdownReason));
}
jobContext.AddIssue(new Issue()
{
Type = IssueType.Error, Message = errorMessage
});
});
// Validate directory permissions.
string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
Trace.Info($"Validating directory permissions for: '{workDirectory}'");
try
{
Directory.CreateDirectory(workDirectory);
IOUtil.ValidateExecutePermission(workDirectory);
}
catch (Exception ex)
{
Trace.Error(ex);
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
jobContext.SetRunnerContext("os", VarUtil.OS);
string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools);
Directory.CreateDirectory(toolsDirectory);
jobContext.SetRunnerContext("tool_cache", toolsDirectory);
// Setup TEMP directories
_tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>();
_tempDirectoryManager.InitializeTempDirectory(jobContext);
// // Expand container properties
// jobContext.Container?.ExpandProperties(jobContext.Variables);
// foreach (var sidecar in jobContext.SidecarContainers)
// {
// sidecar.ExpandProperties(jobContext.Variables);
// }
// Get the job extension.
Trace.Info("Getting job extension.");
IJobExtension jobExtension = HostContext.CreateService <IJobExtension>();
List <IStep> jobSteps = null;
try
{
Trace.Info("Initialize job. Getting all job steps.");
jobSteps = await jobExtension.InitializeJob(jobContext, message);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// set the job to canceled
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job is canceled during initialize.");
Trace.Error($"Caught exception: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
}
catch (Exception ex)
{
// set the job to failed.
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job initialize failed.");
Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// trace out all steps
Trace.Info($"Total job steps: {jobSteps.Count}.");
Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'");
HostContext.WritePerfCounter($"WorkerJobInitialized_{message.RequestId.ToString()}");
// Run all job steps
Trace.Info("Run all job steps.");
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
foreach (var step in jobSteps)
{
jobContext.JobSteps.Enqueue(step);
}
await stepsRunner.RunAsync(jobContext);
}
catch (Exception ex)
{
// StepRunner should never throw exception out.
// End up here mean there is a bug in StepRunner
// Log the error and fail the job.
Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
finally
{
Trace.Info("Finalize job.");
jobExtension.FinalizeJob(jobContext, message, jobStartTimeUtc);
}
Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}");
Trace.Info("Completing the job execution context.");
return(await CompleteJobAsync(jobServer, jobContext, message));
}
finally
{
if (runnerShutdownRegistration != null)
{
runnerShutdownRegistration.Value.Dispose();
runnerShutdownRegistration = null;
}
await ShutdownQueue(throwOnFailure : false);
}
}
public AutoLoadEntry(byte[] Data, uint Offset)
{
Address = IOUtil.ReadU32LE(Data, (int)Offset + 0);
Size = IOUtil.ReadU32LE(Data, (int)Offset + 4);
BssSize = IOUtil.ReadU32LE(Data, (int)Offset + 8);
}
public void DeleteAgentRuntimeOptions()
{
IOUtil.Delete(_runtimeOptionsFilePath, default(CancellationToken));
}
public void DeleteSettings()
{
IOUtil.Delete(_configFilePath, default(CancellationToken));
}
/// <summary>
/// Map a view as a Stream.
/// </summary>
/// <param name="access">desired access to the view</param>
/// <param name="offset">offset of the file mapping object to start view at</param>
/// <param name="size">size of the view</param>
/// <exception cref="ObjectDisposedException">
/// MemoryMappedFile cannot create views after it is disposed.
/// </exception>
/// <exception cref="IOException">Memory mapping failed at the Win32 layer</exception>
private Stream MapView(IOUtil.Win32.MapAccess access, long offset, int size)
{
if (hMap == IOUtil.Win32.NULL_HANDLE)
{
throw new ObjectDisposedException("MemoryMappedFile is disposed and cannot create views");
}
IntPtr baseAddress = IntPtr.Zero;
baseAddress = IOUtil.Win32.MapViewOfFile(hMap, (int)access, 0, 0, size);
if (baseAddress == IntPtr.Zero)
{
throw new IOException("Null return for MapViewOfFile", Marshal.GetHRForLastWin32Error());
}
// Find out what MapProtection to use based on the MapAccess flags:
IOUtil.Win32.Protect protection;
if ((access & IOUtil.Win32.MapAccess.FILE_MAP_READ) != 0)
{
protection = IOUtil.Win32.Protect.PAGE_READONLY;
}
else
{
protection = IOUtil.Win32.Protect.PAGE_READWRITE;
}
return new MapViewStream(baseAddress, size, protection);
}
public override bool ConfigureService(
AgentSettings settings,
CommandSettings command)
{
Trace.Entering();
var _linuxServiceHelper = HostContext.GetService <INativeLinuxServiceHelper>();
CalculateServiceName(settings, ServiceNamePattern, ServiceDisplayNamePattern);
if (!_linuxServiceHelper.CheckIfSystemdExists())
{
Trace.Info("Systemd does not exists, returning");
_term.WriteLine(StringUtil.Loc("SystemdDoesNotExists"));
return(false);
}
if (CheckServiceExists(ServiceName))
{
Trace.Info("Service already exists");
_term.WriteLine(StringUtil.Loc("ServiceAleadyExists"));
StopService();
}
var unitFile = _linuxServiceHelper.GetUnitFile(ServiceName);
try
{
var unitContent = File.ReadAllText(Path.Combine(IOUtil.GetBinPath(), VstsAgentServiceTemplate));
var tokensToReplace = new Dictionary <string, string>
{
{ "{{Description}}", ServiceDisplayName },
{ "{{BinDirectory}}", IOUtil.GetBinPath() },
{ "{{AgentRoot}}", IOUtil.GetRootPath() },
{ "{{ExternalsDirectory}}", IOUtil.GetExternalsPath() },
{ "{{User}}", GetCurrentLoginName() },
{ "{{Path}}", Environment.GetEnvironmentVariable("PATH") }
};
unitContent = tokensToReplace.Aggregate(
unitContent,
(current, item) => current.Replace(item.Key, item.Value));
File.WriteAllText(unitFile, unitContent);
// unit file should not be executable and world writable
chmod(unitFile, Convert.ToInt32("664", 8));
}
catch (UnauthorizedAccessException ex)
{
Trace.Error(ex);
_term.WriteError(StringUtil.Loc("UnauthorizedAccess", unitFile));
throw;
}
catch (Exception ex)
{
Trace.Error(ex);
throw;
}
ReloadSystemd();
InstallService(ServiceName);
_term.WriteLine(StringUtil.Loc("ServiceConfigured", ServiceName));
return(true);
}
public void DeleteCredential()
{
IOUtil.Delete(_credFilePath, default(CancellationToken));
}
/// <summary>
/// _work
/// \_update
/// \bin
/// \externals
/// \run.sh
/// \run.cmd
/// \package.zip //temp download .zip/.tar.gz
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
private async Task DownloadLatestAgent(CancellationToken token)
{
string latestAgentDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Path.UpdateDirectory);
IOUtil.DeleteDirectory(latestAgentDirectory, token);
Directory.CreateDirectory(latestAgentDirectory);
int agentSuffix = 1;
string archiveFile = null;
bool downloadSucceeded = false;
bool validationSucceeded = false;
try
{
// Download the agent, using multiple attempts in order to be resilient against any networking/CDN issues
for (int attempt = 1; attempt <= Constants.AgentDownloadRetryMaxAttempts && !validationSucceeded; attempt++)
{
// Generate an available package name, and do our best effort to clean up stale local zip files
while (true)
{
if (_targetPackage.Platform.StartsWith("win"))
{
archiveFile = Path.Combine(latestAgentDirectory, $"agent{agentSuffix}.zip");
}
else
{
archiveFile = Path.Combine(latestAgentDirectory, $"agent{agentSuffix}.tar.gz");
}
// The package name is generated, check if there is already a file with the same name and path
if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile))
{
Trace.Verbose("Deleting latest agent package zip '{0}'", archiveFile);
try
{
// Such a file already exists, so try deleting it
IOUtil.DeleteFile(archiveFile);
// The file was successfully deleted, so we can use the generated package name
break;
}
catch (Exception ex)
{
// Couldn't delete the file for whatever reason, so generate another package name
Trace.Warning("Failed to delete agent package zip '{0}'. Exception: {1}", archiveFile, ex);
agentSuffix++;
}
}
else
{
// There is no a file with the same name and path, so we can use the generated package name
break;
}
}
// Allow a 15-minute package download timeout, which is good enough to update the agent from a 1 Mbit/s ADSL connection.
var timeoutSeconds = AgentKnobs.AgentDownloadTimeout.GetValue(_knobContext).AsInt();
Trace.Info($"Attempt {attempt}: save latest agent into {archiveFile}.");
using (var downloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var downloadCts = CancellationTokenSource.CreateLinkedTokenSource(downloadTimeout.Token, token))
{
try
{
Trace.Info($"Download agent: begin download");
//open zip stream in async mode
using (var handler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(handler))
using (var fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true))
using (var result = await httpClient.GetStreamAsync(_targetPackage.DownloadUrl))
{
//81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k).
await result.CopyToAsync(fs, 81920, downloadCts.Token);
await fs.FlushAsync(downloadCts.Token);
}
Trace.Info($"Download agent: finished download");
downloadSucceeded = true;
validationSucceeded = HashValidation(archiveFile);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info($"Agent download has been canceled.");
throw;
}
catch (SocketException ex)
{
ExceptionsUtil.HandleSocketException(ex, _targetPackage.DownloadUrl, Trace.Warning);
}
catch (Exception ex)
{
if (downloadCts.Token.IsCancellationRequested)
{
Trace.Warning($"Agent download has timed out after {timeoutSeconds} seconds");
}
Trace.Warning($"Failed to get package '{archiveFile}' from '{_targetPackage.DownloadUrl}'. Exception {ex}");
}
}
}
if (!downloadSucceeded)
{
throw new TaskCanceledException($"Agent package '{archiveFile}' failed after {Constants.AgentDownloadRetryMaxAttempts} download attempts.");
}
if (!validationSucceeded)
{
throw new TaskCanceledException(@"Agent package checksum validation failed.
There are possible reasons why this happened:
1) The agent package was compromised.
2) The agent package was not fully downloaded or was corrupted during the download process.
You can skip checksum validation for the agent package by setting the environment variable DISABLE_HASH_VALIDATION=true");
}
// If we got this far, we know that we've successfully downloadeded the agent package
if (archiveFile.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
{
ZipFile.ExtractToDirectory(archiveFile, latestAgentDirectory);
}
else if (archiveFile.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase))
{
string tar = WhichUtil.Which("tar", trace: Trace);
if (string.IsNullOrEmpty(tar))
{
throw new NotSupportedException($"tar -xzf");
}
// tar -xzf
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Info(args.Data);
}
});
processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Error(args.Data);
}
});
int exitCode = await processInvoker.ExecuteAsync(latestAgentDirectory, tar, $"-xzf \"{archiveFile}\"", null, token);
if (exitCode != 0)
{
throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}.");
}
}
}
else
{
throw new NotSupportedException($"{archiveFile}");
}
Trace.Info($"Finished getting latest agent package at: {latestAgentDirectory}.");
}
finally
{
try
{
// delete .zip file
if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile))
{
Trace.Verbose("Deleting latest agent package zip: {0}", archiveFile);
IOUtil.DeleteFile(archiveFile);
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the .zip file
Trace.Warning("Failed to delete agent package zip '{0}'. Exception: {1}", archiveFile, ex);
}
}
if (!String.IsNullOrEmpty(AgentKnobs.DisableAuthenticodeValidation.GetValue(HostContext).AsString()))
{
Trace.Warning("Authenticode validation skipped for downloaded agent package since it is disabled currently by agent settings.");
}
else
{
if (PlatformUtil.RunningOnWindows)
{
var isValid = this.VerifyAgentAuthenticode(latestAgentDirectory);
if (!isValid)
{
throw new Exception("Authenticode validation of agent assemblies failed.");
}
else
{
Trace.Info("Authenticode validation of agent assemblies passed successfully.");
}
}
else
{
Trace.Info("Authenticode validation skipped since it's not supported on non-Windows platforms at the moment.");
}
}
// copy latest agent into agent root folder
// copy bin from _work/_update -> bin.version under root
string binVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.BinDirectory}.{_targetPackage.Version}");
Directory.CreateDirectory(binVersionDir);
Trace.Info($"Copy {Path.Combine(latestAgentDirectory, Constants.Path.BinDirectory)} to {binVersionDir}.");
IOUtil.CopyDirectory(Path.Combine(latestAgentDirectory, Constants.Path.BinDirectory), binVersionDir, token);
// copy externals from _work/_update -> externals.version under root
string externalsVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.ExternalsDirectory}.{_targetPackage.Version}");
Directory.CreateDirectory(externalsVersionDir);
Trace.Info($"Copy {Path.Combine(latestAgentDirectory, Constants.Path.ExternalsDirectory)} to {externalsVersionDir}.");
IOUtil.CopyDirectory(Path.Combine(latestAgentDirectory, Constants.Path.ExternalsDirectory), externalsVersionDir, token);
// copy and replace all .sh/.cmd files
Trace.Info($"Copy any remaining .sh/.cmd files into agent root.");
foreach (FileInfo file in new DirectoryInfo(latestAgentDirectory).GetFiles() ?? new FileInfo[0])
{
// Copy and replace the file.
file.CopyTo(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), file.Name), true);
}
// for windows service back compat with old windows agent, we need make sure the servicehost.exe is still the old name
// if the current bin folder has VsoAgentService.exe, then the new agent bin folder needs VsoAgentService.exe as well
if (PlatformUtil.RunningOnWindows)
{
if (File.Exists(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "VsoAgentService.exe")))
{
Trace.Info($"Make a copy of AgentService.exe, name it VsoAgentService.exe");
File.Copy(Path.Combine(binVersionDir, "AgentService.exe"), Path.Combine(binVersionDir, "VsoAgentService.exe"), true);
File.Copy(Path.Combine(binVersionDir, "AgentService.exe.config"), Path.Combine(binVersionDir, "VsoAgentService.exe.config"), true);
Trace.Info($"Make a copy of Agent.Listener.exe, name it VsoAgent.exe");
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.exe"), Path.Combine(binVersionDir, "VsoAgent.exe"), true);
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.dll"), Path.Combine(binVersionDir, "VsoAgent.dll"), true);
// in case of we remove all pdb file from agent package.
if (File.Exists(Path.Combine(binVersionDir, "AgentService.pdb")))
{
File.Copy(Path.Combine(binVersionDir, "AgentService.pdb"), Path.Combine(binVersionDir, "VsoAgentService.pdb"), true);
}
if (File.Exists(Path.Combine(binVersionDir, "Agent.Listener.pdb")))
{
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.pdb"), Path.Combine(binVersionDir, "VsoAgent.pdb"), true);
}
}
}
}
internal static void CustomersToFile(String fileName)
{
StringBuilder strCustomers = new StringBuilder();
List <ICustomer> writtenCustomers = new List <ICustomer>(customers);
//Clear file
IOUtil.WriteAllText(fileName, "", PosConfiguration.DefaultEncoding);
int count = 0;
foreach (ICustomer c in customers)
{
strCustomers.Append(String.Format(
"{0:D1}{1:D6}{2}{3}{4}{5}{6}{7:D2}{8:D6}{9:D2}\r\n",
((Customer)c).flag == 1 ? "1" : "0",
c.Number,
c.Code.PadRight(20, ' '),
c.Name.PadRight(20, ' '),
c.Contact[0].PadRight(20, ' ') + c.Contact[1].PadRight(20, ' ') + c.Contact[2].PadRight(20, ' '),
c.Contact[3].PadRight(15, ' '),
c.Contact[4].PadRight(15, ' '),
c.PromotionLimit,
c.CustomerGroup,
c.DefaultDocumentType
));
ICustomer cust = null;
var query = from cc in customers
where cc.Number == c.Number
select cc;
if (query != null && query.Count() > 0)
{
cust = query.First();
}
if (cust != null)
{
writtenCustomers.Remove(cust);
}
//Windows ce memory is less than desktop pc
//So OutOfMemoryException occur if stringbuilder capacity is too big.
if (count >= 5000)
{
//move memory to file than clear momory
IOUtil.AppendAllText(fileName, strCustomers.ToString());
strCustomers = new StringBuilder();
count = 0;
}
count++;
}
foreach (ICustomer cstmr in writtenCustomers)
{
//if (writtenLabels.IndexOf(labelNo) >= 0) continue;
strCustomers.Append(String.Format(
"{0:D1}{1:D6}{2}{3}{4}{5}{6}{7:D2}{8:D6}{9:D2}\r\n",
((Customer)cstmr).flag == 1 ? "1" : "0",
cstmr.Number,
cstmr.Code.PadRight(20, ' '),
cstmr.Name.PadRight(20, ' '),
cstmr.Contact[0].PadRight(20, ' ') + cstmr.Contact[1].PadRight(20, ' ') + cstmr.Contact[2].PadRight(20, ' '),
cstmr.Contact[3].PadRight(15, ' '),
cstmr.Contact[4].PadRight(15, ' '),
cstmr.PromotionLimit,
cstmr.CustomerGroup,
cstmr.DefaultDocumentType
));
if (count >= 5000)
{
IOUtil.AppendAllText(fileName, strCustomers.ToString());
strCustomers = new StringBuilder();
count = 0;
}
count++;
}
IOUtil.AppendAllText(fileName, strCustomers.ToString());
strCustomers = new StringBuilder();
writtenCustomers = new List <ICustomer>();
}
private void DeletePreviousVersionAgentBackup(CancellationToken token)
{
// delete previous backup agent (back compat, can be remove after serval sprints)
// bin.bak.2.99.0
// externals.bak.2.99.0
foreach (string existBackUp in Directory.GetDirectories(HostContext.GetDirectory(WellKnownDirectory.Root), "*.bak.*"))
{
Trace.Info($"Delete existing agent backup at {existBackUp}.");
try
{
IOUtil.DeleteDirectory(existBackUp, token);
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
Trace.Error(ex);
Trace.Info($"Catch exception during delete backup folder {existBackUp}, ignore this error try delete the backup folder on next auto-update.");
}
}
// delete old bin.2.99.0 folder, only leave the current version and the latest download version
var allBinDirs = Directory.GetDirectories(HostContext.GetDirectory(WellKnownDirectory.Root), "bin.*");
if (allBinDirs.Length > 2)
{
// there are more than 2 bin.version folder.
// delete older bin.version folders.
foreach (var oldBinDir in allBinDirs)
{
if (string.Equals(oldBinDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"bin"), StringComparison.OrdinalIgnoreCase) ||
string.Equals(oldBinDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"bin.{BuildConstants.AgentPackage.Version}"), StringComparison.OrdinalIgnoreCase) ||
string.Equals(oldBinDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"bin.{_targetPackage.Version}"), StringComparison.OrdinalIgnoreCase))
{
// skip for current agent version
continue;
}
Trace.Info($"Delete agent bin folder's backup at {oldBinDir}.");
try
{
IOUtil.DeleteDirectory(oldBinDir, token);
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
Trace.Error(ex);
Trace.Info($"Catch exception during delete backup folder {oldBinDir}, ignore this error try delete the backup folder on next auto-update.");
}
}
}
// delete old externals.2.99.0 folder, only leave the current version and the latest download version
var allExternalsDirs = Directory.GetDirectories(HostContext.GetDirectory(WellKnownDirectory.Root), "externals.*");
if (allExternalsDirs.Length > 2)
{
// there are more than 2 externals.version folder.
// delete older externals.version folders.
foreach (var oldExternalDir in allExternalsDirs)
{
if (string.Equals(oldExternalDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"externals"), StringComparison.OrdinalIgnoreCase) ||
string.Equals(oldExternalDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"externals.{BuildConstants.AgentPackage.Version}"), StringComparison.OrdinalIgnoreCase) ||
string.Equals(oldExternalDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"externals.{_targetPackage.Version}"), StringComparison.OrdinalIgnoreCase))
{
// skip for current agent version
continue;
}
Trace.Info($"Delete agent externals folder's backup at {oldExternalDir}.");
try
{
IOUtil.DeleteDirectory(oldExternalDir, token);
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
Trace.Error(ex);
Trace.Info($"Catch exception during delete backup folder {oldExternalDir}, ignore this error try delete the backup folder on next auto-update.");
}
}
}
}
private void AddProxySetting(IVstsAgentWebProxy agentProxy)
{
string appConfig = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost), _appConfigFileName);
ArgUtil.File(appConfig, _appConfigFileName);
string appConfigRestore = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost), _appConfigRestoreFileName);
if (!File.Exists(appConfigRestore))
{
Trace.Info("Take snapshot of current appconfig for restore modified appconfig.");
File.Copy(appConfig, appConfigRestore);
}
else
{
// cleanup any appconfig changes from previous build.
ExecutionContext.Debug("Restore default LegacyVSTSPowerShellHost.exe.config.");
IOUtil.DeleteFile(appConfig);
File.Copy(appConfigRestore, appConfig);
}
XmlDocument psHostAppConfig = new XmlDocument();
using (var appConfigStream = new FileStream(appConfig, FileMode.Open, FileAccess.Read))
{
psHostAppConfig.Load(appConfigStream);
}
var configuration = psHostAppConfig.SelectSingleNode("configuration");
ArgUtil.NotNull(configuration, "configuration");
var exist_defaultProxy = psHostAppConfig.SelectSingleNode("configuration/system.net/defaultProxy");
if (exist_defaultProxy == null)
{
var system_net = psHostAppConfig.SelectSingleNode("configuration/system.net");
if (system_net == null)
{
Trace.Verbose("Create system.net section in appconfg.");
system_net = psHostAppConfig.CreateElement("system.net");
}
Trace.Verbose("Create defaultProxy section in appconfg.");
var defaultProxy = psHostAppConfig.CreateElement("defaultProxy");
defaultProxy.SetAttribute("useDefaultCredentials", "true");
Trace.Verbose("Create proxy section in appconfg.");
var proxy = psHostAppConfig.CreateElement("proxy");
proxy.SetAttribute("proxyaddress", agentProxy.ProxyAddress);
proxy.SetAttribute("bypassonlocal", "true");
if (agentProxy.ProxyBypassList != null && agentProxy.ProxyBypassList.Count > 0)
{
Trace.Verbose("Create bypasslist section in appconfg.");
var bypass = psHostAppConfig.CreateElement("bypasslist");
foreach (string proxyBypass in agentProxy.ProxyBypassList)
{
Trace.Verbose($"Create bypasslist.add section for {proxyBypass} in appconfg.");
var add = psHostAppConfig.CreateElement("add");
add.SetAttribute("address", proxyBypass);
bypass.AppendChild(add);
}
defaultProxy.AppendChild(bypass);
}
defaultProxy.AppendChild(proxy);
system_net.AppendChild(defaultProxy);
configuration.AppendChild(system_net);
using (var appConfigStream = new FileStream(appConfig, FileMode.Open, FileAccess.ReadWrite))
{
psHostAppConfig.Save(appConfigStream);
}
ExecutionContext.Debug("Add Proxy setting in LegacyVSTSPowerShellHost.exe.config file.");
}
else
{
//proxy setting exist.
ExecutionContext.Debug("Proxy setting already exist in LegacyVSTSPowerShellHost.exe.config file.");
}
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(Data, nameof(Data));
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(Inputs, nameof(Inputs));
ArgUtil.Directory(TaskDirectory, nameof(TaskDirectory));
#if !OS_WINDOWS
// Ensure compat vso-task-lib exist at the root of _work folder
// This will make vsts-agent works against 2015 RTM/QU1 TFS, since tasks in those version doesn't package with task lib
// Put the 0.5.5 version vso-task-lib into the root of _work/node_modules folder, so tasks are able to find those lib.
if (!File.Exists(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), "node_modules", "vso-task-lib", "package.json")))
{
string vsoTaskLibFromExternal = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "vso-task-lib");
string compatVsoTaskLibInWork = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), "node_modules", "vso-task-lib");
IOUtil.CopyDirectory(vsoTaskLibFromExternal, compatVsoTaskLibInWork, ExecutionContext.CancellationToken);
}
#endif
// Update the env dictionary.
AddInputsToEnvironment();
AddEndpointsToEnvironment();
AddSecureFilesToEnvironment();
AddVariablesToEnvironment();
AddTaskVariablesToEnvironment();
AddPrependPathToEnvironment();
// Resolve the target script.
string target = Data.Target;
ArgUtil.NotNullOrEmpty(target, nameof(target));
target = Path.Combine(TaskDirectory, target);
ArgUtil.File(target, nameof(target));
// Resolve the working directory.
string workingDirectory = Data.WorkingDirectory;
if (string.IsNullOrEmpty(workingDirectory))
{
workingDirectory = ExecutionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory);
if (string.IsNullOrEmpty(workingDirectory))
{
workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
}
}
// fix vsts-task-lib for node 6.x
// vsts-task-lib 0.6/0.7/0.8/0.9/2.0-preview implemented String.prototype.startsWith and String.prototype.endsWith since Node 5.x doesn't have them.
// however the implementation is added in node 6.x, the implementation in vsts-task-lib is different.
// node 6.x's implementation takes 2 parameters str.endsWith(searchString[, length]) / str.startsWith(searchString[, length])
// the implementation vsts-task-lib had only takes one parameter str.endsWith(searchString) / str.startsWith(searchString).
// as long as vsts-task-lib be loaded into memory, it will overwrite the implementation node 6.x has,
// so any scirpt that use the second parameter (length) will encounter unexpected result.
// to avoid customer hit this error, we will modify the file (extensions.js) under vsts-task-lib module folder when customer choose to use Node 6.x
Trace.Info("Inspect node_modules folder, make sure vsts-task-lib doesn't overwrite String.startsWith/endsWith.");
FixVstsTaskLibModule();
StepHost.OutputDataReceived += OnDataReceived;
StepHost.ErrorDataReceived += OnDataReceived;
string file;
if (!string.IsNullOrEmpty(ExecutionContext.Container?.ContainerBringNodePath))
{
file = ExecutionContext.Container.ContainerBringNodePath;
}
else
{
bool useNode10 = ExecutionContext.Variables.GetBoolean("AGENT_USE_NODE10") ??
StringUtil.ConvertToBoolean(System.Environment.GetEnvironmentVariable("AGENT_USE_NODE10"), false);
bool taskHasNode10Data = Data is Node10HandlerData;
string nodeFolder = (taskHasNode10Data || useNode10) ? "node10" : "node";
Trace.Info($"Task.json has node10 handler data: {taskHasNode10Data}, use node10 for node tasks: {useNode10}");
file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), nodeFolder, "bin", $"node{IOUtil.ExeExtension}");
}
// Format the arguments passed to node.
// 1) Wrap the script file path in double quotes.
// 2) Escape double quotes within the script file path. Double-quote is a valid
// file name character on Linux.
string arguments = StepHost.ResolvePathForStepHost(StringUtil.Format(@"""{0}""", target.Replace(@"""", @"\""")));
#if OS_WINDOWS
// It appears that node.exe outputs UTF8 when not in TTY mode.
Encoding outputEncoding = Encoding.UTF8;
#else
// Let .NET choose the default.
Encoding outputEncoding = null;
#endif
// Execute the process. Exit code 0 should always be returned.
// A non-zero exit code indicates infrastructural failure.
// Task failure should be communicated over STDOUT using ## commands.
Task step = StepHost.ExecuteAsync(workingDirectory: StepHost.ResolvePathForStepHost(workingDirectory),
fileName: StepHost.ResolvePathForStepHost(file),
arguments: arguments,
environment: Environment,
requireExitCodeZero: true,
outputEncoding: outputEncoding,
killProcessOnCancel: false,
cancellationToken: ExecutionContext.CancellationToken);
// Wait for either the node exit or force finish through ##vso command
await System.Threading.Tasks.Task.WhenAny(step, ExecutionContext.ForceCompleted);
if (ExecutionContext.ForceCompleted.IsCompleted)
{
ExecutionContext.Debug("The task was marked as \"done\", but the process has not closed after 5 seconds. Treating the task as complete.");
}
}
public async Task GetSourceAsync(
IExecutionContext executionContext,
ServiceEndpoint endpoint,
CancellationToken cancellationToken)
{
Trace.Entering();
// Validate args.
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(endpoint, nameof(endpoint));
executionContext.Output($"Syncing repository: {endpoint.Name} ({RepositoryType})");
Uri repositoryUrl = endpoint.Url;
if (!repositoryUrl.IsAbsoluteUri)
{
throw new InvalidOperationException("Repository url need to be an absolute uri.");
}
string targetPath = GetEndpointData(endpoint, Constants.EndpointData.SourcesDirectory);
string sourceBranch = GetEndpointData(endpoint, Constants.EndpointData.SourceBranch);
string sourceVersion = GetEndpointData(endpoint, Constants.EndpointData.SourceVersion);
bool clean = false;
if (endpoint.Data.ContainsKey(WellKnownEndpointData.Clean))
{
clean = StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.Clean]);
}
bool checkoutSubmodules = false;
if (endpoint.Data.ContainsKey(WellKnownEndpointData.CheckoutSubmodules))
{
checkoutSubmodules = StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.CheckoutSubmodules]);
}
bool checkoutNestedSubmodules = false;
if (endpoint.Data.ContainsKey(WellKnownEndpointData.CheckoutNestedSubmodules))
{
checkoutNestedSubmodules = StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.CheckoutNestedSubmodules]);
}
int fetchDepth = 0;
if (endpoint.Data.ContainsKey("fetchDepth") &&
(!int.TryParse(endpoint.Data["fetchDepth"], out fetchDepth) || fetchDepth < 0))
{
fetchDepth = 0;
}
// prefer feature variable over endpoint data
fetchDepth = executionContext.Variables.GetInt(Constants.Variables.Features.GitShallowDepth) ?? fetchDepth;
bool gitLfsSupport = false;
if (endpoint.Data.ContainsKey("GitLfsSupport"))
{
gitLfsSupport = StringUtil.ConvertToBoolean(endpoint.Data["GitLfsSupport"]);
}
// prefer feature variable over endpoint data
gitLfsSupport = executionContext.Variables.GetBoolean(Constants.Variables.Features.GitLfsSupport) ?? gitLfsSupport;
bool exposeCred = executionContext.Variables.GetBoolean(Constants.Variables.System.EnableAccessToken) ?? false;
Trace.Info($"Repository url={repositoryUrl}");
Trace.Info($"targetPath={targetPath}");
Trace.Info($"sourceBranch={sourceBranch}");
Trace.Info($"sourceVersion={sourceVersion}");
Trace.Info($"clean={clean}");
Trace.Info($"checkoutSubmodules={checkoutSubmodules}");
Trace.Info($"checkoutNestedSubmodules={checkoutNestedSubmodules}");
Trace.Info($"exposeCred={exposeCred}");
Trace.Info($"fetchDepth={fetchDepth}");
Trace.Info($"gitLfsSupport={gitLfsSupport}");
// Determine which git will be use
// On windows, we prefer the built-in portable git within the agent's externals folder, set system.prefergitfrompath=true can change the behavior,
// agent will find git.exe from %PATH%
// On Linux, we will always use git find in %PATH% regardless of system.prefergitfrompath
bool preferGitFromPath = executionContext.Variables.GetBoolean(Constants.Variables.System.PreferGitFromPath) ?? false;
#if !OS_WINDOWS
// On linux/OSX, we will always find git from %PATH%
preferGitFromPath = true;
#endif
// Determine do we need to provide creds to git operation
_selfManageGitCreds = executionContext.Variables.GetBoolean(Constants.Variables.System.SelfManageGitCreds) ?? false;
if (_selfManageGitCreds)
{
// Customer choose to own git creds by themselves.
executionContext.Output(StringUtil.Loc("SelfManageGitCreds"));
}
// Initialize git command manager
_gitCommandManager = HostContext.GetService <IGitCommandManager>();
await _gitCommandManager.LoadGitExecutionInfo(executionContext, useBuiltInGit : !preferGitFromPath);
// Make sure the build machine met all requirements for the git repository
// For now, the only requirement we have is git version greater than 2.9 for on-prem tfsgit
RequirementCheck(executionContext, endpoint);
// retrieve credential from endpoint.
string username = string.Empty;
string password = string.Empty;
if (!_selfManageGitCreds && endpoint.Authorization != null)
{
switch (endpoint.Authorization.Scheme)
{
case EndpointAuthorizationSchemes.OAuth:
username = EndpointAuthorizationSchemes.OAuth;
if (!endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.AccessToken, out password))
{
password = string.Empty;
}
break;
case EndpointAuthorizationSchemes.UsernamePassword:
if (!endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Username, out username))
{
// leave the username as empty, the username might in the url, like: http://[email protected]
username = string.Empty;
}
if (!endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Password, out password))
{
// we have username, but no password
password = string.Empty;
}
break;
default:
executionContext.Warning($"Unsupport endpoint authorization schemes: {endpoint.Authorization.Scheme}");
break;
}
}
// prepare credentail embedded urls
_repositoryUrlWithCred = UrlUtil.GetCredentialEmbeddedUrl(repositoryUrl, username, password);
var agentProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl) && !agentProxy.IsBypassed(repositoryUrl))
{
_proxyUrlWithCred = UrlUtil.GetCredentialEmbeddedUrl(new Uri(executionContext.Variables.Agent_ProxyUrl), executionContext.Variables.Agent_ProxyUsername, executionContext.Variables.Agent_ProxyPassword);
// uri.absoluteuri will not contains port info if the scheme is http/https and the port is 80/443
// however, git.exe always require you provide port info, if nothing passed in, it will use 1080 as default
// as result, we need prefer the uri.originalstring when it's different than uri.absoluteuri.
if (string.Equals(_proxyUrlWithCred.AbsoluteUri, _proxyUrlWithCred.OriginalString, StringComparison.OrdinalIgnoreCase))
{
_proxyUrlWithCredString = _proxyUrlWithCred.AbsoluteUri;
}
else
{
_proxyUrlWithCredString = _proxyUrlWithCred.OriginalString;
}
}
if (gitLfsSupport)
{
// Construct git-lfs url
UriBuilder gitLfsUrl = new UriBuilder(_repositoryUrlWithCred);
if (gitLfsUrl.Path.EndsWith(".git"))
{
gitLfsUrl.Path = gitLfsUrl.Path + "/info/lfs";
}
else
{
gitLfsUrl.Path = gitLfsUrl.Path + ".git/info/lfs";
}
_gitLfsUrlWithCred = gitLfsUrl.Uri;
}
// Check the current contents of the root folder to see if there is already a repo
// If there is a repo, see if it matches the one we are expecting to be there based on the remote fetch url
// if the repo is not what we expect, remove the folder
if (!await IsRepositoryOriginUrlMatch(executionContext, targetPath, repositoryUrl))
{
// Delete source folder
IOUtil.DeleteDirectory(targetPath, cancellationToken);
}
else
{
// delete the index.lock file left by previous canceled build or any operation casue git.exe crash last time.
string lockFile = Path.Combine(targetPath, ".git\\index.lock");
if (File.Exists(lockFile))
{
try
{
File.Delete(lockFile);
}
catch (Exception ex)
{
executionContext.Debug($"Unable to delete the index.lock file: {lockFile}");
executionContext.Debug(ex.ToString());
}
}
// When repo.clean is selected for a git repo, execute git clean -fdx and git reset --hard HEAD on the current repo.
// This will help us save the time to reclone the entire repo.
// If any git commands exit with non-zero return code or any exception happened during git.exe invoke, fall back to delete the repo folder.
if (clean)
{
Boolean softCleanSucceed = true;
// git clean -fdx
int exitCode_clean = await _gitCommandManager.GitClean(executionContext, targetPath);
if (exitCode_clean != 0)
{
executionContext.Debug($"'git clean -fdx' failed with exit code {exitCode_clean}, this normally caused by:\n 1) Path too long\n 2) Permission issue\n 3) File in use\nFor futher investigation, manually run 'git clean -fdx' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
// git reset --hard HEAD
if (softCleanSucceed)
{
int exitCode_reset = await _gitCommandManager.GitReset(executionContext, targetPath);
if (exitCode_reset != 0)
{
executionContext.Debug($"'git reset --hard HEAD' failed with exit code {exitCode_reset}\nFor futher investigation, manually run 'git reset --hard HEAD' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
}
// git clean -fdx and git reset --hard HEAD for each submodule
if (checkoutSubmodules)
{
if (softCleanSucceed)
{
int exitCode_submoduleclean = await _gitCommandManager.GitSubmoduleClean(executionContext, targetPath);
if (exitCode_submoduleclean != 0)
{
executionContext.Debug($"'git submodule foreach git clean -fdx' failed with exit code {exitCode_submoduleclean}\nFor futher investigation, manually run 'git submodule foreach git clean -fdx' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
}
if (softCleanSucceed)
{
int exitCode_submodulereset = await _gitCommandManager.GitSubmoduleReset(executionContext, targetPath);
if (exitCode_submodulereset != 0)
{
executionContext.Debug($"'git submodule foreach git reset --hard HEAD' failed with exit code {exitCode_submodulereset}\nFor futher investigation, manually run 'git submodule foreach git reset --hard HEAD' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
}
}
if (!softCleanSucceed)
{
//fall back
executionContext.Warning("Unable to run \"git clean -fdx\" and \"git reset --hard HEAD\" successfully, delete source folder instead.");
IOUtil.DeleteDirectory(targetPath, cancellationToken);
}
}
}
// if the folder is missing, create it
if (!Directory.Exists(targetPath))
{
Directory.CreateDirectory(targetPath);
}
// if the folder contains a .git folder, it means the folder contains a git repo that matches the remote url and in a clean state.
// we will run git fetch to update the repo.
if (!Directory.Exists(Path.Combine(targetPath, ".git")))
{
// init git repository
int exitCode_init = await _gitCommandManager.GitInit(executionContext, targetPath);
if (exitCode_init != 0)
{
throw new InvalidOperationException($"Unable to use git.exe init repository under {targetPath}, 'git init' failed with exit code: {exitCode_init}");
}
int exitCode_addremote = await _gitCommandManager.GitRemoteAdd(executionContext, targetPath, "origin", repositoryUrl.AbsoluteUri);
if (exitCode_addremote != 0)
{
throw new InvalidOperationException($"Unable to use git.exe add remote 'origin', 'git remote add' failed with exit code: {exitCode_addremote}");
}
}
cancellationToken.ThrowIfCancellationRequested();
executionContext.Progress(0, "Starting fetch...");
// disable git auto gc
int exitCode_disableGC = await _gitCommandManager.GitDisableAutoGC(executionContext, targetPath);
if (exitCode_disableGC != 0)
{
executionContext.Warning("Unable turn off git auto garbage collection, git fetch operation may trigger auto garbage collection which will affect the performence of fetching.");
}
// always remove any possible left extraheader setting from git config.
if (await _gitCommandManager.GitConfigExist(executionContext, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader"))
{
executionContext.Debug("Remove any extraheader setting from git config.");
await RemoveGitConfig(executionContext, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader", string.Empty);
}
// always remove any possible left proxy setting from git config, the proxy setting may contains credential
if (await _gitCommandManager.GitConfigExist(executionContext, targetPath, $"http.proxy"))
{
executionContext.Debug("Remove any proxy setting from git config.");
await RemoveGitConfig(executionContext, targetPath, $"http.proxy", string.Empty);
}
List <string> additionalFetchArgs = new List <string>();
List <string> additionalLfsFetchArgs = new List <string>();
if (!_selfManageGitCreds)
{
// v2.9 git support provide auth header as cmdline arg.
// as long 2.9 git exist, VSTS repo, TFS repo and Github repo will use this to handle auth challenge.
if (UseAuthHeaderCmdlineArg)
{
additionalFetchArgs.Add($"-c http.extraheader=\"AUTHORIZATION: {GenerateAuthHeader(username, password)}\"");
}
else
{
// Otherwise, inject credential into fetch/push url
// inject credential into fetch url
executionContext.Debug("Inject credential into git remote url.");
ArgUtil.NotNull(_repositoryUrlWithCred, nameof(_repositoryUrlWithCred));
// inject credential into fetch url
executionContext.Debug("Inject credential into git remote fetch url.");
int exitCode_seturl = await _gitCommandManager.GitRemoteSetUrl(executionContext, targetPath, "origin", _repositoryUrlWithCred.AbsoluteUri);
if (exitCode_seturl != 0)
{
throw new InvalidOperationException($"Unable to use git.exe inject credential to git remote fetch url, 'git remote set-url' failed with exit code: {exitCode_seturl}");
}
// inject credential into push url
executionContext.Debug("Inject credential into git remote push url.");
exitCode_seturl = await _gitCommandManager.GitRemoteSetPushUrl(executionContext, targetPath, "origin", _repositoryUrlWithCred.AbsoluteUri);
if (exitCode_seturl != 0)
{
throw new InvalidOperationException($"Unable to use git.exe inject credential to git remote push url, 'git remote set-url --push' failed with exit code: {exitCode_seturl}");
}
}
// Prepare proxy config for fetch.
if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl) && !agentProxy.IsBypassed(repositoryUrl))
{
executionContext.Debug($"Config proxy server '{executionContext.Variables.Agent_ProxyUrl}' for git fetch.");
ArgUtil.NotNullOrEmpty(_proxyUrlWithCredString, nameof(_proxyUrlWithCredString));
additionalFetchArgs.Add($"-c http.proxy=\"{_proxyUrlWithCredString}\"");
additionalLfsFetchArgs.Add($"-c http.proxy=\"{_proxyUrlWithCredString}\"");
}
// Prepare gitlfs url for fetch and checkout
if (gitLfsSupport)
{
// Initialize git lfs by execute 'git lfs install'
executionContext.Debug("Detect git-lfs version");
int exitCode_lfsVersion = await _gitCommandManager.GitLFSVersion(executionContext, targetPath);
if (exitCode_lfsVersion != 0)
{
throw new InvalidOperationException($"Can't detect Git-lfs version, 'git lfs version' failed with exit code: {exitCode_lfsVersion}");
}
// Initialize git lfs by execute 'git lfs install'
executionContext.Debug("Setup the local Git hooks for Git LFS.");
int exitCode_lfsInstall = await _gitCommandManager.GitLFSInstall(executionContext, targetPath);
if (exitCode_lfsInstall != 0)
{
throw new InvalidOperationException($"Git-lfs installation failed with exit code: {exitCode_lfsInstall}");
}
// Inject credential into lfs fetch/push url
executionContext.Debug("Inject credential into git-lfs remote url.");
ArgUtil.NotNull(_gitLfsUrlWithCred, nameof(_gitLfsUrlWithCred));
// inject credential into fetch url
executionContext.Debug("Inject credential into git-lfs remote fetch url.");
_configModifications["remote.origin.lfsurl"] = _gitLfsUrlWithCred.AbsoluteUri;
int exitCode_configlfsurl = await _gitCommandManager.GitConfig(executionContext, targetPath, "remote.origin.lfsurl", _gitLfsUrlWithCred.AbsoluteUri);
if (exitCode_configlfsurl != 0)
{
throw new InvalidOperationException($"Git config failed with exit code: {exitCode_configlfsurl}");
}
// inject credential into push url
executionContext.Debug("Inject credential into git-lfs remote push url.");
_configModifications["remote.origin.lfspushurl"] = _gitLfsUrlWithCred.AbsoluteUri;
int exitCode_configlfspushurl = await _gitCommandManager.GitConfig(executionContext, targetPath, "remote.origin.lfspushurl", _gitLfsUrlWithCred.AbsoluteUri);
if (exitCode_configlfspushurl != 0)
{
throw new InvalidOperationException($"Git config failed with exit code: {exitCode_configlfspushurl}");
}
}
}
// If this is a build for a pull request, then include
// the pull request reference as an additional ref.
List <string> additionalFetchSpecs = new List <string>();
if (IsPullRequest(sourceBranch))
{
additionalFetchSpecs.Add("+refs/heads/*:refs/remotes/origin/*");
additionalFetchSpecs.Add(StringUtil.Format("+{0}:{1}", sourceBranch, GetRemoteRefName(sourceBranch)));
}
int exitCode_fetch = await _gitCommandManager.GitFetch(executionContext, targetPath, "origin", fetchDepth, additionalFetchSpecs, string.Join(" ", additionalFetchArgs), cancellationToken);
if (exitCode_fetch != 0)
{
throw new InvalidOperationException($"Git fetch failed with exit code: {exitCode_fetch}");
}
// Checkout
// sourceToBuild is used for checkout
// if sourceBranch is a PR branch or sourceVersion is null, make sure branch name is a remote branch. we need checkout to detached head.
// (change refs/heads to refs/remotes/origin, refs/pull to refs/remotes/pull, or leava it as it when the branch name doesn't contain refs/...)
// if sourceVersion provide, just use that for checkout, since when you checkout a commit, it will end up in detached head.
cancellationToken.ThrowIfCancellationRequested();
executionContext.Progress(80, "Starting checkout...");
string sourcesToBuild;
if (IsPullRequest(sourceBranch) || string.IsNullOrEmpty(sourceVersion))
{
sourcesToBuild = GetRemoteRefName(sourceBranch);
}
else
{
sourcesToBuild = sourceVersion;
}
// fetch lfs object upfront, this will avoid fetch lfs object during checkout which cause checkout taking forever
// since checkout will fetch lfs object 1 at a time, while git lfs fetch will fetch lfs object in parallel.
if (gitLfsSupport)
{
int exitCode_lfsFetch = await _gitCommandManager.GitLFSFetch(executionContext, targetPath, "origin", sourcesToBuild, string.Join(" ", additionalLfsFetchArgs), cancellationToken);
if (exitCode_lfsFetch != 0)
{
// git lfs fetch failed, get lfs log, the log is critical for debug.
int exitCode_lfsLogs = await _gitCommandManager.GitLFSLogs(executionContext, targetPath);
throw new InvalidOperationException($"Git lfs fetch failed with exit code: {exitCode_lfsFetch}. Git lfs logs returned with exit code: {exitCode_lfsLogs}.");
}
}
// Finally, checkout the sourcesToBuild (if we didn't find a valid git object this will throw)
int exitCode_checkout = await _gitCommandManager.GitCheckout(executionContext, targetPath, sourcesToBuild, cancellationToken);
if (exitCode_checkout != 0)
{
// local repository is shallow repository, checkout may fail due to lack of commits history.
// this will happen when the checkout commit is older than tip -> fetchDepth
if (fetchDepth > 0)
{
executionContext.Warning(StringUtil.Loc("ShallowCheckoutFail", fetchDepth, sourcesToBuild));
}
throw new InvalidOperationException($"Git checkout failed with exit code: {exitCode_checkout}");
}
// Submodule update
if (checkoutSubmodules)
{
cancellationToken.ThrowIfCancellationRequested();
executionContext.Progress(90, "Updating submodules...");
List <string> additionalSubmoduleUpdateArgs = new List <string>();
if (!_selfManageGitCreds)
{
if (UseAuthHeaderCmdlineArg)
{
string authorityUrl = repositoryUrl.AbsoluteUri.Replace(repositoryUrl.PathAndQuery, string.Empty);
additionalSubmoduleUpdateArgs.Add($"-c http.{authorityUrl}.extraheader=\"AUTHORIZATION: {GenerateAuthHeader(username, password)}\"");
}
// Prepare proxy config for submodule update.
if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl) && !agentProxy.IsBypassed(repositoryUrl))
{
executionContext.Debug($"Config proxy server '{executionContext.Variables.Agent_ProxyUrl}' for git submodule update.");
ArgUtil.NotNullOrEmpty(_proxyUrlWithCredString, nameof(_proxyUrlWithCredString));
additionalSubmoduleUpdateArgs.Add($"-c http.proxy=\"{_proxyUrlWithCredString}\"");
}
}
int exitCode_submoduleUpdate = await _gitCommandManager.GitSubmoduleUpdate(executionContext, targetPath, string.Join(" ", additionalSubmoduleUpdateArgs), checkoutNestedSubmodules, cancellationToken);
if (exitCode_submoduleUpdate != 0)
{
throw new InvalidOperationException($"Git submodule update failed with exit code: {exitCode_submoduleUpdate}");
}
}
// handle expose creds, related to 'Allow Scripts to Access OAuth Token' option
if (!_selfManageGitCreds)
{
if (UseAuthHeaderCmdlineArg && exposeCred)
{
string configKey = $"http.{repositoryUrl.AbsoluteUri}.extraheader";
string configValue = $"\"AUTHORIZATION: {GenerateAuthHeader(username, password)}\"";
_configModifications[configKey] = configValue.Trim('\"');
int exitCode_config = await _gitCommandManager.GitConfig(executionContext, targetPath, configKey, configValue);
if (exitCode_config != 0)
{
throw new InvalidOperationException($"Git config failed with exit code: {exitCode_config}");
}
}
if (!UseAuthHeaderCmdlineArg && !exposeCred)
{
// remove cached credential from origin's fetch/push url.
await RemoveCachedCredential(executionContext, targetPath, repositoryUrl, "origin");
}
if (exposeCred)
{
// save proxy setting to git config.
if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl) && !agentProxy.IsBypassed(repositoryUrl))
{
executionContext.Debug($"Save proxy config for proxy server '{executionContext.Variables.Agent_ProxyUrl}' into git config.");
ArgUtil.NotNullOrEmpty(_proxyUrlWithCredString, nameof(_proxyUrlWithCredString));
string proxyConfigKey = "http.proxy";
string proxyConfigValue = $"\"{_proxyUrlWithCredString}\"";
_configModifications[proxyConfigKey] = proxyConfigValue.Trim('\"');
int exitCode_proxyconfig = await _gitCommandManager.GitConfig(executionContext, targetPath, proxyConfigKey, proxyConfigValue);
if (exitCode_proxyconfig != 0)
{
throw new InvalidOperationException($"Git config failed with exit code: {exitCode_proxyconfig}");
}
}
}
if (gitLfsSupport && !exposeCred)
{
executionContext.Debug("Remove git-lfs fetch and push url setting from git config.");
await RemoveGitConfig(executionContext, targetPath, "remote.origin.lfsurl", _gitLfsUrlWithCred.AbsoluteUri);
_configModifications.Remove("remote.origin.lfsurl");
await RemoveGitConfig(executionContext, targetPath, "remote.origin.lfspushurl", _gitLfsUrlWithCred.AbsoluteUri);
_configModifications.Remove("remote.origin.lfspushurl");
}
}
}
private async Task DownloadAsync(IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(task, nameof(task));
ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version));
var taskServer = HostContext.GetService <ITaskServer>();
// first check to see if we already have the task
string destDirectory = GetDirectory(task);
Trace.Info($"Ensuring task exists: ID '{task.Id}', version '{task.Version}', name '{task.Name}', directory '{destDirectory}'.");
if (File.Exists(destDirectory + ".completed"))
{
executionContext.Debug($"Task '{task.Name}' already downloaded at '{destDirectory}'.");
return;
}
// delete existing task folder.
Trace.Verbose("Deleting task destination folder: {0}", destDirectory);
IOUtil.DeleteDirectory(destDirectory, CancellationToken.None);
// Inform the user that a download is taking place. The download could take a while if
// the task zip is large. It would be nice to print the localized name, but it is not
// available from the reference included in the job message.
executionContext.Output(StringUtil.Loc("DownloadingTask0", task.Name, task.Version));
string zipFile = string.Empty;
var version = new TaskVersion(task.Version);
//download and extract task in a temp folder and rename it on success
string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Tasks), "_temp_" + Guid.NewGuid());
try
{
Directory.CreateDirectory(tempDirectory);
int retryCount = 0;
// Allow up to 20 * 60s for any task to be downloaded from service.
// Base on Kusto, the longest we have on the service today is over 850 seconds.
// Timeout limit can be overwrite by environment variable
if (!int.TryParse(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_TIMEOUT") ?? string.Empty, out int timeoutSeconds))
{
timeoutSeconds = 20 * 60;
}
while (retryCount < 3)
{
using (var taskDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var taskDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(taskDownloadTimeout.Token, executionContext.CancellationToken))
{
try
{
zipFile = Path.Combine(tempDirectory, string.Format("{0}.zip", Guid.NewGuid()));
//open zip stream in async mode
using (FileStream fs = new FileStream(zipFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
using (Stream result = await taskServer.GetTaskContentZipAsync(task.Id, version, taskDownloadCancellation.Token))
{
await result.CopyToAsync(fs, _defaultCopyBufferSize, taskDownloadCancellation.Token);
await fs.FlushAsync(taskDownloadCancellation.Token);
// download succeed, break out the retry loop.
break;
}
}
catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested)
{
Trace.Info($"Task download has been cancelled.");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
Trace.Error($"Fail to download task '{task.Id} ({task.Name}/{task.Version})' -- Attempt: {retryCount}");
Trace.Error(ex);
if (taskDownloadTimeout.Token.IsCancellationRequested)
{
// task download didn't finish within timeout
executionContext.Warning(StringUtil.Loc("TaskDownloadTimeout", task.Name, timeoutSeconds));
}
else
{
executionContext.Warning(StringUtil.Loc("TaskDownloadFailed", task.Name, ex.Message));
if (ex.InnerException != null)
{
executionContext.Warning("Inner Exception: {ex.InnerException.Message}");
}
}
}
}
if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_NO_BACKOFF")))
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
Directory.CreateDirectory(destDirectory);
ZipFile.ExtractToDirectory(zipFile, destDirectory);
Trace.Verbose("Create watermark file indicate task download succeed.");
File.WriteAllText(destDirectory + ".completed", DateTime.UtcNow.ToString());
executionContext.Debug($"Task '{task.Name}' has been downloaded into '{destDirectory}'.");
Trace.Info("Finished getting task.");
}
finally
{
try
{
//if the temp folder wasn't moved -> wipe it
if (Directory.Exists(tempDirectory))
{
Trace.Verbose("Deleting task temp folder: {0}", tempDirectory);
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the temp folder
Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex);
executionContext.Warning(StringUtil.Loc("FailedDeletingTempDirectory0Message1", tempDirectory, ex.Message));
}
}
}
public Definition LoadAction(IExecutionContext executionContext, Pipelines.ActionStep action)
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(action, nameof(action));
// Initialize the definition wrapper object.
var definition = new Definition()
{
Data = new ActionDefinitionData()
};
if (action.Reference.Type == Pipelines.ActionSourceType.ContainerRegistry)
{
Trace.Info("Load action that reference container from registry.");
CachedActionContainers.TryGetValue(action.Id, out var container);
ArgUtil.NotNull(container, nameof(container));
definition.Data.Execution = new ContainerActionExecutionData()
{
Image = container.ContainerImage
};
Trace.Info($"Using action container image: {container.ContainerImage}.");
}
else if (action.Reference.Type == Pipelines.ActionSourceType.Repository)
{
string actionDirectory = null;
var repoAction = action.Reference as Pipelines.RepositoryPathReference;
if (string.Equals(repoAction.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase))
{
actionDirectory = executionContext.GetGitHubContext("workspace");
if (!string.IsNullOrEmpty(repoAction.Path))
{
actionDirectory = Path.Combine(actionDirectory, repoAction.Path);
}
}
else
{
actionDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repoAction.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repoAction.Ref);
if (!string.IsNullOrEmpty(repoAction.Path))
{
actionDirectory = Path.Combine(actionDirectory, repoAction.Path);
}
}
Trace.Info($"Load action that reference repository from '{actionDirectory}'");
definition.Directory = actionDirectory;
string manifestFile = Path.Combine(actionDirectory, Constants.Path.ActionManifestYmlFile);
string manifestFileYaml = Path.Combine(actionDirectory, Constants.Path.ActionManifestYamlFile);
string dockerFile = Path.Combine(actionDirectory, "Dockerfile");
string dockerFileLowerCase = Path.Combine(actionDirectory, "dockerfile");
if (File.Exists(manifestFile) || File.Exists(manifestFileYaml))
{
var manifestManager = HostContext.GetService <IActionManifestManager>();
if (File.Exists(manifestFile))
{
definition.Data = manifestManager.Load(executionContext, manifestFile);
}
else
{
definition.Data = manifestManager.Load(executionContext, manifestFileYaml);
}
Trace.Verbose($"Action friendly name: '{definition.Data.Name}'");
Trace.Verbose($"Action description: '{definition.Data.Description}'");
if (definition.Data.Inputs != null)
{
foreach (var input in definition.Data.Inputs)
{
Trace.Verbose($"Action input: '{input.Key.ToString()}' default to '{input.Value.ToString()}'");
}
}
if (definition.Data.Execution.ExecutionType == ActionExecutionType.Container)
{
var containerAction = definition.Data.Execution as ContainerActionExecutionData;
Trace.Info($"Action container Dockerfile/image: {containerAction.Image}.");
if (containerAction.Arguments != null)
{
Trace.Info($"Action container args: {StringUtil.ConvertToJson(containerAction.Arguments)}.");
}
if (containerAction.Environment != null)
{
Trace.Info($"Action container env: {StringUtil.ConvertToJson(containerAction.Environment)}.");
}
if (!string.IsNullOrEmpty(containerAction.EntryPoint))
{
Trace.Info($"Action container entrypoint: {containerAction.EntryPoint}.");
}
if (!string.IsNullOrEmpty(containerAction.Cleanup))
{
Trace.Info($"Action container cleanup entrypoint: {containerAction.Cleanup}.");
}
if (CachedActionContainers.TryGetValue(action.Id, out var container))
{
Trace.Info($"Image '{containerAction.Image}' already built/pulled, use image: {container.ContainerImage}.");
containerAction.Image = container.ContainerImage;
}
}
else if (definition.Data.Execution.ExecutionType == ActionExecutionType.NodeJS)
{
var nodeAction = definition.Data.Execution as NodeJSActionExecutionData;
Trace.Info($"Action node.js file: {nodeAction.Script}.");
Trace.Info($"Action cleanup node.js file: {nodeAction.Cleanup ?? "N/A"}.");
}
else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Plugin)
{
var pluginAction = definition.Data.Execution as PluginActionExecutionData;
var pluginManager = HostContext.GetService <IRunnerPluginManager>();
var plugin = pluginManager.GetPluginAction(pluginAction.Plugin);
ArgUtil.NotNull(plugin, pluginAction.Plugin);
ArgUtil.NotNullOrEmpty(plugin.PluginTypeName, pluginAction.Plugin);
pluginAction.Plugin = plugin.PluginTypeName;
Trace.Info($"Action plugin: {plugin.PluginTypeName}.");
if (!string.IsNullOrEmpty(plugin.PostPluginTypeName))
{
pluginAction.Cleanup = plugin.PostPluginTypeName;
Trace.Info($"Action cleanup plugin: {plugin.PluginTypeName}.");
}
}
else
{
throw new NotSupportedException(definition.Data.Execution.ExecutionType.ToString());
}
}
else if (File.Exists(dockerFile))
{
if (CachedActionContainers.TryGetValue(action.Id, out var container))
{
definition.Data.Execution = new ContainerActionExecutionData()
{
Image = container.ContainerImage
};
}
else
{
definition.Data.Execution = new ContainerActionExecutionData()
{
Image = dockerFile
};
}
}
else if (File.Exists(dockerFileLowerCase))
{
if (CachedActionContainers.TryGetValue(action.Id, out var container))
{
definition.Data.Execution = new ContainerActionExecutionData()
{
Image = container.ContainerImage
};
}
else
{
definition.Data.Execution = new ContainerActionExecutionData()
{
Image = dockerFileLowerCase
};
}
}
else
{
var fullPath = IOUtil.ResolvePath(actionDirectory, "."); // resolve full path without access filesystem.
throw new NotSupportedException($"Can't find 'action.yml', 'action.yaml' or 'Dockerfile' under '{fullPath}'. Did you forget to run actions/checkout before running your local action?");
}
}
else if (action.Reference.Type == Pipelines.ActionSourceType.Script)
{
definition.Data.Execution = new ScriptActionExecutionData();
definition.Data.Name = "Run";
definition.Data.Description = "Execute a script";
}
else
{
throw new NotSupportedException(action.Reference.Type.ToString());
}
return(definition);
}
public void DeleteCredential()
{
ArgUtil.Equal(RunMode.Normal, HostContext.RunMode, nameof(HostContext.RunMode));
IOUtil.Delete(_credFilePath, default(CancellationToken));
}
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
var repositoryReference = repositoryAction.Reference as Pipelines.RepositoryPathReference;
ArgUtil.NotNull(repositoryReference, nameof(repositoryReference));
if (string.Equals(repositoryReference.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase))
{
Trace.Info($"Repository action is in 'self' repository.");
return;
}
if (!string.Equals(repositoryReference.RepositoryType, Pipelines.RepositoryTypes.GitHub, StringComparison.OrdinalIgnoreCase))
{
throw new NotSupportedException(repositoryReference.RepositoryType);
}
ArgUtil.NotNullOrEmpty(repositoryReference.Name, nameof(repositoryReference.Name));
ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref));
string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref);
string watermarkFile = destDirectory + ".completed";
if (File.Exists(watermarkFile))
{
executionContext.Debug($"Action '{repositoryReference.Name}@{repositoryReference.Ref}' already downloaded at '{destDirectory}'.");
return;
}
else
{
// make sure we get a clean folder ready to use.
IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
Directory.CreateDirectory(destDirectory);
executionContext.Output($"Download action repository '{repositoryReference.Name}@{repositoryReference.Ref}'");
}
#if OS_WINDOWS
string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/zipball/{repositoryReference.Ref}";
#else
string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/tarball/{repositoryReference.Ref}";
#endif
Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'.");
//download and extract action in a temp folder and rename it on success
string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid());
Directory.CreateDirectory(tempDirectory);
#if OS_WINDOWS
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.zip");
#else
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz");
#endif
Trace.Info($"Save archive '{archiveLink}' into {archiveFile}.");
try
{
int retryCount = 0;
// Allow up to 20 * 60s for any action to be downloaded from github graph.
int timeoutSeconds = 20 * 60;
while (retryCount < 3)
{
using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken))
{
try
{
//open zip stream in async mode
using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
var configurationStore = HostContext.GetService <IConfigurationStore>();
var isHostedServer = configurationStore.GetSettings().IsHostedServer;
if (isHostedServer)
{
var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
if (string.IsNullOrEmpty(authToken))
{
// TODO: Deprecate the PREVIEW_ACTION_TOKEN
authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN");
}
if (!string.IsNullOrEmpty(authToken))
{
HostContext.SecretMasker.AddValue(authToken);
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
else
{
var accessToken = executionContext.GetGitHubContext("token");
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
}
else
{
// Intentionally empty. Temporary for GHES alpha release, download from dotcom unauthenticated.
}
httpClient.DefaultRequestHeaders.UserAgent.Add(HostContext.UserAgent);
using (var result = await httpClient.GetStreamAsync(archiveLink))
{
await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token);
await fs.FlushAsync(actionDownloadCancellation.Token);
// download succeed, break out the retry loop.
break;
}
}
}
catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested)
{
Trace.Info($"Action download has been cancelled.");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
Trace.Error($"Fail to download archive '{archiveLink}' -- Attempt: {retryCount}");
Trace.Error(ex);
if (actionDownloadTimeout.Token.IsCancellationRequested)
{
// action download didn't finish within timeout
executionContext.Warning($"Action '{archiveLink}' didn't finish download within {timeoutSeconds} seconds.");
}
else
{
executionContext.Warning($"Failed to download action '{archiveLink}'. Error {ex.Message}");
}
}
}
if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF")))
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile));
executionContext.Debug($"Download '{archiveLink}' to '{archiveFile}'");
var stagingDirectory = Path.Combine(tempDirectory, "_staging");
Directory.CreateDirectory(stagingDirectory);
#if OS_WINDOWS
ZipFile.ExtractToDirectory(archiveFile, stagingDirectory);
#else
string tar = WhichUtil.Which("tar", require: true, trace: Trace);
// tar -xzf
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Info(args.Data);
}
});
processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Error(args.Data);
}
});
int exitCode = await processInvoker.ExecuteAsync(stagingDirectory, tar, $"-xzf \"{archiveFile}\"", null, executionContext.CancellationToken);
if (exitCode != 0)
{
throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}.");
}
}
#endif
// repository archive from github always contains a nested folder
var subDirectories = new DirectoryInfo(stagingDirectory).GetDirectories();
if (subDirectories.Length != 1)
{
throw new InvalidOperationException($"'{archiveFile}' contains '{subDirectories.Length}' directories");
}
else
{
executionContext.Debug($"Unwrap '{subDirectories[0].Name}' to '{destDirectory}'");
IOUtil.CopyDirectory(subDirectories[0].FullName, destDirectory, executionContext.CancellationToken);
}
Trace.Verbose("Create watermark file indicate action download succeed.");
File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString());
executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'.");
Trace.Info("Finished getting action repository.");
}
finally
{
try
{
//if the temp folder wasn't moved -> wipe it
if (Directory.Exists(tempDirectory))
{
Trace.Verbose("Deleting action temp folder: {0}", tempDirectory);
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the temp folder
Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex);
}
}
}
public void DeleteSettings()
{
ArgUtil.Equal(RunMode.Normal, HostContext.RunMode, nameof(HostContext.RunMode));
IOUtil.Delete(_configFilePath, default(CancellationToken));
}
/// <summary>Create a new sequence from a file.</summary>
/// <remarks>
/// Create a new sequence from a file.
/// <p>
/// The entire file contents are used.
/// </remarks>
/// <param name="file">the text file.</param>
/// <exception cref="System.IO.IOException">if Exceptions occur while reading the file
/// </exception>
public RawText(FilePath file) : this(IOUtil.ReadFully(file))
{
}
public void DeleteAutoLogonSettings()
{
IOUtil.Delete(_autoLogonSettingsFilePath, default(CancellationToken));
}
/// <summary>
/// Map a view as a Stream.
/// </summary>
/// <param name="access">desired access to the view</param>
private Stream MapView(IOUtil.Win32.MapAccess access)
{
return MapView(access, 0, (int)fileLength);
}
private async Task StartContainerAsync(IExecutionContext executionContext, ContainerInfo container)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(container, nameof(container));
ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage));
Trace.Info($"Container name: {container.ContainerName}");
Trace.Info($"Container image: {container.ContainerImage}");
Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}");
Trace.Info($"Container options: {container.ContainerCreateOptions}");
Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}");
foreach (var port in container.UserPortMappings)
{
Trace.Info($"User provided port: {port.Value}");
}
foreach (var volume in container.UserMountVolumes)
{
Trace.Info($"User provided volume: {volume.Value}");
}
if (container.ImageOS != PlatformUtil.OS.Windows)
{
string defaultWorkingDirectory = executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory);
defaultWorkingDirectory = container.TranslateContainerPathForImageOS(PlatformUtil.HostOS, container.TranslateToContainerPath(defaultWorkingDirectory));
if (string.IsNullOrEmpty(defaultWorkingDirectory))
{
throw new NotSupportedException(StringUtil.Loc("ContainerJobRequireSystemDefaultWorkDir"));
}
string workingDirectory = IOUtil.GetDirectoryName(defaultWorkingDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), container.ImageOS);
string mountWorkingDirectory = container.TranslateToHostPath(workingDirectory);
executionContext.Debug($"Default Working Directory {defaultWorkingDirectory}");
executionContext.Debug($"Working Directory {workingDirectory}");
executionContext.Debug($"Mount Working Directory {mountWorkingDirectory}");
if (!string.IsNullOrEmpty(workingDirectory))
{
container.MountVolumes.Add(new MountVolume(mountWorkingDirectory, workingDirectory, readOnly: container.isReadOnlyVolume(Constants.DefaultContainerMounts.Work)));
}
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks)),
readOnly: container.isReadOnlyVolume(Constants.DefaultContainerMounts.Tasks)));
}
else
{
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work)),
readOnly: container.isReadOnlyVolume(Constants.DefaultContainerMounts.Work)));
}
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools)),
readOnly: container.isReadOnlyVolume(Constants.DefaultContainerMounts.Tools)));
bool externalReadOnly = container.ImageOS != PlatformUtil.OS.Windows || container.isReadOnlyVolume(Constants.DefaultContainerMounts.Externals); // This code was refactored to use PlatformUtils. The previous implementation did not have the externals directory mounted read-only for Windows.
// That seems wrong, but to prevent any potential backwards compatibility issues, we are keeping the same logic
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), externalReadOnly));
if (container.ImageOS != PlatformUtil.OS.Windows)
{
// Ensure .taskkey file exist so we can mount it.
string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey");
if (!File.Exists(taskKeyFile))
{
File.WriteAllText(taskKeyFile, string.Empty);
}
container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile), readOnly: true));
}
if (container.IsJobContainer)
{
// See if this container brings its own Node.js
container.CustomNodePath = await _dockerManger.DockerInspect(context : executionContext,
dockerObject : container.ContainerImage,
options : $"--format=\"{{{{index .Config.Labels \\\"{_nodeJsPathLabel}\\\"}}}}\"");
string node;
if (!string.IsNullOrEmpty(container.CustomNodePath))
{
node = container.CustomNodePath;
}
else
{
node = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}"));
// if on Mac OS X, require container to have node
if (PlatformUtil.RunningOnMacOS)
{
container.CustomNodePath = "node";
node = container.CustomNodePath;
}
// if running on Windows, and attempting to run linux container, require container to have node
else if (PlatformUtil.RunningOnWindows && container.ImageOS == PlatformUtil.OS.Linux)
{
container.CustomNodePath = "node";
node = container.CustomNodePath;
}
}
string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\"";
container.ContainerCommand = sleepCommand;
}
container.ContainerId = await _dockerManger.DockerCreate(executionContext, container);
ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId));
if (container.IsJobContainer)
{
executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId);
}
// Start container
int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId);
if (startExitCode != 0)
{
throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}");
}
try
{
// Make sure container is up and running
var psOutputs = await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --filter status=running --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true)
{
// container is not up and running, pull docker log for this container.
await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId);
if (logsExitCode != 0)
{
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
}
executionContext.Warning($"Docker container {container.ContainerId} is not in running state.");
}
}
catch (Exception ex)
{
// pull container log is best effort.
Trace.Error("Catch exception when check container log and container status.");
Trace.Error(ex);
}
// Get port mappings of running container
if (!container.IsJobContainer)
{
container.AddPortMappings(await _dockerManger.DockerPort(executionContext, container.ContainerId));
foreach (var port in container.PortMappings)
{
executionContext.Variables.Set(
$"{Constants.Variables.Agent.ServicePortPrefix}.{container.ContainerNetworkAlias}.ports.{port.ContainerPort}",
$"{port.HostPort}");
}
}
if (!PlatformUtil.RunningOnWindows)
{
if (container.IsJobContainer)
{
// Ensure bash exist in the image
int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"sh -c \"command -v bash\"");
if (execWhichBashExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}");
}
// Get current username
container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName));
// Get current userId
container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId));
executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId));
// Create an user with same uid as the agent run as user inside the container.
// All command execute in docker will run as Root by default,
// this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container.
// So, we create a user account with same UID inside the container and let all docker exec command run as that user.
string containerUserName = string.Empty;
// We need to find out whether there is a user with same UID inside the container
List <string> userNames = new List <string>();
int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"getent passwd {container.CurrentUserId} | cut -d: -f1 \"", userNames);
if (execGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}");
}
if (userNames.Count > 0)
{
// check all potential username that might match the UID.
foreach (string username in userNames)
{
int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}");
if (execIdExitCode == 0)
{
containerUserName = username;
break;
}
}
}
// Create a new user with same UID
if (string.IsNullOrEmpty(containerUserName))
{
string userNameSuffix = "_azpcontainer";
// Linux allows for a 32-character username
int keepLength = Math.Min(32 - userNameSuffix.Length, container.CurrentUserName.Length);
containerUserName = $"{container.CurrentUserName.Substring(0, keepLength)}{userNameSuffix}";
int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}");
if (execUseraddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}");
}
}
executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName));
// Create a new group for giving sudo permission
int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd azure_pipelines_sudo");
if (execGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}");
}
// Add the new created user to the new created sudo group.
int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G azure_pipelines_sudo {containerUserName}");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}");
}
// Allow the new sudo group run any sudo command without providing password.
int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%azure_pipelines_sudo ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\"");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}");
}
if (AgentKnobs.SetupDockerGroup.GetValue(executionContext).AsBoolean())
{
executionContext.Output(StringUtil.Loc("AllowContainerUserRunDocker", containerUserName));
// Get docker.sock group id on Host
string statFormatOption = "-c %g";
if (PlatformUtil.RunningOnMacOS)
{
statFormatOption = "-f %g";
}
string dockerSockGroupId = (await ExecuteCommandAsync(executionContext, "stat", $"{statFormatOption} /var/run/docker.sock")).FirstOrDefault();
// We need to find out whether there is a group with same GID inside the container
string existingGroupName = null;
List <string> groupsOutput = new List <string>();
int execGroupGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"cat /etc/group\"", groupsOutput);
if (execGroupGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupGrepExitCode}");
}
if (groupsOutput.Count > 0)
{
// check all potential groups that might match the GID.
foreach (string groupOutput in groupsOutput)
{
if (!string.IsNullOrEmpty(groupOutput))
{
var groupSegments = groupOutput.Split(':');
if (groupSegments.Length != 4)
{
Trace.Warning($"Unexpected output from /etc/group: '{groupOutput}'");
}
else
{
// the output of /etc/group should looks like `group:x:gid:`
var groupName = groupSegments[0];
var groupId = groupSegments[2];
if (string.Equals(dockerSockGroupId, groupId))
{
existingGroupName = groupName;
break;
}
}
}
}
}
if (string.IsNullOrEmpty(existingGroupName))
{
// create a new group with same gid
existingGroupName = "azure_pipelines_docker";
int execDockerGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd -g {dockerSockGroupId} azure_pipelines_docker");
if (execDockerGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execDockerGroupaddExitCode}");
}
}
// Add the new created user to the docker socket group.
int execGroupUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G {existingGroupName} {containerUserName}");
if (execGroupUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupUsermodExitCode}");
}
// if path to node is just 'node', with no path, let's make sure it is actually there
if (string.Equals(container.CustomNodePath, "node", StringComparison.OrdinalIgnoreCase))
{
List <string> nodeVersionOutput = new List <string>();
int execNodeVersionExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"node -v\"", nodeVersionOutput);
if (execNodeVersionExitCode != 0)
{
throw new InvalidOperationException($"Unable to get node version on container {container.ContainerId}. Got exit code {execNodeVersionExitCode} from docker exec");
}
if (nodeVersionOutput.Count > 0)
{
executionContext.Output($"Detected Node Version: {nodeVersionOutput[0]}");
Trace.Info($"Using node version {nodeVersionOutput[0]} in container {container.ContainerId}");
}
else
{
throw new InvalidOperationException($"Unable to get node version on container {container.ContainerId}. No output from node -v");
}
}
}
}
}
}
/// <summary>
/// Constructor used internally by MemoryMappedFile.
/// </summary>
/// <param name="baseAddress">base address where the view starts</param>
/// <param name="length">length of view in bytes</param>
/// <param name="protection"></param>
internal MapViewStream(IntPtr baseAddress, long length, IOUtil.Win32.Protect protection)
{
this.bufferBase = baseAddress;
this.length = length;
this.protection = protection;
this.position = 0;
this.isOpen = (baseAddress != IntPtr.Zero);
}
private void CreateModpackButton_Click(object sender, RoutedEventArgs e)
{
if (ModPackName.Text.Equals(string.Empty))
{
if (FlexibleMessageBox.Show(null,
UIMessages.DefaultModPackNameMessage,
UIMessages.NoNameFoundTitle, MessageBoxButtons.OKCancel, MessageBoxIcon.Warning) ==
System.Windows.Forms.DialogResult.OK)
{
ModPackName.Text = "ModPack";
}
else
{
return;
}
}
char[] invalidChars = { '/', '\\', ':', '*', '?', '"', '<', '>', '|' };
if (ModPackName.Text.IndexOfAny(invalidChars) >= 0)
{
if (FlexibleMessageBox.Show(null,
UIMessages.InvalidCharacterModpackNameMessage,
UIMessages.InvalidCharacterModpackNameTitle, MessageBoxButtons.OK, MessageBoxIcon.Warning) ==
System.Windows.Forms.DialogResult.OK)
{
return;
}
}
string verString = ModPackVersion.Text.Replace("_", "0");
// Replace commas with periods for different culture formats such as FR
if (verString.Contains(","))
{
verString = verString.Replace(",", ".");
}
Version versionNumber = Version.Parse(verString);
if (versionNumber.ToString().Equals("0.0.0"))
{
if (FlexibleMessageBox.Show(null,
UIMessages.DefaultModPackVersionMessage,
UIMessages.NoVersionFoundTitle, MessageBoxButtons.OKCancel, MessageBoxIcon.Warning) ==
System.Windows.Forms.DialogResult.OK)
{
versionNumber = new Version(1, 0, 0);
}
else
{
return;
}
}
if (ModPackAuthor.Text.Equals(string.Empty))
{
if (FlexibleMessageBox.Show(null,
UIMessages.DefaultModPackAuthorMessage,
UIMessages.NoAuthorFoundTitle, MessageBoxButtons.OKCancel, MessageBoxIcon.Warning) ==
System.Windows.Forms.DialogResult.OK)
{
ModPackAuthor.Text = "TexTools User";
}
else
{
return;
}
}
if (!String.IsNullOrWhiteSpace(ModPackUrl.Text))
{
var url = IOUtil.ValidateUrl(ModPackUrl.Text);
if (url != null)
{
ModPackUrl.Text = url;
}
else
{
ModPackUrl.Text = "";
}
}
else
{
ModPackUrl.Text = "";
}
_vm.Author = ModPackAuthor.Text;
_vm.Name = ModPackName.Text;
_vm.Version = versionNumber;
_vm.Description = ModPackDescription.Text;
_vm.Url = ModPackUrl.Text;
if (CreateModpack != null)
{
CreateModpack.Invoke(this, _vm);
}
}
