private async Task <Dictionary <string, string> > ParseProjectAsync(IFileOperationsExecuter fileOps, string filePath) { XDocument xdoc; using (var stream = await fileOps.OpenFileAsync(filePath, FileMode.Open, FileAccess.Read)) { try { xdoc = XDocument.Load(stream); } catch (Exception ex) { this.LogWarning($"{filePath} is not a valid XML file: {ex.Message}"); return(null); } } var elements = xdoc.Root .Descendants() .Where(d => d.Name.LocalName == "PackageReference"); var result = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); foreach (var e in elements) { result[(string)e.Attribute("Include")] = (string)e.Attribute("Version"); } return(result); }
private void ApplyConfigTransforms(string sourceFile, string suffix, IFileOperationsExecuter fileOps, HashSet <string> alreadyRun) { if (!suffix.EndsWith(".config", StringComparison.OrdinalIgnoreCase)) { suffix = suffix + ".config"; } // Update the extension of the found config to the suffix extensions string transformFile = Path.ChangeExtension(sourceFile, suffix); // Try and find the transform config to apply to the source file if ((fileOps.FileExists(transformFile) && !string.Equals(sourceFile, transformFile, StringComparison.InvariantCultureIgnoreCase)) && !alreadyRun.Contains(transformFile)) { alreadyRun.Add(transformFile); var transformExePath = Path.Combine(fileOps.GetBaseWorkingDirectory(), @"ExtTemp\WindowsSDK\Resources\ctt.exe"); if (!fileOps.FileExists(transformExePath)) { throw new FileNotFoundException("ctt.exe could not be found on the agent.", transformExePath); } // Get all the arguments that form the executable transform task var arguments = BuildArguments(sourceFile, transformFile); LogInformation("Performing XDT transform..."); // Call the transform executable for the file ExecuteCommandLine(transformExePath, arguments); } }
public void Initialize() { string asmDir = PathEx.GetDirectoryName(typeof(GitTests).Assembly.Location); this.rootDir = PathEx.Combine(asmDir, "test-root"); DirectoryEx.Create(this.rootDir); DirectoryEx.Clear(this.rootDir); if (FileEx.Exists(credentialsFilePath)) { var lines = File.ReadAllLines(credentialsFilePath); this.userName = lines[0]; this.password = SecureStringExtensions.ToSecureString(lines[1]); } var fileOps = new TestFileOperationsExecuter(Path.Combine(this.rootDir, "agent")); //var fileOps = new SimulatedFileOperationsExecuter(fileOps); //fileOps.MessageLogged += (s, e) => TestLogger.Instance.Log(e.Level, e.Message); this.fileOps = fileOps; this.processExecuter = new TestRemoteProcessExecuter(); this.jobExecuter = new TestRemoteJobExecuter(); }
private async Task <Dictionary <string, string> > ParsePackagesConfigAsync(IFileOperationsExecuter fileOps, string filePath) { XDocument xdoc; using (var stream = await fileOps.OpenFileAsync(filePath, FileMode.Open, FileAccess.Read)) { try { xdoc = XDocument.Load(stream); } catch (Exception ex) { this.LogWarning($"{filePath} is not a valid XML file: {ex.Message}"); return(null); } } if (xdoc.Root?.Name.LocalName != "packages") { this.LogWarning($"{filePath} is not a valid NuGet packages.config file: missing root \"packages\" element."); return(null); } return(xdoc.Root .Elements("package") .ToDictionary(e => (string)e.Attribute("id"), e => (string)e.Attribute("version"), StringComparer.OrdinalIgnoreCase)); }
protected static async Task CopyFilesAsync(IFileOperationsExecuter fileOps, string sourceDirectory, string targetDirectory, bool keepInternals = false) { if (!await fileOps.DirectoryExistsAsync(sourceDirectory).ConfigureAwait(false)) { return; } char separator = fileOps.DirectorySeparator; var infos = await fileOps.GetFileSystemInfosAsync(sourceDirectory, keepInternals?MaskingContext.IncludeAll : new MaskingContext(new[] { "**" }, new[] { "**" + separator + ".git**", ".git**" })).ConfigureAwait(false); var directoriesToCreate = infos.OfType <SlimDirectoryInfo>().Select(d => CombinePaths(targetDirectory, d.FullName.Substring(sourceDirectory.Length), separator)).ToArray(); var relativeFileNames = infos.OfType <SlimFileInfo>().Select(f => f.FullName.Substring(sourceDirectory.Length).TrimStart(separator)).ToArray(); await fileOps.CreateDirectoryAsync(targetDirectory).ConfigureAwait(false); foreach (string folder in directoriesToCreate) { await fileOps.CreateDirectoryAsync(folder).ConfigureAwait(false); } await fileOps.FileCopyBatchAsync( sourceDirectory, relativeFileNames, targetDirectory, relativeFileNames, true, true ).ConfigureAwait(false); }
private string GetAbsPath(IFileOperationsExecuter agt, string path) { // TODO This duplicates pathing logic in AgentHelper::GetWorkingDirectory // The reason we have to do this here is because the RemoteActionExecutor // will split the working directory into source and target. Until there is // a way to override or block the splitting, then this logic is required. if (path == null) { return(this.Context.SourceDirectory); } else if (Path.IsPathRooted(path)) { return(path); } else if (path.StartsWith("~")) { return(agt.GetLegacyWorkingDirectory((IGenericBuildMasterContext)this.Context, path)); } else { return(agt.CombinePath( this.Context.SourceDirectory, path )); } }
private async Task UnlockRegistryAsync(IFileOperationsExecuter fileOps) { if (this.LockToken == null) { return; } var fileName = fileOps.CombinePath(this.RegistryRoot, ".lock"); if (!await fileOps.FileExistsAsync(fileName).ConfigureAwait(false)) { return; } string token; using (var lockStream = await fileOps.OpenFileAsync(fileName, FileMode.Open, FileAccess.Read).ConfigureAwait(false)) using (var reader = new StreamReader(lockStream, InedoLib.UTF8Encoding)) { reader.ReadLine(); token = reader.ReadLine(); } if (token == this.LockToken) { await fileOps.DeleteFileAsync(fileName).ConfigureAwait(false); } this.LockToken = null; }
public GitCommandLineClient(string gitExePath, IRemoteProcessExecuter processExecuter, IFileOperationsExecuter fileOps, GitRepositoryInfo repository, ILogSink log, CancellationToken cancellationToken) : base(repository, log) { this.gitExePath = gitExePath ?? throw new ArgumentNullException(nameof(gitExePath)); this.processExecuter = processExecuter ?? throw new ArgumentNullException(nameof(processExecuter)); this.fileOps = fileOps ?? throw new ArgumentNullException(nameof(fileOps)); this.cancellationToken = cancellationToken; }
private string GetXmlOutputPath(IFileOperationsExecuter fileOps) { if (string.IsNullOrWhiteSpace(this.CustomXmlOutputPath)) { return(fileOps.CombinePath(this.Context.TempDirectory, Guid.NewGuid().ToString() + ".xml")); } return(fileOps.CombinePath(this.Context.SourceDirectory, this.CustomXmlOutputPath)); }
public RemoteTemporaryFile(IFileOperationsExecuter fileOps, ILogSink log) { this.fileOps = fileOps; this.log = log; string workingDirectory = fileOps.GetBaseWorkingDirectory(); string fileName = Guid.NewGuid().ToString("n"); this.Path = fileOps.CombinePath(workingDirectory, fileName); }
private static async Task WriteInstalledPackagesAsync(IFileOperationsExecuter fileOps, string registryRoot, IEnumerable <RegisteredPackage> packages) { var fileName = fileOps.CombinePath(registryRoot, "installedPackages.json"); using (var configStream = await fileOps.OpenFileAsync(fileName, FileMode.Create, FileAccess.Write).ConfigureAwait(false)) using (var streamWriter = new StreamWriter(configStream, InedoLib.UTF8Encoding)) using (var jsonWriter = new JsonTextWriter(streamWriter)) { new JsonSerializer { Formatting = Formatting.Indented }.Serialize(jsonWriter, packages.ToArray()); } }
private async Task TransferFileAsync(IFileOperationsExecuter sourceFileOps, SlimFileInfo sourceFile, IFileOperationsExecuter targetFileOps, SlimFileSystemInfo target, string targetDirectory) { if (target == null) { if (this.VerboseLogging) { this.LogDebug($"{sourceFile.Name} does not exist in {targetDirectory}."); } } var targetFile = target as SlimFileInfo; if (targetFile != null) { this.LogDebug($"{sourceFile.Name} already exists in {targetDirectory}."); this.LogDebug($"Source timestamp: {sourceFile.LastWriteTimeUtc}, Target timestamp: {targetFile.LastWriteTimeUtc}"); this.LogDebug($"Source size: {sourceFile.Size}, Target size: {targetFile.Size}"); if (sourceFile.LastWriteTimeUtc == targetFile.LastWriteTimeUtc && sourceFile.Size == targetFile.Size) { this.LogDebug($"Size and timestamp are the same; skipping {sourceFile.Name}..."); return; } } else if (target != null) { this.LogDebug($"{sourceFile.Name} is a file in {sourceFile.DirectoryName}, but a directory in {targetDirectory}."); this.LogDebug($"Deleting directory {target.FullName}..."); await targetFileOps.DeleteDirectoriesAsync(new[] { target.FullName }).ConfigureAwait(false); } else { if (!string.IsNullOrEmpty(targetDirectory)) { await targetFileOps.CreateDirectoryAsync(targetDirectory).ConfigureAwait(false); } } var targetFileName = PathEx.Combine(targetDirectory, sourceFile.Name).Replace(sourceFileOps.DirectorySeparator, targetFileOps.DirectorySeparator); this.LogDebug($"Transferring {sourceFile.Name} to {targetDirectory}..."); using (var sourceStream = await sourceFileOps.OpenFileAsync(sourceFile.FullName, FileMode.Open, FileAccess.Read).ConfigureAwait(false)) using (var targetStream = await targetFileOps.OpenFileAsync(targetFileName, FileMode.Create, FileAccess.Write).ConfigureAwait(false)) { await sourceStream.CopyToAsync(targetStream).ConfigureAwait(false); } }
/// <summary> /// Determine if a directory on a remote agent exists. /// </summary> /// <param name="agent"></param> /// <param name="path"></param> /// <returns></returns> /// <remarks> /// This is hack to work around a limitation in the IFileOperationsExecuter for BM less than 3.6 /// </remarks> public static bool DirectoryExists2(this IFileOperationsExecuter agent, string path) { var proxyAgent = agent as IPersistedObjectExecuter; if (proxyAgent != null) { return((bool)proxyAgent.ExecuteMethodOnXmlPersistedObject(Util.Persistence.SerializeToPersistedObjectXml(new ProxyHelper()), "DirectoryExists", new object[] { path })); } var results = agent.GetDirectoryEntry(new GetDirectoryEntryCommand { Path = path, Recurse = false }); return(results != null && results.Entry != null && results.Exceptions == null); }
private static string GetBuildMasterExecutionBaseWorkingDirectory(IFileOperationsExecuter agent, IOperationExecutionContext context) { if (agent == null) { throw new ArgumentNullException(nameof(agent)); } if (context == null) { throw new ArgumentNullException(nameof(context)); } var relativePath = "_E" + context.ExecutionId; return(agent.CombinePath(agent.GetBaseWorkingDirectory(), relativePath)); }
private string GetNUnitExePath(IFileOperationsExecuter fileOps) { if (!string.IsNullOrWhiteSpace(this.ExePath)) { return(fileOps.CombinePath(this.Context.SourceDirectory, this.ExePath)); } var configurer = (NUnitConfigurer)this.GetExtensionConfigurer(); if (string.IsNullOrWhiteSpace(configurer.NUnitConsoleExePath)) { throw new InvalidOperationException("The path to NUnit was not specified in either the action or the selected NUnit extension's configuration."); } return(fileOps.CombinePath(this.Context.SourceDirectory, configurer.NUnitConsoleExePath)); }
private async Task LockRegistryAsync(IFileOperationsExecuter fileOps, CancellationToken cancellationToken) { var fileName = fileOps.CombinePath(this.RegistryRoot, ".lock"); var lockDescription = "Locked by " + SDK.ProductName; var lockToken = Guid.NewGuid().ToString(); TryAgain: var fileInfo = await getFileInfoAsync().ConfigureAwait(false); string lastToken = null; while (fileInfo.Item1 != null && DateTime.UtcNow - fileInfo.Item1.LastWriteTimeUtc <= new TimeSpan(0, 0, 10)) { if ((lastToken == null || !string.Equals(lastToken, fileInfo.Item3)) && fileInfo.Item3 != null) { this.logger?.LogDebug("Package registry is locked: " + fileInfo.Item2); lastToken = fileInfo.Item3; } await Task.Delay(500, cancellationToken).ConfigureAwait(false); fileInfo = await getFileInfoAsync().ConfigureAwait(false); } // ensure registry root exists await fileOps.CreateDirectoryAsync(this.RegistryRoot).ConfigureAwait(false); try { // write out the lock info using (var lockStream = await fileOps.OpenFileAsync(fileName, FileMode.Create, FileAccess.Write).ConfigureAwait(false)) using (var writer = new StreamWriter(lockStream, InedoLib.UTF8Encoding)) { writer.WriteLine(lockDescription); writer.WriteLine(lockToken); } // verify that we acquired the lock using (var lockStream = await fileOps.OpenFileAsync(fileName, FileMode.Open, FileAccess.Read).ConfigureAwait(false)) using (var reader = new StreamReader(lockStream, InedoLib.UTF8Encoding)) { if (reader.ReadLine() != lockDescription) { goto TryAgain; } if (reader.ReadLine() != lockToken) { goto TryAgain; } } } catch (IOException ex) { this.logger?.LogDebug("Locking package registry failed: " + ex.Message); // file may be in use by other process goto TryAgain; } // at this point, lock is acquired provided everyone is following the rules this.LockToken = lockToken; async Task <(SlimFileInfo, string, string)> getFileInfoAsync() { try { var info = await fileOps.GetFileInfoAsync(fileName).ConfigureAwait(false); string description = null, token = null; try { using (var lockStream = await fileOps.OpenFileAsync(fileName, FileMode.Open, FileAccess.Read).ConfigureAwait(false)) using (var reader = new StreamReader(lockStream, InedoLib.UTF8Encoding)) { description = await reader.ReadLineAsync().ConfigureAwait(false); token = await reader.ReadLineAsync().ConfigureAwait(false); } } catch (IOException) { } return(info, description, token); } catch (FileNotFoundException) { return(null, null, null); } catch (DirectoryNotFoundException) { return(null, null, null); } } }
private string GetXmlOutputPath(IFileOperationsExecuter fileOps) { if (string.IsNullOrWhiteSpace(this.CustomXmlOutputPath)) return fileOps.CombinePath(this.Context.TempDirectory, Guid.NewGuid().ToString() + ".xml"); return fileOps.CombinePath(this.Context.SourceDirectory, this.CustomXmlOutputPath); }
private async Task CopyDirectoryAsync(IFileOperationsExecuter fileOps, string sourcePath, string targetPath, IOperationExecutionContext context) { if (!await fileOps.DirectoryExistsAsync(sourcePath).ConfigureAwait(false)) { this.LogWarning($"Source directory {sourcePath} does not exist."); return; } var infos = await fileOps.GetFileSystemInfosAsync( sourcePath, new MaskingContext(this.Includes, this.Excludes) ).ConfigureAwait(false); var files = infos.OfType <SlimFileInfo>(); foreach (var file in files) { context.CancellationToken.ThrowIfCancellationRequested(); var targetFileName = PathEx.Combine(targetPath, file.FullName.Substring(sourcePath.Length).TrimStart('/', '\\')); if (this.VerboseLogging) { this.LogDebug($"Copying {file.FullName} to {targetFileName}..."); } try { if (!this.Overwrite && await fileOps.FileExistsAsync(targetFileName).ConfigureAwait(false)) { this.LogError($"Target file {targetFileName} already exists and overwrite is set to false."); continue; } await fileOps.CreateDirectoryAsync(file.DirectoryName).ConfigureAwait(false); await fileOps.CopyFileAsync(file.FullName, targetFileName, this.Overwrite).ConfigureAwait(false); this.filesCopied++; } catch (Exception ex) { this.LogError($"Cannot copy {file.FullName}: {ex.Message}"); } } var dirs = infos.OfType <SlimDirectoryInfo>(); foreach (var dir in dirs) { context.CancellationToken.ThrowIfCancellationRequested(); var targetDir = PathEx.Combine(targetPath, dir.FullName.Substring(sourcePath.Length).TrimStart('/', '\\')); if (this.VerboseLogging) { this.LogDebug($"Creating directory {targetDir}..."); } await fileOps.CreateDirectoryAsync(targetDir).ConfigureAwait(false); this.directoriesCopied++; } }
public static string GetFullRepositoryPath(this IGitRepository repo, IFileOperationsExecuter agent) { return(agent.CombinePath(agent.GetBaseWorkingDirectory(), "GitRepositories", repo.RepositoryPath)); }
private static async Task <List <RegisteredPackage> > GetInstalledPackagesAsync(IFileOperationsExecuter fileOps, string registryRoot) { var fileName = fileOps.CombinePath(registryRoot, "installedPackages.json"); if (!await fileOps.FileExistsAsync(fileName).ConfigureAwait(false)) { return(new List <RegisteredPackage>()); } using (var configStream = await fileOps.OpenFileAsync(fileName, FileMode.Open, FileAccess.Read).ConfigureAwait(false)) using (var streamReader = new StreamReader(configStream, InedoLib.UTF8Encoding)) using (var jsonReader = new JsonTextReader(streamReader)) { return((new JsonSerializer().Deserialize <RegisteredPackage[]>(jsonReader) ?? new RegisteredPackage[0]).ToList()); } }
private string GetNUnitExePath(IFileOperationsExecuter fileOps) { if (!string.IsNullOrWhiteSpace(this.ExePath)) return fileOps.GetWorkingDirectory(this.Context.ApplicationId, this.Context.DeployableId ?? 0, this.ExePath); var configurer = (NUnitConfigurer)this.GetExtensionConfigurer(); if (string.IsNullOrWhiteSpace(configurer.NUnitConsoleExePath)) throw new InvalidOperationException("The path to NUnit was not specified in either the action or the selected NUnit extension's configuration."); return fileOps.GetWorkingDirectory(this.Context.ApplicationId, this.Context.DeployableId ?? 0, configurer.NUnitConsoleExePath); }
private void TransferFiles(IFileOperationsExecuter sourceAgentHelper, char srcSeparator, string srcPath, DirectoryEntryInfo sourceDir, int targetServerId) { using (var targetAgent = BuildMasterAgent.Create(targetServerId)) { var fileOps = targetAgent.GetService <IFileOperationsExecuter>(); char targetSeparator = fileOps.DirectorySeparator; var tarPath = fileOps.GetLegacyWorkingDirectory( (IGenericBuildMasterContext)this.Context, this.TargetDirectory ); this.LogDebug("Full target directory: {0}", tarPath); fileOps.CreateDirectory(tarPath); this.LogDebug("Loading target file list..."); var targetDir = fileOps.GetDirectoryEntry( new GetDirectoryEntryCommand { Path = tarPath, IncludeRootPath = true, Recurse = true } ).Entry; this.LogDebug("Performing directory comparison..."); List <string> filesToCopy = new List <string>(), foldersToCopy = new List <string>(), filesToDelete = new List <string>(), foldersToDelete = new List <string>(); Util.Files.Comparison.CompareDirectories( sourceDir, targetDir, filesToCopy, foldersToCopy, filesToDelete, foldersToDelete); // Make sure target files and folders to delete are canonical paths for (int i = 0; i < filesToDelete.Count; i++) { filesToDelete[i] = filesToDelete[i].Replace(srcSeparator, targetSeparator); } for (int i = 0; i < foldersToDelete.Count; i++) { foldersToDelete[i] = foldersToDelete[i].Replace(srcSeparator, targetSeparator); } // Run Filters if (this.IncludeFileMasks.Length != 0 && !(this.IncludeFileMasks.Length == 1 && string.IsNullOrEmpty(this.IncludeFileMasks[0]))) { filesToCopy = new List <string>(Util.Files.Comparison .GetMatches(srcPath, filesToCopy.ToArray(), IncludeFileMasks)); foldersToCopy = new List <string>(Util.Files.Comparison .GetMatches(srcPath, foldersToCopy.ToArray(), IncludeFileMasks)); filesToDelete = new List <string>(Util.Files.Comparison .GetMatches(tarPath, filesToDelete.ToArray(), IncludeFileMasks)); foldersToDelete = new List <string>(Util.Files.Comparison .GetMatches(tarPath, foldersToDelete.ToArray(), IncludeFileMasks)); } if (this.DeleteTarget) { this.LogInformation("Deleting files and directories that are not present in the source directory..."); if (filesToDelete.Count == 0) { this.LogDebug("No files to delete in target directory."); } else { this.LogDebug("Deleting {0} files from target directory...", filesToDelete.Count); } foreach (string path in filesToDelete) { this.LogDebug("\t" + path.Substring(tarPath.Length)); } fileOps.DeleteFiles(filesToDelete.ToArray()); if (foldersToDelete.Count == 0) { this.LogDebug("No directories to delete in target directory."); } else { this.LogDebug("Deleting {0} directories from target directory...", foldersToDelete.Count); } foreach (string path in foldersToDelete) { this.LogDebug("\t" + path.Substring(tarPath.Length)); } fileOps.DeleteDirectories(foldersToDelete.ToArray()); } else { this.LogDebug("Files and directories not present in source directory will not be deleted from target directory."); } this.LogInformation("Creating missing directories in target directory..."); if (foldersToCopy.Count == 0) { this.LogDebug("No directories missing in target directory."); } else { this.LogDebug("Creating {0} directories in target directory...", foldersToCopy.Count); } foreach (string directoryToCopy in foldersToCopy) { string relativeTargetPath = directoryToCopy.Substring(srcPath.Length) .Replace(srcSeparator, targetSeparator); if (relativeTargetPath.StartsWith(targetSeparator.ToString())) { relativeTargetPath = relativeTargetPath.Substring(1); } this.LogDebug("\t" + relativeTargetPath); fileOps.CreateDirectory(fileOps.CombinePath(tarPath, relativeTargetPath)); } this.LogInformation("Copying or transferring modified files from source directory to target directory..."); if (filesToCopy.Count == 0) { this.LogDebug("No files to copy to the target directory."); } else { this.LogDebug("Copying {0} files to the target directory...", filesToCopy.Count); } // Build list of source and target files to copy. var sourceFilesToCopy = new List <string>(filesToCopy.Count); var destFilesToCreate = new List <string>(filesToCopy.Count); foreach (var fileToCopy in filesToCopy) { var relativeSourcePath = fileToCopy.Substring(srcPath.Length) .TrimStart(srcSeparator); sourceFilesToCopy.Add(relativeSourcePath); var relativeDestPath = relativeSourcePath.Replace(srcSeparator, targetSeparator); destFilesToCreate.Add(relativeDestPath); // These are copied in a batch, so can't get logged all at once. if (this.Context.SourceServerId == targetServerId) { this.LogDebug("Copying all files from: {0}", relativeDestPath); } } if (this.Context.SourceServerId == targetServerId) { this.LogDebug("Both source and target servers are the same, performing batch copy of files..."); fileOps.FileCopyBatch(srcPath, sourceFilesToCopy.ToArray(), tarPath, destFilesToCreate.ToArray(), true, true); } else { this.LogDebug("Transferring files from source server to target server..."); for (int i = 0; i < sourceFilesToCopy.Count; i++) { this.LogDebug("\t" + destFilesToCreate[i]); var fullSourcePath = CombinePath(srcPath, sourceFilesToCopy[i], srcSeparator); var fullTargetPath = CombinePath(tarPath, destFilesToCreate[i], targetSeparator); string targetDirectory = PathEx.GetDirectoryName(fullTargetPath); if (!string.IsNullOrEmpty(targetDirectory)) { try { fileOps.CreateDirectory(targetDirectory); } catch (Exception ex) { this.LogDebug("Could not create directory at \"{0}\"; error was: {1}", targetDirectory, ex.Message); } } Util.Files.TransferFile(sourceAgentHelper, fullSourcePath, fileOps, fullTargetPath); var lastModified = sourceAgentHelper.GetFileInfo(fullSourcePath).LastWriteTimeUtc; fileOps.SetLastWriteTime(fullTargetPath, lastModified); } } } }
public static string GetFullRepositoryPath(this IGitRepository repo, IFileOperationsExecuter agent) { return agent.CombinePath(agent.GetBaseWorkingDirectory(), "GitRepositories", repo.RepositoryPath); }