/// <summary>
/// Dumping Agent Azure VM extension logs to the support files folder.
/// </summary>
/// <param name="executionContext">Execution context to write debug messages.</param>
/// <param name="supportFilesFolder">Destination folder for files to be dumped.</param>
/// <param name="jobStartTimeUtc">Date and time to create timestamp.</param>
/// <returns>true, if logs have been dumped successfully; otherwise returns false.</returns>
private bool DumpAgentExtensionLogs(IExecutionContext executionContext, string supportFilesFolder, DateTime jobStartTimeUtc)
{
string pathToLogs = String.Empty;
string archiveName = String.Empty;
string timestamp = jobStartTimeUtc.ToString("yyyyMMdd-HHmmss");
if (PlatformUtil.RunningOnWindows)
{
// the extension creates a subfolder with a version number on Windows, and we're taking the latest one
string pathToExtensionVersions = ExtensionPaths.WindowsPathToExtensionVersions;
if (!Directory.Exists(pathToExtensionVersions))
{
executionContext.Debug("Path to subfolders with Agent Azure VM Windows extension logs (of its different versions) does not exist.");
executionContext.Debug($"(directory \"{pathToExtensionVersions}\" not found)");
return(false);
}
string[] subDirs = Directory.GetDirectories(pathToExtensionVersions).Select(dir => Path.GetFileName(dir)).ToArray();
if (subDirs.Length == 0)
{
executionContext.Debug("Path to Agent Azure VM Windows extension logs (of its different versions) does not contain subfolders.");
executionContext.Debug($"(directory \"{pathToExtensionVersions}\" does not contain subdirectories with logs)");
return(false);
}
Version[] versions = subDirs.Select(dir => new Version(dir)).ToArray();
Version maxVersion = versions.Max();
pathToLogs = Path.Combine(pathToExtensionVersions, maxVersion.ToString());
archiveName = $"AgentWindowsExtensionLogs-{timestamp}-utc.zip";
}
else if (PlatformUtil.RunningOnLinux)
{
// the extension does not create a subfolder with a version number on Linux, and we're just taking this folder
pathToLogs = ExtensionPaths.LinuxPathToExtensionLogs;
if (!Directory.Exists(pathToLogs))
{
executionContext.Debug("Path to Agent Azure VM Linux extension logs does not exist.");
executionContext.Debug($"(directory \"{pathToLogs}\" not found)");
return(false);
}
archiveName = $"AgentLinuxExtensionLogs-{timestamp}-utc.zip";
}
else
{
executionContext.Debug("Dumping Agent Azure VM extension logs implemented for Windows and Linux only.");
return(false);
}
executionContext.Debug($"Path to agent extension logs: {pathToLogs}");
string archivePath = Path.Combine(HostContext.GetDiagDirectory(), archiveName);
executionContext.Debug($"Archiving agent extension logs to: {archivePath}");
ZipFile.CreateFromDirectory(pathToLogs, archivePath);
string copyPath = Path.Combine(supportFilesFolder, archiveName);
executionContext.Debug($"Copying archived agent extension logs to: {copyPath}");
File.Copy(archivePath, copyPath);
return(true);
}
private string GenerateUpdateScript(bool restartInteractiveAgent)
{
int processId = Process.GetCurrentProcess().Id;
string updateLog = Path.Combine(HostContext.GetDiagDirectory(), $"SelfUpdate-{DateTime.UtcNow.ToString("yyyyMMdd-HHmmss")}.log");
string agentRoot = HostContext.GetDirectory(WellKnownDirectory.Root);
string templateName = "update.sh.template";
if (PlatformUtil.RunningOnWindows)
{
templateName = "update.cmd.template";
}
string templatePath = Path.Combine(agentRoot, $"bin.{_targetPackage.Version}", templateName);
string template = File.ReadAllText(templatePath);
template = template.Replace("_PROCESS_ID_", processId.ToString());
template = template.Replace("_AGENT_PROCESS_NAME_", $"Agent.Listener{IOUtil.ExeExtension}");
template = template.Replace("_ROOT_FOLDER_", agentRoot);
template = template.Replace("_EXIST_AGENT_VERSION_", BuildConstants.AgentPackage.Version);
template = template.Replace("_DOWNLOAD_AGENT_VERSION_", _targetPackage.Version);
template = template.Replace("_UPDATE_LOG_", updateLog);
template = template.Replace("_RESTART_INTERACTIVE_AGENT_", restartInteractiveAgent ? "1" : "0");
string scriptName = "_update.sh";
if (PlatformUtil.RunningOnWindows)
{
scriptName = "_update.cmd";
}
string updateScript = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), scriptName);
if (File.Exists(updateScript))
{
IOUtil.DeleteFile(updateScript);
}
File.WriteAllText(updateScript, template);
return(updateScript);
}
public void LogFileChangedAccordingToEnvVariable()
{
try
{
var newPath = Path.Combine(Path.GetDirectoryName(Assembly.GetEntryAssembly().Location), "logs");
Environment.SetEnvironmentVariable("AGENT_DIAGLOGPATH", newPath);
using (var _hc = new HostContext(HostType.Agent))
{
// Act.
var diagFolder = _hc.GetDiagDirectory();
// Assert
Assert.Equal(Path.Combine(newPath, Constants.Path.DiagDirectory), diagFolder);
Directory.Exists(diagFolder);
}
}
finally
{
Environment.SetEnvironmentVariable("AGENT_DIAGLOGPATH", null);
}
}
public async Task UploadDiagnosticLogsAsync(IExecutionContext executionContext,
Pipelines.AgentJobRequestMessage message,
DateTime jobStartTimeUtc)
{
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(message, nameof(message));
executionContext.Debug("Starting diagnostic file upload.");
// Setup folders
// \_layout\_work\_temp\[jobname-support]
executionContext.Debug("Setting up diagnostic log folders.");
string tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp);
ArgUtil.Directory(tempDirectory, nameof(tempDirectory));
string supportRootFolder = Path.Combine(tempDirectory, message.JobName + "-support");
Directory.CreateDirectory(supportRootFolder);
// \_layout\_work\_temp\[jobname-support]\files
executionContext.Debug("Creating diagnostic log files folder.");
string supportFilesFolder = Path.Combine(supportRootFolder, "files");
Directory.CreateDirectory(supportFilesFolder);
// Create the environment file
// \_layout\_work\_temp\[jobname-support]\files\environment.txt
var configurationStore = HostContext.GetService <IConfigurationStore>();
AgentSettings settings = configurationStore.GetSettings();
int agentId = settings.AgentId;
string agentName = settings.AgentName;
int poolId = settings.PoolId;
executionContext.Debug("Creating diagnostic log environment file.");
string environmentFile = Path.Combine(supportFilesFolder, "environment.txt");
string content = await GetEnvironmentContent(agentId, agentName, message.Steps);
File.WriteAllText(environmentFile, content);
// Create the capabilities file
var capabilitiesManager = HostContext.GetService <ICapabilitiesManager>();
Dictionary <string, string> capabilities = await capabilitiesManager.GetCapabilitiesAsync(configurationStore.GetSettings(), default(CancellationToken));
executionContext.Debug("Creating capabilities file.");
string capabilitiesFile = Path.Combine(supportFilesFolder, "capabilities.txt");
string capabilitiesContent = GetCapabilitiesContent(capabilities);
File.WriteAllText(capabilitiesFile, capabilitiesContent);
// Copy worker diag log files
List <string> workerDiagLogFiles = GetWorkerDiagLogFiles(HostContext.GetDiagDirectory(), jobStartTimeUtc);
executionContext.Debug($"Copying {workerDiagLogFiles.Count()} worker diag logs from {HostContext.GetDiagDirectory()}.");
foreach (string workerLogFile in workerDiagLogFiles)
{
ArgUtil.File(workerLogFile, nameof(workerLogFile));
string destination = Path.Combine(supportFilesFolder, Path.GetFileName(workerLogFile));
File.Copy(workerLogFile, destination);
}
// Copy agent diag log files - we are using the worker Host Context and we need the diag folder form the Agent.
List <string> agentDiagLogFiles = GetAgentDiagLogFiles(HostContext.GetDiagDirectory(HostType.Agent), jobStartTimeUtc);
executionContext.Debug($"Copying {agentDiagLogFiles.Count()} agent diag logs from {HostContext.GetDiagDirectory(HostType.Agent)}.");
foreach (string agentLogFile in agentDiagLogFiles)
{
ArgUtil.File(agentLogFile, nameof(agentLogFile));
string destination = Path.Combine(supportFilesFolder, Path.GetFileName(agentLogFile));
File.Copy(agentLogFile, destination);
}
// Read and add to logs waagent.conf settings on Linux
if (PlatformUtil.RunningOnLinux)
{
executionContext.Debug("Dumping of waagent.conf file");
string waagentDumpFile = Path.Combine(supportFilesFolder, "waagentConf.txt");
string configFileName = "waagent.conf";
try
{
string filePath = Directory.GetFiles("/etc", configFileName).FirstOrDefault();
if (!string.IsNullOrWhiteSpace(filePath))
{
string waagentContent = File.ReadAllText(filePath);
File.AppendAllText(waagentDumpFile, "waagent.conf settings");
File.AppendAllText(waagentDumpFile, Environment.NewLine);
File.AppendAllText(waagentDumpFile, waagentContent);
executionContext.Debug("Dumping waagent.conf file is completed.");
}
else
{
executionContext.Debug("waagent.conf file wasn't found. Dumping was not done.");
}
}
catch (Exception ex)
{
string warningMessage = $"Dumping of waagent.conf was not completed successfully. Error message: {ex.Message}";
executionContext.Warning(warningMessage);
}
}
// Copy cloud-init log files from linux machines
if (PlatformUtil.RunningOnLinux)
{
executionContext.Debug("Dumping cloud-init logs.");
string logsFilePath = $"{HostContext.GetDiagDirectory()}/cloudinit-{jobStartTimeUtc.ToString("yyyyMMdd-HHmmss")}-logs.tar.gz";
string resultLogs = await DumpCloudInitLogs(logsFilePath);
executionContext.Debug(resultLogs);
if (File.Exists(logsFilePath))
{
string destination = Path.Combine(supportFilesFolder, Path.GetFileName(logsFilePath));
File.Copy(logsFilePath, destination);
executionContext.Debug("Cloud-init logs added to the diagnostics archive.");
}
else
{
executionContext.Debug("Cloud-init logs were not found.");
}
executionContext.Debug("Dumping cloud-init logs is ended.");
}
// Copy event logs for windows machines
bool dumpJobEventLogs = AgentKnobs.DumpJobEventLogs.GetValue(executionContext).AsBoolean();
if (dumpJobEventLogs && PlatformUtil.RunningOnWindows)
{
executionContext.Debug("Dumping event viewer logs for current job.");
try
{
string eventLogsFile = $"{HostContext.GetDiagDirectory()}/EventViewer-{ jobStartTimeUtc.ToString("yyyyMMdd-HHmmss") }.csv";
await DumpCurrentJobEventLogs(executionContext, eventLogsFile, jobStartTimeUtc);
string destination = Path.Combine(supportFilesFolder, Path.GetFileName(eventLogsFile));
File.Copy(eventLogsFile, destination);
}
catch (Exception ex)
{
executionContext.Debug("Failed to dump event viewer logs. Skipping.");
executionContext.Debug($"Error message: {ex}");
}
}
bool dumpPackagesVerificationResult = AgentKnobs.DumpPackagesVerificationResult.GetValue(executionContext).AsBoolean();
if (dumpPackagesVerificationResult && PlatformUtil.RunningOnLinux && !PlatformUtil.RunningOnRHEL6)
{
executionContext.Debug("Dumping info about invalid MD5 sums of installed packages.");
var debsums = WhichUtil.Which("debsums");
if (debsums == null)
{
executionContext.Debug("Debsums is not installed on the system. Skipping broken packages check.");
}
else
{
try
{
string packageVerificationResults = await GetPackageVerificationResult(debsums);
IEnumerable <string> brokenPackagesInfo = packageVerificationResults
.Split("\n")
.Where((line) => !String.IsNullOrEmpty(line) && !line.EndsWith("OK"));
string brokenPackagesLogsPath = $"{HostContext.GetDiagDirectory()}/BrokenPackages-{ jobStartTimeUtc.ToString("yyyyMMdd-HHmmss") }.log";
File.AppendAllLines(brokenPackagesLogsPath, brokenPackagesInfo);
string destination = Path.Combine(supportFilesFolder, Path.GetFileName(brokenPackagesLogsPath));
File.Copy(brokenPackagesLogsPath, destination);
}
catch (Exception ex)
{
executionContext.Debug("Failed to dump broken packages logs. Skipping.");
executionContext.Debug($"Error message: {ex}");
}
}
}
else
{
executionContext.Debug("The platform is not based on Debian - skipping debsums check.");
}
try
{
executionContext.Debug("Starting dumping Agent Azure VM extension logs.");
bool logsSuccessfullyDumped = DumpAgentExtensionLogs(executionContext, supportFilesFolder, jobStartTimeUtc);
if (logsSuccessfullyDumped)
{
executionContext.Debug("Agent Azure VM extension logs successfully dumped.");
}
else
{
executionContext.Debug("Agent Azure VM extension logs not found. Skipping.");
}
}
catch (Exception ex)
{
executionContext.Debug("Failed to dump Agent Azure VM extension logs. Skipping.");
executionContext.Debug($"Error message: {ex}");
}
executionContext.Debug("Zipping diagnostic files.");
string buildNumber = executionContext.Variables.Build_Number ?? "UnknownBuildNumber";
string buildName = $"Build {buildNumber}";
string phaseName = executionContext.Variables.System_PhaseDisplayName ?? "UnknownPhaseName";
// zip the files
string diagnosticsZipFileName = $"{buildName}-{phaseName}.zip";
string diagnosticsZipFilePath = Path.Combine(supportRootFolder, diagnosticsZipFileName);
ZipFile.CreateFromDirectory(supportFilesFolder, diagnosticsZipFilePath);
// upload the json metadata file
executionContext.Debug("Uploading diagnostic metadata file.");
string metadataFileName = $"diagnostics-{buildName}-{phaseName}.json";
string metadataFilePath = Path.Combine(supportFilesFolder, metadataFileName);
string phaseResult = GetTaskResultAsString(executionContext.Result);
IOUtil.SaveObject(new DiagnosticLogMetadata(agentName, agentId, poolId, phaseName, diagnosticsZipFileName, phaseResult), metadataFilePath);
executionContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: metadataFileName, filePath: metadataFilePath);
executionContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: diagnosticsZipFileName, filePath: diagnosticsZipFilePath);
executionContext.Debug("Diagnostic file upload complete.");
}