public TrackingConfig Create(
IExecutionContext executionContext,
ServiceEndpoint endpoint,
string hashKey,
string file)
{
Trace.Entering();
// Get or create the top-level tracking config.
TopLevelTrackingConfig topLevelConfig;
string topLevelFile = Path.Combine(
IOUtil.GetWorkPath(HostContext),
Constants.Build.Path.SourceRootMappingDirectory,
Constants.Build.Path.TopLevelTrackingConfigFile);
Trace.Verbose($"Loading top-level tracking config if exists: {topLevelFile}");
if (!File.Exists(topLevelFile))
{
topLevelConfig = new TopLevelTrackingConfig();
}
else
{
topLevelConfig = JsonConvert.DeserializeObject<TopLevelTrackingConfig>(
value: File.ReadAllText(topLevelFile));
}
// Update the top-level tracking config.
topLevelConfig.LastBuildDirectoryCreatedOn = DateTimeOffset.Now;
topLevelConfig.LastBuildDirectoryNumber++;
WriteToFile(topLevelFile, topLevelConfig);
// Create the new tracking config.
TrackingConfig config = new TrackingConfig(
executionContext,
endpoint,
topLevelConfig.LastBuildDirectoryNumber,
hashKey);
WriteToFile(file, config);
return config;
}
public void MaintenanceCompleted(TrackingConfig config, string file)
{
Trace.Entering();
config.LastMaintenanceCompletedOn = DateTimeOffset.Now;
WriteToFile(file, config);
}
private async Task PrepareAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(PrepareStep, nameof(PrepareStep));
ArgUtil.NotNull(PrepareStep.ExecutionContext, nameof(PrepareStep.ExecutionContext));
IExecutionContext executionContext = PrepareStep.ExecutionContext;
var directoryManager = HostContext.GetService <IBuildDirectoryManager>();
// This flag can be false for jobs like cleanup artifacts.
// If syncSources = false, we will not set source related build variable, not create build folder, not sync source.
bool syncSources = executionContext.Variables.Build_SyncSources ?? true;
if (!syncSources)
{
Trace.Info($"{Constants.Variables.Build.SyncSources} = false, we will not set source related build variable, not create build folder and not sync source");
return;
}
// Get the repo endpoint and source provider.
if (!TrySetPrimaryEndpointAndProviderInfo(executionContext))
{
throw new Exception(StringUtil.Loc("SupportedRepositoryEndpointNotFound"));
}
executionContext.Debug($"Primary repository: {SourceEndpoint.Name}. repository type: {SourceProvider.RepositoryType}");
// Set the repo variables.
string repositoryId;
if (SourceEndpoint.Data.TryGetValue("repositoryId", out repositoryId)) // TODO: Move to const after source artifacts PR is merged.
{
executionContext.Variables.Set(Constants.Variables.Build.RepoId, repositoryId);
}
executionContext.Variables.Set(Constants.Variables.Build.RepoName, SourceEndpoint.Name);
executionContext.Variables.Set(Constants.Variables.Build.RepoProvider, SourceEndpoint.Type);
executionContext.Variables.Set(Constants.Variables.Build.RepoUri, SourceEndpoint.Url?.AbsoluteUri);
string checkoutSubmoduleText;
if (SourceEndpoint.Data.TryGetValue(WellKnownEndpointData.CheckoutSubmodules, out checkoutSubmoduleText))
{
executionContext.Variables.Set(Constants.Variables.Build.RepoGitSubmoduleCheckout, checkoutSubmoduleText);
}
// overwrite primary repository's clean value if build.repository.clean is sent from server. this is used by tfvc gated check-in
bool?repoClean = executionContext.Variables.GetBoolean(Constants.Variables.Build.RepoClean);
if (repoClean != null)
{
SourceEndpoint.Data[WellKnownEndpointData.Clean] = repoClean.Value.ToString();
}
else
{
string cleanRepoText;
if (SourceEndpoint.Data.TryGetValue(WellKnownEndpointData.Clean, out cleanRepoText))
{
executionContext.Variables.Set(Constants.Variables.Build.RepoClean, cleanRepoText);
}
}
// Prepare the build directory.
executionContext.Debug("Preparing build directory.");
TrackingConfig trackingConfig = directoryManager.PrepareDirectory(
executionContext,
SourceEndpoint,
SourceProvider);
// Set the directory variables.
executionContext.Debug("Set build variables.");
string _workDirectory = IOUtil.GetWorkPath(HostContext);
executionContext.Variables.Set(Constants.Variables.Agent.BuildDirectory, Path.Combine(_workDirectory, trackingConfig.BuildDirectory));
executionContext.Variables.Set(Constants.Variables.System.ArtifactsDirectory, Path.Combine(_workDirectory, trackingConfig.ArtifactsDirectory));
executionContext.Variables.Set(Constants.Variables.System.DefaultWorkingDirectory, Path.Combine(_workDirectory, trackingConfig.SourcesDirectory));
executionContext.Variables.Set(Constants.Variables.Common.TestResultsDirectory, Path.Combine(_workDirectory, trackingConfig.TestResultsDirectory));
executionContext.Variables.Set(Constants.Variables.Build.BinariesDirectory, Path.Combine(_workDirectory, trackingConfig.BuildDirectory, Constants.Build.Path.BinariesDirectory));
executionContext.Variables.Set(Constants.Variables.Build.SourcesDirectory, Path.Combine(_workDirectory, trackingConfig.SourcesDirectory));
executionContext.Variables.Set(Constants.Variables.Build.StagingDirectory, Path.Combine(_workDirectory, trackingConfig.ArtifactsDirectory));
executionContext.Variables.Set(Constants.Variables.Build.ArtifactStagingDirectory, Path.Combine(_workDirectory, trackingConfig.ArtifactsDirectory));
executionContext.Variables.Set(Constants.Variables.Build.RepoLocalPath, Path.Combine(_workDirectory, trackingConfig.SourcesDirectory));
SourceProvider.SetVariablesInEndpoint(executionContext, SourceEndpoint);
// Read skipSyncSource property fron endpoint data
string skipSyncSourceText;
bool skipSyncSource = false;
if (SourceEndpoint.Data.TryGetValue("skipSyncSource", out skipSyncSourceText))
{
skipSyncSource = StringUtil.ConvertToBoolean(skipSyncSourceText, false);
}
// Prefer feature variable over endpoint data
skipSyncSource = executionContext.Variables.GetBoolean(Constants.Variables.Features.SkipSyncSource) ?? skipSyncSource;
if (skipSyncSource)
{
executionContext.Output($"Skip sync source for endpoint: {SourceEndpoint.Name}");
}
else
{
executionContext.Debug($"Sync source for endpoint: {SourceEndpoint.Name}");
await SourceProvider.GetSourceAsync(executionContext, SourceEndpoint, executionContext.CancellationToken);
}
}
private async Task RunSourceProviderMaintenance(IExecutionContext executionContext, ITrackingManager trackingManager, TrackingConfig trackingConfig)
{
var extensionManager = HostContext.GetService <IExtensionManager>();
ISourceProvider sourceProvider = extensionManager.GetExtensions <ISourceProvider>().FirstOrDefault(x => string.Equals(x.RepositoryType, trackingConfig.RepositoryType, StringComparison.OrdinalIgnoreCase));
if (sourceProvider != null)
{
try
{
trackingManager.MaintenanceStarted(trackingConfig);
string repositoryPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), trackingConfig.SourcesDirectory);
await sourceProvider.RunMaintenanceOperations(executionContext, repositoryPath);
trackingManager.MaintenanceCompleted(trackingConfig);
}
catch (Exception ex)
{
executionContext.Error(StringUtil.Loc("ErrorDuringBuildGC", trackingConfig.FileLocation));
executionContext.Error(ex);
}
}
}
private TrackingConfig LoadIfExists(
IExecutionContext executionContext,
string file)
{
Trace.Entering();
Trace.Verbose($"Loading {file}");
// The tracking config will not exist for a new definition.
if (!File.Exists(file))
{
Trace.Verbose($"Tracking file does not exist: {file}");
return null;
}
TrackingConfig result = null;
// Load the content and distinguish between tracking config file
// version 1 and file version 2.
string content = File.ReadAllText(file);
string fileFormatVersionJsonProperty = StringUtil.Format(
@"""{0}""",
TrackingConfig.FileFormatVersionJsonProperty);
if (content.Contains(fileFormatVersionJsonProperty))
{
// The config is the new format.
Trace.Verbose("Parsing new tracking config format.");
result = JsonConvert.DeserializeObject<TrackingConfig>(content);
if (result != null)
{
// if RepositoryTrackingInfo is empty, then we should create an entry so the rest
// of the logic after this will act correctly
if (result.RepositoryTrackingInfo.Count == 0)
{
result.RepositoryTrackingInfo.Add(new Build.RepositoryTrackingInfo
{
Identifier = RepositoryUtil.DefaultPrimaryRepositoryName,
RepositoryType = result.RepositoryType,
RepositoryUrl = result.RepositoryUrl,
SourcesDirectory = result.SourcesDirectory,
});
}
}
}
else
{
// Attempt to parse the legacy format.
Trace.Verbose("Parsing legacy tracking config format.");
var legacyTrackingConfig = LegacyTrackingConfig.TryParse(content);
if (legacyTrackingConfig == null)
{
executionContext.Warning(StringUtil.Loc("UnableToParseBuildTrackingConfig0", content));
}
else
{
// Convert legacy format to the new format.
result = ConvertToNewFormat(
executionContext,
legacyTrackingConfig,
RepositoryUtil.GetCloneDirectory(legacyTrackingConfig.RepositoryUrl),
RepositoryUtil.GuessRepositoryType(legacyTrackingConfig.RepositoryUrl));
}
}
if (result != null)
{
result.FileLocation = file;
}
return result;
}
public TrackingConfig Create(
IExecutionContext executionContext,
ServiceEndpoint endpoint,
string hashKey,
string file,
bool overrideBuildDirectory)
{
Trace.Entering();
// Get or create the top-level tracking config.
TopLevelTrackingConfig topLevelConfig;
string topLevelFile = Path.Combine(
IOUtil.GetWorkPath(HostContext),
Constants.Build.Path.SourceRootMappingDirectory,
Constants.Build.Path.TopLevelTrackingConfigFile);
Trace.Verbose($"Loading top-level tracking config if exists: {topLevelFile}");
if (!File.Exists(topLevelFile))
{
topLevelConfig = new TopLevelTrackingConfig();
}
else
{
topLevelConfig = JsonConvert.DeserializeObject <TopLevelTrackingConfig>(
value: File.ReadAllText(topLevelFile));
}
// Determine the build directory.
if (overrideBuildDirectory)
{
// This should only occur during hosted builds. This was added due to TFVC.
// TFVC does not allow a local path for a single machine to be mapped in multiple
// workspaces. The machine name for a hosted images is not unique.
//
// So if a customer is running two hosted builds at the same time, they could run
// into the local mapping conflict.
//
// The workaround is to force the build directory to be different across all concurrent
// hosted builds (for TFVC). The agent ID will be unique across all concurrent hosted
// builds so that can safely be used as the build directory.
ArgUtil.Equal(default(int), topLevelConfig.LastBuildDirectoryNumber, nameof(topLevelConfig.LastBuildDirectoryNumber));
var configurationStore = HostContext.GetService <IConfigurationStore>();
AgentSettings settings = configurationStore.GetSettings();
topLevelConfig.LastBuildDirectoryNumber = settings.AgentId;
}
else
{
topLevelConfig.LastBuildDirectoryNumber++;
}
// Update the top-level tracking config.
topLevelConfig.LastBuildDirectoryCreatedOn = DateTimeOffset.Now;
WriteToFile(topLevelFile, topLevelConfig);
// Create the new tracking config.
TrackingConfig config = new TrackingConfig(
executionContext,
endpoint,
topLevelConfig.LastBuildDirectoryNumber,
hashKey);
WriteToFile(file, config);
return(config);
}
public void MaintenanceCompleted(TrackingConfig config)
{
Trace.Entering();
config.LastMaintenanceCompletedOn = DateTimeOffset.Now;
WriteToFile(config.FileLocation, config);
}
public void MarkExpiredForGarbageCollection(IExecutionContext executionContext, TimeSpan expiration)
{
Trace.Entering();
Trace.Info("Scan all SourceFolder tracking files.");
string searchRoot = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Build.Path.SourceRootMappingDirectory);
if (!Directory.Exists(searchRoot))
{
executionContext.Output(StringUtil.Loc("GCDirNotExist", searchRoot));
return;
}
var allTrackingFiles = Directory.EnumerateFiles(searchRoot, Constants.Build.Path.TrackingConfigFile, SearchOption.AllDirectories);
Trace.Verbose($"Find {allTrackingFiles.Count()} tracking files.");
executionContext.Output(StringUtil.Loc("DirExpireLimit", expiration.TotalDays));
executionContext.Output(StringUtil.Loc("CurrentUTC", DateTime.UtcNow.ToString("o")));
// scan all sourcefolder tracking file, find which folder has never been used since UTC-expiration
// the scan and garbage discovery should be best effort.
// if the tracking file is in old format, just delete the folder since the first time the folder been use we will convert the tracking file to new format.
foreach (var trackingFile in allTrackingFiles)
{
try
{
executionContext.Output(StringUtil.Loc("EvaluateTrackingFile", trackingFile));
TrackingConfigBase tracking = LoadIfExists(executionContext, trackingFile);
// detect whether the tracking file is in new format.
TrackingConfig newTracking = tracking as TrackingConfig;
if (newTracking == null)
{
LegacyTrackingConfig legacyConfig = tracking as LegacyTrackingConfig;
ArgUtil.NotNull(legacyConfig, nameof(LegacyTrackingConfig));
Trace.Verbose($"{trackingFile} is a old format tracking file.");
executionContext.Output(StringUtil.Loc("GCOldFormatTrackingFile", trackingFile));
MarkForGarbageCollection(executionContext, legacyConfig);
IOUtil.DeleteFile(trackingFile);
}
else
{
Trace.Verbose($"{trackingFile} is a new format tracking file.");
ArgUtil.NotNull(newTracking.LastRunOn, nameof(newTracking.LastRunOn));
executionContext.Output(StringUtil.Loc("BuildDirLastUseTIme", Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newTracking.BuildDirectory), newTracking.LastRunOnString));
if (DateTime.UtcNow - expiration > newTracking.LastRunOn)
{
executionContext.Output(StringUtil.Loc("GCUnusedTrackingFile", trackingFile, expiration.TotalDays));
MarkForGarbageCollection(executionContext, newTracking);
IOUtil.DeleteFile(trackingFile);
}
}
}
catch (Exception ex)
{
executionContext.Error(StringUtil.Loc("ErrorDuringBuildGC", trackingFile));
executionContext.Error(ex);
}
}
}
private string GetDefaultRepoLocalPathValue(IExecutionContext executionContext, IList<Pipelines.JobStep> steps, TrackingConfig trackingConfig, RepositoryInfo repoInfo)
{
string selfRepoPath = null;
// For saving backward compatibility with the behavior of the Build.RepoLocalPath that was before this PR https://github.com/microsoft/azure-pipelines-agent/pull/3237
// We need to change how we set the default value of this variable
// We need to allow the setting of paths from RepositoryTrackingInfo for checkout tasks where path input was provided by the user
// and this input is not point to the default location for this repository
// This is the only case where the value of Build.RepoLocalPath variable is not pointing to the root of sources directory /s.
// The new logic is not affecting single checkout jobs and jobs with multiple checkouts and default paths for Self repository
if (RepositoryUtil.HasMultipleCheckouts(executionContext.JobSettings))
{
// get checkout task for self repo
var selfCheckoutTask = GetSelfCheckoutTask(steps);
// Check if the task has path input with custom path, if so we need to set as a value of selfRepoPath the value of SourcesDirectory from RepositoryTrackingInfo
if (IsCheckoutToCustomPath(trackingConfig, repoInfo, selfCheckoutTask))
{
selfRepoPath = trackingConfig.RepositoryTrackingInfo
.Where(repo => RepositoryUtil.IsPrimaryRepositoryName(repo.Identifier))
.Select(props => props.SourcesDirectory).FirstOrDefault();
}
}
// For single checkout jobs and multicheckout jobs with default paths set selfRepoPath to the default sources directory
if (selfRepoPath == null)
{
selfRepoPath = trackingConfig.SourcesDirectory;
}
return selfRepoPath;
}
// Prepare build directory
// Set all build related variables
public override void InitializeJobExtension(IExecutionContext executionContext, IList <Pipelines.JobStep> steps, Pipelines.WorkspaceOptions workspace)
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
// This flag can be false for jobs like cleanup artifacts.
// If syncSources = false, we will not set source related build variable, not create build folder, not sync source.
bool syncSources = executionContext.Variables.Build_SyncSources ?? true;
if (!syncSources)
{
Trace.Info($"{Constants.Variables.Build.SyncSources} = false, we will not set source related build variable, not create build folder and not sync source");
return;
}
// We only support checkout one repository at this time.
if (!TrySetPrimaryRepositoryAndProviderInfo(executionContext))
{
throw new Exception(StringUtil.Loc("SupportedRepositoryEndpointNotFound"));
}
executionContext.Debug($"Primary repository: {Repository.Properties.Get<string>(Pipelines.RepositoryPropertyNames.Name)}. repository type: {Repository.Type}");
// Set the repo variables.
if (!string.IsNullOrEmpty(Repository.Id)) // TODO: Move to const after source artifacts PR is merged.
{
executionContext.Variables.Set(Constants.Variables.Build.RepoId, Repository.Id);
}
executionContext.Variables.Set(Constants.Variables.Build.RepoName, Repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Name));
executionContext.Variables.Set(Constants.Variables.Build.RepoProvider, ConvertToLegacyRepositoryType(Repository.Type));
executionContext.Variables.Set(Constants.Variables.Build.RepoUri, Repository.Url?.AbsoluteUri);
var checkoutTask = steps.SingleOrDefault(x => x.IsCheckoutTask()) as Pipelines.TaskStep;
if (checkoutTask != null)
{
if (checkoutTask.Inputs.ContainsKey(Pipelines.PipelineConstants.CheckoutTaskInputs.Submodules))
{
executionContext.Variables.Set(Constants.Variables.Build.RepoGitSubmoduleCheckout, Boolean.TrueString);
}
else
{
executionContext.Variables.Set(Constants.Variables.Build.RepoGitSubmoduleCheckout, Boolean.FalseString);
}
// overwrite primary repository's clean value if build.repository.clean is sent from server. this is used by tfvc gated check-in
bool?repoClean = executionContext.Variables.GetBoolean(Constants.Variables.Build.RepoClean);
if (repoClean != null)
{
checkoutTask.Inputs[Pipelines.PipelineConstants.CheckoutTaskInputs.Clean] = repoClean.Value.ToString();
}
else
{
if (checkoutTask.Inputs.ContainsKey(Pipelines.PipelineConstants.CheckoutTaskInputs.Clean))
{
executionContext.Variables.Set(Constants.Variables.Build.RepoClean, checkoutTask.Inputs[Pipelines.PipelineConstants.CheckoutTaskInputs.Clean]);
}
else
{
executionContext.Variables.Set(Constants.Variables.Build.RepoClean, Boolean.FalseString);
}
}
}
// Prepare the build directory.
executionContext.Output(StringUtil.Loc("PrepareBuildDir"));
var directoryManager = HostContext.GetService <IBuildDirectoryManager>();
TrackingConfig trackingConfig = directoryManager.PrepareDirectory(
executionContext,
Repository,
workspace);
// Set the directory variables.
executionContext.Output(StringUtil.Loc("SetBuildVars"));
string _workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
executionContext.SetVariable(Constants.Variables.Agent.BuildDirectory, Path.Combine(_workDirectory, trackingConfig.BuildDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.System.ArtifactsDirectory, Path.Combine(_workDirectory, trackingConfig.ArtifactsDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.System.DefaultWorkingDirectory, Path.Combine(_workDirectory, trackingConfig.SourcesDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Common.TestResultsDirectory, Path.Combine(_workDirectory, trackingConfig.TestResultsDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Build.BinariesDirectory, Path.Combine(_workDirectory, trackingConfig.BuildDirectory, Constants.Build.Path.BinariesDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Build.SourcesDirectory, Path.Combine(_workDirectory, trackingConfig.SourcesDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Build.StagingDirectory, Path.Combine(_workDirectory, trackingConfig.ArtifactsDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Build.ArtifactStagingDirectory, Path.Combine(_workDirectory, trackingConfig.ArtifactsDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Build.RepoLocalPath, Path.Combine(_workDirectory, trackingConfig.SourcesDirectory), isFilePath: true);
Repository.Properties.Set <string>(Pipelines.RepositoryPropertyNames.Path, Path.Combine(_workDirectory, trackingConfig.SourcesDirectory));
}
// Prepare build directory
// Set all build related variables
public override void InitializeJobExtension(IExecutionContext executionContext, IList<Pipelines.JobStep> steps, Pipelines.WorkspaceOptions workspace)
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
// This flag can be false for jobs like cleanup artifacts.
// If syncSources = false, we will not set source related build variable, not create build folder, not sync source.
bool syncSources = executionContext.Variables.Build_SyncSources ?? true;
if (!syncSources)
{
Trace.Info($"{Constants.Variables.Build.SyncSources} = false, we will not set source related build variable, not create build folder and not sync source");
return;
}
// We set the variables based on the 'self' repository
if (!TryGetRepositoryInfo(executionContext, out RepositoryInfo repoInfo))
{
throw new Exception(StringUtil.Loc("SupportedRepositoryEndpointNotFound"));
}
executionContext.Debug($"Triggering repository: {repoInfo.TriggeringRepository.Properties.Get<string>(Pipelines.RepositoryPropertyNames.Name)}. repository type: {repoInfo.TriggeringRepository.Type}");
// Set the repo variables.
if (!string.IsNullOrEmpty(repoInfo.TriggeringRepository.Id)) // TODO: Move to const after source artifacts PR is merged.
{
executionContext.SetVariable(Constants.Variables.Build.RepoId, repoInfo.TriggeringRepository.Id);
}
executionContext.SetVariable(Constants.Variables.Build.RepoName, repoInfo.TriggeringRepository.Properties.Get<string>(Pipelines.RepositoryPropertyNames.Name));
executionContext.SetVariable(Constants.Variables.Build.RepoProvider, ConvertToLegacyRepositoryType(repoInfo.TriggeringRepository.Type));
executionContext.SetVariable(Constants.Variables.Build.RepoUri, repoInfo.TriggeringRepository.Url?.AbsoluteUri);
// Prepare the build directory.
executionContext.Output(StringUtil.Loc("PrepareBuildDir"));
var directoryManager = HostContext.GetService<IBuildDirectoryManager>();
TrackingConfig trackingConfig = directoryManager.PrepareDirectory(
executionContext,
executionContext.Repositories,
workspace);
string _workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
string pipelineWorkspaceDirectory = Path.Combine(_workDirectory, trackingConfig.BuildDirectory);
UpdateCheckoutTasksAndVariables(executionContext, steps, pipelineWorkspaceDirectory);
// Get default value for RepoLocalPath variable
string selfRepoPath = GetDefaultRepoLocalPathValue(executionContext, steps, trackingConfig, repoInfo);
// Set the directory variables.
executionContext.Output(StringUtil.Loc("SetBuildVars"));
executionContext.SetVariable(Constants.Variables.Agent.BuildDirectory, pipelineWorkspaceDirectory, isFilePath: true);
executionContext.SetVariable(Constants.Variables.System.ArtifactsDirectory, Path.Combine(_workDirectory, trackingConfig.ArtifactsDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.System.DefaultWorkingDirectory, Path.Combine(_workDirectory, trackingConfig.SourcesDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Common.TestResultsDirectory, Path.Combine(_workDirectory, trackingConfig.TestResultsDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Build.BinariesDirectory, Path.Combine(_workDirectory, trackingConfig.BuildDirectory, Constants.Build.Path.BinariesDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Build.SourcesDirectory, Path.Combine(_workDirectory, trackingConfig.SourcesDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Build.StagingDirectory, Path.Combine(_workDirectory, trackingConfig.ArtifactsDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Build.ArtifactStagingDirectory, Path.Combine(_workDirectory, trackingConfig.ArtifactsDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Build.RepoLocalPath, Path.Combine(_workDirectory, selfRepoPath), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Pipeline.Workspace, pipelineWorkspaceDirectory, isFilePath: true);
}
private TrackingConfig ConvertToNewFormat(
IExecutionContext executionContext,
ServiceEndpoint endpoint,
TrackingConfigBase config)
{
Trace.Entering();
// If it's already in the new format, return it.
TrackingConfig newConfig = config as TrackingConfig;
if (newConfig != null)
{
return newConfig;
}
// Delete the legacy artifact/staging directories.
LegacyTrackingConfig legacyConfig = config as LegacyTrackingConfig;
DeleteDirectory(
executionContext,
description: "legacy artifacts directory",
path: Path.Combine(legacyConfig.BuildDirectory, Constants.Build.Path.LegacyArtifactsDirectory));
DeleteDirectory(
executionContext,
description: "legacy staging directory",
path: Path.Combine(legacyConfig.BuildDirectory, Constants.Build.Path.LegacyStagingDirectory));
// Determine the source directory name. Check if the directory is named "s" already.
// Convert the source directory to be named "s" if there is a problem with the old name.
string sourcesDirectoryNameOnly = Constants.Build.Path.SourcesDirectory;
if (!Directory.Exists(Path.Combine(legacyConfig.BuildDirectory, sourcesDirectoryNameOnly))
&& !String.Equals(endpoint.Name, Constants.Build.Path.ArtifactsDirectory, StringComparison.OrdinalIgnoreCase)
&& !String.Equals(endpoint.Name, Constants.Build.Path.LegacyArtifactsDirectory, StringComparison.OrdinalIgnoreCase)
&& !String.Equals(endpoint.Name, Constants.Build.Path.LegacyStagingDirectory, StringComparison.OrdinalIgnoreCase)
&& !String.Equals(endpoint.Name, Constants.Build.Path.TestResultsDirectory, StringComparison.OrdinalIgnoreCase)
&& !endpoint.Name.Contains("\\")
&& !endpoint.Name.Contains("/")
&& Directory.Exists(Path.Combine(legacyConfig.BuildDirectory, endpoint.Name)))
{
sourcesDirectoryNameOnly = endpoint.Name;
}
// Convert to the new format.
newConfig = new TrackingConfig(
executionContext,
legacyConfig,
sourcesDirectoryNameOnly,
// The legacy artifacts directory has been deleted at this point - see above - so
// switch the configuration to using the new naming scheme.
useNewArtifactsDirectoryName: true);
return newConfig;
}
// TODO: Updates legacy config.
private TestHostContext Setup(
[CallerMemberName] string name = "",
BuildCleanOption? cleanOption = null,
ExistingConfigKind existingConfigKind = ExistingConfigKind.None)
{
// Setup the host context.
TestHostContext hc = new TestHostContext(this, name);
// Create a random work path.
var configStore = new Mock<IConfigurationStore>();
_workFolder = Path.Combine(
Path.GetDirectoryName(Assembly.GetEntryAssembly().Location),
$"_work_{Path.GetRandomFileName()}");
var settings = new AgentSettings() { WorkFolder = _workFolder };
configStore.Setup(x => x.GetSettings()).Returns(settings);
hc.SetSingleton<IConfigurationStore>(configStore.Object);
// Setup the execution context.
_ec = new Mock<IExecutionContext>();
List<string> warnings;
_variables = new Variables(hc, new Dictionary<string, string>(), new List<MaskHint>(), out warnings);
_variables.Set(Constants.Variables.System.CollectionId, CollectionId);
_variables.Set(Constants.Variables.System.DefinitionId, DefinitionId);
_variables.Set(Constants.Variables.Build.Clean, $"{cleanOption}");
_ec.Setup(x => x.Variables).Returns(_variables);
// Store the expected tracking file path.
_trackingFile = Path.Combine(
_workFolder,
Constants.Build.Path.SourceRootMappingDirectory,
_ec.Object.Variables.System_CollectionId,
_ec.Object.Variables.System_DefinitionId,
Constants.Build.Path.TrackingConfigFile);
// Setup the endpoint.
_endpoint = new ServiceEndpoint()
{
Name = "Some endpoint name",
Url = new Uri("http://contoso.visualstudio.com"),
};
// Setup the source provider.
_sourceProvider = new Mock<ISourceProvider>();
_sourceProvider
.Setup(x => x.GetBuildDirectoryHashKey(_ec.Object, _endpoint))
.Returns(HashKey);
hc.SetSingleton<ISourceProvider>(_sourceProvider.Object);
// Store the existing config object.
switch (existingConfigKind)
{
case ExistingConfigKind.Matching:
_existingConfig = new TrackingConfig(_ec.Object, _endpoint, 1, HashKey);
Assert.Equal("1", _existingConfig.BuildDirectory);
break;
case ExistingConfigKind.Nonmatching:
_existingConfig = new TrackingConfig(_ec.Object, _endpoint, 2, NonmatchingHashKey);
Assert.Equal("2", _existingConfig.BuildDirectory);
break;
case ExistingConfigKind.None:
break;
default:
throw new NotSupportedException();
}
// Store the new config object.
if (existingConfigKind == ExistingConfigKind.Matching)
{
_newConfig = _existingConfig;
}
else
{
_newConfig = new TrackingConfig(_ec.Object, _endpoint, 3, HashKey);
Assert.Equal("3", _newConfig.BuildDirectory);
}
// Setup the tracking manager.
_trackingManager = new Mock<ITrackingManager>();
_trackingManager
.Setup(x => x.LoadIfExists(_ec.Object, _trackingFile))
.Returns(_existingConfig);
if (existingConfigKind == ExistingConfigKind.None || existingConfigKind == ExistingConfigKind.Nonmatching)
{
_trackingManager
.Setup(x => x.Create(_ec.Object, _endpoint, HashKey, _trackingFile))
.Returns(_newConfig);
if (existingConfigKind == ExistingConfigKind.Nonmatching)
{
_trackingManager
.Setup(x => x.MarkForGarbageCollection(_ec.Object, _existingConfig));
}
}
else if (existingConfigKind == ExistingConfigKind.Matching)
{
_trackingManager
.Setup(x => x.UpdateJobRunProperties(_ec.Object, _existingConfig, _trackingFile));
}
else
{
throw new NotSupportedException();
}
hc.SetSingleton<ITrackingManager>(_trackingManager.Object);
// Setup the build directory manager.
_buildDirectoryManager = new BuildDirectoryManager();
_buildDirectoryManager.Initialize(hc);
return hc;
}
public TrackingConfig Create(
IExecutionContext executionContext,
ServiceEndpoint endpoint,
string hashKey,
string file,
bool overrideBuildDirectory)
{
Trace.Entering();
// Get or create the top-level tracking config.
TopLevelTrackingConfig topLevelConfig;
string topLevelFile = Path.Combine(
IOUtil.GetWorkPath(HostContext),
Constants.Build.Path.SourceRootMappingDirectory,
Constants.Build.Path.TopLevelTrackingConfigFile);
Trace.Verbose($"Loading top-level tracking config if exists: {topLevelFile}");
if (!File.Exists(topLevelFile))
{
topLevelConfig = new TopLevelTrackingConfig();
}
else
{
topLevelConfig = JsonConvert.DeserializeObject <TopLevelTrackingConfig>(File.ReadAllText(topLevelFile));
if (topLevelConfig == null)
{
executionContext.Warning($"Rebuild corruptted top-level tracking configure file {topLevelFile}.");
// save the corruptted file in case we need to investigate more.
File.Copy(topLevelFile, $"{topLevelFile}.corruptted", true);
topLevelConfig = new TopLevelTrackingConfig();
DirectoryInfo workDir = new DirectoryInfo(HostContext.GetDirectory(WellKnownDirectory.Work));
foreach (var dir in workDir.EnumerateDirectories())
{
// we scan the entire _work directory and find the directory with the highest integer number.
if (int.TryParse(dir.Name, NumberStyles.Integer, CultureInfo.InvariantCulture, out int lastBuildNumber) &&
lastBuildNumber > topLevelConfig.LastBuildDirectoryNumber)
{
topLevelConfig.LastBuildDirectoryNumber = lastBuildNumber;
}
}
}
}
// Determine the build directory.
if (overrideBuildDirectory)
{
// This should only occur during hosted builds. This was added due to TFVC.
// TFVC does not allow a local path for a single machine to be mapped in multiple
// workspaces. The machine name for a hosted images is not unique.
//
// So if a customer is running two hosted builds at the same time, they could run
// into the local mapping conflict.
//
// The workaround is to force the build directory to be different across all concurrent
// hosted builds (for TFVC). The agent ID will be unique across all concurrent hosted
// builds so that can safely be used as the build directory.
ArgUtil.Equal(default(int), topLevelConfig.LastBuildDirectoryNumber, nameof(topLevelConfig.LastBuildDirectoryNumber));
var configurationStore = HostContext.GetService <IConfigurationStore>();
AgentSettings settings = configurationStore.GetSettings();
topLevelConfig.LastBuildDirectoryNumber = settings.AgentId;
}
else
{
topLevelConfig.LastBuildDirectoryNumber++;
}
// Update the top-level tracking config.
topLevelConfig.LastBuildDirectoryCreatedOn = DateTimeOffset.Now;
WriteToFile(topLevelFile, topLevelConfig);
// Create the new tracking config.
TrackingConfig config = new TrackingConfig(
executionContext,
endpoint,
topLevelConfig.LastBuildDirectoryNumber,
hashKey);
WriteToFile(file, config);
return(config);
}
public void MarkForGarbageCollection(IExecutionContext executionContext, TrackingConfigBase config)
{
Trace.Entering();
// Convert legacy format to the new format.
LegacyTrackingConfig legacyConfig = config as LegacyTrackingConfig;
if (legacyConfig != null)
{
// Convert legacy format to the new format.
config = new TrackingConfig(
executionContext,
legacyConfig,
// The sources folder wasn't stored in the legacy format - only the
// build folder was stored. Since the hash key has changed, it is
// unknown what the source folder was named. Just set the folder name
// to "s" so the property isn't left blank.
sourcesDirectoryNameOnly: Constants.Build.Path.SourcesDirectory);
}
// Write a copy of the tracking config to the GC folder.
string gcDirectory = Path.Combine(
IOUtil.GetWorkPath(HostContext),
Constants.Build.Path.SourceRootMappingDirectory,
Constants.Build.Path.GarbageCollectionDirectory);
string file = Path.Combine(
gcDirectory,
StringUtil.Format("{0}.json", Guid.NewGuid()));
WriteToFile(file, config);
}
public async Task RunMaintenanceOperation(IExecutionContext executionContext)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
// this might be not accurate when the agent is configured for old TFS server
int totalAvailableTimeInMinutes = executionContext.Variables.GetInt("maintenance.jobtimeoutinminutes") ?? 60;
// start a timer to track how much time we used
Stopwatch totalTimeSpent = Stopwatch.StartNew();
var trackingManager = HostContext.GetService <ITrackingManager>();
int staleBuildDirThreshold = executionContext.Variables.GetInt("maintenance.deleteworkingdirectory.daysthreshold") ?? 0;
if (staleBuildDirThreshold > 0)
{
// scan unused build directories
executionContext.Output(StringUtil.Loc("DiscoverBuildDir", staleBuildDirThreshold));
trackingManager.MarkExpiredForGarbageCollection(executionContext, TimeSpan.FromDays(staleBuildDirThreshold));
}
else
{
executionContext.Output(StringUtil.Loc("GCBuildDirNotEnabled"));
return;
}
executionContext.Output(StringUtil.Loc("GCBuildDir"));
// delete unused build directories
trackingManager.DisposeCollectedGarbage(executionContext);
// give source provider a chance to run maintenance operation
Trace.Info("Scan all SourceFolder tracking files.");
string searchRoot = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Build.Path.SourceRootMappingDirectory);
if (!Directory.Exists(searchRoot))
{
executionContext.Output(StringUtil.Loc("GCDirNotExist", searchRoot));
return;
}
// <tracking config, tracking file path>
List <Tuple <TrackingConfig, string> > optimizeTrackingFiles = new List <Tuple <TrackingConfig, string> >();
var allTrackingFiles = Directory.EnumerateFiles(searchRoot, Constants.Build.Path.TrackingConfigFile, SearchOption.AllDirectories);
Trace.Verbose($"Find {allTrackingFiles.Count()} tracking files.");
foreach (var trackingFile in allTrackingFiles)
{
executionContext.Output(StringUtil.Loc("EvaluateTrackingFile", trackingFile));
TrackingConfigBase tracking = trackingManager.LoadIfExists(executionContext, trackingFile);
// detect whether the tracking file is in new format.
TrackingConfig newTracking = tracking as TrackingConfig;
if (newTracking == null)
{
executionContext.Output(StringUtil.Loc("GCOldFormatTrackingFile", trackingFile));
}
else if (string.IsNullOrEmpty(newTracking.RepositoryType))
{
// repository not been set.
executionContext.Output(StringUtil.Loc("SkipTrackingFileWithoutRepoType", trackingFile));
}
else
{
optimizeTrackingFiles.Add(new Tuple <TrackingConfig, string>(newTracking, trackingFile));
}
}
// Sort the all tracking file ASC by last maintenance attempted time
foreach (var trackingInfo in optimizeTrackingFiles.OrderBy(x => x.Item1.LastMaintenanceAttemptedOn))
{
// maintenance has been cancelled.
executionContext.CancellationToken.ThrowIfCancellationRequested();
bool runMainenance = false;
TrackingConfig trackingConfig = trackingInfo.Item1;
string trackingFile = trackingInfo.Item2;
if (trackingConfig.LastMaintenanceAttemptedOn == null)
{
// this folder never run maintenance before, we will do maintenance if there is more than half of the time remains.
if (totalTimeSpent.Elapsed.TotalMinutes < totalAvailableTimeInMinutes / 2) // 50% time left
{
runMainenance = true;
}
else
{
executionContext.Output($"Working directory '{trackingConfig.BuildDirectory}' has never run maintenance before. Skip since we may not have enough time.");
}
}
else if (trackingConfig.LastMaintenanceCompletedOn == null)
{
// this folder did finish maintenance last time, this might indicate we need more time for this working directory
if (totalTimeSpent.Elapsed.TotalMinutes < totalAvailableTimeInMinutes / 4) // 75% time left
{
runMainenance = true;
}
else
{
executionContext.Output($"Working directory '{trackingConfig.BuildDirectory}' didn't finish maintenance last time. Skip since we may not have enough time.");
}
}
else
{
// estimate time for running maintenance
TimeSpan estimateTime = trackingConfig.LastMaintenanceCompletedOn.Value - trackingConfig.LastMaintenanceAttemptedOn.Value;
// there is more than 10 mins left after we run maintenance on this repository directory
if (totalAvailableTimeInMinutes > totalTimeSpent.Elapsed.TotalMinutes + estimateTime.TotalMinutes + 10)
{
runMainenance = true;
}
else
{
executionContext.Output($"Working directory '{trackingConfig.BuildDirectory}' may take about '{estimateTime.TotalMinutes}' mins to finish maintenance. It's too risky since we only have '{totalAvailableTimeInMinutes - totalTimeSpent.Elapsed.TotalMinutes}' mins left for maintenance.");
}
}
if (runMainenance)
{
var extensionManager = HostContext.GetService <IExtensionManager>();
ISourceProvider sourceProvider = extensionManager.GetExtensions <ISourceProvider>().FirstOrDefault(x => string.Equals(x.RepositoryType, trackingConfig.RepositoryType, StringComparison.OrdinalIgnoreCase));
if (sourceProvider != null)
{
try
{
trackingManager.MaintenanceStarted(trackingConfig, trackingFile);
string repositoryPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), trackingConfig.SourcesDirectory);
await sourceProvider.RunMaintenanceOperations(executionContext, repositoryPath);
trackingManager.MaintenanceCompleted(trackingConfig, trackingFile);
}
catch (Exception ex)
{
executionContext.Error(StringUtil.Loc("ErrorDuringBuildGC", trackingFile));
executionContext.Error(ex);
}
}
}
}
}
public void UpdateJobRunProperties(IExecutionContext executionContext, TrackingConfig config, string file)
{
Trace.Entering();
// Update the info properties and save the file.
config.UpdateJobRunProperties(executionContext);
WriteToFile(file, config);
}
public void UpdatesTrackingConfigJobRunProperties()
{
using (TestHostContext hc = Setup())
{
// Arrange.
DateTimeOffset testStartOn = DateTimeOffset.Now;
TrackingConfig config = new TrackingConfig();
string trackingFile = Path.Combine(_workFolder, "trackingconfig.json");
// Act.
_trackingManager.UpdateJobRunProperties(_ec.Object, config, trackingFile);
// Assert.
config = _trackingManager.LoadIfExists(_ec.Object, trackingFile) as TrackingConfig;
Assert.NotNull(config);
Assert.Equal(CollectionUrl, config.CollectionUrl);
Assert.Equal(DefinitionName, config.DefinitionName);
// Manipulate the expected seconds due to loss of granularity when the
// date-time-offset is serialized in a friendly format.
Assert.True(testStartOn.AddSeconds(-1) <= config.LastRunOn);
Assert.True(DateTimeOffset.Now.AddSeconds(1) >= config.LastRunOn);
}
}
// Prepare build directory
// Set all build related variables
public override void InitializeJobExtension(IExecutionContext executionContext)
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
// This flag can be false for jobs like cleanup artifacts.
// If syncSources = false, we will not set source related build variable, not create build folder, not sync source.
bool syncSources = executionContext.Variables.Build_SyncSources ?? true;
if (!syncSources)
{
Trace.Info($"{Constants.Variables.Build.SyncSources} = false, we will not set source related build variable, not create build folder and not sync source");
return;
}
// Get the repo endpoint and source provider.
if (!TrySetPrimaryEndpointAndProviderInfo(executionContext))
{
throw new Exception(StringUtil.Loc("SupportedRepositoryEndpointNotFound"));
}
executionContext.Debug($"Primary repository: {SourceEndpoint.Name}. repository type: {SourceProvider.RepositoryType}");
// Set the repo variables.
string repositoryId;
if (SourceEndpoint.Data.TryGetValue("repositoryId", out repositoryId)) // TODO: Move to const after source artifacts PR is merged.
{
executionContext.Variables.Set(Constants.Variables.Build.RepoId, repositoryId);
}
executionContext.Variables.Set(Constants.Variables.Build.RepoName, SourceEndpoint.Name);
executionContext.Variables.Set(Constants.Variables.Build.RepoProvider, SourceEndpoint.Type);
executionContext.Variables.Set(Constants.Variables.Build.RepoUri, SourceEndpoint.Url?.AbsoluteUri);
string checkoutSubmoduleText;
if (SourceEndpoint.Data.TryGetValue(EndpointData.CheckoutSubmodules, out checkoutSubmoduleText))
{
executionContext.Variables.Set(Constants.Variables.Build.RepoGitSubmoduleCheckout, checkoutSubmoduleText);
}
// overwrite primary repository's clean value if build.repository.clean is sent from server. this is used by tfvc gated check-in
bool?repoClean = executionContext.Variables.GetBoolean(Constants.Variables.Build.RepoClean);
if (repoClean != null)
{
SourceEndpoint.Data[EndpointData.Clean] = repoClean.Value.ToString();
}
else
{
string cleanRepoText;
if (SourceEndpoint.Data.TryGetValue(EndpointData.Clean, out cleanRepoText))
{
executionContext.Variables.Set(Constants.Variables.Build.RepoClean, cleanRepoText);
}
}
// Prepare the build directory.
executionContext.Output(StringUtil.Loc("PrepareBuildDir"));
var directoryManager = HostContext.GetService <IBuildDirectoryManager>();
TrackingConfig trackingConfig = directoryManager.PrepareDirectory(
executionContext,
SourceEndpoint,
SourceProvider);
// Set the directory variables.
executionContext.Output(StringUtil.Loc("SetBuildVars"));
string _workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
executionContext.Variables.Set(Constants.Variables.Agent.BuildDirectory, Path.Combine(_workDirectory, trackingConfig.BuildDirectory));
executionContext.Variables.Set(Constants.Variables.System.ArtifactsDirectory, Path.Combine(_workDirectory, trackingConfig.ArtifactsDirectory));
executionContext.Variables.Set(Constants.Variables.System.DefaultWorkingDirectory, Path.Combine(_workDirectory, trackingConfig.SourcesDirectory));
executionContext.Variables.Set(Constants.Variables.Common.TestResultsDirectory, Path.Combine(_workDirectory, trackingConfig.TestResultsDirectory));
executionContext.Variables.Set(Constants.Variables.Build.BinariesDirectory, Path.Combine(_workDirectory, trackingConfig.BuildDirectory, Constants.Build.Path.BinariesDirectory));
executionContext.Variables.Set(Constants.Variables.Build.SourcesDirectory, Path.Combine(_workDirectory, trackingConfig.SourcesDirectory));
executionContext.Variables.Set(Constants.Variables.Build.StagingDirectory, Path.Combine(_workDirectory, trackingConfig.ArtifactsDirectory));
executionContext.Variables.Set(Constants.Variables.Build.ArtifactStagingDirectory, Path.Combine(_workDirectory, trackingConfig.ArtifactsDirectory));
executionContext.Variables.Set(Constants.Variables.Build.RepoLocalPath, Path.Combine(_workDirectory, trackingConfig.SourcesDirectory));
SourceProvider.SetVariablesInEndpoint(executionContext, SourceEndpoint);
}
