public void MarkForGarbageCollection(IExecutionContext executionContext, TrackingConfigBase config)
{
Trace.Entering();
// Convert legacy format to the new format.
LegacyTrackingConfig legacyConfig = config as LegacyTrackingConfig;
if (legacyConfig != null)
{
// Convert legacy format to the new format.
config = new TrackingConfig(
executionContext,
legacyConfig,
// The sources folder wasn't stored in the legacy format - only the
// build folder was stored. Since the hash key has changed, it is
// unknown what the source folder was named. Just set the folder name
// to "s" so the property isn't left blank.
sourcesDirectoryNameOnly: Constants.Build.Path.SourcesDirectory);
}
// Write a copy of the tracking config to the GC folder.
string gcDirectory = Path.Combine(
IOUtil.GetWorkPath(HostContext),
Constants.Build.Path.SourceRootMappingDirectory,
Constants.Build.Path.GarbageCollectionDirectory);
string file = Path.Combine(
gcDirectory,
StringUtil.Format("{0}.json", Guid.NewGuid()));
WriteToFile(file, config);
}
public TrackingConfigBase LoadIfExists(IExecutionContext executionContext, string file)
{
Trace.Entering();
// The tracking config will not exist for a new definition.
if (!File.Exists(file))
{
return(null);
}
// Load the content and distinguish between tracking config file
// version 1 and file version 2.
string content = File.ReadAllText(file);
string fileFormatVersionJsonProperty = StringUtil.Format(
@"""{0}""",
TrackingConfig.FileFormatVersionJsonProperty);
if (content.Contains(fileFormatVersionJsonProperty))
{
// The config is the new format.
Trace.Verbose("Parsing new tracking config format.");
return(JsonConvert.DeserializeObject <TrackingConfig>(content));
}
// Attempt to parse the legacy format.
Trace.Verbose("Parsing legacy tracking config format.");
LegacyTrackingConfig config = LegacyTrackingConfig.TryParse(content);
if (config == null)
{
executionContext.Warning(StringUtil.Loc("UnableToParseBuildTrackingConfig0", content));
}
return(config);
}
public TrackingConfig(
IExecutionContext executionContext,
LegacyTrackingConfig copy,
string sourcesDirectoryNameOnly,
string repositoryType,
bool useNewArtifactsDirectoryName = false)
: this()
{
// Set the directories.
BuildDirectory = Path.GetFileName(copy.BuildDirectory); // Just take the portion after _work folder.
string artifactsDirectoryNameOnly =
useNewArtifactsDirectoryName ? Constants.Build.Path.ArtifactsDirectory : Constants.Build.Path.LegacyArtifactsDirectory;
ArtifactsDirectory = Path.Combine(BuildDirectory, artifactsDirectoryNameOnly);
SourcesDirectory = Path.Combine(BuildDirectory, sourcesDirectoryNameOnly);
TestResultsDirectory = Path.Combine(BuildDirectory, Constants.Build.Path.TestResultsDirectory);
// Set the other properties.
CollectionId = copy.CollectionId;
CollectionUrl = executionContext.Variables.System_TFCollectionUrl;
DefinitionId = copy.DefinitionId;
HashKey = copy.HashKey;
RepositoryType = repositoryType;
RepositoryUrl = copy.RepositoryUrl;
System = copy.System;
// Let's make sure this file gets cleaned up by the garbage collector
LastRunOn = new DateTime(1, 1, 1, 0, 0, 0, DateTimeKind.Utc);
}
public TrackingConfig(
IExecutionContext executionContext,
LegacyTrackingConfig copy,
string sourcesDirectoryNameOnly,
string repositoryType,
bool useNewArtifactsDirectoryName = false)
{
// Set the directories.
BuildDirectory = Path.GetFileName(copy.BuildDirectory); // Just take the portion after _work folder.
string artifactsDirectoryNameOnly =
useNewArtifactsDirectoryName ? Constants.Build.Path.ArtifactsDirectory : Constants.Build.Path.LegacyArtifactsDirectory;
ArtifactsDirectory = Path.Combine(BuildDirectory, artifactsDirectoryNameOnly);
SourcesDirectory = Path.Combine(BuildDirectory, sourcesDirectoryNameOnly);
TestResultsDirectory = Path.Combine(BuildDirectory, Constants.Build.Path.TestResultsDirectory);
// Set the other properties.
CollectionId = copy.CollectionId;
CollectionUrl = executionContext.Variables.System_TFCollectionUrl;
DefinitionId = copy.DefinitionId;
HashKey = copy.HashKey;
RepositoryType = repositoryType;
RepositoryUrl = copy.RepositoryUrl;
System = copy.System;
}
private TrackingConfig LoadIfExists(
IExecutionContext executionContext,
string file)
{
Trace.Entering();
Trace.Verbose($"Loading {file}");
// The tracking config will not exist for a new definition.
if (!File.Exists(file))
{
Trace.Verbose($"Tracking file does not exist: {file}");
return(null);
}
TrackingConfig result = null;
// Load the content and distinguish between tracking config file
// version 1 and file version 2.
string content = File.ReadAllText(file);
string fileFormatVersionJsonProperty = StringUtil.Format(
@"""{0}""",
TrackingConfig.FileFormatVersionJsonProperty);
if (content.Contains(fileFormatVersionJsonProperty))
{
// The config is the new format.
Trace.Verbose("Parsing new tracking config format.");
result = JsonConvert.DeserializeObject <TrackingConfig>(content);
}
else
{
// Attempt to parse the legacy format.
Trace.Verbose("Parsing legacy tracking config format.");
var legacyTrackingConfig = LegacyTrackingConfig.TryParse(content);
if (legacyTrackingConfig == null)
{
executionContext.Warning(StringUtil.Loc("UnableToParseBuildTrackingConfig0", content));
}
else
{
// Convert legacy format to the new format.
result = ConvertToNewFormat(
executionContext,
legacyTrackingConfig,
RepositoryUtil.GetCloneDirectory(legacyTrackingConfig.RepositoryUrl),
RepositoryUtil.GuessRepositoryType(legacyTrackingConfig.RepositoryUrl));
}
}
if (result != null)
{
result.FileLocation = file;
}
return(result);
}
private TrackingConfig ConvertToNewFormat(
IExecutionContext executionContext,
RepositoryResource repository,
TrackingConfigBase config)
{
Trace.Entering();
// If it's already in the new format, return it.
TrackingConfig newConfig = config as TrackingConfig;
if (newConfig != null)
{
return(newConfig);
}
// Delete the legacy artifact/staging directories.
LegacyTrackingConfig legacyConfig = config as LegacyTrackingConfig;
DeleteDirectory(
executionContext,
description: "legacy artifacts directory",
path: Path.Combine(legacyConfig.BuildDirectory, Constants.Build.Path.LegacyArtifactsDirectory));
DeleteDirectory(
executionContext,
description: "legacy staging directory",
path: Path.Combine(legacyConfig.BuildDirectory, Constants.Build.Path.LegacyStagingDirectory));
// Determine the source directory name. Check if the directory is named "s" already.
// Convert the source directory to be named "s" if there is a problem with the old name.
string sourcesDirectoryNameOnly = Constants.Build.Path.SourcesDirectory;
string repositoryName = repository.Properties.Get <string>(RepositoryPropertyNames.Name);
if (!Directory.Exists(Path.Combine(legacyConfig.BuildDirectory, sourcesDirectoryNameOnly)) &&
!String.Equals(repositoryName, Constants.Build.Path.ArtifactsDirectory, StringComparison.OrdinalIgnoreCase) &&
!String.Equals(repositoryName, Constants.Build.Path.LegacyArtifactsDirectory, StringComparison.OrdinalIgnoreCase) &&
!String.Equals(repositoryName, Constants.Build.Path.LegacyStagingDirectory, StringComparison.OrdinalIgnoreCase) &&
!String.Equals(repositoryName, Constants.Build.Path.TestResultsDirectory, StringComparison.OrdinalIgnoreCase) &&
!repositoryName.Contains("\\") &&
!repositoryName.Contains("/") &&
Directory.Exists(Path.Combine(legacyConfig.BuildDirectory, repositoryName)))
{
sourcesDirectoryNameOnly = repositoryName;
}
// Convert to the new format.
newConfig = new TrackingConfig(
executionContext,
legacyConfig,
sourcesDirectoryNameOnly,
repository.Type,
// The legacy artifacts directory has been deleted at this point - see above - so
// switch the configuration to using the new naming scheme.
useNewArtifactsDirectoryName: true);
return(newConfig);
}
public static LegacyTrackingConfig TryParse(string content)
{
ArgUtil.NotNull(content, nameof(content));
// Fix the content to be valid JSON syntax. The file version 1 files
// were written as:
// {
// "system"" : "[...]",
// "collectionId"" = "[...]",
// "definitionId"" = "[...]",
// "repositoryUrl"" = "[...]",
// "sourceFolder" = "[...]",
// "hashKey" = "[...]"
// }
//
// Furthermore, the values were not JSON-escaped.
content =
content
// Escape special characters.
.Replace(@"\", @"\\")
// Change "=" to ":".
.Replace(@"""collectionId"" = ", @"""collectionId"": ")
.Replace(@"""definitionId"" = ", @"""definitionId"": ")
.Replace(@"""repositoryUrl"" = ", @"""repositoryUrl"": ")
.Replace(@"""sourceFolder"" = ", @"""sourceFolder"": ")
.Replace(@"""hashKey"" = ", @"""hashKey"": ");
LegacyTrackingConfig config = null;
try
{
config = JsonConvert.DeserializeObject <LegacyTrackingConfig>(content);
}
catch (Exception)
{
}
if (config != null &&
!string.IsNullOrEmpty(config.BuildDirectory) &&
!string.IsNullOrEmpty(config.CollectionId) &&
!string.IsNullOrEmpty(config.DefinitionId) &&
!string.IsNullOrEmpty(config.HashKey) &&
!string.IsNullOrEmpty(config.RepositoryUrl) &&
!string.IsNullOrEmpty(config.System))
{
return(config);
}
return(null);
}
private TrackingConfig ConvertToNewFormat(
IExecutionContext executionContext,
LegacyTrackingConfig legacyConfig,
string suggestedSourceFolderName,
string repositoryType)
{
Trace.Entering();
if (legacyConfig == null)
{
return(null);
}
// Determine the source directory name. Check if the directory is named "s" already.
// Convert the source directory to be named "s" if there is a problem with the old name.
string sourcesDirectoryNameOnly = Constants.Build.Path.SourcesDirectory;
if (!Directory.Exists(Path.Combine(legacyConfig.BuildDirectory, sourcesDirectoryNameOnly)) &&
!String.Equals(suggestedSourceFolderName, Constants.Build.Path.ArtifactsDirectory, StringComparison.OrdinalIgnoreCase) &&
!String.Equals(suggestedSourceFolderName, Constants.Build.Path.LegacyArtifactsDirectory, StringComparison.OrdinalIgnoreCase) &&
!String.Equals(suggestedSourceFolderName, Constants.Build.Path.LegacyStagingDirectory, StringComparison.OrdinalIgnoreCase) &&
!String.Equals(suggestedSourceFolderName, Constants.Build.Path.TestResultsDirectory, StringComparison.OrdinalIgnoreCase) &&
!suggestedSourceFolderName.Contains("\\") &&
!suggestedSourceFolderName.Contains("/") &&
Directory.Exists(Path.Combine(legacyConfig.BuildDirectory, suggestedSourceFolderName)))
{
sourcesDirectoryNameOnly = suggestedSourceFolderName;
}
// Convert to the new format.
var newConfig = new TrackingConfig(
executionContext,
legacyConfig,
sourcesDirectoryNameOnly,
repositoryType,
useNewArtifactsDirectoryName: false);
return(newConfig);
}
public TrackingConfig(
IExecutionContext executionContext,
LegacyTrackingConfig copy,
string sourcesDirectoryNameOnly,
bool useNewArtifactsDirectoryName = false)
{
// Set the directories.
BuildDirectory = Path.GetFileName(copy.BuildDirectory); // Just take the portion after _work folder.
string artifactsDirectoryNameOnly =
useNewArtifactsDirectoryName ? Constants.Build.Path.ArtifactsDirectory : Constants.Build.Path.LegacyArtifactsDirectory;
ArtifactsDirectory = Path.Combine(BuildDirectory, artifactsDirectoryNameOnly);
SourcesDirectory = Path.Combine(BuildDirectory, sourcesDirectoryNameOnly);
TestResultsDirectory = Path.Combine(BuildDirectory, Constants.Build.Path.TestResultsDirectory);
// Set the other properties.
CollectionId = copy.CollectionId;
CollectionUrl = executionContext.Variables.System_TFCollectionUrl;
DefinitionId = copy.DefinitionId;
HashKey = copy.HashKey;
RepositoryUrl = copy.RepositoryUrl;
System = copy.System;
}
private TrackingConfig LoadIfExists(
IExecutionContext executionContext,
string file)
{
Trace.Entering();
Trace.Verbose($"Loading {file}");
// The tracking config will not exist for a new definition.
if (!File.Exists(file))
{
Trace.Verbose($"Tracking file does not exist: {file}");
return(null);
}
TrackingConfig result = null;
// Load the content and distinguish between tracking config file
// version 1 and file version 2.
string content = File.ReadAllText(file);
string fileFormatVersionJsonProperty = StringUtil.Format(
@"""{0}""",
TrackingConfig.FileFormatVersionJsonProperty);
if (content.Contains(fileFormatVersionJsonProperty))
{
// The config is the new format.
Trace.Verbose("Parsing new tracking config format.");
result = JsonConvert.DeserializeObject <TrackingConfig>(content);
if (result != null)
{
// if RepositoryTrackingInfo is empty, then we should create an entry so the rest
// of the logic after this will act correctly
if (result.RepositoryTrackingInfo.Count == 0)
{
result.RepositoryTrackingInfo.Add(new Build.RepositoryTrackingInfo
{
Identifier = RepositoryUtil.DefaultPrimaryRepositoryName,
RepositoryType = result.RepositoryType,
RepositoryUrl = result.RepositoryUrl,
SourcesDirectory = result.SourcesDirectory,
});
}
}
}
else
{
// Attempt to parse the legacy format.
Trace.Verbose("Parsing legacy tracking config format.");
var legacyTrackingConfig = LegacyTrackingConfig.TryParse(content);
if (legacyTrackingConfig == null)
{
executionContext.Warning(StringUtil.Loc("UnableToParseBuildTrackingConfig0", content));
}
else
{
// Convert legacy format to the new format.
result = ConvertToNewFormat(
executionContext,
legacyTrackingConfig,
RepositoryUtil.GetCloneDirectory(legacyTrackingConfig.RepositoryUrl),
RepositoryUtil.GuessRepositoryType(legacyTrackingConfig.RepositoryUrl));
}
}
if (result != null)
{
result.FileLocation = file;
}
return(result);
}
public void MarkExpiredForGarbageCollection(IExecutionContext executionContext, TimeSpan expiration)
{
Trace.Entering();
Trace.Info("Scan all SourceFolder tracking files.");
string searchRoot = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Build.Path.SourceRootMappingDirectory);
if (!Directory.Exists(searchRoot))
{
executionContext.Output(StringUtil.Loc("GCDirNotExist", searchRoot));
return;
}
var allTrackingFiles = Directory.EnumerateFiles(searchRoot, Constants.Build.Path.TrackingConfigFile, SearchOption.AllDirectories);
Trace.Verbose($"Find {allTrackingFiles.Count()} tracking files.");
executionContext.Output(StringUtil.Loc("DirExpireLimit", expiration.TotalDays));
executionContext.Output(StringUtil.Loc("CurrentUTC", DateTime.UtcNow.ToString("o")));
// scan all sourcefolder tracking file, find which folder has never been used since UTC-expiration
// the scan and garbage discovery should be best effort.
// if the tracking file is in old format, just delete the folder since the first time the folder been use we will convert the tracking file to new format.
foreach (var trackingFile in allTrackingFiles)
{
try
{
executionContext.Output(StringUtil.Loc("EvaluateTrackingFile", trackingFile));
TrackingConfigBase tracking = LoadIfExists(executionContext, trackingFile);
// detect whether the tracking file is in new format.
TrackingConfig newTracking = tracking as TrackingConfig;
if (newTracking == null)
{
LegacyTrackingConfig legacyConfig = tracking as LegacyTrackingConfig;
ArgUtil.NotNull(legacyConfig, nameof(LegacyTrackingConfig));
Trace.Verbose($"{trackingFile} is a old format tracking file.");
executionContext.Output(StringUtil.Loc("GCOldFormatTrackingFile", trackingFile));
MarkForGarbageCollection(executionContext, legacyConfig);
IOUtil.DeleteFile(trackingFile);
}
else
{
Trace.Verbose($"{trackingFile} is a new format tracking file.");
ArgUtil.NotNull(newTracking.LastRunOn, nameof(newTracking.LastRunOn));
executionContext.Output(StringUtil.Loc("BuildDirLastUseTIme", Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newTracking.BuildDirectory), newTracking.LastRunOnString));
if (DateTime.UtcNow - expiration > newTracking.LastRunOn)
{
executionContext.Output(StringUtil.Loc("GCUnusedTrackingFile", trackingFile, expiration.TotalDays));
MarkForGarbageCollection(executionContext, newTracking);
IOUtil.DeleteFile(trackingFile);
}
}
}
catch (Exception ex)
{
executionContext.Error(StringUtil.Loc("ErrorDuringBuildGC", trackingFile));
executionContext.Error(ex);
}
}
}
