/// <summary>
/// Cleans up the successors in the dependency graph.
/// </summary>
/// <param name="graph">Node to start with</param>
internal void Clean(IGraph graph)
{
if (!_silentMode)
{
_logger.LogMsg("Cleaning all components...");
}
Logger.Instance().Log(TraceLevel.Info, "{0}: Cleaning components ...", CommandType);
var dwSvc = new DownloadWatermarkService(graph.RootComponentTargetPath);
//TODO: Use parallel foreach to gain cleanup speed. might infere with logging and therefore we are currently using single threaded only
var cachedComponents = dwSvc.GetStoredDependencyWatermarks();
foreach (var cacheComponent in cachedComponents)
{
// Properties of cachedComponent: Item1 = name, cachedComponent Item2 = version
RevertComponent(dwSvc, cacheComponent.Item1, cacheComponent.Item2);
}
// Finish documents
if (!_silentMode)
{
_logger.LogMsg("Cleaned components successfully!\n");
}
Logger.Instance().Log(TraceLevel.Info, "{0}: Cleaning components finished successfully", CommandType);
}
/// <summary>
/// Reverts an individual component
/// </summary>
/// <param name="dwSvc">The watermark service to use</param>
/// <param name="name">The name of the component</param>
/// <param name="version">The version of the component</param>
internal void RevertComponent(DownloadWatermarkService dwSvc, string name, string version)
{
var dFac = new DownloaderFactory();
IDependencyDownloaderWatermark wm = null;
var wmName = name;
var wmVersion = version;
try
{
wm = dwSvc.Load(name, version);
if (null == wm)
{
// TODO: might consider logging this to a trace file
Debug.Fail(
string.Format(
"Unable to load a watermark for component {0}@{1}. This situation should never occur due to the fact that the same method is reading and writing the watermarks",
name,
version));
}
// Read name and version information from watermark (saving can convert name and version back to hashes).
if (wm.Tags.ContainsKey("name"))
{
wmName = wm.Tags["name"];
}
if (wm.Tags.ContainsKey("version"))
{
wmVersion = wm.Tags["version"];
}
var cleaner = dFac.GetDownloader(wm.DownloadType);
Logger.Instance().Log(TraceLevel.Info, "{0}: Cleaning component {1}#{2} ...", CommandType, wmName, wmVersion);
cleaner.RevertDownload(wm);
dwSvc.Delete(wm);
}
catch
{
if (null != wm)
{
dwSvc.Save(wm, name, version);
}
}
if (!_silentMode)
{
_logger.LogMsg(string.Format(" * Component {0} (Version:{1}) cleaned.", wmName, wmVersion));
}
Logger.Instance().Log(TraceLevel.Info, "{0}: Component {1}#{2} successfully cleaned", CommandType, wmName, wmVersion);
}
/// <summary>
/// Determines the source, target location and worker settings and initializes download workers according to their type.
/// </summary>
/// <param name="dws">The downloader wartermark service.</param>
/// <param name="component">The component node in graph to start download from.</param>
/// <param name="recursive">Indicates if the dependencies should be fetched recursively or not.</param>
/// <param name="force">If set to <c>true</c> it indicates that the get operation is forced and all files have to be overwritten. Otherwise false.</param>
private void DownloadComponent(DownloadWatermarkService dws, IComponent component, bool recursive, bool force)
{
var df = new DownloaderFactory();
var componentAlreadyDownloaded = false;
var settings = new Settings <DownloaderValidSettings>();
IDependencyDownloader worker;
string destLocation;
string targetLocation;
bool removeTempFiles = false;
Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component {1}#{2} ...", CommandType, component.Name, component.Version);
// Create worker and settings according to type
if (component.Type.Equals(ComponentType.FileShare))
{
var fileShareRootPath = component.GetFieldValue(DependencyProviderValidSettingName.FileShareRootPath);
destLocation = Path.Combine(fileShareRootPath, component.Name.Path, component.Version.Version);
var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath);
targetLocation =
relativeTargetPath == null
? Path.GetFullPath(Path.Combine(_defaultDownloadFolder, _defaultRelativeOutputPath))
: Path.GetFullPath(Path.Combine(_defaultDownloadFolder, relativeTargetPath));
LoadFilterSettings(component, settings);
}
else if (component.Type.Equals(ComponentType.VNextBuildResult))
{
var url = component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl);
if (!Uri.IsWellFormedUriString(url, UriKind.Absolute))
{
Logger.Instance().Log(
TraceLevel.Error,
"{0}: Invalid BuildTeamProjectCollectionUrl setting '{1}' was found for component {2}#{3}",
CommandType,
url,
component.Name,
component.Version);
throw new DependencyServiceException(
string.Format(
" ! Invalid download information was found for build result control component {0} (BuildTeamProjectCollectionUrl setting)",
component.Name.GetName()));
}
// Connect to tfs server
var tpc = TfsTeamProjectCollectionFactory.GetTeamProjectCollection(new Uri(url));
tpc.EnsureAuthenticated();
// Connect to version control service
var versionControl = tpc.GetService <VersionControlServer>();
if (versionControl == null)
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Connection to version control server failed for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format(" ! Could not connect to TFS version control server for team project collection {0}", component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl)));
}
var tpcUrl = new Uri(url);
var connection = new VssConnection(tpcUrl, new VssClientCredentials(true));
var client = connection.GetClient <BuildHttpClient>();
var builds = client.GetBuildsAsync(project: component.Name.TeamProject, type: DefinitionType.Build).Result;
var buildResult = builds.FirstOrDefault(r => r.BuildNumber == component.Version.BuildNumber);
if (buildResult == null)
{
Logger.Instance().Log(TraceLevel.Error, "{0}: No vnext build result could not be determined for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format(" ! Could not determine build result component {0}", component.Name.GetName()));
}
var artifacts = client.GetArtifactsAsync(component.Name.TeamProject, buildResult.Id).Result;
destLocation = Path.Combine(Path.GetTempPath(), "VNextDM_" + Guid.NewGuid());
Directory.CreateDirectory(destLocation);
removeTempFiles = true;
foreach (var artifact in artifacts)
{
if (artifact.Resource.Type == "FilePath")
{
var sourceDirName = $"{artifact.Resource.Data}/{artifact.Name}";
DirectoryCopy(sourceDirName, destLocation, true);
}
else
{
var content = client.GetArtifactContentZipAsync(component.Name.TeamProject, buildResult.Id, artifact.Name);
using (var zipArchive = new ZipArchive(content.Result))
{
zipArchive.ExtractToDirectory(destLocation);
}
}
}
var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath);
targetLocation =
relativeTargetPath == null
? Path.GetFullPath(Path.Combine(_defaultDownloadFolder, _defaultRelativeOutputPath))
: Path.GetFullPath(Path.Combine(_defaultDownloadFolder, relativeTargetPath));
LoadFilterSettings(component, settings);
}
else if (component.Type.Equals(ComponentType.BuildResult))
{
if (!Uri.IsWellFormedUriString(component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl), UriKind.Absolute))
{
Logger.Instance().Log(
TraceLevel.Error,
"{0}: Invalid BuildTeamProjectCollectionUrl setting '{1}' was found for component {2}#{3}",
CommandType,
component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl),
component.Name,
component.Version);
throw new DependencyServiceException(
string.Format(
" ! Invalid download information was found for build result control component {0} (BuildTeamProjectCollectionUrl setting)",
component.Name.GetName()));
}
// Connect to tfs server
var tpc = TfsTeamProjectCollectionFactory.GetTeamProjectCollection(new Uri(component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl)));
tpc.EnsureAuthenticated();
// Connect to version control service
var versionControl = tpc.GetService <VersionControlServer>();
if (versionControl == null)
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Connection to version control server failed for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format(" ! Could not connect to TFS version control server for team project collection {0}", component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl)));
}
// Connect to build server
var buildServer = tpc.GetService <IBuildServer>();
if (buildServer == null)
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Connection to build server failed for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format(" ! Could not connect to TFS build server for team project collection {0}", component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl)));
}
var buildDef = buildServer.GetBuildDefinition(component.Name.TeamProject, component.Name.BuildDefinition);
var buildDetailSpec = buildServer.CreateBuildDetailSpec(buildDef);
buildDetailSpec.BuildNumber = component.Version.BuildNumber;
buildDetailSpec.InformationTypes = new string[] { };
var buildResult = buildServer.QueryBuilds(buildDetailSpec);
if (buildResult == null || buildResult.Builds == null || buildResult.Builds.Length == 0)
{
Logger.Instance().Log(TraceLevel.Error, "{0}: No build result could not be determined for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format(" ! Could not determine drop location for build result component {0}", component.Name.GetName()));
}
// Determine source location
var dropLocation = buildResult.Builds[0].DropLocation; // TODO dga: Is this a bug? it returns localhost for the computer name while we are not running this code on the server (I guess)?
if (string.IsNullOrEmpty(dropLocation))
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Drop location could not be determined for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format("! Could not determine drop location for build result component {0}", component.Name.GetName()));
}
destLocation = dropLocation;
var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath);
targetLocation =
relativeTargetPath == null
? Path.GetFullPath(Path.Combine(_defaultDownloadFolder, _defaultRelativeOutputPath))
: Path.GetFullPath(Path.Combine(_defaultDownloadFolder, relativeTargetPath));
LoadFilterSettings(component, settings);
}
else if (component.Type.Equals(ComponentType.SourceControl))
{
if (!Uri.IsWellFormedUriString(component.GetFieldValue(DependencyProviderValidSettingName.WorkspaceTeamProjectCollectionUrl), UriKind.Absolute))
{
Logger.Instance().Log(
TraceLevel.Error,
"{0}: Invalid WorkspaceTeamProjectCollectionUrl setting '{1}' was found for component {2}#{3}",
CommandType,
component.GetFieldValue(DependencyProviderValidSettingName.WorkspaceTeamProjectCollectionUrl),
component.Name,
component.Version);
throw new DependencyServiceException(
string.Format(
" ! Invalid download information was found for source control component {0} (WorkspaceTeamProjectCollectionUrl setting)",
component.Name.GetName()));
}
var workspaceName = component.GetFieldValue(DependencyProviderValidSettingName.WorkspaceName);
if (string.IsNullOrEmpty(workspaceName))
{
Logger.Instance().Log(TraceLevel.Error, "{0}: WorkspaceName setting was not specified for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(
string.Format(
" ! Invalid download information was found for source control component {0} (WorkspaceName setting)",
component.Name.GetName()));
}
var workspaceOwner = component.GetFieldValue(DependencyProviderValidSettingName.WorkspaceOwner);
if (string.IsNullOrEmpty(workspaceOwner))
{
Logger.Instance().Log(TraceLevel.Error, "{0}: WorkspaceOwner setting was not specified for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(
string.Format(
" ! Invalid download information was found for source control component {0} (WorkspaceOwner setting)",
component.Name.GetName()));
}
destLocation = component.Name.ToString();
settings.AddSetting(new KeyValuePair <DownloaderValidSettings, string>(DownloaderValidSettings.VersionString, component.Version.ToString()));
var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath);
targetLocation =
Path.GetFullPath(
relativeTargetPath == null
? Path.Combine(_defaultDownloadFolder, "..", VersionControlPath.GetFileName(component.Name.ToString()))
: Path.Combine(_defaultDownloadFolder, relativeTargetPath));
}
else if (component.Type.Equals(ComponentType.SourceControlCopy))
{
destLocation = component.Name.ToString();
settings.AddSetting(new KeyValuePair <DownloaderValidSettings, string>(DownloaderValidSettings.VersionString, component.Version.ToString()));
var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath);
targetLocation =
Path.GetFullPath(
relativeTargetPath == null
? Path.Combine(_defaultDownloadFolder, "..", VersionControlPath.GetFileName(component.Name.ToString()))
: Path.Combine(_defaultDownloadFolder, relativeTargetPath));
LoadFilterSettings(component, settings);
}
else if (component.Type.Equals(ComponentType.BinaryRepository))
{
var repositoryTeamProject = component.GetFieldValue(DependencyProviderValidSettingName.BinaryRepositoryTeamProject);
if (string.IsNullOrEmpty(repositoryTeamProject))
{
Logger.Instance().Log(TraceLevel.Error, "{0}: BinaryRepositoryTeamProject setting was not specified for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(
string.Format(
" ! Invalid download information was found for binary repository component {0} (BinaryRepositoryTeamProject setting)",
component.Name.GetName()));
}
destLocation = VersionControlPath.Combine(VersionControlPath.Combine(VersionControlPath.Combine(VersionControlPath.RootFolder, repositoryTeamProject), component.Name.GetName()), component.Version.GetVersion());
settings.AddSetting(new KeyValuePair <DownloaderValidSettings, string>(DownloaderValidSettings.VersionString, VersionSpec.Latest.DisplayString));
var relativeOutputPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath);
targetLocation = Path.GetFullPath(
relativeOutputPath == null
? Path.Combine(_defaultDownloadFolder, _defaultRelativeOutputPath)
: Path.Combine(_defaultDownloadFolder, relativeOutputPath));
LoadFilterSettings(component, settings);
}
else if (component.Type.Equals(ComponentType.Subversion))
{
destLocation = string.Format("{0}", component.Name);
settings.AddSetting(new KeyValuePair <DownloaderValidSettings, string>(DownloaderValidSettings.VersionString, component.Version.ToString()));
var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath);
targetLocation =
relativeTargetPath == null
? Path.GetFullPath(Path.Combine(_defaultDownloadFolder, _defaultRelativeOutputPath))
: Path.GetFullPath(Path.Combine(_defaultDownloadFolder, relativeTargetPath));
LoadFilterSettings(component, settings);
}
else
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Unknown dependency type '{1}' was found in dependency graph", CommandType, component.Type.GetType());
throw new DependencyServiceException(" ! Invalid dependency node found in graph!");
}
//Determine Multi-Site path for BuildResult and FileShare-Provider
if (component.Type.Equals(ComponentType.FileShare) || (component.Type.Equals(ComponentType.BuildResult)))
{
destLocation = DetermineMultiSiteSourceLocation(destLocation);
}
if (string.IsNullOrEmpty(destLocation))
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Source location was not set for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format(" ! Error occured while preparing to download component {0} (Source location was not set)", component.Name.GetName()));
}
if (string.IsNullOrEmpty(targetLocation))
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Target location was not set for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format(" ! Error occured while preparing to download component {0} (Destination location was not set)", component.Name.GetName()));
}
// Download files
try
{
IDependencyDownloaderWatermark wm = null;
try
{
worker = df.GetDownloader(component);
wm = dws.Load(component.Name.GetName(), component.Version.GetVersion()) ?? new DownloaderWatermark(worker);
worker.Download(destLocation, targetLocation, wm, force, settings);
}
finally
{
if (null != wm)
{
dws.Save(wm, component.Name.GetName(), component.Version.GetVersion());
}
if (removeTempFiles)
{
Directory.Delete(destLocation, true);
}
}
}
catch (Exception e)
{
if (e is DependencyServiceException)
{
throw;
}
// ReSharper disable RedundantIfElseBlock
else if (e is ComponentAlreadyDownloadedException)
// ReSharper restore RedundantIfElseBlock
{
componentAlreadyDownloaded = true;
}
else
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Exception {1} occured while downloading files: {2}", CommandType, e, e.Message);
throw new DependencyServiceException(string.Format(" ! Download of component {0} failed ({1})", component.Name.GetName(), e.Message));
}
}
// Log download
if (!_silentMode)
{
_logger.LogMsg(
!componentAlreadyDownloaded
? string.Format(
" * Component {0} (Version:{1}) downloaded to target directory {2}",
component.Name.GetName(),
component.Version.GetVersion(),
targetLocation)
: string.Format(
" * Skipped component {0} (Version:{1}). Already present in target directory {2}",
component.Name.GetName(),
component.Version.GetVersion(),
targetLocation));
}
Logger.Instance().Log(
TraceLevel.Info,
!componentAlreadyDownloaded
? "{0}: Component {1}#{2} successfully downloaded to target directory {3}"
: "{0}: Component {1}#{2} download was skipped. Already present in target directory {3}",
CommandType,
component.Name,
component.Version,
targetLocation);
}
/// <summary>
/// Downloads the successors in the dependency graph.
/// </summary>
/// <param name="graph">Node to start with</param>
/// <param name="recursive">Indicates if the dependencies should be fetched recursively or not.</param>
/// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten</param>
internal void Download(IGraph graph, bool recursive, bool force)
{
if (graph.RootComponent == null)
{
throw new DependencyServiceException("! Dependency graph does have an invalid root node!");
}
if (!_silentMode)
{
_logger.LogMsg("Downloading components...");
}
Logger.Instance().Log(TraceLevel.Info, "{0}: {1} downloading components {2} ...", CommandType, recursive ? "Recursively " : "Non-recursively ", force ? "(Forced Get mode) ..." : "(Normal mode)");
// Retrieve watermark service for currently called dependency definition (root component)
var wms = new DownloadWatermarkService(graph.RootComponentTargetPath);
// Clean previously downloaded components that are not part of the list anymore (removed or version changed)
var previousWatermarks = wms.GetStoredDependencyWatermarks();
var currentComponentList = graph.GetFlattenedGraph(false, recursive);
// Copy currentComponentList into new Tuple<string, string> list
var localComponentsToKeep = currentComponentList.Select(component => new Tuple <string, string>(component.Name.GetName(), component.Version.GetVersion())).ToList();
var cleaner = new Cleaner(_logger, false);
foreach (var previousWatermark in previousWatermarks)
{
if (!wms.IsComponentInList(localComponentsToKeep, previousWatermark))
{
// Revert component which is not used anymore
cleaner.RevertComponent(wms, previousWatermark.Item1, previousWatermark.Item2);
}
}
// Download current components
foreach (var component in currentComponentList)
{
DownloadComponent(wms, component, recursive, force);
}
#region # Output detected problematic dependencies
if (graph.SideBySideDependencies.Count() > 0 || graph.CircularDependencies.Count() > 0)
{
if (!_silentMode)
{
_logger.LogMsg("- The following dependency anomalies have been detected:");
}
else
{
Debug.WriteLine("- The following dependency anomalies have been detected:");
}
if (graph.SideBySideDependencies.Count() > 0)
{
if (!_silentMode)
{
_logger.LogMsg(" x Side-by-side dependency warning:");
}
Logger.Instance().Log(TraceLevel.Info, "{0}: Side-by-side dependencies detected:", CommandType);
foreach (var validationError in graph.SideBySideDependencies)
{
if (!_silentMode)
{
_logger.LogMsg(" * " + validationError.Message);
}
Logger.Instance().Log(TraceLevel.Info, "{0}: {1}", CommandType, validationError.Message);
}
}
if (graph.CircularDependencies.Count() > 0)
{
if (!_silentMode)
{
_logger.LogMsg(" x Circular dependency warning:");
}
Logger.Instance().Log(TraceLevel.Info, "{0}: Circular dependencies detected:", CommandType);
foreach (var validationError in graph.CircularDependencies)
{
if (!_silentMode)
{
_logger.LogMsg(" * " + validationError.Message);
}
Logger.Instance().Log(TraceLevel.Info, "{0}: {1}", CommandType, validationError.Message);
}
}
}
#endregion
// Finish documents
if (!_silentMode)
{
_logger.LogMsg("Downloaded components successfully!\n");
}
Logger.Instance().Log(TraceLevel.Info, "{0}: Downloaded components successfully!", CommandType);
}
