/// <summary>
/// Downloads a component from source control to a local path.
/// </summary>
/// <param name="source">Path to source location</param>
/// <param name="destination">Path to destination folder</param>
/// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations</param>
/// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten</param>
/// <param name="settings">Setting which contain the version to fetch</param>
public void Download(string source, string destination, IDependencyDownloaderWatermark watermark, bool force, ISettings <DownloaderValidSettings> settings)
{
if (string.IsNullOrEmpty(source))
{
throw new InvalidComponentException(
"Source control path to component folder was empty in DownloaderSourceControlCopy");
}
if (string.IsNullOrEmpty(destination))
{
throw new InvalidComponentException(
"Destination path for component was empty in DownloaderSourceControlCopy");
}
if (null == watermark)
{
throw new ArgumentNullException("watermark");
}
if (null == settings)
{
throw new ArgumentNullException("settings");
}
var versionString = settings.GetSetting(DownloaderValidSettings.VersionString);
if (string.IsNullOrEmpty(versionString))
{
throw new InvalidComponentException(
"Version string for component was empty in DownloaderSourceControlCopy");
}
var folderMappings = InitializeMappings(settings.GetSetting(DownloaderValidSettings.FolderMappings));
var includeFilter = InitializeIncludeFilter(settings.GetSetting(DownloaderValidSettings.IncludedFilesFilter));
var excludeFilter = InitializeExcludeFilter(settings.GetSetting(DownloaderValidSettings.ExcludedFilesFilter));
// ReSharper disable PossibleMultipleEnumeration
if (folderMappings.Any())
// ReSharper restore PossibleMultipleEnumeration
{
// ReSharper disable PossibleMultipleEnumeration
foreach (var mapping in folderMappings)
// ReSharper restore PossibleMultipleEnumeration
{
var subSource = VersionControlPath.Combine(source, mapping.Key);
var subDest = Path.Combine(destination, mapping.Value);
Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component" +
" subfolder {1} with source control version '{2}' to {3}", DownloadType, mapping.Key, versionString, subDest);
DownloadFolder(source, subSource, subDest, versionString, includeFilter, excludeFilter, watermark, force);
}
Logger.Instance().Log(TraceLevel.Info, "{0}: Component {1} download finished successfully", DownloadType, source);
}
else
{
Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component {1} with source control version '{2}' to {3}", DownloadType, source, versionString, destination);
DownloadFolder(source, source, destination, versionString, includeFilter, excludeFilter, watermark, force);
Logger.Instance().Log(TraceLevel.Info, "{0}: Component {1} download finished successfully", DownloadType, source);
}
}
/// <summary>
/// Deletes the specified watermark file.
/// </summary>
/// <param name="wm">The wm.</param>
/// <exception cref="System.ArgumentNullException">wm</exception>
internal void Delete(IDependencyDownloaderWatermark wm)
{
if (null == wm)
{
throw new ArgumentNullException("wm");
}
Delete(GetCacheFilePath(wm.Tags["name"], wm.Tags["version"]));
}
/// <summary>
/// This will remove any previously downloaded local folders, so in case a mapping was changed,
/// the clean will remove the previously mapped folders also.
/// The currently active mapping can be retrieved using the watermark (server -> local path)
/// </summary>
/// <param name="watermark">The watermark to use.</param>
public void RevertDownload(IDependencyDownloaderWatermark watermark)
{
if (null == watermark)
{
throw new ArgumentNullException("watermark");
}
Logger.Instance().Log(TraceLevel.Info, "{0}: Reverting download via watermark ...", DownloadType);
var itemsToClean = watermark.ArtifactsToClean.OrderByDescending(x => x.Length).ToList();
foreach (var localFolder in itemsToClean)
{
// Check local folder
var di = new DirectoryInfo(localFolder);
if (!di.Exists)
{
continue;
}
// Determine workspace
var wi = Workstation.Current.GetLocalWorkspaceInfo(localFolder);
if (wi == null)
{
continue;
}
var localCollection = TfsTeamProjectCollectionFactory.GetTeamProjectCollection(wi.ServerUri);
var localWorkspace = wi.GetWorkspace(localCollection);
var localVcs = localWorkspace.VersionControlServer;
// Clean
// ReSharper disable AccessToForEachVariableInClosure
var mappingsToRemove = localWorkspace.Folders.Where(item => item.LocalItem != null) // ignore cloaked
.Where(mapping => mapping.LocalItem.StartsWith(localFolder, StringComparison.OrdinalIgnoreCase));
// ReSharper restore AccessToForEachVariableInClosure
// Remove outdated mappings
foreach (var mapping in mappingsToRemove)
{
// Get version C1 in order to remove all downloaded files without edits (Conflict resolution: "Keep Local Version")
var getRequest = new GetRequest(mapping.ServerItem, RecursionType.Full, new ChangesetVersionSpec(1));
localVcs.Getting += RemovingHandler;
localWorkspace.Get(getRequest, GetOptions.None);
localVcs.Getting -= RemovingHandler;
// Remove from workspace
localWorkspace.DeleteMapping(mapping);
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Removed workspace folder mapping {1} -> {2}", DownloadType, mapping.ServerItem, mapping.LocalItem);
watermark.ArtifactsToClean.Remove(localFolder);
}
}
}
/// <summary>
/// Reverts an individual component
/// </summary>
/// <param name="dwSvc">The watermark service to use</param>
/// <param name="name">The name of the component</param>
/// <param name="version">The version of the component</param>
internal void RevertComponent(DownloadWatermarkService dwSvc, string name, string version)
{
var dFac = new DownloaderFactory();
IDependencyDownloaderWatermark wm = null;
var wmName = name;
var wmVersion = version;
try
{
wm = dwSvc.Load(name, version);
if (null == wm)
{
// TODO: might consider logging this to a trace file
Debug.Fail(
string.Format(
"Unable to load a watermark for component {0}@{1}. This situation should never occur due to the fact that the same method is reading and writing the watermarks",
name,
version));
}
// Read name and version information from watermark (saving can convert name and version back to hashes).
if (wm.Tags.ContainsKey("name"))
{
wmName = wm.Tags["name"];
}
if (wm.Tags.ContainsKey("version"))
{
wmVersion = wm.Tags["version"];
}
var cleaner = dFac.GetDownloader(wm.DownloadType);
Logger.Instance().Log(TraceLevel.Info, "{0}: Cleaning component {1}#{2} ...", CommandType, wmName, wmVersion);
cleaner.RevertDownload(wm);
dwSvc.Delete(wm);
}
catch
{
if (null != wm)
{
dwSvc.Save(wm, name, version);
}
}
if (!_silentMode)
{
_logger.LogMsg(string.Format(" * Component {0} (Version:{1}) cleaned.", wmName, wmVersion));
}
Logger.Instance().Log(TraceLevel.Info, "{0}: Component {1}#{2} successfully cleaned", CommandType, wmName, wmVersion);
}
/// <summary>
/// Performs a revert operation by removing all the files which have been downloaded previously. The watermark write during the Download will be provided
/// </summary>
/// <param name="watermark">The watermark that has been used for the download operation</param>
public void RevertDownload(IDependencyDownloaderWatermark watermark)
{
if (null == watermark)
{
throw new ArgumentNullException("watermark");
}
Logger.Instance().Log(TraceLevel.Info, "{0}: Reverting download via watermark ...", DownloadType);
// We sort it by length. This means that we delete the deepest files first and we
// hit the directories after all files have been deleted
var itemsToClean = watermark.ArtifactsToClean.OrderByDescending(x => x.Length).ToList();
foreach (var item in itemsToClean)
{
var fi = new FileInfo(item);
if (fi.Exists)
{
fi.Attributes = FileAttributes.Normal;
fi.Delete();
watermark.ArtifactsToClean.Remove(item);
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Deleted file {1}", DownloadType, fi.FullName);
continue;
}
var di = new DirectoryInfo(item);
if (di.Exists)
{
if (!di.EnumerateDirectories().Any() && !di.EnumerateFiles().Any())
{
// Delete empty directories only
di.Delete();
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Deleted empty directory {1}", DownloadType, di.FullName);
}
// But remove from artifacts anyway
watermark.ArtifactsToClean.Remove(item);
}
}
// In the end, we will remove all watermarks in case someone else removed some files before
watermark.ArtifactsToClean.Clear();
watermark.Watermarks.Clear();
Logger.Instance().Log(TraceLevel.Info, "{0}: Reverting download finished successfully", DownloadType);
}
internal void Save(IDependencyDownloaderWatermark wm, string name, string version)
{
if (null == wm)
{
throw new ArgumentNullException("wm");
}
if (string.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (string.IsNullOrEmpty(version))
{
throw new ArgumentNullException("version");
}
// Store name and version as Tags
wm.UpdateTag("name", name);
wm.UpdateTag("version", version);
Store(wm, GetCacheFilePath(name, version));
}
private static void Store(IDependencyDownloaderWatermark watermark, string path)
{
if (null == watermark)
{
throw new ArgumentNullException("watermark");
}
if (null == path)
{
throw new ArgumentNullException("path");
}
if (0 == watermark.ArtifactsToClean.Count && 0 == watermark.Watermarks.Count)
{
File.Delete(path);
return;
}
using (var stream = File.Open(path, FileMode.Create))
{
var serializer = new XmlSerializer(typeof(DownloaderWatermark));
serializer.Serialize(stream, watermark);
}
}
/// <summary>
/// Copy files in directory and calls CopyDirectoryContent recursively for sub directories.
/// </summary>
/// <param name="root">The root of the recursive copy operation.</param>
/// <param name="source">The directory to copy files from.</param>
/// <param name="target">The directory to copy files to.</param>
/// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations.</param>
/// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten.</param>
/// <param name="includeFilters">Filter that defines what files and folders to include.</param>
/// <param name="excludeFilters">Files that should be excluded</param>
private void CopyDirectoryContent(DirectoryInfo root, DirectoryInfo source, DirectoryInfo target, IList <string> includeFilters, IList <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force)
{
if (string.Equals(source.FullName, target.FullName, StringComparison.OrdinalIgnoreCase))
{
return;
}
//Get the folders only, if filters are provided and the list with excluded and included files and folders has not been initalized yet.
// getting the folders only once, will increase the copy speed
if (includeFilters != null || excludeFilters != null)
{
if (_includedFoldersList == null || _excludedFoldersList == null || _includedFilesList == null || _excludedFilesList == null)
{
// explicitly and implicitly included and excluded folders
_includedFoldersList = includeFilters.Where(x => x.EndsWith("\\")).SelectMany(x => GetFoldersFromRoot(x, root)).ToList();
_excludedFoldersList = excludeFilters.Where(x => x.EndsWith("\\")).SelectMany(x => GetFoldersFromRoot(x, root)).ToList();
_includedFoldersList = _includedFoldersList.Concat(includeFilters.Where(x => x.EndsWith("\\")).Select(x => x.Trim('\\')).SelectMany(x => GetFoldersFromRoot(x, root)).ToList()).ToList();
_excludedFoldersList = _excludedFoldersList.Concat(excludeFilters.Where(x => x.EndsWith("\\")).Select(x => x.Trim('\\')).SelectMany(x => GetFoldersFromRoot(x, root)).ToList()).ToList();
//// generally and specifically included and excluded files
_includedFilesList = includeFilters.Where(x => x.EndsWith("\\") == false).SelectMany(x => GetFilesFromRoot(x, root)).ToList();
_excludedFilesList = excludeFilters.Where(x => x.EndsWith("\\") == false).SelectMany(x => GetFilesFromRoot(x, root)).ToList();
//Sort the list to move the root folder first
_includedFoldersList = _includedFoldersList.OrderBy(o => o.FullName.Length).ToList();
if (_excludedFoldersList.Any(x => source.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase)))
{
return;
}
}
}
watermark.ArtifactsToClean.Add(target.FullName);
if (!target.Exists)
{
// Since preliminary deletes might lead to file or folder locks, we need to retry this
var retries = _retryHelper.RetryAction(target.Create, 10, 1000);
if (retries > 0)
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Folder {1} successfully created after {2} retries", DownloadType, target, retries);
}
}
//The Move Operation Type.
if (_operationType == OperationType.Move)
{
MoveFolder(source, target, watermark);
}
else
{
// get excluded and included files
foreach (var sf in source.GetFiles())
{
var isInIncludedFolder = _includedFoldersList.Any(x => sf.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase));
var isInExcludedFolder = _excludedFoldersList.Any(x => sf.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase));
var isIncludedFile = _includedFilesList.Any(x => IsSameFile(x.FullName, sf.FullName));
var isExcludedFile = _excludedFilesList.Any(x => IsSameFile(x.FullName, sf.FullName));
if ((isIncludedFile == false && isInIncludedFolder == false) || isExcludedFile || isInExcludedFolder)
{
continue;
}
var df = new FileInfo(Path.Combine(target.FullName, sf.Name));
var wm = watermark.GetWatermark <DateTime>(sf.FullName);
if (force || wm == DateTime.MinValue || wm != df.LastWriteTimeUtc || !df.Exists)
{
if (df.Exists)
{
df.Attributes = FileAttributes.Normal;
}
var localSourceFile = sf;
watermark.ArtifactsToClean.Add(df.FullName);
var retriesCopy = _retryHelper.RetryAction(() => localSourceFile.CopyTo(df.FullName, true), 5, 1000);
if (retriesCopy > 0)
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully downloaded after {2} retries", DownloadType, df.FullName, retriesCopy);
}
//To determine the the LastWrite time the object must be refreshed after copy the file.
df.Refresh();
watermark.UpdateWatermark(sf.FullName, df.LastWriteTimeUtc);
df.Attributes = FileAttributes.Normal;
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, sf.FullName);
}
else
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, sf.FullName);
}
}
foreach (var subdirectory in source.GetDirectories())
{
// a folder is included if one of its parents is included (unless it is explicitly exluded)
var files = _includedFilesList.Where(i => !_excludedFilesList.Any(e => IsSameFile(e.FullName, i.FullName))).ToList();
var isIncluded =
_includedFoldersList.Any(x => x.FullName.StartsWith(subdirectory.FullName, StringComparison.InvariantCultureIgnoreCase)) ||
!_excludedFoldersList.Any(x => x.FullName.EndsWith(subdirectory.FullName));
if (isIncluded)
{
var targetDi = new DirectoryInfo(Path.Combine(target.FullName, subdirectory.Name));
Logger.Instance()
.Log(TraceLevel.Verbose, "{0}: Downloading folder {1}", DownloadType,
Path.Combine(source.FullName, subdirectory.Name));
CopyDirectoryContent(root, subdirectory, targetDi, includeFilters, excludeFilters, watermark,
force);
}
}
}
}
/// <summary>
/// Downloads a component from source control to a local path.
/// </summary>
/// <param name="source">Path to source location</param>
/// <param name="destination">Path to destination folder</param>
/// <param name="watermark">The watermark to adapt for clean support</param>
/// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten</param>
/// <param name="settings">Setting which contain the version to fetch</param>
public void Download(string source, string destination, IDependencyDownloaderWatermark watermark, bool force, ISettings <DownloaderValidSettings> settings)
{
if (string.IsNullOrEmpty(source))
{
throw new ArgumentNullException(
"source", "Source control path to component folder was empty in DownloaderSourceControl");
}
if (string.IsNullOrEmpty(destination))
{
throw new ArgumentNullException(
"destination", "Destination path for component was empty in DownloaderSourceControl");
}
if (null == watermark)
{
throw new ArgumentNullException("watermark");
}
if (null == settings)
{
throw new ArgumentNullException("settings");
}
var versionString = settings.GetSetting(DownloaderValidSettings.VersionString);
if (string.IsNullOrEmpty(versionString))
{
throw new InvalidComponentException(
"VersionSpec string for component was empty in DownloaderSourceControl");
}
var version = VersionSpec.ParseSingleSpec(versionString, _workspace.OwnerName);
if (version == null)
{
throw new InvalidComponentException("VersionSpec for component was invalid in DownloaderSourceControl");
}
// Adapt workspace mapping
// The mapping we want to end up with: source -> destination
// The currently active workspace mappings targeting our local root paths
var workspaceMappings = _workspace.Folders.Where(item => item.LocalItem != null) // ignore cloaked
.Where(mapping => mapping.LocalItem.StartsWith(destination, StringComparison.OrdinalIgnoreCase));
// Items mapped in earlier runs of this tool, maybe configuration has changed or items no longer belong to the dependencies.
var outdatedMappings =
workspaceMappings.Where(
mapping => !source.Equals(mapping.ServerItem, StringComparison.OrdinalIgnoreCase));
// Map and Get operations
// Remove outdated mappings
foreach (var mapping in outdatedMappings)
{
_workspace.DeleteMapping(mapping);
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Deleted old workspace folder mapping {1} -> {2}", DownloadType, mapping.ServerItem, mapping.LocalItem);
}
// Ensure desired mappings are mapped.
var localPath = _workspace.TryGetLocalItemForServerItem(source);
if (!string.Equals(localPath, destination))
{
_workspace.Map(source, destination);
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Created workspace folder mapping {1} -> {2}", DownloadType, source, destination);
}
// Build a get request for every mapping to ensure we have the desired version. Always need to do a full get since the mapping might
// not have changed but only the label, or we map to latest version and there is a newer version available.
var getRequest = new GetRequest(source, RecursionType.Full, version);
try
{
_vcs.Getting += GettingHandler;
_workspace.Get(getRequest, force ? ForceGetOptions : IncrementalGetOptions);
watermark.ArtifactsToClean.Add(destination);
}
finally
{
_vcs.Getting -= GettingHandler;
}
}
/// <summary>
/// Determines the source, target location and worker settings and initializes download workers according to their type.
/// </summary>
/// <param name="dws">The downloader wartermark service.</param>
/// <param name="component">The component node in graph to start download from.</param>
/// <param name="recursive">Indicates if the dependencies should be fetched recursively or not.</param>
/// <param name="force">If set to <c>true</c> it indicates that the get operation is forced and all files have to be overwritten. Otherwise false.</param>
private void DownloadComponent(DownloadWatermarkService dws, IComponent component, bool recursive, bool force)
{
var df = new DownloaderFactory();
var componentAlreadyDownloaded = false;
var settings = new Settings <DownloaderValidSettings>();
IDependencyDownloader worker;
string destLocation;
string targetLocation;
bool removeTempFiles = false;
Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component {1}#{2} ...", CommandType, component.Name, component.Version);
// Create worker and settings according to type
if (component.Type.Equals(ComponentType.FileShare))
{
var fileShareRootPath = component.GetFieldValue(DependencyProviderValidSettingName.FileShareRootPath);
destLocation = Path.Combine(fileShareRootPath, component.Name.Path, component.Version.Version);
var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath);
targetLocation =
relativeTargetPath == null
? Path.GetFullPath(Path.Combine(_defaultDownloadFolder, _defaultRelativeOutputPath))
: Path.GetFullPath(Path.Combine(_defaultDownloadFolder, relativeTargetPath));
LoadFilterSettings(component, settings);
}
else if (component.Type.Equals(ComponentType.VNextBuildResult))
{
var url = component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl);
if (!Uri.IsWellFormedUriString(url, UriKind.Absolute))
{
Logger.Instance().Log(
TraceLevel.Error,
"{0}: Invalid BuildTeamProjectCollectionUrl setting '{1}' was found for component {2}#{3}",
CommandType,
url,
component.Name,
component.Version);
throw new DependencyServiceException(
string.Format(
" ! Invalid download information was found for build result control component {0} (BuildTeamProjectCollectionUrl setting)",
component.Name.GetName()));
}
// Connect to tfs server
var tpc = TfsTeamProjectCollectionFactory.GetTeamProjectCollection(new Uri(url));
tpc.EnsureAuthenticated();
// Connect to version control service
var versionControl = tpc.GetService <VersionControlServer>();
if (versionControl == null)
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Connection to version control server failed for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format(" ! Could not connect to TFS version control server for team project collection {0}", component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl)));
}
var tpcUrl = new Uri(url);
var connection = new VssConnection(tpcUrl, new VssClientCredentials(true));
var client = connection.GetClient <BuildHttpClient>();
var builds = client.GetBuildsAsync(project: component.Name.TeamProject, type: DefinitionType.Build).Result;
var buildResult = builds.FirstOrDefault(r => r.BuildNumber == component.Version.BuildNumber);
if (buildResult == null)
{
Logger.Instance().Log(TraceLevel.Error, "{0}: No vnext build result could not be determined for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format(" ! Could not determine build result component {0}", component.Name.GetName()));
}
var artifacts = client.GetArtifactsAsync(component.Name.TeamProject, buildResult.Id).Result;
destLocation = Path.Combine(Path.GetTempPath(), "VNextDM_" + Guid.NewGuid());
Directory.CreateDirectory(destLocation);
removeTempFiles = true;
foreach (var artifact in artifacts)
{
if (artifact.Resource.Type == "FilePath")
{
var sourceDirName = $"{artifact.Resource.Data}/{artifact.Name}";
DirectoryCopy(sourceDirName, destLocation, true);
}
else
{
var content = client.GetArtifactContentZipAsync(component.Name.TeamProject, buildResult.Id, artifact.Name);
using (var zipArchive = new ZipArchive(content.Result))
{
zipArchive.ExtractToDirectory(destLocation);
}
}
}
var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath);
targetLocation =
relativeTargetPath == null
? Path.GetFullPath(Path.Combine(_defaultDownloadFolder, _defaultRelativeOutputPath))
: Path.GetFullPath(Path.Combine(_defaultDownloadFolder, relativeTargetPath));
LoadFilterSettings(component, settings);
}
else if (component.Type.Equals(ComponentType.BuildResult))
{
if (!Uri.IsWellFormedUriString(component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl), UriKind.Absolute))
{
Logger.Instance().Log(
TraceLevel.Error,
"{0}: Invalid BuildTeamProjectCollectionUrl setting '{1}' was found for component {2}#{3}",
CommandType,
component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl),
component.Name,
component.Version);
throw new DependencyServiceException(
string.Format(
" ! Invalid download information was found for build result control component {0} (BuildTeamProjectCollectionUrl setting)",
component.Name.GetName()));
}
// Connect to tfs server
var tpc = TfsTeamProjectCollectionFactory.GetTeamProjectCollection(new Uri(component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl)));
tpc.EnsureAuthenticated();
// Connect to version control service
var versionControl = tpc.GetService <VersionControlServer>();
if (versionControl == null)
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Connection to version control server failed for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format(" ! Could not connect to TFS version control server for team project collection {0}", component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl)));
}
// Connect to build server
var buildServer = tpc.GetService <IBuildServer>();
if (buildServer == null)
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Connection to build server failed for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format(" ! Could not connect to TFS build server for team project collection {0}", component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl)));
}
var buildDef = buildServer.GetBuildDefinition(component.Name.TeamProject, component.Name.BuildDefinition);
var buildDetailSpec = buildServer.CreateBuildDetailSpec(buildDef);
buildDetailSpec.BuildNumber = component.Version.BuildNumber;
buildDetailSpec.InformationTypes = new string[] { };
var buildResult = buildServer.QueryBuilds(buildDetailSpec);
if (buildResult == null || buildResult.Builds == null || buildResult.Builds.Length == 0)
{
Logger.Instance().Log(TraceLevel.Error, "{0}: No build result could not be determined for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format(" ! Could not determine drop location for build result component {0}", component.Name.GetName()));
}
// Determine source location
var dropLocation = buildResult.Builds[0].DropLocation; // TODO dga: Is this a bug? it returns localhost for the computer name while we are not running this code on the server (I guess)?
if (string.IsNullOrEmpty(dropLocation))
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Drop location could not be determined for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format("! Could not determine drop location for build result component {0}", component.Name.GetName()));
}
destLocation = dropLocation;
var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath);
targetLocation =
relativeTargetPath == null
? Path.GetFullPath(Path.Combine(_defaultDownloadFolder, _defaultRelativeOutputPath))
: Path.GetFullPath(Path.Combine(_defaultDownloadFolder, relativeTargetPath));
LoadFilterSettings(component, settings);
}
else if (component.Type.Equals(ComponentType.SourceControl))
{
if (!Uri.IsWellFormedUriString(component.GetFieldValue(DependencyProviderValidSettingName.WorkspaceTeamProjectCollectionUrl), UriKind.Absolute))
{
Logger.Instance().Log(
TraceLevel.Error,
"{0}: Invalid WorkspaceTeamProjectCollectionUrl setting '{1}' was found for component {2}#{3}",
CommandType,
component.GetFieldValue(DependencyProviderValidSettingName.WorkspaceTeamProjectCollectionUrl),
component.Name,
component.Version);
throw new DependencyServiceException(
string.Format(
" ! Invalid download information was found for source control component {0} (WorkspaceTeamProjectCollectionUrl setting)",
component.Name.GetName()));
}
var workspaceName = component.GetFieldValue(DependencyProviderValidSettingName.WorkspaceName);
if (string.IsNullOrEmpty(workspaceName))
{
Logger.Instance().Log(TraceLevel.Error, "{0}: WorkspaceName setting was not specified for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(
string.Format(
" ! Invalid download information was found for source control component {0} (WorkspaceName setting)",
component.Name.GetName()));
}
var workspaceOwner = component.GetFieldValue(DependencyProviderValidSettingName.WorkspaceOwner);
if (string.IsNullOrEmpty(workspaceOwner))
{
Logger.Instance().Log(TraceLevel.Error, "{0}: WorkspaceOwner setting was not specified for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(
string.Format(
" ! Invalid download information was found for source control component {0} (WorkspaceOwner setting)",
component.Name.GetName()));
}
destLocation = component.Name.ToString();
settings.AddSetting(new KeyValuePair <DownloaderValidSettings, string>(DownloaderValidSettings.VersionString, component.Version.ToString()));
var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath);
targetLocation =
Path.GetFullPath(
relativeTargetPath == null
? Path.Combine(_defaultDownloadFolder, "..", VersionControlPath.GetFileName(component.Name.ToString()))
: Path.Combine(_defaultDownloadFolder, relativeTargetPath));
}
else if (component.Type.Equals(ComponentType.SourceControlCopy))
{
destLocation = component.Name.ToString();
settings.AddSetting(new KeyValuePair <DownloaderValidSettings, string>(DownloaderValidSettings.VersionString, component.Version.ToString()));
var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath);
targetLocation =
Path.GetFullPath(
relativeTargetPath == null
? Path.Combine(_defaultDownloadFolder, "..", VersionControlPath.GetFileName(component.Name.ToString()))
: Path.Combine(_defaultDownloadFolder, relativeTargetPath));
LoadFilterSettings(component, settings);
}
else if (component.Type.Equals(ComponentType.BinaryRepository))
{
var repositoryTeamProject = component.GetFieldValue(DependencyProviderValidSettingName.BinaryRepositoryTeamProject);
if (string.IsNullOrEmpty(repositoryTeamProject))
{
Logger.Instance().Log(TraceLevel.Error, "{0}: BinaryRepositoryTeamProject setting was not specified for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(
string.Format(
" ! Invalid download information was found for binary repository component {0} (BinaryRepositoryTeamProject setting)",
component.Name.GetName()));
}
destLocation = VersionControlPath.Combine(VersionControlPath.Combine(VersionControlPath.Combine(VersionControlPath.RootFolder, repositoryTeamProject), component.Name.GetName()), component.Version.GetVersion());
settings.AddSetting(new KeyValuePair <DownloaderValidSettings, string>(DownloaderValidSettings.VersionString, VersionSpec.Latest.DisplayString));
var relativeOutputPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath);
targetLocation = Path.GetFullPath(
relativeOutputPath == null
? Path.Combine(_defaultDownloadFolder, _defaultRelativeOutputPath)
: Path.Combine(_defaultDownloadFolder, relativeOutputPath));
LoadFilterSettings(component, settings);
}
else if (component.Type.Equals(ComponentType.Subversion))
{
destLocation = string.Format("{0}", component.Name);
settings.AddSetting(new KeyValuePair <DownloaderValidSettings, string>(DownloaderValidSettings.VersionString, component.Version.ToString()));
var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath);
targetLocation =
relativeTargetPath == null
? Path.GetFullPath(Path.Combine(_defaultDownloadFolder, _defaultRelativeOutputPath))
: Path.GetFullPath(Path.Combine(_defaultDownloadFolder, relativeTargetPath));
LoadFilterSettings(component, settings);
}
else
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Unknown dependency type '{1}' was found in dependency graph", CommandType, component.Type.GetType());
throw new DependencyServiceException(" ! Invalid dependency node found in graph!");
}
//Determine Multi-Site path for BuildResult and FileShare-Provider
if (component.Type.Equals(ComponentType.FileShare) || (component.Type.Equals(ComponentType.BuildResult)))
{
destLocation = DetermineMultiSiteSourceLocation(destLocation);
}
if (string.IsNullOrEmpty(destLocation))
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Source location was not set for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format(" ! Error occured while preparing to download component {0} (Source location was not set)", component.Name.GetName()));
}
if (string.IsNullOrEmpty(targetLocation))
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Target location was not set for component {1}#{2}", CommandType, component.Name, component.Version);
throw new DependencyServiceException(string.Format(" ! Error occured while preparing to download component {0} (Destination location was not set)", component.Name.GetName()));
}
// Download files
try
{
IDependencyDownloaderWatermark wm = null;
try
{
worker = df.GetDownloader(component);
wm = dws.Load(component.Name.GetName(), component.Version.GetVersion()) ?? new DownloaderWatermark(worker);
worker.Download(destLocation, targetLocation, wm, force, settings);
}
finally
{
if (null != wm)
{
dws.Save(wm, component.Name.GetName(), component.Version.GetVersion());
}
if (removeTempFiles)
{
Directory.Delete(destLocation, true);
}
}
}
catch (Exception e)
{
if (e is DependencyServiceException)
{
throw;
}
// ReSharper disable RedundantIfElseBlock
else if (e is ComponentAlreadyDownloadedException)
// ReSharper restore RedundantIfElseBlock
{
componentAlreadyDownloaded = true;
}
else
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Exception {1} occured while downloading files: {2}", CommandType, e, e.Message);
throw new DependencyServiceException(string.Format(" ! Download of component {0} failed ({1})", component.Name.GetName(), e.Message));
}
}
// Log download
if (!_silentMode)
{
_logger.LogMsg(
!componentAlreadyDownloaded
? string.Format(
" * Component {0} (Version:{1}) downloaded to target directory {2}",
component.Name.GetName(),
component.Version.GetVersion(),
targetLocation)
: string.Format(
" * Skipped component {0} (Version:{1}). Already present in target directory {2}",
component.Name.GetName(),
component.Version.GetVersion(),
targetLocation));
}
Logger.Instance().Log(
TraceLevel.Info,
!componentAlreadyDownloaded
? "{0}: Component {1}#{2} successfully downloaded to target directory {3}"
: "{0}: Component {1}#{2} download was skipped. Already present in target directory {3}",
CommandType,
component.Name,
component.Version,
targetLocation);
}
/// <summary>
/// Extracts the compressed files in directory and recursively in subdirectories.
/// Searches for zipped and 7-zipped files (LZMA)
/// </summary>
/// <param name="source">The directory with compressed files</param>
/// <param name="target">The target direcotry of the zip files content</param>
/// <param name="watermark">The watermark which is be used to perform incremental updates and cleanup operations.</param>
/// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten.</param>
private void ExtractCompressedFiles(string source, string target, IDependencyDownloaderWatermark watermark, bool force)
{
string[] allowedExtensions = { ".zip", ".7z" };
foreach (var compressedFile in Directory.EnumerateFiles(source, "*.*", SearchOption.AllDirectories)
.Where(s => allowedExtensions.Any(ext => ext == Path.GetExtension(s))))
{
Logger.Instance().Log(TraceLevel.Info, "Start extracting compressed file {0}.", compressedFile);
try
{
// regard if the zip file is in a subdirectory
var targetSubdir = compressedFile.Replace(source, string.Empty);
if (targetSubdir.Contains("\\"))
{
targetSubdir = targetSubdir.Substring(0, targetSubdir.LastIndexOf("\\", StringComparison.Ordinal));
// cut the leading Backslash "\", if one exists
if ((targetSubdir.StartsWith("\\")) && (targetSubdir.Length > 0))
{
targetSubdir = targetSubdir.Substring(1);
}
}
else
{
// contains only the filename of the zip archive
targetSubdir = string.Empty;
}
//If Sevenzip is supported, extract and zip file also with sevenzip
if (compressedFile.EndsWith(".zip"))
{
if (string.IsNullOrWhiteSpace(ResolveSevenZipExecutable()))
{
ExtractStandardZipFile(target, targetSubdir, compressedFile, watermark, force);
}
else
{
ExtractSevenZipFile(target, targetSubdir, compressedFile, watermark, force);
}
}
else if (compressedFile.EndsWith(".7z"))
{
ExtractSevenZipFile(target, targetSubdir, compressedFile, watermark, force);
}
else
{
throw new Exception("The file you are trying to extract is not supported ");
}
//Delete
if (_isDeleteArchiveFiles)
{
DeleteArchiveFile(compressedFile);
}
}
catch (InvalidDataException)
{
Logger.Instance().Log(TraceLevel.Warning, "Failed to extract file {0}. Format not supported!", compressedFile);
throw new Exception();
}
}
}
/// <summary>
/// Downloads a component to a local path.
/// </summary>
/// <param name="source">The path to source folder.</param>
/// <param name="destination">The path to destination folder.</param>
/// <param name="watermark">The watermark which is be used to perform incremental updates and cleanup operations.</param>
/// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten.</param>
/// <param name="settings">The settings which include the pattern for files/directories to include and folder mappings.</param>
public void Download(string source, string destination, IDependencyDownloaderWatermark watermark, bool force, ISettings <DownloaderValidSettings> settings)
{
if (string.IsNullOrEmpty(source))
{
throw new ArgumentNullException("source");
}
if (string.IsNullOrEmpty(destination))
{
throw new ArgumentNullException("destination");
}
if (watermark == null)
{
throw new ArgumentNullException("watermark");
}
if (settings == null)
{
throw new ArgumentNullException("settings");
}
var versionSpec = settings.GetSetting(DownloaderValidSettings.VersionString);
if (string.IsNullOrEmpty(versionSpec))
{
throw new InvalidComponentException("Version string for component was empty in DownloaderSubversion");
}
try
{
if (!ProviderSubversion.Instance.ItemExists(source, versionSpec))
{
Logger.Instance().Log(TraceLevel.Error, "{0}: Folder not exists in Subversion {1}", DownloadType, source);
throw new InvalidProviderConfigurationException(string.Format("Could not connect to Subversion {0}", source));
}
}
catch (SvnAuthenticationException)
{
throw new SvnAuthenticationException();
}
Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component {1} to {2} ...", DownloadType, source, destination);
var folderMappings = InitializeMappings(settings.GetSetting(DownloaderValidSettings.FolderMappings));
var includeFilter = InitializeIncludeFilter(settings.GetSetting(DownloaderValidSettings.IncludedFilesFilter));
var excludeFilter = InitializeExcludeFilter(settings.GetSetting(DownloaderValidSettings.ExcludedFilesFilter));
if (folderMappings.Any())
{
foreach (var mapping in folderMappings)
{
//var subSource = VersionControlPath.Combine(source, mapping.Key);
var subSource = string.Format("{0}/{1}", source, mapping.Key);
var subDest = Path.Combine(destination, mapping.Value);
Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component subfolder {1} with source control version to folder {2}", DownloadType, mapping.Key, subDest);
Export(source, subSource, subDest, versionSpec, includeFilter, excludeFilter, watermark, force);
}
Logger.Instance().Log(TraceLevel.Info, "{0}: Component {1} download finished successfully", DownloadType, source);
}
else
{
Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component {1} to folder {2}", DownloadType, source, destination);
Export(source, source, destination, versionSpec, includeFilter, excludeFilter, watermark, force);
Logger.Instance().Log(TraceLevel.Info, "{0}: Component {1} download finished successfully", DownloadType, source);
}
}
private void Export(string root, string source, string destination, string versionSpec, IEnumerable <string> includeFilters, IEnumerable <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force)
{
var includes = ConvertFilterToRegex(includeFilters, root);
var excludes = ConvertFilterToRegex(excludeFilters, root);
var items = new Dictionary <string, string>();
var allItems = ProviderSubversion.Instance.GetItems(source, ProviderSubversion.ItemType.File, true, versionSpec);
// apply include and exclude filter
foreach (var item in allItems)
{
if ((includes.Any(x => x.IsMatch(item.Key))) && (!excludes.Any(x => x.IsMatch(item.Key))))
{
items.Add(item.Key, item.Value);
}
}
var revision = 0L;
foreach (var item in items)
{
var destinationpath = destination;
var subpath = item.Value.Substring(source.Length).Trim(new[] { '/' }).Replace("/", "\\");
//Remove filename
if (subpath.Contains("\\"))
{
subpath = subpath.Remove(subpath.LastIndexOf("\\"));
destinationpath = Path.Combine(destination, subpath);
}
// Determine revision one-time. All items have same revision.
if (revision == 0)
{
revision = ProviderSubversion.Instance.GetHeadRevision(item.Key);
}
var wm = watermark.GetWatermark <long>(item.Key);
var targetFile = Path.Combine(destinationpath, Path.GetFileName(item.Value));
if (force || 0 == wm || revision != wm || !File.Exists(targetFile))
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, item.Key);
ProviderSubversion.Instance.GetFile(item.Key, destinationpath, force, versionSpec);
//Save files and folders, which included in the item
watermark.ArtifactsToClean.Add(destinationpath);
foreach (var file in Directory.GetFiles(destinationpath))
{
watermark.ArtifactsToClean.Add(file);
}
//Update information about source (path and revision)
watermark.UpdateWatermark(item.Key, revision);
}
else
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, item);
}
}
}
public void Download(string source, string destination, IDependencyDownloaderWatermark watermark, bool force, ISettings <DownloaderValidSettings> settings)
{
if (source == null)
{
throw new ArgumentNullException("source");
}
if (destination == null)
{
throw new ArgumentNullException("destination");
}
if (watermark == null)
{
throw new ArgumentNullException("watermark");
}
if (settings == null)
{
throw new ArgumentNullException("settings");
}
Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component {1} to {2} ...", DownloadType, source, destination);
// Save compressed files to temp path and temporarily store watermarks. Turn of mappings as they only apply to the extracted folder structure
var tempPath = Path.Combine(destination, "..\\", "DM_TEMP");
//Create the temp Path
if (!Directory.Exists(tempPath))
{
DirectoryInfo di = Directory.CreateDirectory(tempPath);
}
//save the folder mapping
var tempFolderMapping = settings.GetSetting(DownloaderValidSettings.FolderMappings);
var remainingArtefacts = new HashSet <string>(watermark.ArtifactsToClean);
//Download the zip files and the rest of the contents ignore any mappings to get everything in the temppath
settings.SettingsDictionary[DownloaderValidSettings.FolderMappings] = null;
_downloader
.Download(source, tempPath, watermark, force, settings);
//var zipWatermarks = new Dictionary<string, object>(watermark.Watermarks);
// The DownloaderFileShareCopy / DownloaderSourceControlCopy creates artifacts for cleanup,
// which are deleted by the ZippedDependencyDownloader and must not be saved.
watermark.ArtifactsToClean = remainingArtefacts;
//Extract the zipped files in the temp path
ExtractCompressedFiles(tempPath, tempPath, watermark, force);
//Set the settings to a move operation
settings.AddSetting(new KeyValuePair <DownloaderValidSettings, string>(DownloaderValidSettings.OperationType, "Move"));
//Set the temporary saved mapping again
settings.SettingsDictionary[DownloaderValidSettings.FolderMappings] = tempFolderMapping;
//The SourceControlCopy-Provider needs the FileShareCopy-Provider to copy the extracted files from the temp folder to the
//target folder, because the SourceControlCopy-Provider works not with local folders as source folder
if (_downloader.DownloadType == "Downloader_SourceControlCopy")
{
IDependencyDownloader tmpDownloader = new DownloaderFileShareCopy();
tmpDownloader.Download(tempPath, destination, watermark, force, settings);
}
// Provider: FileShare, BuildResult
else
{
//Copy the files from the temp path to the final destination
_downloader.Download(tempPath, destination, watermark, force, settings);
}
//Cleanup: Delete the temp path
try
{
DeleteTempDirectory(tempPath);
}
catch (Exception e)
{
Logger.Instance().Log(TraceLevel.Error, "Deleting temp folder {0} did not succeed: {1}", tempPath, e.Message);
Logger.Instance().Log(TraceLevel.Error, "Stacktrace: {0}", e.StackTrace);
}
}
/// <summary>
/// Performs a revert operation by removing all the files which have been downloaded previously. The watermark written during the Download will be provided
/// </summary>
/// <param name="watermark">The watermark that has been used for the download operation</param>
public void RevertDownload(IDependencyDownloaderWatermark watermark)
{
// Todo MRI: Refactor to use PluginFactory
(new DownloaderFileShareCopy()).RevertDownload(watermark);
}
/// <summary>
/// Extract a standard zip file
/// </summary>
/// <param name="target"></param>
/// <param name="targetSubdir"></param>
/// <param name="archiveFile"></param>
/// <param name="watermark"></param>
/// <param name="force"></param>
private void ExtractStandardZipFile(string target, string targetSubdir, string archiveFile, IDependencyDownloaderWatermark watermark, bool force)
{
using (var archive = ZipFile.OpenRead(archiveFile))
{
foreach (var file in archive.Entries)
{
// Extract only files, not subdirectories. If a subdirectory not exists, create it;
if (file.FullName.EndsWith("/"))
{
var subdir = Path.Combine(target, targetSubdir, file.FullName.Substring(0, file.FullName.Length - 1));
subdir = subdir.Replace("/", "\\");
if (!Directory.Exists(subdir))
{
Directory.CreateDirectory(subdir);
// For the CreateDirectory method the folder depth its not relevant, but for the cleanup, so each subfolder must
// be logged separately
if (FolderDepth(subdir) > FolderDepth(target) + 1)
{
var tmp = subdir;
do
{
watermark.ArtifactsToClean.Add(tmp);
tmp = tmp.Substring(0, tmp.LastIndexOf("\\"));
} while (tmp.Length > target.Length);
}
}
continue;
}
var df = new FileInfo(Path.Combine(target, targetSubdir, file.FullName));
var wm = watermark.GetWatermark <DateTime>(file.FullName);
if (force || wm == DateTime.MinValue || wm != df.LastWriteTimeUtc || !df.Exists)
{
// Remove possible readonly flag
if (df.Exists)
{
df.Attributes = FileAttributes.Normal;
}
watermark.ArtifactsToClean.Add(df.FullName);
try
{
file.ExtractToFile(df.FullName, true);
}
catch (Exception ex)
{
Logger.Instance().Log(TraceLevel.Error, "{0}: The archive cannot be extracted. The compression method is not supported. {1}", ex.Message, file.FullName);
throw new Exception(String.Format("The file {0} cannot be extracted. If the file is compressed with a non-standard algorithm (e.g. LZMA), make sure the support for 7-zip is enabled and all dlls are set properly in the Dependecy Manager settings.", archiveFile));
}
df.Attributes = FileAttributes.Normal;
//TODO: Klaus will auch die enthaltenen Datien im Output Window sehen!
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, file.FullName);
}
else
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, file.FullName);
}
}
}
Logger.Instance().Log(TraceLevel.Info, "Extracted zipped file {0} finished successfully.", archiveFile);
}
/// <summary>
/// Seven zip (lzma) support for the DM. The libraries for sevenzip sharp are referenced dynamically
/// </summary>
/// <param name="target"></param>
/// <param name="targetSubdir"></param>
/// <param name="archiveFile"></param>
/// <param name="watermark"></param>
/// <param name="force"></param>
private void ExtractSevenZipFile(string target, string targetSubdir, string archiveFile, IDependencyDownloaderWatermark watermark, bool force)
{
var sevenZipExecutable = ResolveSevenZipExecutable();
if (string.IsNullOrWhiteSpace(sevenZipExecutable))
{
Logger.Instance().Log(TraceLevel.Warning, "Unable to resolve the path to the 7 zip executable. Make sure that 7 zip is installed and that the dependency manager is configured properly");
Logger.Instance().Log(TraceLevel.Warning, "Your dependecy contains *.7z archives. Please make sure the SevenZip support is enabled in the Dependency Manager settings.");
return;
}
var process = new Process();
process.StartInfo = new ProcessStartInfo();
process.StartInfo.FileName = sevenZipExecutable;
process.StartInfo.Arguments = string.Format("x \"{0}\" -y", archiveFile);
process.StartInfo.WindowStyle = ProcessWindowStyle.Hidden;
process.StartInfo.UseShellExecute = false;
process.StartInfo.CreateNoWindow = true;
process.StartInfo.RedirectStandardError = true;
process.StartInfo.RedirectStandardOutput = true;
process.StartInfo.WorkingDirectory = Path.Combine(target, targetSubdir);
process.Start();
process.OutputDataReceived += (x, y) =>
{
if (!string.IsNullOrWhiteSpace(y.Data) && y.Data.StartsWith("Extracting", StringComparison.OrdinalIgnoreCase))
{
var file = y.Data.Replace("Extracting", string.Empty).Trim();
var fullName = Path.Combine(target, file);
watermark.ArtifactsToClean.Add(fullName);
}
};
process.BeginOutputReadLine();
process.WaitForExit();
if (0 != process.ExitCode)
{
//Todo Replace by other and more specific exception
Logger.Instance().Log(TraceLevel.Error, "An error occured while executing {0} {1}. The exit code is {2}", process.StartInfo.FileName, process.StartInfo.Arguments, process.ExitCode);
throw new Exception(string.Format("An error occured while executing {0} {1}. The exit code is {2}", process.StartInfo.FileName, process.StartInfo.Arguments, process.ExitCode));
}
Logger.Instance().Log(TraceLevel.Info, "Extracted zipped file {0} finished successfully.", archiveFile);
}
/// <summary>
/// Recursive method to move a folder
/// A folder is moved by moving its subfolders and all files within the folder
/// If the target folder already exsist the method checks if the children of the target folder are the same.
/// It will delete existing folders and will then move them to the folder
///
/// </summary>
/// <param name="source"></param>
/// <param name="target"></param>
/// <param name="watermark"></param>
private void MoveFolder(DirectoryInfo source, DirectoryInfo target, IDependencyDownloaderWatermark watermark)
{
//Move the content of the current folder to the given output folder
foreach (string tempPath in Directory.GetDirectories(source.FullName, "*", SearchOption.TopDirectoryOnly))
{
////Check if the current folder exists on the target
//if (Directory.Exists(tempPath.Replace(source.FullName, target.FullName)))
//{
// //Check if the target contains any children. If there are no children, --> delete the folder
// //If the folder has children, check if the target children are equal to the source children --> If they are equal --> Delete
// var children = Directory.GetDirectories(tempPath.Replace(source.FullName, target.FullName), "*", SearchOption.TopDirectoryOnly);
// if (children.Count() == 0)
// {
// DeleteDirectory(tempPath.Replace(source.FullName, target.FullName));
// }
// else
// {
// //Delete each path that is similar within the children
// foreach (var tempPathChildren in Directory.GetDirectories(tempPath, "*", SearchOption.TopDirectoryOnly))
// {
// if (Directory.Exists(tempPathChildren.Replace(source.FullName, target.FullName)))
// {
// DeleteDirectory(tempPathChildren.Replace(source.FullName, target.FullName));
// }
// }
// }
//}
//If the target folder still exists, the children and files of the source can be moved
//If the target does not exsist, the whole source folder can be moved
if (Directory.Exists(tempPath.Replace(source.FullName, target.FullName)))
{
MoveFolder(new DirectoryInfo(tempPath), new DirectoryInfo(tempPath.Replace(source.FullName, target.FullName)), watermark);
}
else
{
var retriesMove = _retryHelper.RetryAction(() => Directory.Move(tempPath, tempPath.Replace(source.FullName, target.FullName)), 5, 1000);
if (retriesMove > 0)
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully moved after {2} retries", DownloadType, tempPath, retriesMove);
}
}
//Get the file info and set the FileAttributes to Normal
var fi = new DirectoryInfo(tempPath.Replace(source.FullName, target.FullName));
fi.Attributes = FileAttributes.Normal;
//Add all files in the folder to the watermarks
foreach (var file in fi.GetFiles("*.*", SearchOption.AllDirectories))
{
//Add the files to the watermark
watermark.UpdateWatermark(file.FullName, file.LastWriteTimeUtc);
}
}
//Move all the files & Replaces any files with the same name by deleting them
foreach (string tempPath in Directory.GetFiles(source.FullName, "*.*", SearchOption.TopDirectoryOnly))
{
if (File.Exists(tempPath.Replace(source.FullName, target.FullName)))
{
File.SetAttributes(tempPath.Replace(source.FullName, target.FullName), FileAttributes.Normal);
File.Delete(tempPath.Replace(source.FullName, target.FullName));
}
var retriesMove = _retryHelper.RetryAction(() => File.Move(tempPath, tempPath.Replace(source.FullName, target.FullName)), 5, 1000);
if (retriesMove > 0)
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully moved after {2} retries", DownloadType, tempPath, retriesMove);
}
//Get the file info and set the FileAttributes to Normal
var fi = new FileInfo(tempPath.Replace(source.FullName, target.FullName));
fi.Attributes = FileAttributes.Normal;
//Add the files to the watermark
watermark.UpdateWatermark(fi.FullName, fi.LastWriteTimeUtc);
}
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Moving Directory {1} to {2}", DownloadType, source.FullName, target.FullName);
}
/// <summary>
/// Downloads a component to a local path.
/// </summary>
/// <param name="source">The path to source folder.</param>
/// <param name="destination">The path to destination folder.</param>
/// <param name="watermark">The watermark which is be used to perform incremental updates and cleanup operations.</param>
/// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten.</param>
/// <param name="settings">The settings which include the pattern for files/directories to include and folder mappings.</param>
public void Download(string source, string destination, IDependencyDownloaderWatermark watermark, bool force, ISettings <DownloaderValidSettings> settings)
{
if (string.IsNullOrEmpty(source))
{
throw new ArgumentNullException("source");
}
if (string.IsNullOrEmpty(destination))
{
throw new ArgumentNullException("destination");
}
if (watermark == null)
{
throw new ArgumentNullException("watermark");
}
if (settings == null)
{
throw new ArgumentNullException("settings");
}
if (!Directory.Exists(source))
{
throw new DirectoryNotFoundException(string.Format("The directory {0} does not exit", source));
}
Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component {1} to {2} ...", DownloadType, source, destination);
var sourceDi = new DirectoryInfo(source);
var destinationDi = new DirectoryInfo(destination);
var folderMappings = InitializeMappings(settings.GetSetting(DownloaderValidSettings.FolderMappings));
var filters = InitializeIncludeFilter(settings.GetSetting(DownloaderValidSettings.IncludedFilesFilter));
var excludedFiles = InitializeExcludeFilter(settings.GetSetting(DownloaderValidSettings.ExcludedFilesFilter));
_operationType = InitializeOperationType(settings.GetSetting(DownloaderValidSettings.OperationType));
// Reset the list of excluded and included files when a new download occurs
_includedFoldersList = null;
_excludedFoldersList = null;
_includedFilesList = null;
_excludedFilesList = null;
// ReSharper disable PossibleMultipleEnumeration
if (folderMappings.Any())
// ReSharper restore PossibleMultipleEnumeration
{
// ReSharper disable PossibleMultipleEnumeration
foreach (var mapping in folderMappings)
// ReSharper restore PossibleMultipleEnumeration
{
var subSourceDi = new DirectoryInfo(Path.Combine(sourceDi.ToString(), mapping.Key));
var subDestDi = new DirectoryInfo(Path.Combine(destinationDi.ToString(), mapping.Value));
Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component subfolder {1} to {2}", DownloadType, mapping.Key, subDestDi);
CopyDirectoryContent(sourceDi, subSourceDi, subDestDi, filters, excludedFiles, watermark, force);
}
Logger.Instance().Log(TraceLevel.Info, "{0}: Component {1} download finished successfully", DownloadType, source);
}
else
{
CopyDirectoryContent(sourceDi, sourceDi, destinationDi, filters, excludedFiles, watermark, force);
Logger.Instance().Log(TraceLevel.Info, "{0}: Component {1} download finished successfully", DownloadType, source);
}
}
/// <summary>
/// Downloads all items in current folder and items in subfolders by calling itself.
/// </summary>
/// <param name="root">Root of copy operation</param>
/// <param name="source">A valid TFS version control path to a folder that shall be downloaded.</param>
/// <param name="destination">A valid directory to which all the files will be downloaded to.</param>
/// <param name="versionString">The version string to determine the version to download.</param>
/// <param name="includeFilters">The filter for file types to include.</param>
/// <param name="excludeFilters">List of filters to exclude files and folders.</param>
/// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations.</param>
/// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten</param>
private void DownloadFolder(string root, string source, string destination, string versionString, IEnumerable <string> includeFilters, IEnumerable <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force)
{
var version = VersionSpec.ParseSingleSpec(versionString, _workspace.OwnerName);
var includes = ConvertFilterToRegex(includeFilters, root);
var excludes = ConvertFilterToRegex(excludeFilters, root);
var items = _vcs.GetItems(source, version, RecursionType.Full, DeletedState.NonDeleted, ItemType.File, true).Items;
items = items.Where(x => includes.Any(y => y.IsMatch(x.ServerItem))).ToArray();
items = items.Where(x => !excludes.Any(y => y.IsMatch(x.ServerItem))).ToArray();
Parallel.ForEach(items, new ParallelOptions {
MaxDegreeOfParallelism = 8
}, item =>
{
var subpath = item.ServerItem.Substring(source.Length).Trim(new[] { '/' }).Replace("//", "\\");
var destinationpath = Path.Combine(destination, subpath);
var wm = watermark.GetWatermark <int>(item.ServerItem);
if (force || 0 == wm || item.ChangesetId != wm || !File.Exists(destinationpath))
{
if (File.Exists(destinationpath))
{
File.SetAttributes(destinationpath, FileAttributes.Normal);
}
lock (this)
{
watermark.ArtifactsToClean.Add(destinationpath);
}
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, item.ServerItem);
item.DownloadFile(destinationpath);
lock (this)
{
watermark.UpdateWatermark(item.ServerItem, item.ChangesetId);
}
}
else
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, item.ServerItem);
}
});
}
