/// <summary> /// Downloads a component from source control to a local path. /// </summary> /// <param name="source">Path to source location</param> /// <param name="destination">Path to destination folder</param> /// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations</param> /// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten</param> /// <param name="settings">Setting which contain the version to fetch</param> public void Download(string source, string destination, IDependencyDownloaderWatermark watermark, bool force, ISettings <DownloaderValidSettings> settings) { if (string.IsNullOrEmpty(source)) { throw new InvalidComponentException( "Source control path to component folder was empty in DownloaderSourceControlCopy"); } if (string.IsNullOrEmpty(destination)) { throw new InvalidComponentException( "Destination path for component was empty in DownloaderSourceControlCopy"); } if (null == watermark) { throw new ArgumentNullException("watermark"); } if (null == settings) { throw new ArgumentNullException("settings"); } var versionString = settings.GetSetting(DownloaderValidSettings.VersionString); if (string.IsNullOrEmpty(versionString)) { throw new InvalidComponentException( "Version string for component was empty in DownloaderSourceControlCopy"); } var folderMappings = InitializeMappings(settings.GetSetting(DownloaderValidSettings.FolderMappings)); var includeFilter = InitializeIncludeFilter(settings.GetSetting(DownloaderValidSettings.IncludedFilesFilter)); var excludeFilter = InitializeExcludeFilter(settings.GetSetting(DownloaderValidSettings.ExcludedFilesFilter)); // ReSharper disable PossibleMultipleEnumeration if (folderMappings.Any()) // ReSharper restore PossibleMultipleEnumeration { // ReSharper disable PossibleMultipleEnumeration foreach (var mapping in folderMappings) // ReSharper restore PossibleMultipleEnumeration { var subSource = VersionControlPath.Combine(source, mapping.Key); var subDest = Path.Combine(destination, mapping.Value); Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component" + " subfolder {1} with source control version '{2}' to {3}", DownloadType, mapping.Key, versionString, subDest); DownloadFolder(source, subSource, subDest, versionString, includeFilter, excludeFilter, watermark, force); } Logger.Instance().Log(TraceLevel.Info, "{0}: Component {1} download finished successfully", DownloadType, source); } else { Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component {1} with source control version '{2}' to {3}", DownloadType, source, versionString, destination); DownloadFolder(source, source, destination, versionString, includeFilter, excludeFilter, watermark, force); Logger.Instance().Log(TraceLevel.Info, "{0}: Component {1} download finished successfully", DownloadType, source); } }
/// <summary> /// Deletes the specified watermark file. /// </summary> /// <param name="wm">The wm.</param> /// <exception cref="System.ArgumentNullException">wm</exception> internal void Delete(IDependencyDownloaderWatermark wm) { if (null == wm) { throw new ArgumentNullException("wm"); } Delete(GetCacheFilePath(wm.Tags["name"], wm.Tags["version"])); }
/// <summary> /// This will remove any previously downloaded local folders, so in case a mapping was changed, /// the clean will remove the previously mapped folders also. /// The currently active mapping can be retrieved using the watermark (server -> local path) /// </summary> /// <param name="watermark">The watermark to use.</param> public void RevertDownload(IDependencyDownloaderWatermark watermark) { if (null == watermark) { throw new ArgumentNullException("watermark"); } Logger.Instance().Log(TraceLevel.Info, "{0}: Reverting download via watermark ...", DownloadType); var itemsToClean = watermark.ArtifactsToClean.OrderByDescending(x => x.Length).ToList(); foreach (var localFolder in itemsToClean) { // Check local folder var di = new DirectoryInfo(localFolder); if (!di.Exists) { continue; } // Determine workspace var wi = Workstation.Current.GetLocalWorkspaceInfo(localFolder); if (wi == null) { continue; } var localCollection = TfsTeamProjectCollectionFactory.GetTeamProjectCollection(wi.ServerUri); var localWorkspace = wi.GetWorkspace(localCollection); var localVcs = localWorkspace.VersionControlServer; // Clean // ReSharper disable AccessToForEachVariableInClosure var mappingsToRemove = localWorkspace.Folders.Where(item => item.LocalItem != null) // ignore cloaked .Where(mapping => mapping.LocalItem.StartsWith(localFolder, StringComparison.OrdinalIgnoreCase)); // ReSharper restore AccessToForEachVariableInClosure // Remove outdated mappings foreach (var mapping in mappingsToRemove) { // Get version C1 in order to remove all downloaded files without edits (Conflict resolution: "Keep Local Version") var getRequest = new GetRequest(mapping.ServerItem, RecursionType.Full, new ChangesetVersionSpec(1)); localVcs.Getting += RemovingHandler; localWorkspace.Get(getRequest, GetOptions.None); localVcs.Getting -= RemovingHandler; // Remove from workspace localWorkspace.DeleteMapping(mapping); Logger.Instance().Log(TraceLevel.Verbose, "{0}: Removed workspace folder mapping {1} -> {2}", DownloadType, mapping.ServerItem, mapping.LocalItem); watermark.ArtifactsToClean.Remove(localFolder); } } }
/// <summary> /// Reverts an individual component /// </summary> /// <param name="dwSvc">The watermark service to use</param> /// <param name="name">The name of the component</param> /// <param name="version">The version of the component</param> internal void RevertComponent(DownloadWatermarkService dwSvc, string name, string version) { var dFac = new DownloaderFactory(); IDependencyDownloaderWatermark wm = null; var wmName = name; var wmVersion = version; try { wm = dwSvc.Load(name, version); if (null == wm) { // TODO: might consider logging this to a trace file Debug.Fail( string.Format( "Unable to load a watermark for component {0}@{1}. This situation should never occur due to the fact that the same method is reading and writing the watermarks", name, version)); } // Read name and version information from watermark (saving can convert name and version back to hashes). if (wm.Tags.ContainsKey("name")) { wmName = wm.Tags["name"]; } if (wm.Tags.ContainsKey("version")) { wmVersion = wm.Tags["version"]; } var cleaner = dFac.GetDownloader(wm.DownloadType); Logger.Instance().Log(TraceLevel.Info, "{0}: Cleaning component {1}#{2} ...", CommandType, wmName, wmVersion); cleaner.RevertDownload(wm); dwSvc.Delete(wm); } catch { if (null != wm) { dwSvc.Save(wm, name, version); } } if (!_silentMode) { _logger.LogMsg(string.Format(" * Component {0} (Version:{1}) cleaned.", wmName, wmVersion)); } Logger.Instance().Log(TraceLevel.Info, "{0}: Component {1}#{2} successfully cleaned", CommandType, wmName, wmVersion); }
/// <summary> /// Performs a revert operation by removing all the files which have been downloaded previously. The watermark write during the Download will be provided /// </summary> /// <param name="watermark">The watermark that has been used for the download operation</param> public void RevertDownload(IDependencyDownloaderWatermark watermark) { if (null == watermark) { throw new ArgumentNullException("watermark"); } Logger.Instance().Log(TraceLevel.Info, "{0}: Reverting download via watermark ...", DownloadType); // We sort it by length. This means that we delete the deepest files first and we // hit the directories after all files have been deleted var itemsToClean = watermark.ArtifactsToClean.OrderByDescending(x => x.Length).ToList(); foreach (var item in itemsToClean) { var fi = new FileInfo(item); if (fi.Exists) { fi.Attributes = FileAttributes.Normal; fi.Delete(); watermark.ArtifactsToClean.Remove(item); Logger.Instance().Log(TraceLevel.Verbose, "{0}: Deleted file {1}", DownloadType, fi.FullName); continue; } var di = new DirectoryInfo(item); if (di.Exists) { if (!di.EnumerateDirectories().Any() && !di.EnumerateFiles().Any()) { // Delete empty directories only di.Delete(); Logger.Instance().Log(TraceLevel.Verbose, "{0}: Deleted empty directory {1}", DownloadType, di.FullName); } // But remove from artifacts anyway watermark.ArtifactsToClean.Remove(item); } } // In the end, we will remove all watermarks in case someone else removed some files before watermark.ArtifactsToClean.Clear(); watermark.Watermarks.Clear(); Logger.Instance().Log(TraceLevel.Info, "{0}: Reverting download finished successfully", DownloadType); }
internal void Save(IDependencyDownloaderWatermark wm, string name, string version) { if (null == wm) { throw new ArgumentNullException("wm"); } if (string.IsNullOrEmpty(name)) { throw new ArgumentNullException("name"); } if (string.IsNullOrEmpty(version)) { throw new ArgumentNullException("version"); } // Store name and version as Tags wm.UpdateTag("name", name); wm.UpdateTag("version", version); Store(wm, GetCacheFilePath(name, version)); }
private static void Store(IDependencyDownloaderWatermark watermark, string path) { if (null == watermark) { throw new ArgumentNullException("watermark"); } if (null == path) { throw new ArgumentNullException("path"); } if (0 == watermark.ArtifactsToClean.Count && 0 == watermark.Watermarks.Count) { File.Delete(path); return; } using (var stream = File.Open(path, FileMode.Create)) { var serializer = new XmlSerializer(typeof(DownloaderWatermark)); serializer.Serialize(stream, watermark); } }
/// <summary> /// Copy files in directory and calls CopyDirectoryContent recursively for sub directories. /// </summary> /// <param name="root">The root of the recursive copy operation.</param> /// <param name="source">The directory to copy files from.</param> /// <param name="target">The directory to copy files to.</param> /// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations.</param> /// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten.</param> /// <param name="includeFilters">Filter that defines what files and folders to include.</param> /// <param name="excludeFilters">Files that should be excluded</param> private void CopyDirectoryContent(DirectoryInfo root, DirectoryInfo source, DirectoryInfo target, IList <string> includeFilters, IList <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force) { if (string.Equals(source.FullName, target.FullName, StringComparison.OrdinalIgnoreCase)) { return; } //Get the folders only, if filters are provided and the list with excluded and included files and folders has not been initalized yet. // getting the folders only once, will increase the copy speed if (includeFilters != null || excludeFilters != null) { if (_includedFoldersList == null || _excludedFoldersList == null || _includedFilesList == null || _excludedFilesList == null) { // explicitly and implicitly included and excluded folders _includedFoldersList = includeFilters.Where(x => x.EndsWith("\\")).SelectMany(x => GetFoldersFromRoot(x, root)).ToList(); _excludedFoldersList = excludeFilters.Where(x => x.EndsWith("\\")).SelectMany(x => GetFoldersFromRoot(x, root)).ToList(); _includedFoldersList = _includedFoldersList.Concat(includeFilters.Where(x => x.EndsWith("\\")).Select(x => x.Trim('\\')).SelectMany(x => GetFoldersFromRoot(x, root)).ToList()).ToList(); _excludedFoldersList = _excludedFoldersList.Concat(excludeFilters.Where(x => x.EndsWith("\\")).Select(x => x.Trim('\\')).SelectMany(x => GetFoldersFromRoot(x, root)).ToList()).ToList(); //// generally and specifically included and excluded files _includedFilesList = includeFilters.Where(x => x.EndsWith("\\") == false).SelectMany(x => GetFilesFromRoot(x, root)).ToList(); _excludedFilesList = excludeFilters.Where(x => x.EndsWith("\\") == false).SelectMany(x => GetFilesFromRoot(x, root)).ToList(); //Sort the list to move the root folder first _includedFoldersList = _includedFoldersList.OrderBy(o => o.FullName.Length).ToList(); if (_excludedFoldersList.Any(x => source.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase))) { return; } } } watermark.ArtifactsToClean.Add(target.FullName); if (!target.Exists) { // Since preliminary deletes might lead to file or folder locks, we need to retry this var retries = _retryHelper.RetryAction(target.Create, 10, 1000); if (retries > 0) { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Folder {1} successfully created after {2} retries", DownloadType, target, retries); } } //The Move Operation Type. if (_operationType == OperationType.Move) { MoveFolder(source, target, watermark); } else { // get excluded and included files foreach (var sf in source.GetFiles()) { var isInIncludedFolder = _includedFoldersList.Any(x => sf.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase)); var isInExcludedFolder = _excludedFoldersList.Any(x => sf.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase)); var isIncludedFile = _includedFilesList.Any(x => IsSameFile(x.FullName, sf.FullName)); var isExcludedFile = _excludedFilesList.Any(x => IsSameFile(x.FullName, sf.FullName)); if ((isIncludedFile == false && isInIncludedFolder == false) || isExcludedFile || isInExcludedFolder) { continue; } var df = new FileInfo(Path.Combine(target.FullName, sf.Name)); var wm = watermark.GetWatermark <DateTime>(sf.FullName); if (force || wm == DateTime.MinValue || wm != df.LastWriteTimeUtc || !df.Exists) { if (df.Exists) { df.Attributes = FileAttributes.Normal; } var localSourceFile = sf; watermark.ArtifactsToClean.Add(df.FullName); var retriesCopy = _retryHelper.RetryAction(() => localSourceFile.CopyTo(df.FullName, true), 5, 1000); if (retriesCopy > 0) { Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully downloaded after {2} retries", DownloadType, df.FullName, retriesCopy); } //To determine the the LastWrite time the object must be refreshed after copy the file. df.Refresh(); watermark.UpdateWatermark(sf.FullName, df.LastWriteTimeUtc); df.Attributes = FileAttributes.Normal; Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, sf.FullName); } else { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, sf.FullName); } } foreach (var subdirectory in source.GetDirectories()) { // a folder is included if one of its parents is included (unless it is explicitly exluded) var files = _includedFilesList.Where(i => !_excludedFilesList.Any(e => IsSameFile(e.FullName, i.FullName))).ToList(); var isIncluded = _includedFoldersList.Any(x => x.FullName.StartsWith(subdirectory.FullName, StringComparison.InvariantCultureIgnoreCase)) || !_excludedFoldersList.Any(x => x.FullName.EndsWith(subdirectory.FullName)); if (isIncluded) { var targetDi = new DirectoryInfo(Path.Combine(target.FullName, subdirectory.Name)); Logger.Instance() .Log(TraceLevel.Verbose, "{0}: Downloading folder {1}", DownloadType, Path.Combine(source.FullName, subdirectory.Name)); CopyDirectoryContent(root, subdirectory, targetDi, includeFilters, excludeFilters, watermark, force); } } } }
/// <summary> /// Downloads a component from source control to a local path. /// </summary> /// <param name="source">Path to source location</param> /// <param name="destination">Path to destination folder</param> /// <param name="watermark">The watermark to adapt for clean support</param> /// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten</param> /// <param name="settings">Setting which contain the version to fetch</param> public void Download(string source, string destination, IDependencyDownloaderWatermark watermark, bool force, ISettings <DownloaderValidSettings> settings) { if (string.IsNullOrEmpty(source)) { throw new ArgumentNullException( "source", "Source control path to component folder was empty in DownloaderSourceControl"); } if (string.IsNullOrEmpty(destination)) { throw new ArgumentNullException( "destination", "Destination path for component was empty in DownloaderSourceControl"); } if (null == watermark) { throw new ArgumentNullException("watermark"); } if (null == settings) { throw new ArgumentNullException("settings"); } var versionString = settings.GetSetting(DownloaderValidSettings.VersionString); if (string.IsNullOrEmpty(versionString)) { throw new InvalidComponentException( "VersionSpec string for component was empty in DownloaderSourceControl"); } var version = VersionSpec.ParseSingleSpec(versionString, _workspace.OwnerName); if (version == null) { throw new InvalidComponentException("VersionSpec for component was invalid in DownloaderSourceControl"); } // Adapt workspace mapping // The mapping we want to end up with: source -> destination // The currently active workspace mappings targeting our local root paths var workspaceMappings = _workspace.Folders.Where(item => item.LocalItem != null) // ignore cloaked .Where(mapping => mapping.LocalItem.StartsWith(destination, StringComparison.OrdinalIgnoreCase)); // Items mapped in earlier runs of this tool, maybe configuration has changed or items no longer belong to the dependencies. var outdatedMappings = workspaceMappings.Where( mapping => !source.Equals(mapping.ServerItem, StringComparison.OrdinalIgnoreCase)); // Map and Get operations // Remove outdated mappings foreach (var mapping in outdatedMappings) { _workspace.DeleteMapping(mapping); Logger.Instance().Log(TraceLevel.Verbose, "{0}: Deleted old workspace folder mapping {1} -> {2}", DownloadType, mapping.ServerItem, mapping.LocalItem); } // Ensure desired mappings are mapped. var localPath = _workspace.TryGetLocalItemForServerItem(source); if (!string.Equals(localPath, destination)) { _workspace.Map(source, destination); Logger.Instance().Log(TraceLevel.Verbose, "{0}: Created workspace folder mapping {1} -> {2}", DownloadType, source, destination); } // Build a get request for every mapping to ensure we have the desired version. Always need to do a full get since the mapping might // not have changed but only the label, or we map to latest version and there is a newer version available. var getRequest = new GetRequest(source, RecursionType.Full, version); try { _vcs.Getting += GettingHandler; _workspace.Get(getRequest, force ? ForceGetOptions : IncrementalGetOptions); watermark.ArtifactsToClean.Add(destination); } finally { _vcs.Getting -= GettingHandler; } }
/// <summary> /// Determines the source, target location and worker settings and initializes download workers according to their type. /// </summary> /// <param name="dws">The downloader wartermark service.</param> /// <param name="component">The component node in graph to start download from.</param> /// <param name="recursive">Indicates if the dependencies should be fetched recursively or not.</param> /// <param name="force">If set to <c>true</c> it indicates that the get operation is forced and all files have to be overwritten. Otherwise false.</param> private void DownloadComponent(DownloadWatermarkService dws, IComponent component, bool recursive, bool force) { var df = new DownloaderFactory(); var componentAlreadyDownloaded = false; var settings = new Settings <DownloaderValidSettings>(); IDependencyDownloader worker; string destLocation; string targetLocation; bool removeTempFiles = false; Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component {1}#{2} ...", CommandType, component.Name, component.Version); // Create worker and settings according to type if (component.Type.Equals(ComponentType.FileShare)) { var fileShareRootPath = component.GetFieldValue(DependencyProviderValidSettingName.FileShareRootPath); destLocation = Path.Combine(fileShareRootPath, component.Name.Path, component.Version.Version); var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath); targetLocation = relativeTargetPath == null ? Path.GetFullPath(Path.Combine(_defaultDownloadFolder, _defaultRelativeOutputPath)) : Path.GetFullPath(Path.Combine(_defaultDownloadFolder, relativeTargetPath)); LoadFilterSettings(component, settings); } else if (component.Type.Equals(ComponentType.VNextBuildResult)) { var url = component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl); if (!Uri.IsWellFormedUriString(url, UriKind.Absolute)) { Logger.Instance().Log( TraceLevel.Error, "{0}: Invalid BuildTeamProjectCollectionUrl setting '{1}' was found for component {2}#{3}", CommandType, url, component.Name, component.Version); throw new DependencyServiceException( string.Format( " ! Invalid download information was found for build result control component {0} (BuildTeamProjectCollectionUrl setting)", component.Name.GetName())); } // Connect to tfs server var tpc = TfsTeamProjectCollectionFactory.GetTeamProjectCollection(new Uri(url)); tpc.EnsureAuthenticated(); // Connect to version control service var versionControl = tpc.GetService <VersionControlServer>(); if (versionControl == null) { Logger.Instance().Log(TraceLevel.Error, "{0}: Connection to version control server failed for component {1}#{2}", CommandType, component.Name, component.Version); throw new DependencyServiceException(string.Format(" ! Could not connect to TFS version control server for team project collection {0}", component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl))); } var tpcUrl = new Uri(url); var connection = new VssConnection(tpcUrl, new VssClientCredentials(true)); var client = connection.GetClient <BuildHttpClient>(); var builds = client.GetBuildsAsync(project: component.Name.TeamProject, type: DefinitionType.Build).Result; var buildResult = builds.FirstOrDefault(r => r.BuildNumber == component.Version.BuildNumber); if (buildResult == null) { Logger.Instance().Log(TraceLevel.Error, "{0}: No vnext build result could not be determined for component {1}#{2}", CommandType, component.Name, component.Version); throw new DependencyServiceException(string.Format(" ! Could not determine build result component {0}", component.Name.GetName())); } var artifacts = client.GetArtifactsAsync(component.Name.TeamProject, buildResult.Id).Result; destLocation = Path.Combine(Path.GetTempPath(), "VNextDM_" + Guid.NewGuid()); Directory.CreateDirectory(destLocation); removeTempFiles = true; foreach (var artifact in artifacts) { if (artifact.Resource.Type == "FilePath") { var sourceDirName = $"{artifact.Resource.Data}/{artifact.Name}"; DirectoryCopy(sourceDirName, destLocation, true); } else { var content = client.GetArtifactContentZipAsync(component.Name.TeamProject, buildResult.Id, artifact.Name); using (var zipArchive = new ZipArchive(content.Result)) { zipArchive.ExtractToDirectory(destLocation); } } } var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath); targetLocation = relativeTargetPath == null ? Path.GetFullPath(Path.Combine(_defaultDownloadFolder, _defaultRelativeOutputPath)) : Path.GetFullPath(Path.Combine(_defaultDownloadFolder, relativeTargetPath)); LoadFilterSettings(component, settings); } else if (component.Type.Equals(ComponentType.BuildResult)) { if (!Uri.IsWellFormedUriString(component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl), UriKind.Absolute)) { Logger.Instance().Log( TraceLevel.Error, "{0}: Invalid BuildTeamProjectCollectionUrl setting '{1}' was found for component {2}#{3}", CommandType, component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl), component.Name, component.Version); throw new DependencyServiceException( string.Format( " ! Invalid download information was found for build result control component {0} (BuildTeamProjectCollectionUrl setting)", component.Name.GetName())); } // Connect to tfs server var tpc = TfsTeamProjectCollectionFactory.GetTeamProjectCollection(new Uri(component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl))); tpc.EnsureAuthenticated(); // Connect to version control service var versionControl = tpc.GetService <VersionControlServer>(); if (versionControl == null) { Logger.Instance().Log(TraceLevel.Error, "{0}: Connection to version control server failed for component {1}#{2}", CommandType, component.Name, component.Version); throw new DependencyServiceException(string.Format(" ! Could not connect to TFS version control server for team project collection {0}", component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl))); } // Connect to build server var buildServer = tpc.GetService <IBuildServer>(); if (buildServer == null) { Logger.Instance().Log(TraceLevel.Error, "{0}: Connection to build server failed for component {1}#{2}", CommandType, component.Name, component.Version); throw new DependencyServiceException(string.Format(" ! Could not connect to TFS build server for team project collection {0}", component.GetFieldValue(DependencyProviderValidSettingName.BuildTeamProjectCollectionUrl))); } var buildDef = buildServer.GetBuildDefinition(component.Name.TeamProject, component.Name.BuildDefinition); var buildDetailSpec = buildServer.CreateBuildDetailSpec(buildDef); buildDetailSpec.BuildNumber = component.Version.BuildNumber; buildDetailSpec.InformationTypes = new string[] { }; var buildResult = buildServer.QueryBuilds(buildDetailSpec); if (buildResult == null || buildResult.Builds == null || buildResult.Builds.Length == 0) { Logger.Instance().Log(TraceLevel.Error, "{0}: No build result could not be determined for component {1}#{2}", CommandType, component.Name, component.Version); throw new DependencyServiceException(string.Format(" ! Could not determine drop location for build result component {0}", component.Name.GetName())); } // Determine source location var dropLocation = buildResult.Builds[0].DropLocation; // TODO dga: Is this a bug? it returns localhost for the computer name while we are not running this code on the server (I guess)? if (string.IsNullOrEmpty(dropLocation)) { Logger.Instance().Log(TraceLevel.Error, "{0}: Drop location could not be determined for component {1}#{2}", CommandType, component.Name, component.Version); throw new DependencyServiceException(string.Format("! Could not determine drop location for build result component {0}", component.Name.GetName())); } destLocation = dropLocation; var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath); targetLocation = relativeTargetPath == null ? Path.GetFullPath(Path.Combine(_defaultDownloadFolder, _defaultRelativeOutputPath)) : Path.GetFullPath(Path.Combine(_defaultDownloadFolder, relativeTargetPath)); LoadFilterSettings(component, settings); } else if (component.Type.Equals(ComponentType.SourceControl)) { if (!Uri.IsWellFormedUriString(component.GetFieldValue(DependencyProviderValidSettingName.WorkspaceTeamProjectCollectionUrl), UriKind.Absolute)) { Logger.Instance().Log( TraceLevel.Error, "{0}: Invalid WorkspaceTeamProjectCollectionUrl setting '{1}' was found for component {2}#{3}", CommandType, component.GetFieldValue(DependencyProviderValidSettingName.WorkspaceTeamProjectCollectionUrl), component.Name, component.Version); throw new DependencyServiceException( string.Format( " ! Invalid download information was found for source control component {0} (WorkspaceTeamProjectCollectionUrl setting)", component.Name.GetName())); } var workspaceName = component.GetFieldValue(DependencyProviderValidSettingName.WorkspaceName); if (string.IsNullOrEmpty(workspaceName)) { Logger.Instance().Log(TraceLevel.Error, "{0}: WorkspaceName setting was not specified for component {1}#{2}", CommandType, component.Name, component.Version); throw new DependencyServiceException( string.Format( " ! Invalid download information was found for source control component {0} (WorkspaceName setting)", component.Name.GetName())); } var workspaceOwner = component.GetFieldValue(DependencyProviderValidSettingName.WorkspaceOwner); if (string.IsNullOrEmpty(workspaceOwner)) { Logger.Instance().Log(TraceLevel.Error, "{0}: WorkspaceOwner setting was not specified for component {1}#{2}", CommandType, component.Name, component.Version); throw new DependencyServiceException( string.Format( " ! Invalid download information was found for source control component {0} (WorkspaceOwner setting)", component.Name.GetName())); } destLocation = component.Name.ToString(); settings.AddSetting(new KeyValuePair <DownloaderValidSettings, string>(DownloaderValidSettings.VersionString, component.Version.ToString())); var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath); targetLocation = Path.GetFullPath( relativeTargetPath == null ? Path.Combine(_defaultDownloadFolder, "..", VersionControlPath.GetFileName(component.Name.ToString())) : Path.Combine(_defaultDownloadFolder, relativeTargetPath)); } else if (component.Type.Equals(ComponentType.SourceControlCopy)) { destLocation = component.Name.ToString(); settings.AddSetting(new KeyValuePair <DownloaderValidSettings, string>(DownloaderValidSettings.VersionString, component.Version.ToString())); var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath); targetLocation = Path.GetFullPath( relativeTargetPath == null ? Path.Combine(_defaultDownloadFolder, "..", VersionControlPath.GetFileName(component.Name.ToString())) : Path.Combine(_defaultDownloadFolder, relativeTargetPath)); LoadFilterSettings(component, settings); } else if (component.Type.Equals(ComponentType.BinaryRepository)) { var repositoryTeamProject = component.GetFieldValue(DependencyProviderValidSettingName.BinaryRepositoryTeamProject); if (string.IsNullOrEmpty(repositoryTeamProject)) { Logger.Instance().Log(TraceLevel.Error, "{0}: BinaryRepositoryTeamProject setting was not specified for component {1}#{2}", CommandType, component.Name, component.Version); throw new DependencyServiceException( string.Format( " ! Invalid download information was found for binary repository component {0} (BinaryRepositoryTeamProject setting)", component.Name.GetName())); } destLocation = VersionControlPath.Combine(VersionControlPath.Combine(VersionControlPath.Combine(VersionControlPath.RootFolder, repositoryTeamProject), component.Name.GetName()), component.Version.GetVersion()); settings.AddSetting(new KeyValuePair <DownloaderValidSettings, string>(DownloaderValidSettings.VersionString, VersionSpec.Latest.DisplayString)); var relativeOutputPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath); targetLocation = Path.GetFullPath( relativeOutputPath == null ? Path.Combine(_defaultDownloadFolder, _defaultRelativeOutputPath) : Path.Combine(_defaultDownloadFolder, relativeOutputPath)); LoadFilterSettings(component, settings); } else if (component.Type.Equals(ComponentType.Subversion)) { destLocation = string.Format("{0}", component.Name); settings.AddSetting(new KeyValuePair <DownloaderValidSettings, string>(DownloaderValidSettings.VersionString, component.Version.ToString())); var relativeTargetPath = component.GetFieldValue(DependencyProviderValidSettingName.RelativeOutputPath); targetLocation = relativeTargetPath == null ? Path.GetFullPath(Path.Combine(_defaultDownloadFolder, _defaultRelativeOutputPath)) : Path.GetFullPath(Path.Combine(_defaultDownloadFolder, relativeTargetPath)); LoadFilterSettings(component, settings); } else { Logger.Instance().Log(TraceLevel.Error, "{0}: Unknown dependency type '{1}' was found in dependency graph", CommandType, component.Type.GetType()); throw new DependencyServiceException(" ! Invalid dependency node found in graph!"); } //Determine Multi-Site path for BuildResult and FileShare-Provider if (component.Type.Equals(ComponentType.FileShare) || (component.Type.Equals(ComponentType.BuildResult))) { destLocation = DetermineMultiSiteSourceLocation(destLocation); } if (string.IsNullOrEmpty(destLocation)) { Logger.Instance().Log(TraceLevel.Error, "{0}: Source location was not set for component {1}#{2}", CommandType, component.Name, component.Version); throw new DependencyServiceException(string.Format(" ! Error occured while preparing to download component {0} (Source location was not set)", component.Name.GetName())); } if (string.IsNullOrEmpty(targetLocation)) { Logger.Instance().Log(TraceLevel.Error, "{0}: Target location was not set for component {1}#{2}", CommandType, component.Name, component.Version); throw new DependencyServiceException(string.Format(" ! Error occured while preparing to download component {0} (Destination location was not set)", component.Name.GetName())); } // Download files try { IDependencyDownloaderWatermark wm = null; try { worker = df.GetDownloader(component); wm = dws.Load(component.Name.GetName(), component.Version.GetVersion()) ?? new DownloaderWatermark(worker); worker.Download(destLocation, targetLocation, wm, force, settings); } finally { if (null != wm) { dws.Save(wm, component.Name.GetName(), component.Version.GetVersion()); } if (removeTempFiles) { Directory.Delete(destLocation, true); } } } catch (Exception e) { if (e is DependencyServiceException) { throw; } // ReSharper disable RedundantIfElseBlock else if (e is ComponentAlreadyDownloadedException) // ReSharper restore RedundantIfElseBlock { componentAlreadyDownloaded = true; } else { Logger.Instance().Log(TraceLevel.Error, "{0}: Exception {1} occured while downloading files: {2}", CommandType, e, e.Message); throw new DependencyServiceException(string.Format(" ! Download of component {0} failed ({1})", component.Name.GetName(), e.Message)); } } // Log download if (!_silentMode) { _logger.LogMsg( !componentAlreadyDownloaded ? string.Format( " * Component {0} (Version:{1}) downloaded to target directory {2}", component.Name.GetName(), component.Version.GetVersion(), targetLocation) : string.Format( " * Skipped component {0} (Version:{1}). Already present in target directory {2}", component.Name.GetName(), component.Version.GetVersion(), targetLocation)); } Logger.Instance().Log( TraceLevel.Info, !componentAlreadyDownloaded ? "{0}: Component {1}#{2} successfully downloaded to target directory {3}" : "{0}: Component {1}#{2} download was skipped. Already present in target directory {3}", CommandType, component.Name, component.Version, targetLocation); }
/// <summary> /// Extracts the compressed files in directory and recursively in subdirectories. /// Searches for zipped and 7-zipped files (LZMA) /// </summary> /// <param name="source">The directory with compressed files</param> /// <param name="target">The target direcotry of the zip files content</param> /// <param name="watermark">The watermark which is be used to perform incremental updates and cleanup operations.</param> /// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten.</param> private void ExtractCompressedFiles(string source, string target, IDependencyDownloaderWatermark watermark, bool force) { string[] allowedExtensions = { ".zip", ".7z" }; foreach (var compressedFile in Directory.EnumerateFiles(source, "*.*", SearchOption.AllDirectories) .Where(s => allowedExtensions.Any(ext => ext == Path.GetExtension(s)))) { Logger.Instance().Log(TraceLevel.Info, "Start extracting compressed file {0}.", compressedFile); try { // regard if the zip file is in a subdirectory var targetSubdir = compressedFile.Replace(source, string.Empty); if (targetSubdir.Contains("\\")) { targetSubdir = targetSubdir.Substring(0, targetSubdir.LastIndexOf("\\", StringComparison.Ordinal)); // cut the leading Backslash "\", if one exists if ((targetSubdir.StartsWith("\\")) && (targetSubdir.Length > 0)) { targetSubdir = targetSubdir.Substring(1); } } else { // contains only the filename of the zip archive targetSubdir = string.Empty; } //If Sevenzip is supported, extract and zip file also with sevenzip if (compressedFile.EndsWith(".zip")) { if (string.IsNullOrWhiteSpace(ResolveSevenZipExecutable())) { ExtractStandardZipFile(target, targetSubdir, compressedFile, watermark, force); } else { ExtractSevenZipFile(target, targetSubdir, compressedFile, watermark, force); } } else if (compressedFile.EndsWith(".7z")) { ExtractSevenZipFile(target, targetSubdir, compressedFile, watermark, force); } else { throw new Exception("The file you are trying to extract is not supported "); } //Delete if (_isDeleteArchiveFiles) { DeleteArchiveFile(compressedFile); } } catch (InvalidDataException) { Logger.Instance().Log(TraceLevel.Warning, "Failed to extract file {0}. Format not supported!", compressedFile); throw new Exception(); } } }
/// <summary> /// Downloads a component to a local path. /// </summary> /// <param name="source">The path to source folder.</param> /// <param name="destination">The path to destination folder.</param> /// <param name="watermark">The watermark which is be used to perform incremental updates and cleanup operations.</param> /// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten.</param> /// <param name="settings">The settings which include the pattern for files/directories to include and folder mappings.</param> public void Download(string source, string destination, IDependencyDownloaderWatermark watermark, bool force, ISettings <DownloaderValidSettings> settings) { if (string.IsNullOrEmpty(source)) { throw new ArgumentNullException("source"); } if (string.IsNullOrEmpty(destination)) { throw new ArgumentNullException("destination"); } if (watermark == null) { throw new ArgumentNullException("watermark"); } if (settings == null) { throw new ArgumentNullException("settings"); } var versionSpec = settings.GetSetting(DownloaderValidSettings.VersionString); if (string.IsNullOrEmpty(versionSpec)) { throw new InvalidComponentException("Version string for component was empty in DownloaderSubversion"); } try { if (!ProviderSubversion.Instance.ItemExists(source, versionSpec)) { Logger.Instance().Log(TraceLevel.Error, "{0}: Folder not exists in Subversion {1}", DownloadType, source); throw new InvalidProviderConfigurationException(string.Format("Could not connect to Subversion {0}", source)); } } catch (SvnAuthenticationException) { throw new SvnAuthenticationException(); } Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component {1} to {2} ...", DownloadType, source, destination); var folderMappings = InitializeMappings(settings.GetSetting(DownloaderValidSettings.FolderMappings)); var includeFilter = InitializeIncludeFilter(settings.GetSetting(DownloaderValidSettings.IncludedFilesFilter)); var excludeFilter = InitializeExcludeFilter(settings.GetSetting(DownloaderValidSettings.ExcludedFilesFilter)); if (folderMappings.Any()) { foreach (var mapping in folderMappings) { //var subSource = VersionControlPath.Combine(source, mapping.Key); var subSource = string.Format("{0}/{1}", source, mapping.Key); var subDest = Path.Combine(destination, mapping.Value); Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component subfolder {1} with source control version to folder {2}", DownloadType, mapping.Key, subDest); Export(source, subSource, subDest, versionSpec, includeFilter, excludeFilter, watermark, force); } Logger.Instance().Log(TraceLevel.Info, "{0}: Component {1} download finished successfully", DownloadType, source); } else { Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component {1} to folder {2}", DownloadType, source, destination); Export(source, source, destination, versionSpec, includeFilter, excludeFilter, watermark, force); Logger.Instance().Log(TraceLevel.Info, "{0}: Component {1} download finished successfully", DownloadType, source); } }
private void Export(string root, string source, string destination, string versionSpec, IEnumerable <string> includeFilters, IEnumerable <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force) { var includes = ConvertFilterToRegex(includeFilters, root); var excludes = ConvertFilterToRegex(excludeFilters, root); var items = new Dictionary <string, string>(); var allItems = ProviderSubversion.Instance.GetItems(source, ProviderSubversion.ItemType.File, true, versionSpec); // apply include and exclude filter foreach (var item in allItems) { if ((includes.Any(x => x.IsMatch(item.Key))) && (!excludes.Any(x => x.IsMatch(item.Key)))) { items.Add(item.Key, item.Value); } } var revision = 0L; foreach (var item in items) { var destinationpath = destination; var subpath = item.Value.Substring(source.Length).Trim(new[] { '/' }).Replace("/", "\\"); //Remove filename if (subpath.Contains("\\")) { subpath = subpath.Remove(subpath.LastIndexOf("\\")); destinationpath = Path.Combine(destination, subpath); } // Determine revision one-time. All items have same revision. if (revision == 0) { revision = ProviderSubversion.Instance.GetHeadRevision(item.Key); } var wm = watermark.GetWatermark <long>(item.Key); var targetFile = Path.Combine(destinationpath, Path.GetFileName(item.Value)); if (force || 0 == wm || revision != wm || !File.Exists(targetFile)) { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, item.Key); ProviderSubversion.Instance.GetFile(item.Key, destinationpath, force, versionSpec); //Save files and folders, which included in the item watermark.ArtifactsToClean.Add(destinationpath); foreach (var file in Directory.GetFiles(destinationpath)) { watermark.ArtifactsToClean.Add(file); } //Update information about source (path and revision) watermark.UpdateWatermark(item.Key, revision); } else { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, item); } } }
public void Download(string source, string destination, IDependencyDownloaderWatermark watermark, bool force, ISettings <DownloaderValidSettings> settings) { if (source == null) { throw new ArgumentNullException("source"); } if (destination == null) { throw new ArgumentNullException("destination"); } if (watermark == null) { throw new ArgumentNullException("watermark"); } if (settings == null) { throw new ArgumentNullException("settings"); } Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component {1} to {2} ...", DownloadType, source, destination); // Save compressed files to temp path and temporarily store watermarks. Turn of mappings as they only apply to the extracted folder structure var tempPath = Path.Combine(destination, "..\\", "DM_TEMP"); //Create the temp Path if (!Directory.Exists(tempPath)) { DirectoryInfo di = Directory.CreateDirectory(tempPath); } //save the folder mapping var tempFolderMapping = settings.GetSetting(DownloaderValidSettings.FolderMappings); var remainingArtefacts = new HashSet <string>(watermark.ArtifactsToClean); //Download the zip files and the rest of the contents ignore any mappings to get everything in the temppath settings.SettingsDictionary[DownloaderValidSettings.FolderMappings] = null; _downloader .Download(source, tempPath, watermark, force, settings); //var zipWatermarks = new Dictionary<string, object>(watermark.Watermarks); // The DownloaderFileShareCopy / DownloaderSourceControlCopy creates artifacts for cleanup, // which are deleted by the ZippedDependencyDownloader and must not be saved. watermark.ArtifactsToClean = remainingArtefacts; //Extract the zipped files in the temp path ExtractCompressedFiles(tempPath, tempPath, watermark, force); //Set the settings to a move operation settings.AddSetting(new KeyValuePair <DownloaderValidSettings, string>(DownloaderValidSettings.OperationType, "Move")); //Set the temporary saved mapping again settings.SettingsDictionary[DownloaderValidSettings.FolderMappings] = tempFolderMapping; //The SourceControlCopy-Provider needs the FileShareCopy-Provider to copy the extracted files from the temp folder to the //target folder, because the SourceControlCopy-Provider works not with local folders as source folder if (_downloader.DownloadType == "Downloader_SourceControlCopy") { IDependencyDownloader tmpDownloader = new DownloaderFileShareCopy(); tmpDownloader.Download(tempPath, destination, watermark, force, settings); } // Provider: FileShare, BuildResult else { //Copy the files from the temp path to the final destination _downloader.Download(tempPath, destination, watermark, force, settings); } //Cleanup: Delete the temp path try { DeleteTempDirectory(tempPath); } catch (Exception e) { Logger.Instance().Log(TraceLevel.Error, "Deleting temp folder {0} did not succeed: {1}", tempPath, e.Message); Logger.Instance().Log(TraceLevel.Error, "Stacktrace: {0}", e.StackTrace); } }
/// <summary> /// Performs a revert operation by removing all the files which have been downloaded previously. The watermark written during the Download will be provided /// </summary> /// <param name="watermark">The watermark that has been used for the download operation</param> public void RevertDownload(IDependencyDownloaderWatermark watermark) { // Todo MRI: Refactor to use PluginFactory (new DownloaderFileShareCopy()).RevertDownload(watermark); }
/// <summary> /// Extract a standard zip file /// </summary> /// <param name="target"></param> /// <param name="targetSubdir"></param> /// <param name="archiveFile"></param> /// <param name="watermark"></param> /// <param name="force"></param> private void ExtractStandardZipFile(string target, string targetSubdir, string archiveFile, IDependencyDownloaderWatermark watermark, bool force) { using (var archive = ZipFile.OpenRead(archiveFile)) { foreach (var file in archive.Entries) { // Extract only files, not subdirectories. If a subdirectory not exists, create it; if (file.FullName.EndsWith("/")) { var subdir = Path.Combine(target, targetSubdir, file.FullName.Substring(0, file.FullName.Length - 1)); subdir = subdir.Replace("/", "\\"); if (!Directory.Exists(subdir)) { Directory.CreateDirectory(subdir); // For the CreateDirectory method the folder depth its not relevant, but for the cleanup, so each subfolder must // be logged separately if (FolderDepth(subdir) > FolderDepth(target) + 1) { var tmp = subdir; do { watermark.ArtifactsToClean.Add(tmp); tmp = tmp.Substring(0, tmp.LastIndexOf("\\")); } while (tmp.Length > target.Length); } } continue; } var df = new FileInfo(Path.Combine(target, targetSubdir, file.FullName)); var wm = watermark.GetWatermark <DateTime>(file.FullName); if (force || wm == DateTime.MinValue || wm != df.LastWriteTimeUtc || !df.Exists) { // Remove possible readonly flag if (df.Exists) { df.Attributes = FileAttributes.Normal; } watermark.ArtifactsToClean.Add(df.FullName); try { file.ExtractToFile(df.FullName, true); } catch (Exception ex) { Logger.Instance().Log(TraceLevel.Error, "{0}: The archive cannot be extracted. The compression method is not supported. {1}", ex.Message, file.FullName); throw new Exception(String.Format("The file {0} cannot be extracted. If the file is compressed with a non-standard algorithm (e.g. LZMA), make sure the support for 7-zip is enabled and all dlls are set properly in the Dependecy Manager settings.", archiveFile)); } df.Attributes = FileAttributes.Normal; //TODO: Klaus will auch die enthaltenen Datien im Output Window sehen! Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, file.FullName); } else { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, file.FullName); } } } Logger.Instance().Log(TraceLevel.Info, "Extracted zipped file {0} finished successfully.", archiveFile); }
/// <summary> /// Seven zip (lzma) support for the DM. The libraries for sevenzip sharp are referenced dynamically /// </summary> /// <param name="target"></param> /// <param name="targetSubdir"></param> /// <param name="archiveFile"></param> /// <param name="watermark"></param> /// <param name="force"></param> private void ExtractSevenZipFile(string target, string targetSubdir, string archiveFile, IDependencyDownloaderWatermark watermark, bool force) { var sevenZipExecutable = ResolveSevenZipExecutable(); if (string.IsNullOrWhiteSpace(sevenZipExecutable)) { Logger.Instance().Log(TraceLevel.Warning, "Unable to resolve the path to the 7 zip executable. Make sure that 7 zip is installed and that the dependency manager is configured properly"); Logger.Instance().Log(TraceLevel.Warning, "Your dependecy contains *.7z archives. Please make sure the SevenZip support is enabled in the Dependency Manager settings."); return; } var process = new Process(); process.StartInfo = new ProcessStartInfo(); process.StartInfo.FileName = sevenZipExecutable; process.StartInfo.Arguments = string.Format("x \"{0}\" -y", archiveFile); process.StartInfo.WindowStyle = ProcessWindowStyle.Hidden; process.StartInfo.UseShellExecute = false; process.StartInfo.CreateNoWindow = true; process.StartInfo.RedirectStandardError = true; process.StartInfo.RedirectStandardOutput = true; process.StartInfo.WorkingDirectory = Path.Combine(target, targetSubdir); process.Start(); process.OutputDataReceived += (x, y) => { if (!string.IsNullOrWhiteSpace(y.Data) && y.Data.StartsWith("Extracting", StringComparison.OrdinalIgnoreCase)) { var file = y.Data.Replace("Extracting", string.Empty).Trim(); var fullName = Path.Combine(target, file); watermark.ArtifactsToClean.Add(fullName); } }; process.BeginOutputReadLine(); process.WaitForExit(); if (0 != process.ExitCode) { //Todo Replace by other and more specific exception Logger.Instance().Log(TraceLevel.Error, "An error occured while executing {0} {1}. The exit code is {2}", process.StartInfo.FileName, process.StartInfo.Arguments, process.ExitCode); throw new Exception(string.Format("An error occured while executing {0} {1}. The exit code is {2}", process.StartInfo.FileName, process.StartInfo.Arguments, process.ExitCode)); } Logger.Instance().Log(TraceLevel.Info, "Extracted zipped file {0} finished successfully.", archiveFile); }
/// <summary> /// Recursive method to move a folder /// A folder is moved by moving its subfolders and all files within the folder /// If the target folder already exsist the method checks if the children of the target folder are the same. /// It will delete existing folders and will then move them to the folder /// /// </summary> /// <param name="source"></param> /// <param name="target"></param> /// <param name="watermark"></param> private void MoveFolder(DirectoryInfo source, DirectoryInfo target, IDependencyDownloaderWatermark watermark) { //Move the content of the current folder to the given output folder foreach (string tempPath in Directory.GetDirectories(source.FullName, "*", SearchOption.TopDirectoryOnly)) { ////Check if the current folder exists on the target //if (Directory.Exists(tempPath.Replace(source.FullName, target.FullName))) //{ // //Check if the target contains any children. If there are no children, --> delete the folder // //If the folder has children, check if the target children are equal to the source children --> If they are equal --> Delete // var children = Directory.GetDirectories(tempPath.Replace(source.FullName, target.FullName), "*", SearchOption.TopDirectoryOnly); // if (children.Count() == 0) // { // DeleteDirectory(tempPath.Replace(source.FullName, target.FullName)); // } // else // { // //Delete each path that is similar within the children // foreach (var tempPathChildren in Directory.GetDirectories(tempPath, "*", SearchOption.TopDirectoryOnly)) // { // if (Directory.Exists(tempPathChildren.Replace(source.FullName, target.FullName))) // { // DeleteDirectory(tempPathChildren.Replace(source.FullName, target.FullName)); // } // } // } //} //If the target folder still exists, the children and files of the source can be moved //If the target does not exsist, the whole source folder can be moved if (Directory.Exists(tempPath.Replace(source.FullName, target.FullName))) { MoveFolder(new DirectoryInfo(tempPath), new DirectoryInfo(tempPath.Replace(source.FullName, target.FullName)), watermark); } else { var retriesMove = _retryHelper.RetryAction(() => Directory.Move(tempPath, tempPath.Replace(source.FullName, target.FullName)), 5, 1000); if (retriesMove > 0) { Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully moved after {2} retries", DownloadType, tempPath, retriesMove); } } //Get the file info and set the FileAttributes to Normal var fi = new DirectoryInfo(tempPath.Replace(source.FullName, target.FullName)); fi.Attributes = FileAttributes.Normal; //Add all files in the folder to the watermarks foreach (var file in fi.GetFiles("*.*", SearchOption.AllDirectories)) { //Add the files to the watermark watermark.UpdateWatermark(file.FullName, file.LastWriteTimeUtc); } } //Move all the files & Replaces any files with the same name by deleting them foreach (string tempPath in Directory.GetFiles(source.FullName, "*.*", SearchOption.TopDirectoryOnly)) { if (File.Exists(tempPath.Replace(source.FullName, target.FullName))) { File.SetAttributes(tempPath.Replace(source.FullName, target.FullName), FileAttributes.Normal); File.Delete(tempPath.Replace(source.FullName, target.FullName)); } var retriesMove = _retryHelper.RetryAction(() => File.Move(tempPath, tempPath.Replace(source.FullName, target.FullName)), 5, 1000); if (retriesMove > 0) { Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully moved after {2} retries", DownloadType, tempPath, retriesMove); } //Get the file info and set the FileAttributes to Normal var fi = new FileInfo(tempPath.Replace(source.FullName, target.FullName)); fi.Attributes = FileAttributes.Normal; //Add the files to the watermark watermark.UpdateWatermark(fi.FullName, fi.LastWriteTimeUtc); } Logger.Instance().Log(TraceLevel.Verbose, "{0}: Moving Directory {1} to {2}", DownloadType, source.FullName, target.FullName); }
/// <summary> /// Downloads a component to a local path. /// </summary> /// <param name="source">The path to source folder.</param> /// <param name="destination">The path to destination folder.</param> /// <param name="watermark">The watermark which is be used to perform incremental updates and cleanup operations.</param> /// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten.</param> /// <param name="settings">The settings which include the pattern for files/directories to include and folder mappings.</param> public void Download(string source, string destination, IDependencyDownloaderWatermark watermark, bool force, ISettings <DownloaderValidSettings> settings) { if (string.IsNullOrEmpty(source)) { throw new ArgumentNullException("source"); } if (string.IsNullOrEmpty(destination)) { throw new ArgumentNullException("destination"); } if (watermark == null) { throw new ArgumentNullException("watermark"); } if (settings == null) { throw new ArgumentNullException("settings"); } if (!Directory.Exists(source)) { throw new DirectoryNotFoundException(string.Format("The directory {0} does not exit", source)); } Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component {1} to {2} ...", DownloadType, source, destination); var sourceDi = new DirectoryInfo(source); var destinationDi = new DirectoryInfo(destination); var folderMappings = InitializeMappings(settings.GetSetting(DownloaderValidSettings.FolderMappings)); var filters = InitializeIncludeFilter(settings.GetSetting(DownloaderValidSettings.IncludedFilesFilter)); var excludedFiles = InitializeExcludeFilter(settings.GetSetting(DownloaderValidSettings.ExcludedFilesFilter)); _operationType = InitializeOperationType(settings.GetSetting(DownloaderValidSettings.OperationType)); // Reset the list of excluded and included files when a new download occurs _includedFoldersList = null; _excludedFoldersList = null; _includedFilesList = null; _excludedFilesList = null; // ReSharper disable PossibleMultipleEnumeration if (folderMappings.Any()) // ReSharper restore PossibleMultipleEnumeration { // ReSharper disable PossibleMultipleEnumeration foreach (var mapping in folderMappings) // ReSharper restore PossibleMultipleEnumeration { var subSourceDi = new DirectoryInfo(Path.Combine(sourceDi.ToString(), mapping.Key)); var subDestDi = new DirectoryInfo(Path.Combine(destinationDi.ToString(), mapping.Value)); Logger.Instance().Log(TraceLevel.Info, "{0}: Downloading component subfolder {1} to {2}", DownloadType, mapping.Key, subDestDi); CopyDirectoryContent(sourceDi, subSourceDi, subDestDi, filters, excludedFiles, watermark, force); } Logger.Instance().Log(TraceLevel.Info, "{0}: Component {1} download finished successfully", DownloadType, source); } else { CopyDirectoryContent(sourceDi, sourceDi, destinationDi, filters, excludedFiles, watermark, force); Logger.Instance().Log(TraceLevel.Info, "{0}: Component {1} download finished successfully", DownloadType, source); } }
/// <summary> /// Downloads all items in current folder and items in subfolders by calling itself. /// </summary> /// <param name="root">Root of copy operation</param> /// <param name="source">A valid TFS version control path to a folder that shall be downloaded.</param> /// <param name="destination">A valid directory to which all the files will be downloaded to.</param> /// <param name="versionString">The version string to determine the version to download.</param> /// <param name="includeFilters">The filter for file types to include.</param> /// <param name="excludeFilters">List of filters to exclude files and folders.</param> /// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations.</param> /// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten</param> private void DownloadFolder(string root, string source, string destination, string versionString, IEnumerable <string> includeFilters, IEnumerable <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force) { var version = VersionSpec.ParseSingleSpec(versionString, _workspace.OwnerName); var includes = ConvertFilterToRegex(includeFilters, root); var excludes = ConvertFilterToRegex(excludeFilters, root); var items = _vcs.GetItems(source, version, RecursionType.Full, DeletedState.NonDeleted, ItemType.File, true).Items; items = items.Where(x => includes.Any(y => y.IsMatch(x.ServerItem))).ToArray(); items = items.Where(x => !excludes.Any(y => y.IsMatch(x.ServerItem))).ToArray(); Parallel.ForEach(items, new ParallelOptions { MaxDegreeOfParallelism = 8 }, item => { var subpath = item.ServerItem.Substring(source.Length).Trim(new[] { '/' }).Replace("//", "\\"); var destinationpath = Path.Combine(destination, subpath); var wm = watermark.GetWatermark <int>(item.ServerItem); if (force || 0 == wm || item.ChangesetId != wm || !File.Exists(destinationpath)) { if (File.Exists(destinationpath)) { File.SetAttributes(destinationpath, FileAttributes.Normal); } lock (this) { watermark.ArtifactsToClean.Add(destinationpath); } Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, item.ServerItem); item.DownloadFile(destinationpath); lock (this) { watermark.UpdateWatermark(item.ServerItem, item.ChangesetId); } } else { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, item.ServerItem); } }); }