/// <summary> /// Downloads all items in current folder and items in subfolders by calling itself. /// </summary> /// <param name="root">Root of copy operation</param> /// <param name="source">A valid TFS version control path to a folder that shall be downloaded.</param> /// <param name="destination">A valid directory to which all the files will be downloaded to.</param> /// <param name="versionString">The version string to determine the version to download.</param> /// <param name="includeFilters">The filter for file types to include.</param> /// <param name="excludeFilters">List of filters to exclude files and folders.</param> /// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations.</param> /// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten</param> private void DownloadFolder(string root, string source, string destination, string versionString, IEnumerable <string> includeFilters, IEnumerable <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force) { var version = VersionSpec.ParseSingleSpec(versionString, _workspace.OwnerName); var includes = ConvertFilterToRegex(includeFilters, root); var excludes = ConvertFilterToRegex(excludeFilters, root); var items = _vcs.GetItems(source, version, RecursionType.Full, DeletedState.NonDeleted, ItemType.File, true).Items; items = items.Where(x => includes.Any(y => y.IsMatch(x.ServerItem))).ToArray(); items = items.Where(x => !excludes.Any(y => y.IsMatch(x.ServerItem))).ToArray(); Parallel.ForEach(items, new ParallelOptions { MaxDegreeOfParallelism = 8 }, item => { var subpath = item.ServerItem.Substring(source.Length).Trim(new[] { '/' }).Replace("//", "\\"); var destinationpath = Path.Combine(destination, subpath); var wm = watermark.GetWatermark <int>(item.ServerItem); if (force || 0 == wm || item.ChangesetId != wm || !File.Exists(destinationpath)) { if (File.Exists(destinationpath)) { File.SetAttributes(destinationpath, FileAttributes.Normal); } lock (this) { watermark.ArtifactsToClean.Add(destinationpath); } Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, item.ServerItem); item.DownloadFile(destinationpath); lock (this) { watermark.UpdateWatermark(item.ServerItem, item.ChangesetId); } } else { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, item.ServerItem); } }); }
/// <summary> /// Recursive method to move a folder /// A folder is moved by moving its subfolders and all files within the folder /// If the target folder already exsist the method checks if the children of the target folder are the same. /// It will delete existing folders and will then move them to the folder /// /// </summary> /// <param name="source"></param> /// <param name="target"></param> /// <param name="watermark"></param> private void MoveFolder(DirectoryInfo source, DirectoryInfo target, IDependencyDownloaderWatermark watermark) { //Move the content of the current folder to the given output folder foreach (string tempPath in Directory.GetDirectories(source.FullName, "*", SearchOption.TopDirectoryOnly)) { ////Check if the current folder exists on the target //if (Directory.Exists(tempPath.Replace(source.FullName, target.FullName))) //{ // //Check if the target contains any children. If there are no children, --> delete the folder // //If the folder has children, check if the target children are equal to the source children --> If they are equal --> Delete // var children = Directory.GetDirectories(tempPath.Replace(source.FullName, target.FullName), "*", SearchOption.TopDirectoryOnly); // if (children.Count() == 0) // { // DeleteDirectory(tempPath.Replace(source.FullName, target.FullName)); // } // else // { // //Delete each path that is similar within the children // foreach (var tempPathChildren in Directory.GetDirectories(tempPath, "*", SearchOption.TopDirectoryOnly)) // { // if (Directory.Exists(tempPathChildren.Replace(source.FullName, target.FullName))) // { // DeleteDirectory(tempPathChildren.Replace(source.FullName, target.FullName)); // } // } // } //} //If the target folder still exists, the children and files of the source can be moved //If the target does not exsist, the whole source folder can be moved if (Directory.Exists(tempPath.Replace(source.FullName, target.FullName))) { MoveFolder(new DirectoryInfo(tempPath), new DirectoryInfo(tempPath.Replace(source.FullName, target.FullName)), watermark); } else { var retriesMove = _retryHelper.RetryAction(() => Directory.Move(tempPath, tempPath.Replace(source.FullName, target.FullName)), 5, 1000); if (retriesMove > 0) { Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully moved after {2} retries", DownloadType, tempPath, retriesMove); } } //Get the file info and set the FileAttributes to Normal var fi = new DirectoryInfo(tempPath.Replace(source.FullName, target.FullName)); fi.Attributes = FileAttributes.Normal; //Add all files in the folder to the watermarks foreach (var file in fi.GetFiles("*.*", SearchOption.AllDirectories)) { //Add the files to the watermark watermark.UpdateWatermark(file.FullName, file.LastWriteTimeUtc); } } //Move all the files & Replaces any files with the same name by deleting them foreach (string tempPath in Directory.GetFiles(source.FullName, "*.*", SearchOption.TopDirectoryOnly)) { if (File.Exists(tempPath.Replace(source.FullName, target.FullName))) { File.SetAttributes(tempPath.Replace(source.FullName, target.FullName), FileAttributes.Normal); File.Delete(tempPath.Replace(source.FullName, target.FullName)); } var retriesMove = _retryHelper.RetryAction(() => File.Move(tempPath, tempPath.Replace(source.FullName, target.FullName)), 5, 1000); if (retriesMove > 0) { Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully moved after {2} retries", DownloadType, tempPath, retriesMove); } //Get the file info and set the FileAttributes to Normal var fi = new FileInfo(tempPath.Replace(source.FullName, target.FullName)); fi.Attributes = FileAttributes.Normal; //Add the files to the watermark watermark.UpdateWatermark(fi.FullName, fi.LastWriteTimeUtc); } Logger.Instance().Log(TraceLevel.Verbose, "{0}: Moving Directory {1} to {2}", DownloadType, source.FullName, target.FullName); }
/// <summary> /// Copy files in directory and calls CopyDirectoryContent recursively for sub directories. /// </summary> /// <param name="root">The root of the recursive copy operation.</param> /// <param name="source">The directory to copy files from.</param> /// <param name="target">The directory to copy files to.</param> /// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations.</param> /// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten.</param> /// <param name="includeFilters">Filter that defines what files and folders to include.</param> /// <param name="excludeFilters">Files that should be excluded</param> private void CopyDirectoryContent(DirectoryInfo root, DirectoryInfo source, DirectoryInfo target, IList <string> includeFilters, IList <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force) { if (string.Equals(source.FullName, target.FullName, StringComparison.OrdinalIgnoreCase)) { return; } //Get the folders only, if filters are provided and the list with excluded and included files and folders has not been initalized yet. // getting the folders only once, will increase the copy speed if (includeFilters != null || excludeFilters != null) { if (_includedFoldersList == null || _excludedFoldersList == null || _includedFilesList == null || _excludedFilesList == null) { // explicitly and implicitly included and excluded folders _includedFoldersList = includeFilters.Where(x => x.EndsWith("\\")).SelectMany(x => GetFoldersFromRoot(x, root)).ToList(); _excludedFoldersList = excludeFilters.Where(x => x.EndsWith("\\")).SelectMany(x => GetFoldersFromRoot(x, root)).ToList(); _includedFoldersList = _includedFoldersList.Concat(includeFilters.Where(x => x.EndsWith("\\")).Select(x => x.Trim('\\')).SelectMany(x => GetFoldersFromRoot(x, root)).ToList()).ToList(); _excludedFoldersList = _excludedFoldersList.Concat(excludeFilters.Where(x => x.EndsWith("\\")).Select(x => x.Trim('\\')).SelectMany(x => GetFoldersFromRoot(x, root)).ToList()).ToList(); //// generally and specifically included and excluded files _includedFilesList = includeFilters.Where(x => x.EndsWith("\\") == false).SelectMany(x => GetFilesFromRoot(x, root)).ToList(); _excludedFilesList = excludeFilters.Where(x => x.EndsWith("\\") == false).SelectMany(x => GetFilesFromRoot(x, root)).ToList(); //Sort the list to move the root folder first _includedFoldersList = _includedFoldersList.OrderBy(o => o.FullName.Length).ToList(); if (_excludedFoldersList.Any(x => source.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase))) { return; } } } watermark.ArtifactsToClean.Add(target.FullName); if (!target.Exists) { // Since preliminary deletes might lead to file or folder locks, we need to retry this var retries = _retryHelper.RetryAction(target.Create, 10, 1000); if (retries > 0) { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Folder {1} successfully created after {2} retries", DownloadType, target, retries); } } //The Move Operation Type. if (_operationType == OperationType.Move) { MoveFolder(source, target, watermark); } else { // get excluded and included files foreach (var sf in source.GetFiles()) { var isInIncludedFolder = _includedFoldersList.Any(x => sf.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase)); var isInExcludedFolder = _excludedFoldersList.Any(x => sf.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase)); var isIncludedFile = _includedFilesList.Any(x => IsSameFile(x.FullName, sf.FullName)); var isExcludedFile = _excludedFilesList.Any(x => IsSameFile(x.FullName, sf.FullName)); if ((isIncludedFile == false && isInIncludedFolder == false) || isExcludedFile || isInExcludedFolder) { continue; } var df = new FileInfo(Path.Combine(target.FullName, sf.Name)); var wm = watermark.GetWatermark <DateTime>(sf.FullName); if (force || wm == DateTime.MinValue || wm != df.LastWriteTimeUtc || !df.Exists) { if (df.Exists) { df.Attributes = FileAttributes.Normal; } var localSourceFile = sf; watermark.ArtifactsToClean.Add(df.FullName); var retriesCopy = _retryHelper.RetryAction(() => localSourceFile.CopyTo(df.FullName, true), 5, 1000); if (retriesCopy > 0) { Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully downloaded after {2} retries", DownloadType, df.FullName, retriesCopy); } //To determine the the LastWrite time the object must be refreshed after copy the file. df.Refresh(); watermark.UpdateWatermark(sf.FullName, df.LastWriteTimeUtc); df.Attributes = FileAttributes.Normal; Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, sf.FullName); } else { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, sf.FullName); } } foreach (var subdirectory in source.GetDirectories()) { // a folder is included if one of its parents is included (unless it is explicitly exluded) var files = _includedFilesList.Where(i => !_excludedFilesList.Any(e => IsSameFile(e.FullName, i.FullName))).ToList(); var isIncluded = _includedFoldersList.Any(x => x.FullName.StartsWith(subdirectory.FullName, StringComparison.InvariantCultureIgnoreCase)) || !_excludedFoldersList.Any(x => x.FullName.EndsWith(subdirectory.FullName)); if (isIncluded) { var targetDi = new DirectoryInfo(Path.Combine(target.FullName, subdirectory.Name)); Logger.Instance() .Log(TraceLevel.Verbose, "{0}: Downloading folder {1}", DownloadType, Path.Combine(source.FullName, subdirectory.Name)); CopyDirectoryContent(root, subdirectory, targetDi, includeFilters, excludeFilters, watermark, force); } } } }
private void Export(string root, string source, string destination, string versionSpec, IEnumerable <string> includeFilters, IEnumerable <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force) { var includes = ConvertFilterToRegex(includeFilters, root); var excludes = ConvertFilterToRegex(excludeFilters, root); var items = new Dictionary <string, string>(); var allItems = ProviderSubversion.Instance.GetItems(source, ProviderSubversion.ItemType.File, true, versionSpec); // apply include and exclude filter foreach (var item in allItems) { if ((includes.Any(x => x.IsMatch(item.Key))) && (!excludes.Any(x => x.IsMatch(item.Key)))) { items.Add(item.Key, item.Value); } } var revision = 0L; foreach (var item in items) { var destinationpath = destination; var subpath = item.Value.Substring(source.Length).Trim(new[] { '/' }).Replace("/", "\\"); //Remove filename if (subpath.Contains("\\")) { subpath = subpath.Remove(subpath.LastIndexOf("\\")); destinationpath = Path.Combine(destination, subpath); } // Determine revision one-time. All items have same revision. if (revision == 0) { revision = ProviderSubversion.Instance.GetHeadRevision(item.Key); } var wm = watermark.GetWatermark <long>(item.Key); var targetFile = Path.Combine(destinationpath, Path.GetFileName(item.Value)); if (force || 0 == wm || revision != wm || !File.Exists(targetFile)) { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, item.Key); ProviderSubversion.Instance.GetFile(item.Key, destinationpath, force, versionSpec); //Save files and folders, which included in the item watermark.ArtifactsToClean.Add(destinationpath); foreach (var file in Directory.GetFiles(destinationpath)) { watermark.ArtifactsToClean.Add(file); } //Update information about source (path and revision) watermark.UpdateWatermark(item.Key, revision); } else { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, item); } } }