/// <summary> /// Downloads all items in current folder and items in subfolders by calling itself. /// </summary> /// <param name="root">Root of copy operation</param> /// <param name="source">A valid TFS version control path to a folder that shall be downloaded.</param> /// <param name="destination">A valid directory to which all the files will be downloaded to.</param> /// <param name="versionString">The version string to determine the version to download.</param> /// <param name="includeFilters">The filter for file types to include.</param> /// <param name="excludeFilters">List of filters to exclude files and folders.</param> /// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations.</param> /// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten</param> private void DownloadFolder(string root, string source, string destination, string versionString, IEnumerable <string> includeFilters, IEnumerable <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force) { var version = VersionSpec.ParseSingleSpec(versionString, _workspace.OwnerName); var includes = ConvertFilterToRegex(includeFilters, root); var excludes = ConvertFilterToRegex(excludeFilters, root); var items = _vcs.GetItems(source, version, RecursionType.Full, DeletedState.NonDeleted, ItemType.File, true).Items; items = items.Where(x => includes.Any(y => y.IsMatch(x.ServerItem))).ToArray(); items = items.Where(x => !excludes.Any(y => y.IsMatch(x.ServerItem))).ToArray(); Parallel.ForEach(items, new ParallelOptions { MaxDegreeOfParallelism = 8 }, item => { var subpath = item.ServerItem.Substring(source.Length).Trim(new[] { '/' }).Replace("//", "\\"); var destinationpath = Path.Combine(destination, subpath); var wm = watermark.GetWatermark <int>(item.ServerItem); if (force || 0 == wm || item.ChangesetId != wm || !File.Exists(destinationpath)) { if (File.Exists(destinationpath)) { File.SetAttributes(destinationpath, FileAttributes.Normal); } lock (this) { watermark.ArtifactsToClean.Add(destinationpath); } Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, item.ServerItem); item.DownloadFile(destinationpath); lock (this) { watermark.UpdateWatermark(item.ServerItem, item.ChangesetId); } } else { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, item.ServerItem); } }); }
/// <summary> /// Copy files in directory and calls CopyDirectoryContent recursively for sub directories. /// </summary> /// <param name="root">The root of the recursive copy operation.</param> /// <param name="source">The directory to copy files from.</param> /// <param name="target">The directory to copy files to.</param> /// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations.</param> /// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten.</param> /// <param name="includeFilters">Filter that defines what files and folders to include.</param> /// <param name="excludeFilters">Files that should be excluded</param> private void CopyDirectoryContent(DirectoryInfo root, DirectoryInfo source, DirectoryInfo target, IList <string> includeFilters, IList <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force) { if (string.Equals(source.FullName, target.FullName, StringComparison.OrdinalIgnoreCase)) { return; } //Get the folders only, if filters are provided and the list with excluded and included files and folders has not been initalized yet. // getting the folders only once, will increase the copy speed if (includeFilters != null || excludeFilters != null) { if (_includedFoldersList == null || _excludedFoldersList == null || _includedFilesList == null || _excludedFilesList == null) { // explicitly and implicitly included and excluded folders _includedFoldersList = includeFilters.Where(x => x.EndsWith("\\")).SelectMany(x => GetFoldersFromRoot(x, root)).ToList(); _excludedFoldersList = excludeFilters.Where(x => x.EndsWith("\\")).SelectMany(x => GetFoldersFromRoot(x, root)).ToList(); _includedFoldersList = _includedFoldersList.Concat(includeFilters.Where(x => x.EndsWith("\\")).Select(x => x.Trim('\\')).SelectMany(x => GetFoldersFromRoot(x, root)).ToList()).ToList(); _excludedFoldersList = _excludedFoldersList.Concat(excludeFilters.Where(x => x.EndsWith("\\")).Select(x => x.Trim('\\')).SelectMany(x => GetFoldersFromRoot(x, root)).ToList()).ToList(); //// generally and specifically included and excluded files _includedFilesList = includeFilters.Where(x => x.EndsWith("\\") == false).SelectMany(x => GetFilesFromRoot(x, root)).ToList(); _excludedFilesList = excludeFilters.Where(x => x.EndsWith("\\") == false).SelectMany(x => GetFilesFromRoot(x, root)).ToList(); //Sort the list to move the root folder first _includedFoldersList = _includedFoldersList.OrderBy(o => o.FullName.Length).ToList(); if (_excludedFoldersList.Any(x => source.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase))) { return; } } } watermark.ArtifactsToClean.Add(target.FullName); if (!target.Exists) { // Since preliminary deletes might lead to file or folder locks, we need to retry this var retries = _retryHelper.RetryAction(target.Create, 10, 1000); if (retries > 0) { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Folder {1} successfully created after {2} retries", DownloadType, target, retries); } } //The Move Operation Type. if (_operationType == OperationType.Move) { MoveFolder(source, target, watermark); } else { // get excluded and included files foreach (var sf in source.GetFiles()) { var isInIncludedFolder = _includedFoldersList.Any(x => sf.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase)); var isInExcludedFolder = _excludedFoldersList.Any(x => sf.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase)); var isIncludedFile = _includedFilesList.Any(x => IsSameFile(x.FullName, sf.FullName)); var isExcludedFile = _excludedFilesList.Any(x => IsSameFile(x.FullName, sf.FullName)); if ((isIncludedFile == false && isInIncludedFolder == false) || isExcludedFile || isInExcludedFolder) { continue; } var df = new FileInfo(Path.Combine(target.FullName, sf.Name)); var wm = watermark.GetWatermark <DateTime>(sf.FullName); if (force || wm == DateTime.MinValue || wm != df.LastWriteTimeUtc || !df.Exists) { if (df.Exists) { df.Attributes = FileAttributes.Normal; } var localSourceFile = sf; watermark.ArtifactsToClean.Add(df.FullName); var retriesCopy = _retryHelper.RetryAction(() => localSourceFile.CopyTo(df.FullName, true), 5, 1000); if (retriesCopy > 0) { Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully downloaded after {2} retries", DownloadType, df.FullName, retriesCopy); } //To determine the the LastWrite time the object must be refreshed after copy the file. df.Refresh(); watermark.UpdateWatermark(sf.FullName, df.LastWriteTimeUtc); df.Attributes = FileAttributes.Normal; Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, sf.FullName); } else { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, sf.FullName); } } foreach (var subdirectory in source.GetDirectories()) { // a folder is included if one of its parents is included (unless it is explicitly exluded) var files = _includedFilesList.Where(i => !_excludedFilesList.Any(e => IsSameFile(e.FullName, i.FullName))).ToList(); var isIncluded = _includedFoldersList.Any(x => x.FullName.StartsWith(subdirectory.FullName, StringComparison.InvariantCultureIgnoreCase)) || !_excludedFoldersList.Any(x => x.FullName.EndsWith(subdirectory.FullName)); if (isIncluded) { var targetDi = new DirectoryInfo(Path.Combine(target.FullName, subdirectory.Name)); Logger.Instance() .Log(TraceLevel.Verbose, "{0}: Downloading folder {1}", DownloadType, Path.Combine(source.FullName, subdirectory.Name)); CopyDirectoryContent(root, subdirectory, targetDi, includeFilters, excludeFilters, watermark, force); } } } }
private void Export(string root, string source, string destination, string versionSpec, IEnumerable <string> includeFilters, IEnumerable <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force) { var includes = ConvertFilterToRegex(includeFilters, root); var excludes = ConvertFilterToRegex(excludeFilters, root); var items = new Dictionary <string, string>(); var allItems = ProviderSubversion.Instance.GetItems(source, ProviderSubversion.ItemType.File, true, versionSpec); // apply include and exclude filter foreach (var item in allItems) { if ((includes.Any(x => x.IsMatch(item.Key))) && (!excludes.Any(x => x.IsMatch(item.Key)))) { items.Add(item.Key, item.Value); } } var revision = 0L; foreach (var item in items) { var destinationpath = destination; var subpath = item.Value.Substring(source.Length).Trim(new[] { '/' }).Replace("/", "\\"); //Remove filename if (subpath.Contains("\\")) { subpath = subpath.Remove(subpath.LastIndexOf("\\")); destinationpath = Path.Combine(destination, subpath); } // Determine revision one-time. All items have same revision. if (revision == 0) { revision = ProviderSubversion.Instance.GetHeadRevision(item.Key); } var wm = watermark.GetWatermark <long>(item.Key); var targetFile = Path.Combine(destinationpath, Path.GetFileName(item.Value)); if (force || 0 == wm || revision != wm || !File.Exists(targetFile)) { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, item.Key); ProviderSubversion.Instance.GetFile(item.Key, destinationpath, force, versionSpec); //Save files and folders, which included in the item watermark.ArtifactsToClean.Add(destinationpath); foreach (var file in Directory.GetFiles(destinationpath)) { watermark.ArtifactsToClean.Add(file); } //Update information about source (path and revision) watermark.UpdateWatermark(item.Key, revision); } else { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, item); } } }
/// <summary> /// Extract a standard zip file /// </summary> /// <param name="target"></param> /// <param name="targetSubdir"></param> /// <param name="archiveFile"></param> /// <param name="watermark"></param> /// <param name="force"></param> private void ExtractStandardZipFile(string target, string targetSubdir, string archiveFile, IDependencyDownloaderWatermark watermark, bool force) { using (var archive = ZipFile.OpenRead(archiveFile)) { foreach (var file in archive.Entries) { // Extract only files, not subdirectories. If a subdirectory not exists, create it; if (file.FullName.EndsWith("/")) { var subdir = Path.Combine(target, targetSubdir, file.FullName.Substring(0, file.FullName.Length - 1)); subdir = subdir.Replace("/", "\\"); if (!Directory.Exists(subdir)) { Directory.CreateDirectory(subdir); // For the CreateDirectory method the folder depth its not relevant, but for the cleanup, so each subfolder must // be logged separately if (FolderDepth(subdir) > FolderDepth(target) + 1) { var tmp = subdir; do { watermark.ArtifactsToClean.Add(tmp); tmp = tmp.Substring(0, tmp.LastIndexOf("\\")); } while (tmp.Length > target.Length); } } continue; } var df = new FileInfo(Path.Combine(target, targetSubdir, file.FullName)); var wm = watermark.GetWatermark <DateTime>(file.FullName); if (force || wm == DateTime.MinValue || wm != df.LastWriteTimeUtc || !df.Exists) { // Remove possible readonly flag if (df.Exists) { df.Attributes = FileAttributes.Normal; } watermark.ArtifactsToClean.Add(df.FullName); try { file.ExtractToFile(df.FullName, true); } catch (Exception ex) { Logger.Instance().Log(TraceLevel.Error, "{0}: The archive cannot be extracted. The compression method is not supported. {1}", ex.Message, file.FullName); throw new Exception(String.Format("The file {0} cannot be extracted. If the file is compressed with a non-standard algorithm (e.g. LZMA), make sure the support for 7-zip is enabled and all dlls are set properly in the Dependecy Manager settings.", archiveFile)); } df.Attributes = FileAttributes.Normal; //TODO: Klaus will auch die enthaltenen Datien im Output Window sehen! Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, file.FullName); } else { Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, file.FullName); } } } Logger.Instance().Log(TraceLevel.Info, "Extracted zipped file {0} finished successfully.", archiveFile); }