/// <summary>
        /// Downloads all items in current folder and items in subfolders by calling itself.
        /// </summary>
        /// <param name="root">Root of copy operation</param>
        /// <param name="source">A valid TFS version control path to a folder that shall be downloaded.</param>
        /// <param name="destination">A valid directory to which all the files will be downloaded to.</param>
        /// <param name="versionString">The version string to determine the version to download.</param>
        /// <param name="includeFilters">The filter for file types to include.</param>
        /// <param name="excludeFilters">List of filters to exclude files and folders.</param>
        /// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations.</param>
        /// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten</param>
        private void DownloadFolder(string root, string source, string destination, string versionString, IEnumerable <string> includeFilters, IEnumerable <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force)
        {
            var version = VersionSpec.ParseSingleSpec(versionString, _workspace.OwnerName);

            var includes = ConvertFilterToRegex(includeFilters, root);
            var excludes = ConvertFilterToRegex(excludeFilters, root);

            var items = _vcs.GetItems(source, version, RecursionType.Full, DeletedState.NonDeleted, ItemType.File, true).Items;

            items = items.Where(x => includes.Any(y => y.IsMatch(x.ServerItem))).ToArray();
            items = items.Where(x => !excludes.Any(y => y.IsMatch(x.ServerItem))).ToArray();

            Parallel.ForEach(items, new ParallelOptions {
                MaxDegreeOfParallelism = 8
            }, item =>
            {
                var subpath         = item.ServerItem.Substring(source.Length).Trim(new[] { '/' }).Replace("//", "\\");
                var destinationpath = Path.Combine(destination, subpath);

                var wm = watermark.GetWatermark <int>(item.ServerItem);
                if (force || 0 == wm || item.ChangesetId != wm || !File.Exists(destinationpath))
                {
                    if (File.Exists(destinationpath))
                    {
                        File.SetAttributes(destinationpath, FileAttributes.Normal);
                    }

                    lock (this)
                    {
                        watermark.ArtifactsToClean.Add(destinationpath);
                    }

                    Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, item.ServerItem);
                    item.DownloadFile(destinationpath);

                    lock (this)
                    {
                        watermark.UpdateWatermark(item.ServerItem, item.ChangesetId);
                    }
                }
                else
                {
                    Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, item.ServerItem);
                }
            });
        }
        /// <summary>
        /// Recursive method to move a folder
        /// A folder is moved by moving its subfolders and all files within the folder
        /// If the target folder already exsist the method checks if the children of the target folder are the same.
        /// It will delete existing folders and will then move them to the folder
        ///
        /// </summary>
        /// <param name="source"></param>
        /// <param name="target"></param>
        /// <param name="watermark"></param>
        private void MoveFolder(DirectoryInfo source, DirectoryInfo target, IDependencyDownloaderWatermark watermark)
        {
            //Move the content of the current folder to the given output folder
            foreach (string tempPath in Directory.GetDirectories(source.FullName, "*", SearchOption.TopDirectoryOnly))
            {
                ////Check if the current folder exists on the target
                //if (Directory.Exists(tempPath.Replace(source.FullName, target.FullName)))
                //{
                //    //Check if the target contains any children. If there are no children, --> delete the folder
                //    //If the folder has children, check if the target children are equal to the source children --> If they are equal --> Delete
                //    var children = Directory.GetDirectories(tempPath.Replace(source.FullName, target.FullName), "*", SearchOption.TopDirectoryOnly);
                //    if (children.Count() == 0)
                //    {
                //        DeleteDirectory(tempPath.Replace(source.FullName, target.FullName));
                //    }
                //    else
                //    {
                //        //Delete each path that is similar within the children
                //        foreach (var tempPathChildren in Directory.GetDirectories(tempPath, "*", SearchOption.TopDirectoryOnly))
                //        {
                //            if (Directory.Exists(tempPathChildren.Replace(source.FullName, target.FullName)))
                //            {
                //                DeleteDirectory(tempPathChildren.Replace(source.FullName, target.FullName));
                //            }
                //        }
                //    }
                //}

                //If the target folder still exists, the children and files of the source can be moved
                //If the target does not exsist, the whole source folder can be moved
                if (Directory.Exists(tempPath.Replace(source.FullName, target.FullName)))
                {
                    MoveFolder(new DirectoryInfo(tempPath), new DirectoryInfo(tempPath.Replace(source.FullName, target.FullName)), watermark);
                }
                else
                {
                    var retriesMove = _retryHelper.RetryAction(() => Directory.Move(tempPath, tempPath.Replace(source.FullName, target.FullName)), 5, 1000);
                    if (retriesMove > 0)
                    {
                        Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully moved after {2} retries", DownloadType, tempPath, retriesMove);
                    }
                }

                //Get the file info and set the FileAttributes to Normal
                var fi = new DirectoryInfo(tempPath.Replace(source.FullName, target.FullName));
                fi.Attributes = FileAttributes.Normal;

                //Add all files in the folder to the watermarks
                foreach (var file in fi.GetFiles("*.*", SearchOption.AllDirectories))
                {
                    //Add the files to the watermark
                    watermark.UpdateWatermark(file.FullName, file.LastWriteTimeUtc);
                }
            }

            //Move all the files & Replaces any files with the same name by deleting them
            foreach (string tempPath in Directory.GetFiles(source.FullName, "*.*", SearchOption.TopDirectoryOnly))
            {
                if (File.Exists(tempPath.Replace(source.FullName, target.FullName)))
                {
                    File.SetAttributes(tempPath.Replace(source.FullName, target.FullName), FileAttributes.Normal);
                    File.Delete(tempPath.Replace(source.FullName, target.FullName));
                }

                var retriesMove = _retryHelper.RetryAction(() => File.Move(tempPath, tempPath.Replace(source.FullName, target.FullName)), 5, 1000);
                if (retriesMove > 0)
                {
                    Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully moved after {2} retries", DownloadType, tempPath, retriesMove);
                }

                //Get the file info and set the FileAttributes to Normal
                var fi = new FileInfo(tempPath.Replace(source.FullName, target.FullName));
                fi.Attributes = FileAttributes.Normal;

                //Add the files to the watermark
                watermark.UpdateWatermark(fi.FullName, fi.LastWriteTimeUtc);
            }

            Logger.Instance().Log(TraceLevel.Verbose, "{0}: Moving Directory {1} to {2}", DownloadType, source.FullName, target.FullName);
        }
        /// <summary>
        /// Copy files in directory and calls CopyDirectoryContent recursively for sub directories.
        /// </summary>
        /// <param name="root">The root of the recursive copy operation.</param>
        /// <param name="source">The directory to copy files from.</param>
        /// <param name="target">The directory to copy files to.</param>
        /// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations.</param>
        /// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten.</param>
        /// <param name="includeFilters">Filter that defines what files and folders to include.</param>
        /// <param name="excludeFilters">Files that should be excluded</param>
        private void CopyDirectoryContent(DirectoryInfo root, DirectoryInfo source, DirectoryInfo target, IList <string> includeFilters, IList <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force)
        {
            if (string.Equals(source.FullName, target.FullName, StringComparison.OrdinalIgnoreCase))
            {
                return;
            }

            //Get the folders only, if filters are provided and the list with excluded and included files and folders has not been initalized yet.
            // getting the folders only once, will increase the copy speed
            if (includeFilters != null || excludeFilters != null)
            {
                if (_includedFoldersList == null || _excludedFoldersList == null || _includedFilesList == null || _excludedFilesList == null)
                {
                    // explicitly and implicitly included and excluded folders
                    _includedFoldersList = includeFilters.Where(x => x.EndsWith("\\")).SelectMany(x => GetFoldersFromRoot(x, root)).ToList();
                    _excludedFoldersList = excludeFilters.Where(x => x.EndsWith("\\")).SelectMany(x => GetFoldersFromRoot(x, root)).ToList();
                    _includedFoldersList = _includedFoldersList.Concat(includeFilters.Where(x => x.EndsWith("\\")).Select(x => x.Trim('\\')).SelectMany(x => GetFoldersFromRoot(x, root)).ToList()).ToList();
                    _excludedFoldersList = _excludedFoldersList.Concat(excludeFilters.Where(x => x.EndsWith("\\")).Select(x => x.Trim('\\')).SelectMany(x => GetFoldersFromRoot(x, root)).ToList()).ToList();
                    //// generally and specifically included and excluded files
                    _includedFilesList = includeFilters.Where(x => x.EndsWith("\\") == false).SelectMany(x => GetFilesFromRoot(x, root)).ToList();
                    _excludedFilesList = excludeFilters.Where(x => x.EndsWith("\\") == false).SelectMany(x => GetFilesFromRoot(x, root)).ToList();


                    //Sort the list to move the root folder first
                    _includedFoldersList = _includedFoldersList.OrderBy(o => o.FullName.Length).ToList();

                    if (_excludedFoldersList.Any(x => source.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase)))
                    {
                        return;
                    }
                }
            }

            watermark.ArtifactsToClean.Add(target.FullName);
            if (!target.Exists)
            {
                // Since preliminary deletes might lead to file or folder locks, we need to retry this
                var retries = _retryHelper.RetryAction(target.Create, 10, 1000);
                if (retries > 0)
                {
                    Logger.Instance().Log(TraceLevel.Verbose, "{0}: Folder {1} successfully created after {2} retries", DownloadType, target, retries);
                }
            }

            //The Move Operation Type.
            if (_operationType == OperationType.Move)
            {
                MoveFolder(source, target, watermark);
            }

            else
            {
                // get excluded and included files
                foreach (var sf in source.GetFiles())
                {
                    var isInIncludedFolder = _includedFoldersList.Any(x => sf.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase));
                    var isInExcludedFolder = _excludedFoldersList.Any(x => sf.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase));
                    var isIncludedFile     = _includedFilesList.Any(x => IsSameFile(x.FullName, sf.FullName));
                    var isExcludedFile     = _excludedFilesList.Any(x => IsSameFile(x.FullName, sf.FullName));

                    if ((isIncludedFile == false && isInIncludedFolder == false) || isExcludedFile || isInExcludedFolder)
                    {
                        continue;
                    }
                    var df = new FileInfo(Path.Combine(target.FullName, sf.Name));
                    var wm = watermark.GetWatermark <DateTime>(sf.FullName);

                    if (force || wm == DateTime.MinValue || wm != df.LastWriteTimeUtc || !df.Exists)
                    {
                        if (df.Exists)
                        {
                            df.Attributes = FileAttributes.Normal;
                        }
                        var localSourceFile = sf;

                        watermark.ArtifactsToClean.Add(df.FullName);

                        var retriesCopy = _retryHelper.RetryAction(() => localSourceFile.CopyTo(df.FullName, true), 5, 1000);
                        if (retriesCopy > 0)
                        {
                            Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully downloaded after {2} retries", DownloadType, df.FullName, retriesCopy);
                        }

                        //To determine the the LastWrite time the object must be refreshed after copy the file.
                        df.Refresh();

                        watermark.UpdateWatermark(sf.FullName, df.LastWriteTimeUtc);

                        df.Attributes = FileAttributes.Normal;
                        Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, sf.FullName);
                    }
                    else
                    {
                        Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, sf.FullName);
                    }
                }

                foreach (var subdirectory in source.GetDirectories())
                {
                    // a folder is included if one of its parents is included (unless it is explicitly exluded)
                    var files      = _includedFilesList.Where(i => !_excludedFilesList.Any(e => IsSameFile(e.FullName, i.FullName))).ToList();
                    var isIncluded =
                        _includedFoldersList.Any(x => x.FullName.StartsWith(subdirectory.FullName, StringComparison.InvariantCultureIgnoreCase)) ||
                        !_excludedFoldersList.Any(x => x.FullName.EndsWith(subdirectory.FullName));


                    if (isIncluded)
                    {
                        var targetDi = new DirectoryInfo(Path.Combine(target.FullName, subdirectory.Name));
                        Logger.Instance()
                        .Log(TraceLevel.Verbose, "{0}: Downloading folder {1}", DownloadType,
                             Path.Combine(source.FullName, subdirectory.Name));
                        CopyDirectoryContent(root, subdirectory, targetDi, includeFilters, excludeFilters, watermark,
                                             force);
                    }
                }
            }
        }
Exemplo n.º 4
0
        private void Export(string root, string source, string destination, string versionSpec, IEnumerable <string> includeFilters, IEnumerable <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force)
        {
            var includes = ConvertFilterToRegex(includeFilters, root);
            var excludes = ConvertFilterToRegex(excludeFilters, root);

            var items    = new Dictionary <string, string>();
            var allItems = ProviderSubversion.Instance.GetItems(source, ProviderSubversion.ItemType.File, true, versionSpec);

            // apply include and exclude filter
            foreach (var item in allItems)
            {
                if ((includes.Any(x => x.IsMatch(item.Key))) && (!excludes.Any(x => x.IsMatch(item.Key))))
                {
                    items.Add(item.Key, item.Value);
                }
            }

            var revision = 0L;

            foreach (var item in items)
            {
                var destinationpath = destination;

                var subpath = item.Value.Substring(source.Length).Trim(new[] { '/' }).Replace("/", "\\");

                //Remove filename
                if (subpath.Contains("\\"))
                {
                    subpath         = subpath.Remove(subpath.LastIndexOf("\\"));
                    destinationpath = Path.Combine(destination, subpath);
                }

                // Determine revision one-time. All items have same revision.
                if (revision == 0)
                {
                    revision = ProviderSubversion.Instance.GetHeadRevision(item.Key);
                }

                var wm = watermark.GetWatermark <long>(item.Key);

                var targetFile = Path.Combine(destinationpath, Path.GetFileName(item.Value));

                if (force || 0 == wm || revision != wm || !File.Exists(targetFile))
                {
                    Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, item.Key);

                    ProviderSubversion.Instance.GetFile(item.Key, destinationpath, force, versionSpec);

                    //Save files and folders, which included in the item
                    watermark.ArtifactsToClean.Add(destinationpath);

                    foreach (var file in Directory.GetFiles(destinationpath))
                    {
                        watermark.ArtifactsToClean.Add(file);
                    }

                    //Update information about source (path and revision)
                    watermark.UpdateWatermark(item.Key, revision);
                }
                else
                {
                    Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, item);
                }
            }
        }