/// <summary>
/// Downloads all items in current folder and items in subfolders by calling itself.
/// </summary>
/// <param name="root">Root of copy operation</param>
/// <param name="source">A valid TFS version control path to a folder that shall be downloaded.</param>
/// <param name="destination">A valid directory to which all the files will be downloaded to.</param>
/// <param name="versionString">The version string to determine the version to download.</param>
/// <param name="includeFilters">The filter for file types to include.</param>
/// <param name="excludeFilters">List of filters to exclude files and folders.</param>
/// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations.</param>
/// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten</param>
private void DownloadFolder(string root, string source, string destination, string versionString, IEnumerable <string> includeFilters, IEnumerable <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force)
{
var version = VersionSpec.ParseSingleSpec(versionString, _workspace.OwnerName);
var includes = ConvertFilterToRegex(includeFilters, root);
var excludes = ConvertFilterToRegex(excludeFilters, root);
var items = _vcs.GetItems(source, version, RecursionType.Full, DeletedState.NonDeleted, ItemType.File, true).Items;
items = items.Where(x => includes.Any(y => y.IsMatch(x.ServerItem))).ToArray();
items = items.Where(x => !excludes.Any(y => y.IsMatch(x.ServerItem))).ToArray();
Parallel.ForEach(items, new ParallelOptions {
MaxDegreeOfParallelism = 8
}, item =>
{
var subpath = item.ServerItem.Substring(source.Length).Trim(new[] { '/' }).Replace("//", "\\");
var destinationpath = Path.Combine(destination, subpath);
var wm = watermark.GetWatermark <int>(item.ServerItem);
if (force || 0 == wm || item.ChangesetId != wm || !File.Exists(destinationpath))
{
if (File.Exists(destinationpath))
{
File.SetAttributes(destinationpath, FileAttributes.Normal);
}
lock (this)
{
watermark.ArtifactsToClean.Add(destinationpath);
}
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, item.ServerItem);
item.DownloadFile(destinationpath);
lock (this)
{
watermark.UpdateWatermark(item.ServerItem, item.ChangesetId);
}
}
else
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, item.ServerItem);
}
});
}
/// <summary>
/// Recursive method to move a folder
/// A folder is moved by moving its subfolders and all files within the folder
/// If the target folder already exsist the method checks if the children of the target folder are the same.
/// It will delete existing folders and will then move them to the folder
///
/// </summary>
/// <param name="source"></param>
/// <param name="target"></param>
/// <param name="watermark"></param>
private void MoveFolder(DirectoryInfo source, DirectoryInfo target, IDependencyDownloaderWatermark watermark)
{
//Move the content of the current folder to the given output folder
foreach (string tempPath in Directory.GetDirectories(source.FullName, "*", SearchOption.TopDirectoryOnly))
{
////Check if the current folder exists on the target
//if (Directory.Exists(tempPath.Replace(source.FullName, target.FullName)))
//{
// //Check if the target contains any children. If there are no children, --> delete the folder
// //If the folder has children, check if the target children are equal to the source children --> If they are equal --> Delete
// var children = Directory.GetDirectories(tempPath.Replace(source.FullName, target.FullName), "*", SearchOption.TopDirectoryOnly);
// if (children.Count() == 0)
// {
// DeleteDirectory(tempPath.Replace(source.FullName, target.FullName));
// }
// else
// {
// //Delete each path that is similar within the children
// foreach (var tempPathChildren in Directory.GetDirectories(tempPath, "*", SearchOption.TopDirectoryOnly))
// {
// if (Directory.Exists(tempPathChildren.Replace(source.FullName, target.FullName)))
// {
// DeleteDirectory(tempPathChildren.Replace(source.FullName, target.FullName));
// }
// }
// }
//}
//If the target folder still exists, the children and files of the source can be moved
//If the target does not exsist, the whole source folder can be moved
if (Directory.Exists(tempPath.Replace(source.FullName, target.FullName)))
{
MoveFolder(new DirectoryInfo(tempPath), new DirectoryInfo(tempPath.Replace(source.FullName, target.FullName)), watermark);
}
else
{
var retriesMove = _retryHelper.RetryAction(() => Directory.Move(tempPath, tempPath.Replace(source.FullName, target.FullName)), 5, 1000);
if (retriesMove > 0)
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully moved after {2} retries", DownloadType, tempPath, retriesMove);
}
}
//Get the file info and set the FileAttributes to Normal
var fi = new DirectoryInfo(tempPath.Replace(source.FullName, target.FullName));
fi.Attributes = FileAttributes.Normal;
//Add all files in the folder to the watermarks
foreach (var file in fi.GetFiles("*.*", SearchOption.AllDirectories))
{
//Add the files to the watermark
watermark.UpdateWatermark(file.FullName, file.LastWriteTimeUtc);
}
}
//Move all the files & Replaces any files with the same name by deleting them
foreach (string tempPath in Directory.GetFiles(source.FullName, "*.*", SearchOption.TopDirectoryOnly))
{
if (File.Exists(tempPath.Replace(source.FullName, target.FullName)))
{
File.SetAttributes(tempPath.Replace(source.FullName, target.FullName), FileAttributes.Normal);
File.Delete(tempPath.Replace(source.FullName, target.FullName));
}
var retriesMove = _retryHelper.RetryAction(() => File.Move(tempPath, tempPath.Replace(source.FullName, target.FullName)), 5, 1000);
if (retriesMove > 0)
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully moved after {2} retries", DownloadType, tempPath, retriesMove);
}
//Get the file info and set the FileAttributes to Normal
var fi = new FileInfo(tempPath.Replace(source.FullName, target.FullName));
fi.Attributes = FileAttributes.Normal;
//Add the files to the watermark
watermark.UpdateWatermark(fi.FullName, fi.LastWriteTimeUtc);
}
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Moving Directory {1} to {2}", DownloadType, source.FullName, target.FullName);
}
/// <summary>
/// Copy files in directory and calls CopyDirectoryContent recursively for sub directories.
/// </summary>
/// <param name="root">The root of the recursive copy operation.</param>
/// <param name="source">The directory to copy files from.</param>
/// <param name="target">The directory to copy files to.</param>
/// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations.</param>
/// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten.</param>
/// <param name="includeFilters">Filter that defines what files and folders to include.</param>
/// <param name="excludeFilters">Files that should be excluded</param>
private void CopyDirectoryContent(DirectoryInfo root, DirectoryInfo source, DirectoryInfo target, IList <string> includeFilters, IList <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force)
{
if (string.Equals(source.FullName, target.FullName, StringComparison.OrdinalIgnoreCase))
{
return;
}
//Get the folders only, if filters are provided and the list with excluded and included files and folders has not been initalized yet.
// getting the folders only once, will increase the copy speed
if (includeFilters != null || excludeFilters != null)
{
if (_includedFoldersList == null || _excludedFoldersList == null || _includedFilesList == null || _excludedFilesList == null)
{
// explicitly and implicitly included and excluded folders
_includedFoldersList = includeFilters.Where(x => x.EndsWith("\\")).SelectMany(x => GetFoldersFromRoot(x, root)).ToList();
_excludedFoldersList = excludeFilters.Where(x => x.EndsWith("\\")).SelectMany(x => GetFoldersFromRoot(x, root)).ToList();
_includedFoldersList = _includedFoldersList.Concat(includeFilters.Where(x => x.EndsWith("\\")).Select(x => x.Trim('\\')).SelectMany(x => GetFoldersFromRoot(x, root)).ToList()).ToList();
_excludedFoldersList = _excludedFoldersList.Concat(excludeFilters.Where(x => x.EndsWith("\\")).Select(x => x.Trim('\\')).SelectMany(x => GetFoldersFromRoot(x, root)).ToList()).ToList();
//// generally and specifically included and excluded files
_includedFilesList = includeFilters.Where(x => x.EndsWith("\\") == false).SelectMany(x => GetFilesFromRoot(x, root)).ToList();
_excludedFilesList = excludeFilters.Where(x => x.EndsWith("\\") == false).SelectMany(x => GetFilesFromRoot(x, root)).ToList();
//Sort the list to move the root folder first
_includedFoldersList = _includedFoldersList.OrderBy(o => o.FullName.Length).ToList();
if (_excludedFoldersList.Any(x => source.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase)))
{
return;
}
}
}
watermark.ArtifactsToClean.Add(target.FullName);
if (!target.Exists)
{
// Since preliminary deletes might lead to file or folder locks, we need to retry this
var retries = _retryHelper.RetryAction(target.Create, 10, 1000);
if (retries > 0)
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Folder {1} successfully created after {2} retries", DownloadType, target, retries);
}
}
//The Move Operation Type.
if (_operationType == OperationType.Move)
{
MoveFolder(source, target, watermark);
}
else
{
// get excluded and included files
foreach (var sf in source.GetFiles())
{
var isInIncludedFolder = _includedFoldersList.Any(x => sf.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase));
var isInExcludedFolder = _excludedFoldersList.Any(x => sf.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase));
var isIncludedFile = _includedFilesList.Any(x => IsSameFile(x.FullName, sf.FullName));
var isExcludedFile = _excludedFilesList.Any(x => IsSameFile(x.FullName, sf.FullName));
if ((isIncludedFile == false && isInIncludedFolder == false) || isExcludedFile || isInExcludedFolder)
{
continue;
}
var df = new FileInfo(Path.Combine(target.FullName, sf.Name));
var wm = watermark.GetWatermark <DateTime>(sf.FullName);
if (force || wm == DateTime.MinValue || wm != df.LastWriteTimeUtc || !df.Exists)
{
if (df.Exists)
{
df.Attributes = FileAttributes.Normal;
}
var localSourceFile = sf;
watermark.ArtifactsToClean.Add(df.FullName);
var retriesCopy = _retryHelper.RetryAction(() => localSourceFile.CopyTo(df.FullName, true), 5, 1000);
if (retriesCopy > 0)
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully downloaded after {2} retries", DownloadType, df.FullName, retriesCopy);
}
//To determine the the LastWrite time the object must be refreshed after copy the file.
df.Refresh();
watermark.UpdateWatermark(sf.FullName, df.LastWriteTimeUtc);
df.Attributes = FileAttributes.Normal;
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, sf.FullName);
}
else
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, sf.FullName);
}
}
foreach (var subdirectory in source.GetDirectories())
{
// a folder is included if one of its parents is included (unless it is explicitly exluded)
var files = _includedFilesList.Where(i => !_excludedFilesList.Any(e => IsSameFile(e.FullName, i.FullName))).ToList();
var isIncluded =
_includedFoldersList.Any(x => x.FullName.StartsWith(subdirectory.FullName, StringComparison.InvariantCultureIgnoreCase)) ||
!_excludedFoldersList.Any(x => x.FullName.EndsWith(subdirectory.FullName));
if (isIncluded)
{
var targetDi = new DirectoryInfo(Path.Combine(target.FullName, subdirectory.Name));
Logger.Instance()
.Log(TraceLevel.Verbose, "{0}: Downloading folder {1}", DownloadType,
Path.Combine(source.FullName, subdirectory.Name));
CopyDirectoryContent(root, subdirectory, targetDi, includeFilters, excludeFilters, watermark,
force);
}
}
}
}
private void Export(string root, string source, string destination, string versionSpec, IEnumerable <string> includeFilters, IEnumerable <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force)
{
var includes = ConvertFilterToRegex(includeFilters, root);
var excludes = ConvertFilterToRegex(excludeFilters, root);
var items = new Dictionary <string, string>();
var allItems = ProviderSubversion.Instance.GetItems(source, ProviderSubversion.ItemType.File, true, versionSpec);
// apply include and exclude filter
foreach (var item in allItems)
{
if ((includes.Any(x => x.IsMatch(item.Key))) && (!excludes.Any(x => x.IsMatch(item.Key))))
{
items.Add(item.Key, item.Value);
}
}
var revision = 0L;
foreach (var item in items)
{
var destinationpath = destination;
var subpath = item.Value.Substring(source.Length).Trim(new[] { '/' }).Replace("/", "\\");
//Remove filename
if (subpath.Contains("\\"))
{
subpath = subpath.Remove(subpath.LastIndexOf("\\"));
destinationpath = Path.Combine(destination, subpath);
}
// Determine revision one-time. All items have same revision.
if (revision == 0)
{
revision = ProviderSubversion.Instance.GetHeadRevision(item.Key);
}
var wm = watermark.GetWatermark <long>(item.Key);
var targetFile = Path.Combine(destinationpath, Path.GetFileName(item.Value));
if (force || 0 == wm || revision != wm || !File.Exists(targetFile))
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, item.Key);
ProviderSubversion.Instance.GetFile(item.Key, destinationpath, force, versionSpec);
//Save files and folders, which included in the item
watermark.ArtifactsToClean.Add(destinationpath);
foreach (var file in Directory.GetFiles(destinationpath))
{
watermark.ArtifactsToClean.Add(file);
}
//Update information about source (path and revision)
watermark.UpdateWatermark(item.Key, revision);
}
else
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, item);
}
}
}