/// <summary>
/// Downloads all items in current folder and items in subfolders by calling itself.
/// </summary>
/// <param name="root">Root of copy operation</param>
/// <param name="source">A valid TFS version control path to a folder that shall be downloaded.</param>
/// <param name="destination">A valid directory to which all the files will be downloaded to.</param>
/// <param name="versionString">The version string to determine the version to download.</param>
/// <param name="includeFilters">The filter for file types to include.</param>
/// <param name="excludeFilters">List of filters to exclude files and folders.</param>
/// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations.</param>
/// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten</param>
private void DownloadFolder(string root, string source, string destination, string versionString, IEnumerable <string> includeFilters, IEnumerable <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force)
{
var version = VersionSpec.ParseSingleSpec(versionString, _workspace.OwnerName);
var includes = ConvertFilterToRegex(includeFilters, root);
var excludes = ConvertFilterToRegex(excludeFilters, root);
var items = _vcs.GetItems(source, version, RecursionType.Full, DeletedState.NonDeleted, ItemType.File, true).Items;
items = items.Where(x => includes.Any(y => y.IsMatch(x.ServerItem))).ToArray();
items = items.Where(x => !excludes.Any(y => y.IsMatch(x.ServerItem))).ToArray();
Parallel.ForEach(items, new ParallelOptions {
MaxDegreeOfParallelism = 8
}, item =>
{
var subpath = item.ServerItem.Substring(source.Length).Trim(new[] { '/' }).Replace("//", "\\");
var destinationpath = Path.Combine(destination, subpath);
var wm = watermark.GetWatermark <int>(item.ServerItem);
if (force || 0 == wm || item.ChangesetId != wm || !File.Exists(destinationpath))
{
if (File.Exists(destinationpath))
{
File.SetAttributes(destinationpath, FileAttributes.Normal);
}
lock (this)
{
watermark.ArtifactsToClean.Add(destinationpath);
}
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, item.ServerItem);
item.DownloadFile(destinationpath);
lock (this)
{
watermark.UpdateWatermark(item.ServerItem, item.ChangesetId);
}
}
else
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, item.ServerItem);
}
});
}
/// <summary>
/// Copy files in directory and calls CopyDirectoryContent recursively for sub directories.
/// </summary>
/// <param name="root">The root of the recursive copy operation.</param>
/// <param name="source">The directory to copy files from.</param>
/// <param name="target">The directory to copy files to.</param>
/// <param name="watermark">The watermark can be used to perform incremental updates and cleanup operations.</param>
/// <param name="force">Indicates that we want to force a get operation and all files have to be overwritten.</param>
/// <param name="includeFilters">Filter that defines what files and folders to include.</param>
/// <param name="excludeFilters">Files that should be excluded</param>
private void CopyDirectoryContent(DirectoryInfo root, DirectoryInfo source, DirectoryInfo target, IList <string> includeFilters, IList <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force)
{
if (string.Equals(source.FullName, target.FullName, StringComparison.OrdinalIgnoreCase))
{
return;
}
//Get the folders only, if filters are provided and the list with excluded and included files and folders has not been initalized yet.
// getting the folders only once, will increase the copy speed
if (includeFilters != null || excludeFilters != null)
{
if (_includedFoldersList == null || _excludedFoldersList == null || _includedFilesList == null || _excludedFilesList == null)
{
// explicitly and implicitly included and excluded folders
_includedFoldersList = includeFilters.Where(x => x.EndsWith("\\")).SelectMany(x => GetFoldersFromRoot(x, root)).ToList();
_excludedFoldersList = excludeFilters.Where(x => x.EndsWith("\\")).SelectMany(x => GetFoldersFromRoot(x, root)).ToList();
_includedFoldersList = _includedFoldersList.Concat(includeFilters.Where(x => x.EndsWith("\\")).Select(x => x.Trim('\\')).SelectMany(x => GetFoldersFromRoot(x, root)).ToList()).ToList();
_excludedFoldersList = _excludedFoldersList.Concat(excludeFilters.Where(x => x.EndsWith("\\")).Select(x => x.Trim('\\')).SelectMany(x => GetFoldersFromRoot(x, root)).ToList()).ToList();
//// generally and specifically included and excluded files
_includedFilesList = includeFilters.Where(x => x.EndsWith("\\") == false).SelectMany(x => GetFilesFromRoot(x, root)).ToList();
_excludedFilesList = excludeFilters.Where(x => x.EndsWith("\\") == false).SelectMany(x => GetFilesFromRoot(x, root)).ToList();
//Sort the list to move the root folder first
_includedFoldersList = _includedFoldersList.OrderBy(o => o.FullName.Length).ToList();
if (_excludedFoldersList.Any(x => source.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase)))
{
return;
}
}
}
watermark.ArtifactsToClean.Add(target.FullName);
if (!target.Exists)
{
// Since preliminary deletes might lead to file or folder locks, we need to retry this
var retries = _retryHelper.RetryAction(target.Create, 10, 1000);
if (retries > 0)
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Folder {1} successfully created after {2} retries", DownloadType, target, retries);
}
}
//The Move Operation Type.
if (_operationType == OperationType.Move)
{
MoveFolder(source, target, watermark);
}
else
{
// get excluded and included files
foreach (var sf in source.GetFiles())
{
var isInIncludedFolder = _includedFoldersList.Any(x => sf.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase));
var isInExcludedFolder = _excludedFoldersList.Any(x => sf.FullName.StartsWith(x.FullName, StringComparison.InvariantCultureIgnoreCase));
var isIncludedFile = _includedFilesList.Any(x => IsSameFile(x.FullName, sf.FullName));
var isExcludedFile = _excludedFilesList.Any(x => IsSameFile(x.FullName, sf.FullName));
if ((isIncludedFile == false && isInIncludedFolder == false) || isExcludedFile || isInExcludedFolder)
{
continue;
}
var df = new FileInfo(Path.Combine(target.FullName, sf.Name));
var wm = watermark.GetWatermark <DateTime>(sf.FullName);
if (force || wm == DateTime.MinValue || wm != df.LastWriteTimeUtc || !df.Exists)
{
if (df.Exists)
{
df.Attributes = FileAttributes.Normal;
}
var localSourceFile = sf;
watermark.ArtifactsToClean.Add(df.FullName);
var retriesCopy = _retryHelper.RetryAction(() => localSourceFile.CopyTo(df.FullName, true), 5, 1000);
if (retriesCopy > 0)
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: File {1} successfully downloaded after {2} retries", DownloadType, df.FullName, retriesCopy);
}
//To determine the the LastWrite time the object must be refreshed after copy the file.
df.Refresh();
watermark.UpdateWatermark(sf.FullName, df.LastWriteTimeUtc);
df.Attributes = FileAttributes.Normal;
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, sf.FullName);
}
else
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, sf.FullName);
}
}
foreach (var subdirectory in source.GetDirectories())
{
// a folder is included if one of its parents is included (unless it is explicitly exluded)
var files = _includedFilesList.Where(i => !_excludedFilesList.Any(e => IsSameFile(e.FullName, i.FullName))).ToList();
var isIncluded =
_includedFoldersList.Any(x => x.FullName.StartsWith(subdirectory.FullName, StringComparison.InvariantCultureIgnoreCase)) ||
!_excludedFoldersList.Any(x => x.FullName.EndsWith(subdirectory.FullName));
if (isIncluded)
{
var targetDi = new DirectoryInfo(Path.Combine(target.FullName, subdirectory.Name));
Logger.Instance()
.Log(TraceLevel.Verbose, "{0}: Downloading folder {1}", DownloadType,
Path.Combine(source.FullName, subdirectory.Name));
CopyDirectoryContent(root, subdirectory, targetDi, includeFilters, excludeFilters, watermark,
force);
}
}
}
}
private void Export(string root, string source, string destination, string versionSpec, IEnumerable <string> includeFilters, IEnumerable <string> excludeFilters, IDependencyDownloaderWatermark watermark, bool force)
{
var includes = ConvertFilterToRegex(includeFilters, root);
var excludes = ConvertFilterToRegex(excludeFilters, root);
var items = new Dictionary <string, string>();
var allItems = ProviderSubversion.Instance.GetItems(source, ProviderSubversion.ItemType.File, true, versionSpec);
// apply include and exclude filter
foreach (var item in allItems)
{
if ((includes.Any(x => x.IsMatch(item.Key))) && (!excludes.Any(x => x.IsMatch(item.Key))))
{
items.Add(item.Key, item.Value);
}
}
var revision = 0L;
foreach (var item in items)
{
var destinationpath = destination;
var subpath = item.Value.Substring(source.Length).Trim(new[] { '/' }).Replace("/", "\\");
//Remove filename
if (subpath.Contains("\\"))
{
subpath = subpath.Remove(subpath.LastIndexOf("\\"));
destinationpath = Path.Combine(destination, subpath);
}
// Determine revision one-time. All items have same revision.
if (revision == 0)
{
revision = ProviderSubversion.Instance.GetHeadRevision(item.Key);
}
var wm = watermark.GetWatermark <long>(item.Key);
var targetFile = Path.Combine(destinationpath, Path.GetFileName(item.Value));
if (force || 0 == wm || revision != wm || !File.Exists(targetFile))
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, item.Key);
ProviderSubversion.Instance.GetFile(item.Key, destinationpath, force, versionSpec);
//Save files and folders, which included in the item
watermark.ArtifactsToClean.Add(destinationpath);
foreach (var file in Directory.GetFiles(destinationpath))
{
watermark.ArtifactsToClean.Add(file);
}
//Update information about source (path and revision)
watermark.UpdateWatermark(item.Key, revision);
}
else
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, item);
}
}
}
/// <summary>
/// Extract a standard zip file
/// </summary>
/// <param name="target"></param>
/// <param name="targetSubdir"></param>
/// <param name="archiveFile"></param>
/// <param name="watermark"></param>
/// <param name="force"></param>
private void ExtractStandardZipFile(string target, string targetSubdir, string archiveFile, IDependencyDownloaderWatermark watermark, bool force)
{
using (var archive = ZipFile.OpenRead(archiveFile))
{
foreach (var file in archive.Entries)
{
// Extract only files, not subdirectories. If a subdirectory not exists, create it;
if (file.FullName.EndsWith("/"))
{
var subdir = Path.Combine(target, targetSubdir, file.FullName.Substring(0, file.FullName.Length - 1));
subdir = subdir.Replace("/", "\\");
if (!Directory.Exists(subdir))
{
Directory.CreateDirectory(subdir);
// For the CreateDirectory method the folder depth its not relevant, but for the cleanup, so each subfolder must
// be logged separately
if (FolderDepth(subdir) > FolderDepth(target) + 1)
{
var tmp = subdir;
do
{
watermark.ArtifactsToClean.Add(tmp);
tmp = tmp.Substring(0, tmp.LastIndexOf("\\"));
} while (tmp.Length > target.Length);
}
}
continue;
}
var df = new FileInfo(Path.Combine(target, targetSubdir, file.FullName));
var wm = watermark.GetWatermark <DateTime>(file.FullName);
if (force || wm == DateTime.MinValue || wm != df.LastWriteTimeUtc || !df.Exists)
{
// Remove possible readonly flag
if (df.Exists)
{
df.Attributes = FileAttributes.Normal;
}
watermark.ArtifactsToClean.Add(df.FullName);
try
{
file.ExtractToFile(df.FullName, true);
}
catch (Exception ex)
{
Logger.Instance().Log(TraceLevel.Error, "{0}: The archive cannot be extracted. The compression method is not supported. {1}", ex.Message, file.FullName);
throw new Exception(String.Format("The file {0} cannot be extracted. If the file is compressed with a non-standard algorithm (e.g. LZMA), make sure the support for 7-zip is enabled and all dlls are set properly in the Dependecy Manager settings.", archiveFile));
}
df.Attributes = FileAttributes.Normal;
//TODO: Klaus will auch die enthaltenen Datien im Output Window sehen!
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Downloading file {1}", DownloadType, file.FullName);
}
else
{
Logger.Instance().Log(TraceLevel.Verbose, "{0}: Skipping file {1}", DownloadType, file.FullName);
}
}
}
Logger.Instance().Log(TraceLevel.Info, "Extracted zipped file {0} finished successfully.", archiveFile);
}