/// <summary>
/// Removes any directory at the specified path which has a file matching the provided name
/// older than the specified number of days. Used by code that writes a .token file to temp
/// folders
/// </summary>
/// <param name="InPath"></param>
/// <param name="FileName"></param>
/// <param name="Days"></param>
public static void CleanupMarkedDirectories(string InPath, int Days)
{
DirectoryInfo Di = new DirectoryInfo(InPath);
if (Di.Exists == false)
{
return;
}
foreach (DirectoryInfo SubDir in Di.GetDirectories())
{
bool HasFile =
SubDir.GetFiles().Where(F => {
int DaysOld = (DateTime.Now - F.LastWriteTime).Days;
if (DaysOld >= Days)
{
// use the old and new tokennames
return(string.Equals(F.Name, "gauntlet.tempdir", StringComparison.OrdinalIgnoreCase) ||
string.Equals(F.Name, "gauntlet.token", StringComparison.OrdinalIgnoreCase));
}
return(false);
}).Count() > 0;
if (HasFile)
{
Log.Info("Removing old directory {0}", SubDir.Name);
try
{
SubDir.Delete(true);
}
catch (Exception Ex)
{
Log.Warning("Failed to remove old directory {0}. {1}", SubDir.FullName, Ex.Message);
}
}
else
{
CleanupMarkedDirectories(SubDir.FullName, Days);
}
}
}
/// <summary>
/// Copies src to dest by comparing files sizes and time stamps and only copying files that are different in src. Basically a more flexible
/// robocopy
/// </summary>
/// <param name="SourcePath"></param>
/// <param name="DestPath"></param>
/// <param name="Verbose"></param>
public static void CopyDirectory(string SourceDirPath, string DestDirPath, CopyOptions Options, Func <string, string> Transform, int RetryCount = 5)
{
DateTime StartTime = DateTime.Now;
DirectoryInfo SourceDir = new DirectoryInfo(SourceDirPath);
DirectoryInfo DestDir = new DirectoryInfo(DestDirPath);
if (DestDir.Exists == false)
{
DestDir = Directory.CreateDirectory(DestDir.FullName);
}
System.IO.FileInfo[] SourceFiles = SourceDir.GetFiles("*", SearchOption.AllDirectories);
System.IO.FileInfo[] DestFiles = DestDir.GetFiles("*", SearchOption.AllDirectories);
// Convert dest into a map of relative paths to absolute
Dictionary <string, System.IO.FileInfo> DestStructure = new Dictionary <string, System.IO.FileInfo>();
foreach (FileInfo Info in DestFiles)
{
string RelativePath = Info.FullName.Replace(DestDir.FullName, "");
// remove leading seperator
if (RelativePath.First() == Path.DirectorySeparatorChar)
{
RelativePath = RelativePath.Substring(1);
}
DestStructure[RelativePath] = Info;
}
// List of relative-path files to copy to dest
List <string> CopyList = new List <string>();
// List of relative path files in dest to delete
List <string> DeletionList = new List <string>();
foreach (FileInfo SourceInfo in SourceFiles)
{
string SourceFilePath = SourceInfo.FullName.Replace(SourceDir.FullName, "");
// remove leading seperator
if (SourceFilePath.First() == Path.DirectorySeparatorChar)
{
SourceFilePath = SourceFilePath.Substring(1);
}
string DestFilePath = Transform(SourceFilePath);
if (DestStructure.ContainsKey(DestFilePath) == false)
{
// No copy in dest, add it to the list
CopyList.Add(SourceFilePath);
}
else
{
// Check the file is the same version
FileInfo DestInfo = DestStructure[DestFilePath];
// Difference in ticks. Even though we set the dest to the src there still appears to be minute
// differences in ticks. 1ms is 10k ticks...
Int64 TimeDelta = Math.Abs(DestInfo.LastWriteTime.Ticks - SourceInfo.LastWriteTime.Ticks);
Int64 Threshhold = 100000;
if (DestInfo.Length != SourceInfo.Length ||
TimeDelta > Threshhold)
{
CopyList.Add(SourceFilePath);
}
// Remove it from the map
DestStructure.Remove(DestFilePath);
}
}
// If set to mirror, delete all the files that were not in source
if ((Options & CopyOptions.Mirror) == CopyOptions.Mirror)
{
// Now go through the remaining map items and delete them
foreach (var Pair in DestStructure)
{
DeletionList.Add(Pair.Key);
}
foreach (string RelativePath in DeletionList)
{
FileInfo DestInfo = new FileInfo(Path.Combine(DestDir.FullName, RelativePath));
Log.Verbose("Deleting extra file {0}", DestInfo.FullName);
try
{
// avoid an UnauthorizedAccessException by making sure file isn't read only
DestInfo.IsReadOnly = false;
DestInfo.Delete();
}
catch (Exception Ex)
{
Log.Warning("Failed to delete file {0}. {1}", DestInfo.FullName, Ex);
}
}
// delete empty directories
DirectoryInfo DestDirInfo = new DirectoryInfo(DestDirPath);
DirectoryInfo[] AllSubDirs = DestDirInfo.GetDirectories("*", SearchOption.AllDirectories);
foreach (DirectoryInfo SubDir in AllSubDirs)
{
try
{
if (SubDir.GetFiles().Length == 0 && SubDir.GetDirectories().Length == 0)
{
Log.Verbose("Deleting empty dir {0}", SubDir.FullName);
SubDir.Delete(true);
}
}
catch (Exception Ex)
{
// handle the case where a file is locked
Log.Info("Failed to delete directory {0}. {1}", SubDir.FullName, Ex);
}
}
}
CancellationTokenSource CTS = new CancellationTokenSource();
// todo - make param..
var POptions = new ParallelOptions {
MaxDegreeOfParallelism = 1, CancellationToken = CTS.Token
};
// install a cancel handler so we can stop parallel-for gracefully
Action CancelHandler = delegate()
{
CTS.Cancel();
};
Globals.AbortHandlers.Add(CancelHandler);
// now do the work
Parallel.ForEach(CopyList, POptions, RelativePath =>
{
// ensure path exists
string DestPath = Path.Combine(DestDir.FullName, RelativePath);
if (Transform != null)
{
DestPath = Transform(DestPath);
}
FileInfo DestInfo = new FileInfo(DestPath);
FileInfo SrcInfo = new FileInfo(Path.Combine(SourceDir.FullName, RelativePath));
// ensure directory exists
DestInfo.Directory.Create();
string DestFile = DestInfo.FullName;
if (Transform != null)
{
DestFile = Transform(DestFile);
}
int Tries = 0;
bool Copied = false;
do
{
try
{
Log.Verbose("Copying to {0}", DestFile);
SrcInfo.CopyTo(DestFile, true);
// Clear and read-only attrib and set last write time
FileInfo DestFileInfo = new FileInfo(DestFile);
DestFileInfo.IsReadOnly = false;
DestFileInfo.LastWriteTime = SrcInfo.LastWriteTime;
Copied = true;
}
catch (Exception ex)
{
if (Tries++ < RetryCount)
{
Log.Info("Copy to {0} failed, retrying {1} of {2} in 30 secs..", DestFile, Tries, RetryCount);
// todo - make param..
Thread.Sleep(30000);
}
else
{
Log.Error("File Copy failed with {0}.", ex.Message);
throw new Exception(string.Format("File Copy failed with {0}.", ex.Message));
}
}
} while (Copied == false);
});
TimeSpan Duration = DateTime.Now - StartTime;
if (Duration.TotalSeconds > 10)
{
Log.Verbose("Copied Directory in {0}", Duration.ToString(@"mm\m\:ss\s"));
}
// remove cancel handler
Globals.AbortHandlers.Remove(CancelHandler);
}
/// <summary>
/// Copies src to dest by comparing files sizes and time stamps and only copying files that are different in src. Basically a more flexible
/// robocopy
/// </summary>
/// <param name="SourcePath"></param>
/// <param name="DestPath"></param>
/// <param name="Verbose"></param>
public static void CopyDirectory(string SourceDirPath, string DestDirPath, CopyDirectoryOptions Options)
{
DateTime StartTime = DateTime.Now;
DirectoryInfo SourceDir = new DirectoryInfo(SourceDirPath);
DirectoryInfo DestDir = new DirectoryInfo(DestDirPath);
if (DestDir.Exists == false)
{
DestDir = Directory.CreateDirectory(DestDir.FullName);
}
bool IsMirroring = (Options.Mode & CopyOptions.Mirror) == CopyOptions.Mirror;
if (IsMirroring && !Options.IsDirectoryPattern)
{
Log.Warning("Can only use mirror with pattern that includes whole directories (e.g. '*')");
IsMirroring = false;
}
IEnumerable <FileInfo> SourceFiles = null;
FileInfo[] DestFiles = null;
// find all files. If a directory get them all, else use the pattern/regex
if (Options.IsDirectoryPattern)
{
SourceFiles = SourceDir.GetFiles("*", Options.Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly);
}
else
{
if (Options.Regex == null)
{
SourceFiles = SourceDir.GetFiles(Options.Pattern, Options.Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly);
}
else
{
SourceFiles = SourceDir.GetFiles("*", Options.Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly);
SourceFiles = SourceFiles.Where(F => Options.Regex.IsMatch(F.Name));
}
}
// Convert dest into a map of relative paths to absolute
Dictionary <string, System.IO.FileInfo> DestStructure = new Dictionary <string, System.IO.FileInfo>();
if (IsMirroring)
{
DestFiles = DestDir.GetFiles("*", SearchOption.AllDirectories);
foreach (FileInfo Info in DestFiles)
{
string RelativePath = Info.FullName.Replace(DestDir.FullName, "");
// remove leading seperator
if (RelativePath.First() == Path.DirectorySeparatorChar)
{
RelativePath = RelativePath.Substring(1);
}
DestStructure[RelativePath] = Info;
}
}
// List of relative-path files to copy to dest
List <string> CopyList = new List <string>();
// List of relative path files in dest to delete
List <string> DeletionList = new List <string>();
foreach (FileInfo SourceInfo in SourceFiles)
{
string SourceFilePath = SourceInfo.FullName.Replace(SourceDir.FullName, "");
// remove leading seperator
if (SourceFilePath.First() == Path.DirectorySeparatorChar)
{
SourceFilePath = SourceFilePath.Substring(1);
}
string DestFilePath = Options.Transform(SourceFilePath);
FileInfo DestInfo = null;
// We may have destination info if mirroring where we prebuild it all, if not
// grab it now
if (DestStructure.ContainsKey(DestFilePath))
{
DestInfo = DestStructure[DestFilePath];
}
else
{
string FullDestPath = Path.Combine(DestDir.FullName, DestFilePath);
if (File.Exists(FullDestPath))
{
DestInfo = new FileInfo(FullDestPath);
}
}
if (DestInfo == null)
{
// No copy in dest, add it to the list
CopyList.Add(SourceFilePath);
}
else
{
// Check the file is the same version
// Difference in ticks. Even though we set the dest to the src there still appears to be minute
// differences in ticks. 1ms is 10k ticks...
Int64 TimeDelta = Math.Abs(DestInfo.LastWriteTime.Ticks - SourceInfo.LastWriteTime.Ticks);
Int64 Threshhold = 100000;
if (DestInfo.Length != SourceInfo.Length ||
TimeDelta > Threshhold)
{
CopyList.Add(SourceFilePath);
}
else
{
if (Options.Verbose)
{
Log.Info("Will skip copy to {0}. File up to date.", DestInfo.FullName);
}
else
{
Log.Verbose("Will skip copy to {0}. File up to date.", DestInfo.FullName);
}
}
// Remove it from the map
DestStructure.Remove(DestFilePath);
}
}
// If set to mirror, delete all the files that were not in source
if (IsMirroring)
{
// Now go through the remaining map items and delete them
foreach (var Pair in DestStructure)
{
DeletionList.Add(Pair.Key);
}
foreach (string RelativePath in DeletionList)
{
FileInfo DestInfo = new FileInfo(Path.Combine(DestDir.FullName, RelativePath));
if (Options.Verbose)
{
Log.Info("Deleting extra file {0}", DestInfo.FullName);
}
else
{
Log.Verbose("Deleting extra file {0}", DestInfo.FullName);
}
try
{
// avoid an UnauthorizedAccessException by making sure file isn't read only
DestInfo.IsReadOnly = false;
DestInfo.Delete();
}
catch (Exception Ex)
{
Log.Warning("Failed to delete file {0}. {1}", DestInfo.FullName, Ex);
}
}
// delete empty directories
DirectoryInfo DestDirInfo = new DirectoryInfo(DestDirPath);
DirectoryInfo[] AllSubDirs = DestDirInfo.GetDirectories("*", SearchOption.AllDirectories);
foreach (DirectoryInfo SubDir in AllSubDirs)
{
try
{
if (SubDir.GetFiles().Length == 0 && SubDir.GetDirectories().Length == 0)
{
if (Options.Verbose)
{
Log.Info("Deleting empty dir {0}", SubDir.FullName);
}
else
{
Log.Verbose("Deleting empty dir {0}", SubDir.FullName);
}
SubDir.Delete(true);
}
}
catch (Exception Ex)
{
// handle the case where a file is locked
Log.Info("Failed to delete directory {0}. {1}", SubDir.FullName, Ex);
}
}
}
CancellationTokenSource CTS = new CancellationTokenSource();
// todo - make param..
var POptions = new ParallelOptions {
MaxDegreeOfParallelism = 1, CancellationToken = CTS.Token
};
// install a cancel handler so we can stop parallel-for gracefully
Action CancelHandler = delegate()
{
CTS.Cancel();
};
Globals.AbortHandlers.Add(CancelHandler);
// now do the work
Parallel.ForEach(CopyList, POptions, RelativePath =>
{
// ensure path exists
string DestPath = Path.Combine(DestDir.FullName, RelativePath);
if (Options.Transform != null)
{
DestPath = Options.Transform(DestPath);
}
string SourcePath = Path.Combine(SourceDir.FullName, RelativePath);
FileInfo DestInfo;
FileInfo SrcInfo;
// wrap FileInfo creation with exception handler as can throw and want informative error
try
{
DestInfo = new FileInfo(DestPath);
SrcInfo = new FileInfo(SourcePath);
}
catch (Exception Ex)
{
throw new Exception(string.Format("FileInfo creation failed for Source:{0}, Dest:{1}, with: {2}", SourcePath, DestPath, Ex.Message));
}
// ensure directory exists
DestInfo.Directory.Create();
string DestFile = DestInfo.FullName;
if (Options.Transform != null)
{
DestFile = Options.Transform(DestFile);
}
int Tries = 0;
bool Copied = false;
do
{
try
{
if (Options.Verbose)
{
Log.Info("Copying to {0}", DestFile);
}
else
{
Log.Verbose("Copying to {0}", DestFile);
}
SrcInfo.CopyTo(DestFile, true);
// Clear and read-only attrib and set last write time
FileInfo DestFileInfo = new FileInfo(DestFile);
DestFileInfo.IsReadOnly = false;
DestFileInfo.LastWriteTime = SrcInfo.LastWriteTime;
Copied = true;
}
catch (Exception ex)
{
if (Tries++ < Options.Retries)
{
Log.Info("Copy to {0} failed, retrying {1} of {2} in 30 secs..", DestFile, Tries, Options.Retries);
// todo - make param..
Thread.Sleep(30000);
}
else
{
using (var PauseEC = new ScopedSuspendECErrorParsing())
{
Log.Error("File Copy failed with {0}.", ex.Message);
}
throw new Exception(string.Format("File Copy failed with {0}.", ex.Message));
}
}
} while (Copied == false);
});
TimeSpan Duration = DateTime.Now - StartTime;
if (Duration.TotalSeconds > 10)
{
if (Options.Verbose)
{
Log.Info("Copied Directory in {0}", Duration.ToString(@"mm\m\:ss\s"));
}
else
{
Log.Verbose("Copied Directory in {0}", Duration.ToString(@"mm\m\:ss\s"));
}
}
// remove cancel handler
Globals.AbortHandlers.Remove(CancelHandler);
}