public void Create_MissingElapsed_ReturnsValidInstance()
{
var stats = CopyStats.Create(_summary, null);
Assert.Equal(0, stats.ElapsedMilliseconds);
Assert.Equal(0, stats.BytesPerSecond);
}
public void Create_AllOk_ReturnsValidInstance()
{
var stopwatch = Stopwatch.StartNew();
stopwatch.Stop();
var stats = CopyStats.Create(_summary, stopwatch.Elapsed);
Assert.Equal(stopwatch.ElapsedMilliseconds, stats.ElapsedMilliseconds);
Assert.Equal((long)(stats.TotalBytesCopied / stopwatch.Elapsed.TotalSeconds), stats.BytesPerSecond);
}
/// <summary>
/// Execute the command
/// </summary>
public override void ExecuteBuild()
{
Console.WriteLine();
// Parse the command line arguments
string LocalDirName = ParseParamValue("LocalDir", null);
if (LocalDirName == null)
{
throw new AutomationException("Missing -LocalDir=... argument");
}
long MaxSize;
if (!TryParseSize(ParseParamValue("MaxSize", "0mb"), out MaxSize))
{
throw new AutomationException("Invalid -MaxSize=... argument");
}
string[] RemoteDirNames = ParseParamValues("RemoteDir");
if (RemoteDirNames.Length == 0)
{
throw new AutomationException("Missing -RemoteDir=... argument");
}
int MaxDays;
if (!int.TryParse(ParseParamValue("MaxDays", "3"), out MaxDays))
{
throw new AutomationException("Invalid -MaxDays=... argument");
}
int TimeLimit;
if (!TryParseTime(ParseParamValue("TimeLimit", "0m"), out TimeLimit))
{
throw new AutomationException("Invalid -TimeLimit=... argument");
}
bool bPreview = ParseParam("Preview");
// Make sure the source directory exists
List <DirectoryInfo> RemoteDirs = new List <DirectoryInfo>();
foreach (string RemoteDirName in RemoteDirNames)
{
DirectoryInfo RemoteDir = new DirectoryInfo(RemoteDirName);
if (!RemoteDir.Exists)
{
throw new AutomationException("Remote directory '{0}' does not exist", RemoteDirName);
}
RemoteDirs.Add(RemoteDir);
}
// Get the local directory
DirectoryInfo LocalDir = new DirectoryInfo(LocalDirName);
if (!LocalDir.Exists)
{
LocalDir.Create();
}
// Create all the base DDC directory names. These are three entries deep, each numbered 0-9.
List <string> BasePathPrefixes = new List <string>();
for (int IndexA = 0; IndexA <= 9; IndexA++)
{
for (int IndexB = 0; IndexB <= 9; IndexB++)
{
for (int IndexC = 0; IndexC <= 9; IndexC++)
{
BasePathPrefixes.Add(String.Format("{0}{3}{1}{3}{2}{3}", IndexA, IndexB, IndexC, Path.DirectorySeparatorChar));
}
}
}
// Find all the local files
ConcurrentBag <CacheFile> LocalFiles = new ConcurrentBag <CacheFile>();
Console.WriteLine("Enumerating local files from {0}", LocalDir.FullName);
ForEach(BasePathPrefixes, (BasePath, Messages) => (() => EnumerateFiles(LocalDir, BasePath, LocalFiles)), "Enumerating files...");
Console.WriteLine("Found {0} files, {1}mb.", LocalFiles.Count, LocalFiles.Sum(x => x.Length) / (1024 * 1024));
Console.WriteLine();
// Find all the remote files
ConcurrentBag <CacheFile> RemoteFiles = new ConcurrentBag <CacheFile>();
foreach (DirectoryInfo RemoteDir in RemoteDirs)
{
Console.WriteLine("Enumerating remote files from {0}", RemoteDir.FullName);
ForEach(BasePathPrefixes, (BasePath, Messages) => (() => EnumerateFiles(RemoteDir, BasePath, RemoteFiles)), "Enumerating files...");
Console.WriteLine("Found {0} files, {1}mb.", RemoteFiles.Count, RemoteFiles.Sum(x => x.Length) / (1024 * 1024));
Console.WriteLine();
}
// Get the oldest file that we want to copy
DateTime OldestLastWriteTimeUtc = DateTime.Now - TimeSpan.FromDays(MaxDays);
// Build a lookup of remote files by name
Dictionary <string, CacheFile> RelativePathToRemoteFile = new Dictionary <string, CacheFile>(StringComparer.InvariantCultureIgnoreCase);
foreach (CacheFile RemoteFile in RemoteFiles)
{
if (RemoteFile.LastWriteTimeUtc > OldestLastWriteTimeUtc)
{
RelativePathToRemoteFile[RemoteFile.RelativePath] = RemoteFile;
}
}
// Build a lookup of local files by name
Dictionary <string, CacheFile> RelativePathToLocalFile = LocalFiles.ToDictionary(x => x.RelativePath, x => x, StringComparer.InvariantCultureIgnoreCase);
// Build a list of target files that we want in the DDC
long TotalSize = 0;
Dictionary <string, CacheFile> RelativePathToTargetFile = new Dictionary <string, CacheFile>(StringComparer.InvariantCultureIgnoreCase);
foreach (CacheFile TargetFile in Enumerable.Concat <CacheFile>(RelativePathToLocalFile.Values, RelativePathToRemoteFile.Values).OrderByDescending(x => x.LastWriteTimeUtc))
{
if (MaxSize > 0 && TotalSize + TargetFile.Length > MaxSize)
{
break;
}
if (!RelativePathToTargetFile.ContainsKey(TargetFile.RelativePath))
{
RelativePathToTargetFile.Add(TargetFile.RelativePath, TargetFile);
TotalSize += TargetFile.Length;
}
}
// Print measure of how coherent the cache is
double CoherencyPct = RelativePathToTargetFile.Values.Count(x => RelativePathToLocalFile.ContainsKey(x.RelativePath)) * 100.0 / RelativePathToTargetFile.Count;
Console.WriteLine("Cache is {0:0.0}% coherent with remote.", CoherencyPct);
Console.WriteLine();
// Remove any outdated files
List <CacheFile> FilesToRemove = RelativePathToLocalFile.Values.Where(x => !RelativePathToTargetFile.ContainsKey(x.RelativePath)).ToList();
if (bPreview)
{
Console.WriteLine("Sync would remove {0} files ({1}mb)", FilesToRemove.Count, FilesToRemove.Sum(x => x.Length) / (1024 * 1024));
}
else if (FilesToRemove.Count > 0)
{
Console.WriteLine("Deleting {0} files ({1}mb)...", FilesToRemove.Count, FilesToRemove.Sum(x => x.Length) / (1024 * 1024));
ForEach(FilesToRemove, (File, Messages) => (() => RemoveFile(LocalDir, File.RelativePath, Messages)), "Deleting files");
Console.WriteLine();
}
// Add any new files
List <CacheFile> FilesToAdd = RelativePathToTargetFile.Values.Where(x => !RelativePathToLocalFile.ContainsKey(x.RelativePath)).ToList();
if (bPreview)
{
Console.WriteLine("Sync would add {0} files ({1}mb)", FilesToAdd.Count, FilesToAdd.Sum(x => x.Length) / (1024 * 1024));
}
else if (FilesToAdd.Count > 0)
{
Console.WriteLine("Copying {0} files ({1}mb)...", FilesToAdd.Count, FilesToAdd.Sum(x => x.Length) / (1024 * 1024));
CancellationTokenSource CancellationTokenSource = new CancellationTokenSource();
if (TimeLimit > 0)
{
CancellationTokenSource.CancelAfter(TimeLimit * 1000);
}
DateTime StartTime = DateTime.UtcNow;
CopyStats Stats = new CopyStats();
ForEach(FilesToAdd, (File, Messages) => (() => CopyFile(File, LocalDir, Messages, Stats)), "Copying files...", CancellationTokenSource.Token);
double TotalSizeMb = Stats.NumBytes / (1024.0 * 1024.0);
Console.WriteLine("Copied {0} files totalling {1:0.0}mb ({2:0.00}mb/s).", Stats.NumFiles, TotalSizeMb, TotalSizeMb / (DateTime.UtcNow - StartTime).TotalSeconds);
double FinalCoherencyPct = (RelativePathToTargetFile.Values.Count(x => RelativePathToLocalFile.ContainsKey(x.RelativePath)) + Stats.NumFiles) * 100.0 / RelativePathToTargetFile.Count;
Console.WriteLine();
Console.WriteLine("Final cache is {0:0.0}% coherent with remote.", FinalCoherencyPct);
if (CancellationTokenSource.IsCancellationRequested)
{
Console.WriteLine("Halting due to expired time limit.");
}
}
Console.WriteLine();
}
/// <summary>
/// Copies a file from one DDC to another
/// </summary>
/// <param name="SourceFile">File to copy</param>
/// <param name="TargetDir">Target DDC directory</param>
/// <param name="Messages">Queue to receieve error messages</param>
/// <param name="Stats">Stats for the files copied</param>
static void CopyFile(CacheFile SourceFile, DirectoryInfo TargetDir, ConcurrentQueue <string> Messages, CopyStats Stats)
{
int NumRetries = 0;
for (;;)
{
// Try to copy the file, and return if we succeed
string Message;
if (TryCopyFile(SourceFile, TargetDir, out Message))
{
Interlocked.Increment(ref Stats.NumFiles);
Interlocked.Add(ref Stats.NumBytes, SourceFile.Length);
break;
}
// Increment the number of retries
NumRetries++;
// Give up after retrying 5 times, and report the last error
if (NumRetries >= 3)
{
// Check that the source file still actually exists. If it doesn't, we'll ignore the errors.
try
{
if (!File.Exists(SourceFile.FullName))
{
Interlocked.Increment(ref Stats.NumFiles);
break;
}
}
catch
{
}
// Otherwise queue up the error message
Messages.Enqueue(Message);
break;
}
}
}
public void Create_SummaryMissing_ThrowsException()
{
var error = Assert.Throws <ArgumentNullException>(() => CopyStats.Create(null, null));
Assert.Equal("summary", error.ParamName);
}