예제 #1
0
        public void Create_MissingElapsed_ReturnsValidInstance()
        {
            var stats = CopyStats.Create(_summary, null);

            Assert.Equal(0, stats.ElapsedMilliseconds);
            Assert.Equal(0, stats.BytesPerSecond);
        }
예제 #2
0
        public void Create_AllOk_ReturnsValidInstance()
        {
            var stopwatch = Stopwatch.StartNew();

            stopwatch.Stop();

            var stats = CopyStats.Create(_summary, stopwatch.Elapsed);

            Assert.Equal(stopwatch.ElapsedMilliseconds, stats.ElapsedMilliseconds);
            Assert.Equal((long)(stats.TotalBytesCopied / stopwatch.Elapsed.TotalSeconds), stats.BytesPerSecond);
        }
예제 #3
0
        /// <summary>
        /// Execute the command
        /// </summary>
        public override void ExecuteBuild()
        {
            Console.WriteLine();

            // Parse the command line arguments
            string LocalDirName = ParseParamValue("LocalDir", null);

            if (LocalDirName == null)
            {
                throw new AutomationException("Missing -LocalDir=... argument");
            }

            long MaxSize;

            if (!TryParseSize(ParseParamValue("MaxSize", "0mb"), out MaxSize))
            {
                throw new AutomationException("Invalid -MaxSize=... argument");
            }

            string[] RemoteDirNames = ParseParamValues("RemoteDir");
            if (RemoteDirNames.Length == 0)
            {
                throw new AutomationException("Missing -RemoteDir=... argument");
            }

            int MaxDays;

            if (!int.TryParse(ParseParamValue("MaxDays", "3"), out MaxDays))
            {
                throw new AutomationException("Invalid -MaxDays=... argument");
            }

            int TimeLimit;

            if (!TryParseTime(ParseParamValue("TimeLimit", "0m"), out TimeLimit))
            {
                throw new AutomationException("Invalid -TimeLimit=... argument");
            }

            bool bPreview = ParseParam("Preview");

            // Make sure the source directory exists
            List <DirectoryInfo> RemoteDirs = new List <DirectoryInfo>();

            foreach (string RemoteDirName in RemoteDirNames)
            {
                DirectoryInfo RemoteDir = new DirectoryInfo(RemoteDirName);
                if (!RemoteDir.Exists)
                {
                    throw new AutomationException("Remote directory '{0}' does not exist", RemoteDirName);
                }
                RemoteDirs.Add(RemoteDir);
            }

            // Get the local directory
            DirectoryInfo LocalDir = new DirectoryInfo(LocalDirName);

            if (!LocalDir.Exists)
            {
                LocalDir.Create();
            }

            // Create all the base DDC directory names. These are three entries deep, each numbered 0-9.
            List <string> BasePathPrefixes = new List <string>();

            for (int IndexA = 0; IndexA <= 9; IndexA++)
            {
                for (int IndexB = 0; IndexB <= 9; IndexB++)
                {
                    for (int IndexC = 0; IndexC <= 9; IndexC++)
                    {
                        BasePathPrefixes.Add(String.Format("{0}{3}{1}{3}{2}{3}", IndexA, IndexB, IndexC, Path.DirectorySeparatorChar));
                    }
                }
            }

            // Find all the local files
            ConcurrentBag <CacheFile> LocalFiles = new ConcurrentBag <CacheFile>();

            Console.WriteLine("Enumerating local files from {0}", LocalDir.FullName);
            ForEach(BasePathPrefixes, (BasePath, Messages) => (() => EnumerateFiles(LocalDir, BasePath, LocalFiles)), "Enumerating files...");
            Console.WriteLine("Found {0} files, {1}mb.", LocalFiles.Count, LocalFiles.Sum(x => x.Length) / (1024 * 1024));
            Console.WriteLine();

            // Find all the remote files
            ConcurrentBag <CacheFile> RemoteFiles = new ConcurrentBag <CacheFile>();

            foreach (DirectoryInfo RemoteDir in RemoteDirs)
            {
                Console.WriteLine("Enumerating remote files from {0}", RemoteDir.FullName);
                ForEach(BasePathPrefixes, (BasePath, Messages) => (() => EnumerateFiles(RemoteDir, BasePath, RemoteFiles)), "Enumerating files...");
                Console.WriteLine("Found {0} files, {1}mb.", RemoteFiles.Count, RemoteFiles.Sum(x => x.Length) / (1024 * 1024));
                Console.WriteLine();
            }

            // Get the oldest file that we want to copy
            DateTime OldestLastWriteTimeUtc = DateTime.Now - TimeSpan.FromDays(MaxDays);

            // Build a lookup of remote files by name
            Dictionary <string, CacheFile> RelativePathToRemoteFile = new Dictionary <string, CacheFile>(StringComparer.InvariantCultureIgnoreCase);

            foreach (CacheFile RemoteFile in RemoteFiles)
            {
                if (RemoteFile.LastWriteTimeUtc > OldestLastWriteTimeUtc)
                {
                    RelativePathToRemoteFile[RemoteFile.RelativePath] = RemoteFile;
                }
            }

            // Build a lookup of local files by name
            Dictionary <string, CacheFile> RelativePathToLocalFile = LocalFiles.ToDictionary(x => x.RelativePath, x => x, StringComparer.InvariantCultureIgnoreCase);

            // Build a list of target files that we want in the DDC
            long TotalSize = 0;
            Dictionary <string, CacheFile> RelativePathToTargetFile = new Dictionary <string, CacheFile>(StringComparer.InvariantCultureIgnoreCase);

            foreach (CacheFile TargetFile in Enumerable.Concat <CacheFile>(RelativePathToLocalFile.Values, RelativePathToRemoteFile.Values).OrderByDescending(x => x.LastWriteTimeUtc))
            {
                if (MaxSize > 0 && TotalSize + TargetFile.Length > MaxSize)
                {
                    break;
                }
                if (!RelativePathToTargetFile.ContainsKey(TargetFile.RelativePath))
                {
                    RelativePathToTargetFile.Add(TargetFile.RelativePath, TargetFile);
                    TotalSize += TargetFile.Length;
                }
            }

            // Print measure of how coherent the cache is
            double CoherencyPct = RelativePathToTargetFile.Values.Count(x => RelativePathToLocalFile.ContainsKey(x.RelativePath)) * 100.0 / RelativePathToTargetFile.Count;

            Console.WriteLine("Cache is {0:0.0}% coherent with remote.", CoherencyPct);
            Console.WriteLine();

            // Remove any outdated files
            List <CacheFile> FilesToRemove = RelativePathToLocalFile.Values.Where(x => !RelativePathToTargetFile.ContainsKey(x.RelativePath)).ToList();

            if (bPreview)
            {
                Console.WriteLine("Sync would remove {0} files ({1}mb)", FilesToRemove.Count, FilesToRemove.Sum(x => x.Length) / (1024 * 1024));
            }
            else if (FilesToRemove.Count > 0)
            {
                Console.WriteLine("Deleting {0} files ({1}mb)...", FilesToRemove.Count, FilesToRemove.Sum(x => x.Length) / (1024 * 1024));
                ForEach(FilesToRemove, (File, Messages) => (() => RemoveFile(LocalDir, File.RelativePath, Messages)), "Deleting files");
                Console.WriteLine();
            }

            // Add any new files
            List <CacheFile> FilesToAdd = RelativePathToTargetFile.Values.Where(x => !RelativePathToLocalFile.ContainsKey(x.RelativePath)).ToList();

            if (bPreview)
            {
                Console.WriteLine("Sync would add {0} files ({1}mb)", FilesToAdd.Count, FilesToAdd.Sum(x => x.Length) / (1024 * 1024));
            }
            else if (FilesToAdd.Count > 0)
            {
                Console.WriteLine("Copying {0} files ({1}mb)...", FilesToAdd.Count, FilesToAdd.Sum(x => x.Length) / (1024 * 1024));

                CancellationTokenSource CancellationTokenSource = new CancellationTokenSource();
                if (TimeLimit > 0)
                {
                    CancellationTokenSource.CancelAfter(TimeLimit * 1000);
                }

                DateTime  StartTime = DateTime.UtcNow;
                CopyStats Stats     = new CopyStats();
                ForEach(FilesToAdd, (File, Messages) => (() => CopyFile(File, LocalDir, Messages, Stats)), "Copying files...", CancellationTokenSource.Token);

                double TotalSizeMb = Stats.NumBytes / (1024.0 * 1024.0);
                Console.WriteLine("Copied {0} files totalling {1:0.0}mb ({2:0.00}mb/s).", Stats.NumFiles, TotalSizeMb, TotalSizeMb / (DateTime.UtcNow - StartTime).TotalSeconds);

                double FinalCoherencyPct = (RelativePathToTargetFile.Values.Count(x => RelativePathToLocalFile.ContainsKey(x.RelativePath)) + Stats.NumFiles) * 100.0 / RelativePathToTargetFile.Count;
                Console.WriteLine();
                Console.WriteLine("Final cache is {0:0.0}% coherent with remote.", FinalCoherencyPct);

                if (CancellationTokenSource.IsCancellationRequested)
                {
                    Console.WriteLine("Halting due to expired time limit.");
                }
            }

            Console.WriteLine();
        }
예제 #4
0
        /// <summary>
        /// Copies a file from one DDC to another
        /// </summary>
        /// <param name="SourceFile">File to copy</param>
        /// <param name="TargetDir">Target DDC directory</param>
        /// <param name="Messages">Queue to receieve error messages</param>
        /// <param name="Stats">Stats for the files copied</param>
        static void CopyFile(CacheFile SourceFile, DirectoryInfo TargetDir, ConcurrentQueue <string> Messages, CopyStats Stats)
        {
            int NumRetries = 0;

            for (;;)
            {
                // Try to copy the file, and return if we succeed
                string Message;
                if (TryCopyFile(SourceFile, TargetDir, out Message))
                {
                    Interlocked.Increment(ref Stats.NumFiles);
                    Interlocked.Add(ref Stats.NumBytes, SourceFile.Length);
                    break;
                }

                // Increment the number of retries
                NumRetries++;

                // Give up after retrying 5 times, and report the last error
                if (NumRetries >= 3)
                {
                    // Check that the source file still actually exists. If it doesn't, we'll ignore the errors.
                    try
                    {
                        if (!File.Exists(SourceFile.FullName))
                        {
                            Interlocked.Increment(ref Stats.NumFiles);
                            break;
                        }
                    }
                    catch
                    {
                    }

                    // Otherwise queue up the error message
                    Messages.Enqueue(Message);
                    break;
                }
            }
        }
예제 #5
0
        public void Create_SummaryMissing_ThrowsException()
        {
            var error = Assert.Throws <ArgumentNullException>(() => CopyStats.Create(null, null));

            Assert.Equal("summary", error.ParamName);
        }