private void CopyFileUsingLocks(string sourceFilePath, string targetFilePath) { var sourceFile = new FileInfo(sourceFilePath); if (!sourceFile.Exists) { Assert.Fail("Source file not found: " + sourceFile); } // This copy will overwrite the file mFileTools.CopyFileUsingLocks(sourceFilePath, targetFilePath, true); System.Threading.Thread.Sleep(150); // This copy will not overwrite the file, though CopyFileUsingLocks returns True because the file exists var success1 = mFileTools.CopyFileUsingLocks(sourceFilePath, targetFilePath, false); Assert.IsTrue(success1, "CopyFileUsingLocks did not return true (test 1)"); // This copy will not overwrite the file, though CopyFileUsingLocks returns True because the file exists var success2 = mFileTools.CopyFileUsingLocks(sourceFilePath, targetFilePath, "TestClass"); Assert.IsTrue(success2, "CopyFileUsingLocks did not return true (test 2)"); System.Threading.Thread.Sleep(150); // Copy the file again, but with overwrite = false // This should raise an exception bool exceptionRaised; try { mFileTools.CopyFile(sourceFilePath, targetFilePath, false); exceptionRaised = false; } catch (Exception) { exceptionRaised = true; } Assert.IsTrue(exceptionRaised, "File copy with overwrite = false did not raise an exception; it should have"); }
/// <summary> /// Copy a file from a remote path and store it locally, including creating a .hashcheck file and a .lastused file /// If the file exists and the SHA1 hash matches, do not re-copy the file /// </summary> /// <param name="sourceFilePath">Source file path</param> /// <param name="targetDirectoryPath">Target directory path</param> /// <param name="errorMessage">Output: error message</param> /// <param name="recheckIntervalDays"> /// If the .hashcheck file is more than this number of days old, re-compute the hash value of the local file and compare to the hashcheck file /// Set to 0 to check the hash on every call to this method /// </param> /// <param name="hashType">Hash type for newly created .hashcheck files</param> /// <returns></returns> // ReSharper disable once UnusedMember.Global public bool CopyFileToLocal( string sourceFilePath, string targetDirectoryPath, out string errorMessage, int recheckIntervalDays = 0, HashUtilities.HashTypeConstants hashType = HashUtilities.HashTypeConstants.SHA1) { try { // Look for the source file var sourceFile = new FileInfo(sourceFilePath); if (!sourceFile.Exists) { errorMessage = "File not found: " + sourceFile; return(false); } var sourceHashcheckFile = new FileInfo(sourceFile.FullName + HashUtilities.HASHCHECK_FILE_SUFFIX); var sourceHashInfo = new HashUtilities.HashInfoType(); sourceHashInfo.Clear(); var targetDirectory = new DirectoryInfo(targetDirectoryPath); // Look for the local .hashcheck file // If there is a hash validation error, we might delay re-copying the file, depending on whether this local .hashcheck file exists or was changed recently var localHashCheckFile = new FileInfo(Path.Combine(targetDirectory.FullName, sourceFile.Name + HashUtilities.HASHCHECK_FILE_SUFFIX)); if (sourceHashcheckFile.Exists) { // Read the .hashcheck file sourceHashInfo = HashUtilities.ReadHashcheckFile(sourceHashcheckFile.FullName); } else { // .hashcheck file not found; create it for the source file (in the source directory) // Raise a warning if unable to create it, but continue try { OnStatusEvent(string.Format("Creating .hashcheck file for {0}", sourceFile.FullName)); HashUtilities.CreateHashcheckFile(sourceFile.FullName, hashType, out var hashValueSource, out var warningMessage); if (string.IsNullOrWhiteSpace(hashValueSource)) { if (string.IsNullOrWhiteSpace(warningMessage)) { OnWarningEvent("Unable to create the hash value for remote file " + sourceFile.FullName); } else { OnWarningEvent(warningMessage); } } else { if (!string.IsNullOrWhiteSpace(warningMessage)) { OnWarningEvent(warningMessage); } sourceHashInfo.HashValue = hashValueSource; sourceHashInfo.HashType = hashType; sourceHashInfo.FileSize = sourceFile.Length; sourceHashInfo.FileDateUtc = sourceFile.LastWriteTimeUtc; } } catch (Exception ex2) { // Treat this as a non-critical error OnWarningEvent(string.Format("Unable to create the .hashcheck file for source file {0}: {1}", sourceFile.FullName, ex2.Message)); } } // Validate the target directory if (!targetDirectory.Exists) { OnStatusEvent(string.Format("Creating directory {0}", targetDirectory.FullName)); targetDirectory.Create(); } // Look for the target file in the target directory var targetFile = new FileInfo(Path.Combine(targetDirectory.FullName, sourceFile.Name)); if (!targetFile.Exists) { DeleteHashCheckFileForDataFile(targetFile); OnStatusEvent(string.Format("Copying {0} to {1}", sourceFile.FullName, targetDirectory.FullName)); // Copy the source file locally mFileTools.CopyFileUsingLocks(sourceFile, targetFile.FullName, true); // Create the local .hashcheck file, sending localFilePath and the hash info of the source file var validNewFile = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays); return(validNewFile); } OnDebugEvent(string.Format("Validating {0} vs. expected hash {1}", targetFile.FullName, sourceHashInfo.HashValue)); // The target file exists // Create or validate the local .hashcheck file, sending localFilePath and the hash info of the source file var validFile = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays); if (validFile) { return(true); } // Existing local file and/or local file hash does not match the source file hash if (localHashCheckFile.Exists && DateTime.UtcNow.Subtract(localHashCheckFile.LastWriteTimeUtc).TotalMinutes > 10) { // The local hash check file already existed and is over 10 minutes old // Do not use a delay; immediately re-copy the file locally } else { // Wait for a random time between 5 and 15 seconds, plus an additional 1 second per 50 MB, to give other processes a chance to copy the file var rand = new Random(); var fileSizeMB = sourceFile.Length / 1024.0 / 1024; var waitTimeSeconds = rand.Next(5, 15) + fileSizeMB / 50; OnStatusEvent(string.Format("Hashcheck mismatch for {0}; waiting {1} seconds then re-checking", targetFile.FullName, waitTimeSeconds)); ConsoleMsgUtils.SleepSeconds(waitTimeSeconds); // Repeat the validation of the .hashcheck file // If valid, return true // Otherwise, delete the local file and the local hashcheck file and re-try the copy to the local directory var validFileB = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays: 0); if (validFileB) { OnStatusEvent(string.Format("Hash value is now the expected value: {0}", sourceHashInfo.HashValue)); return(true); } } OnWarningEvent(string.Format("Hash for local file does not match the remote file; recopying {0} to {1}", sourceFile.FullName, targetDirectory.FullName)); DeleteHashCheckFileForDataFile(targetFile); OnStatusEvent(string.Format("Copying {0} to {1}", sourceFile.FullName, targetDirectory.FullName)); // Repeat copying the remote file locally mFileTools.CopyFileUsingLocks(sourceFile, targetFile.FullName, true); // Create the local .hashcheck file, sending localFilePath and the hash info of the source file var validFileC = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays: 0); return(validFileC); } catch (Exception ex) { errorMessage = "Error retrieving/validating " + sourceFilePath + ": " + ex.Message; OnWarningEvent(errorMessage); return(false); } }