/// <summary> /// Copy a file from a remote path and store it locally, including creating a .hashcheck file and a .lastused file /// If the file exists and the SHA1 hash matches, do not re-copy the file /// </summary> /// <param name="sourceFilePath">Source file path</param> /// <param name="targetDirectoryPath">Target directory path</param> /// <param name="errorMessage">Output: error message</param> /// <param name="recheckIntervalDays"> /// If the .hashcheck file is more than this number of days old, re-compute the hash value of the local file and compare to the hashcheck file /// Set to 0 to check the hash on every call to this method /// </param> /// <param name="hashType">Hash type for newly created .hashcheck files</param> /// <returns></returns> // ReSharper disable once UnusedMember.Global public bool CopyFileToLocal( string sourceFilePath, string targetDirectoryPath, out string errorMessage, int recheckIntervalDays = 0, HashUtilities.HashTypeConstants hashType = HashUtilities.HashTypeConstants.SHA1) { try { // Look for the source file var sourceFile = new FileInfo(sourceFilePath); if (!sourceFile.Exists) { errorMessage = "File not found: " + sourceFile; return(false); } var sourceHashcheckFile = new FileInfo(sourceFile.FullName + HashUtilities.HASHCHECK_FILE_SUFFIX); var sourceHashInfo = new HashUtilities.HashInfoType(); sourceHashInfo.Clear(); var targetDirectory = new DirectoryInfo(targetDirectoryPath); // Look for the local .hashcheck file // If there is a hash validation error, we might delay re-copying the file, depending on whether this local .hashcheck file exists or was changed recently var localHashCheckFile = new FileInfo(Path.Combine(targetDirectory.FullName, sourceFile.Name + HashUtilities.HASHCHECK_FILE_SUFFIX)); if (sourceHashcheckFile.Exists) { // Read the .hashcheck file sourceHashInfo = HashUtilities.ReadHashcheckFile(sourceHashcheckFile.FullName); } else { // .hashcheck file not found; create it for the source file (in the source directory) // Raise a warning if unable to create it, but continue try { OnStatusEvent(string.Format("Creating .hashcheck file for {0}", sourceFile.FullName)); HashUtilities.CreateHashcheckFile(sourceFile.FullName, hashType, out var hashValueSource, out var warningMessage); if (string.IsNullOrWhiteSpace(hashValueSource)) { if (string.IsNullOrWhiteSpace(warningMessage)) { OnWarningEvent("Unable to create the hash value for remote file " + sourceFile.FullName); } else { OnWarningEvent(warningMessage); } } else { if (!string.IsNullOrWhiteSpace(warningMessage)) { OnWarningEvent(warningMessage); } sourceHashInfo.HashValue = hashValueSource; sourceHashInfo.HashType = hashType; sourceHashInfo.FileSize = sourceFile.Length; sourceHashInfo.FileDateUtc = sourceFile.LastWriteTimeUtc; } } catch (Exception ex2) { // Treat this as a non-critical error OnWarningEvent(string.Format("Unable to create the .hashcheck file for source file {0}: {1}", sourceFile.FullName, ex2.Message)); } } // Validate the target directory if (!targetDirectory.Exists) { OnStatusEvent(string.Format("Creating directory {0}", targetDirectory.FullName)); targetDirectory.Create(); } // Look for the target file in the target directory var targetFile = new FileInfo(Path.Combine(targetDirectory.FullName, sourceFile.Name)); if (!targetFile.Exists) { DeleteHashCheckFileForDataFile(targetFile); OnStatusEvent(string.Format("Copying {0} to {1}", sourceFile.FullName, targetDirectory.FullName)); // Copy the source file locally mFileTools.CopyFileUsingLocks(sourceFile, targetFile.FullName, true); // Create the local .hashcheck file, sending localFilePath and the hash info of the source file var validNewFile = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays); return(validNewFile); } OnDebugEvent(string.Format("Validating {0} vs. expected hash {1}", targetFile.FullName, sourceHashInfo.HashValue)); // The target file exists // Create or validate the local .hashcheck file, sending localFilePath and the hash info of the source file var validFile = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays); if (validFile) { return(true); } // Existing local file and/or local file hash does not match the source file hash if (localHashCheckFile.Exists && DateTime.UtcNow.Subtract(localHashCheckFile.LastWriteTimeUtc).TotalMinutes > 10) { // The local hash check file already existed and is over 10 minutes old // Do not use a delay; immediately re-copy the file locally } else { // Wait for a random time between 5 and 15 seconds, plus an additional 1 second per 50 MB, to give other processes a chance to copy the file var rand = new Random(); var fileSizeMB = sourceFile.Length / 1024.0 / 1024; var waitTimeSeconds = rand.Next(5, 15) + fileSizeMB / 50; OnStatusEvent(string.Format("Hashcheck mismatch for {0}; waiting {1} seconds then re-checking", targetFile.FullName, waitTimeSeconds)); ConsoleMsgUtils.SleepSeconds(waitTimeSeconds); // Repeat the validation of the .hashcheck file // If valid, return true // Otherwise, delete the local file and the local hashcheck file and re-try the copy to the local directory var validFileB = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays: 0); if (validFileB) { OnStatusEvent(string.Format("Hash value is now the expected value: {0}", sourceHashInfo.HashValue)); return(true); } } OnWarningEvent(string.Format("Hash for local file does not match the remote file; recopying {0} to {1}", sourceFile.FullName, targetDirectory.FullName)); DeleteHashCheckFileForDataFile(targetFile); OnStatusEvent(string.Format("Copying {0} to {1}", sourceFile.FullName, targetDirectory.FullName)); // Repeat copying the remote file locally mFileTools.CopyFileUsingLocks(sourceFile, targetFile.FullName, true); // Create the local .hashcheck file, sending localFilePath and the hash info of the source file var validFileC = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays: 0); return(validFileC); } catch (Exception ex) { errorMessage = "Error retrieving/validating " + sourceFilePath + ": " + ex.Message; OnWarningEvent(errorMessage); return(false); } }
/// <summary> /// Validate that the hash value of a local file matches the expected hash info, creating the .hashcheck file if missing /// </summary> /// <param name="localFilePath">Local file path</param> /// <param name="hashCheckFilePath">Hashcheck file for the given data file (auto-defined if blank)</param> /// <param name="errorMessage">Output: error message</param> /// <param name="expectedHashInfo">Expected hash info (e.g. based on a remote file)</param> /// <param name="checkDate">If True, compares UTC modification time; times must agree within 2 seconds</param> /// <param name="computeHash">If true, compute the file hash every recheckIntervalDays (or every time if recheckIntervalDays is 0)</param> /// <param name="checkSize">If true, compare the actual file size to that in the hashcheck file</param> /// <param name="recheckIntervalDays"> /// If the .hashcheck file is more than this number of days old, re-compute the hash value of the local file and compare to the hashcheck file /// Set to 0 to check the hash on every call to this method /// </param> /// <returns>True if the file is valid, otherwise false</returns> /// <remarks> /// Will create the .hashcheck file if missing /// Will also update the .lastused file for the local file /// </remarks> public static bool ValidateFileVsHashcheck( string localFilePath, string hashCheckFilePath, out string errorMessage, HashUtilities.HashInfoType expectedHashInfo, bool checkDate = true, bool computeHash = true, bool checkSize = true, int recheckIntervalDays = 0) { try { var localFile = new FileInfo(localFilePath); if (!localFile.Exists) { errorMessage = "File not found: " + localFilePath; ConsoleMsgUtils.ShowWarning(errorMessage); return(false); } FileInfo localHashcheckFile; if (string.IsNullOrWhiteSpace(hashCheckFilePath)) { localHashcheckFile = new FileInfo(localFile.FullName + HashUtilities.HASHCHECK_FILE_SUFFIX); } else { localHashcheckFile = new FileInfo(hashCheckFilePath); } if (!localHashcheckFile.Exists) { // Local .hashcheck file not found; create it if (expectedHashInfo.HashType == HashUtilities.HashTypeConstants.Undefined) { expectedHashInfo.HashType = HashUtilities.HashTypeConstants.SHA1; } HashUtilities.CreateHashcheckFile(localFile.FullName, expectedHashInfo.HashType, out var localFileHash, out var warningMessage); if (string.IsNullOrWhiteSpace(localFileHash)) { if (string.IsNullOrWhiteSpace(warningMessage)) { errorMessage = "Unable to compute the hash value for local file " + localFile.FullName; } else { errorMessage = warningMessage; } ConsoleMsgUtils.ShowWarning(errorMessage); return(false); } if (!string.IsNullOrWhiteSpace(warningMessage)) { ConsoleMsgUtils.ShowWarning(warningMessage); } // Compare the hash to expectedHashInfo.HashValue (if .HashValue is not "") if (!string.IsNullOrWhiteSpace(expectedHashInfo.HashValue) && !localFileHash.Equals(expectedHashInfo.HashValue)) { errorMessage = string.Format("Mismatch between the expected hash value and the actual hash value for {0}: {1} vs. {2}", localFile.Name, expectedHashInfo.HashValue, localFileHash); ConsoleMsgUtils.ShowWarning(errorMessage); return(false); } // Create/update the .lastused file UpdateLastUsedFile(localFile); errorMessage = string.Empty; return(true); } // Local .hashcheck file exists var localHashInfo = HashUtilities.ReadHashcheckFile(localHashcheckFile.FullName); if (expectedHashInfo.HashType != HashUtilities.HashTypeConstants.Undefined && !string.IsNullOrWhiteSpace(expectedHashInfo.HashValue) && !localHashInfo.HashValue.Equals(expectedHashInfo.HashValue)) { errorMessage = string.Format("Hash mismatch for {0}: expected {1} but actually {2}", localFile.Name, expectedHashInfo.HashValue, localHashInfo.HashValue); ConsoleMsgUtils.ShowWarning(errorMessage); return(false); } if (checkSize && localFile.Length != localHashInfo.FileSize) { errorMessage = string.Format("File size mismatch for {0}: expected {1:#,##0} bytes but actually {2:#,##0} bytes", localFile.Name, localHashInfo.FileSize, localFile.Length); ConsoleMsgUtils.ShowWarning(errorMessage); return(false); } // Only compare dates if we are not comparing hash values if (!computeHash && checkDate) { if (Math.Abs(localFile.LastWriteTimeUtc.Subtract(localHashInfo.FileDateUtc).TotalSeconds) > 2) { errorMessage = string.Format("File date mismatch for {0}: expected {1} UTC but actually {2} UTC", localFile.Name, localHashInfo.FileDateUtc.ToString(HashUtilities.DATE_TIME_FORMAT), localFile.LastWriteTimeUtc.ToString(HashUtilities.DATE_TIME_FORMAT)); ConsoleMsgUtils.ShowWarning(errorMessage); return(false); } } if (computeHash) { var lastCheckDays = DateTime.UtcNow.Subtract(localHashcheckFile.LastWriteTimeUtc).TotalDays; if (recheckIntervalDays <= 0 || lastCheckDays > recheckIntervalDays) { // Compute the hash of the file if (localHashInfo.HashType == HashUtilities.HashTypeConstants.Undefined) { errorMessage = "Hashtype is undefined; cannot compute the file hash to compare to the .hashcheck file"; ConsoleMsgUtils.ShowWarning(errorMessage); return(false); } var actualHash = HashUtilities.ComputeFileHash(localFilePath, localHashInfo.HashType); if (!actualHash.Equals(localHashInfo.HashValue)) { errorMessage = "Hash mismatch: expecting " + localHashInfo.HashValue + " but computed " + actualHash; ConsoleMsgUtils.ShowWarning(errorMessage); return(false); } } } // Create/update the .lastused file UpdateLastUsedFile(localFile); errorMessage = string.Empty; return(true); } catch (Exception ex) { errorMessage = "Error validating " + localFilePath + " against the expected hash: " + ex.Message; ConsoleMsgUtils.ShowWarning(errorMessage); return(false); } }