/// <summary>
/// Copy a file from a remote path and store it locally, including creating a .hashcheck file and a .lastused file
/// If the file exists and the SHA1 hash matches, do not re-copy the file
/// </summary>
/// <param name="sourceFilePath">Source file path</param>
/// <param name="targetDirectoryPath">Target directory path</param>
/// <param name="errorMessage">Output: error message</param>
/// <param name="recheckIntervalDays">
/// If the .hashcheck file is more than this number of days old, re-compute the hash value of the local file and compare to the hashcheck file
/// Set to 0 to check the hash on every call to this method
/// </param>
/// <param name="hashType">Hash type for newly created .hashcheck files</param>
/// <returns></returns>
// ReSharper disable once UnusedMember.Global
public bool CopyFileToLocal(
string sourceFilePath,
string targetDirectoryPath,
out string errorMessage,
int recheckIntervalDays = 0,
HashUtilities.HashTypeConstants hashType = HashUtilities.HashTypeConstants.SHA1)
{
try
{
// Look for the source file
var sourceFile = new FileInfo(sourceFilePath);
if (!sourceFile.Exists)
{
errorMessage = "File not found: " + sourceFile;
return(false);
}
var sourceHashcheckFile = new FileInfo(sourceFile.FullName + HashUtilities.HASHCHECK_FILE_SUFFIX);
var sourceHashInfo = new HashUtilities.HashInfoType();
sourceHashInfo.Clear();
var targetDirectory = new DirectoryInfo(targetDirectoryPath);
// Look for the local .hashcheck file
// If there is a hash validation error, we might delay re-copying the file, depending on whether this local .hashcheck file exists or was changed recently
var localHashCheckFile = new FileInfo(Path.Combine(targetDirectory.FullName, sourceFile.Name + HashUtilities.HASHCHECK_FILE_SUFFIX));
if (sourceHashcheckFile.Exists)
{
// Read the .hashcheck file
sourceHashInfo = HashUtilities.ReadHashcheckFile(sourceHashcheckFile.FullName);
}
else
{
// .hashcheck file not found; create it for the source file (in the source directory)
// Raise a warning if unable to create it, but continue
try
{
OnStatusEvent(string.Format("Creating .hashcheck file for {0}", sourceFile.FullName));
HashUtilities.CreateHashcheckFile(sourceFile.FullName, hashType, out var hashValueSource, out var warningMessage);
if (string.IsNullOrWhiteSpace(hashValueSource))
{
if (string.IsNullOrWhiteSpace(warningMessage))
{
OnWarningEvent("Unable to create the hash value for remote file " + sourceFile.FullName);
}
else
{
OnWarningEvent(warningMessage);
}
}
else
{
if (!string.IsNullOrWhiteSpace(warningMessage))
{
OnWarningEvent(warningMessage);
}
sourceHashInfo.HashValue = hashValueSource;
sourceHashInfo.HashType = hashType;
sourceHashInfo.FileSize = sourceFile.Length;
sourceHashInfo.FileDateUtc = sourceFile.LastWriteTimeUtc;
}
}
catch (Exception ex2)
{
// Treat this as a non-critical error
OnWarningEvent(string.Format("Unable to create the .hashcheck file for source file {0}: {1}",
sourceFile.FullName, ex2.Message));
}
}
// Validate the target directory
if (!targetDirectory.Exists)
{
OnStatusEvent(string.Format("Creating directory {0}", targetDirectory.FullName));
targetDirectory.Create();
}
// Look for the target file in the target directory
var targetFile = new FileInfo(Path.Combine(targetDirectory.FullName, sourceFile.Name));
if (!targetFile.Exists)
{
DeleteHashCheckFileForDataFile(targetFile);
OnStatusEvent(string.Format("Copying {0} to {1}", sourceFile.FullName, targetDirectory.FullName));
// Copy the source file locally
mFileTools.CopyFileUsingLocks(sourceFile, targetFile.FullName, true);
// Create the local .hashcheck file, sending localFilePath and the hash info of the source file
var validNewFile = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays);
return(validNewFile);
}
OnDebugEvent(string.Format("Validating {0} vs. expected hash {1}", targetFile.FullName, sourceHashInfo.HashValue));
// The target file exists
// Create or validate the local .hashcheck file, sending localFilePath and the hash info of the source file
var validFile = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays);
if (validFile)
{
return(true);
}
// Existing local file and/or local file hash does not match the source file hash
if (localHashCheckFile.Exists && DateTime.UtcNow.Subtract(localHashCheckFile.LastWriteTimeUtc).TotalMinutes > 10)
{
// The local hash check file already existed and is over 10 minutes old
// Do not use a delay; immediately re-copy the file locally
}
else
{
// Wait for a random time between 5 and 15 seconds, plus an additional 1 second per 50 MB, to give other processes a chance to copy the file
var rand = new Random();
var fileSizeMB = sourceFile.Length / 1024.0 / 1024;
var waitTimeSeconds = rand.Next(5, 15) + fileSizeMB / 50;
OnStatusEvent(string.Format("Hashcheck mismatch for {0}; waiting {1} seconds then re-checking", targetFile.FullName, waitTimeSeconds));
ConsoleMsgUtils.SleepSeconds(waitTimeSeconds);
// Repeat the validation of the .hashcheck file
// If valid, return true
// Otherwise, delete the local file and the local hashcheck file and re-try the copy to the local directory
var validFileB = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays: 0);
if (validFileB)
{
OnStatusEvent(string.Format("Hash value is now the expected value: {0}", sourceHashInfo.HashValue));
return(true);
}
}
OnWarningEvent(string.Format("Hash for local file does not match the remote file; recopying {0} to {1}",
sourceFile.FullName, targetDirectory.FullName));
DeleteHashCheckFileForDataFile(targetFile);
OnStatusEvent(string.Format("Copying {0} to {1}", sourceFile.FullName, targetDirectory.FullName));
// Repeat copying the remote file locally
mFileTools.CopyFileUsingLocks(sourceFile, targetFile.FullName, true);
// Create the local .hashcheck file, sending localFilePath and the hash info of the source file
var validFileC = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays: 0);
return(validFileC);
}
catch (Exception ex)
{
errorMessage = "Error retrieving/validating " + sourceFilePath + ": " + ex.Message;
OnWarningEvent(errorMessage);
return(false);
}
}
/// <summary>
/// Validate that the hash value of a local file matches the expected hash info, creating the .hashcheck file if missing
/// </summary>
/// <param name="localFilePath">Local file path</param>
/// <param name="hashCheckFilePath">Hashcheck file for the given data file (auto-defined if blank)</param>
/// <param name="errorMessage">Output: error message</param>
/// <param name="expectedHashInfo">Expected hash info (e.g. based on a remote file)</param>
/// <param name="checkDate">If True, compares UTC modification time; times must agree within 2 seconds</param>
/// <param name="computeHash">If true, compute the file hash every recheckIntervalDays (or every time if recheckIntervalDays is 0)</param>
/// <param name="checkSize">If true, compare the actual file size to that in the hashcheck file</param>
/// <param name="recheckIntervalDays">
/// If the .hashcheck file is more than this number of days old, re-compute the hash value of the local file and compare to the hashcheck file
/// Set to 0 to check the hash on every call to this method
/// </param>
/// <returns>True if the file is valid, otherwise false</returns>
/// <remarks>
/// Will create the .hashcheck file if missing
/// Will also update the .lastused file for the local file
/// </remarks>
public static bool ValidateFileVsHashcheck(
string localFilePath, string hashCheckFilePath,
out string errorMessage,
HashUtilities.HashInfoType expectedHashInfo,
bool checkDate = true, bool computeHash = true, bool checkSize = true,
int recheckIntervalDays = 0)
{
try
{
var localFile = new FileInfo(localFilePath);
if (!localFile.Exists)
{
errorMessage = "File not found: " + localFilePath;
ConsoleMsgUtils.ShowWarning(errorMessage);
return(false);
}
FileInfo localHashcheckFile;
if (string.IsNullOrWhiteSpace(hashCheckFilePath))
{
localHashcheckFile = new FileInfo(localFile.FullName + HashUtilities.HASHCHECK_FILE_SUFFIX);
}
else
{
localHashcheckFile = new FileInfo(hashCheckFilePath);
}
if (!localHashcheckFile.Exists)
{
// Local .hashcheck file not found; create it
if (expectedHashInfo.HashType == HashUtilities.HashTypeConstants.Undefined)
{
expectedHashInfo.HashType = HashUtilities.HashTypeConstants.SHA1;
}
HashUtilities.CreateHashcheckFile(localFile.FullName, expectedHashInfo.HashType, out var localFileHash, out var warningMessage);
if (string.IsNullOrWhiteSpace(localFileHash))
{
if (string.IsNullOrWhiteSpace(warningMessage))
{
errorMessage = "Unable to compute the hash value for local file " + localFile.FullName;
}
else
{
errorMessage = warningMessage;
}
ConsoleMsgUtils.ShowWarning(errorMessage);
return(false);
}
if (!string.IsNullOrWhiteSpace(warningMessage))
{
ConsoleMsgUtils.ShowWarning(warningMessage);
}
// Compare the hash to expectedHashInfo.HashValue (if .HashValue is not "")
if (!string.IsNullOrWhiteSpace(expectedHashInfo.HashValue) && !localFileHash.Equals(expectedHashInfo.HashValue))
{
errorMessage = string.Format("Mismatch between the expected hash value and the actual hash value for {0}: {1} vs. {2}",
localFile.Name, expectedHashInfo.HashValue, localFileHash);
ConsoleMsgUtils.ShowWarning(errorMessage);
return(false);
}
// Create/update the .lastused file
UpdateLastUsedFile(localFile);
errorMessage = string.Empty;
return(true);
}
// Local .hashcheck file exists
var localHashInfo = HashUtilities.ReadHashcheckFile(localHashcheckFile.FullName);
if (expectedHashInfo.HashType != HashUtilities.HashTypeConstants.Undefined &&
!string.IsNullOrWhiteSpace(expectedHashInfo.HashValue) &&
!localHashInfo.HashValue.Equals(expectedHashInfo.HashValue))
{
errorMessage = string.Format("Hash mismatch for {0}: expected {1} but actually {2}",
localFile.Name, expectedHashInfo.HashValue, localHashInfo.HashValue);
ConsoleMsgUtils.ShowWarning(errorMessage);
return(false);
}
if (checkSize && localFile.Length != localHashInfo.FileSize)
{
errorMessage = string.Format("File size mismatch for {0}: expected {1:#,##0} bytes but actually {2:#,##0} bytes",
localFile.Name, localHashInfo.FileSize, localFile.Length);
ConsoleMsgUtils.ShowWarning(errorMessage);
return(false);
}
// Only compare dates if we are not comparing hash values
if (!computeHash && checkDate)
{
if (Math.Abs(localFile.LastWriteTimeUtc.Subtract(localHashInfo.FileDateUtc).TotalSeconds) > 2)
{
errorMessage = string.Format("File date mismatch for {0}: expected {1} UTC but actually {2} UTC",
localFile.Name,
localHashInfo.FileDateUtc.ToString(HashUtilities.DATE_TIME_FORMAT),
localFile.LastWriteTimeUtc.ToString(HashUtilities.DATE_TIME_FORMAT));
ConsoleMsgUtils.ShowWarning(errorMessage);
return(false);
}
}
if (computeHash)
{
var lastCheckDays = DateTime.UtcNow.Subtract(localHashcheckFile.LastWriteTimeUtc).TotalDays;
if (recheckIntervalDays <= 0 || lastCheckDays > recheckIntervalDays)
{
// Compute the hash of the file
if (localHashInfo.HashType == HashUtilities.HashTypeConstants.Undefined)
{
errorMessage = "Hashtype is undefined; cannot compute the file hash to compare to the .hashcheck file";
ConsoleMsgUtils.ShowWarning(errorMessage);
return(false);
}
var actualHash = HashUtilities.ComputeFileHash(localFilePath, localHashInfo.HashType);
if (!actualHash.Equals(localHashInfo.HashValue))
{
errorMessage = "Hash mismatch: expecting " + localHashInfo.HashValue + " but computed " + actualHash;
ConsoleMsgUtils.ShowWarning(errorMessage);
return(false);
}
}
}
// Create/update the .lastused file
UpdateLastUsedFile(localFile);
errorMessage = string.Empty;
return(true);
}
catch (Exception ex)
{
errorMessage = "Error validating " + localFilePath + " against the expected hash: " + ex.Message;
ConsoleMsgUtils.ShowWarning(errorMessage);
return(false);
}
}
