/// <summary>
/// Validate that the hash value of a local file matches the expected hash info, creating the .hashcheck file if missing
/// </summary>
/// <param name="localFilePath">Local file path</param>
/// <param name="hashCheckFilePath">Hashcheck file for the given data file (auto-defined if blank)</param>
/// <param name="errorMessage">Output: error message</param>
/// <param name="expectedHashType">Hash type that is presumed to be in the hashcheck file</param>
/// <param name="recheckIntervalDays">
/// If the .hashcheck file is more than this number of days old, re-compute the hash value of the local file and compare to the hashcheck file
/// Set to 0 to check the hash on every call to this method
/// </param>
/// <returns>True if the file is valid, otherwise false</returns>
/// <remarks>
/// Will create the .hashcheck file if missing
/// Will also update the .lastused file for the local file
/// </remarks>
// ReSharper disable once UnusedMember.Global
public static bool ValidateFileVsHashcheck(
string localFilePath, string hashCheckFilePath,
out string errorMessage,
HashUtilities.HashTypeConstants expectedHashType,
int recheckIntervalDays)
{
const bool checkDate = true;
const bool computeHash = true;
const bool checkSize = true;
var expectedHashInfo = new HashUtilities.HashInfoType
{
HashType = expectedHashType
};
return(ValidateFileVsHashcheck(localFilePath, hashCheckFilePath, out errorMessage, expectedHashInfo, checkDate, computeHash, checkSize, recheckIntervalDays));
}
/// <summary>
/// Looks for a .hashcheck file for the specified data file; returns false if not found
/// If found, compares the stored values to the actual values (size, modification_date_utc, and hash)
/// </summary>
/// <param name="localFilePath">Data file to check</param>
/// <param name="hashCheckFilePath">Hashcheck file for the given data file (auto-defined if blank)</param>
/// <param name="errorMessage">Output: error message</param>
/// <param name="checkDate">If True, compares UTC modification time; times must agree within 2 seconds</param>
/// <param name="computeHash">If true, compute the file hash (every time); if false, only compare file size and date</param>
/// <param name="checkSize">If true, compare the actual file size to that in the hashcheck file</param>
/// <param name="assumedHashType">Hash type to assume if the .hashcheck file does not have a hashtype entry</param>
/// <returns>True if the hashcheck file exists and the actual file matches the expected values; false if a mismatch, if .hashcheck is missing, or if a problem</returns>
/// <remarks>The .hashcheck file has the same name as the data file, but with ".hashcheck" appended</remarks>
// ReSharper disable once UnusedMember.Global
public static bool ValidateFileVsHashcheck(
string localFilePath, string hashCheckFilePath, out string errorMessage,
bool checkDate = true, bool computeHash = true, bool checkSize = true,
HashUtilities.HashTypeConstants assumedHashType = HashUtilities.HashTypeConstants.MD5)
{
errorMessage = string.Empty;
try
{
var localFile = new FileInfo(localFilePath);
if (!localFile.Exists)
{
errorMessage = "File not found: " + localFile.FullName;
return(false);
}
var localHashcheckFile = new FileInfo(hashCheckFilePath);
if (!localHashcheckFile.Exists)
{
errorMessage = "Data file at " + localFile.FullName + " does not have a corresponding .hashcheck file named " + localHashcheckFile.Name;
return(false);
}
var expectedHashInfo = new HashUtilities.HashInfoType
{
HashType = assumedHashType
};
var validFile = ValidateFileVsHashcheck(localFilePath, hashCheckFilePath, out errorMessage,
expectedHashInfo, checkDate, computeHash, checkSize, recheckIntervalDays: 0);
return(validFile);
}
catch (Exception ex)
{
ConsoleMsgUtils.ShowWarning("Error in ValidateLocalFile: " + ex.Message);
return(false);
}
}
public void Clear()
{
Length = 0;
Hash = string.Empty;
HashType = HashUtilities.HashTypeConstants.Undefined;
}
/// <summary>
/// Copy a file from a remote path and store it locally, including creating a .hashcheck file and a .lastused file
/// If the file exists and the SHA1 hash matches, do not re-copy the file
/// </summary>
/// <param name="sourceFilePath">Source file path</param>
/// <param name="targetDirectoryPath">Target directory path</param>
/// <param name="errorMessage">Output: error message</param>
/// <param name="recheckIntervalDays">
/// If the .hashcheck file is more than this number of days old, re-compute the hash value of the local file and compare to the hashcheck file
/// Set to 0 to check the hash on every call to this method
/// </param>
/// <param name="hashType">Hash type for newly created .hashcheck files</param>
/// <returns></returns>
// ReSharper disable once UnusedMember.Global
public bool CopyFileToLocal(
string sourceFilePath,
string targetDirectoryPath,
out string errorMessage,
int recheckIntervalDays = 0,
HashUtilities.HashTypeConstants hashType = HashUtilities.HashTypeConstants.SHA1)
{
try
{
// Look for the source file
var sourceFile = new FileInfo(sourceFilePath);
if (!sourceFile.Exists)
{
errorMessage = "File not found: " + sourceFile;
return(false);
}
var sourceHashcheckFile = new FileInfo(sourceFile.FullName + HashUtilities.HASHCHECK_FILE_SUFFIX);
var sourceHashInfo = new HashUtilities.HashInfoType();
sourceHashInfo.Clear();
var targetDirectory = new DirectoryInfo(targetDirectoryPath);
// Look for the local .hashcheck file
// If there is a hash validation error, we might delay re-copying the file, depending on whether this local .hashcheck file exists or was changed recently
var localHashCheckFile = new FileInfo(Path.Combine(targetDirectory.FullName, sourceFile.Name + HashUtilities.HASHCHECK_FILE_SUFFIX));
if (sourceHashcheckFile.Exists)
{
// Read the .hashcheck file
sourceHashInfo = HashUtilities.ReadHashcheckFile(sourceHashcheckFile.FullName);
}
else
{
// .hashcheck file not found; create it for the source file (in the source directory)
// Raise a warning if unable to create it, but continue
try
{
OnStatusEvent(string.Format("Creating .hashcheck file for {0}", sourceFile.FullName));
HashUtilities.CreateHashcheckFile(sourceFile.FullName, hashType, out var hashValueSource, out var warningMessage);
if (string.IsNullOrWhiteSpace(hashValueSource))
{
if (string.IsNullOrWhiteSpace(warningMessage))
{
OnWarningEvent("Unable to create the hash value for remote file " + sourceFile.FullName);
}
else
{
OnWarningEvent(warningMessage);
}
}
else
{
if (!string.IsNullOrWhiteSpace(warningMessage))
{
OnWarningEvent(warningMessage);
}
sourceHashInfo.HashValue = hashValueSource;
sourceHashInfo.HashType = hashType;
sourceHashInfo.FileSize = sourceFile.Length;
sourceHashInfo.FileDateUtc = sourceFile.LastWriteTimeUtc;
}
}
catch (Exception ex2)
{
// Treat this as a non-critical error
OnWarningEvent(string.Format("Unable to create the .hashcheck file for source file {0}: {1}",
sourceFile.FullName, ex2.Message));
}
}
// Validate the target directory
if (!targetDirectory.Exists)
{
OnStatusEvent(string.Format("Creating directory {0}", targetDirectory.FullName));
targetDirectory.Create();
}
// Look for the target file in the target directory
var targetFile = new FileInfo(Path.Combine(targetDirectory.FullName, sourceFile.Name));
if (!targetFile.Exists)
{
DeleteHashCheckFileForDataFile(targetFile);
OnStatusEvent(string.Format("Copying {0} to {1}", sourceFile.FullName, targetDirectory.FullName));
// Copy the source file locally
mFileTools.CopyFileUsingLocks(sourceFile, targetFile.FullName, true);
// Create the local .hashcheck file, sending localFilePath and the hash info of the source file
var validNewFile = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays);
return(validNewFile);
}
OnDebugEvent(string.Format("Validating {0} vs. expected hash {1}", targetFile.FullName, sourceHashInfo.HashValue));
// The target file exists
// Create or validate the local .hashcheck file, sending localFilePath and the hash info of the source file
var validFile = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays);
if (validFile)
{
return(true);
}
// Existing local file and/or local file hash does not match the source file hash
if (localHashCheckFile.Exists && DateTime.UtcNow.Subtract(localHashCheckFile.LastWriteTimeUtc).TotalMinutes > 10)
{
// The local hash check file already existed and is over 10 minutes old
// Do not use a delay; immediately re-copy the file locally
}
else
{
// Wait for a random time between 5 and 15 seconds, plus an additional 1 second per 50 MB, to give other processes a chance to copy the file
var rand = new Random();
var fileSizeMB = sourceFile.Length / 1024.0 / 1024;
var waitTimeSeconds = rand.Next(5, 15) + fileSizeMB / 50;
OnStatusEvent(string.Format("Hashcheck mismatch for {0}; waiting {1} seconds then re-checking", targetFile.FullName, waitTimeSeconds));
ConsoleMsgUtils.SleepSeconds(waitTimeSeconds);
// Repeat the validation of the .hashcheck file
// If valid, return true
// Otherwise, delete the local file and the local hashcheck file and re-try the copy to the local directory
var validFileB = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays: 0);
if (validFileB)
{
OnStatusEvent(string.Format("Hash value is now the expected value: {0}", sourceHashInfo.HashValue));
return(true);
}
}
OnWarningEvent(string.Format("Hash for local file does not match the remote file; recopying {0} to {1}",
sourceFile.FullName, targetDirectory.FullName));
DeleteHashCheckFileForDataFile(targetFile);
OnStatusEvent(string.Format("Copying {0} to {1}", sourceFile.FullName, targetDirectory.FullName));
// Repeat copying the remote file locally
mFileTools.CopyFileUsingLocks(sourceFile, targetFile.FullName, true);
// Create the local .hashcheck file, sending localFilePath and the hash info of the source file
var validFileC = ValidateFileVsHashcheck(targetFile.FullName, out errorMessage, sourceHashInfo, recheckIntervalDays: 0);
return(validFileC);
}
catch (Exception ex)
{
errorMessage = "Error retrieving/validating " + sourceFilePath + ": " + ex.Message;
OnWarningEvent(errorMessage);
return(false);
}
}
/// <summary>
/// Validate that the hash value of a local file matches the expected hash value
/// </summary>
/// <param name="localFilePath">Local file path</param>
/// <param name="errorMessage">Output: error message</param>
/// <param name="expectedHash">Expected hash value</param>
/// <param name="expectedHashType">Hash type (CRC32, MD5, or SHA1)</param>
/// <returns>True if the file is valid, otherwise false</returns>
// ReSharper disable once UnusedMember.Global
public bool ValidateFileVsHashcheck(string localFilePath, out string errorMessage, string expectedHash, HashUtilities.HashTypeConstants expectedHashType)
{
var expectedHashInfo = new HashUtilities.HashInfoType
{
HashValue = expectedHash,
HashType = expectedHashType
};
var hashCheckFilePath = string.Empty;
return(ValidateFileVsHashcheck(localFilePath, hashCheckFilePath, out errorMessage, expectedHashInfo));
}
/// <summary>
/// Add an instrument file, optionally including its file hash
/// </summary>
/// <param name="instrumentFileRelativePath"></param>
/// <param name="fileSizeBytes"></param>
/// <param name="hashValue"></param>
/// <param name="hashType"></param>
public void AddInstrumentFile(string instrumentFileRelativePath, long fileSizeBytes, string hashValue, HashUtilities.HashTypeConstants hashType)
{
if (InstrumentFiles.ContainsKey(instrumentFileRelativePath))
{
throw new DuplicateNameException("Duplicate key in AddInstrumentFile; Instrument file already defined: " +
instrumentFileRelativePath);
}
var instFileInfo = new InstrumentFileInfo
{
Length = fileSizeBytes
};
if (string.IsNullOrWhiteSpace(hashValue))
{
instFileInfo.Hash = string.Empty;
instFileInfo.HashType = HashUtilities.HashTypeConstants.Undefined;
}
else
{
instFileInfo.Hash = hashValue;
instFileInfo.HashType = hashType;
}
InstrumentFiles.Add(instrumentFileRelativePath, instFileInfo);
}
