public void HasNullHashBeforeClosingAndFinishedReadingInReadMode()
{
// Arrange
var hashing = new HashingStream(new MemoryStream(), SHA1.Create(), true);
// Act, Assert
hashing.Hash.Should().BeNull();
}
/// <summary>
/// Prefixes the data then signs it.
/// </summary>
/// <param name="signingStream">The signing stream.</param>
/// <param name="extra">The extra data passed by prefixData.</param>
protected override void PrefixDataSign(HashingStream signingStream, object extra)
{
base.PrefixDataSign(signingStream, extra);
var expiration = FromDateTime((DateTime)extra);
var buffer = Utility.GetBytes(expiration);
signingStream.Write(buffer, 0, buffer.Length);
}
public void HasNullHashBeforeClosingInWriteMode()
{
// Arrange
var hashing = new HashingStream(new MemoryStream(), SHA1.Create(), false);
// Act, Assert
hashing.Hash.Should().BeNull();
}
public void CannotWriteForWriteMode()
{
// Arrange
var hashing = new HashingStream(new MemoryStream(), SHA1.Create(), false);
// Act, Assert
Action action = () => hashing.Read(new byte[1], 0, 1);
action.ShouldThrow <NotSupportedException>().Which.Message.Should().Be("The hashing stream is configured to write, not read.");
}
public void HasHashAfterFinishedReadingInReadMode()
{
// Arrange
var hashing = new HashingStream(new MemoryStream(new byte[0]), SHA1.Create(), true);
// Act
var read = hashing.Read(new byte[1], 0, 1);
// Assert
read.Should().Be(0);
hashing.Hash.Should().NotBeNull();
GetHash(hashing.Hash).Should().Be("DA39A3EE5E6B4B0D3255BFEF95601890AFD80709");
}
private byte[] GetIndexHash()
{
if (this.shouldHashIndex)
{
using (Stream fileStream = new FileStream(this.indexLockPath, FileMode.Open, FileAccess.Read, FileShare.Write))
using (HashingStream hasher = new HashingStream(fileStream))
{
hasher.CopyTo(Stream.Null);
return(hasher.Hash);
}
}
return(new byte[20]);
}
public void HashesPartialReadInReadMode()
{
// Arrange
var inner = new MemoryStream(Encoding.ASCII.GetBytes("foobar"));
var hashing = new HashingStream(inner, SHA1.Create(), true);
var buffer = new byte[3];
// Act
var read = hashing.Read(buffer, 0, 3);
hashing.Close();
// Assert
read.Should().Be(3);
GetHash(hashing.Hash).Should().Be("0BEEC7B5EA3F0FDBC95D0DD47F3C5BC275DA8A33");
}
public void GeneratesHashWhileWriting(HashAlgorithm hashAlgorithm, string expectedContent, int bufferSize, string expectedHash)
{
// Arrange
var inner = new MemoryStream();
var hashing = new HashingStream(inner, hashAlgorithm, false);
// Act
using (var writer = new StreamWriter(hashing, Encoding.ASCII, bufferSize))
{
writer.Write(expectedContent);
}
// Assert
var actualContent = Encoding.ASCII.GetString(inner.ToArray());
actualContent.Should().Be(expectedContent);
var actualHash = GetHash(hashing.Hash);
actualHash.Should().Be(expectedHash);
}
public void GeneratesHashWhileReading(HashAlgorithm hashAlgorithm, string expectedContent, int bufferSize, string expectedHash)
{
// Arrange
var inner = new MemoryStream(Encoding.ASCII.GetBytes(expectedContent));
var hashing = new HashingStream(inner, hashAlgorithm, true);
// Act
string actualContent;
using (var reader = new StreamReader(hashing, Encoding.ASCII, false, bufferSize))
{
actualContent = reader.ReadToEnd();
}
// Assert
actualContent.Should().Be(expectedContent);
var actualHash = GetHash(hashing.Hash);
actualHash.Should().Be(expectedHash);
}
private void thread()
{
while (true)
{
try
{
// Hash metadata of everything in StaticPath
var hashStream = new HashingStream(new VoidStream(), MD5.Create()); // no need to dispose of it
var writer = new BinaryWriter(hashStream);
foreach (var path in new PathManager(StaticPath).GetFiles().OrderBy(f => f.FullName))
{
writer.Write(path.FullName);
writer.Write(path.Length);
writer.Write(path.LastWriteTimeUtc.ToBinary());
}
var hash = hashStream.Hash.ToHex();
SendUpdate(new ReloadDto {
ValidUntilUtc = DateTime.UtcNow.AddYears(10), StaticFilesHash = hash
});
// Sleep until we observe a change, but rescan fully every N minutes even we didn't see any changes
var minimumWait = DateTime.UtcNow.AddSeconds(2);
var watcher = new FileSystemWatcher();
watcher.IncludeSubdirectories = true;
watcher.Path = StaticPath;
watcher.WaitForChanged(WatcherChangeTypes.All, 20 * 60 * 1000);
// Enforce a minimum wait time in case the watcher breaks and starts firing endlessly for whatever reason
if (DateTime.UtcNow < minimumWait)
{
Thread.Sleep(minimumWait - DateTime.UtcNow);
}
}
catch
{
Thread.Sleep(TimeSpan.FromSeconds(60));
}
}
}
/// <summary>
/// Postfixes the data then signs it.
/// </summary>
/// <param name="signingStream">The signing stream.</param>
/// <param name="extra">The extra data passed by postfixData.</param>
protected virtual void PostfixDataSign(HashingStream signingStream, object extra)
{
signingStream.Write(FormatBytes, 0, FormatBytes.Length);
}
/// <summary>
/// Prefixes the data then signs it.
/// </summary>
/// <param name="signingStream">The signing stream.</param>
/// <param name="extra">The extra data passed by prefixData.</param>
protected virtual void PrefixDataSign(HashingStream signingStream, object extra)
{
}
/// <summary>
/// Postfixes the data then signs it.
/// </summary>
/// <param name="signingStream">The signing stream.</param>
/// <param name="extra">The extra data passed by postfixData.</param>
protected override void PostfixDataSign(HashingStream signingStream, object extra)
{
}
private async Task <Possible <IReadOnlyList <ContentHash> > > TryGetBuildManifestHashFromLocalFileAsync(string fullFilePath, ContentHash hash, IList <HashType> requestedTypes, int retryAttempt = 0)
{
Contract.Assert(requestedTypes.Count > 0, "Must request at least one hash type");
var result = new List <ContentHash>();
if (retryAttempt >= GetBuildManifestHashFromLocalFileRetryLimit)
{
string message = $"GetBuildManifestHashFromLocalFileRetryLimit exceeded at path '{fullFilePath}'";
Tracing.Logger.Log.ApiServerForwardedIpcServerMessage(m_loggingContext, "BuildManifest", message);
return(new Failure <string>(message));
}
if (retryAttempt > 0)
{
await Task.Delay(TimeSpan.FromMilliseconds(Math.Pow(2, retryAttempt) * GetBuildManifestHashFromLocalFileRetryMultiplierMs));
}
if (!File.Exists(fullFilePath))
{
Tracing.Logger.Log.ApiServerForwardedIpcServerMessage(m_loggingContext, "BuildManifest", $"Local file not found at path '{fullFilePath}' while computing BuildManifest Hash. Trying other methods to obtain hash.");
return(new Failure <string>($"File doesn't exist: '{fullFilePath}'"));
}
var hashers = new HashingStream[requestedTypes.Count - 1];
HashingStream validationStream = null;
try
{
using var fs = FileUtilities.CreateFileStream(fullFilePath, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.Read, FileOptions.SequentialScan).AssertHasLength();
// If enabled, create a hashing stream for content validation. Using HashType.Unknown uses the default hashtype
validationStream = m_verifyFileContentOnBuildManifestHashComputation ? ContentHashingUtilities.GetContentHasher(HashType.Unknown).CreateReadHashingStream(fs) : null;
// Create a series of nested ReadHashingStream so we compute all the hashes in parallel
StreamWithLength outerStream = validationStream?.AssertHasLength() ?? fs;
for (var i = 0; i < hashers.Length; i++)
{
// Hashers has size (requestedTypes.Count - 1)
// requestedTypes[0] will be used to hash+consume the resulting outerStream (see below)
var hashType = requestedTypes[i + 1];
hashers[i] = ContentHashingUtilities.GetContentHasher(hashType).CreateReadHashingStream(outerStream);
outerStream = hashers[i].AssertHasLength();
}
// Hashing the outermost stream will cause all the nested hashers to also do their processing
var firstManifestHash = await ContentHashingUtilities.HashContentStreamAsync(outerStream, requestedTypes[0]);
result.Add(firstManifestHash);
for (int i = 0; i < hashers.Length; i++)
{
result.Add(hashers[i].GetContentHash());
}
if (m_verifyFileContentOnBuildManifestHashComputation)
{
var actualHash = validationStream.GetContentHash();
if (hash != actualHash)
{
return(new Failure <string>($"Unexpected file content during build manifest hash computation. Path: '{fullFilePath}', expected hash '{hash}', actual hash '{actualHash}'."));
}
}
}
catch (Exception ex) when(ex is BuildXLException || ex is IOException)
{
Tracing.Logger.Log.ApiServerForwardedIpcServerMessage(m_loggingContext, "BuildManifest",
$"Local file found at path '{fullFilePath}' but threw exception while computing BuildManifest Hash. Retry attempt {retryAttempt} out of {GetBuildManifestHashFromLocalFileRetryLimit}. Exception: {ex}");
return(await TryGetBuildManifestHashFromLocalFileAsync(fullFilePath, hash, requestedTypes, retryAttempt + 1));
}
finally
{
validationStream?.Dispose();
for (var i = 0; i < hashers.Length; i++)
{
hashers[i]?.Dispose();
}
}
return(result);
}
