public void ToStringCorrect(HashType hashType, string expected)
{
var hashLength = HashInfoLookup.Find(hashType).ByteLength;
var hash = new ContentHash(hashType, Enumerable.Range(0, hashLength).Select(i => (byte)i).ToArray());
Assert.Equal(expected, hash.ToString());
}
/// <summary>
/// See <see cref="ITwoPhaseFingerprintStore.TryPublishCacheEntryAsync"/>
/// </summary>
public virtual async Task <Possible <CacheEntryPublishResult, Failure> > TryPublishCacheEntryAsync(
Pip pip,
WeakContentFingerprint weakFingerprint,
ContentHash pathSetHash,
StrongContentFingerprint strongFingerprint,
CacheEntry entry,
CacheEntryPublishMode mode = CacheEntryPublishMode.CreateNew)
{
Contract.Requires(pathSetHash.IsValid);
Contract.Requires(entry.MetadataHash.IsValid);
var result = await TwoPhaseFingerprintStore.TryPublishCacheEntryAsync(
weakFingerprint,
pathSetHash,
strongFingerprint,
entry,
mode);
if (result.Succeeded)
{
var publishedEntry = result.Result.Status == CacheEntryPublishStatus.Published ? entry : result.Result.ConflictingEntry;
Tracing.Logger.Log.PipTwoPhaseCachePublishCacheEntry(
LoggingContext,
pip.GetDescription(Context),
weakFingerprint.ToString(),
pathSetHash.ToString(),
strongFingerprint.ToString(),
entry.MetadataHash.ToString(),
result.Result.Status.ToString(),
publishedEntry.MetadataHash.ToString());
}
return(result);
}
private async Task <BoolResult> GetFileWithDedupAsync(Context context, ContentHash contentHash, string path, CancellationToken cts)
{
VstsBlobIdentifier blobId = ToVstsBlobIdentifier(contentHash.ToBlobIdentifier());
VstsDedupIdentifier dedupId = blobId.ToDedupIdentifier();
try
{
await TryGatedArtifactOperationAsync <Object>(
context,
contentHash.ToString(),
"DownloadToFileAsync",
async innerCts =>
{
await DedupStoreClient.DownloadToFileAsync(dedupId, path, null, null, EdgeCache.Allowed, innerCts);
return(null);
}, cts);
}
catch (NullReferenceException) // Null reference thrown when DedupIdentifier doesn't exist in VSTS.
{
return(new BoolResult("DedupIdentifier not found."));
}
catch (Exception ex)
{
return(new BoolResult(ex));
}
return(BoolResult.Success);
}
public override void OpenStreamStart(Context context, ContentHash contentHash)
{
if (_eventSource.IsEnabled())
{
_eventSource.OpenStreamStart(context.Id.ToString(), contentHash.ToString());
}
base.OpenStreamStart(context, contentHash);
}
public void PinStart(Context context, ContentHash contentHash)
{
if (_eventSource.IsEnabled())
{
_eventSource.PinStart(context.TraceId, contentHash.ToString());
}
PinStart(context);
}
public override void PutFileStart(Context context, AbsolutePath path, FileRealizationMode mode, ContentHash contentHash, bool trusted)
{
if (_eventSource.IsEnabled())
{
_eventSource.PutFileStart(context.TraceId, path.Path, (int)mode, contentHash.ToString());
}
base.PutFileStart(context, path, mode, contentHash, trusted);
}
public void EvictStart(ContentHash contentHash)
{
if (_eventSource.IsEnabled())
{
_eventSource.EvictStart(contentHash.ToString());
}
_evictCallCounter.Started();
}
public override void PutStreamStart(Context context, ContentHash contentHash)
{
if (_eventSource.IsEnabled())
{
_eventSource.PutStreamStart(context.TraceId, contentHash.ToString());
}
base.PutStreamStart(context, contentHash);
}
protected IDownloadFileSettings GetSampleData(string url, DownloadArchiveType archiveType, ContentHash?contentHash = null)
{
return(new DownloadFileSettings
{
ModuleName = "TestDownload",
ArchiveType = archiveType,
Url = url,
Hash = contentHash?.ToString()
});
}
public void RoundtripFullBuffer(HashType hashType)
{
var buffer = new byte[ContentHash.SerializedLength];
var h1 = ContentHash.Random(hashType);
h1.Serialize(buffer);
var h2 = new ContentHash(buffer);
Assert.Equal(hashType, h2.HashType);
Assert.Equal(h1.ToString(), h2.ToString());
}
public void RoundtripFullBufferPositiveOffset(HashType hashType)
{
const int offset = 3;
var buffer = new byte[ContentHash.SerializedLength + offset];
var h1 = ContentHash.Random(hashType);
h1.Serialize(buffer, offset);
var h2 = new ContentHash(buffer, offset);
Assert.Equal(hashType, h2.HashType);
Assert.Equal(h1.ToString(), h2.ToString());
}
/// <summary>
/// Gets the string representation of <see cref="GraphAgnosticIncrementalSchedulingStateId"/>.
/// </summary>
public override string ToString()
{
var stringBuilder = new StringBuilder();
stringBuilder.AppendLine("[");
stringBuilder.AppendLine(I($"\tMachine name : {m_machineName}"));
stringBuilder.AppendLine(I($"\tSubst source : {m_substSource}"));
stringBuilder.AppendLine(I($"\tSubst target : {m_substTarget}"));
stringBuilder.AppendLine(I($"\tPreserve output salt : {m_preserveOutputSalt.ToString()}"));
stringBuilder.AppendLine("]");
return(stringBuilder.ToString());
}
public override void PlaceFileStart(
Context context,
ContentHash contentHash,
AbsolutePath path,
FileAccessMode accessMode,
FileReplacementMode replacementMode,
FileRealizationMode realizationMode)
{
if (_eventSource.IsEnabled())
{
_eventSource.PlaceFileStart(
context.TraceId, contentHash.ToString(), path.Path, (int)accessMode, (int)replacementMode, (int)realizationMode);
}
base.PlaceFileStart(context, contentHash, path, accessMode, replacementMode, realizationMode);
}
public void PlaceFileCopy(Context context, AbsolutePath path, ContentHash contentHash, TimeSpan duration)
{
if (_eventSource.IsEnabled())
{
_eventSource.PlaceFileCopy(context.TraceId, path.Path, contentHash.ToString());
}
_placeFileCopyCallCounter.Completed(duration.Ticks);
if (!context.IsEnabled)
{
return;
}
var ms = (long)duration.TotalMilliseconds;
TraceDiagnostic(context, $"{Name}.PlaceFileCopy({path},{contentHash.ToShortString()}) {ms}ms", TimeSpan.FromMilliseconds(ms), operation: "PlaceFileCopy");
}
public void PlaceFileCopy(Context context, AbsolutePath path, ContentHash contentHash, TimeSpan duration)
{
if (_eventSource.IsEnabled())
{
_eventSource.PlaceFileCopy(context.Id.ToString(), path.Path, contentHash.ToString());
}
_placeFileCopyCallCounter.Completed(duration.Ticks);
if (!context.IsEnabled)
{
return;
}
var ms = (long)duration.TotalMilliseconds;
Debug(context, $"{Name}.PlaceFileCopy({path},{contentHash}) {ms}ms");
}
public void RoundtripFullBinary(HashType hashType)
{
using (var ms = new MemoryStream())
{
using (var writer = new BinaryWriter(ms))
{
var h1 = ContentHash.Random(hashType);
h1.Serialize(writer);
Assert.Equal(ContentHash.SerializedLength, ms.Length);
ms.Position = 0;
using (var reader = new BinaryReader(ms))
{
var h2 = new ContentHash(reader);
Assert.Equal(hashType, h2.HashType);
Assert.Equal(h1.ToString(), h2.ToString());
}
}
}
}
/// <summary>
/// See <see cref="ITwoPhaseFingerprintStore.TryGetCacheEntryAsync"/>
/// </summary>
public virtual async Task <Possible <CacheEntry?, Failure> > TryGetCacheEntryAsync(
Pip pip,
WeakContentFingerprint weakFingerprint,
ContentHash pathSetHash,
StrongContentFingerprint strongFingerprint)
{
var result = await TwoPhaseFingerprintStore.TryGetCacheEntryAsync(weakFingerprint, pathSetHash, strongFingerprint);
if (result.Succeeded)
{
Tracing.Logger.Log.PipTwoPhaseCacheGetCacheEntry(
LoggingContext,
pip.GetDescription(Context),
weakFingerprint.ToString(),
pathSetHash.ToString(),
strongFingerprint.ToString(),
result.Result.HasValue ? result.Result.Value.MetadataHash.ToString() : "<NOVALUE>");
}
return(result);
}
public async Task <string> GetDiagnosticsAsync()
{
Contract.Requires(IsValid);
FileInfo existingFile = default;
bool hasWritableACL = false;
uint linkCount = 0;
string existingHashStr = string.Empty;
if (FileUtilities.FileExistsNoFollow(Path))
{
ContentHash existingHash = await ContentHashingUtilities.HashFileAsync(Path);
existingHashStr = existingHash.ToString();
existingFile = new FileInfo(Path);
hasWritableACL = FileUtilities.HasWritableAccessControl(Path);
linkCount = FileUtilities.GetHardLinkCount(Path);
}
string fileToDeleteTimestamp = m_fileInfoPriorDeletion != null?m_fileInfoPriorDeletion.CreationTime.ToString("MM/dd/yyyy hh:mm:ss.fff tt") : "Non-existent";
string existingFileTimestamp = existingFile != null?existingFile.CreationTime.ToString("MM/dd/yyyy hh:mm:ss.fff tt") : "Non-existent";
string deletionTime = m_deletionTime.ToLocalTime().ToString("MM/dd/yyyy hh:mm:ss.fff tt");
var info = new[]
{
I($"File: {Path}"),
I($"Deletion attempt time: {deletionTime}"),
I($"Creation time of file to delete: {fileToDeleteTimestamp}"),
I($"Creation time of existing file: {existingFileTimestamp}"),
I($"Content hash of existing file: {existingHashStr}"),
I($"Has writable ACL: {hasWritableACL}"),
I($"Link count: {linkCount}")
};
return(string.Join(" | ", info));
}
/// <inherit />
public override string ToString()
{
return($"({Hash.ToString()}: is{(IsAvailable ? "" : " not")} available, {BytesTransferred}B transferred from {SourceCache}){(Failure != null ? " due to Failure: " + Failure.Describe() : "")}");
}
/// <inherit />
public override string ToString()
{
return($"({Hash.ToString()}: is{(IsAvailable ? "" : " not")} available, {BytesTransferred}B transferred from {SourceCache})");
}
/// <summary>
/// Implements a copy file request.
/// </summary>
public async Task CopyFileAsync(CopyFileRequest request, IServerStreamWriter <CopyFileResponse> responseStream, ServerCallContext context)
{
try
{
LogRequestHandling();
// Get the content stream.
Context cacheContext = new Context(new Guid(request.TraceId), _logger);
HashType type = (HashType)request.HashType;
ContentHash hash = request.ContentHash.ToContentHash((HashType)request.HashType);
OpenStreamResult result = await GetFileStreamAsync(cacheContext, hash);
using (result.Stream)
{
// Figure out response headers.
CopyCompression compression = CopyCompression.None;
Metadata headers = new Metadata();
headers.Add("ContentHash", hash.ToString());
switch (result.Code)
{
case OpenStreamResult.ResultCode.ContentNotFound:
headers.Add("Exception", "ContentNotFound");
headers.Add("Message", $"Requested content {hash} not found.");
break;
case OpenStreamResult.ResultCode.Error:
Debug.Assert(result.Exception != null);
headers.Add("Exception", result.Exception.GetType().Name);
headers.Add("Message", result.Exception.Message);
break;
case OpenStreamResult.ResultCode.Success:
Debug.Assert(result.Stream != null);
long size = result.Stream.Length;
headers.Add("FileSize", size.ToString());
if ((request.Compression == CopyCompression.Gzip) && (size > ContentStore.Grpc.CopyConstants.DefaultBufferSize))
{
compression = CopyCompression.Gzip;
}
headers.Add("Compression", compression.ToString());
headers.Add("ChunkSize", ContentStore.Grpc.CopyConstants.DefaultBufferSize.ToString());
break;
default:
throw new NotImplementedException();
}
// Send the response headers.
await context.WriteResponseHeadersAsync(headers);
// Send the content.
if (result.Succeeded)
{
byte[] buffer = new byte[ContentStore.Grpc.CopyConstants.DefaultBufferSize];
switch (compression)
{
case CopyCompression.None:
await StreamContentAsync(result.Stream, buffer, responseStream, context.CancellationToken);
break;
case CopyCompression.Gzip:
await StreamContentWithCompressionAsync(result.Stream, buffer, responseStream, context.CancellationToken);
break;
}
}
}
}
catch (Exception)
{
throw;
}
}
public void HashAlgorithmIsStable()
{
// We want to test with multiple blocks.
var bytes = new byte[VsoHash.BlockSize * 5];
using (var hashAlgorithm = new VsoHashAlgorithm())
{
hashAlgorithm.Initialize();
byte[] hashAlgoBytes = hashAlgorithm.ComputeHash(bytes);
var hash = new ContentHash(HashType.Vso0, hashAlgoBytes);
Assert.Equal("VSO0:36668B653DB0B48D3AA1F2FDDCEA481B34A310C166B9B041A5B23B59BE02E5DB00", hash.ToString());
}
}
/// <inheritdoc />
public override string ToString()
{
string content = Hash.HashType != HashType.Unknown ? Hash.ToString() : "<UnknownHashType>";
return(I($"[Content {content} (Length: {m_length})]"));
}
private void EnsureRemote(ContentHash hash)
{
bool entryExists = m_content.TryGetValue(hash, out CacheEntry entry);
Contract.Assert(entryExists);
Contract.Assert(entry.Sites != CacheSites.None);
GetPaths(hash, out string localPath, out string remotePath);
if ((entry.Sites & CacheSites.Remote) == 0)
{
Contract.Requires((entry.Sites & CacheSites.Local) != 0);
ExceptionUtilities.HandleRecoverableIOException(
() => Upload(hash),
ex => throw new BuildXLException(I($"Unable to ensure remote replication of content hash '{hash.ToString()}'")));
}
else
{
Contract.Assert(File.Exists(remotePath));
}
entry.Sites |= CacheSites.Remote;
}
/// <summary>
/// Deserializes an <see cref="IPipFingerprintEntryData"/> of the appropriate concrete type (or null if the type is unknown).
/// <see cref="PipFingerprintEntry"/> is a tagged polymorphic container (see <see cref="Kind"/>),
/// and this operation unwraps it. The returned instance contains all relevant data for the entry.
/// </summary>
public IPipFingerprintEntryData Deserialize(CacheQueryData cacheQueryData = null)
{
if (Kind == PipFingerprintEntryKind.Unknown)
{
return(null);
}
if (m_deserializedEntryData != null)
{
return(m_deserializedEntryData);
}
try
{
InputBuffer buffer = new InputBuffer(DataBlob);
CompactBinaryReader <InputBuffer> reader = new CompactBinaryReader <InputBuffer>(buffer);
IPipFingerprintEntryData deserialized;
switch (Kind)
{
case PipFingerprintEntryKind.DescriptorV1:
deserialized = Deserialize <PipCacheDescriptor> .From(reader);
break;
case PipFingerprintEntryKind.DescriptorV2:
deserialized = Deserialize <PipCacheDescriptorV2Metadata> .From(reader);
break;
case PipFingerprintEntryKind.GraphDescriptor:
deserialized = Deserialize <PipGraphCacheDescriptor> .From(reader);
break;
case PipFingerprintEntryKind.FileDownload:
deserialized = Deserialize <FileDownloadDescriptor> .From(reader);
break;
case PipFingerprintEntryKind.PackageDownload:
deserialized = Deserialize <PackageDownloadDescriptor> .From(reader);
break;
case PipFingerprintEntryKind.GraphInputDescriptor:
deserialized = Deserialize <PipGraphInputDescriptor> .From(reader);
break;
default:
throw Contract.AssertFailure("Unhandled PipFingerprintEntryKind");
}
Interlocked.CompareExchange(ref m_deserializedEntryData, deserialized, comparand: null);
return(Volatile.Read(ref m_deserializedEntryData));
}
catch (Exception exception)
{
if (IsCorrupted)
{
OutputBuffer valueBuffer = new OutputBuffer(1024);
CompactBinaryWriter <OutputBuffer> writer = new CompactBinaryWriter <OutputBuffer>(valueBuffer);
Serialize.To(writer, this);
// Include in the log the hash of this instance so that we can trace it in the cache log, and obtain the file in the CAS.
ContentHash valueHash = ContentHashingUtilities.HashBytes(
valueBuffer.Data.Array,
valueBuffer.Data.Offset,
valueBuffer.Data.Count);
const string Unspecified = "<Unspecified>";
string actualEntryBlob = Unspecified;
string actualEntryHash = Unspecified;
if (cacheQueryData != null && cacheQueryData.ContentCache != null)
{
var maybeActualContent = cacheQueryData.ContentCache.TryLoadContent(
cacheQueryData.MetadataHash,
failOnNonSeekableStream: true,
byteLimit: 20 * 1024 * 1024).Result;
if (maybeActualContent.Succeeded && maybeActualContent.Result != null)
{
actualEntryBlob = ToByteArrayString(maybeActualContent.Result);
actualEntryHash = ContentHashingUtilities.HashBytes(maybeActualContent.Result).ToString();
}
}
Logger.Log.DeserializingCorruptedPipFingerprintEntry(
Events.StaticContext,
kind: Kind.ToString(),
weakFingerprint: cacheQueryData?.WeakContentFingerprint.ToString() ?? Unspecified,
pathSetHash: cacheQueryData?.PathSetHash.ToString() ?? Unspecified,
strongFingerprint: cacheQueryData?.StrongContentFingerprint.ToString() ?? Unspecified,
expectedHash: cacheQueryData?.MetadataHash.ToString() ?? Unspecified, // expected metadata hash
hash: valueHash.ToString(), // re-computed hash
blob: ToDataBlobString(), // blob of data carried by pip fingerprint entry
actualHash: actualEntryHash, // actual pip fingerprint entry hash
actualEntryBlob: actualEntryBlob); // actual pip fingerprint entry blob
throw new BuildXLException("Deserializing corrupted pip fingerprint entry", exception, ExceptionRootCause.CorruptedCache);
}
// We don't expect this to happen so rethrow the exception.
throw;
}
}
private string GetFileNameFromHash(ContentHash hash) => hash.ToString().Replace(':', '_').ToUpperInvariant();
