Result <MachineLocation[]> IDistributedContentCopierHost.GetDesignatedLocations(ContentHash hash)
{
return(_contentLocationStore.GetDesignatedLocations(hash));
}
/// <summary>
/// Initializes a new instance of the <see cref="PutResult" /> class.
/// </summary>
public PutResult(ResultBase other, ContentHash contentHash, string message = null)
: base(other, message)
{
ContentHash = contentHash;
}
/// <inheritdoc />
public async Task <FileExistenceResult> CheckFileExistsAsync(Context context, ContentHash contentHash)
{
if (InnerContentStore is IStreamStore innerStreamStore)
{
return(await innerStreamStore.CheckFileExistsAsync(context, contentHash));
}
return(new FileExistenceResult(FileExistenceResult.ResultCode.Error, $"{InnerContentStore} does not implement {nameof(IStreamStore)} in {nameof(DistributedContentStore<T>)}."));
}
/// <nodoc />
protected abstract Task <OpenStreamResult> OpenStreamCoreAsync(
OperationContext operationContext,
ContentHash contentHash,
UrgencyHint urgencyHint,
Counter retryCounter);
/// <inheritdoc />
protected override Task <PutResult> PutStreamCoreAsync(OperationContext operationContext, ContentHash contentHash, Stream stream, UrgencyHint urgencyHint, Counter retryCounter)
{
return(_innerSession.PutStreamAsync(operationContext, contentHash, stream, operationContext.Token, urgencyHint));
}
/// <inheritdoc />
protected override async Task <PlaceFileResult> PlaceFileCoreAsync(OperationContext operationContext, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter)
{
if (replacementMode != FileReplacementMode.ReplaceExisting && _fileSystem.FileExists(path))
{
if (replacementMode == FileReplacementMode.SkipIfExists)
{
return(new PlaceFileResult(PlaceFileResult.ResultCode.NotPlacedAlreadyExists));
}
else if (replacementMode == FileReplacementMode.FailIfExists)
{
return(new PlaceFileResult(
PlaceFileResult.ResultCode.Error,
$"File exists at destination {path} with FailIfExists specified"));
}
}
var virtualPath = _contentManager.ToVirtualPath(path);
if (virtualPath == null)
{
return(await _innerSession.PlaceFileAsync(operationContext, contentHash, path, accessMode, replacementMode, realizationMode, operationContext.Token, urgencyHint));
}
_contentManager.Tree.AddFileNode(virtualPath, new VfsFilePlacementData(contentHash, realizationMode, accessMode));
return(new PlaceFileResult(GetPlaceResultCode(realizationMode, accessMode), fileSize: -1 /* Unknown */));
}
private Task <ProactiveCopyResult> RequestProactiveCopyIfNeededAsync(OperationContext context, ContentHash hash)
{
if (!_pendingProactivePuts.Add(hash))
{
return(Task.FromResult(ProactiveCopyResult.CopyNotRequiredResult));
}
return(context.PerformOperationAsync(
Tracer,
traceErrorsOnly: true,
operation: async() =>
{
try
{
var hashArray = _buildIdHash != null
? new[] { hash, _buildIdHash.Value }
: new[] { hash };
// First check in local location store, then global if failed.
var getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local);
if (getLocationsResult.Succeeded && getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
_counters[Counters.GetLocationsSatisfiedFromLocal].Increment();
return ProactiveCopyResult.CopyNotRequiredResult;
}
else
{
getLocationsResult += await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global).ThrowIfFailure();
_counters[Counters.GetLocationsSatisfiedFromRemote].Increment();
}
if (getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
return ProactiveCopyResult.CopyNotRequiredResult;
}
IReadOnlyList <MachineLocation> buildRingMachines;
// Get random machine inside build ring
Task <BoolResult> insideRingCopyTask;
if (_buildIdHash != null)
{
buildRingMachines = getLocationsResult.ContentHashesInfo[getLocationsResult.ContentHashesInfo.Count - 1].Locations;
var candidates = buildRingMachines.Where(m => !m.Equals(LocalCacheRootMachineLocation)).ToArray();
if (candidates.Length > 0)
{
var candidate = candidates[ThreadSafeRandom.Generator.Next(0, candidates.Length)];
Tracer.Info(context, $"{nameof(RequestProactiveCopyIfNeededAsync)}: Copying {hash.ToShortString()} to machine '{candidate}' in build ring (of {candidates.Length} machines).");
insideRingCopyTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate);
}
else
{
insideRingCopyTask = Task.FromResult(new BoolResult("Could not find any machines belonging to the build ring."));
}
}
else
{
insideRingCopyTask = Task.FromResult(new BoolResult("BuildId was not specified, so machines in the build ring cannot be found."));
buildRingMachines = new[] { LocalCacheRootMachineLocation };
}
Task <BoolResult> outsideRingCopyTask;
var getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: buildRingMachines);
if (getLocationResult.Succeeded)
{
var candidate = getLocationResult.Value;
Tracer.Info(context, $"{nameof(RequestProactiveCopyIfNeededAsync)}: Copying {hash.ToShortString()} to machine '{candidate}' outside build ring.");
outsideRingCopyTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate);
}
else
{
outsideRingCopyTask = Task.FromResult(new BoolResult(getLocationResult));
}
return new ProactiveCopyResult(await insideRingCopyTask, await outsideRingCopyTask);
}
finally
{
_pendingProactivePuts.Remove(hash);
}
}));
}
protected override void CorruptContent(TestFileSystemContentStoreInternal store, ContentHash contentHash)
{
store.CorruptContent(contentHash);
}
public static ReplicaRank GetReplicaRank(
ContentHash hash,
ContentLocationEntry entry,
MachineId localMachineId,
LocalLocationStoreConfiguration configuration,
DateTime now)
{
var locationsCount = entry.Locations.Count;
var desiredReplicaCount = configuration.DesiredReplicaRetention;
if (desiredReplicaCount == 0 ||
locationsCount == 0) // It is possible for the entry to have 0 locations.
// For instance, the database has 1 location but the machine is in the bad state
{
return(ReplicaRank.None);
}
if (locationsCount <= desiredReplicaCount
// If using throttled eviction, we might need to upgrade the rank to Protected
// so don't return here
&& configuration.ThrottledEvictionInterval == TimeSpan.Zero)
{
return(ReplicaRank.Important);
}
// Making sure that probabilistically, some locations are considered important for the current machine.
long contentHashCode = unchecked ((uint)HashCodeHelper.Combine(hash[0] | hash[1] << 8, hash[1]));
var importantRangeStart = contentHashCode % locationsCount;
// Getting an index of a current location in the location list
int currentMachineLocationIndex = entry.Locations.GetMachineIdIndex(localMachineId);
if (currentMachineLocationIndex == -1)
{
// This is used for testing only. The machine Id should be part of the machines.
// But in tests it is useful to control the behavior of this method and in some cases to guarantee that some replica won't be important.
return(ReplicaRank.None);
}
// In case of important range wrapping around end of location list to start of location list
// we need to compute a positive offset from the range start to see if the replica exists in the range
// i.e. range start = 5, location count = 7, and desired location count = 3
// important range contains [5, 6 and 0] since it overflows the end of the list
var offset = currentMachineLocationIndex - importantRangeStart;
if (offset < 0)
{
offset += locationsCount;
}
var lastImportantReplicaOffset = Math.Min(desiredReplicaCount, locationsCount) - 1;
if (offset >= desiredReplicaCount)
{
return(ReplicaRank.None);
}
if (offset != lastImportantReplicaOffset)
{
// All but last important replica are always Protected
return(ReplicaRank.Protected);
}
if (configuration.ThrottledEvictionInterval == TimeSpan.Zero)
{
// Throttled eviction is disabled. Just mark the replica as important
// since its in the important range
return(ReplicaRank.Important);
}
// How throttled eviction works:
// 1. Compute which machines consider the content important
// This is done by computing a hash code from the content hash modulo location count to
// generate a start index into the list replicas.
// For instance,
// given locations: [4, 11, 22, 35, 73, 89]
// locationCount = 6,
// if contentHashCode % locationCount = 2 and DesiredReplicaCount = 3
// then the machines considering content important are [22, 35, 73]
// 2. All but last important replica must be consider Protected (i.e. 22, 35 have rank Protected)
// 3. Compute if last replica is protected.
// This is based of to time ranges or buckets of duration ThrottledEvictionInterval
// For instance,
// if ThrottleInterval = 20 minutes
// 10:00AM-10:20AM -> (timeBucketIndex = 23045230) % DesiredReplicaCount = 2 = evictableOffset
// 10:20AM-10:40AM -> (timeBucketIndex = 23045231) % DesiredReplicaCount = 0 = evictableOffset
// 10:40AM-11:00AM -> (timeBucketIndex = 23045232) % DesiredReplicaCount = 1 = evictableOffset
// 11:00AM-11:20AM -> (timeBucketIndex = 23045233) % DesiredReplicaCount = 2 = evictableOffset
// So for times 10:00AM-10:20AM and 11:00AM-11:20AM the last important replica is evictable
var timeBucketIndex = now.Ticks / configuration.ThrottledEvictionInterval.Ticks;
// NOTE: We add contentHashCode to timeBucketIndex so that not all Protected content is considered evictable
// at the same time
var evictableOffset = (contentHashCode + timeBucketIndex) % desiredReplicaCount;
if (evictableOffset == offset)
{
return(ReplicaRank.Important);
}
else
{
// The replica is not currently evictable. Mark it as protected which will give it the minimum effective age
// so that it is only evicted as a last resort
return(ReplicaRank.Protected);
}
}
/// <nodoc />
public static Task <EvictResult> EvictAsync(
this ContentStoreInternalTracer tracer, OperationContext context, ContentHash contentHash, Func <Task <EvictResult> > func)
{
return(EvictCall.RunAsync(tracer, context, contentHash, func));
}
public override string ToString()
{
return($"[ContentHash={ContentHash.ToShortString()} Size={Size} LocationCount={Locations?.Count}]");
}
public ContentHashWithSizeAndLocations(ContentHash contentHash, long size = -1)
{
ContentHash = contentHash;
Size = size;
}
private async Task <ProactiveCopyResult> ProactiveCopyIfNeededAsync(OperationContext operationContext, ContentHash hash)
{
var sessionResult = await ProactiveCopySession.Value;
if (sessionResult)
{
return(await sessionResult.Value.ProactiveCopyIfNeededAsync(
operationContext,
hash,
tryBuildRing : false,
reason : ProactiveCopyReason.Replication));
}
return(new ProactiveCopyResult(sessionResult, "Failed to retrieve session for proactive copies."));
}
private async Task CopyFileAsync(CopyFileRequest request, IServerStreamWriter <CopyFileResponse> responseStream, ServerCallContext context)
{
OperationStarted();
// Get the content stream.
Context cacheContext = new Context(new Guid(request.TraceId), Logger);
ContentHash hash = request.GetContentHash();
OpenStreamResult result = await GetFileStreamAsync(cacheContext, hash);
// If result is unsuccessful, then result.Stream is null, but using(null) is just a no op.
using (result.Stream)
{
// Figure out response headers.
CopyCompression compression = CopyCompression.None;
Metadata headers = new Metadata();
switch (result.Code)
{
case OpenStreamResult.ResultCode.ContentNotFound:
headers.Add("Exception", "ContentNotFound");
headers.Add("Message", $"Requested content at {hash} not found.");
break;
case OpenStreamResult.ResultCode.Error:
Contract.AssertNotNull(result.Exception);
headers.Add("Exception", result.Exception.GetType().Name);
headers.Add("Message", result.Exception.Message);
break;
case OpenStreamResult.ResultCode.Success:
Contract.AssertNotNull(result.Stream);
long size = result.Stream.Length;
headers.Add("FileSize", size.ToString());
if ((request.Compression == CopyCompression.Gzip) && (size > _gzipSizeBarrier))
{
compression = CopyCompression.Gzip;
}
headers.Add("Compression", compression.ToString());
headers.Add("ChunkSize", _bufferSize.ToString());
break;
default:
throw new NotImplementedException($"Unknown result.Code '{result.Code}'.");
}
// Send the response headers.
await context.WriteResponseHeadersAsync(headers);
// Send the content.
if (result.Succeeded)
{
var operationContext = new OperationContext(cacheContext, context.CancellationToken);
using (var arrayHandle = _pool.Get())
{
StreamContentDelegate streamContent = compression == CopyCompression.None ? (StreamContentDelegate)StreamContentAsync : StreamContentWithCompressionAsync;
byte[] buffer = arrayHandle.Value;
await operationContext.PerformOperationAsync(
_tracer,
() => streamContent(result.Stream !, buffer, responseStream, context.CancellationToken),
traceOperationStarted : false, // Tracing only stop messages
extraEndMessage : r => $"Hash={hash.ToShortString()}, GZip={(compression == CopyCompression.Gzip ? "on" : "off")}.")
.IgnoreFailure(); // The error was already logged.
}
}
}
}
/// <summary>
/// Deserializes an <see cref="IPipFingerprintEntryData"/> of the appropriate concrete type (or null if the type is unknown).
/// <see cref="PipFingerprintEntry"/> is a tagged polymorphic container (see <see cref="Kind"/>),
/// and this operation unwraps it. The returned instance contains all relevant data for the entry.
/// </summary>
public IPipFingerprintEntryData Deserialize(CacheQueryData cacheQueryData = null)
{
if (Kind == PipFingerprintEntryKind.Unknown)
{
return(null);
}
if (m_deserializedEntryData != null)
{
return(m_deserializedEntryData);
}
try
{
InputBuffer buffer = new InputBuffer(DataBlob);
CompactBinaryReader <InputBuffer> reader = new CompactBinaryReader <InputBuffer>(buffer);
IPipFingerprintEntryData deserialized;
switch (Kind)
{
case PipFingerprintEntryKind.DescriptorV1:
deserialized = Deserialize <PipCacheDescriptor> .From(reader);
break;
case PipFingerprintEntryKind.DescriptorV2:
deserialized = Deserialize <PipCacheDescriptorV2Metadata> .From(reader);
break;
case PipFingerprintEntryKind.GraphDescriptor:
deserialized = Deserialize <PipGraphCacheDescriptor> .From(reader);
break;
case PipFingerprintEntryKind.FileDownload:
deserialized = Deserialize <FileDownloadDescriptor> .From(reader);
break;
case PipFingerprintEntryKind.PackageDownload:
deserialized = Deserialize <PackageDownloadDescriptor> .From(reader);
break;
case PipFingerprintEntryKind.GraphInputDescriptor:
deserialized = Deserialize <PipGraphInputDescriptor> .From(reader);
break;
default:
throw Contract.AssertFailure("Unhandled PipFingerprintEntryKind");
}
Interlocked.CompareExchange(ref m_deserializedEntryData, deserialized, comparand: null);
return(Volatile.Read(ref m_deserializedEntryData));
}
catch (Exception exception)
{
if (IsCorrupted)
{
OutputBuffer valueBuffer = new OutputBuffer(1024);
CompactBinaryWriter <OutputBuffer> writer = new CompactBinaryWriter <OutputBuffer>(valueBuffer);
Serialize.To(writer, this);
// Include in the log the hash of this instance so that we can trace it in the cache log, and obtain the file in the CAS.
ContentHash valueHash = ContentHashingUtilities.HashBytes(
valueBuffer.Data.Array,
valueBuffer.Data.Offset,
valueBuffer.Data.Count);
const string Unspecified = "<Unspecified>";
string actualEntryBlob = Unspecified;
string actualEntryHash = Unspecified;
if (cacheQueryData != null && cacheQueryData.ContentCache != null)
{
var maybeActualContent = cacheQueryData.ContentCache.TryLoadContent(
cacheQueryData.MetadataHash,
failOnNonSeekableStream: true,
byteLimit: 20 * 1024 * 1024).Result;
if (maybeActualContent.Succeeded && maybeActualContent.Result != null)
{
actualEntryBlob = ToByteArrayString(maybeActualContent.Result);
actualEntryHash = ContentHashingUtilities.HashBytes(maybeActualContent.Result).ToString();
}
}
Logger.Log.DeserializingCorruptedPipFingerprintEntry(
Events.StaticContext,
kind: Kind.ToString(),
weakFingerprint: cacheQueryData?.WeakContentFingerprint.ToString() ?? Unspecified,
pathSetHash: cacheQueryData?.PathSetHash.ToString() ?? Unspecified,
strongFingerprint: cacheQueryData?.StrongContentFingerprint.ToString() ?? Unspecified,
expectedHash: cacheQueryData?.MetadataHash.ToString() ?? Unspecified, // expected metadata hash
hash: valueHash.ToString(), // re-computed hash
blob: ToDataBlobString(), // blob of data carried by pip fingerprint entry
actualHash: actualEntryHash, // actual pip fingerprint entry hash
actualEntryBlob: actualEntryBlob); // actual pip fingerprint entry blob
throw new BuildXLException("Deserializing corrupted pip fingerprint entry", exception, ExceptionRootCause.CorruptedCache);
}
// We don't expect this to happen so rethrow the exception.
throw;
}
}
/// <nodoc />
public DownloadIncrementalState(DownloadData downloadData, ContentHash contentHash)
{
m_downloadData = downloadData;
ContentHash = contentHash;
}
/// <inheritdoc /> public PathBase GeneratePath(ContentHash contentHash, byte[] contentLocationIdContent) => null;
/// <inheritdoc />
protected override Task <PutResult> PutFileCoreAsync(OperationContext operationContext, ContentHash contentHash, AbsolutePath path, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter)
{
return(_innerSession.PutFileAsync(operationContext, contentHash, path, realizationMode, operationContext.Token, urgencyHint));
}
public override AbsolutePath GeneratePath(ContentHash contentHash, byte[] contentLocationIdContent)
{
LastContentLocation = contentLocationIdContent;
return(_root);
}
/// <inheritdoc />
protected override Task <PinResult> PinCoreAsync(OperationContext operationContext, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter)
{
return(_innerSession.PinAsync(operationContext, contentHash, operationContext.Token, urgencyHint));
}
public PutResult(bool succeeded, ContentHash contentHash, string errorMessage, string diagnostics = null)
: base(succeeded, errorMessage, diagnostics)
{
ContentHash = contentHash;
}
/// <summary>
/// Records a <see cref="ContentHash" /> for the given file handle. This hash mapping will be persisted to disk if the
/// table is saved with <see cref="SaveAsync" />. The given file handle should be opened with at most Read sharing
/// (having the handle should ensure the file is not being written).
/// This returns a <see cref="VersionedFileIdentityAndContentInfo"/>:
/// - The identity has the kind <see cref="VersionedFileIdentity.IdentityKind.StrongUsn"/> if a USN-based identity was successfully established;
/// the identity may have kind <see cref="VersionedFileIdentity.IdentityKind.Anonymous"/> if such an identity was unavailable.
/// - Regardless, the contained <see cref="FileContentInfo"/> contains the actual length of the stream corresponding to <paramref name="hash"/>.
/// </summary>
/// <remarks>
/// An overload taking a file path is intentionally not provided. This should be called after hashing or writing a file,
/// but before closing the handle. This way, there is no race between establishing the file's hash, some unrelated writer,
/// and recording its file version (e.g., USN) to hash mapping.
/// Note that this results in a small amount of I/O (e.g., on Windows, a file open and USN query), but never hashes the file or reads its contents.
/// The <paramref name="strict"/> corresponds to the <c>flush</c> parameter of <see cref="VersionedFileIdentity.TryEstablishStrong"/>
/// </remarks>
public VersionedFileIdentity RecordContentHash(
string path,
SafeFileHandle handle,
ContentHash hash,
long length,
bool?strict = default)
{
Contract.Requires(handle != null);
Contract.Requires(!string.IsNullOrWhiteSpace(path));
using (Counters.StartStopwatch(FileContentTableCounters.RecordContentHashDuration))
{
// TODO: The contract below looks very nice but breaks tons of UT
// Fix the tests and enable the contract.
// Contract.Requires(FileContentInfo.IsValidLength(length, hash));
// Here we write a new change journal record for this file to get a 'strong' identity. This means that the USN -> hash table
// only ever contains USNs whose records have the 'close' reason set. Recording USNs without that
// reason set would not be correct; it would be possible that multiple separate changes (e.g. writes)
// were represented with the same USN, and so intermediate USNs do not necessarily correspond to exactly
// one snapshot of a file. See http://msdn.microsoft.com/en-us/library/windows/desktop/aa363803(v=vs.85).aspx
Possible <VersionedFileIdentity, Failure <VersionedFileIdentity.IdentityUnavailabilityReason> > possibleVersionedIdentity =
TryEstablishStrongIdentity(handle, flush: strict == true);
if (!possibleVersionedIdentity.Succeeded)
{
if (Interlocked.CompareExchange(ref m_changeJournalWarningLogged, 1, 0) == 0)
{
Tracing.Logger.Log.StorageFileContentTableIgnoringFileSinceVersionedFileIdentityIsNotSupported(
Events.StaticContext,
path,
possibleVersionedIdentity.Failure.DescribeIncludingInnerFailures());
}
return(VersionedFileIdentity.Anonymous);
}
VersionedFileIdentity identity = possibleVersionedIdentity.Result;
var newEntry = new Entry(identity.Usn, hash, length, EntryTimeToLive);
// We allow concurrent update attempts with different observed USNs.
// This is useful and relevant for two reasons:
// - Querying a 'strong' identity (TryEstablishStrongIdentity) generates a new CLOSE record every time.
// - Creating hardlinks generates 'hardlink change' records.
// So, concurrently creating and recording (or even just recording) different links is possible, and
// keeping the last stored entry (rather than highest-USN entry) can introduce false positives.
var fileIdAndVolumeId = new FileIdAndVolumeId(identity.VolumeSerialNumber, identity.FileId);
m_entries.AddOrUpdate(
new FileIdAndVolumeId(identity.VolumeSerialNumber, identity.FileId),
newEntry,
updateValueFactory: (key, existingEntry) =>
{
if (existingEntry.Usn > newEntry.Usn)
{
return(existingEntry);
}
if (newEntry.Hash == existingEntry.Hash)
{
Counters.IncrementCounter(FileContentTableCounters.NumUsnMismatch);
Tracing.Logger.Log.StorageUsnMismatchButContentMatch(
Events.StaticContext,
path,
existingEntry.Usn.Value,
newEntry.Usn.Value,
existingEntry.Hash.ToHex());
}
else
{
// Stale USN.
Counters.IncrementCounter(FileContentTableCounters.NumContentMismatch);
}
return(newEntry);
});
Tracing.Logger.Log.StorageRecordNewKnownUsn(
Events.StaticContext,
path,
identity.FileId.High,
identity.FileId.Low,
identity.VolumeSerialNumber,
identity.Usn.Value,
hash.ToHex());
return(identity);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="PutResult"/> class.
/// </summary>
public PutResult(ContentHash contentHash, string errorMessage, string diagnostics = null)
: base(errorMessage, diagnostics)
{
ContentHash = contentHash;
}
/// <inheritdoc />
public Task <DeleteResult> DeleteAsync(Context context, ContentHash contentHash)
{
throw new NotImplementedException();
}
/// <summary>
/// Initializes a new instance of the <see cref="PutResult" /> class.
/// </summary>
public PutResult(Exception exception, ContentHash contentHash, string message = null)
: base(exception, message)
{
ContentHash = contentHash;
}
/// <nodoc />
protected abstract Task <PinResult> PinCoreAsync(
OperationContext operatonContext,
ContentHash contentHash,
UrgencyHint urgencyHint,
Counter retryCounter);
internal static string GetRedisKey(ContentHash hash)
{
// Use the string representation short hash used in other parts of the system (db and event stream) as the redis key
return(new ShortHash(hash).ToString());
}
