/// <summary>
/// Bond-serializes a given <typeparamref name="T"/> and stores the result to the content cache.
/// The returned content hash can be used to later deserialize the structure with <see cref="TryLoadAndDeserializeContent{T}"/>.
/// </summary>
public static Task <Possible <ContentHash> > TrySerializeAndStoreContent <T>(
this IArtifactContentCache contentCache,
T valueToSerialize,
BoxRef <long> contentSize = null,
StoreArtifactOptions options = default)
{
return(BondExtensions.TrySerializeAndStoreContent(
valueToSerialize,
async(valueHash, valueBuffer) =>
{
using (var entryStream = new MemoryStream(
valueBuffer.Array,
valueBuffer.Offset,
valueBuffer.Count,
writable: false))
{
Possible <Unit, Failure> maybeStored = await contentCache.TryStoreAsync(
entryStream,
contentHash: valueHash,
options: options);
return maybeStored.WithGenericFailure();
}
},
contentSize));
}
/// <summary>
/// Bond-serializes a given <typeparamref name="T"/> and stores the result to the content cache.
/// The returned content hash can be used to later deserialize the structure with <see cref="TryLoadAndDeserializeContent{T}"/>.
/// </summary>
public static Task <Possible <ContentHash> > TrySerializeAndStoreContent <T>(
this IArtifactContentCache contentCache,
T valueToSerialize,
BoxRef <long> contentSize = null)
{
return(BondExtensions.TrySerializeAndStoreContent(
contentCache,
valueToSerialize,
TryStoreContentAsync,
contentSize));
}
/// <summary>
/// Deserialize metadata from reader
/// </summary>
public static PipCacheDescriptorV2Metadata ReadPipCacheDescriptor(BuildXLReader reader)
{
if (reader.ReadBoolean())
{
var length = reader.ReadInt32Compact();
var blob = new ArraySegment <byte>(reader.ReadBytes(length));
return(BondExtensions.Deserialize <PipCacheDescriptorV2Metadata>(blob));
}
else
{
return(null);
}
}
/// <summary>
/// Serialize metadata to writer
/// </summary>
public static void WritePipCacheDescriptor(BuildXLWriter writer, PipCacheDescriptorV2Metadata metadata)
{
if (metadata != null)
{
writer.Write(true);
var blob = BondExtensions.Serialize(metadata);
writer.WriteCompact(blob.Count);
writer.Write(blob.Array, blob.Offset, blob.Count);
}
else
{
writer.Write(false);
}
}
/// <summary>
/// Store the fingerprint store directory to the cache
/// </summary>
public static async Task <Possible <long> > TrySaveFingerprintStoreAsync(
this EngineCache cache,
LoggingContext loggingContext,
AbsolutePath path,
PathTable pathTable,
string key,
string environment)
{
var fingerprint = ComputeFingerprint(pathTable, key, environment);
var pathStr = path.ToString(pathTable);
BoxRef <long> size = 0;
SemaphoreSlim concurrencyLimiter = new SemaphoreSlim(8);
var tasks = new List <Task <Possible <StringKeyedHash, Failure> > >();
FileUtilities.EnumerateDirectoryEntries(pathStr, (name, attr) =>
{
var task = Task.Run(async() =>
{
using (await concurrencyLimiter.AcquireAsync())
{
var filePath = path.Combine(pathTable, name);
var storeResult = await cache.ArtifactContentCache.TryStoreAsync(
FileRealizationMode.Copy,
filePath.Expand(pathTable));
if (storeResult.Succeeded)
{
Interlocked.Add(ref size.Value, new FileInfo(filePath.ToString(pathTable)).Length);
}
var result = storeResult.Then(result => new StringKeyedHash()
{
Key = path.ExpandRelative(pathTable, filePath),
ContentHash = result.ToBondContentHash()
});
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Saving fingerprint store to cache: Success='{storeResult.Succeeded}', FilePath='{filePath}' Key='{result.Result.Key}' Hash='{result.Result.ContentHash.ToContentHash()}'"));
return(result);
}
});
tasks.Add(task);
});
var storedFiles = await Task.WhenAll(tasks);
var failure = storedFiles.Where(p => !p.Succeeded).Select(p => p.Failure).FirstOrDefault();
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Saving fingerprint store to cache: Success='{failure == null}', FileCount={storedFiles.Length} Size={size.Value}"));
if (failure != null)
{
return(failure);
}
PackageDownloadDescriptor descriptor = new PackageDownloadDescriptor()
{
TraceInfo = loggingContext.Session.Environment,
FriendlyName = nameof(FingerprintStore),
Contents = storedFiles.Select(p => p.Result).ToList()
};
// Publish contents before saving the descriptor to make sure content exist before the descriptor exist to avoid fingerprintstore opening errors
var storeDescriptorBuffer = BondExtensions.Serialize(descriptor);
var storeDescriptorHash = ContentHashingUtilities.HashBytes(
storeDescriptorBuffer.Array,
storeDescriptorBuffer.Offset,
storeDescriptorBuffer.Count);
var associatedFileHashes = descriptor.Contents.Select(s => s.ContentHash.ToContentHash()).ToArray().ToReadOnlyArray().GetSubView(0);
var cacheEntry = new CacheEntry(storeDescriptorHash, null, associatedFileHashes);
var publishResult = await cache.TwoPhaseFingerprintStore.TryPublishTemporalCacheEntryAsync(loggingContext, fingerprint, cacheEntry);
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Publishing fingerprint store to cache: Fingerprint='{fingerprint}' Hash={storeDescriptorHash} Success='{publishResult.Succeeded}'"));
var storeDescriptorResult = await cache.ArtifactContentCache.TryStoreContent(storeDescriptorHash, storeDescriptorBuffer);
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Saving fingerprint store descriptor to cache: Success='{storeDescriptorResult.Succeeded}'"));
if (!storeDescriptorResult.Succeeded)
{
return(storeDescriptorResult.Failure);
}
return(size.Value);
}
