/// <summary>
/// This is a self-contained operation to obtain and deserialize a previously stored structure by its hash.
/// First, this tries to load the given content with <see cref="IArtifactContentCache.TryLoadAvailableContentAsync"/>.
/// If the content is available, returns a Bond-deserialized <typeparamref name="T"/>.
/// If the content is unavailable, returns <c>null</c>.
/// To store a structure (the inverse of this method), use <see cref="TrySerializeAndStoreContent{T}"/>.
/// </summary>
public static async Task <Possible <T, Failure> > TryLoadAndDeserializeContent <T>(
this IArtifactContentCache contentCache,
ContentHash contentHash,
BoxRef <long> contentSize = null)
where T : class
{
var maybeStream = await TryGetStreamFromContentHash(contentCache, contentHash, contentSize);
if (!maybeStream.Succeeded)
{
return(maybeStream.Failure);
}
var stream = maybeStream.Result;
if (stream == null)
{
return(default(T));
}
using (stream)
{
int streamLength = (int)stream.Length;
// Use default bond deserializer
return(DeserializeWithInputStream <T>(stream, streamLength));
}
}
/// <summary>
/// Bond-serializes a given <typeparamref name="T"/> and stores the result to the content cache.
/// The returned content hash can be used to later deserialize the structure with <see cref="TryLoadAndDeserializeContent{T}"/>.
/// </summary>
public static Task <Possible <ContentHash> > TrySerializeAndStoreContent <T>(
this IArtifactContentCache contentCache,
T valueToSerialize,
BoxRef <long> contentSize = null,
StoreArtifactOptions options = default)
{
return(BondExtensions.TrySerializeAndStoreContent(
valueToSerialize,
async(valueHash, valueBuffer) =>
{
using (var entryStream = new MemoryStream(
valueBuffer.Array,
valueBuffer.Offset,
valueBuffer.Count,
writable: false))
{
Possible <Unit, Failure> maybeStored = await contentCache.TryStoreAsync(
entryStream,
contentHash: valueHash,
options: options);
return maybeStored.WithGenericFailure();
}
},
contentSize));
}
/// <summary>
/// Bond-serializes a given <typeparamref name="T"/> and stores the result to the content cache.
/// The returned content hash can be used to later deserialize the structure with <see cref="TryLoadAndDeserializeContent{T}"/>.
/// </summary>
public static Task <Possible <ContentHash> > TrySerializeAndStoreContent <T>(
this IArtifactContentCache contentCache,
T valueToSerialize,
BoxRef <long> contentSize = null)
{
return(BondExtensions.TrySerializeAndStoreContent(
contentCache,
valueToSerialize,
TryStoreContentAsync,
contentSize));
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="captureScheduler">indicates the scheduler should be captured and not disposed with engine.</param>
/// <param name="captureFrontEndEngineAbstraction">indicates the created FrontEndEngineAbstraction should be captured and not disposed with engine.</param>
public EngineTestHooksData(bool captureScheduler = false, bool captureFrontEndEngineAbstraction = false)
{
if (captureScheduler)
{
Scheduler = new BoxRef <Scheduler.Scheduler>();
}
if (captureFrontEndEngineAbstraction)
{
FrontEndEngineAbstraction = new BoxRef <FrontEndEngineAbstraction>();
}
}
/// <summary>
/// Gets the cache descriptor metadata given its content hash
/// </summary>
/// <returns>the metadata, or a Failure{<see cref="PipFingerprintEntry"/>} if metadata was retrieved
/// but was a different kind, null if content was not available, or standard <see cref="Failure"/></returns>
public virtual async Task <Possible <PipCacheDescriptorV2Metadata> > TryRetrieveMetadataAsync(
Pip pip,
// TODO: Do we need these fingerprints given that the metadata hash is provided by this interface in the first place
WeakContentFingerprint weakFingerprint,
StrongContentFingerprint strongFingerprint,
ContentHash metadataHash,
ContentHash pathSetHash)
{
BoxRef <long> metadataSize = new BoxRef <long>();
Possible <PipFingerprintEntry> maybeMetadata =
await ArtifactContentCache.TryLoadAndDeserializeContentWithRetry <PipFingerprintEntry>(
LoggingContext,
metadataHash,
contentSize : metadataSize,
cancellationToken : Context.CancellationToken,
shouldRetry : possibleResult => !possibleResult.Succeeded || (possibleResult.Result != null && possibleResult.Result.IsCorrupted),
maxRetry : PipFingerprintEntry.LoadingAndDeserializingRetries);
if (!maybeMetadata.Succeeded)
{
return(maybeMetadata.Failure);
}
Counters.IncrementCounter(PipCachingCounter.LoadedMetadataCount);
Counters.AddToCounter(PipCachingCounter.LoadedMetadataSize, metadataSize.Value);
var metadataEntry = maybeMetadata.Result;
if (metadataEntry == null)
{
return((PipCacheDescriptorV2Metadata)null);
}
if (metadataEntry.Kind != PipFingerprintEntryKind.DescriptorV2)
{
// Metadata is incorrect kind.
var message = I($"Expected metadata kind is '{nameof(PipFingerprintEntryKind.DescriptorV2)}' but got '{metadataEntry.Kind}'");
return(new Failure <PipFingerprintEntry>(metadataEntry, new Failure <string>(message)));
}
return((PipCacheDescriptorV2Metadata)metadataEntry.Deserialize(
Context.CancellationToken,
new CacheQueryData
{
WeakContentFingerprint = weakFingerprint,
PathSetHash = pathSetHash,
StrongContentFingerprint = strongFingerprint,
MetadataHash = metadataHash,
ContentCache = ArtifactContentCache
}));
}
/// <summary>
/// Stores cache descriptor metadata for pip
/// </summary>
public virtual async Task <Possible <ContentHash> > TryStoreMetadataAsync(PipCacheDescriptorV2Metadata metadata)
{
BoxRef <long> metadataSize = new BoxRef <long>();
var result = await ArtifactContentCache.TrySerializeAndStoreContent(metadata.ToEntry(), metadataSize);
if (result.Succeeded)
{
Counters.IncrementCounter(PipCachingCounter.StoredMetadataCount);
Counters.AddToCounter(PipCachingCounter.StoredMetadataSize, metadataSize.Value);
}
return(result);
}
private static async Task <Possible <Stream, Failure> > TryGetStreamFromContentHash(
IArtifactContentCache contentCache,
ContentHash contentHash,
CancellationToken cancellationToken,
BoxRef <long> contentSize = null)
{
if (!EngineEnvironmentSettings.SkipExtraneousPins)
{
Possible <ContentAvailabilityBatchResult, Failure> maybeAvailable =
await contentCache.TryLoadAvailableContentAsync(new[] { contentHash }, cancellationToken);
if (!maybeAvailable.Succeeded)
{
return(maybeAvailable.Failure);
}
bool contentIsAvailable = maybeAvailable.Result.AllContentAvailable;
if (!contentIsAvailable)
{
return(default(Stream));
}
}
var maybeStream = await contentCache.TryOpenContentStreamAsync(contentHash);
if (!maybeStream.Succeeded)
{
if (maybeStream.Failure is NoCasEntryFailure)
{
return(default(Stream));
}
return(maybeStream.Failure);
}
Stream stream = maybeStream.Result;
if (contentSize != null)
{
contentSize.Value = stream.Length;
}
return(stream);
}
/// <summary>
/// Configures the host builder to use the given values as services rather than creating its own on
/// application initialization.
/// </summary>
private static void UseExternalServices(IHostBuilder hostBuilder, BoxRef <OperationContext> operationContext, BoxRef <HostParameters> hostParameters)
{
hostBuilder
.ConfigureWebHostDefaults(webBuilder =>
{
webBuilder.UseStartup <DeploymentProxyStartup>();
})
.ConfigureHostConfiguration(configBuilder =>
{
configBuilder.AddInMemoryCollection(new Dictionary <string, string>()
{
{ UseExternalServicesKey, bool.TrueString }
});
})
.ConfigureServices(services =>
{
services.AddSingleton <ILogger>(s => operationContext.Value.TracingContext.Logger);
services.AddSingleton <BoxRef <OperationContext> >(s => operationContext);
services.AddSingleton <HostParameters>(s => hostParameters.Value);
});
}
/// <summary>
/// Runs <see cref="TryLoadAndDeserializeContent{T}(IArtifactContentCache, ContentHash, BoxRef{long})"/> with some retry logic.
/// </summary>
public static async Task <Possible <T, Failure> > TryLoadAndDeserializeContentWithRetry <T>(
this IArtifactContentCache contentCache,
LoggingContext loggingContext,
ContentHash contentHash,
Func <Possible <T, Failure>, bool> shouldRetry,
BoxRef <long> contentSize = null,
int maxRetry = 1)
where T : class
{
Contract.Requires(loggingContext != null);
Contract.Requires(shouldRetry != null);
int retryCount = 0;
Possible <T, Failure> result;
do
{
result = await TryLoadAndDeserializeContent <T>(contentCache, contentHash, contentSize);
if (!shouldRetry(result))
{
if (retryCount > 0)
{
Tracing.Logger.Log.RetryOnLoadingAndDeserializingMetadata(loggingContext, true, retryCount);
}
return(result);
}
++retryCount;
}while (retryCount < maxRetry);
if (retryCount > 1)
{
Tracing.Logger.Log.RetryOnLoadingAndDeserializingMetadata(loggingContext, false, retryCount);
}
return(result);
}
private static async Task <Possible <Stream, Failure> > TryGetStreamFromContentHash(
IArtifactContentCache contentCache,
ContentHash contentHash,
BoxRef <long> contentSize = null)
{
Possible <ContentAvailabilityBatchResult, Failure> maybeAvailable =
await contentCache.TryLoadAvailableContentAsync(new[] { contentHash });
if (!maybeAvailable.Succeeded)
{
return(maybeAvailable.Failure);
}
bool contentIsAvailable = maybeAvailable.Result.AllContentAvailable;
if (!contentIsAvailable)
{
return(default(Stream));
}
var maybeStream = await contentCache.TryOpenContentStreamAsync(contentHash);
if (!maybeStream.Succeeded)
{
return(maybeStream.Failure);
}
Stream stream = maybeStream.Result;
if (contentSize != null)
{
contentSize.Value = stream.Length;
}
return(stream);
}
/// <summary>
/// Store the fingerprint store directory to the cache
/// </summary>
public static async Task <Possible <long> > TrySaveFingerprintStoreAsync(
this EngineCache cache,
LoggingContext loggingContext,
AbsolutePath path,
PathTable pathTable,
string key,
string environment)
{
var fingerprint = ComputeFingerprint(pathTable, key, environment);
var pathStr = path.ToString(pathTable);
BoxRef <long> size = 0;
SemaphoreSlim concurrencyLimiter = new SemaphoreSlim(8);
var tasks = new List <Task <Possible <StringKeyedHash, Failure> > >();
FileUtilities.EnumerateDirectoryEntries(pathStr, (name, attr) =>
{
var task = Task.Run(async() =>
{
using (await concurrencyLimiter.AcquireAsync())
{
var filePath = path.Combine(pathTable, name);
var storeResult = await cache.ArtifactContentCache.TryStoreAsync(
FileRealizationMode.Copy,
filePath.Expand(pathTable));
if (storeResult.Succeeded)
{
Interlocked.Add(ref size.Value, new FileInfo(filePath.ToString(pathTable)).Length);
}
return(storeResult.Then(result => new StringKeyedHash()
{
Key = path.ExpandRelative(pathTable, filePath),
ContentHash = result.ToBondContentHash()
}));
}
});
tasks.Add(task);
});
var storedFiles = await Task.WhenAll(tasks);
var failure = storedFiles.Where(p => !p.Succeeded).Select(p => p.Failure).FirstOrDefault();
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Saving fingerprint store to cache: Success='{failure == null}', FileCount={storedFiles.Length} Size={size.Value}"));
if (failure != null)
{
return(failure);
}
PackageDownloadDescriptor descriptor = new PackageDownloadDescriptor()
{
TraceInfo = loggingContext.Session.Environment,
FriendlyName = nameof(FingerprintStore),
Contents = storedFiles.Select(p => p.Result).ToList()
};
var storeDescriptorResult = await cache.ArtifactContentCache.TrySerializeAndStoreContent(descriptor);
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Saving fingerprint store descriptor to cache: Success='{storeDescriptorResult.Succeeded}'"));
if (!storeDescriptorResult.Succeeded)
{
return(storeDescriptorResult.Failure);
}
var associatedFileHashes = descriptor.Contents.Select(s => s.ContentHash.ToContentHash()).ToArray().ToReadOnlyArray().GetSubView(0);
var cacheEntry = new CacheEntry(storeDescriptorResult.Result, null, associatedFileHashes);
var publishResult = await cache.TwoPhaseFingerprintStore.TryPublishTemporalCacheEntryAsync(loggingContext, fingerprint, cacheEntry);
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Publishing fingerprint store to cache: Fingerprint='{fingerprint}' Hash={storeDescriptorResult.Result}"));
return(size.Value);
}
/// <summary>
/// Try load contants.
/// </summary>
public static async Task <Possible <byte[], Failure> > TryLoadContent(
this IArtifactContentCache contentCache,
ContentHash contentHash,
BoxRef <long> contentSize = null,
bool failOnNonSeekableStream = false,
int byteLimit = int.MaxValue)
{
var maybeStream = await TryGetStreamFromContentHash(contentCache, contentHash, contentSize);
if (!maybeStream.Succeeded)
{
return(maybeStream.Failure);
}
var stream = maybeStream.Result;
if (stream == null)
{
return(default(byte[]));
}
try
{
MemoryStream memoryStream;
Stream streamToRead;
if (!stream.CanSeek)
{
if (failOnNonSeekableStream)
{
return(new Failure <string>("Stream is not seekable"));
}
memoryStream = new MemoryStream();
streamToRead = memoryStream;
}
else
{
memoryStream = null;
streamToRead = stream;
}
using (memoryStream)
{
if (memoryStream != null)
{
await stream.CopyToAsync(memoryStream);
memoryStream.Position = 0;
}
Contract.Assert(streamToRead.CanSeek);
if (streamToRead.Length > byteLimit)
{
return(new Failure <string>(I($"Stream exceeds limit: Length: {streamToRead.Length} byte(s) | Limit: {byteLimit} byte(s)")));
}
var length = (int)streamToRead.Length;
var contentBytesLocal = new byte[length];
int read = 0;
while (read < length)
{
int readThisIteration = await streamToRead.ReadAsync(contentBytesLocal, read, length - read);
if (readThisIteration == 0)
{
return(new Failure <string>("Unexpected end of stream"));
}
read += readThisIteration;
}
return(contentBytesLocal);
}
}
catch (Exception e)
{
return(new Failure <string>(e.GetLogEventMessage()));
}
}
public async Task TestHistoricMetadataPathStringRoundtrip()
{
LoggingContext loggingContext = CreateLoggingContextForTest();
PipExecutionContext context;
HistoricMetadataCache cache = null;
var hmcFolderName = "hmc";
for (int i = 0; i < 3; i++)
{
CreateHistoricCache(loggingContext, hmcFolderName, out context, out cache, out var memoryArtifactCache);
var process1 = SchedulerTest.CreateDummyProcess(context, new PipId(1));
var process2 = SchedulerTest.CreateDummyProcess(context, new PipId(2));
var pathTable = context.PathTable;
// Add some random paths to ensure path table indices are different after loading
AbsolutePath.Create(pathTable, X("/H/aslj/sfas/832.stxt"));
AbsolutePath.Create(pathTable, X("/R/f/s/Historic"));
AbsolutePath.Create(pathTable, X("/M/hgf/sf4as/83afsd"));
AbsolutePath.Create(pathTable, X("/Z/bd/sfas/Cache"));
var abPath1 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/p1OUT.bin"));
var abPath2 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/P2.txt"));
var pathSet1 = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/X/a/b/c"),
X("/X/d/e"),
X("/X/a/b/c/d"));
PipCacheDescriptorV2Metadata metadata1 =
new PipCacheDescriptorV2Metadata
{
StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo>
{
new AbsolutePathFileMaterializationInfo
{
AbsolutePath = abPath1.GetName(pathTable).ToString(context.StringTable),
Info = new BondFileMaterializationInfo {
FileName = "p1OUT.bin"
}
}
}
};
var storedPathSet1 = await cache.TryStorePathSetAsync(pathSet1, preservePathCasing : false);
var storedMetadata1 = await cache.TryStoreMetadataAsync(metadata1);
var weakFingerprint1 = new WeakContentFingerprint(FingerprintUtilities.CreateRandom());
var strongFingerprint1 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom());
var cacheEntry = new CacheEntry(storedMetadata1.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty);
var publishedCacheEntry = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet1.Result, strongFingerprint1, cacheEntry);
var pathSet2 = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/F/a/y/c"),
X("/B/d/e"),
X("/G/a/z/c/d"),
X("/B/a/b/c"));
PipCacheDescriptorV2Metadata metadata2 =
new PipCacheDescriptorV2Metadata
{
StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo>
{
new AbsolutePathFileMaterializationInfo
{
AbsolutePath = abPath2.ToString(pathTable),
Info = new BondFileMaterializationInfo {
FileName = abPath2.GetName(pathTable).ToString(context.StringTable)
}
}
},
DynamicOutputs = new List <List <RelativePathFileMaterializationInfo> >
{
new List <RelativePathFileMaterializationInfo>
{
new RelativePathFileMaterializationInfo
{
RelativePath = @"dir\P2Dynamic.txt",
Info = new BondFileMaterializationInfo {
FileName = "p2dynamic.txt"
}
},
new RelativePathFileMaterializationInfo
{
RelativePath = @"dir\P2dynout2.txt",
Info = new BondFileMaterializationInfo {
FileName = null
}
}
}
}
};
var storedPathSet2 = await cache.TryStorePathSetAsync(pathSet2, preservePathCasing : false);
var storedMetadata2 = await cache.TryStoreMetadataAsync(metadata2);
var cacheEntry2 = new CacheEntry(storedMetadata2.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty);
var strongFingerprint2 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom());
var publishedCacheEntry2 = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet2.Result, strongFingerprint2, cacheEntry2);
await cache.CloseAsync();
memoryArtifactCache.Clear();
PipExecutionContext loadedContext;
HistoricMetadataCache loadedCache;
TaskSourceSlim <bool> loadCompletionSource = TaskSourceSlim.Create <bool>();
TaskSourceSlim <bool> loadCalled = TaskSourceSlim.Create <bool>();
BoxRef <bool> calledLoad = new BoxRef <bool>();
CreateHistoricCache(loggingContext, "hmc", out loadedContext, out loadedCache, out memoryArtifactCache, loadTask: async hmc =>
{
loadCalled.SetResult(true);
await loadCompletionSource.Task;
});
var operationContext = OperationContext.CreateUntracked(loggingContext);
var retrievePathSet1Task = loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet1.Result);
var retrievdMetadata1Task = loadedCache.TryRetrieveMetadataAsync(
process1,
WeakContentFingerprint.Zero,
StrongContentFingerprint.Zero,
storedMetadata1.Result,
storedPathSet1.Result);
var getCacheEntry1Task = loadedCache.TryGetCacheEntryAsync(
process1,
weakFingerprint1,
storedPathSet1.Result,
strongFingerprint1);
Assert.False(retrievePathSet1Task.IsCompleted, "Before load task completes. TryRetrievePathSetAsync operations should block");
Assert.False(retrievdMetadata1Task.IsCompleted, "Before load task completes. TryRetrieveMetadataAsync operations should block");
Assert.False(getCacheEntry1Task.IsCompleted, "Before load task completes. TryGetCacheEntryAsync operations should block");
Assert.True(loadCalled.Task.Wait(TimeSpan.FromSeconds(10)) && loadCalled.Task.Result, "Load should have been called in as a result of querying");
loadCompletionSource.SetResult(true);
var maybeLoadedPathSet1 = await retrievePathSet1Task;
var maybeLoadedMetadata1 = await retrievdMetadata1Task;
var maybeLoadedCacheEntry1 = await getCacheEntry1Task;
Assert.Equal(storedMetadata1.Result, maybeLoadedCacheEntry1.Result.Value.MetadataHash);
var maybeLoadedPathSet2 = await loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet2.Result);
var maybeLoadedMetadata2 = await loadedCache.TryRetrieveMetadataAsync(
process2,
WeakContentFingerprint.Zero,
StrongContentFingerprint.Zero,
storedMetadata2.Result,
storedPathSet2.Result);
AssertPathSetEquals(pathTable, pathSet1, loadedContext.PathTable, maybeLoadedPathSet1.Result);
AssertPathSetEquals(pathTable, pathSet2, loadedContext.PathTable, maybeLoadedPathSet2.Result);
AssertMetadataEquals(metadata1, maybeLoadedMetadata1.Result);
AssertMetadataEquals(metadata2, maybeLoadedMetadata2.Result);
await loadedCache.CloseAsync();
}
}
public async Task TestFullDeployment()
{
var sources = new Dictionary <string, string>()
{
{ @"Stamp3\info.txt", "" },
{ @"Env\RootFile.json", "{ 'key1': 1, 'key2': 2 }" },
{ @"Env\Subfolder\Hello.txt", "Hello world" },
{ @"Env\Foo.txt", "Baz" },
{ @"Files\Foo.txt", "Bar" },
};
Dictionary <string, string> getSourceDrop(string root, string prefix)
{
return(sources.Where(e => e.Key.StartsWith(root))
.ToDictionary(e => e.Key.Substring(prefix.Length), e => e.Value));
}
var drops = new Dictionary <string, Dictionary <string, string> >()
{
{
"https://dev.azure.com/buildxlcachetest/drop/drops/dev/testdrop1?root=release/win-x64",
new Dictionary <string, string>
{
{ @"file1.bin", "File content 1" },
{ @"file2.txt", "File content 2" },
{ @"sub\file3.dll", "File content 3" }
}
},
{
"https://dev.azure.com/buildxlcachetest/drop/drops/dev/testdrop2?root=debug",
new Dictionary <string, string>
{
{ @"file1.bin", "File content 1" },
{ @"file2.txt", "File content 2 changed" },
{ @"sub\file5.dll", "File content 5" }
}
},
{
DeploymentUtilities.ConfigDropUri.OriginalString,
new Dictionary <string, string>()
{
{ DeploymentUtilities.DeploymentConfigurationFileName, ConfigString }
}
},
{
"file://Env", getSourceDrop(@"Env\", @"Env\")
},
{
"file://Files/Foo.txt", getSourceDrop(@"Files\Foo.txt", @"Files\")
},
{
"file://Env/Foo.txt", getSourceDrop(@"Env\Foo.txt", @"Env\")
},
{
"file://Stamp3", getSourceDrop(@"Stamp3\", @"Stamp3\")
}
};
var deploymentRoot = TestRootDirectoryPath / "deploy";
var ingester = new DeploymentIngester(
Context,
sourceRoot: base.TestRootDirectoryPath / "src",
deploymentRoot: deploymentRoot,
deploymentConfigurationPath: base.TestRootDirectoryPath / "DeploymentConfiguration.json",
FileSystem,
dropExeFilePath: base.TestRootDirectoryPath / @"dropbin\drop.exe",
retentionSizeGb: 1,
dropToken: DropToken);
// Write source files
WriteFiles(ingester.SourceRoot, sources);
FileSystem.WriteAllText(ingester.DeploymentConfigurationPath, ConfigString);
ingester.OverrideLaunchDropProcess = t =>
{
var dropContents = drops[t.dropUrl];
WriteFiles(new AbsolutePath(t.targetDirectory) / (t.relativeRoot ?? ""), dropContents);
return(BoolResult.Success);
};
await ingester.RunAsync().ShouldBeSuccess();
var manifestText = FileSystem.ReadAllText(ingester.DeploymentManifestPath);
var deploymentManifest = JsonSerializer.Deserialize <DeploymentManifest>(manifestText);
foreach (var drop in drops)
{
var uri = new Uri(drop.Key);
var expectedDropContents = drops[drop.Key];
var layoutSpec = deploymentManifest.Drops[drop.Key];
layoutSpec.Count.Should().Be(expectedDropContents.Count);
foreach (var fileAndContent in expectedDropContents)
{
var hash = new ContentHash(layoutSpec[fileAndContent.Key].Hash);
var expectedPath = ingester.DeploymentRoot / DeploymentUtilities.GetContentRelativePath(hash);
var text = FileSystem.ReadAllText(expectedPath);
text.Should().Be(fileAndContent.Value);
}
}
var clock = new MemoryClock();
var deploymentService = new DeploymentService(new DeploymentServiceConfiguration(), deploymentRoot, _ => new TestSecretsProvider(), clock);
BoxRef <Task> uploadFileCompletion = Task.CompletedTask;
deploymentService.OverrideCreateCentralStorage = t => new DeploymentTestCentralStorage(t.storageSecretName)
{
UploadFileCompletion = uploadFileCompletion
};
await verifyLaunchManifestAsync(new DeploymentParameters()
{
Stamp = "ST_S3",
Ring = "Ring_1"
},
new HashSet <(string targetPath, string drop)>()
{
("bin", "https://dev.azure.com/buildxlcachetest/drop/drops/dev/testdrop2?root=debug"),
("", "file://Env"),
("info", "file://Stamp3"),
});
async Task <LauncherManifest> verifyLaunchManifestAsync(DeploymentParameters parameters, HashSet <(string targetPath, string drop)> expectedDrops)
{
var launchManifest = await deploymentService.UploadFilesAndGetManifestAsync(Context, parameters, waitForCompletion : true);
var expectedDeploymentPathToHashMap = new Dictionary <string, string>();
launchManifest.Drops.Count.Should().Be(expectedDrops.Count);
foreach (var drop in launchManifest.Drops)
{
var targetRelativePath = drop.TargetRelativePath ?? string.Empty;
expectedDrops.Should().Contain((targetRelativePath, drop.Url));
var dropSpec = deploymentManifest.Drops[drop.Url];
foreach (var dropFile in drops[drop.Url])
{
expectedDeploymentPathToHashMap[Path.Combine(targetRelativePath, dropFile.Key)] = dropSpec[dropFile.Key].Hash;
}
}
launchManifest.Deployment.Count.Should().Be(expectedDeploymentPathToHashMap.Count);
foreach (var file in launchManifest.Deployment)
{
var expectedHash = expectedDeploymentPathToHashMap[file.Key];
file.Value.Hash.Should().Be(expectedHash);
}
return(launchManifest);
}
}
