public async Task TestHandleResponseAsync()
{
using (MemoryStream blobContent = new MemoryStream(Encoding.UTF8.GetBytes("some BLOB content")))
{
BlobDescriptor descriptor = await Digests.ComputeDigestAsync(blobContent).ConfigureAwait(false);
DescriptorDigest testBlobDigest = descriptor.GetDigest();
blobContent.Position = 0;
using (HttpResponseMessage mockResponse = new HttpResponseMessage()
{
Content = new StringContent("some BLOB content")
})
{
LongAdder byteCount = new LongAdder();
BlobPuller blobPuller =
new BlobPuller(
fakeRegistryEndpointRequestProperties,
testBlobDigest,
layerOutputStream,
size => Assert.AreEqual("some BLOB content".Length, size),
byteCount.Add);
await blobPuller.HandleResponseAsync(mockResponse).ConfigureAwait(false);
Assert.AreEqual(
"some BLOB content",
Encoding.UTF8.GetString(layerContentOutputStream.ToArray()));
Assert.AreEqual(testBlobDigest, layerOutputStream.ComputeDigest().GetDigest());
Assert.AreEqual("some BLOB content".Length, byteCount.Sum());
}
}
}
/**
* Writes an uncompressed {@code layerBlob} to the {@code layerDirectory}.
*
* @param uncompressedLayerBlob the uncompressed layer {@link Blob}
* @param layerDirectory the directory for the layer
* @return a {@link WrittenLayer} with the written layer information
* @throws IOException if an I/O exception occurs
*/
private async Task <WrittenLayer> WriteUncompressedLayerBlobToDirectoryAsync(
IBlob uncompressedLayerBlob, SystemPath layerDirectory)
{
using (TemporaryFile temporaryLayerFile = CacheStorageFiles.GetTemporaryLayerFile(layerDirectory))
{
DescriptorDigest layerDiffId;
BlobDescriptor blobDescriptor;
// Writes the layer with GZIP compression. The original bytes are captured as the layer's
// diff ID and the bytes outputted from the GZIP compression are captured as the layer's
// content descriptor.
using (CountingDigestOutputStream compressedDigestOutputStream =
new CountingDigestOutputStream(
Files.NewOutputStream(temporaryLayerFile.Path)))
{
using (GZipStream compressorStream = new GZipStream(compressedDigestOutputStream, CompressionMode.Compress, true))
{
BlobDescriptor descriptor = await uncompressedLayerBlob.WriteToAsync(compressorStream).ConfigureAwait(false);
layerDiffId = descriptor.GetDigest();
}
// The GZIPOutputStream must be closed in order to write out the remaining compressed data.
blobDescriptor = compressedDigestOutputStream.ComputeDigest();
}
DescriptorDigest layerDigest = blobDescriptor.GetDigest();
long layerSize = blobDescriptor.GetSize();
// Renames the temporary layer file to the correct filename.
SystemPath layerFile = layerDirectory.Resolve(cacheStorageFiles.GetLayerFilename(layerDiffId));
temporaryLayerFile.MoveIfDoesNotExist(layerFile);
return(new WrittenLayer(layerDigest, layerDiffId, layerSize));
}
}
public async Task TestHandleResponse_unexpectedDigestAsync()
{
using (MemoryStream blobContent = new MemoryStream(Encoding.UTF8.GetBytes("some BLOB content")))
{
BlobDescriptor descriptor = await Digests.ComputeDigestAsync(blobContent).ConfigureAwait(false);
DescriptorDigest testBlobDigest = descriptor.GetDigest();
blobContent.Position = 0;
using (HttpResponseMessage mockResponse = new HttpResponseMessage()
{
Content = new StringContent("some BLOB content")
})
{
try
{
await testBlobPuller.HandleResponseAsync(mockResponse).ConfigureAwait(false);
Assert.Fail("Receiving an unexpected digest should fail");
}
catch (UnexpectedBlobDigestException ex)
{
Assert.AreEqual(
"The pulled BLOB has digest '"
+ testBlobDigest
+ "', but the request digest was '"
+ fakeDigest
+ "'",
ex.Message);
}
}
}
}
public async Task TestPullAsync()
{
// Pulls the busybox image.
localRegistry.PullAndPushToLocal("busybox", "busybox");
RegistryClient registryClient =
RegistryClient.CreateFactory(EventHandlers.NONE, "localhost:5000", "busybox")
.SetAllowInsecureRegistries(true)
.NewRegistryClient();
V21ManifestTemplate manifestTemplate =
await registryClient.PullManifestAsync <V21ManifestTemplate>("latest").ConfigureAwait(false);
DescriptorDigest realDigest = manifestTemplate.GetLayerDigests().First();
// Pulls a layer BLOB of the busybox image.
LongAdder totalByteCount = new LongAdder();
LongAdder expectedSize = new LongAdder();
IBlob pulledBlob =
registryClient.PullBlob(
realDigest,
size =>
{
Assert.AreEqual(0, expectedSize.Sum());
expectedSize.Add(size);
},
totalByteCount.Add);
BlobDescriptor blobDescriptor = await pulledBlob.WriteToAsync(Stream.Null).ConfigureAwait(false);
Assert.AreEqual(realDigest, blobDescriptor.GetDigest());
Assert.IsTrue(expectedSize.Sum() > 0);
Assert.AreEqual(expectedSize.Sum(), totalByteCount.Sum());
}
public async Task <BuildResult> CallAsync()
{
IReadOnlyList <BlobDescriptor> baseImageDescriptors = await pushBaseImageLayersStep.GetFuture().ConfigureAwait(false);
IReadOnlyList <BlobDescriptor> appLayerDescriptors = await pushApplicationLayersStep.GetFuture().ConfigureAwait(false);
BlobDescriptor containerConfigurationBlobDescriptor = await pushContainerConfigurationStep.GetFuture().ConfigureAwait(false);
ImmutableHashSet <string> targetImageTags = buildConfiguration.GetAllTargetImageTags();
using (var progressEventDispatcher =
progressEventDispatcherFactory.Create("pushing image manifest", this.Index))
using (var factory =
progressEventDispatcher.NewChildProducer()("[child progress]pushing image manifest", targetImageTags.Count))
using (TimerEventDispatcher ignored =
new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), DESCRIPTION))
{
RegistryClient registryClient =
buildConfiguration
.NewTargetImageRegistryClientFactory()
.SetAuthorization(await authenticatePushStep.GetFuture().ConfigureAwait(false))
.NewRegistryClient();
// Constructs the image.
ImageToJsonTranslator imageToJsonTranslator =
new ImageToJsonTranslator(await buildImageStep.GetFuture().ConfigureAwait(false));
// Gets the image manifest to push.
IBuildableManifestTemplate manifestTemplate =
imageToJsonTranslator.GetManifestTemplate(
buildConfiguration.GetTargetFormat(), containerConfigurationBlobDescriptor);
// Pushes to all target image tags.
IList <Task <DescriptorDigest> > pushAllTagsFutures = new List <Task <DescriptorDigest> >();
var idx = 0;
ProgressEventDispatcher.Factory progressEventDispatcherFactory =
factory.NewChildProducer();
foreach (string tag in targetImageTags)
{
idx++;
using (progressEventDispatcherFactory.Create("tagging with " + tag, idx))
{
buildConfiguration.GetEventHandlers().Dispatch(LogEvent.Info("Tagging with " + tag + "..."));
pushAllTagsFutures.Add(registryClient.PushManifestAsync(manifestTemplate, tag));
}
}
DescriptorDigest imageDigest =
await Digests.ComputeJsonDigestAsync(manifestTemplate).ConfigureAwait(false);
DescriptorDigest imageId = containerConfigurationBlobDescriptor.GetDigest();
BuildResult result = new BuildResult(imageDigest, imageId);
await Task.WhenAll(pushAllTagsFutures).ConfigureAwait(false);
return(result);
}
}
/**
* Gets a {@link BuildResult} from an {@link Image}.
*
* @param image the image
* @param targetFormat the target format of the image
* @return a new {@link BuildResult} with the image's digest and id
* @throws IOException if writing the digest or container configuration fails
*/
public static async Task <BuildResult> FromImageAsync(Image image, ManifestFormat targetFormat)
{
ImageToJsonTranslator imageToJsonTranslator = new ImageToJsonTranslator(image);
ContainerConfigurationTemplate configurationTemplate = imageToJsonTranslator.GetContainerConfiguration();
BlobDescriptor containerConfigurationBlobDescriptor =
await Digests.ComputeJsonDescriptorAsync(configurationTemplate).ConfigureAwait(false);
IBuildableManifestTemplate manifestTemplate =
imageToJsonTranslator.GetManifestTemplate(
targetFormat, containerConfigurationBlobDescriptor);
DescriptorDigest imageDigest =
await Digests.ComputeJsonDigestAsync(manifestTemplate).ConfigureAwait(false);
DescriptorDigest imageId = containerConfigurationBlobDescriptor.GetDigest();
return(new BuildResult(imageDigest, imageId));
}
/**
* Gets the manifest as a JSON template. The {@code containerConfigurationBlobDescriptor} must be
* the {@link BlobDescriptor} obtained by writing out the container configuration JSON returned
* from {@link #getContainerConfiguration()}.
*
* @param <T> child type of {@link BuildableManifestTemplate}.
* @param manifestTemplateClass the JSON template to translate the image to.
* @param containerConfigurationBlobDescriptor the container configuration descriptor.
* @return the image contents serialized as JSON.
*/
public IBuildableManifestTemplate GetManifestTemplate(
ManifestFormat manifestFormat, BlobDescriptor containerConfigurationBlobDescriptor)
{
containerConfigurationBlobDescriptor =
containerConfigurationBlobDescriptor
?? throw new ArgumentNullException(nameof(containerConfigurationBlobDescriptor));
try
{
IBuildableManifestTemplate template;
// ISet up the JSON template.
switch (manifestFormat)
{
case ManifestFormat.V22:
template = new V22ManifestTemplate();
break;
case ManifestFormat.OCI:
template = new OCIManifestTemplate();
break;
default:
throw new ArgumentOutOfRangeException(nameof(manifestFormat));
}
IBuildableManifestTemplate buildableTemplate = template;
// Adds the container configuration reference.
DescriptorDigest containerConfigurationDigest =
containerConfigurationBlobDescriptor.GetDigest();
long containerConfigurationSize = containerConfigurationBlobDescriptor.GetSize();
buildableTemplate.SetContainerConfiguration(containerConfigurationSize, containerConfigurationDigest);
// Adds the layers.
foreach (ILayer layer in image.GetLayers())
{
buildableTemplate.AddLayer(
layer.GetBlobDescriptor().GetSize(), layer.GetBlobDescriptor().GetDigest());
}
// Serializes into JSON.
return(template);
}
catch (JsonException ex)
{
throw new ArgumentException(manifestFormat + " cannot be instantiated", ex);
}
}
public async Task Test_smokeTestAsync()
{
foreach (KeyValuePair <string, string> knownHash in KNOWN_SHA256_HASHES)
{
string toHash = knownHash.Key;
string expectedHash = knownHash.Value;
byte[] bytesToHash = Encoding.UTF8.GetBytes(toHash);
Stream underlyingOutputStream = new MemoryStream();
using (CountingDigestOutputStream countingDigestOutputStream = new CountingDigestOutputStream(underlyingOutputStream))
using (Stream toHashInputStream = new MemoryStream(bytesToHash))
{
await toHashInputStream.CopyToAsync(countingDigestOutputStream).ConfigureAwait(false);
BlobDescriptor blobDescriptor = countingDigestOutputStream.ComputeDigest();
Assert.AreEqual(DescriptorDigest.FromHash(expectedHash), blobDescriptor.GetDigest());
Assert.AreEqual(bytesToHash.Length, blobDescriptor.GetSize());
}
}
}
private async Task VerifyCachedLayerAsync(CachedLayer cachedLayer, IBlob uncompressedLayerBlob)
{
BlobDescriptor layerBlobDescriptor = await GetDigestAsync(Compress(uncompressedLayerBlob)).ConfigureAwait(false);
BlobDescriptor layerDiffDescriptor = await GetDigestAsync(uncompressedLayerBlob).ConfigureAwait(false);
DescriptorDigest layerDiffId = layerDiffDescriptor.GetDigest();
// Verifies cachedLayer is correct.
Assert.AreEqual(layerBlobDescriptor.GetDigest(), cachedLayer.GetDigest());
Assert.AreEqual(layerDiffId, cachedLayer.GetDiffId());
Assert.AreEqual(layerBlobDescriptor.GetSize(), cachedLayer.GetSize());
CollectionAssert.AreEqual(
await Blobs.WriteToByteArrayAsync(uncompressedLayerBlob).ConfigureAwait(false),
await Blobs.WriteToByteArrayAsync(await DecompressAsync(cachedLayer.GetBlob()).ConfigureAwait(false)).ConfigureAwait(false));
// Verifies that the files are present.
Assert.IsTrue(
Files.Exists(
cacheStorageFiles.GetLayerFile(cachedLayer.GetDigest(), cachedLayer.GetDiffId())));
}
public async Task TestWrite_uncompressedAsync()
{
IBlob uncompressedLayerBlob = Blobs.From("uncompressedLayerBlob");
BlobDescriptor layerDigestDescriptor = await GetDigestAsync(Compress(uncompressedLayerBlob)).ConfigureAwait(false);
DescriptorDigest layerDigest = layerDigestDescriptor.GetDigest();
BlobDescriptor selectorDescriptor = await GetDigestAsync(Blobs.From("selector")).ConfigureAwait(false);
DescriptorDigest selector = selectorDescriptor.GetDigest();
CachedLayer cachedLayer =
await new CacheStorageWriter(cacheStorageFiles)
.WriteUncompressedAsync(uncompressedLayerBlob, selector).ConfigureAwait(false);
await VerifyCachedLayerAsync(cachedLayer, uncompressedLayerBlob).ConfigureAwait(false);
// Verifies that the files are present.
SystemPath selectorFile = cacheStorageFiles.GetSelectorFile(selector);
Assert.IsTrue(Files.Exists(selectorFile));
Assert.AreEqual(layerDigest.GetHash(), await Blobs.WriteToStringAsync(Blobs.From(selectorFile)).ConfigureAwait(false));
}
/** Checks that the {@link Blob} streams the expected string. */
private async Task VerifyBlobWriteToAsync(string expected, IBlob blob)
{
using (MemoryStream outputStream = new MemoryStream())
{
BlobDescriptor blobDescriptor = await blob.WriteToAsync(outputStream).ConfigureAwait(false);
string output = Encoding.UTF8.GetString(outputStream.ToArray());
Assert.AreEqual(expected, output);
byte[] expectedBytes = Encoding.UTF8.GetBytes(expected);
Assert.AreEqual(expectedBytes.Length, blobDescriptor.GetSize());
using (MemoryStream stream = new MemoryStream(expectedBytes))
{
BlobDescriptor digestDescriptor = await Digests.ComputeDigestAsync(stream).ConfigureAwait(false);
DescriptorDigest expectedDigest =
digestDescriptor.GetDigest();
Assert.AreEqual(expectedDigest, blobDescriptor.GetDigest());
}
}
}
/**
* Gets the digest of {@code blob}.
*
* @param blob the {@link Blob}
* @return the {@link DescriptorDigest} of {@code blob}
* @throws IOException if an I/O exception occurs
*/
private static async Task <DescriptorDigest> DigestOfAsync(IBlob blob)
{
BlobDescriptor descriptor = await blob.WriteToAsync(Stream.Null).ConfigureAwait(false);
return(descriptor.GetDigest());
}
public DescriptorDigest GetDigest()
{
return(blobDescriptor.GetDigest());
}
public Uri GetApiRoute(string apiRouteBase)
{
return(new Uri(
apiRouteBase + registryEndpointRequestProperties.GetImageName() + "/blobs/" + blobDescriptor.GetDigest()));
}