public async Task TestHandleResponseAsync()
{
using (MemoryStream blobContent = new MemoryStream(Encoding.UTF8.GetBytes("some BLOB content")))
{
BlobDescriptor descriptor = await Digests.ComputeDigestAsync(blobContent).ConfigureAwait(false);
DescriptorDigest testBlobDigest = descriptor.GetDigest();
blobContent.Position = 0;
using (HttpResponseMessage mockResponse = new HttpResponseMessage()
{
Content = new StringContent("some BLOB content")
})
{
LongAdder byteCount = new LongAdder();
BlobPuller blobPuller =
new BlobPuller(
fakeRegistryEndpointRequestProperties,
testBlobDigest,
layerOutputStream,
size => Assert.AreEqual("some BLOB content".Length, size),
byteCount.Add);
await blobPuller.HandleResponseAsync(mockResponse).ConfigureAwait(false);
Assert.AreEqual(
"some BLOB content",
Encoding.UTF8.GetString(layerContentOutputStream.ToArray()));
Assert.AreEqual(testBlobDigest, layerOutputStream.ComputeDigest().GetDigest());
Assert.AreEqual("some BLOB content".Length, byteCount.Sum());
}
}
}
public async Task TestPullAsync()
{
// Pulls the busybox image.
localRegistry.PullAndPushToLocal("busybox", "busybox");
RegistryClient registryClient =
RegistryClient.CreateFactory(EventHandlers.NONE, "localhost:5000", "busybox")
.SetAllowInsecureRegistries(true)
.NewRegistryClient();
V21ManifestTemplate manifestTemplate =
await registryClient.PullManifestAsync <V21ManifestTemplate>("latest").ConfigureAwait(false);
DescriptorDigest realDigest = manifestTemplate.GetLayerDigests().First();
// Pulls a layer BLOB of the busybox image.
LongAdder totalByteCount = new LongAdder();
LongAdder expectedSize = new LongAdder();
IBlob pulledBlob =
registryClient.PullBlob(
realDigest,
size =>
{
Assert.AreEqual(0, expectedSize.Sum());
expectedSize.Add(size);
},
totalByteCount.Add);
BlobDescriptor blobDescriptor = await pulledBlob.WriteToAsync(Stream.Null).ConfigureAwait(false);
Assert.AreEqual(realDigest, blobDescriptor.GetDigest());
Assert.IsTrue(expectedSize.Sum() > 0);
Assert.AreEqual(expectedSize.Sum(), totalByteCount.Sum());
}
public BlobChecker(
RegistryEndpointRequestProperties registryEndpointRequestProperties,
BlobDescriptor blobDigest)
{
this.registryEndpointRequestProperties = registryEndpointRequestProperties;
blobDescriptor = blobDigest;
}
/**
* Computes the hash and returns it along with the size of the bytes written to compute the hash.
* The buffer resets after this method is called, so this method should only be called once per
* computation.
*
* @return the computed hash and the size of the bytes consumed
*/
public BlobDescriptor ComputeDigest()
{
Flush();
try
{
byte[] hashedBytes = MessageDigest.Digest();
// Encodes each hashed byte into 2-character hexadecimal representation.
StringBuilder stringBuilder = new StringBuilder(2 * hashedBytes.Length);
foreach (byte b in hashedBytes)
{
stringBuilder.Append($"{b:x2}");
}
string hash = stringBuilder.ToString();
BlobDescriptor blobDescriptor =
new BlobDescriptor(bytesSoFar, DescriptorDigest.FromHash(hash));
bytesSoFar = 0;
return(blobDescriptor);
}
catch (DigestException ex)
{
throw new Exception("SHA-256 algorithm produced invalid hash: " + ex.Message, ex);
}
}
private BlobDescriptor GetBlobDescriptor(Blob blob)
{
var match = Regex.Match(blob.Name, BlobNameRegex);
if (!match.Success)
{
throw new InvalidOperationException("Unable to match blob name with regex; all blob names");
}
var blobDescriptor = new BlobDescriptor
{
Container = match.Groups["Container"].Value,
ContentMD5 = blob.Md5Hash,
ContentType = blob.ContentType,
ETag = blob.ETag,
LastModified = DateTimeOffset.Parse(blob.UpdatedRaw),
Length = Convert.ToInt64(blob.Size),
Name = match.Groups["Blob"].Value,
Security = blob.Acl != null &&
blob.Acl.Any(acl => acl.Entity.ToLowerInvariant() == "allusers") ? BlobSecurity.Public : BlobSecurity.Private,
Url = blob.MediaLink
};
return(blobDescriptor);
}
/**
* Writes an uncompressed {@code layerBlob} to the {@code layerDirectory}.
*
* @param uncompressedLayerBlob the uncompressed layer {@link Blob}
* @param layerDirectory the directory for the layer
* @return a {@link WrittenLayer} with the written layer information
* @throws IOException if an I/O exception occurs
*/
private async Task <WrittenLayer> WriteUncompressedLayerBlobToDirectoryAsync(
IBlob uncompressedLayerBlob, SystemPath layerDirectory)
{
using (TemporaryFile temporaryLayerFile = CacheStorageFiles.GetTemporaryLayerFile(layerDirectory))
{
DescriptorDigest layerDiffId;
BlobDescriptor blobDescriptor;
// Writes the layer with GZIP compression. The original bytes are captured as the layer's
// diff ID and the bytes outputted from the GZIP compression are captured as the layer's
// content descriptor.
using (CountingDigestOutputStream compressedDigestOutputStream =
new CountingDigestOutputStream(
Files.NewOutputStream(temporaryLayerFile.Path)))
{
using (GZipStream compressorStream = new GZipStream(compressedDigestOutputStream, CompressionMode.Compress, true))
{
BlobDescriptor descriptor = await uncompressedLayerBlob.WriteToAsync(compressorStream).ConfigureAwait(false);
layerDiffId = descriptor.GetDigest();
}
// The GZIPOutputStream must be closed in order to write out the remaining compressed data.
blobDescriptor = compressedDigestOutputStream.ComputeDigest();
}
DescriptorDigest layerDigest = blobDescriptor.GetDigest();
long layerSize = blobDescriptor.GetSize();
// Renames the temporary layer file to the correct filename.
SystemPath layerFile = layerDirectory.Resolve(cacheStorageFiles.GetLayerFilename(layerDiffId));
temporaryLayerFile.MoveIfDoesNotExist(layerFile);
return(new WrittenLayer(layerDigest, layerDiffId, layerSize));
}
}
public async Task TestHandleResponse_unexpectedDigestAsync()
{
using (MemoryStream blobContent = new MemoryStream(Encoding.UTF8.GetBytes("some BLOB content")))
{
BlobDescriptor descriptor = await Digests.ComputeDigestAsync(blobContent).ConfigureAwait(false);
DescriptorDigest testBlobDigest = descriptor.GetDigest();
blobContent.Position = 0;
using (HttpResponseMessage mockResponse = new HttpResponseMessage()
{
Content = new StringContent("some BLOB content")
})
{
try
{
await testBlobPuller.HandleResponseAsync(mockResponse).ConfigureAwait(false);
Assert.Fail("Receiving an unexpected digest should fail");
}
catch (UnexpectedBlobDigestException ex)
{
Assert.AreEqual(
"The pulled BLOB has digest '"
+ testBlobDigest
+ "', but the request digest was '"
+ fakeDigest
+ "'",
ex.Message);
}
}
}
}
public async Task <BuildResult> CallAsync()
{
IReadOnlyList <BlobDescriptor> baseImageDescriptors = await pushBaseImageLayersStep.GetFuture().ConfigureAwait(false);
IReadOnlyList <BlobDescriptor> appLayerDescriptors = await pushApplicationLayersStep.GetFuture().ConfigureAwait(false);
BlobDescriptor containerConfigurationBlobDescriptor = await pushContainerConfigurationStep.GetFuture().ConfigureAwait(false);
ImmutableHashSet <string> targetImageTags = buildConfiguration.GetAllTargetImageTags();
using (var progressEventDispatcher =
progressEventDispatcherFactory.Create("pushing image manifest", this.Index))
using (var factory =
progressEventDispatcher.NewChildProducer()("[child progress]pushing image manifest", targetImageTags.Count))
using (TimerEventDispatcher ignored =
new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), DESCRIPTION))
{
RegistryClient registryClient =
buildConfiguration
.NewTargetImageRegistryClientFactory()
.SetAuthorization(await authenticatePushStep.GetFuture().ConfigureAwait(false))
.NewRegistryClient();
// Constructs the image.
ImageToJsonTranslator imageToJsonTranslator =
new ImageToJsonTranslator(await buildImageStep.GetFuture().ConfigureAwait(false));
// Gets the image manifest to push.
IBuildableManifestTemplate manifestTemplate =
imageToJsonTranslator.GetManifestTemplate(
buildConfiguration.GetTargetFormat(), containerConfigurationBlobDescriptor);
// Pushes to all target image tags.
IList <Task <DescriptorDigest> > pushAllTagsFutures = new List <Task <DescriptorDigest> >();
var idx = 0;
ProgressEventDispatcher.Factory progressEventDispatcherFactory =
factory.NewChildProducer();
foreach (string tag in targetImageTags)
{
idx++;
using (progressEventDispatcherFactory.Create("tagging with " + tag, idx))
{
buildConfiguration.GetEventHandlers().Dispatch(LogEvent.Info("Tagging with " + tag + "..."));
pushAllTagsFutures.Add(registryClient.PushManifestAsync(manifestTemplate, tag));
}
}
DescriptorDigest imageDigest =
await Digests.ComputeJsonDigestAsync(manifestTemplate).ConfigureAwait(false);
DescriptorDigest imageId = containerConfigurationBlobDescriptor.GetDigest();
BuildResult result = new BuildResult(imageDigest, imageId);
await Task.WhenAll(pushAllTagsFutures).ConfigureAwait(false);
return(result);
}
}
private CachedLayer(
DescriptorDigest layerDigest, DescriptorDigest layerDiffId, long layerSize, IBlob layerBlob)
{
blobDescriptor =
new BlobDescriptor(layerSize, layerDigest ?? throw new ArgumentNullException(nameof(layerDigest)));
this.layerDiffId = layerDiffId ?? throw new ArgumentNullException(nameof(layerDiffId));
this.layerBlob = layerBlob ?? throw new ArgumentNullException(nameof(layerBlob));
}
protected async Task AssertBlobDescriptor(BlobDescriptor descriptor, CloudBlockBlob blobRef)
{
Assert.NotNull(descriptor);
await blobRef.FetchAttributesAsync();
Assert.Equal(blobRef.Container.Name, descriptor.Container);
Assert.Equal(blobRef.Properties.ContentMD5, descriptor.ContentMD5);
Assert.Equal(blobRef.Properties.ContentType, descriptor.ContentType);
Assert.NotEmpty(descriptor.ETag);
Assert.NotNull(descriptor.LastModified);
Assert.Equal(blobRef.Properties.Length, descriptor.Length);
Assert.Equal(blobRef.Name, descriptor.Name);
Assert.Equal(BlobSecurity.Public, descriptor.Security);
}
/** Tests translation of image to {@link BuildableManifestTemplate}. */
private async Task TestGetManifestAsync(
ManifestFormat manifestTemplateClass, string translatedJsonFilename)
{
// Loads the expected JSON string.
SystemPath jsonFile = Paths.Get(TestResources.GetResource(translatedJsonFilename).ToURI());
string expectedJson = Encoding.UTF8.GetString(Files.ReadAllBytes(jsonFile));
// Translates the image to the manifest and writes the JSON string.
ContainerConfigurationTemplate containerConfiguration = imageToJsonTranslator.GetContainerConfiguration();
BlobDescriptor blobDescriptor = await Digests.ComputeJsonDescriptorAsync(containerConfiguration).ConfigureAwait(false);
IBuildableManifestTemplate manifestTemplate =
imageToJsonTranslator.GetManifestTemplate(manifestTemplateClass, blobDescriptor);
Assert.AreEqual(expectedJson, JsonTemplateMapper.ToUtf8String(manifestTemplate));
}
/**
* Gets the manifest as a JSON template. The {@code containerConfigurationBlobDescriptor} must be
* the {@link BlobDescriptor} obtained by writing out the container configuration JSON returned
* from {@link #getContainerConfiguration()}.
*
* @param <T> child type of {@link BuildableManifestTemplate}.
* @param manifestTemplateClass the JSON template to translate the image to.
* @param containerConfigurationBlobDescriptor the container configuration descriptor.
* @return the image contents serialized as JSON.
*/
public IBuildableManifestTemplate GetManifestTemplate(
ManifestFormat manifestFormat, BlobDescriptor containerConfigurationBlobDescriptor)
{
containerConfigurationBlobDescriptor =
containerConfigurationBlobDescriptor
?? throw new ArgumentNullException(nameof(containerConfigurationBlobDescriptor));
try
{
IBuildableManifestTemplate template;
// ISet up the JSON template.
switch (manifestFormat)
{
case ManifestFormat.V22:
template = new V22ManifestTemplate();
break;
case ManifestFormat.OCI:
template = new OCIManifestTemplate();
break;
default:
throw new ArgumentOutOfRangeException(nameof(manifestFormat));
}
IBuildableManifestTemplate buildableTemplate = template;
// Adds the container configuration reference.
DescriptorDigest containerConfigurationDigest =
containerConfigurationBlobDescriptor.GetDigest();
long containerConfigurationSize = containerConfigurationBlobDescriptor.GetSize();
buildableTemplate.SetContainerConfiguration(containerConfigurationSize, containerConfigurationDigest);
// Adds the layers.
foreach (ILayer layer in image.GetLayers())
{
buildableTemplate.AddLayer(
layer.GetBlobDescriptor().GetSize(), layer.GetBlobDescriptor().GetDigest());
}
// Serializes into JSON.
return(template);
}
catch (JsonException ex)
{
throw new ArgumentException(manifestFormat + " cannot be instantiated", ex);
}
}
/**
* Gets a {@link BuildResult} from an {@link Image}.
*
* @param image the image
* @param targetFormat the target format of the image
* @return a new {@link BuildResult} with the image's digest and id
* @throws IOException if writing the digest or container configuration fails
*/
public static async Task <BuildResult> FromImageAsync(Image image, ManifestFormat targetFormat)
{
ImageToJsonTranslator imageToJsonTranslator = new ImageToJsonTranslator(image);
ContainerConfigurationTemplate configurationTemplate = imageToJsonTranslator.GetContainerConfiguration();
BlobDescriptor containerConfigurationBlobDescriptor =
await Digests.ComputeJsonDescriptorAsync(configurationTemplate).ConfigureAwait(false);
IBuildableManifestTemplate manifestTemplate =
imageToJsonTranslator.GetManifestTemplate(
targetFormat, containerConfigurationBlobDescriptor);
DescriptorDigest imageDigest =
await Digests.ComputeJsonDigestAsync(manifestTemplate).ConfigureAwait(false);
DescriptorDigest imageId = containerConfigurationBlobDescriptor.GetDigest();
return(new BuildResult(imageDigest, imageId));
}
public async Task Test_smokeTestAsync()
{
foreach (KeyValuePair <string, string> knownHash in KNOWN_SHA256_HASHES)
{
string toHash = knownHash.Key;
string expectedHash = knownHash.Value;
byte[] bytesToHash = Encoding.UTF8.GetBytes(toHash);
Stream underlyingOutputStream = new MemoryStream();
using (CountingDigestOutputStream countingDigestOutputStream = new CountingDigestOutputStream(underlyingOutputStream))
using (Stream toHashInputStream = new MemoryStream(bytesToHash))
{
await toHashInputStream.CopyToAsync(countingDigestOutputStream).ConfigureAwait(false);
BlobDescriptor blobDescriptor = countingDigestOutputStream.ComputeDigest();
Assert.AreEqual(DescriptorDigest.FromHash(expectedHash), blobDescriptor.GetDigest());
Assert.AreEqual(bytesToHash.Length, blobDescriptor.GetSize());
}
}
}
public void SetUpFakes()
{
DescriptorDigest mockDescriptorDigest1 = DescriptorDigest.FromHash(new string('a', 64));
DescriptorDigest mockDescriptorDigest2 = DescriptorDigest.FromHash(new string('b', 64));
DescriptorDigest mockDescriptorDigest3 = DescriptorDigest.FromHash(new string('c', 64));
BlobDescriptor layerBlobDescriptor = new BlobDescriptor(0, mockDescriptorDigest1);
BlobDescriptor referenceLayerBlobDescriptor = new BlobDescriptor(0, mockDescriptorDigest2);
BlobDescriptor referenceNoDiffIdLayerBlobDescriptor =
new BlobDescriptor(0, mockDescriptorDigest3);
// Intentionally the same digest as the mockLayer.
BlobDescriptor anotherBlobDescriptor = new BlobDescriptor(0, mockDescriptorDigest1);
Mock.Get(mockLayer).Setup(m => m.GetBlobDescriptor()).Returns(layerBlobDescriptor);
Mock.Get(mockReferenceLayer).Setup(m => m.GetBlobDescriptor()).Returns(referenceLayerBlobDescriptor);
Mock.Get(mockDigestOnlyLayer).Setup(m => m.GetBlobDescriptor()).Returns(referenceNoDiffIdLayerBlobDescriptor);
Mock.Get(mockLayer2).Setup(m => m.GetBlobDescriptor()).Returns(anotherBlobDescriptor);
}
private async Task VerifyCachedLayerAsync(CachedLayer cachedLayer, IBlob uncompressedLayerBlob)
{
BlobDescriptor layerBlobDescriptor = await GetDigestAsync(Compress(uncompressedLayerBlob)).ConfigureAwait(false);
BlobDescriptor layerDiffDescriptor = await GetDigestAsync(uncompressedLayerBlob).ConfigureAwait(false);
DescriptorDigest layerDiffId = layerDiffDescriptor.GetDigest();
// Verifies cachedLayer is correct.
Assert.AreEqual(layerBlobDescriptor.GetDigest(), cachedLayer.GetDigest());
Assert.AreEqual(layerDiffId, cachedLayer.GetDiffId());
Assert.AreEqual(layerBlobDescriptor.GetSize(), cachedLayer.GetSize());
CollectionAssert.AreEqual(
await Blobs.WriteToByteArrayAsync(uncompressedLayerBlob).ConfigureAwait(false),
await Blobs.WriteToByteArrayAsync(await DecompressAsync(cachedLayer.GetBlob()).ConfigureAwait(false)).ConfigureAwait(false));
// Verifies that the files are present.
Assert.IsTrue(
Files.Exists(
cacheStorageFiles.GetLayerFile(cachedLayer.GetDigest(), cachedLayer.GetDiffId())));
}
public async Task TestWrite_uncompressedAsync()
{
IBlob uncompressedLayerBlob = Blobs.From("uncompressedLayerBlob");
BlobDescriptor layerDigestDescriptor = await GetDigestAsync(Compress(uncompressedLayerBlob)).ConfigureAwait(false);
DescriptorDigest layerDigest = layerDigestDescriptor.GetDigest();
BlobDescriptor selectorDescriptor = await GetDigestAsync(Blobs.From("selector")).ConfigureAwait(false);
DescriptorDigest selector = selectorDescriptor.GetDigest();
CachedLayer cachedLayer =
await new CacheStorageWriter(cacheStorageFiles)
.WriteUncompressedAsync(uncompressedLayerBlob, selector).ConfigureAwait(false);
await VerifyCachedLayerAsync(cachedLayer, uncompressedLayerBlob).ConfigureAwait(false);
// Verifies that the files are present.
SystemPath selectorFile = cacheStorageFiles.GetSelectorFile(selector);
Assert.IsTrue(Files.Exists(selectorFile));
Assert.AreEqual(layerDigest.GetHash(), await Blobs.WriteToStringAsync(Blobs.From(selectorFile)).ConfigureAwait(false));
}
public BlobViewModel(BlobDescriptor blob)
{
Metadata = new ObservableCollection <Property>();
if (blob.CloudBlob.Metadata != null)
{
foreach (string key in blob.CloudBlob.Metadata.AllKeys)
{
Metadata.Add(new Property(key, blob.CloudBlob.Metadata[key]));
}
}
TextSpinnerVisibility = Visibility.Visible;
PreviewTextVisibility = Visibility.Collapsed;
ImageSpinnerVisibility = Visibility.Visible;
PreviewImageVisibility = Visibility.Collapsed;
Blob = blob;
base.DisplayName = blob.CloudBlob.Uri.LocalPath;
}
/** Checks that the {@link Blob} streams the expected string. */
private async Task VerifyBlobWriteToAsync(string expected, IBlob blob)
{
using (MemoryStream outputStream = new MemoryStream())
{
BlobDescriptor blobDescriptor = await blob.WriteToAsync(outputStream).ConfigureAwait(false);
string output = Encoding.UTF8.GetString(outputStream.ToArray());
Assert.AreEqual(expected, output);
byte[] expectedBytes = Encoding.UTF8.GetBytes(expected);
Assert.AreEqual(expectedBytes.Length, blobDescriptor.GetSize());
using (MemoryStream stream = new MemoryStream(expectedBytes))
{
BlobDescriptor digestDescriptor = await Digests.ComputeDigestAsync(stream).ConfigureAwait(false);
DescriptorDigest expectedDigest =
digestDescriptor.GetDigest();
Assert.AreEqual(expectedDigest, blobDescriptor.GetDigest());
}
}
}
/**
* @param blobDigest the blob digest to check for
* @return the BLOB's {@link BlobDescriptor} if the BLOB exists on the registry, or {@code null}
* if it doesn't
* @throws IOException if communicating with the endpoint fails
* @throws RegistryException if communicating with the endpoint fails
*/
public async Task <bool> CheckBlobAsync(BlobDescriptor blobDigest)
{
BlobChecker blobChecker = new BlobChecker(registryEndpointRequestProperties, blobDigest);
return(await CallRegistryEndpointAsync(blobChecker).ConfigureAwait(false));
}
/**
* Gets the digest of {@code blob}.
*
* @param blob the {@link Blob}
* @return the {@link DescriptorDigest} of {@code blob}
* @throws IOException if an I/O exception occurs
*/
private static async Task <DescriptorDigest> DigestOfAsync(IBlob blob)
{
BlobDescriptor descriptor = await blob.WriteToAsync(Stream.Null).ConfigureAwait(false);
return(descriptor.GetDigest());
}
/**
* Instantiate with a {@link BlobDescriptor} and diff ID.
*
* @param blobDescriptor the blob descriptor
* @param diffId the diff ID
*/
public ReferenceLayer(BlobDescriptor blobDescriptor, DescriptorDigest diffId)
{
this.blobDescriptor = blobDescriptor;
this.diffId = diffId;
}
/**
* Instantiate with a {@link BlobDescriptor} and no diff ID.
*
* @param blobDescriptor the blob descriptor
*/
public ReferenceNoDiffIdLayer(BlobDescriptor blobDescriptor)
{
this.blobDescriptor = blobDescriptor;
}
/**
* Instantiate with a {@link DescriptorDigest}.
*
* @param digest the digest to instantiate the {@link DigestOnlyLayer} from
*/
public DigestOnlyLayer(DescriptorDigest digest)
{
blobDescriptor = new BlobDescriptor(digest);
}
public void SaveProperties()
{
CloudBlob blob = Blob.CloudBlob;
foreach (Property prop in Properties)
{
switch (prop.PropertyName)
{
case "(Name)":
if (prop.PropertyValue != BlobDescriptor.BlobName(blob))
{
CloudBlobContainer container = blob.Container;
if (container != null)
{
CloudBlob newBlob = container.GetBlobReference(prop.PropertyValue);
newBlob.UploadText(String.Empty);
newBlob.CopyFromBlob(blob);
blob.Delete();
blob = newBlob;
}
}
break;
case "Blob Type":
// Can't be set - blob.Properties.BlobType = prop.PropertyValue;
break;
case "CacheControl":
blob.Properties.CacheControl = prop.PropertyValue;
break;
case "ContentEncoding":
blob.Properties.ContentEncoding = prop.PropertyValue;
break;
case "ContentLanguage":
blob.Properties.ContentLanguage = prop.PropertyValue;
break;
case "ContentMD5":
blob.Properties.ContentMD5 = prop.PropertyValue;
break;
case "ContentType":
blob.Properties.ContentType = prop.PropertyValue;
break;
case "ETag":
//Can't be set - blob.Properties.ETag = prop.PropertyValue;
break;
case "LastModifiedUtc":
//Can't be set
break;
case "Length":
break;
}
}
blob.SetProperties();
}
