//private static DescriptorDigest fakeDigest = DescriptorDigest.fromHash(new string('a', 64));
private void SetUp(ManifestFormat imageFormat)
{
Image.Builder testImageBuilder =
Image.CreateBuilder(imageFormat)
.SetCreated(Instant.FromUnixTimeSeconds(20))
.SetArchitecture("wasm")
.SetOs("js")
.AddEnvironmentVariable("VAR1", "VAL1")
.AddEnvironmentVariable("VAR2", "VAL2")
.SetEntrypoint(new[] { "some", "entrypoint", "command" })
.SetProgramArguments(new[] { "arg1", "arg2" })
.SetHealthCheck(
DockerHealthCheck.FromCommand(ImmutableArray.Create("CMD-SHELL", "/checkhealth"))
.SetInterval(Duration.FromSeconds(3))
.SetTimeout(Duration.FromSeconds(1))
.SetStartPeriod(Duration.FromSeconds(2))
.SetRetries(3)
.Build())
.AddExposedPorts(ImmutableHashSet.Create(Port.Tcp(1000), Port.Tcp(2000), Port.Udp(3000)))
.AddVolumes(
ImmutableHashSet.Create(
AbsoluteUnixPath.Get("/var/job-result-data"),
AbsoluteUnixPath.Get("/var/log/my-app-logs")))
.AddLabels(ImmutableDic.Of("key1", "value1", "key2", "value2"))
.SetWorkingDirectory("/some/workspace")
.SetUser("tomcat");
DescriptorDigest fakeDigest =
DescriptorDigest.FromDigest(
"sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad");
testImageBuilder.AddLayer(
new FakeLayer(fakeDigest));
testImageBuilder.AddHistory(
HistoryEntry.CreateBuilder()
.SetCreationTimestamp(Instant.FromUnixTimeSeconds(0))
.SetAuthor("Bazel")
.SetCreatedBy("bazel build ...")
.SetEmptyLayer(true)
.Build());
testImageBuilder.AddHistory(
HistoryEntry.CreateBuilder()
.SetCreationTimestamp(Instant.FromUnixTimeSeconds(20))
.SetAuthor("Fib")
.SetCreatedBy("fib")
.Build());
imageToJsonTranslator = new ImageToJsonTranslator(testImageBuilder.Build());
}
public async Task TestHandleResponse_invalidDigestAsync()
{
DescriptorDigest expectedDigest = await Digests.ComputeJsonDigestAsync(fakeManifestTemplate).ConfigureAwait(false);
using (HttpResponseMessage mockResponse = new HttpResponseMessage
{
Headers = { { "Docker-Content-Digest", new List <string> {
"not valid"
} } }
})
{
Assert.AreEqual(expectedDigest, await testManifestPusher.HandleResponseAsync(mockResponse).ConfigureAwait(false));
Mock.Get(mockEventHandlers).Verify(m => m.Dispatch(
LogEvent.Warn("Expected image digest " + expectedDigest + ", but received: not valid")));
}
}
public async Task TestPushAsync()
{
localRegistry.PullAndPushToLocal("busybox", "busybox");
IBlob testBlob = Blobs.From("crepecake");
// Known digest for 'crepecake'
DescriptorDigest testBlobDigest =
DescriptorDigest.FromHash(
"52a9e4d4ba4333ce593707f98564fee1e6d898db0d3602408c0b2a6a424d357c");
RegistryClient registryClient =
RegistryClient.CreateFactory(EventHandlers.NONE, "localhost:5000", "testimage")
.SetAllowInsecureRegistries(true)
.NewRegistryClient();
Assert.IsFalse(await registryClient.PushBlobAsync(testBlobDigest, testBlob, null, _ => { }).ConfigureAwait(false));
}
public void TestGetDiffId()
{
Assert.AreEqual(
DescriptorDigest.FromHash(
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"),
TEST_CACHE_STORAGE_FILES.GetDiffId(
Paths.Get(
"layer",
"file",
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")));
Assert.AreEqual(
DescriptorDigest.FromHash(
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
TEST_CACHE_STORAGE_FILES.GetDiffId(
Paths.Get("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")));
}
/**
* Gets the manifest as a JSON template. The {@code containerConfigurationBlobDescriptor} must be
* the {@link BlobDescriptor} obtained by writing out the container configuration JSON returned
* from {@link #getContainerConfiguration()}.
*
* @param <T> child type of {@link BuildableManifestTemplate}.
* @param manifestTemplateClass the JSON template to translate the image to.
* @param containerConfigurationBlobDescriptor the container configuration descriptor.
* @return the image contents serialized as JSON.
*/
public IBuildableManifestTemplate GetManifestTemplate(
ManifestFormat manifestFormat, BlobDescriptor containerConfigurationBlobDescriptor)
{
containerConfigurationBlobDescriptor =
containerConfigurationBlobDescriptor
?? throw new ArgumentNullException(nameof(containerConfigurationBlobDescriptor));
try
{
IBuildableManifestTemplate template;
// ISet up the JSON template.
switch (manifestFormat)
{
case ManifestFormat.V22:
template = new V22ManifestTemplate();
break;
case ManifestFormat.OCI:
template = new OCIManifestTemplate();
break;
default:
throw new ArgumentOutOfRangeException(nameof(manifestFormat));
}
IBuildableManifestTemplate buildableTemplate = template;
// Adds the container configuration reference.
DescriptorDigest containerConfigurationDigest =
containerConfigurationBlobDescriptor.GetDigest();
long containerConfigurationSize = containerConfigurationBlobDescriptor.GetSize();
buildableTemplate.SetContainerConfiguration(containerConfigurationSize, containerConfigurationDigest);
// Adds the layers.
foreach (ILayer layer in image.GetLayers())
{
buildableTemplate.AddLayer(
layer.GetBlobDescriptor().GetSize(), layer.GetBlobDescriptor().GetDigest());
}
// Serializes into JSON.
return(template);
}
catch (JsonException ex)
{
throw new ArgumentException(manifestFormat + " cannot be instantiated", ex);
}
}
public void SetUpFakes()
{
layerContentOutputStream = new MemoryStream();
layerOutputStream = new CountingDigestOutputStream(layerContentOutputStream);
fakeDigest =
DescriptorDigest.FromHash(
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
testBlobPuller =
new BlobPuller(
fakeRegistryEndpointRequestProperties,
fakeDigest,
layerOutputStream,
_ => { },
_ => { });
}
/**
* Gets a {@link BuildResult} from an {@link Image}.
*
* @param image the image
* @param targetFormat the target format of the image
* @return a new {@link BuildResult} with the image's digest and id
* @throws IOException if writing the digest or container configuration fails
*/
public static async Task <BuildResult> FromImageAsync(Image image, ManifestFormat targetFormat)
{
ImageToJsonTranslator imageToJsonTranslator = new ImageToJsonTranslator(image);
ContainerConfigurationTemplate configurationTemplate = imageToJsonTranslator.GetContainerConfiguration();
BlobDescriptor containerConfigurationBlobDescriptor =
await Digests.ComputeJsonDescriptorAsync(configurationTemplate).ConfigureAwait(false);
IBuildableManifestTemplate manifestTemplate =
imageToJsonTranslator.GetManifestTemplate(
targetFormat, containerConfigurationBlobDescriptor);
DescriptorDigest imageDigest =
await Digests.ComputeJsonDigestAsync(manifestTemplate).ConfigureAwait(false);
DescriptorDigest imageId = containerConfigurationBlobDescriptor.GetDigest();
return(new BuildResult(imageDigest, imageId));
}
public async Task TestRetrieveAsync()
{
CacheStorageFiles cacheStorageFiles =
new CacheStorageFiles(temporaryFolder.NewFolder().ToPath());
CacheStorageReader cacheStorageReader = new CacheStorageReader(cacheStorageFiles);
// Creates the test layer directory.
DescriptorDigest layerDigest = layerDigest1;
DescriptorDigest layerDiffId = layerDigest2;
Files.CreateDirectories(cacheStorageFiles.GetLayerDirectory(layerDigest));
using (Stream @out =
Files.NewOutputStream(cacheStorageFiles.GetLayerFile(layerDigest, layerDiffId)))
{
byte[] bytes = Encoding.UTF8.GetBytes("layerBlob");
await @out.WriteAsync(bytes, 0, bytes.Length).ConfigureAwait(false);
}
// Checks that the CachedLayer is retrieved correctly.
Maybe <CachedLayer> optionalCachedLayer = cacheStorageReader.Retrieve(layerDigest);
Assert.IsTrue(optionalCachedLayer.IsPresent());
Assert.AreEqual(layerDigest, optionalCachedLayer.Get().GetDigest());
Assert.AreEqual(layerDiffId, optionalCachedLayer.Get().GetDiffId());
Assert.AreEqual("layerBlob".Length, optionalCachedLayer.Get().GetSize());
Assert.AreEqual("layerBlob", await Blobs.WriteToStringAsync(optionalCachedLayer.Get().GetBlob()).ConfigureAwait(false));
// Checks that multiple .layer files means the cache is corrupted.
Files.CreateFile(cacheStorageFiles.GetLayerFile(layerDigest, layerDigest));
try
{
cacheStorageReader.Retrieve(layerDigest);
Assert.Fail("Should have thrown CacheCorruptedException");
}
catch (CacheCorruptedException ex)
{
Assert.That(
ex.Message, Does.StartWith(
"Multiple layer files found for layer with digest "
+ layerDigest.GetHash()
+ " in directory: "
+ cacheStorageFiles.GetLayerDirectory(layerDigest)));
}
}
public void TestGetLayerFile()
{
DescriptorDigest layerDigest =
DescriptorDigest.FromHash(
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
DescriptorDigest diffId =
DescriptorDigest.FromHash(
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb");
Assert.AreEqual(
Paths.Get(
"cache",
"directory",
"layers",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"),
TEST_CACHE_STORAGE_FILES.GetLayerFile(layerDigest, diffId));
}
public void TestSelect()
{
CacheStorageFiles cacheStorageFiles =
new CacheStorageFiles(temporaryFolder.NewFolder().ToPath());
CacheStorageReader cacheStorageReader = new CacheStorageReader(cacheStorageFiles);
DescriptorDigest selector = layerDigest1;
SystemPath selectorFile = cacheStorageFiles.GetSelectorFile(selector);
Files.CreateDirectories(selectorFile.GetParent());
Files.Write(selectorFile, Encoding.UTF8.GetBytes(layerDigest2.GetHash()));
Maybe <DescriptorDigest> selectedLayerDigest = cacheStorageReader.Select(selector);
Assert.IsTrue(selectedLayerDigest.IsPresent());
Assert.AreEqual(layerDigest2, selectedLayerDigest.Get());
}
public async Task Test_smokeTestAsync()
{
foreach (KeyValuePair <string, string> knownHash in KNOWN_SHA256_HASHES)
{
string toHash = knownHash.Key;
string expectedHash = knownHash.Value;
byte[] bytesToHash = Encoding.UTF8.GetBytes(toHash);
Stream underlyingOutputStream = new MemoryStream();
using (CountingDigestOutputStream countingDigestOutputStream = new CountingDigestOutputStream(underlyingOutputStream))
using (Stream toHashInputStream = new MemoryStream(bytesToHash))
{
await toHashInputStream.CopyToAsync(countingDigestOutputStream).ConfigureAwait(false);
BlobDescriptor blobDescriptor = countingDigestOutputStream.ComputeDigest();
Assert.AreEqual(DescriptorDigest.FromHash(expectedHash), blobDescriptor.GetDigest());
Assert.AreEqual(bytesToHash.Length, blobDescriptor.GetSize());
}
}
}
/**
* Translates {@link BuildableManifestTemplate} to {@link Image}. Uses the corresponding {@link
* ContainerConfigurationTemplate} to get the layer diff IDs.
*
* @param manifestTemplate the template containing the image layers.
* @param containerConfigurationTemplate the template containing the diff IDs and container
* configuration properties.
* @return the translated {@link Image}.
* @throws LayerCountMismatchException if the manifest and configuration contain conflicting layer
* information.
* @throws LayerPropertyNotFoundException if adding image layers fails.
* @throws BadContainerConfigurationFormatException if the container configuration is in a bad
* format
*/
public static Image ToImage <T>(
T manifestTemplate,
ContainerConfigurationTemplate containerConfigurationTemplate) where T : IBuildableManifestTemplate
{
containerConfigurationTemplate =
containerConfigurationTemplate
?? throw new ArgumentNullException(nameof(containerConfigurationTemplate));
IList <ReferenceNoDiffIdLayer> layers = new List <ReferenceNoDiffIdLayer>();
foreach (ContentDescriptorTemplate layerObjectTemplate in
manifestTemplate.Layers)
{
if (layerObjectTemplate.Digest == null)
{
throw new ArgumentException(Resources.JsonToImageTranslatorMissingDigestExceptionMessage);
}
layers.Add(new ReferenceNoDiffIdLayer(
new BlobDescriptor(layerObjectTemplate.Size, layerObjectTemplate.Digest)));
}
IList <DescriptorDigest> diffIds = containerConfigurationTemplate.GetDiffIds();
if (layers.Count != diffIds.Count)
{
throw new LayerCountMismatchException(Resources.JsonToImageTranslatorDiffIdMismatchExceptionMessage);
}
Image.Builder imageBuilder = Image.CreateBuilder(manifestTemplate.GetFormat());
for (int layerIndex = 0; layerIndex < layers.Count; layerIndex++)
{
ReferenceNoDiffIdLayer noDiffIdLayer = layers[layerIndex];
DescriptorDigest diffId = diffIds[layerIndex];
imageBuilder.AddLayer(new ReferenceLayer(noDiffIdLayer.GetBlobDescriptor(), diffId));
}
ConfigureBuilderWithContainerConfiguration(imageBuilder, containerConfigurationTemplate);
return(imageBuilder.Build());
}
public async Task TestGenerateSelector_fileModifiedAsync()
{
SystemPath layerFile = temporaryFolder.NewFolder("testFolder").ToPath().Resolve("file");
Files.Write(layerFile, Encoding.UTF8.GetBytes("hello"));
Files.SetLastModifiedTime(layerFile, FileTime.From(Instant.FromUnixTimeSeconds(0)));
LayerEntry layerEntry = DefaultLayerEntry(layerFile, AbsoluteUnixPath.Get("/extraction/path"));
DescriptorDigest expectedSelector =
await GenerateSelectorAsync(ImmutableArray.Create(layerEntry)).ConfigureAwait(false);
// Verify that changing modified time generates a different selector
Files.SetLastModifiedTime(layerFile, FileTime.From(Instant.FromUnixTimeSeconds(1)));
Assert.AreNotEqual(
expectedSelector, await GenerateSelectorAsync(ImmutableArray.Create(layerEntry)).ConfigureAwait(false));
// Verify that changing modified time back generates same selector
Files.SetLastModifiedTime(layerFile, FileTime.From(Instant.FromUnixTimeSeconds(0)));
Assert.AreEqual(
expectedSelector,
await GenerateSelectorAsync(ImmutableArray.Create(layerEntry)).ConfigureAwait(false));
}
public void SetUpFakes()
{
DescriptorDigest mockDescriptorDigest1 = DescriptorDigest.FromHash(new string('a', 64));
DescriptorDigest mockDescriptorDigest2 = DescriptorDigest.FromHash(new string('b', 64));
DescriptorDigest mockDescriptorDigest3 = DescriptorDigest.FromHash(new string('c', 64));
BlobDescriptor layerBlobDescriptor = new BlobDescriptor(0, mockDescriptorDigest1);
BlobDescriptor referenceLayerBlobDescriptor = new BlobDescriptor(0, mockDescriptorDigest2);
BlobDescriptor referenceNoDiffIdLayerBlobDescriptor =
new BlobDescriptor(0, mockDescriptorDigest3);
// Intentionally the same digest as the mockLayer.
BlobDescriptor anotherBlobDescriptor = new BlobDescriptor(0, mockDescriptorDigest1);
Mock.Get(mockLayer).Setup(m => m.GetBlobDescriptor()).Returns(layerBlobDescriptor);
Mock.Get(mockReferenceLayer).Setup(m => m.GetBlobDescriptor()).Returns(referenceLayerBlobDescriptor);
Mock.Get(mockDigestOnlyLayer).Setup(m => m.GetBlobDescriptor()).Returns(referenceNoDiffIdLayerBlobDescriptor);
Mock.Get(mockLayer2).Setup(m => m.GetBlobDescriptor()).Returns(anotherBlobDescriptor);
}
private async Task VerifyCachedLayerAsync(CachedLayer cachedLayer, IBlob uncompressedLayerBlob)
{
BlobDescriptor layerBlobDescriptor = await GetDigestAsync(Compress(uncompressedLayerBlob)).ConfigureAwait(false);
BlobDescriptor layerDiffDescriptor = await GetDigestAsync(uncompressedLayerBlob).ConfigureAwait(false);
DescriptorDigest layerDiffId = layerDiffDescriptor.GetDigest();
// Verifies cachedLayer is correct.
Assert.AreEqual(layerBlobDescriptor.GetDigest(), cachedLayer.GetDigest());
Assert.AreEqual(layerDiffId, cachedLayer.GetDiffId());
Assert.AreEqual(layerBlobDescriptor.GetSize(), cachedLayer.GetSize());
CollectionAssert.AreEqual(
await Blobs.WriteToByteArrayAsync(uncompressedLayerBlob).ConfigureAwait(false),
await Blobs.WriteToByteArrayAsync(await DecompressAsync(cachedLayer.GetBlob()).ConfigureAwait(false)).ConfigureAwait(false));
// Verifies that the files are present.
Assert.IsTrue(
Files.Exists(
cacheStorageFiles.GetLayerFile(cachedLayer.GetDigest(), cachedLayer.GetDiffId())));
}
/** Checks that the {@link Blob} streams the expected string. */
private async Task VerifyBlobWriteToAsync(string expected, IBlob blob)
{
using (MemoryStream outputStream = new MemoryStream())
{
BlobDescriptor blobDescriptor = await blob.WriteToAsync(outputStream).ConfigureAwait(false);
string output = Encoding.UTF8.GetString(outputStream.ToArray());
Assert.AreEqual(expected, output);
byte[] expectedBytes = Encoding.UTF8.GetBytes(expected);
Assert.AreEqual(expectedBytes.Length, blobDescriptor.GetSize());
using (MemoryStream stream = new MemoryStream(expectedBytes))
{
BlobDescriptor digestDescriptor = await Digests.ComputeDigestAsync(stream).ConfigureAwait(false);
DescriptorDigest expectedDigest =
digestDescriptor.GetDigest();
Assert.AreEqual(expectedDigest, blobDescriptor.GetDigest());
}
}
}
public async Task TestWrite_uncompressedAsync()
{
IBlob uncompressedLayerBlob = Blobs.From("uncompressedLayerBlob");
BlobDescriptor layerDigestDescriptor = await GetDigestAsync(Compress(uncompressedLayerBlob)).ConfigureAwait(false);
DescriptorDigest layerDigest = layerDigestDescriptor.GetDigest();
BlobDescriptor selectorDescriptor = await GetDigestAsync(Blobs.From("selector")).ConfigureAwait(false);
DescriptorDigest selector = selectorDescriptor.GetDigest();
CachedLayer cachedLayer =
await new CacheStorageWriter(cacheStorageFiles)
.WriteUncompressedAsync(uncompressedLayerBlob, selector).ConfigureAwait(false);
await VerifyCachedLayerAsync(cachedLayer, uncompressedLayerBlob).ConfigureAwait(false);
// Verifies that the files are present.
SystemPath selectorFile = cacheStorageFiles.GetSelectorFile(selector);
Assert.IsTrue(Files.Exists(selectorFile));
Assert.AreEqual(layerDigest.GetHash(), await Blobs.WriteToStringAsync(Blobs.From(selectorFile)).ConfigureAwait(false));
}
public async Task TestExecuteAsync_OutputsResult()
{
_objectUnderTest.ConfigFile =
Path.Combine(TestContext.CurrentContext.TestDirectory, "Resources", "trivial-config.json");
Mock.Get(_objectUnderTest.Containerizer)
.Setup(c => c.CreateStepsRunner(It.IsAny <BuildConfiguration>()).RunAsync())
.Returns(
Task.FromResult(
Mock.Of <IBuildResult>(
r => r.GetImageDigest()
== DescriptorDigest.FromHash(new string('a', DescriptorDigest.HashLength)) &&
r.GetImageId()
== DescriptorDigest.FromHash(new string('b', DescriptorDigest.HashLength)))));
await _objectUnderTest.ExecuteAsync(_output, _error).ConfigureAwait(false);
Assert.That(
_output.ToString(),
Contains.Substring($"ImageDigest:sha256:{new string('a', DescriptorDigest.HashLength)}"));
Assert.That(
_output.ToString(),
Contains.Substring($"ImageId:sha256:{new string('b', DescriptorDigest.HashLength)}"));
}
public void TestToJson()
{
// Loads the expected JSON string.
SystemPath jsonFile = Paths.Get(TestResources.GetResource("core/json/ocimanifest.json").ToURI());
string expectedJson = Encoding.UTF8.GetString(Files.ReadAllBytes(jsonFile));
// Creates the JSON object to serialize.
OCIManifestTemplate manifestJson = new OCIManifestTemplate();
manifestJson.SetContainerConfiguration(
1000,
DescriptorDigest.FromDigest(
"sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad"));
manifestJson.AddLayer(
1000_000,
DescriptorDigest.FromHash(
"4945ba5011739b0b98c4a41afe224e417f47c7c99b2ce76830999c9a0861b236"));
// Serializes the JSON object.
Assert.AreEqual(expectedJson, JsonTemplateMapper.ToUtf8String(manifestJson));
}
/**
* Pushes the BLOB. If the {@code sourceRepository} is provided then the remote registry may skip
* if the BLOB already exists on the registry.
*
* @param blobDigest the digest of the BLOB, used for existence-check
* @param blob the BLOB to push
* @param sourceRepository if pushing to the same registry then the source image, or {@code null}
* otherwise; used to optimize the BLOB push
* @param writtenByteCountListener listens on byte count written to the registry during the push
* @return {@code true} if the BLOB already exists on the registry and pushing was skipped; false
* if the BLOB was pushed
* @throws IOException if communicating with the endpoint fails
* @throws RegistryException if communicating with the endpoint fails
*/
public async Task <bool> PushBlobAsync(
DescriptorDigest blobDigest,
IBlob blob,
string sourceRepository,
Action <long> writtenByteCountListener)
{
BlobPusher blobPusher =
new BlobPusher(registryEndpointRequestProperties, blobDigest, blob, sourceRepository);
using (TimerEventDispatcher timerEventDispatcher =
new TimerEventDispatcher(eventHandlers, "pushBlob"))
{
timerEventDispatcher.Lap("pushBlob POST " + blobDigest);
// POST /v2/<name>/blobs/uploads/ OR
// POST /v2/<name>/blobs/uploads/?mount={blob.digest}&from={sourceRepository}
Uri patchLocation = await CallRegistryEndpointAsync(blobPusher.CreateInitializer()).ConfigureAwait(false);
if (patchLocation == null)
{
// The BLOB exists already.
return(true);
}
timerEventDispatcher.Lap("pushBlob PATCH " + blobDigest);
// PATCH <Location> with BLOB
Uri putLocation =
await CallRegistryEndpointAsync(blobPusher.CreateWriter(patchLocation, writtenByteCountListener)).ConfigureAwait(false);
Preconditions.CheckNotNull(putLocation);
timerEventDispatcher.Lap("pushBlob PUT " + blobDigest);
// PUT <Location>?digest={blob.digest}
await CallRegistryEndpointAsync(blobPusher.CreateCommitter(putLocation)).ConfigureAwait(false);
return(false);
}
}
public void TestFromJson()
{
// Loads the JSON string.
SystemPath jsonFile = Paths.Get(TestResources.GetResource("core/json/ocimanifest.json").ToURI());
// Deserializes into a manifest JSON object.
OCIManifestTemplate manifestJson =
JsonTemplateMapper.ReadJsonFromFile <OCIManifestTemplate>(jsonFile);
Assert.AreEqual(
DescriptorDigest.FromDigest(
"sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad"),
manifestJson.GetContainerConfiguration().Digest);
Assert.AreEqual(1000, manifestJson.GetContainerConfiguration().Size);
Assert.AreEqual(
DescriptorDigest.FromHash(
"4945ba5011739b0b98c4a41afe224e417f47c7c99b2ce76830999c9a0861b236"),
manifestJson.Layers[0].Digest);
Assert.AreEqual(1000_000, manifestJson.Layers[0].Size);
}
public void TestFromJson()
{
// Loads the JSON string.
SystemPath jsonFile = Paths.Get(TestResources.GetResource("core/json/v21manifest.json").ToURI());
// Deserializes into a manifest JSON object.
V21ManifestTemplate manifestJson =
JsonTemplateMapper.ReadJsonFromFile <V21ManifestTemplate>(jsonFile);
Assert.AreEqual(
DescriptorDigest.FromDigest(
"sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad"),
manifestJson.FsLayers[0].GetDigest());
ContainerConfigurationTemplate containerConfiguration =
manifestJson.GetContainerConfiguration().OrElse(null);
Assert.AreEqual(
new[] { "JAVA_HOME=/opt/openjdk", "PATH=/opt/openjdk/bin" },
containerConfiguration.GetContainerEnvironment());
Assert.AreEqual(
new[] { "/opt/openjdk/bin/java" }, containerConfiguration.GetContainerEntrypoint());
}
/**
* Lists all the layer digests stored.
*
* @return the list of layer digests
* @throws CacheCorruptedException if the cache was found to be corrupted
* @throws IOException if an I/O exception occurs
*/
public ISet <DescriptorDigest> FetchDigests()
{
IEnumerable <SystemPath> layerDirectories = Files.List(cacheStorageFiles.GetLayersDirectory());
IList <SystemPath> layerDirectoriesList = layerDirectories.ToList();
ISet <DescriptorDigest> layerDigests = new HashSet <DescriptorDigest>();
foreach (SystemPath layerDirectory in layerDirectoriesList)
{
try
{
layerDigests.Add(DescriptorDigest.FromHash(layerDirectory.GetFileName().ToString()));
}
catch (DigestException ex)
{
throw new CacheCorruptedException(
cacheStorageFiles.GetCacheDirectory(),
Resources.CacheStorageReaderNonDigestFileExceptionMessage,
ex);
}
}
return(layerDigests);
}
/**
* Gets the BLOB referenced by {@code blobDigest}. Note that the BLOB is only pulled when it is
* written out.
*
* @param blobDigest the digest of the BLOB to download
* @param blobSizeListener callback to receive the total size of the BLOb to pull
* @param writtenByteCountListener listens on byte count written to an output stream during the
* pull
* @return a {@link Blob}
*/
public IBlob PullBlob(
DescriptorDigest blobDigest,
Action <long> blobSizeListener,
Action <long> writtenByteCountListener)
{
return(Blobs.From(
async outputStream =>
{
try
{
await CallRegistryEndpointAsync(new BlobPuller(
registryEndpointRequestProperties,
blobDigest,
outputStream,
blobSizeListener,
writtenByteCountListener)).ConfigureAwait(false);
}
catch (RegistryException ex)
{
throw new IOException("", ex);
}
}, -1));
}
/**
* Writes a compressed {@code layerBlob} to the {@code layerDirectory}.
*
* @param compressedLayerBlob the compressed layer {@link Blob}
* @param layerDirectory the directory for the layer
* @return a {@link WrittenLayer} with the written layer information
* @throws IOException if an I/O exception occurs
*/
private async Task <WrittenLayer> WriteCompressedLayerBlobToDirectoryAsync(
IBlob compressedLayerBlob, SystemPath layerDirectory)
{
// Writes the layer file to the temporary directory.
using (TemporaryFile temporaryLayerFile = CacheStorageFiles.GetTemporaryLayerFile(layerDirectory))
{
BlobDescriptor layerBlobDescriptor;
using (Stream fileOutputStream = Files.NewOutputStream(temporaryLayerFile.Path))
{
layerBlobDescriptor = await compressedLayerBlob.WriteToAsync(fileOutputStream).ConfigureAwait(false);
}
// Gets the diff ID.
DescriptorDigest layerDiffId = await GetDiffIdByDecompressingFileAsync(temporaryLayerFile.Path).ConfigureAwait(false);
// Renames the temporary layer file to the correct filename.
SystemPath layerFile = layerDirectory.Resolve(cacheStorageFiles.GetLayerFilename(layerDiffId));
temporaryLayerFile.MoveIfDoesNotExist(layerFile);
return(new WrittenLayer(
layerBlobDescriptor.GetDigest(), layerDiffId, layerBlobDescriptor.GetSize()));
}
}
public void TestToImage_v21()
{
// Loads the JSON string.
SystemPath jsonFile =
Paths.Get(TestResources.GetResource("core/json/v21manifest.json").ToURI());
// Deserializes into a manifest JSON object.
V21ManifestTemplate manifestTemplate =
JsonTemplateMapper.ReadJsonFromFile <V21ManifestTemplate>(jsonFile);
Image image = JsonToImageTranslator.ToImage(manifestTemplate);
IList <ILayer> layers = image.GetLayers();
Assert.AreEqual(2, layers.Count);
Assert.AreEqual(
DescriptorDigest.FromDigest(
"sha256:5bd451067f9ab05e97cda8476c82f86d9b69c2dffb60a8ad2fe3723942544ab3"),
layers[0].GetBlobDescriptor().GetDigest());
Assert.AreEqual(
DescriptorDigest.FromDigest(
"sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad"),
layers[1].GetBlobDescriptor().GetDigest());
}
public void TestUseAsMapKey()
{
DescriptorDigest descriptorDigestA1 = DescriptorDigest.FromHash(CreateGoodHash('a'));
DescriptorDigest descriptorDigestA2 = DescriptorDigest.FromHash(CreateGoodHash('a'));
DescriptorDigest descriptorDigestA3 =
DescriptorDigest.FromDigest("sha256:" + CreateGoodHash('a'));
DescriptorDigest descriptorDigestB = DescriptorDigest.FromHash(CreateGoodHash('b'));
IDictionary <DescriptorDigest, string> digestMap = new Dictionary <DescriptorDigest, string>
{
[descriptorDigestA1] = "digest with a"
};
Assert.AreEqual("digest with a", digestMap[descriptorDigestA1]);
Assert.AreEqual("digest with a", digestMap[descriptorDigestA2]);
Assert.AreEqual("digest with a", digestMap[descriptorDigestA3]);
Assert.IsFalse(digestMap.ContainsKey(descriptorDigestB));
digestMap[descriptorDigestA2] = "digest with a";
Assert.AreEqual("digest with a", digestMap[descriptorDigestA1]);
Assert.AreEqual("digest with a", digestMap[descriptorDigestA2]);
Assert.AreEqual("digest with a", digestMap[descriptorDigestA3]);
Assert.IsFalse(digestMap.ContainsKey(descriptorDigestB));
digestMap[descriptorDigestA3] = "digest with a";
Assert.AreEqual("digest with a", digestMap[descriptorDigestA1]);
Assert.AreEqual("digest with a", digestMap[descriptorDigestA2]);
Assert.AreEqual("digest with a", digestMap[descriptorDigestA3]);
Assert.IsFalse(digestMap.ContainsKey(descriptorDigestB));
digestMap[descriptorDigestB] = "digest with b";
Assert.AreEqual("digest with a", digestMap[descriptorDigestA1]);
Assert.AreEqual("digest with a", digestMap[descriptorDigestA2]);
Assert.AreEqual("digest with a", digestMap[descriptorDigestA3]);
Assert.AreEqual("digest with b", digestMap[descriptorDigestB]);
}
/**
* Writes the {@code selector} to a file in the selectors directory, with contents {@code
* layerDigest}.
*
* @param selector the selector
* @param layerDigest the layer digest it selects
* @throws IOException if an I/O exception occurs
*/
private void WriteSelector(DescriptorDigest selector, DescriptorDigest layerDigest)
{
SystemPath selectorFile = cacheStorageFiles.GetSelectorFile(selector);
// Creates the selectors directory if it doesn't exist.
Files.CreateDirectories(selectorFile.GetParent());
// Writes the selector to a temporary file and then moves the file to the intended location.
using (TemporaryFile temporarySelectorFile = Files.CreateTempFile())
{
using (Stream fileOut = FileOperations.NewLockingOutputStream(temporarySelectorFile.Path))
{
byte[] bytes = Encoding.UTF8.GetBytes(layerDigest.GetHash());
fileOut.Write(bytes, 0, bytes.Length);
}
// Attempts an atomic move first, and falls back to non-atomic if the file system does not
// support atomic moves.
Files.Move(
temporarySelectorFile.Path,
selectorFile,
StandardCopyOption.REPLACE_EXISTING);
}
}
/**
* Pulls the base image.
*
* @param registryAuthorization authentication credentials to possibly use
* @param progressEventDispatcher the {@link ProgressEventDispatcher} for emitting {@link
* ProgressEvent}s
* @return the pulled image
* @throws IOException when an I/O exception occurs during the pulling
* @throws RegistryException if communicating with the registry caused a known error
* @throws LayerCountMismatchException if the manifest and configuration contain conflicting layer
* information
* @throws LayerPropertyNotFoundException if adding image layers fails
* @throws BadContainerConfigurationFormatException if the container configuration is in a bad
* format
*/
private async Task <Image> PullBaseImageAsync(
Authorization registryAuthorization,
ProgressEventDispatcher progressEventDispatcher)
{
RegistryClient registryClient =
buildConfiguration
.NewBaseImageRegistryClientFactory()
.SetAuthorization(registryAuthorization)
.NewRegistryClient();
IManifestTemplate manifestTemplate =
await registryClient.PullManifestAsync(buildConfiguration.GetBaseImageConfiguration().GetImageTag()).ConfigureAwait(false);
// TODO: Make schema version be enum.
switch (manifestTemplate.SchemaVersion)
{
case 1:
V21ManifestTemplate v21ManifestTemplate = (V21ManifestTemplate)manifestTemplate;
await buildConfiguration
.GetBaseImageLayersCache()
.WriteMetadataAsync(
buildConfiguration.GetBaseImageConfiguration().GetImage(), v21ManifestTemplate).ConfigureAwait(false);
return(JsonToImageTranslator.ToImage(v21ManifestTemplate));
case 2:
IBuildableManifestTemplate buildableManifestTemplate =
(IBuildableManifestTemplate)manifestTemplate;
if (buildableManifestTemplate.GetContainerConfiguration() == null ||
buildableManifestTemplate.GetContainerConfiguration().Digest == null)
{
throw new UnknownManifestFormatException(
"Invalid container configuration in Docker V2.2/OCI manifest: \n"
+ JsonTemplateMapper.ToUtf8String(buildableManifestTemplate));
}
DescriptorDigest containerConfigurationDigest =
buildableManifestTemplate.GetContainerConfiguration().Digest;
using (ThrottledProgressEventDispatcherWrapper progressEventDispatcherWrapper =
new ThrottledProgressEventDispatcherWrapper(
progressEventDispatcher.NewChildProducer(),
"pull container configuration " + containerConfigurationDigest))
{
string containerConfigurationString =
await Blobs.WriteToStringAsync(
registryClient.PullBlob(
containerConfigurationDigest,
progressEventDispatcherWrapper.SetProgressTarget,
progressEventDispatcherWrapper.DispatchProgress)).ConfigureAwait(false);
ContainerConfigurationTemplate containerConfigurationTemplate =
JsonTemplateMapper.ReadJson <ContainerConfigurationTemplate>(
containerConfigurationString);
await buildConfiguration
.GetBaseImageLayersCache()
.WriteMetadataAsync(
buildConfiguration.GetBaseImageConfiguration().GetImage(),
buildableManifestTemplate,
containerConfigurationTemplate).ConfigureAwait(false);
return(JsonToImageTranslator.ToImage(
buildableManifestTemplate, containerConfigurationTemplate));
}
}
throw new InvalidOperationException(Resources.PullBaseImageStepUnknownManifestErrorMessage);
}
public void SetContainerConfiguration(long size, DescriptorDigest digest)
{
Config = new ContentDescriptorTemplate(CONTAINER_CONFIGURATION_MEDIA_TYPE, size, digest);
}
