public void TestListDigests()
{
CacheStorageFiles cacheStorageFiles =
new CacheStorageFiles(temporaryFolder.NewFolder().ToPath());
CacheStorageReader cacheStorageReader = new CacheStorageReader(cacheStorageFiles);
// Creates test layer directories.
Files.CreateDirectories(cacheStorageFiles.GetLayersDirectory().Resolve(layerDigest1.GetHash()));
Files.CreateDirectories(cacheStorageFiles.GetLayersDirectory().Resolve(layerDigest2.GetHash()));
// Checks that layer directories created are all listed.
Assert.AreEqual(
new HashSet <DescriptorDigest>(new[] { layerDigest1, layerDigest2 }),
cacheStorageReader.FetchDigests());
// Checks that non-digest directories means the cache is corrupted.
Files.CreateDirectory(cacheStorageFiles.GetLayersDirectory().Resolve("not a hash"));
try
{
cacheStorageReader.FetchDigests();
Assert.Fail("Listing digests should have failed");
}
catch (CacheCorruptedException ex)
{
Assert.That(
ex.Message, Does.StartWith("Found non-digest file in layers directory"));
Assert.IsInstanceOf <DigestException>(ex.InnerException);
}
}
/**
* Retrieves the {@link CachedLayer} for the layer with digest {@code layerDigest}.
*
* @param layerDigest the layer digest
* @return the {@link CachedLayer} referenced by the layer digest, if found
* @throws CacheCorruptedException if the cache was found to be corrupted
* @throws IOException if an I/O exception occurs
*/
public Maybe <CachedLayer> Retrieve(DescriptorDigest layerDigest)
{
layerDigest = layerDigest ?? throw new ArgumentNullException(nameof(layerDigest));
SystemPath layerDirectory = cacheStorageFiles.GetLayerDirectory(layerDigest);
if (!Files.Exists(layerDirectory))
{
return(Maybe.Empty <CachedLayer>());
}
CachedLayer.Builder cachedLayerBuilder = CachedLayer.CreateBuilder().SetLayerDigest(layerDigest);
foreach (SystemPath fileInLayerDirectory in Files.List(layerDirectory))
{
if (CacheStorageFiles.IsLayerFile(fileInLayerDirectory))
{
if (cachedLayerBuilder.HasLayerBlob())
{
throw new CacheCorruptedException(
cacheStorageFiles.GetCacheDirectory(),
"Multiple layer files found for layer with digest "
+ layerDigest.GetHash()
+ " in directory: "
+ layerDirectory);
}
cachedLayerBuilder
.SetLayerBlob(Blobs.From(fileInLayerDirectory))
.SetLayerDiffId(cacheStorageFiles.GetDiffId(fileInLayerDirectory))
.SetLayerSize(Files.Size(fileInLayerDirectory));
}
}
return(Maybe.Of(cachedLayerBuilder.Build()));
}
/**
* Retrieves the layer digest selected by the {@code selector}.
*
* @param selector the selector
* @return the layer digest {@code selector} selects, if found
* @throws CacheCorruptedException if the selector file contents was not a valid layer digest
* @throws IOException if an I/O exception occurs
*/
public Maybe <DescriptorDigest> Select(DescriptorDigest selector)
{
selector = selector ?? throw new ArgumentNullException(nameof(selector));
SystemPath selectorFile = cacheStorageFiles.GetSelectorFile(selector);
if (!Files.Exists(selectorFile))
{
return(Maybe.Empty <DescriptorDigest>());
}
string selectorFileContents =
File.ReadAllText(selectorFile.ToFile().FullName, Encoding.UTF8);
try
{
return(Maybe.Of(DescriptorDigest.FromHash(selectorFileContents)));
}
catch (DigestException)
{
throw new CacheCorruptedException(
cacheStorageFiles.GetCacheDirectory(),
"Expected valid layer digest as contents of selector file `"
+ selectorFile
+ "` for selector `"
+ selector.GetHash()
+ "`, but got: "
+ selectorFileContents);
}
}
public void TestCreateFromHash_pass()
{
string goodHash = CreateGoodHash('a');
DescriptorDigest descriptorDigest = DescriptorDigest.FromHash(goodHash);
Assert.AreEqual(goodHash, descriptorDigest.GetHash());
Assert.AreEqual("sha256:" + goodHash, descriptorDigest.ToString());
}
public async Task TestRetrieveAsync()
{
CacheStorageFiles cacheStorageFiles =
new CacheStorageFiles(temporaryFolder.NewFolder().ToPath());
CacheStorageReader cacheStorageReader = new CacheStorageReader(cacheStorageFiles);
// Creates the test layer directory.
DescriptorDigest layerDigest = layerDigest1;
DescriptorDigest layerDiffId = layerDigest2;
Files.CreateDirectories(cacheStorageFiles.GetLayerDirectory(layerDigest));
using (Stream @out =
Files.NewOutputStream(cacheStorageFiles.GetLayerFile(layerDigest, layerDiffId)))
{
byte[] bytes = Encoding.UTF8.GetBytes("layerBlob");
await @out.WriteAsync(bytes, 0, bytes.Length).ConfigureAwait(false);
}
// Checks that the CachedLayer is retrieved correctly.
Maybe <CachedLayer> optionalCachedLayer = cacheStorageReader.Retrieve(layerDigest);
Assert.IsTrue(optionalCachedLayer.IsPresent());
Assert.AreEqual(layerDigest, optionalCachedLayer.Get().GetDigest());
Assert.AreEqual(layerDiffId, optionalCachedLayer.Get().GetDiffId());
Assert.AreEqual("layerBlob".Length, optionalCachedLayer.Get().GetSize());
Assert.AreEqual("layerBlob", await Blobs.WriteToStringAsync(optionalCachedLayer.Get().GetBlob()).ConfigureAwait(false));
// Checks that multiple .layer files means the cache is corrupted.
Files.CreateFile(cacheStorageFiles.GetLayerFile(layerDigest, layerDigest));
try
{
cacheStorageReader.Retrieve(layerDigest);
Assert.Fail("Should have thrown CacheCorruptedException");
}
catch (CacheCorruptedException ex)
{
Assert.That(
ex.Message, Does.StartWith(
"Multiple layer files found for layer with digest "
+ layerDigest.GetHash()
+ " in directory: "
+ cacheStorageFiles.GetLayerDirectory(layerDigest)));
}
}
public async Task TestWrite_uncompressedAsync()
{
IBlob uncompressedLayerBlob = Blobs.From("uncompressedLayerBlob");
BlobDescriptor layerDigestDescriptor = await GetDigestAsync(Compress(uncompressedLayerBlob)).ConfigureAwait(false);
DescriptorDigest layerDigest = layerDigestDescriptor.GetDigest();
BlobDescriptor selectorDescriptor = await GetDigestAsync(Blobs.From("selector")).ConfigureAwait(false);
DescriptorDigest selector = selectorDescriptor.GetDigest();
CachedLayer cachedLayer =
await new CacheStorageWriter(cacheStorageFiles)
.WriteUncompressedAsync(uncompressedLayerBlob, selector).ConfigureAwait(false);
await VerifyCachedLayerAsync(cachedLayer, uncompressedLayerBlob).ConfigureAwait(false);
// Verifies that the files are present.
SystemPath selectorFile = cacheStorageFiles.GetSelectorFile(selector);
Assert.IsTrue(Files.Exists(selectorFile));
Assert.AreEqual(layerDigest.GetHash(), await Blobs.WriteToStringAsync(Blobs.From(selectorFile)).ConfigureAwait(false));
}
/**
* Writes the {@code selector} to a file in the selectors directory, with contents {@code
* layerDigest}.
*
* @param selector the selector
* @param layerDigest the layer digest it selects
* @throws IOException if an I/O exception occurs
*/
private void WriteSelector(DescriptorDigest selector, DescriptorDigest layerDigest)
{
SystemPath selectorFile = cacheStorageFiles.GetSelectorFile(selector);
// Creates the selectors directory if it doesn't exist.
Files.CreateDirectories(selectorFile.GetParent());
// Writes the selector to a temporary file and then moves the file to the intended location.
using (TemporaryFile temporarySelectorFile = Files.CreateTempFile())
{
using (Stream fileOut = FileOperations.NewLockingOutputStream(temporarySelectorFile.Path))
{
byte[] bytes = Encoding.UTF8.GetBytes(layerDigest.GetHash());
fileOut.Write(bytes, 0, bytes.Length);
}
// Attempts an atomic move first, and falls back to non-atomic if the file system does not
// support atomic moves.
Files.Move(
temporarySelectorFile.Path,
selectorFile,
StandardCopyOption.REPLACE_EXISTING);
}
}
/**
* Gets the directory for the layer with digest {@code layerDigest}.
*
* @param layerDigest the digest of the layer
* @return the directory for that {@code layerDigest}
*/
public SystemPath GetLayerDirectory(DescriptorDigest layerDigest)
{
layerDigest = layerDigest ?? throw new ArgumentNullException(nameof(layerDigest));
return(GetLayersDirectory().Resolve(layerDigest.GetHash()));
}
/**
* Resolves a selector file.
*
* @param selector the selector digest
* @return the selector file
*/
public SystemPath GetSelectorFile(DescriptorDigest selector)
{
selector = selector ?? throw new ArgumentNullException(nameof(selector));
return(cacheDirectory.Resolve(SELECTORS_DIRECTORY).Resolve(selector.GetHash()));
}
/**
* Gets the filename for the layer file. The filename is in the form {@code <layer diff
* ID>.layer}.
*
* @param layerDiffId the layer's diff ID
* @return the layer filename
*/
public virtual string GetLayerFilename(DescriptorDigest layerDiffId)
{
layerDiffId = layerDiffId ?? throw new ArgumentNullException(nameof(layerDiffId));
return(layerDiffId.GetHash());
}
public async Task TestWriteToAsync()
{
SystemPath fileA = Paths.Get(TestResources.GetResource("core/fileA").ToURI());
SystemPath fileB = Paths.Get(TestResources.GetResource("core/fileB").ToURI());
long fileASize = Files.Size(fileA);
long fileBSize = Files.Size(fileB);
DescriptorDigest fakeDigestA =
DescriptorDigest.FromHash(
"5994471abb01112afcc18159f6cc74b4f511b99806da59b3caf5a9c173cacfc5");
DescriptorDigest fakeDigestB =
DescriptorDigest.FromHash(
"5994471abb01112afcc18159f6cc74b4f511b99806da59b3caf5a9c173cacfc6");
Mock.Get(mockLayer1).Setup(m => m.GetBlob()).Returns(Blobs.From(fileA));
Mock.Get(mockLayer1).Setup(m => m.GetBlobDescriptor()).Returns(new BlobDescriptor(fileASize, fakeDigestA));
Mock.Get(mockLayer1).Setup(m => m.GetDiffId()).Returns(fakeDigestA);
Mock.Get(mockLayer2).Setup(m => m.GetBlob()).Returns(Blobs.From(fileB));
Mock.Get(mockLayer2).Setup(m => m.GetBlobDescriptor()).Returns(new BlobDescriptor(fileBSize, fakeDigestB));
Mock.Get(mockLayer2).Setup(m => m.GetDiffId()).Returns(fakeDigestB);
Image testImage =
Image.CreateBuilder(ManifestFormat.V22).AddLayer(mockLayer1).AddLayer(mockLayer2).Build();
ImageTarball imageToTarball = new ImageTarball(testImage, ImageReference.Parse("my/image:tag"));
MemoryStream @out = new MemoryStream();
await imageToTarball.WriteToAsync(@out).ConfigureAwait(false);
MemoryStream @in = new MemoryStream(@out.ToArray());
using (TarInputStream tarArchiveInputStream = new TarInputStream(@in))
{
// Verifies layer with fileA was added.
TarEntry headerFileALayer = tarArchiveInputStream.GetNextEntry();
Assert.AreEqual(fakeDigestA.GetHash() + ".tar.gz", headerFileALayer.Name);
string fileAString =
await new StreamReader(tarArchiveInputStream).ReadToEndAsync().ConfigureAwait(false);
Assert.AreEqual(await Blobs.WriteToStringAsync(Blobs.From(fileA)).ConfigureAwait(false), fileAString);
// Verifies layer with fileB was added.
TarEntry headerFileBLayer = tarArchiveInputStream.GetNextEntry();
Assert.AreEqual(fakeDigestB.GetHash() + ".tar.gz", headerFileBLayer.Name);
string fileBString = await new StreamReader(tarArchiveInputStream).ReadToEndAsync().ConfigureAwait(false);
Assert.AreEqual(await Blobs.WriteToStringAsync(Blobs.From(fileB)).ConfigureAwait(false), fileBString);
// Verifies container configuration was added.
TarEntry headerContainerConfiguration = tarArchiveInputStream.GetNextEntry();
Assert.AreEqual("config.json", headerContainerConfiguration.Name);
string containerConfigJson = await new StreamReader(tarArchiveInputStream).ReadToEndAsync().ConfigureAwait(false);
JsonTemplateMapper.ReadJson <ContainerConfigurationTemplate>(containerConfigJson);
// Verifies manifest was added.
TarEntry headerManifest = tarArchiveInputStream.GetNextEntry();
Assert.AreEqual("manifest.json", headerManifest.Name);
string manifestJson = await new StreamReader(tarArchiveInputStream).ReadToEndAsync().ConfigureAwait(false);
JsonTemplateMapper.ReadListOfJson <DockerLoadManifestEntryTemplate>(manifestJson);
}
}