public static async Task CreateImageAsync()
{
SystemPath root = imageLocation.GetRoot().ToPath();
SystemPath fileA = Files.CreateFile(root.Resolve("fileA.txt"));
SystemPath fileB = Files.CreateFile(root.Resolve("fileB.txt"));
SystemPath fileC = Files.CreateFile(root.Resolve("fileC.txt"));
SystemPath subdir = Files.CreateDirectory(root.Resolve("dir"));
SystemPath subsubdir = Files.CreateDirectory(subdir.Resolve("subdir"));
Files.CreateFile(subdir.Resolve("fileD.txt"));
Files.CreateFile(subsubdir.Resolve("fileE.txt"));
imageTar = new FileInfo(Path.Combine(imageLocation.GetRoot().FullName, "image.tar"));
Containerizer containerizer =
Containerizer.To(TarImage.Named("fibdotnet-core/reproducible").SaveTo(imageTar.ToPath()));
await FibContainerBuilder.FromScratch()
.SetEntrypoint("echo", "Hello World")
.AddLayer(ImmutableArray.Create(fileA), AbsoluteUnixPath.Get("/app"))
// layer with out-of-order files
.AddLayer(ImmutableArray.Create(fileC, fileB), "/app")
.AddLayer(
LayerConfiguration.CreateBuilder()
.AddEntryRecursive(subdir, AbsoluteUnixPath.Get("/app"))
.Build())
.ContainerizeAsync(containerizer).ConfigureAwait(false);
}
/**
* Writes a V2.1 manifest for a given image reference.
*
* @param imageReference the image reference to store the metadata for
* @param manifestTemplate the manifest
*/
public async Task WriteMetadataAsync(IImageReference imageReference, V21ManifestTemplate manifestTemplate)
{
SystemPath imageDirectory = cacheStorageFiles.GetImageDirectory(imageReference);
Files.CreateDirectories(imageDirectory);
using (LockFile ignored1 = LockFile.Create(imageDirectory.Resolve("lock")))
{
await WriteMetadataAsync(manifestTemplate, imageDirectory.Resolve("manifest.json")).ConfigureAwait(false);
}
}
private static void SetupCachedMetadataV22(SystemPath cacheDirectory)
{
SystemPath imageDirectory = cacheDirectory.Resolve("images/test/image!tag");
Files.CreateDirectories(imageDirectory);
Files.Copy(
Paths.Get(TestResources.GetResource("core/json/v22manifest.json").ToURI()),
imageDirectory.Resolve("manifest.json"));
Files.Copy(
Paths.Get(TestResources.GetResource("core/json/containerconfig.json").ToURI()),
imageDirectory.Resolve("config.json"));
}
/**
* Writes an uncompressed {@code layerBlob} to the {@code layerDirectory}.
*
* @param uncompressedLayerBlob the uncompressed layer {@link Blob}
* @param layerDirectory the directory for the layer
* @return a {@link WrittenLayer} with the written layer information
* @throws IOException if an I/O exception occurs
*/
private async Task <WrittenLayer> WriteUncompressedLayerBlobToDirectoryAsync(
IBlob uncompressedLayerBlob, SystemPath layerDirectory)
{
using (TemporaryFile temporaryLayerFile = CacheStorageFiles.GetTemporaryLayerFile(layerDirectory))
{
DescriptorDigest layerDiffId;
BlobDescriptor blobDescriptor;
// Writes the layer with GZIP compression. The original bytes are captured as the layer's
// diff ID and the bytes outputted from the GZIP compression are captured as the layer's
// content descriptor.
using (CountingDigestOutputStream compressedDigestOutputStream =
new CountingDigestOutputStream(
Files.NewOutputStream(temporaryLayerFile.Path)))
{
using (GZipStream compressorStream = new GZipStream(compressedDigestOutputStream, CompressionMode.Compress, true))
{
BlobDescriptor descriptor = await uncompressedLayerBlob.WriteToAsync(compressorStream).ConfigureAwait(false);
layerDiffId = descriptor.GetDigest();
}
// The GZIPOutputStream must be closed in order to write out the remaining compressed data.
blobDescriptor = compressedDigestOutputStream.ComputeDigest();
}
DescriptorDigest layerDigest = blobDescriptor.GetDigest();
long layerSize = blobDescriptor.GetSize();
// Renames the temporary layer file to the correct filename.
SystemPath layerFile = layerDirectory.Resolve(cacheStorageFiles.GetLayerFilename(layerDiffId));
temporaryLayerFile.MoveIfDoesNotExist(layerFile);
return(new WrittenLayer(layerDigest, layerDiffId, layerSize));
}
}
public void TestClose_directoryNotDeletedIfMoved()
{
SystemPath destinationParent = temporaryFolder.NewFolder().ToPath();
using (TemporaryDirectory temporaryDirectory =
new TemporaryDirectory(temporaryFolder.NewFolder().ToPath()))
{
CreateFilesInDirectory(temporaryDirectory.GetDirectory());
Assert.IsFalse(Files.Exists(destinationParent.Resolve("destination")));
Files.Move(temporaryDirectory.GetDirectory(), destinationParent.Resolve("destination"));
temporaryDirectory.Dispose();
Assert.IsFalse(Files.Exists(temporaryDirectory.GetDirectory()));
Assert.IsTrue(Files.Exists(destinationParent.Resolve("destination")));
}
}
private static void CreateFilesInDirectory(SystemPath directory)
{
SystemPath testFilesDirectory = Paths.Get(TestResources.GetResource("core/layer").ToURI());
new DirectoryWalker(testFilesDirectory)
.FilterRoot()
.Walk(path =>
{
if (File.Exists(path))
{
Files.Copy(path, directory.Resolve(testFilesDirectory.Relativize(path)));
}
else if (Directory.Exists(path))
{
Directory.CreateDirectory(directory.Resolve(testFilesDirectory.Relativize(path)));
}
});
}
private SystemPath CreateFile(SystemPath root, string filename, string content, long lastModifiedTime)
{
SystemPath newFile =
Files.Write(
root.Resolve(filename),
Encoding.UTF8.GetBytes(content));
Files.SetLastModifiedTime(newFile, FileTime.FromMillis(lastModifiedTime));
return(newFile);
}
/**
* Saves the manifest and container configuration for a V2.2 or OCI image.
*
* @param imageReference the image reference to store the metadata for
* @param manifestTemplate the manifest
* @param containerConfiguration the container configuration
*/
public async Task WriteMetadataAsync(
IImageReference imageReference,
IBuildableManifestTemplate manifestTemplate,
ContainerConfigurationTemplate containerConfiguration)
{
manifestTemplate = manifestTemplate ?? throw new ArgumentNullException(nameof(manifestTemplate));
Preconditions.CheckNotNull(manifestTemplate.GetContainerConfiguration());
Preconditions.CheckNotNull(manifestTemplate.GetContainerConfiguration().Digest);
SystemPath imageDirectory = cacheStorageFiles.GetImageDirectory(imageReference);
Files.CreateDirectories(imageDirectory);
using (LockFile ignored1 = LockFile.Create(imageDirectory.Resolve("lock")))
{
await WriteMetadataAsync(manifestTemplate, imageDirectory.Resolve("manifest.json")).ConfigureAwait(false);
await WriteMetadataAsync(containerConfiguration, imageDirectory.Resolve("config.json")).ConfigureAwait(false);
}
}
public void SetUp()
{
SystemPath folder = temporaryFolder.NewFolder().ToPath();
SystemPath file1 = Files.CreateDirectory(folder.Resolve("files"));
SystemPath file2 = Files.CreateFile(folder.Resolve("files").Resolve("two"));
SystemPath file3 = Files.CreateFile(folder.Resolve("gile"));
LayerEntry testLayerEntry1 = DefaultLayerEntry(file1, AbsoluteUnixPath.Get("/extraction/path"));
LayerEntry testLayerEntry2 = DefaultLayerEntry(file2, AbsoluteUnixPath.Get("/extraction/path"));
LayerEntry testLayerEntry3 = DefaultLayerEntry(file3, AbsoluteUnixPath.Get("/extraction/path"));
LayerEntry testLayerEntry4 =
new LayerEntry(
file3,
AbsoluteUnixPath.Get("/extraction/path"),
FilePermissions.FromOctalString("755"),
LayerConfiguration.DefaultModifiedTime);
LayerEntry testLayerEntry5 =
DefaultLayerEntry(file3, AbsoluteUnixPath.Get("/extraction/patha"));
LayerEntry testLayerEntry6 =
new LayerEntry(
file3,
AbsoluteUnixPath.Get("/extraction/patha"),
FilePermissions.FromOctalString("755"),
LayerConfiguration.DefaultModifiedTime);
outOfOrderLayerEntries =
ImmutableArray.Create(
testLayerEntry4,
testLayerEntry2,
testLayerEntry6,
testLayerEntry3,
testLayerEntry1,
testLayerEntry5);
inOrderLayerEntries =
ImmutableArray.Create(
testLayerEntry1,
testLayerEntry2,
testLayerEntry3,
testLayerEntry4,
testLayerEntry5,
testLayerEntry6);
}
public async Task TestWriteMetadata_v21Async()
{
SystemPath manifestJsonFile =
Paths.Get(TestResources.GetResource("core/json/v21manifest.json").ToURI());
V21ManifestTemplate manifestTemplate =
JsonTemplateMapper.ReadJsonFromFile <V21ManifestTemplate>(manifestJsonFile);
ImageReference imageReference = ImageReference.Parse("image.reference/project/thing:tag");
await new CacheStorageWriter(cacheStorageFiles).WriteMetadataAsync(imageReference, manifestTemplate).ConfigureAwait(false);
SystemPath savedManifestPath =
cacheRoot.Resolve("images/image.reference/project/thing!tag/manifest.json");
Assert.IsTrue(Files.Exists(savedManifestPath));
V21ManifestTemplate savedManifest =
JsonTemplateMapper.ReadJsonFromFile <V21ManifestTemplate>(savedManifestPath);
Assert.AreEqual("amd64", savedManifest.GetContainerConfiguration().Get().Architecture);
}
public void TestGetImageDirectory()
{
SystemPath imagesDirectory = Paths.Get("cache", "directory", "images");
Assert.AreEqual(imagesDirectory, TEST_CACHE_STORAGE_FILES.GetImagesDirectory());
Assert.AreEqual(
imagesDirectory.Resolve("reg.istry/repo/sitory!tag"),
TEST_CACHE_STORAGE_FILES.GetImageDirectory(
ImageReference.Parse("reg.istry/repo/sitory:tag")));
Assert.AreEqual(
imagesDirectory.Resolve(
"reg.istry/repo!sha256!aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
TEST_CACHE_STORAGE_FILES.GetImageDirectory(
ImageReference.Parse(
"reg.istry/repo@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")));
Assert.AreEqual(
imagesDirectory.Resolve("reg.istry!5000/repo/sitory!tag"),
TEST_CACHE_STORAGE_FILES.GetImageDirectory(
ImageReference.Parse("reg.istry:5000/repo/sitory:tag")));
}
public void TestAddEntryRecursive_defaults()
{
SystemPath testDirectory = TestResources.GetResource("core/layer");
SystemPath testFile = TestResources.GetResource("core/fileA");
ILayerConfiguration layerConfiguration =
LayerConfiguration.CreateBuilder()
.AddEntryRecursive(testDirectory, AbsoluteUnixPath.Get("/app/layer/"))
.AddEntryRecursive(testFile, AbsoluteUnixPath.Get("/app/fileA"))
.Build();
ImmutableHashSet <LayerEntry> expectedLayerEntries =
ImmutableHashSet.Create(
DefaultLayerEntry(testDirectory, AbsoluteUnixPath.Get("/app/layer/")),
DefaultLayerEntry(testDirectory.Resolve("a"), AbsoluteUnixPath.Get("/app/layer/a/")),
DefaultLayerEntry(
testDirectory.Resolve("a/b"), AbsoluteUnixPath.Get("/app/layer/a/b/")),
DefaultLayerEntry(
testDirectory.Resolve("a/b/bar"), AbsoluteUnixPath.Get("/app/layer/a/b/bar/")),
DefaultLayerEntry(testDirectory.Resolve("c/"), AbsoluteUnixPath.Get("/app/layer/c")),
DefaultLayerEntry(
testDirectory.Resolve("c/cat/"), AbsoluteUnixPath.Get("/app/layer/c/cat")),
DefaultLayerEntry(testDirectory.Resolve("foo"), AbsoluteUnixPath.Get("/app/layer/foo")),
DefaultLayerEntry(testFile, AbsoluteUnixPath.Get("/app/fileA")));
CollectionAssert.AreEquivalent(
expectedLayerEntries, ImmutableHashSet.CreateRange(layerConfiguration.LayerEntries));
}
public async Task SetUpAsync()
{
SystemPath directory = temporaryFolder.NewFolder().ToPath();
Files.CreateDirectory(directory.Resolve("source"));
Files.CreateFile(directory.Resolve("source/file"));
Files.CreateDirectories(directory.Resolve("another/source"));
Files.CreateFile(directory.Resolve("another/source/file"));
layerBlob1 = Blobs.From("layerBlob1");
layerDigest1 = await DigestOfAsync(Compress(layerBlob1)).ConfigureAwait(false);
layerDiffId1 = await DigestOfAsync(layerBlob1).ConfigureAwait(false);
layerSize1 = await SizeOfAsync(Compress(layerBlob1)).ConfigureAwait(false);
layerEntries1 =
ImmutableArray.Create(
DefaultLayerEntry(
directory.Resolve("source/file"), AbsoluteUnixPath.Get("/extraction/path")),
DefaultLayerEntry(
directory.Resolve("another/source/file"),
AbsoluteUnixPath.Get("/another/extraction/path")));
layerBlob2 = Blobs.From("layerBlob2");
layerDigest2 = await DigestOfAsync(Compress(layerBlob2)).ConfigureAwait(false);
layerDiffId2 = await DigestOfAsync(layerBlob2).ConfigureAwait(false);
layerSize2 = await SizeOfAsync(Compress(layerBlob2)).ConfigureAwait(false);
layerEntries2 = ImmutableArray.Create <LayerEntry>();
}
/** Starts the registry */
public async Task StartAsync()
{
// Runs the Docker registry.
List <string> dockerTokens = new List <string> {
"run",
"--rm",
"-d",
"-p",
port + ":5000",
"--name",
containerName
};
if (username != null && password != null)
{
// Generate the htpasswd file to store credentials
string credentialString =
new Command(
"docker",
"run",
"--rm",
"--entrypoint",
"htpasswd",
"registry",
"-Bbn",
username,
password)
.Run();
// Creates the temporary directory in /tmp since that is one of the default directories
// mounted into Docker.
// See: https://docs.docker.com/docker-for-mac/osxfs
SystemPath tempFolder = Files.CreateTempDirectory(Paths.Get(Path.GetTempPath()), "");
Files.Write(
tempFolder.Resolve("htpasswd"), Encoding.UTF8.GetBytes(credentialString));
// Run the Docker registry
dockerTokens.AddRange(new[] {
"-v",
// Volume mount used for storing credentials
tempFolder + ":/auth",
"-e",
"REGISTRY_AUTH=htpasswd",
"-e",
"REGISTRY_AUTH_HTPASSWD_REALM=\"Registry Realm\"",
"-e",
"REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd"
});
}
dockerTokens.Add("registry");
new Command("docker", dockerTokens).Run();
await WaitUntilReadyAsync().ConfigureAwait(false);
}
public void TestCopy()
{
SystemPath destDir = temporaryFolder.NewFolder().ToPath();
SystemPath libraryA =
Paths.Get(TestResources.GetResource("core/application/dependencies/libraryA.jar").ToURI());
SystemPath libraryB =
Paths.Get(TestResources.GetResource("core/application/dependencies/libraryB.jar").ToURI());
SystemPath dirLayer = Paths.Get(TestResources.GetResource("core/layer").ToURI());
FileOperations.Copy(ImmutableArray.Create(libraryA, libraryB, dirLayer), destDir);
AssertFilesEqual(libraryA, destDir.Resolve("libraryA.jar"));
AssertFilesEqual(libraryB, destDir.Resolve("libraryB.jar"));
Assert.IsTrue(Files.Exists(destDir.Resolve("layer").Resolve("a").Resolve("b")));
Assert.IsTrue(Files.Exists(destDir.Resolve("layer").Resolve("c")));
AssertFilesEqual(
dirLayer.Resolve("a").Resolve("b").Resolve("bar"),
destDir.Resolve("layer").Resolve("a").Resolve("b").Resolve("bar"));
AssertFilesEqual(
dirLayer.Resolve("c").Resolve("cat"), destDir.Resolve("layer").Resolve("c").Resolve("cat"));
AssertFilesEqual(dirLayer.Resolve("foo"), destDir.Resolve("layer").Resolve("foo"));
}
public async Task TestToBlob_reproducibilityAsync()
{
SystemPath testRoot = temporaryFolder.GetRoot().ToPath();
SystemPath root1 = Files.CreateDirectories(testRoot.Resolve("files1"));
SystemPath root2 = Files.CreateDirectories(testRoot.Resolve("files2"));
// TODO: Currently this test only covers variation in order and modified time, even though
// TODO: the code is designed to clean up userid/groupid, this test does not check that yet.
const string contentA = "abcabc";
SystemPath fileA1 = CreateFile(root1, "fileA", contentA, 10000);
SystemPath fileA2 = CreateFile(root2, "fileA", contentA, 20000);
const string contentB = "yumyum";
SystemPath fileB1 = CreateFile(root1, "fileB", contentB, 10000);
SystemPath fileB2 = CreateFile(root2, "fileB", contentB, 20000);
// check if modified times are off
Assert.AreNotEqual(Files.GetLastModifiedTime(fileA1), Files.GetLastModifiedTime(fileA2));
Assert.AreNotEqual(Files.GetLastModifiedTime(fileB1), Files.GetLastModifiedTime(fileB2));
// create layers of exact same content but ordered differently and with different timestamps
IBlob layer =
new ReproducibleLayerBuilder(
ImmutableArray.Create(
DefaultLayerEntry(fileA1, AbsoluteUnixPath.Get("/somewhere/fileA")),
DefaultLayerEntry(fileB1, AbsoluteUnixPath.Get("/somewhere/fileB"))))
.Build();
IBlob reproduced =
new ReproducibleLayerBuilder(
ImmutableArray.Create(
DefaultLayerEntry(fileB2, AbsoluteUnixPath.Get("/somewhere/fileB")),
DefaultLayerEntry(fileA2, AbsoluteUnixPath.Get("/somewhere/fileA"))))
.Build();
byte[] layerContent = await Blobs.WriteToByteArrayAsync(layer).ConfigureAwait(false);
byte[] reproducedLayerContent = await Blobs.WriteToByteArrayAsync(reproduced).ConfigureAwait(false);
Assert.AreEqual(layerContent, reproducedLayerContent);
}
/**
* Gets the directory corresponding to the given image reference.
*
* @param imageReference the image reference
* @return a path in the form of {@code
* (fib-cache)/images/registry[!port]/repository!(tag|digest-type!digest)}
*/
public SystemPath GetImageDirectory(IImageReference imageReference)
{
imageReference = imageReference ?? throw new ArgumentNullException(nameof(imageReference));
// Replace ':' and '@' with '!' to avoid directory-naming restrictions
string replacedReference = imageReference.ToStringWithTag().Replace(':', '!').Replace('@', '!');
// Split image reference on '/' to build directory structure
IEnumerable <string> directories = Splitter.On('/').Split(replacedReference);
SystemPath destination = GetImagesDirectory();
foreach (string dir in directories)
{
destination = destination.Resolve(dir);
}
return(destination);
}
public async Task TestBuild_permissionsAsync()
{
SystemPath testRoot = temporaryFolder.GetRoot().ToPath();
SystemPath folder = Files.CreateDirectories(testRoot.Resolve("files1"));
SystemPath fileA = CreateFile(testRoot, "fileA", "abc", 54321);
SystemPath fileB = CreateFile(testRoot, "fileB", "def", 54321);
IBlob blob =
new ReproducibleLayerBuilder(
ImmutableArray.Create(
DefaultLayerEntry(fileA, AbsoluteUnixPath.Get("/somewhere/fileA")),
new LayerEntry(
fileB,
AbsoluteUnixPath.Get("/somewhere/fileB"),
FilePermissions.FromOctalString("123"),
LayerConfiguration.DefaultModifiedTime),
new LayerEntry(
folder,
AbsoluteUnixPath.Get("/somewhere/folder"),
FilePermissions.FromOctalString("456"),
LayerConfiguration.DefaultModifiedTime)))
.Build();
SystemPath tarFile = temporaryFolder.NewFile().ToPath();
using (Stream @out = new BufferedStream(Files.NewOutputStream(tarFile)))
{
await blob.WriteToAsync(@out).ConfigureAwait(false);
}
using (TarInputStream @in = new TarInputStream(Files.NewInputStream(tarFile)))
{
// Root folder (default folder permissions)
TarEntry rootEntry = @in.GetNextEntry();
// fileA (default file permissions)
TarEntry fileAEntry = @in.GetNextEntry();
// fileB (custom file permissions)
TarEntry fileBEntry = @in.GetNextEntry();
// folder (custom folder permissions)
TarEntry folderEntry = @in.GetNextEntry();
Assert.AreEqual("755", rootEntry.GetMode().ToOctalString());
Assert.AreEqual("644", fileAEntry.GetMode().ToOctalString());
Assert.AreEqual("123", fileBEntry.GetMode().ToOctalString());
Assert.AreEqual("456", folderEntry.GetMode().ToOctalString());
}
}
/**
* Writes a compressed {@code layerBlob} to the {@code layerDirectory}.
*
* @param compressedLayerBlob the compressed layer {@link Blob}
* @param layerDirectory the directory for the layer
* @return a {@link WrittenLayer} with the written layer information
* @throws IOException if an I/O exception occurs
*/
private async Task <WrittenLayer> WriteCompressedLayerBlobToDirectoryAsync(
IBlob compressedLayerBlob, SystemPath layerDirectory)
{
// Writes the layer file to the temporary directory.
using (TemporaryFile temporaryLayerFile = CacheStorageFiles.GetTemporaryLayerFile(layerDirectory))
{
BlobDescriptor layerBlobDescriptor;
using (Stream fileOutputStream = Files.NewOutputStream(temporaryLayerFile.Path))
{
layerBlobDescriptor = await compressedLayerBlob.WriteToAsync(fileOutputStream).ConfigureAwait(false);
}
// Gets the diff ID.
DescriptorDigest layerDiffId = await GetDiffIdByDecompressingFileAsync(temporaryLayerFile.Path).ConfigureAwait(false);
// Renames the temporary layer file to the correct filename.
SystemPath layerFile = layerDirectory.Resolve(cacheStorageFiles.GetLayerFilename(layerDiffId));
temporaryLayerFile.MoveIfDoesNotExist(layerFile);
return(new WrittenLayer(
layerBlobDescriptor.GetDigest(), layerDiffId, layerBlobDescriptor.GetSize()));
}
}
public void TestWalk()
{
new DirectoryWalker(testDir).Walk(addToWalkedPaths);
ISet <SystemPath> expectedPaths = new HashSet <SystemPath>
{
testDir,
testDir.Resolve("a"),
testDir.Resolve("a").Resolve("b"),
testDir.Resolve("a").Resolve("b").Resolve("bar"),
testDir.Resolve("c"),
testDir.Resolve("c").Resolve("cat"),
testDir.Resolve("foo")
};
Assert.AreEqual(expectedPaths, walkedPaths);
}
public async Task TestBuild_parentDirBehaviorAsync()
{
SystemPath testRoot = temporaryFolder.GetRoot().ToPath();
// the path doesn't really matter on source files, but these are structured
SystemPath parent = Files.CreateDirectories(testRoot.Resolve("aaa"));
SystemPath fileA = Files.CreateFile(parent.Resolve("fileA"));
SystemPath ignoredParent = Files.CreateDirectories(testRoot.Resolve("bbb-ignored"));
SystemPath fileB = Files.CreateFile(ignoredParent.Resolve("fileB"));
SystemPath fileC =
Files.CreateFile(Files.CreateDirectories(testRoot.Resolve("ccc-absent")).Resolve("fileC"));
IBlob layer =
new ReproducibleLayerBuilder(
ImmutableArray.Create(
new LayerEntry(
parent,
AbsoluteUnixPath.Get("/root/aaa"),
FilePermissions.FromOctalString("111"),
Instant.FromUnixTimeSeconds(10)),
new LayerEntry(
fileA,
AbsoluteUnixPath.Get("/root/aaa/fileA"),
FilePermissions.FromOctalString("222"),
Instant.FromUnixTimeSeconds(20)),
new LayerEntry(
fileB,
AbsoluteUnixPath.Get("/root/bbb-ignored/fileB"),
FilePermissions.FromOctalString("333"),
Instant.FromUnixTimeSeconds(30)),
new LayerEntry(
ignoredParent,
AbsoluteUnixPath.Get("/root/bbb-ignored"),
FilePermissions.FromOctalString("444"),
Instant.FromUnixTimeSeconds(40)),
new LayerEntry(
fileC,
AbsoluteUnixPath.Get("/root/ccc-absent/file3"),
FilePermissions.FromOctalString("555"),
Instant.FromUnixTimeSeconds(50))))
.Build();
SystemPath tarFile = temporaryFolder.NewFile().ToPath();
using (Stream @out = new BufferedStream(Files.NewOutputStream(tarFile)))
{
await layer.WriteToAsync(@out).ConfigureAwait(false);
}
using (TarInputStream @in = new TarInputStream(Files.NewInputStream(tarFile)))
{
// root (default folder permissions)
TarEntry root = @in.GetNextEntry();
Assert.AreEqual("755", root.GetMode().ToOctalString());
Assert.AreEqual(Instant.FromUnixTimeSeconds(1), Instant.FromDateTimeUtc(DateTime.SpecifyKind(root.ModTime, DateTimeKind.Utc)));
// parentAAA (custom permissions, custom timestamp)
TarEntry rootParentAAA = @in.GetNextEntry();
Assert.AreEqual("111", rootParentAAA.GetMode().ToOctalString());
Assert.AreEqual(Instant.FromUnixTimeSeconds(10), Instant.FromDateTimeUtc(DateTime.SpecifyKind(rootParentAAA.ModTime, DateTimeKind.Utc)));
// skip over fileA
@in.GetNextEntry();
// parentBBB (default permissions - ignored custom permissions, since fileB added first)
TarEntry rootParentBBB = @in.GetNextEntry();
// TODO (#1650): we want 040444 here.
Assert.AreEqual("755", rootParentBBB.GetMode().ToOctalString());
// TODO (#1650): we want Instant.ofEpochSecond(40) here.
Assert.AreEqual(Instant.FromUnixTimeSeconds(1), Instant.FromDateTimeUtc(DateTime.SpecifyKind(root.ModTime, DateTimeKind.Utc)));
// skip over fileB
@in.GetNextEntry();
// parentCCC (default permissions - no entry provided)
TarEntry rootParentCCC = @in.GetNextEntry();
Assert.AreEqual("755", rootParentCCC.GetMode().ToOctalString());
Assert.AreEqual(Instant.FromUnixTimeSeconds(1), Instant.FromDateTimeUtc(DateTime.SpecifyKind(root.ModTime, DateTimeKind.Utc)));
// we don't care about fileC
}
}
/**
* Retrieves the cached manifest and container configuration for an image reference.
*
* @param imageReference the image reference
* @return the manifest and container configuration for the image reference, if found
* @throws IOException if an I/O exception occurs
* @throws CacheCorruptedException if the cache is corrupted
*/
public Maybe <ManifestAndConfig> RetrieveMetadata(IImageReference imageReference)
{
SystemPath imageDirectory = cacheStorageFiles.GetImageDirectory(imageReference);
SystemPath manifestPath = imageDirectory.Resolve("manifest.json");
if (!Files.Exists(manifestPath))
{
return(Maybe.Empty <ManifestAndConfig>());
}
// TODO: Consolidate with ManifestPuller
JToken token;
using (JsonTextReader reader = new JsonTextReader(File.OpenText(manifestPath)))
{
token = JToken.ReadFrom(reader);
}
if (!(token is JObject node))
{
throw new CacheCorruptedException(
cacheStorageFiles.GetCacheDirectory(),
Resources.CacheStorageReaderNotJsonExecpetionMessage);
}
if (!node.ContainsKey("schemaVersion"))
{
throw new CacheCorruptedException(
cacheStorageFiles.GetCacheDirectory(),
Resources.CacheStorageReaderSchemaVersionMissingExecpetionMessage);
}
int schemaVersion = node["schemaVersion"].Value <int>();
if (schemaVersion == -1)
{
throw new CacheCorruptedException(
cacheStorageFiles.GetCacheDirectory(),
Resources.CacheStorageReaderInvalidSchemaVersionExecpetionMessageFormat);
}
if (schemaVersion == 1)
{
return(Maybe.Of(
new ManifestAndConfig(
JsonTemplateMapper.ReadJsonFromFile <V21ManifestTemplate>(manifestPath),
null)));
}
if (schemaVersion == 2)
{
// 'schemaVersion' of 2 can be either Docker V2.2 or OCI.
string mediaType = node["mediaType"].Value <string>();
IManifestTemplate manifestTemplate;
if (V22ManifestTemplate.ManifestMediaType == mediaType)
{
manifestTemplate =
JsonTemplateMapper.ReadJsonFromFile <V22ManifestTemplate>(manifestPath);
}
else if (OCIManifestTemplate.ManifestMediaType == mediaType)
{
manifestTemplate =
JsonTemplateMapper.ReadJsonFromFile <OCIManifestTemplate>(manifestPath);
}
else
{
throw new CacheCorruptedException(
cacheStorageFiles.GetCacheDirectory(),
string.Format(
CultureInfo.CurrentCulture,
Resources.CacheStorageReaderUnknownMediaTypeExecpetionMessageFormat, mediaType));
}
SystemPath configPath = imageDirectory.Resolve("config.json");
if (!Files.Exists(configPath))
{
throw new CacheCorruptedException(
cacheStorageFiles.GetCacheDirectory(),
Resources.CacheStorageReaderContainerConfigurationMissingExecpetionMessage);
}
ContainerConfigurationTemplate config =
JsonTemplateMapper.ReadJsonFromFile <ContainerConfigurationTemplate>(configPath);
return(Maybe.Of(new ManifestAndConfig(manifestTemplate, config)));
}
throw new CacheCorruptedException(
cacheStorageFiles.GetCacheDirectory(),
string.Format(
CultureInfo.CurrentCulture,
Resources.CacheStorageReaderInvalidSchemaVersionExecpetionMessageFormat, schemaVersion));
}
internal static TemporaryFile CreateTempFile(SystemPath systemPath)
{
return(new TemporaryFile(systemPath.Resolve(Path.GetRandomFileName())));
}
/**
* Resolves a selector file.
*
* @param selector the selector digest
* @return the selector file
*/
public SystemPath GetSelectorFile(DescriptorDigest selector)
{
selector = selector ?? throw new ArgumentNullException(nameof(selector));
return(cacheDirectory.Resolve(SELECTORS_DIRECTORY).Resolve(selector.GetHash()));
}
public void TestAddEntryRecursive_permissionsAndTimestamps()
{
SystemPath testDirectory = TestResources.GetResource("core/layer");
SystemPath testFile = TestResources.GetResource("core/fileA");
FilePermissions permissions1 = FilePermissions.FromOctalString("111");
FilePermissions permissions2 = FilePermissions.FromOctalString("777");
Instant timestamp1 = Instant.FromUnixTimeSeconds(123);
Instant timestamp2 = Instant.FromUnixTimeSeconds(987);
FilePermissions permissionsProvider(SystemPath _, AbsoluteUnixPath destination) =>
destination.ToString().StartsWith("/app/layer/a", StringComparison.Ordinal) ? permissions1 : permissions2;
Instant timestampProvider(SystemPath _, AbsoluteUnixPath destination) =>
destination.ToString().StartsWith("/app/layer/a", StringComparison.Ordinal) ? timestamp1 : timestamp2;
ILayerConfiguration layerConfiguration =
LayerConfiguration.CreateBuilder()
.AddEntryRecursive(
testDirectory,
AbsoluteUnixPath.Get("/app/layer/"),
permissionsProvider,
timestampProvider)
.AddEntryRecursive(
testFile,
AbsoluteUnixPath.Get("/app/fileA"),
permissionsProvider,
timestampProvider)
.Build();
ImmutableHashSet <LayerEntry> expectedLayerEntries =
ImmutableHashSet.Create(
new LayerEntry(
testDirectory, AbsoluteUnixPath.Get("/app/layer/"), permissions2, timestamp2),
new LayerEntry(
testDirectory.Resolve("a"),
AbsoluteUnixPath.Get("/app/layer/a/"),
permissions1,
timestamp1),
new LayerEntry(
testDirectory.Resolve("a/b"),
AbsoluteUnixPath.Get("/app/layer/a/b/"),
permissions1,
timestamp1),
new LayerEntry(
testDirectory.Resolve("a/b/bar"),
AbsoluteUnixPath.Get("/app/layer/a/b/bar/"),
permissions1,
timestamp1),
new LayerEntry(
testDirectory.Resolve("c/"),
AbsoluteUnixPath.Get("/app/layer/c"),
permissions2,
timestamp2),
new LayerEntry(
testDirectory.Resolve("c/cat/"),
AbsoluteUnixPath.Get("/app/layer/c/cat"),
permissions2,
timestamp2),
new LayerEntry(
testDirectory.Resolve("foo"),
AbsoluteUnixPath.Get("/app/layer/foo"),
permissions2,
timestamp2),
new LayerEntry(testFile, AbsoluteUnixPath.Get("/app/fileA"), permissions2, timestamp2));
CollectionAssert.AreEquivalent(
expectedLayerEntries, ImmutableHashSet.CreateRange(layerConfiguration.LayerEntries));
}
/**
* Resolves a file to use as a temporary file to write layer contents to.
*
* @param layerDirectory the directory in which to resolve the temporary layer file
* @return the temporary layer file
*/
public static TemporaryFile GetTemporaryLayerFile(SystemPath layerDirectory)
{
layerDirectory = layerDirectory ?? throw new ArgumentNullException(nameof(layerDirectory));
return(new TemporaryFile(layerDirectory.Resolve(TEMPORARY_LAYER_FILE_NAME)));
}
