/**
* Retrieves the cached base image.
*
* @return the cached image
* @throws IOException when an I/O exception occurs
* @throws CacheCorruptedException if the cache is corrupted
* @throws LayerPropertyNotFoundException if adding image layers fails
* @throws BadContainerConfigurationFormatException if the container configuration is in a bad
* format
*/
private Image PullBaseImageOffline()
{
IImageReference baseImage = buildConfiguration.GetBaseImageConfiguration().GetImage();
Maybe <ManifestAndConfig> metadata =
buildConfiguration.GetBaseImageLayersCache().RetrieveMetadata(baseImage);
if (!metadata.IsPresent())
{
throw new IOException(
"Cannot run Fib in offline mode; " + baseImage + " not found in local Fib cache");
}
IManifestTemplate manifestTemplate = metadata.Get().GetManifest();
if (manifestTemplate is V21ManifestTemplate v21ManifestTemplate)
{
return(JsonToImageTranslator.ToImage(v21ManifestTemplate));
}
ContainerConfigurationTemplate configurationTemplate =
metadata.Get().GetConfig().OrElseThrow(() => new InvalidOperationException());
return(JsonToImageTranslator.ToImage(
(IBuildableManifestTemplate)manifestTemplate, configurationTemplate));
}
public void TestToJson()
{
// Loads the expected JSON string.
SystemPath jsonFile = Paths.Get(TestResources.GetResource("core/json/containerconfig.json").ToURI());
string expectedJson = Encoding.UTF8.GetString(Files.ReadAllBytes(jsonFile));
// Creates the JSON object to serialize.
ContainerConfigurationTemplate containerConfigJson = new ContainerConfigurationTemplate
{
Created = "1970-01-01T00:00:20Z",
Architecture = "wasm",
Os = "js"
};
containerConfigJson.SetContainerEnvironment(new[] { "VAR1=VAL1", "VAR2=VAL2" });
containerConfigJson.SetContainerEntrypoint(new[] { "some", "entrypoint", "command" });
containerConfigJson.SetContainerCmd(new[] { "arg1", "arg2" });
containerConfigJson.SetContainerHealthCheckTest(new[] { "CMD-SHELL", "/checkhealth" });
containerConfigJson.SetContainerHealthCheckInterval(3000000000L);
containerConfigJson.SetContainerHealthCheckTimeout(1000000000L);
containerConfigJson.SetContainerHealthCheckStartPeriod(2000000000L);
containerConfigJson.SetContainerHealthCheckRetries(3);
containerConfigJson.SetContainerExposedPorts(
new Dictionary <string, IDictionary <object, object> >
{
["1000/tcp"] =
ImmutableDictionary.Create <object, object>(),
["2000/tcp"] =
ImmutableDictionary.Create <object, object>(),
["3000/udp"] =
ImmutableDictionary.Create <object, object>()
}.ToImmutableSortedDictionary());
containerConfigJson.SetContainerLabels(ImmutableDic.Of("key1", "value1", "key2", "value2"));
containerConfigJson.SetContainerVolumes(
ImmutableDic.Of <string, IDictionary <object, object> >(
"/var/job-result-data", ImmutableDictionary.Create <object, object>(), "/var/log/my-app-logs", ImmutableDictionary.Create <object, object>()));
containerConfigJson.SetContainerWorkingDir("/some/workspace");
containerConfigJson.SetContainerUser("tomcat");
containerConfigJson.AddLayerDiffId(
DescriptorDigest.FromDigest(
"sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad"));
containerConfigJson.AddHistoryEntry(
HistoryEntry.CreateBuilder()
.SetCreationTimestamp(Instant.FromUnixTimeSeconds(0))
.SetAuthor("Bazel")
.SetCreatedBy("bazel build ...")
.SetEmptyLayer(true)
.Build());
containerConfigJson.AddHistoryEntry(
HistoryEntry.CreateBuilder()
.SetCreationTimestamp(Instant.FromUnixTimeSeconds(20))
.SetAuthor("Fib")
.SetCreatedBy("fib")
.Build());
// Serializes the JSON object.
Assert.AreEqual(expectedJson, JsonTemplateMapper.ToUtf8String(containerConfigJson));
}
/**
* Saves a manifest and container configuration for a V2.2 or OCI image.
*
* @param imageReference the image reference to save the manifest and container configuration for
* @param manifestTemplate the V2.2 or OCI manifest
* @param containerConfigurationTemplate the container configuration
* @throws IOException if an I/O exception occurs
*/
public async Task WriteMetadataAsync(
IImageReference imageReference,
IBuildableManifestTemplate manifestTemplate,
ContainerConfigurationTemplate containerConfigurationTemplate)
{
await cacheStorageWriter.WriteMetadataAsync(
imageReference, manifestTemplate, containerConfigurationTemplate).ConfigureAwait(false);
}
public void TestFromJson()
{
// Loads the JSON string.
SystemPath jsonFile = Paths.Get(TestResources.GetResource("core/json/containerconfig.json").ToURI());
// Deserializes into a manifest JSON object.
ContainerConfigurationTemplate containerConfigJson =
JsonTemplateMapper.ReadJsonFromFile <ContainerConfigurationTemplate>(jsonFile);
Assert.AreEqual("1970-01-01T00:00:20Z", containerConfigJson.Created);
Assert.AreEqual("wasm", containerConfigJson.Architecture);
Assert.AreEqual("js", containerConfigJson.Os);
Assert.AreEqual(
new[] { "VAR1=VAL1", "VAR2=VAL2" }, containerConfigJson.GetContainerEnvironment());
Assert.AreEqual(
new[] { "some", "entrypoint", "command" },
containerConfigJson.GetContainerEntrypoint());
Assert.AreEqual(new[] { "arg1", "arg2" }, containerConfigJson.GetContainerCmd());
Assert.AreEqual(
new[] { "CMD-SHELL", "/checkhealth" }, containerConfigJson.GetContainerHealthTest());
Assert.IsNotNull(containerConfigJson.GetContainerHealthInterval());
Assert.AreEqual(3000000000L, containerConfigJson.GetContainerHealthInterval().GetValueOrDefault());
Assert.IsNotNull(containerConfigJson.GetContainerHealthTimeout());
Assert.AreEqual(1000000000L, containerConfigJson.GetContainerHealthTimeout().GetValueOrDefault());
Assert.IsNotNull(containerConfigJson.GetContainerHealthStartPeriod());
Assert.AreEqual(
2000000000L, containerConfigJson.GetContainerHealthStartPeriod().GetValueOrDefault());
Assert.IsNotNull(containerConfigJson.GetContainerHealthRetries());
Assert.AreEqual(3, containerConfigJson.GetContainerHealthRetries().GetValueOrDefault());
Assert.AreEqual(
ImmutableDic.Of("key1", "value1", "key2", "value2"),
containerConfigJson.GetContainerLabels());
Assert.AreEqual("/some/workspace", containerConfigJson.GetContainerWorkingDir());
Assert.AreEqual(
DescriptorDigest.FromDigest(
"sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad"),
containerConfigJson.GetLayerDiffId(0));
Assert.AreEqual(
ImmutableArray.Create(
HistoryEntry.CreateBuilder()
.SetCreationTimestamp(Instant.FromUnixTimeSeconds(0))
.SetAuthor("Bazel")
.SetCreatedBy("bazel build ...")
.SetEmptyLayer(true)
.Build(),
HistoryEntry.CreateBuilder()
.SetCreationTimestamp(Instant.FromUnixTimeSeconds(20))
.SetAuthor("Fib")
.SetCreatedBy("fib")
.Build()),
containerConfigJson.History);
}
public void TestGetContainerConfiguration()
{
SetUp(ManifestFormat.V22);
// Loads the expected JSON string.
SystemPath jsonFile = Paths.Get(TestResources.GetResource("core/json/containerconfig.json").ToURI());
string expectedJson = Encoding.UTF8.GetString(Files.ReadAllBytes(jsonFile));
// Translates the image to the container configuration and writes the JSON string.
ContainerConfigurationTemplate containerConfiguration = imageToJsonTranslator.GetContainerConfiguration();
Assert.AreEqual(expectedJson, JsonTemplateMapper.ToUtf8String(containerConfiguration));
}
/** Tests translation of image to {@link BuildableManifestTemplate}. */
private async Task TestGetManifestAsync(
ManifestFormat manifestTemplateClass, string translatedJsonFilename)
{
// Loads the expected JSON string.
SystemPath jsonFile = Paths.Get(TestResources.GetResource(translatedJsonFilename).ToURI());
string expectedJson = Encoding.UTF8.GetString(Files.ReadAllBytes(jsonFile));
// Translates the image to the manifest and writes the JSON string.
ContainerConfigurationTemplate containerConfiguration = imageToJsonTranslator.GetContainerConfiguration();
BlobDescriptor blobDescriptor = await Digests.ComputeJsonDescriptorAsync(containerConfiguration).ConfigureAwait(false);
IBuildableManifestTemplate manifestTemplate =
imageToJsonTranslator.GetManifestTemplate(manifestTemplateClass, blobDescriptor);
Assert.AreEqual(expectedJson, JsonTemplateMapper.ToUtf8String(manifestTemplate));
}
/**
* Gets a {@link BuildResult} from an {@link Image}.
*
* @param image the image
* @param targetFormat the target format of the image
* @return a new {@link BuildResult} with the image's digest and id
* @throws IOException if writing the digest or container configuration fails
*/
public static async Task <BuildResult> FromImageAsync(Image image, ManifestFormat targetFormat)
{
ImageToJsonTranslator imageToJsonTranslator = new ImageToJsonTranslator(image);
ContainerConfigurationTemplate configurationTemplate = imageToJsonTranslator.GetContainerConfiguration();
BlobDescriptor containerConfigurationBlobDescriptor =
await Digests.ComputeJsonDescriptorAsync(configurationTemplate).ConfigureAwait(false);
IBuildableManifestTemplate manifestTemplate =
imageToJsonTranslator.GetManifestTemplate(
targetFormat, containerConfigurationBlobDescriptor);
DescriptorDigest imageDigest =
await Digests.ComputeJsonDigestAsync(manifestTemplate).ConfigureAwait(false);
DescriptorDigest imageId = containerConfigurationBlobDescriptor.GetDigest();
return(new BuildResult(imageDigest, imageId));
}
public void TestRetrieveContainerConfiguration()
{
SystemPath cacheDirectory = temporaryFolder.NewFolder().ToPath();
SetupCachedMetadataV22(cacheDirectory);
CacheStorageFiles cacheStorageFiles = new CacheStorageFiles(cacheDirectory);
CacheStorageReader cacheStorageReader = new CacheStorageReader(cacheStorageFiles);
ContainerConfigurationTemplate configurationTemplate =
cacheStorageReader
.RetrieveMetadata(ImageReference.Of("test", "image", "tag"))
.Get()
.GetConfig()
.Get();
Assert.AreEqual("wasm", configurationTemplate.Architecture);
Assert.AreEqual("js", configurationTemplate.Os);
}
/**
* Saves the manifest and container configuration for a V2.2 or OCI image.
*
* @param imageReference the image reference to store the metadata for
* @param manifestTemplate the manifest
* @param containerConfiguration the container configuration
*/
public async Task WriteMetadataAsync(
IImageReference imageReference,
IBuildableManifestTemplate manifestTemplate,
ContainerConfigurationTemplate containerConfiguration)
{
manifestTemplate = manifestTemplate ?? throw new ArgumentNullException(nameof(manifestTemplate));
Preconditions.CheckNotNull(manifestTemplate.GetContainerConfiguration());
Preconditions.CheckNotNull(manifestTemplate.GetContainerConfiguration().Digest);
SystemPath imageDirectory = cacheStorageFiles.GetImageDirectory(imageReference);
Files.CreateDirectories(imageDirectory);
using (LockFile ignored1 = LockFile.Create(imageDirectory.Resolve("lock")))
{
await WriteMetadataAsync(manifestTemplate, imageDirectory.Resolve("manifest.json")).ConfigureAwait(false);
await WriteMetadataAsync(containerConfiguration, imageDirectory.Resolve("config.json")).ConfigureAwait(false);
}
}
public void TestFromJson()
{
// Loads the JSON string.
SystemPath jsonFile = Paths.Get(TestResources.GetResource("core/json/v21manifest.json").ToURI());
// Deserializes into a manifest JSON object.
V21ManifestTemplate manifestJson =
JsonTemplateMapper.ReadJsonFromFile <V21ManifestTemplate>(jsonFile);
Assert.AreEqual(
DescriptorDigest.FromDigest(
"sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad"),
manifestJson.FsLayers[0].GetDigest());
ContainerConfigurationTemplate containerConfiguration =
manifestJson.GetContainerConfiguration().OrElse(null);
Assert.AreEqual(
new[] { "JAVA_HOME=/opt/openjdk", "PATH=/opt/openjdk/bin" },
containerConfiguration.GetContainerEnvironment());
Assert.AreEqual(
new[] { "/opt/openjdk/bin/java" }, containerConfiguration.GetContainerEntrypoint());
}
public async Task TestWriteMetadata_v22Async()
{
SystemPath containerConfigurationJsonFile =
Paths.Get(
TestResources.GetResource("core/json/containerconfig.json").ToURI());
ContainerConfigurationTemplate containerConfigurationTemplate =
JsonTemplateMapper.ReadJsonFromFile <ContainerConfigurationTemplate>(
containerConfigurationJsonFile);
SystemPath manifestJsonFile =
Paths.Get(TestResources.GetResource("core/json/v22manifest.json").ToURI());
IBuildableManifestTemplate manifestTemplate =
JsonTemplateMapper.ReadJsonFromFile <V22ManifestTemplate>(manifestJsonFile);
ImageReference imageReference = ImageReference.Parse("image.reference/project/thing:tag");
await new CacheStorageWriter(cacheStorageFiles)
.WriteMetadataAsync(imageReference, manifestTemplate, containerConfigurationTemplate).ConfigureAwait(false);
SystemPath savedManifestPath =
cacheRoot.Resolve("images/image.reference/project/thing!tag/manifest.json");
SystemPath savedConfigPath =
cacheRoot.Resolve("images/image.reference/project/thing!tag/config.json");
Assert.IsTrue(Files.Exists(savedManifestPath));
Assert.IsTrue(Files.Exists(savedConfigPath));
V22ManifestTemplate savedManifest =
JsonTemplateMapper.ReadJsonFromFile <V22ManifestTemplate>(savedManifestPath);
Assert.AreEqual(
"8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad",
savedManifest.GetContainerConfiguration().Digest.GetHash());
ContainerConfigurationTemplate savedContainerConfig =
JsonTemplateMapper.ReadJsonFromFile <ContainerConfigurationTemplate>(savedConfigPath);
Assert.AreEqual("wasm", savedContainerConfig.Architecture);
}
/**
* Pulls the base image.
*
* @param registryAuthorization authentication credentials to possibly use
* @param progressEventDispatcher the {@link ProgressEventDispatcher} for emitting {@link
* ProgressEvent}s
* @return the pulled image
* @throws IOException when an I/O exception occurs during the pulling
* @throws RegistryException if communicating with the registry caused a known error
* @throws LayerCountMismatchException if the manifest and configuration contain conflicting layer
* information
* @throws LayerPropertyNotFoundException if adding image layers fails
* @throws BadContainerConfigurationFormatException if the container configuration is in a bad
* format
*/
private async Task <Image> PullBaseImageAsync(
Authorization registryAuthorization,
ProgressEventDispatcher progressEventDispatcher)
{
RegistryClient registryClient =
buildConfiguration
.NewBaseImageRegistryClientFactory()
.SetAuthorization(registryAuthorization)
.NewRegistryClient();
IManifestTemplate manifestTemplate =
await registryClient.PullManifestAsync(buildConfiguration.GetBaseImageConfiguration().GetImageTag()).ConfigureAwait(false);
// TODO: Make schema version be enum.
switch (manifestTemplate.SchemaVersion)
{
case 1:
V21ManifestTemplate v21ManifestTemplate = (V21ManifestTemplate)manifestTemplate;
await buildConfiguration
.GetBaseImageLayersCache()
.WriteMetadataAsync(
buildConfiguration.GetBaseImageConfiguration().GetImage(), v21ManifestTemplate).ConfigureAwait(false);
return(JsonToImageTranslator.ToImage(v21ManifestTemplate));
case 2:
IBuildableManifestTemplate buildableManifestTemplate =
(IBuildableManifestTemplate)manifestTemplate;
if (buildableManifestTemplate.GetContainerConfiguration() == null ||
buildableManifestTemplate.GetContainerConfiguration().Digest == null)
{
throw new UnknownManifestFormatException(
"Invalid container configuration in Docker V2.2/OCI manifest: \n"
+ JsonTemplateMapper.ToUtf8String(buildableManifestTemplate));
}
DescriptorDigest containerConfigurationDigest =
buildableManifestTemplate.GetContainerConfiguration().Digest;
using (ThrottledProgressEventDispatcherWrapper progressEventDispatcherWrapper =
new ThrottledProgressEventDispatcherWrapper(
progressEventDispatcher.NewChildProducer(),
"pull container configuration " + containerConfigurationDigest))
{
string containerConfigurationString =
await Blobs.WriteToStringAsync(
registryClient.PullBlob(
containerConfigurationDigest,
progressEventDispatcherWrapper.SetProgressTarget,
progressEventDispatcherWrapper.DispatchProgress)).ConfigureAwait(false);
ContainerConfigurationTemplate containerConfigurationTemplate =
JsonTemplateMapper.ReadJson <ContainerConfigurationTemplate>(
containerConfigurationString);
await buildConfiguration
.GetBaseImageLayersCache()
.WriteMetadataAsync(
buildConfiguration.GetBaseImageConfiguration().GetImage(),
buildableManifestTemplate,
containerConfigurationTemplate).ConfigureAwait(false);
return(JsonToImageTranslator.ToImage(
buildableManifestTemplate, containerConfigurationTemplate));
}
}
throw new InvalidOperationException(Resources.PullBaseImageStepUnknownManifestErrorMessage);
}
/**
* Retrieves the cached manifest and container configuration for an image reference.
*
* @param imageReference the image reference
* @return the manifest and container configuration for the image reference, if found
* @throws IOException if an I/O exception occurs
* @throws CacheCorruptedException if the cache is corrupted
*/
public Maybe <ManifestAndConfig> RetrieveMetadata(IImageReference imageReference)
{
SystemPath imageDirectory = cacheStorageFiles.GetImageDirectory(imageReference);
SystemPath manifestPath = imageDirectory.Resolve("manifest.json");
if (!Files.Exists(manifestPath))
{
return(Maybe.Empty <ManifestAndConfig>());
}
// TODO: Consolidate with ManifestPuller
JToken token;
using (JsonTextReader reader = new JsonTextReader(File.OpenText(manifestPath)))
{
token = JToken.ReadFrom(reader);
}
if (!(token is JObject node))
{
throw new CacheCorruptedException(
cacheStorageFiles.GetCacheDirectory(),
Resources.CacheStorageReaderNotJsonExecpetionMessage);
}
if (!node.ContainsKey("schemaVersion"))
{
throw new CacheCorruptedException(
cacheStorageFiles.GetCacheDirectory(),
Resources.CacheStorageReaderSchemaVersionMissingExecpetionMessage);
}
int schemaVersion = node["schemaVersion"].Value <int>();
if (schemaVersion == -1)
{
throw new CacheCorruptedException(
cacheStorageFiles.GetCacheDirectory(),
Resources.CacheStorageReaderInvalidSchemaVersionExecpetionMessageFormat);
}
if (schemaVersion == 1)
{
return(Maybe.Of(
new ManifestAndConfig(
JsonTemplateMapper.ReadJsonFromFile <V21ManifestTemplate>(manifestPath),
null)));
}
if (schemaVersion == 2)
{
// 'schemaVersion' of 2 can be either Docker V2.2 or OCI.
string mediaType = node["mediaType"].Value <string>();
IManifestTemplate manifestTemplate;
if (V22ManifestTemplate.ManifestMediaType == mediaType)
{
manifestTemplate =
JsonTemplateMapper.ReadJsonFromFile <V22ManifestTemplate>(manifestPath);
}
else if (OCIManifestTemplate.ManifestMediaType == mediaType)
{
manifestTemplate =
JsonTemplateMapper.ReadJsonFromFile <OCIManifestTemplate>(manifestPath);
}
else
{
throw new CacheCorruptedException(
cacheStorageFiles.GetCacheDirectory(),
string.Format(
CultureInfo.CurrentCulture,
Resources.CacheStorageReaderUnknownMediaTypeExecpetionMessageFormat, mediaType));
}
SystemPath configPath = imageDirectory.Resolve("config.json");
if (!Files.Exists(configPath))
{
throw new CacheCorruptedException(
cacheStorageFiles.GetCacheDirectory(),
Resources.CacheStorageReaderContainerConfigurationMissingExecpetionMessage);
}
ContainerConfigurationTemplate config =
JsonTemplateMapper.ReadJsonFromFile <ContainerConfigurationTemplate>(configPath);
return(Maybe.Of(new ManifestAndConfig(manifestTemplate, config)));
}
throw new CacheCorruptedException(
cacheStorageFiles.GetCacheDirectory(),
string.Format(
CultureInfo.CurrentCulture,
Resources.CacheStorageReaderInvalidSchemaVersionExecpetionMessageFormat, schemaVersion));
}
private void TestToImage_buildable <T>(
string jsonFilename) where T : IBuildableManifestTemplate
{
// Loads the container configuration JSON.
SystemPath containerConfigurationJsonFile =
Paths.Get(
TestResources.GetResource("core/json/containerconfig.json").ToURI());
ContainerConfigurationTemplate containerConfigurationTemplate =
JsonTemplateMapper.ReadJsonFromFile <ContainerConfigurationTemplate>(
containerConfigurationJsonFile);
// Loads the manifest JSON.
SystemPath manifestJsonFile =
Paths.Get(TestResources.GetResource(jsonFilename).ToURI());
T manifestTemplate =
JsonTemplateMapper.ReadJsonFromFile <T>(manifestJsonFile);
Image image = JsonToImageTranslator.ToImage(manifestTemplate, containerConfigurationTemplate);
IList <ILayer> layers = image.GetLayers();
Assert.AreEqual(1, layers.Count);
Assert.AreEqual(
new BlobDescriptor(
1000000,
DescriptorDigest.FromDigest(
"sha256:4945ba5011739b0b98c4a41afe224e417f47c7c99b2ce76830999c9a0861b236")),
layers[0].GetBlobDescriptor());
Assert.AreEqual(
DescriptorDigest.FromDigest(
"sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad"),
layers[0].GetDiffId());
CollectionAssert.AreEqual(
ImmutableArray.Create(
HistoryEntry.CreateBuilder()
.SetCreationTimestamp(Instant.FromUnixTimeSeconds(0))
.SetAuthor("Bazel")
.SetCreatedBy("bazel build ...")
.SetEmptyLayer(true)
.Build(),
HistoryEntry.CreateBuilder()
.SetCreationTimestamp(Instant.FromUnixTimeSeconds(20))
.SetAuthor("Fib")
.SetCreatedBy("fib")
.Build()),
image.GetHistory());
Assert.AreEqual(Instant.FromUnixTimeSeconds(20), image.GetCreated());
Assert.AreEqual(new[] { "some", "entrypoint", "command" }, image.GetEntrypoint());
Assert.AreEqual(ImmutableDic.Of("VAR1", "VAL1", "VAR2", "VAL2"), image.GetEnvironment());
Assert.AreEqual("/some/workspace", image.GetWorkingDirectory());
Assert.AreEqual(
ImmutableHashSet.Create(Port.Tcp(1000), Port.Tcp(2000), Port.Udp(3000)), image.GetExposedPorts());
Assert.AreEqual(
ImmutableHashSet.Create(
AbsoluteUnixPath.Get("/var/job-result-data"),
AbsoluteUnixPath.Get("/var/log/my-app-logs")),
image.GetVolumes());
Assert.AreEqual("tomcat", image.GetUser());
Assert.AreEqual("value1", image.GetLabels()["key1"]);
Assert.AreEqual("value2", image.GetLabels()["key2"]);
Assert.AreEqual(2, image.GetLabels().Count);
}
