public async Task <IReadOnlyList <ICachedLayer> > CallAsync()
{
BaseImageWithAuthorization pullBaseImageStepResult = await pullBaseImageStep.GetFuture().ConfigureAwait(false);
ImmutableArray <ILayer> baseImageLayers = pullBaseImageStepResult.GetBaseImage().GetLayers();
var checkIndex = 0;
using (var progressEventDispatcher = progressEventDispatcherFactory.Create(
"checking base image layers", this.Index))
using (var factory = progressEventDispatcher.NewChildProducer()("[child progress]checking base image layers", baseImageLayers.Length))
using (TimerEventDispatcher ignored =
new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), DESCRIPTION))
{
List <Task <ICachedLayer> > pullAndCacheBaseImageLayerStepsBuilder = new List <Task <ICachedLayer> >();
foreach (ILayer layer in baseImageLayers)
{
checkIndex++;
pullAndCacheBaseImageLayerStepsBuilder.Add(
new PullAndCacheBaseImageLayerStep(
buildConfiguration,
factory.NewChildProducer(),
layer.GetBlobDescriptor().GetDigest(),
pullBaseImageStepResult.GetBaseImageAuthorization())
{
Index = checkIndex
}.GetFuture());
}
return(await Task.WhenAll(pullAndCacheBaseImageLayerStepsBuilder).ConfigureAwait(false));
}
}
public async Task <BlobDescriptor> CallAsync()
{
Authorization authorization = await authenticatePushStep.GetFuture().ConfigureAwait(false);
using (ProgressEventDispatcher progressEventDispatcher =
progressEventDipatcherFactory.Create(
"pushing blob " + blobDescriptor.GetDigest(), blobDescriptor.GetSize()))
using (TimerEventDispatcher ignored =
new TimerEventDispatcher(
buildConfiguration.GetEventHandlers(), DESCRIPTION + blobDescriptor))
using (ThrottledAccumulatingConsumer throttledProgressReporter =
new ThrottledAccumulatingConsumer(progressEventDispatcher.DispatchProgress))
{
RegistryClient registryClient =
buildConfiguration
.NewTargetImageRegistryClientFactory()
.SetAuthorization(authorization)
.NewRegistryClient();
// check if the BLOB is available
if (await registryClient.CheckBlobAsync(blobDescriptor).ConfigureAwait(false))
{
buildConfiguration
.GetEventHandlers()
.Dispatch(LogEvent.Info("BLOB : " + blobDescriptor + " already exists on registry"));
return(blobDescriptor);
}
// todo: leverage cross-repository mounts
await registryClient.PushBlobAsync(blobDescriptor.GetDigest(), blob, null, throttledProgressReporter.Accept).ConfigureAwait(false);
return(blobDescriptor);
}
}
/**
* Makes a list of {@link BuildAndCacheApplicationLayerStep} for dependencies, resources, and
* classes layers. Optionally adds an extra layer if configured to do so.
*/
public static IAsyncStep <IReadOnlyList <ICachedLayer> > MakeList(
IBuildConfiguration buildConfiguration,
ProgressEventDispatcher.Factory progressEventDispatcherFactory)
{
buildConfiguration = buildConfiguration ?? throw new ArgumentNullException(nameof(buildConfiguration));
int layerCount = buildConfiguration.GetLayerConfigurations().Length;
using (ProgressEventDispatcher progressEventDispatcher =
progressEventDispatcherFactory.Create(
"setting up to build application layers", layerCount))
using (TimerEventDispatcher ignored =
new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), Description))
{
List <Task <ICachedLayer> > buildAndCacheApplicationLayerSteps = new List <Task <ICachedLayer> >();
foreach (LayerConfiguration layerConfiguration in buildConfiguration.GetLayerConfigurations())
{
// Skips the layer if empty.
if (layerConfiguration.LayerEntries.Length == 0)
{
continue;
}
buildAndCacheApplicationLayerSteps.Add(
new BuildAndCacheApplicationLayerStep(
buildConfiguration,
progressEventDispatcher.NewChildProducer(),
layerConfiguration.Name,
layerConfiguration).GetFuture());
}
return(AsyncSteps.FromTasks(buildAndCacheApplicationLayerSteps));
}
}
public async Task <BuildResult> CallAsync()
{
await pullAndCacheBaseImageLayersStep.GetFuture().ConfigureAwait(false);
await buildAndCacheApplicationLayersStep.GetFuture().ConfigureAwait(false);
await buildImageStep.GetFuture().ConfigureAwait(false);
string description = string.Format(
CultureInfo.CurrentCulture,
Resources.WriteTarFileStepDescriptionFormat,
outputPath.GetFileName());
buildConfiguration.GetEventHandlers().Dispatch(LogEvent.Progress(description));
using (progressEventDispatcherFactory.Create(description, this.Index))
{
Image image = await buildImageStep.GetFuture().ConfigureAwait(false);
// Builds the image to a tarball.
Files.CreateDirectories(outputPath.GetParent());
using (Stream outputStream =
new BufferedStream(FileOperations.NewLockingOutputStream(outputPath)))
{
await new ImageTarball(image, buildConfiguration.GetTargetImageConfiguration().GetImage())
.WriteToAsync(outputStream).ConfigureAwait(false);
}
return(await BuildResult.FromImageAsync(image, buildConfiguration.GetTargetFormat()).ConfigureAwait(false));
}
}
public async Task <Authorization> CallAsync()
{
Credential registryCredential = await retrieveTargetRegistryCredentialsStep.GetFuture().ConfigureAwait(false);
string registry = buildConfiguration.GetTargetImageConfiguration().GetImageRegistry();
try
{
using (progressEventDispatcherFactory.Create("authenticating push to " + registry, 1))
using (new TimerEventDispatcher(
buildConfiguration.GetEventHandlers(), string.Format(CultureInfo.CurrentCulture, DESCRIPTION, registry)))
{
RegistryAuthenticator registryAuthenticator =
await buildConfiguration
.NewTargetImageRegistryClientFactory()
.NewRegistryClient()
.GetRegistryAuthenticatorAsync().ConfigureAwait(false);
if (registryAuthenticator != null)
{
return(await registryAuthenticator.AuthenticatePushAsync(registryCredential).ConfigureAwait(false));
}
}
}
catch (InsecureRegistryException)
{
// Cannot skip certificate validation or use HTTP; fall through.
}
return(registryCredential?.IsOAuth2RefreshToken() != false
? null
: Authorization.FromBasicCredentials(
registryCredential.GetUsername(), registryCredential.GetPassword()));
}
public async Task <ICachedLayer> CallAsync()
{
string description = "Building " + layerType + " layer";
buildConfiguration.GetEventHandlers().Dispatch(LogEvent.Progress(description + "..."));
using (ProgressEventDispatcher ignored =
progressEventDispatcherFactory.Create("building " + layerType + " layer", 1))
using (TimerEventDispatcher ignored2 =
new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), description))
{
LayersCache cache = buildConfiguration.GetApplicationLayersCache();
// Don't build the layer if it exists already.
Maybe <CachedLayer> optionalCachedLayer =
await cache.RetrieveAsync(layerConfiguration.LayerEntries).ConfigureAwait(false);
if (optionalCachedLayer.IsPresent())
{
return(new CachedLayerWithType(optionalCachedLayer.Get(), GetLayerType()));
}
IBlob layerBlob = new ReproducibleLayerBuilder(layerConfiguration.LayerEntries).Build();
CachedLayer cachedLayer =
await cache.WriteUncompressedLayerAsync(layerBlob, layerConfiguration.LayerEntries).ConfigureAwait(false);
buildConfiguration
.GetEventHandlers()
.Dispatch(LogEvent.Debug(description + " built " + cachedLayer.GetDigest()));
return(new CachedLayerWithType(cachedLayer, GetLayerType()));
}
}
public async Task <IReadOnlyList <BlobDescriptor> > CallAsync()
{
IReadOnlyList <ICachedLayer> cachedLayers = await cachedLayerStep.GetFuture().ConfigureAwait(false);
var idx = 0;
using (var progressEventDispatcher =
progressEventDispatcherFactory.Create("setting up to push layers", this.Index))
using (TimerEventDispatcher ignored =
new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), DESCRIPTION))
{
using (var factory =
progressEventDispatcher.NewChildProducer()("[child progress]setting up to push layers", cachedLayers.Count))
{
// Constructs a PushBlobStep for each layer.
var pushBlobSteps = new List <Task <BlobDescriptor> >();
foreach (ICachedLayer cachedLayer in cachedLayers)
{
idx++;
pushBlobSteps.Add(PushBlobAsync(cachedLayer, factory.NewChildProducer(), idx));
}
return(await Task.WhenAll(pushBlobSteps).ConfigureAwait(false));
}
}
}
public Credential Call()
{
string description = MakeDescription(registry);
buildConfiguration.GetEventHandlers().Dispatch(LogEvent.Progress(description + "..."));
using (progressEventDispatcherFactory.Create("retrieving credentials for " + registry, this.Index))
using (new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), description))
{
foreach (CredentialRetriever credentialRetriever in credentialRetrievers)
{
Maybe <Credential> optionalCredential = credentialRetriever.Retrieve();
if (optionalCredential.IsPresent())
{
buildConfiguration
.GetEventHandlers()
.Dispatch(LogEvent.Info("credentials was be retrieved for registry " + registry));
return(optionalCredential.Get());
}
}
// If no credentials found, give an info (not warning because in most cases, the base image is
// public and does not need extra credentials) and return null.
buildConfiguration
.GetEventHandlers()
.Dispatch(LogEvent.Info("No credentials could be retrieved for registry " + registry));
return(null);
}
}
public async Task <BuildResult> CallAsync()
{
IReadOnlyList <BlobDescriptor> baseImageDescriptors = await pushBaseImageLayersStep.GetFuture().ConfigureAwait(false);
IReadOnlyList <BlobDescriptor> appLayerDescriptors = await pushApplicationLayersStep.GetFuture().ConfigureAwait(false);
BlobDescriptor containerConfigurationBlobDescriptor = await pushContainerConfigurationStep.GetFuture().ConfigureAwait(false);
ImmutableHashSet <string> targetImageTags = buildConfiguration.GetAllTargetImageTags();
using (var progressEventDispatcher =
progressEventDispatcherFactory.Create("pushing image manifest", this.Index))
using (var factory =
progressEventDispatcher.NewChildProducer()("[child progress]pushing image manifest", targetImageTags.Count))
using (TimerEventDispatcher ignored =
new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), DESCRIPTION))
{
RegistryClient registryClient =
buildConfiguration
.NewTargetImageRegistryClientFactory()
.SetAuthorization(await authenticatePushStep.GetFuture().ConfigureAwait(false))
.NewRegistryClient();
// Constructs the image.
ImageToJsonTranslator imageToJsonTranslator =
new ImageToJsonTranslator(await buildImageStep.GetFuture().ConfigureAwait(false));
// Gets the image manifest to push.
IBuildableManifestTemplate manifestTemplate =
imageToJsonTranslator.GetManifestTemplate(
buildConfiguration.GetTargetFormat(), containerConfigurationBlobDescriptor);
// Pushes to all target image tags.
IList <Task <DescriptorDigest> > pushAllTagsFutures = new List <Task <DescriptorDigest> >();
var idx = 0;
ProgressEventDispatcher.Factory progressEventDispatcherFactory =
factory.NewChildProducer();
foreach (string tag in targetImageTags)
{
idx++;
using (progressEventDispatcherFactory.Create("tagging with " + tag, idx))
{
buildConfiguration.GetEventHandlers().Dispatch(LogEvent.Info("Tagging with " + tag + "..."));
pushAllTagsFutures.Add(registryClient.PushManifestAsync(manifestTemplate, tag));
}
}
DescriptorDigest imageDigest =
await Digests.ComputeJsonDigestAsync(manifestTemplate).ConfigureAwait(false);
DescriptorDigest imageId = containerConfigurationBlobDescriptor.GetDigest();
BuildResult result = new BuildResult(imageDigest, imageId);
await Task.WhenAll(pushAllTagsFutures).ConfigureAwait(false);
return(result);
}
}
public void SetProgressTarget(long allocationUnits)
{
if (progressEventDispatcher != null)
{
throw new InvalidOperationException(Resources.ThrottledProgressEventDispatcherWrapperResetExceptionMessage);
}
progressEventDispatcher = progressEventDispatcherFactory.Create(description, allocationUnits);
throttledDispatcher =
new ThrottledAccumulatingConsumer(progressEventDispatcher.DispatchProgress);
}
public async Task <ICachedLayer> CallAsync()
{
using (ProgressEventDispatcher progressEventDispatcher =
progressEventDispatcherFactory.Create("checking base image layer " + layerDigest, 1))
using (TimerEventDispatcher ignored =
new TimerEventDispatcher(
buildConfiguration.GetEventHandlers(), string.Format(CultureInfo.CurrentCulture, Description, layerDigest)))
{
LayersCache cache = buildConfiguration.GetBaseImageLayersCache();
// Checks if the layer already exists in the cache.
Maybe <CachedLayer> optionalCachedLayer = cache.Retrieve(layerDigest);
if (optionalCachedLayer.IsPresent())
{
return(optionalCachedLayer.Get());
}
else if (buildConfiguration.IsOffline())
{
throw new IOException(
"Cannot run Fib in offline mode; local Fib cache for base image is missing image layer "
+ layerDigest
+ ". You may need to rerun Fib in online mode to re-download the base image layers.");
}
RegistryClient registryClient =
buildConfiguration
.NewBaseImageRegistryClientFactory()
.SetAuthorization(pullAuthorization)
.NewRegistryClient();
using (ThrottledProgressEventDispatcherWrapper progressEventDispatcherWrapper =
new ThrottledProgressEventDispatcherWrapper(
progressEventDispatcher.NewChildProducer(),
"pulling base image layer " + layerDigest))
{
return(await cache.WriteCompressedLayerAsync(
registryClient.PullBlob(
layerDigest,
progressEventDispatcherWrapper.SetProgressTarget,
progressEventDispatcherWrapper.DispatchProgress)).ConfigureAwait(false));
}
}
}
public async Task <BuildResult> CallAsync()
{
await pullAndCacheBaseImageLayersStep.GetFuture().ConfigureAwait(false);
await buildAndCacheApplicationLayersStep.GetFuture().ConfigureAwait(false);
await buildImageStep.GetFuture().ConfigureAwait(false);
buildConfiguration
.GetEventHandlers()
.Dispatch(LogEvent.Progress(Resources.LoadDockerStepDescription));
using (progressEventDispatcherFactory.Create(Resources.LoadDockerStepDescription, 1))
{
Image image = await buildImageStep.GetFuture().ConfigureAwait(false);
IImageReference targetImageReference =
buildConfiguration.GetTargetImageConfiguration().GetImage();
// Load the image to docker daemon.
buildConfiguration
.GetEventHandlers()
.Dispatch(
LogEvent.Debug(await dockerClient.LoadAsync(new ImageTarball(image, targetImageReference)).ConfigureAwait(false)));
// Tags the image with all the additional tags, skipping the one 'docker load' already loaded.
foreach (string tag in buildConfiguration.GetAllTargetImageTags())
{
if (tag.Equals(targetImageReference.GetTag(), StringComparison.Ordinal))
{
continue;
}
ImageReference taggedImageReference = targetImageReference.WithTag(tag);
await dockerClient.TagAsync(targetImageReference, taggedImageReference).ConfigureAwait(false);
}
return(await BuildResult.FromImageAsync(image, buildConfiguration.GetTargetFormat()).ConfigureAwait(false));
}
}
public async Task <BlobDescriptor> CallAsync()
{
Image image = await buildImageStep.GetFuture().ConfigureAwait(false);
using (ProgressEventDispatcher progressEventDispatcher =
progressEventDispatcherFactory.Create("pushing container configuration", this.Index))
using (TimerEventDispatcher ignored =
new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), DESCRIPTION))
{
ContainerConfigurationTemplate containerConfiguration =
new ImageToJsonTranslator(image).GetContainerConfiguration();
BlobDescriptor blobDescriptor =
await Digests.ComputeJsonDescriptorAsync(containerConfiguration).ConfigureAwait(false);
return(await new PushBlobStep(
buildConfiguration,
progressEventDispatcher.NewChildProducer(),
authenticatePushStep,
blobDescriptor,
Blobs.FromJson(containerConfiguration)).GetFuture().ConfigureAwait(false));
}
}
public async Task <BaseImageWithAuthorization> CallAsync()
{
IEventHandlers eventHandlers = buildConfiguration.GetEventHandlers();
// Skip this step if this is a scratch image
ImageConfiguration baseImageConfiguration = buildConfiguration.GetBaseImageConfiguration();
string description = string.Format(
CultureInfo.CurrentCulture,
Resources.PullBaseImageStepDescriptionFormat,
buildConfiguration.GetBaseImageConfiguration().GetImage());
eventHandlers.Dispatch(LogEvent.Progress(description));
if (baseImageConfiguration.GetImage().IsScratch())
{
return(new BaseImageWithAuthorization(
Image.CreateBuilder(buildConfiguration.GetTargetFormat()).Build(), null));
}
if (buildConfiguration.IsOffline())
{
return(new BaseImageWithAuthorization(PullBaseImageOffline(), null));
}
using (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.Create(description, 2))
using (new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), description))
{
// First, try with no credentials.
try
{
return(new BaseImageWithAuthorization(await PullBaseImageAsync(null, progressEventDispatcher).ConfigureAwait(false), null));
}
catch (RegistryUnauthorizedException)
{
eventHandlers.Dispatch(
LogEvent.Lifecycle(
"The base image requires auth. Trying again for "
+ buildConfiguration.GetBaseImageConfiguration().GetImage()
+ "..."));
// If failed, then, retrieve base registry credentials and try with retrieved credentials.
// TODO: Refactor the logic in RetrieveRegistryCredentialsStep out to
// registry.credentials.RegistryCredentialsRetriever to avoid this direct executor hack.
RetrieveRegistryCredentialsStep retrieveBaseRegistryCredentialsStep =
RetrieveRegistryCredentialsStep.ForBaseImage(
buildConfiguration,
progressEventDispatcher.NewChildProducer());
Credential registryCredential = await retrieveBaseRegistryCredentialsStep.GetFuture().ConfigureAwait(false);
Authorization registryAuthorization =
registryCredential?.IsOAuth2RefreshToken() != false
? null
: Authorization.FromBasicCredentials(
registryCredential.GetUsername(), registryCredential.GetPassword());
try
{
return(new BaseImageWithAuthorization(
await PullBaseImageAsync(registryAuthorization, progressEventDispatcher).ConfigureAwait(false), registryAuthorization));
}
catch (RegistryUnauthorizedException)
{
// The registry requires us to authenticate using the Docker Token Authentication.
// See https://docs.docker.com/registry/spec/auth/token
try
{
RegistryAuthenticator registryAuthenticator =
await buildConfiguration
.NewBaseImageRegistryClientFactory()
.NewRegistryClient()
.GetRegistryAuthenticatorAsync().ConfigureAwait(false);
if (registryAuthenticator != null)
{
Authorization pullAuthorization =
await registryAuthenticator.AuthenticatePullAsync(registryCredential).ConfigureAwait(false);
return(new BaseImageWithAuthorization(
await PullBaseImageAsync(pullAuthorization, progressEventDispatcher).ConfigureAwait(false), pullAuthorization));
}
}
catch (InsecureRegistryException)
{
// Cannot skip certificate validation or use HTTP; fall through.
}
eventHandlers.Dispatch(LogEvent.Error(Resources.PullBaseImageStepAuthenticationErrorMessage));
throw;
}
}
}
}
public async Task <Image> CallAsync()
{
BaseImageWithAuthorization baseImageWithAuthorization = await pullBaseImageStep.GetFuture().ConfigureAwait(false);
IReadOnlyList <ICachedLayer> baseImageLayers = await pullAndCacheBaseImageLayersStep.GetFuture().ConfigureAwait(false);
IReadOnlyList <ICachedLayer> applicationLayers = await buildAndCacheApplicationLayersStep.GetFuture().ConfigureAwait(false);
using (progressEventDispatcherFactory.Create("building image format", 1))
using (new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), DESCRIPTION))
{
// Constructs the image.
Image.Builder imageBuilder = Image.CreateBuilder(buildConfiguration.GetTargetFormat());
Image baseImage = baseImageWithAuthorization.GetBaseImage();
IContainerConfiguration containerConfiguration =
buildConfiguration.GetContainerConfiguration();
// Base image layers
foreach (ICachedLayer pullAndCacheBaseImageLayer in baseImageLayers)
{
imageBuilder.AddLayer(pullAndCacheBaseImageLayer);
}
// Passthrough config and count non-empty history entries
int nonEmptyLayerCount = 0;
foreach (HistoryEntry historyObject in baseImage.GetHistory())
{
imageBuilder.AddHistory(historyObject);
if (!historyObject.HasCorrespondingLayer())
{
nonEmptyLayerCount++;
}
}
imageBuilder
.SetArchitecture(baseImage.GetArchitecture())
.SetOs(baseImage.GetOs())
.AddEnvironment(baseImage.GetEnvironment())
.AddLabels(baseImage.GetLabels())
.SetHealthCheck(baseImage.GetHealthCheck())
.AddExposedPorts(baseImage.GetExposedPorts())
.AddVolumes(baseImage.GetVolumes())
.SetWorkingDirectory(baseImage.GetWorkingDirectory());
// Add history elements for non-empty layers that don't have one yet
Instant layerCreationTime =
containerConfiguration == null
? ContainerConfiguration.DefaultCreationTime
: containerConfiguration.GetCreationTime();
for (int count = 0; count < baseImageLayers.Count - nonEmptyLayerCount; count++)
{
imageBuilder.AddHistory(
HistoryEntry.CreateBuilder()
.SetCreationTimestamp(layerCreationTime)
.SetComment("auto-generated by Fib")
.Build());
}
// Add built layers/configuration
foreach (ICachedLayer applicationLayer in applicationLayers)
{
HistoryEntry.Builder historyBuilder = HistoryEntry.CreateBuilder();
if (buildConfiguration.GetToolName() != null)
{
historyBuilder.SetCreatedBy(buildConfiguration.GetToolName() + ":" + (buildConfiguration.GetToolVersion() ?? "null"));
}
else
{
historyBuilder.SetCreatedBy(ProjectInfo.TOOL_NAME + ":" + ProjectInfo.VERSION);
}
imageBuilder
.AddLayer(applicationLayer)
.AddHistory(
historyBuilder
.SetCreationTimestamp(layerCreationTime)
.SetAuthor("Fib")
.SetComment(applicationLayer.GetLayerType())
.Build());
}
if (containerConfiguration != null)
{
imageBuilder
.AddEnvironment(containerConfiguration.GetEnvironmentMap())
.SetCreated(containerConfiguration.GetCreationTime())
.SetUser(containerConfiguration.GetUser())
.SetEntrypoint(ComputeEntrypoint(baseImage, containerConfiguration))
.SetProgramArguments(ComputeProgramArguments(baseImage, containerConfiguration))
.AddExposedPorts(containerConfiguration.GetExposedPorts())
.AddVolumes(containerConfiguration.GetVolumes())
.AddLabels(containerConfiguration.GetLabels());
if (containerConfiguration.GetWorkingDirectory() != null)
{
imageBuilder.SetWorkingDirectory(containerConfiguration.GetWorkingDirectory().ToString());
}
}
// Gets the container configuration content descriptor.
return(imageBuilder.Build());
}
}