public async Task <BlobDescriptor> CallAsync() { Authorization authorization = await authenticatePushStep.GetFuture().ConfigureAwait(false); using (ProgressEventDispatcher progressEventDispatcher = progressEventDipatcherFactory.Create( "pushing blob " + blobDescriptor.GetDigest(), blobDescriptor.GetSize())) using (TimerEventDispatcher ignored = new TimerEventDispatcher( buildConfiguration.GetEventHandlers(), DESCRIPTION + blobDescriptor)) using (ThrottledAccumulatingConsumer throttledProgressReporter = new ThrottledAccumulatingConsumer(progressEventDispatcher.DispatchProgress)) { RegistryClient registryClient = buildConfiguration .NewTargetImageRegistryClientFactory() .SetAuthorization(authorization) .NewRegistryClient(); // check if the BLOB is available if (await registryClient.CheckBlobAsync(blobDescriptor).ConfigureAwait(false)) { buildConfiguration .GetEventHandlers() .Dispatch(LogEvent.Info("BLOB : " + blobDescriptor + " already exists on registry")); return(blobDescriptor); } // todo: leverage cross-repository mounts await registryClient.PushBlobAsync(blobDescriptor.GetDigest(), blob, null, throttledProgressReporter.Accept).ConfigureAwait(false); return(blobDescriptor); } }
public async Task <IReadOnlyList <ICachedLayer> > CallAsync() { BaseImageWithAuthorization pullBaseImageStepResult = await pullBaseImageStep.GetFuture().ConfigureAwait(false); ImmutableArray <ILayer> baseImageLayers = pullBaseImageStepResult.GetBaseImage().GetLayers(); var checkIndex = 0; using (var progressEventDispatcher = progressEventDispatcherFactory.Create( "checking base image layers", this.Index)) using (var factory = progressEventDispatcher.NewChildProducer()("[child progress]checking base image layers", baseImageLayers.Length)) using (TimerEventDispatcher ignored = new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), DESCRIPTION)) { List <Task <ICachedLayer> > pullAndCacheBaseImageLayerStepsBuilder = new List <Task <ICachedLayer> >(); foreach (ILayer layer in baseImageLayers) { checkIndex++; pullAndCacheBaseImageLayerStepsBuilder.Add( new PullAndCacheBaseImageLayerStep( buildConfiguration, factory.NewChildProducer(), layer.GetBlobDescriptor().GetDigest(), pullBaseImageStepResult.GetBaseImageAuthorization()) { Index = checkIndex }.GetFuture()); } return(await Task.WhenAll(pullAndCacheBaseImageLayerStepsBuilder).ConfigureAwait(false)); } }
/** * Makes a list of {@link BuildAndCacheApplicationLayerStep} for dependencies, resources, and * classes layers. Optionally adds an extra layer if configured to do so. */ public static IAsyncStep <IReadOnlyList <ICachedLayer> > MakeList( IBuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory) { buildConfiguration = buildConfiguration ?? throw new ArgumentNullException(nameof(buildConfiguration)); int layerCount = buildConfiguration.GetLayerConfigurations().Length; using (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.Create( "setting up to build application layers", layerCount)) using (TimerEventDispatcher ignored = new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), Description)) { List <Task <ICachedLayer> > buildAndCacheApplicationLayerSteps = new List <Task <ICachedLayer> >(); foreach (LayerConfiguration layerConfiguration in buildConfiguration.GetLayerConfigurations()) { // Skips the layer if empty. if (layerConfiguration.LayerEntries.Length == 0) { continue; } buildAndCacheApplicationLayerSteps.Add( new BuildAndCacheApplicationLayerStep( buildConfiguration, progressEventDispatcher.NewChildProducer(), layerConfiguration.Name, layerConfiguration).GetFuture()); } return(AsyncSteps.FromTasks(buildAndCacheApplicationLayerSteps)); } }
public async Task <ICachedLayer> CallAsync() { string description = "Building " + layerType + " layer"; buildConfiguration.GetEventHandlers().Dispatch(LogEvent.Progress(description + "...")); using (ProgressEventDispatcher ignored = progressEventDispatcherFactory.Create("building " + layerType + " layer", 1)) using (TimerEventDispatcher ignored2 = new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), description)) { LayersCache cache = buildConfiguration.GetApplicationLayersCache(); // Don't build the layer if it exists already. Maybe <CachedLayer> optionalCachedLayer = await cache.RetrieveAsync(layerConfiguration.LayerEntries).ConfigureAwait(false); if (optionalCachedLayer.IsPresent()) { return(new CachedLayerWithType(optionalCachedLayer.Get(), GetLayerType())); } IBlob layerBlob = new ReproducibleLayerBuilder(layerConfiguration.LayerEntries).Build(); CachedLayer cachedLayer = await cache.WriteUncompressedLayerAsync(layerBlob, layerConfiguration.LayerEntries).ConfigureAwait(false); buildConfiguration .GetEventHandlers() .Dispatch(LogEvent.Debug(description + " built " + cachedLayer.GetDigest())); return(new CachedLayerWithType(cachedLayer, GetLayerType())); } }
public async Task <IReadOnlyList <BlobDescriptor> > CallAsync() { IReadOnlyList <ICachedLayer> cachedLayers = await cachedLayerStep.GetFuture().ConfigureAwait(false); var idx = 0; using (var progressEventDispatcher = progressEventDispatcherFactory.Create("setting up to push layers", this.Index)) using (TimerEventDispatcher ignored = new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), DESCRIPTION)) { using (var factory = progressEventDispatcher.NewChildProducer()("[child progress]setting up to push layers", cachedLayers.Count)) { // Constructs a PushBlobStep for each layer. var pushBlobSteps = new List <Task <BlobDescriptor> >(); foreach (ICachedLayer cachedLayer in cachedLayers) { idx++; pushBlobSteps.Add(PushBlobAsync(cachedLayer, factory.NewChildProducer(), idx)); } return(await Task.WhenAll(pushBlobSteps).ConfigureAwait(false)); } } }
public async Task <BuildResult> CallAsync() { IReadOnlyList <BlobDescriptor> baseImageDescriptors = await pushBaseImageLayersStep.GetFuture().ConfigureAwait(false); IReadOnlyList <BlobDescriptor> appLayerDescriptors = await pushApplicationLayersStep.GetFuture().ConfigureAwait(false); BlobDescriptor containerConfigurationBlobDescriptor = await pushContainerConfigurationStep.GetFuture().ConfigureAwait(false); ImmutableHashSet <string> targetImageTags = buildConfiguration.GetAllTargetImageTags(); using (var progressEventDispatcher = progressEventDispatcherFactory.Create("pushing image manifest", this.Index)) using (var factory = progressEventDispatcher.NewChildProducer()("[child progress]pushing image manifest", targetImageTags.Count)) using (TimerEventDispatcher ignored = new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), DESCRIPTION)) { RegistryClient registryClient = buildConfiguration .NewTargetImageRegistryClientFactory() .SetAuthorization(await authenticatePushStep.GetFuture().ConfigureAwait(false)) .NewRegistryClient(); // Constructs the image. ImageToJsonTranslator imageToJsonTranslator = new ImageToJsonTranslator(await buildImageStep.GetFuture().ConfigureAwait(false)); // Gets the image manifest to push. IBuildableManifestTemplate manifestTemplate = imageToJsonTranslator.GetManifestTemplate( buildConfiguration.GetTargetFormat(), containerConfigurationBlobDescriptor); // Pushes to all target image tags. IList <Task <DescriptorDigest> > pushAllTagsFutures = new List <Task <DescriptorDigest> >(); var idx = 0; ProgressEventDispatcher.Factory progressEventDispatcherFactory = factory.NewChildProducer(); foreach (string tag in targetImageTags) { idx++; using (progressEventDispatcherFactory.Create("tagging with " + tag, idx)) { buildConfiguration.GetEventHandlers().Dispatch(LogEvent.Info("Tagging with " + tag + "...")); pushAllTagsFutures.Add(registryClient.PushManifestAsync(manifestTemplate, tag)); } } DescriptorDigest imageDigest = await Digests.ComputeJsonDigestAsync(manifestTemplate).ConfigureAwait(false); DescriptorDigest imageId = containerConfigurationBlobDescriptor.GetDigest(); BuildResult result = new BuildResult(imageDigest, imageId); await Task.WhenAll(pushAllTagsFutures).ConfigureAwait(false); return(result); } }
public void TestLogging() { EventHandlers eventHandlers = EventHandlers.CreateBuilder().Add <TimerEvent>(timerEventQueue.Enqueue).Build(); Mock.Get(mockClock).Setup(m => m.GetCurrentInstant()).Returns(Instant.FromUnixTimeSeconds(0)); using (TimerEventDispatcher parentTimerEventDispatcher = new TimerEventDispatcher(eventHandlers, "description", mockClock, null)) { Mock.Get(mockClock).Setup(m => m.GetCurrentInstant()).Returns(Instant.FromUnixTimeSeconds(0) + Duration.FromMilliseconds(1)); parentTimerEventDispatcher.Lap(); Mock.Get(mockClock).Setup(m => m.GetCurrentInstant()).Returns((Instant.FromUnixTimeSeconds(0) + Duration.FromMilliseconds(1)).PlusNanoseconds(1)); using (TimerEventDispatcher ignored = parentTimerEventDispatcher.SubTimer("child description")) { Mock.Get(mockClock).Setup(m => m.GetCurrentInstant()).Returns(Instant.FromUnixTimeSeconds(0) + Duration.FromMilliseconds(2)); // Laps on close. } } TimerEvent timerEvent = GetNextTimerEvent(); VerifyStartState(timerEvent); VerifyDescription(timerEvent, "description"); timerEvent = GetNextTimerEvent(); VerifyStateFirstLap(timerEvent, State.LAP); VerifyDescription(timerEvent, "description"); timerEvent = GetNextTimerEvent(); VerifyStartState(timerEvent); VerifyDescription(timerEvent, "child description"); timerEvent = GetNextTimerEvent(); VerifyStateFirstLap(timerEvent, State.FINISHED); VerifyDescription(timerEvent, "child description"); timerEvent = GetNextTimerEvent(); VerifyStateNotFirstLap(timerEvent, State.FINISHED); VerifyDescription(timerEvent, "description"); Assert.IsTrue(timerEventQueue.Count == 0); }
public async Task <ICachedLayer> CallAsync() { using (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.Create("checking base image layer " + layerDigest, 1)) using (TimerEventDispatcher ignored = new TimerEventDispatcher( buildConfiguration.GetEventHandlers(), string.Format(CultureInfo.CurrentCulture, Description, layerDigest))) { LayersCache cache = buildConfiguration.GetBaseImageLayersCache(); // Checks if the layer already exists in the cache. Maybe <CachedLayer> optionalCachedLayer = cache.Retrieve(layerDigest); if (optionalCachedLayer.IsPresent()) { return(optionalCachedLayer.Get()); } else if (buildConfiguration.IsOffline()) { throw new IOException( "Cannot run Fib in offline mode; local Fib cache for base image is missing image layer " + layerDigest + ". You may need to rerun Fib in online mode to re-download the base image layers."); } RegistryClient registryClient = buildConfiguration .NewBaseImageRegistryClientFactory() .SetAuthorization(pullAuthorization) .NewRegistryClient(); using (ThrottledProgressEventDispatcherWrapper progressEventDispatcherWrapper = new ThrottledProgressEventDispatcherWrapper( progressEventDispatcher.NewChildProducer(), "pulling base image layer " + layerDigest)) { return(await cache.WriteCompressedLayerAsync( registryClient.PullBlob( layerDigest, progressEventDispatcherWrapper.SetProgressTarget, progressEventDispatcherWrapper.DispatchProgress)).ConfigureAwait(false)); } } }
public async Task <BlobDescriptor> CallAsync() { Image image = await buildImageStep.GetFuture().ConfigureAwait(false); using (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.Create("pushing container configuration", this.Index)) using (TimerEventDispatcher ignored = new TimerEventDispatcher(buildConfiguration.GetEventHandlers(), DESCRIPTION)) { ContainerConfigurationTemplate containerConfiguration = new ImageToJsonTranslator(image).GetContainerConfiguration(); BlobDescriptor blobDescriptor = await Digests.ComputeJsonDescriptorAsync(containerConfiguration).ConfigureAwait(false); return(await new PushBlobStep( buildConfiguration, progressEventDispatcher.NewChildProducer(), authenticatePushStep, blobDescriptor, Blobs.FromJson(containerConfiguration)).GetFuture().ConfigureAwait(false)); } }
/** * Pushes the BLOB. If the {@code sourceRepository} is provided then the remote registry may skip * if the BLOB already exists on the registry. * * @param blobDigest the digest of the BLOB, used for existence-check * @param blob the BLOB to push * @param sourceRepository if pushing to the same registry then the source image, or {@code null} * otherwise; used to optimize the BLOB push * @param writtenByteCountListener listens on byte count written to the registry during the push * @return {@code true} if the BLOB already exists on the registry and pushing was skipped; false * if the BLOB was pushed * @throws IOException if communicating with the endpoint fails * @throws RegistryException if communicating with the endpoint fails */ public async Task <bool> PushBlobAsync( DescriptorDigest blobDigest, IBlob blob, string sourceRepository, Action <long> writtenByteCountListener) { BlobPusher blobPusher = new BlobPusher(registryEndpointRequestProperties, blobDigest, blob, sourceRepository); using (TimerEventDispatcher timerEventDispatcher = new TimerEventDispatcher(eventHandlers, "pushBlob")) { timerEventDispatcher.Lap("pushBlob POST " + blobDigest); // POST /v2/<name>/blobs/uploads/ OR // POST /v2/<name>/blobs/uploads/?mount={blob.digest}&from={sourceRepository} Uri patchLocation = await CallRegistryEndpointAsync(blobPusher.CreateInitializer()).ConfigureAwait(false); if (patchLocation == null) { // The BLOB exists already. return(true); } timerEventDispatcher.Lap("pushBlob PATCH " + blobDigest); // PATCH <Location> with BLOB Uri putLocation = await CallRegistryEndpointAsync(blobPusher.CreateWriter(patchLocation, writtenByteCountListener)).ConfigureAwait(false); Preconditions.CheckNotNull(putLocation); timerEventDispatcher.Lap("pushBlob PUT " + blobDigest); // PUT <Location>?digest={blob.digest} await CallRegistryEndpointAsync(blobPusher.CreateCommitter(putLocation)).ConfigureAwait(false); return(false); } }