Esempio n. 1
0
 public CollectCmd(AppCfg cfg, PipeAppCtx appCtx, PipeAppCfg pipeAppCfg, IPipeCtx pipeCtx)
 {
     Cfg        = cfg;
     AppCtx     = appCtx;
     PipeAppCfg = pipeAppCfg;
     PipeCtx    = pipeCtx;
 }
Esempio n. 2
0
 public UpgradePartitionsCmd(BlobStores stores, ILogger log, WarehouseCfg cfg, Stage stage, IPipeCtx ctx)
 {
     Stores = stores;
     Log    = log;
     Cfg    = cfg;
     Stage  = stage;
     Ctx    = ctx;
 }
Esempio n. 3
0
 public YtCollector(BlobStores stores, AppCfg cfg, SnowflakeConnectionProvider sf, IPipeCtx pipeCtx, YtWeb ytWeb,
                    YtClient api, ILogger log)
 {
     DbStore = new(stores.Store(DataStoreType.DbStage), log);
     Cfg     = cfg;
     Sf      = sf;
     PipeCtx = pipeCtx;
     Scraper = ytWeb;
     Api     = api;
 }
Esempio n. 4
0
 public YtCollector(YtStore store, AppCfg cfg, SnowflakeConnectionProvider sf, IPipeCtx pipeCtx, WebScraper webScraper, ChromeScraper chromeScraper,
                    YtClient api)
 {
     Store         = store;
     Cfg           = cfg;
     Sf            = sf;
     PipeCtx       = pipeCtx;
     Scraper       = webScraper;
     ChromeScraper = chromeScraper;
     Api           = api;
 }
Esempio n. 5
0
        /// <summary>Run a batch of containers. Must have already created state for them. Waits till the batch is complete and
        ///   returns the status.</summary>
        public async Task <IReadOnlyCollection <PipeRunMetadata> > Launch(IPipeCtx ctx, IReadOnlyCollection <PipeRunId> ids, bool returnOnRunning,
                                                                          bool exclusive, ILogger log, CancellationToken cancel)
        {
            var res = await ids.BlockFunc(async runId => {
                var runCfg = runId.PipeCfg(ctx.PipeCfg); // id is for the sub-pipe, ctx is for the root

                var tag           = await FindPipeTag(runCfg.Container.ImageName);
                var fullImageName = runCfg.Container.FullContainerImageName(tag);
                var pipeLog       = log.ForContext("Image", fullImageName).ForContext("Pipe", runId.Name);

                var containerGroup = runId.ContainerGroupName(exclusive, Version);
                var containerName  = runCfg.Container.ImageName.ToLowerInvariant();

                var(group, launchDur) = await Launch(runCfg.Container, containerGroup, containerName,
                                                     fullImageName, ctx.AppCtx.EnvironmentVariables,
                                                     runId.PipeArgs(), returnOnRunning, ctx.AppCtx.CustomRegion, pipeLog, cancel).WithDuration();

                var logTxt  = await group.GetLogContentAsync(containerName);
                var logPath = new StringPath($"{runId.StatePath()}.log.txt");

                var launchState = group.State();

                var errorMsg = launchState.In(ContainerState.Failed, ContainerState.Terminated, ContainerState.Unknown)
          ? $"The container is in an error state '{group.State}', see {logPath}"
          : null;
                if (errorMsg.HasValue())
                {
                    pipeLog.Error("{RunId} - failed: {Log}", runId.ToString(), logTxt);
                }

                var md = new PipeRunMetadata {
                    Id           = runId,
                    Duration     = launchDur,
                    Containers   = group.Containers.Select(c => c.Value).ToArray(),
                    RawState     = group.State,
                    State        = group.State(),
                    RunCfg       = runCfg,
                    ErrorMessage = errorMsg
                };
                await Task.WhenAll(
                    ctx.Store.Save(logPath, logTxt.AsStream(), pipeLog),
                    md.Save(ctx.Store, pipeLog));

                // delete succeeded non-exclusive containers. Failed, and rutned on running will be cleaned up by another process
                if (group.State() == ContainerState.Succeeded && !(exclusive && runId.Num > 0))
                {
                    await DeleteContainer(containerGroup, log);
                }

                return(md);
            }, ctx.PipeCfg.Azure.Parallel);

            return(res);
        }
Esempio n. 6
0
        public async Task <IReadOnlyCollection <PipeRunMetadata> > Launch(IPipeCtx ctx, IReadOnlyCollection <PipeRunId> ids, ILogger log, CancellationToken cancel)
        {
            var res = await ids.BlockFunc(async id => {
                await ctx.DoPipeWork(id, cancel);
                var md = new PipeRunMetadata {
                    Id = id
                };
                await md.Save(ctx.Store, log);
                return(md);
            });

            return(res);
        }
Esempio n. 7
0
 public PipeApp(ILogger log, IPipeCtx ctx)
 {
     Log = log;
     Ctx = ctx;
 }
Esempio n. 8
0
 public UpdateCmd(YtUpdater updater, IPipeCtx pipeCtx, ILogger log)
 {
     Updater = updater;
     PipeCtx = pipeCtx;
     Log     = log;
 }
Esempio n. 9
0
 public static async Task <(PipeRunMetadata Metadata, TOut State)> Run <TOut, TPipeInstance>(this IPipeCtx ctx,
                                                                                             Expression <Func <TPipeInstance, Task <TOut> > > expression, PipeRunOptions options = null, ILogger log = null)
 {
Esempio n. 10
0
 public ContainerLauncher(PipeAppCfg cfg, IPipeCtx ctx)
 {
     Cfg = cfg;
     Ctx = ctx;
 }
Esempio n. 11
0
 public Task <IReadOnlyCollection <PipeRunMetadata> > Launch(IPipeCtx ctx, IReadOnlyCollection <PipeRunId> ids, ILogger log, CancellationToken cancel) =>
 Launch(ctx, ids, returnOnRunning: false, exclusive: false, log: log, cancel: cancel);
Esempio n. 12
0
 public PipeCmd(IPipeCtx pipeCtx, ILogger log)
 {
     PipeCtx = pipeCtx;
     Log     = log;
 }