public CollectCmd(AppCfg cfg, PipeAppCtx appCtx, PipeAppCfg pipeAppCfg, IPipeCtx pipeCtx)
{
Cfg = cfg;
AppCtx = appCtx;
PipeAppCfg = pipeAppCfg;
PipeCtx = pipeCtx;
}
public UpgradePartitionsCmd(BlobStores stores, ILogger log, WarehouseCfg cfg, Stage stage, IPipeCtx ctx)
{
Stores = stores;
Log = log;
Cfg = cfg;
Stage = stage;
Ctx = ctx;
}
public YtCollector(BlobStores stores, AppCfg cfg, SnowflakeConnectionProvider sf, IPipeCtx pipeCtx, YtWeb ytWeb,
YtClient api, ILogger log)
{
DbStore = new(stores.Store(DataStoreType.DbStage), log);
Cfg = cfg;
Sf = sf;
PipeCtx = pipeCtx;
Scraper = ytWeb;
Api = api;
}
public YtCollector(YtStore store, AppCfg cfg, SnowflakeConnectionProvider sf, IPipeCtx pipeCtx, WebScraper webScraper, ChromeScraper chromeScraper,
YtClient api)
{
Store = store;
Cfg = cfg;
Sf = sf;
PipeCtx = pipeCtx;
Scraper = webScraper;
ChromeScraper = chromeScraper;
Api = api;
}
/// <summary>Run a batch of containers. Must have already created state for them. Waits till the batch is complete and
/// returns the status.</summary>
public async Task <IReadOnlyCollection <PipeRunMetadata> > Launch(IPipeCtx ctx, IReadOnlyCollection <PipeRunId> ids, bool returnOnRunning,
bool exclusive, ILogger log, CancellationToken cancel)
{
var res = await ids.BlockFunc(async runId => {
var runCfg = runId.PipeCfg(ctx.PipeCfg); // id is for the sub-pipe, ctx is for the root
var tag = await FindPipeTag(runCfg.Container.ImageName);
var fullImageName = runCfg.Container.FullContainerImageName(tag);
var pipeLog = log.ForContext("Image", fullImageName).ForContext("Pipe", runId.Name);
var containerGroup = runId.ContainerGroupName(exclusive, Version);
var containerName = runCfg.Container.ImageName.ToLowerInvariant();
var(group, launchDur) = await Launch(runCfg.Container, containerGroup, containerName,
fullImageName, ctx.AppCtx.EnvironmentVariables,
runId.PipeArgs(), returnOnRunning, ctx.AppCtx.CustomRegion, pipeLog, cancel).WithDuration();
var logTxt = await group.GetLogContentAsync(containerName);
var logPath = new StringPath($"{runId.StatePath()}.log.txt");
var launchState = group.State();
var errorMsg = launchState.In(ContainerState.Failed, ContainerState.Terminated, ContainerState.Unknown)
? $"The container is in an error state '{group.State}', see {logPath}"
: null;
if (errorMsg.HasValue())
{
pipeLog.Error("{RunId} - failed: {Log}", runId.ToString(), logTxt);
}
var md = new PipeRunMetadata {
Id = runId,
Duration = launchDur,
Containers = group.Containers.Select(c => c.Value).ToArray(),
RawState = group.State,
State = group.State(),
RunCfg = runCfg,
ErrorMessage = errorMsg
};
await Task.WhenAll(
ctx.Store.Save(logPath, logTxt.AsStream(), pipeLog),
md.Save(ctx.Store, pipeLog));
// delete succeeded non-exclusive containers. Failed, and rutned on running will be cleaned up by another process
if (group.State() == ContainerState.Succeeded && !(exclusive && runId.Num > 0))
{
await DeleteContainer(containerGroup, log);
}
return(md);
}, ctx.PipeCfg.Azure.Parallel);
return(res);
}
public async Task <IReadOnlyCollection <PipeRunMetadata> > Launch(IPipeCtx ctx, IReadOnlyCollection <PipeRunId> ids, ILogger log, CancellationToken cancel)
{
var res = await ids.BlockFunc(async id => {
await ctx.DoPipeWork(id, cancel);
var md = new PipeRunMetadata {
Id = id
};
await md.Save(ctx.Store, log);
return(md);
});
return(res);
}
public PipeApp(ILogger log, IPipeCtx ctx)
{
Log = log;
Ctx = ctx;
}
public UpdateCmd(YtUpdater updater, IPipeCtx pipeCtx, ILogger log)
{
Updater = updater;
PipeCtx = pipeCtx;
Log = log;
}
public static async Task <(PipeRunMetadata Metadata, TOut State)> Run <TOut, TPipeInstance>(this IPipeCtx ctx,
Expression <Func <TPipeInstance, Task <TOut> > > expression, PipeRunOptions options = null, ILogger log = null)
{
public ContainerLauncher(PipeAppCfg cfg, IPipeCtx ctx)
{
Cfg = cfg;
Ctx = ctx;
}
public Task <IReadOnlyCollection <PipeRunMetadata> > Launch(IPipeCtx ctx, IReadOnlyCollection <PipeRunId> ids, ILogger log, CancellationToken cancel) => Launch(ctx, ids, returnOnRunning: false, exclusive: false, log: log, cancel: cancel);
public PipeCmd(IPipeCtx pipeCtx, ILogger log)
{
PipeCtx = pipeCtx;
Log = log;
}