public async ValueTask ExecuteAsync(IConsole console)
{
var includedDirs = Dirs?.UnJoin('|');
var dirs = new[] { "videos", "recs", "captions" }.Where(d => includedDirs == null || includedDirs.Contains(d));
var store = Stores.Store(DataStoreType.DbStage);
foreach (var dir in dirs)
{
Log.Information("upgrade-partitions - {Dir} started", dir);
var files = await store.Files(dir, allDirectories : true).SelectMany()
.Where(f => f.Path.Tokens.Count == 3) // only optimise from within partitions
.ToListAsync();
var plan = JsonlStoreExtensions.OptimisePlan(dir, files, Cfg.Optimise, Log);
if (plan.Count < 10) // if the plan is small, run locally, otherwise on many machines
{
await store.Optimise(Cfg.Optimise, plan, Log);
}
else
{
await plan.Process(Ctx,
b => Stage.ProcessOptimisePlan(b, store, PipeArg.Inject <ILogger>()),
new() { MaxParallel = 12, MinWorkItems = 1 },
log : Log, cancel : console.GetCancellationToken());
}
}
}
public async ValueTask ExecuteAsync(IConsole console)
{
console.GetCancellationToken().Register(() => Log.Information("Cancellation requested"));
var options = new UpdateOptions {
Actions = Actions?.UnJoin('|'),
Channels = Channels?.UnJoin('|'),
Parts = Parts?.UnJoin('|').Select(p => p.ParseEnum <CollectPart>()).ToArray(),
Tables = Tables?.UnJoin('|'),
Results = Results?.UnJoin('|'),
Indexes = Indexes?.UnJoin('|'),
FullLoad = FullLoad,
DisableChannelDiscover = DisableChannelDiscover,
SearchConditions = SearchConditions?.UnJoin('|').Select(t => {
var(index, condition, _) = t.UnJoin(':');
return(index, condition);
}).ToArray(),
SearchIndexes = SearchIndexes?.UnJoin('|'),
UserScrapeInit = UserScrapeInit,
UserScrapeTrial = UserScrapeTrial,
UserScrapeAccounts = UserScrapeAccounts?.UnJoin('|')
};
await PipeCtx.Run((YtUpdater u) => u.Update(options, PipeArg.Inject <CancellationToken>()),
new PipeRunOptions { Location = Location, Exclusive = true }, Log, console.GetCancellationToken());
}
}
public async ValueTask ExecuteAsync(IConsole console)
{
var channels = ChannelIds?.UnJoin('|').ToArray();
// make a new app context with a custom region defined
var appCtx = new PipeAppCtx(AppCtx)
{
CustomRegion = () => Rand.Choice(Regions)
};
// run the work using the pipe entry point, forced to be local
PipeAppCfg.Location = PipeRunLocation.Local;
var pipeCtx = new PipeCtx(PipeAppCfg, appCtx, PipeCtx.Store, PipeCtx.Log);
await pipeCtx.Run((YtCollector d) => d.Collect(PipeArg.Inject <ILogger>(), ForceUpdate, false, channels, PipeArg.Inject <CancellationToken>()));
}