Example #1
0
        public async ValueTask ExecuteAsync(IConsole console)
        {
            var includedDirs = Dirs?.UnJoin('|');
            var dirs = new[] { "videos", "recs", "captions" }.Where(d => includedDirs == null || includedDirs.Contains(d));
            var store = Stores.Store(DataStoreType.DbStage);

            foreach (var dir in dirs)
            {
                Log.Information("upgrade-partitions - {Dir} started", dir);
                var files = await store.Files(dir, allDirectories : true).SelectMany()
                            .Where(f => f.Path.Tokens.Count == 3) // only optimise from within partitions
                            .ToListAsync();

                var plan = JsonlStoreExtensions.OptimisePlan(dir, files, Cfg.Optimise, Log);

                if (plan.Count < 10) // if the plan is small, run locally, otherwise on many machines
                {
                    await store.Optimise(Cfg.Optimise, plan, Log);
                }
                else
                {
                    await plan.Process(Ctx,
                                       b => Stage.ProcessOptimisePlan(b, store, PipeArg.Inject <ILogger>()),
                                       new() { MaxParallel = 12, MinWorkItems = 1 },
                                       log : Log, cancel : console.GetCancellationToken());
                }
            }
        }
Example #2
0
        public async ValueTask ExecuteAsync(IConsole console)
        {
            console.GetCancellationToken().Register(() => Log.Information("Cancellation requested"));
            var options = new UpdateOptions {
                Actions  = Actions?.UnJoin('|'),
                Channels = Channels?.UnJoin('|'),
                Parts    = Parts?.UnJoin('|').Select(p => p.ParseEnum <CollectPart>()).ToArray(),
                Tables   = Tables?.UnJoin('|'),
                Results  = Results?.UnJoin('|'),
                Indexes  = Indexes?.UnJoin('|'),
                FullLoad = FullLoad,
                DisableChannelDiscover = DisableChannelDiscover,
                SearchConditions       = SearchConditions?.UnJoin('|').Select(t => {
                    var(index, condition, _) = t.UnJoin(':');
                    return(index, condition);
                }).ToArray(),
                SearchIndexes      = SearchIndexes?.UnJoin('|'),
                UserScrapeInit     = UserScrapeInit,
                UserScrapeTrial    = UserScrapeTrial,
                UserScrapeAccounts = UserScrapeAccounts?.UnJoin('|')
            };

            await PipeCtx.Run((YtUpdater u) => u.Update(options, PipeArg.Inject <CancellationToken>()),
                              new PipeRunOptions { Location = Location, Exclusive = true }, Log, console.GetCancellationToken());
        }
    }
Example #3
0
        public async ValueTask ExecuteAsync(IConsole console)
        {
            var channels = ChannelIds?.UnJoin('|').ToArray();

            // make a new app context with a custom region defined
            var appCtx = new PipeAppCtx(AppCtx)
            {
                CustomRegion = () => Rand.Choice(Regions)
            };

            // run the work using the pipe entry point, forced to be local
            PipeAppCfg.Location = PipeRunLocation.Local;
            var pipeCtx = new PipeCtx(PipeAppCfg, appCtx, PipeCtx.Store, PipeCtx.Log);
            await pipeCtx.Run((YtCollector d) => d.Collect(PipeArg.Inject <ILogger>(), ForceUpdate, false, channels, PipeArg.Inject <CancellationToken>()));
        }