public async ValueTask ExecuteAsync(IConsole console) { var includedDirs = Dirs?.UnJoin('|'); var dirs = new[] { "videos", "recs", "captions" }.Where(d => includedDirs == null || includedDirs.Contains(d)); var store = Stores.Store(DataStoreType.DbStage); foreach (var dir in dirs) { Log.Information("upgrade-partitions - {Dir} started", dir); var files = await store.Files(dir, allDirectories : true).SelectMany() .Where(f => f.Path.Tokens.Count == 3) // only optimise from within partitions .ToListAsync(); var plan = JsonlStoreExtensions.OptimisePlan(dir, files, Cfg.Optimise, Log); if (plan.Count < 10) // if the plan is small, run locally, otherwise on many machines { await store.Optimise(Cfg.Optimise, plan, Log); } else { await plan.Process(Ctx, b => Stage.ProcessOptimisePlan(b, store, PipeArg.Inject <ILogger>()), new() { MaxParallel = 12, MinWorkItems = 1 }, log : Log, cancel : console.GetCancellationToken()); } } }
public async Task Append(IEnumerable <T> items, ILogger log = null) { log ??= Log; await items.GroupBy(Partition).BlockDo(async g => { var ts = g.Max(GetTs); var path = JsonlStoreExtensions.FilePath(FilePath(g.Key), ts, Version); using var memStream = await g.ToJsonlGzStream(IJsonlStore.JCfg); await Store.Save(path, memStream, log).WithDuration(); }, Parallel); }
async Task UpdateRecs_0to1() { var toUpgrade = await FilesToUpgrade("recs", 0); V0UpdateTime = DateTime.Parse("2019-11-02T13:50:00Z").ToUniversalTime(); await toUpgrade.BlockAction(async f => { var existingJs = await Jsonl(f); var upgradedJs = existingJs.GroupBy(j => j["FromVideoId"].Value <string>()).SelectMany(g => { return(g.Select((j, i) => { var newJ = j.DeepClone(); newJ["Updated"] = V0UpdateTime; newJ["Rank"] = i + 1; return newJ; })); }); var newPath = JsonlStoreExtensions.FilePath(f.Path.Parent, V0UpdateTime.FileSafeTimestamp(), "1"); await ReplaceJsonLFile(f, newPath, upgradedJs); }, Cfg.DefaultParallel); }
static StringPath NewFilePath(StoreFileMd f, int version) => JsonlStoreExtensions.FilePath(f.Path.Parent, StoreFileMd.GetTs(f.Path), version.ToString());