public async ValueTask ExecuteAsync(IConsole console)
{
var includedDirs = Dirs?.UnJoin('|');
var dirs = new[] { "videos", "recs", "captions" }.Where(d => includedDirs == null || includedDirs.Contains(d));
var store = Stores.Store(DataStoreType.DbStage);
foreach (var dir in dirs)
{
Log.Information("upgrade-partitions - {Dir} started", dir);
var files = await store.Files(dir, allDirectories : true).SelectMany()
.Where(f => f.Path.Tokens.Count == 3) // only optimise from within partitions
.ToListAsync();
var plan = JsonlStoreExtensions.OptimisePlan(dir, files, Cfg.Optimise, Log);
if (plan.Count < 10) // if the plan is small, run locally, otherwise on many machines
{
await store.Optimise(Cfg.Optimise, plan, Log);
}
else
{
await plan.Process(Ctx,
b => Stage.ProcessOptimisePlan(b, store, PipeArg.Inject <ILogger>()),
new() { MaxParallel = 12, MinWorkItems = 1 },
log : Log, cancel : console.GetCancellationToken());
}
}
}
public async Task Append(IEnumerable <T> items, ILogger log = null)
{
log ??= Log;
await items.GroupBy(Partition).BlockDo(async g => {
var ts = g.Max(GetTs);
var path = JsonlStoreExtensions.FilePath(FilePath(g.Key), ts, Version);
using var memStream = await g.ToJsonlGzStream(IJsonlStore.JCfg);
await Store.Save(path, memStream, log).WithDuration();
}, Parallel);
}
async Task UpdateRecs_0to1()
{
var toUpgrade = await FilesToUpgrade("recs", 0);
V0UpdateTime = DateTime.Parse("2019-11-02T13:50:00Z").ToUniversalTime();
await toUpgrade.BlockAction(async f => {
var existingJs = await Jsonl(f);
var upgradedJs = existingJs.GroupBy(j => j["FromVideoId"].Value <string>()).SelectMany(g => {
return(g.Select((j, i) => {
var newJ = j.DeepClone();
newJ["Updated"] = V0UpdateTime;
newJ["Rank"] = i + 1;
return newJ;
}));
});
var newPath = JsonlStoreExtensions.FilePath(f.Path.Parent, V0UpdateTime.FileSafeTimestamp(), "1");
await ReplaceJsonLFile(f, newPath, upgradedJs);
}, Cfg.DefaultParallel);
}
static StringPath NewFilePath(StoreFileMd f, int version) => JsonlStoreExtensions.FilePath(f.Path.Parent, StoreFileMd.GetTs(f.Path), version.ToString());
