public async Task Append(IEnumerable <T> items, ILogger log = null) { log ??= Log; await items.GroupBy(Partition).BlockDo(async g => { var ts = g.Max(GetTs); var path = JsonlStoreExtensions.FilePath(FilePath(g.Key), ts, Version); using var memStream = await g.ToJsonlGzStream(IJsonlStore.JCfg); await Store.Save(path, memStream, log).WithDuration(); }, Parallel); }
async Task UpdateRecs_0to1() { var toUpgrade = await FilesToUpgrade("recs", 0); V0UpdateTime = DateTime.Parse("2019-11-02T13:50:00Z").ToUniversalTime(); await toUpgrade.BlockAction(async f => { var existingJs = await Jsonl(f); var upgradedJs = existingJs.GroupBy(j => j["FromVideoId"].Value <string>()).SelectMany(g => { return(g.Select((j, i) => { var newJ = j.DeepClone(); newJ["Updated"] = V0UpdateTime; newJ["Rank"] = i + 1; return newJ; })); }); var newPath = JsonlStoreExtensions.FilePath(f.Path.Parent, V0UpdateTime.FileSafeTimestamp(), "1"); await ReplaceJsonLFile(f, newPath, upgradedJs); }, Cfg.DefaultParallel); }
static StringPath NewFilePath(StoreFileMd f, int version) => JsonlStoreExtensions.FilePath(f.Path.Parent, StoreFileMd.GetTs(f.Path), version.ToString());