public FixReviewsCmd(SheetsCfg sheetsCfg, ILogger log, YtStore store, SnowflakeConnectionProvider sf) { SheetsCfg = sheetsCfg; Log = log; Store = store; Sf = sf; }
public YtCollector(BlobStores stores, AppCfg cfg, SnowflakeConnectionProvider sf, IPipeCtx pipeCtx, YtWeb ytWeb, YtClient api, ILogger log) { DbStore = new(stores.Store(DataStoreType.DbStage), log); Cfg = cfg; Sf = sf; PipeCtx = pipeCtx; Scraper = ytWeb; Api = api; }
public async Task SaveChannelRelationData() { var cfg = await Setup.LoadCfg(); var log = Setup.CreateTestLogger(); log.Information("started test"); var store = new YtStore(new YtClient(cfg.App, log), cfg.DataStore()); var analysis = new YtCollect(store, cfg.DataStore(cfg.App.Storage.AnalysisPath), cfg.App, log); await analysis.SaveChannelRelationData(); }
public async Task SaveCaptions() { var cfg = await Setup.LoadCfg(); var log = Setup.CreateTestLogger(); var videoIds = new[] { "P0dcyAXXmCw" }; var store = new YtStore(new YtClient(cfg.App, log), cfg.DataStore()); foreach (var videoId in videoIds) { var txt = await store.GetAndUpdateVideoCaptions(videoId); } }
public async Task SaveCaptions() { var cfg = await Setup.LoadCfg(); var log = Setup.CreateTestLogger(); var store = new YtStore(new YtClient(cfg.App, log), cfg.DataStore()); var channelCfg = await cfg.App.LoadChannelConfig(); foreach (var c in channelCfg.Seeds.Randomize()) { var existingCaptionIds = (await store.Store.List(StringPath.Relative("VideoCaptions", c.Id))) .Select(b => b.NameSansExtension).ToHashSet(); if (existingCaptionIds.Any()) { continue; } var cvc = await store.ChannelVideosCollection.Get(c.Id); var toUpdate = cvc.Vids.OrderByDescending(v => v.PublishedAt).Take(50) .Where(v => !existingCaptionIds.Contains(v.VideoId)).ToList(); await toUpdate.BlockAction(v => store.GetAndUpdateVideoCaptions(c.Id, v.VideoId, log), cfg.App.ParallelCollect); } }
public UserScrape(AzureContainers containers, RootCfg rootCfg, UserScrapeCfg cfg, SemVersion version, YtStore store) { Containers = containers; RootCfg = rootCfg; Cfg = cfg; Version = version; Store = store.Store; }