public FixReviewsCmd(SheetsCfg sheetsCfg, ILogger log, YtStore store, SnowflakeConnectionProvider sf)
{
SheetsCfg = sheetsCfg;
Log = log;
Store = store;
Sf = sf;
}
public YtCollector(BlobStores stores, AppCfg cfg, SnowflakeConnectionProvider sf, IPipeCtx pipeCtx, YtWeb ytWeb,
YtClient api, ILogger log)
{
DbStore = new(stores.Store(DataStoreType.DbStage), log);
Cfg = cfg;
Sf = sf;
PipeCtx = pipeCtx;
Scraper = ytWeb;
Api = api;
}
public async Task SaveChannelRelationData()
{
var cfg = await Setup.LoadCfg();
var log = Setup.CreateTestLogger();
log.Information("started test");
var store = new YtStore(new YtClient(cfg.App, log), cfg.DataStore());
var analysis = new YtCollect(store, cfg.DataStore(cfg.App.Storage.AnalysisPath), cfg.App, log);
await analysis.SaveChannelRelationData();
}
public async Task SaveCaptions()
{
var cfg = await Setup.LoadCfg();
var log = Setup.CreateTestLogger();
var videoIds = new[] { "P0dcyAXXmCw" };
var store = new YtStore(new YtClient(cfg.App, log), cfg.DataStore());
foreach (var videoId in videoIds)
{
var txt = await store.GetAndUpdateVideoCaptions(videoId);
}
}
public async Task SaveCaptions()
{
var cfg = await Setup.LoadCfg();
var log = Setup.CreateTestLogger();
var store = new YtStore(new YtClient(cfg.App, log), cfg.DataStore());
var channelCfg = await cfg.App.LoadChannelConfig();
foreach (var c in channelCfg.Seeds.Randomize())
{
var existingCaptionIds = (await store.Store.List(StringPath.Relative("VideoCaptions", c.Id)))
.Select(b => b.NameSansExtension).ToHashSet();
if (existingCaptionIds.Any())
{
continue;
}
var cvc = await store.ChannelVideosCollection.Get(c.Id);
var toUpdate = cvc.Vids.OrderByDescending(v => v.PublishedAt).Take(50)
.Where(v => !existingCaptionIds.Contains(v.VideoId)).ToList();
await toUpdate.BlockAction(v => store.GetAndUpdateVideoCaptions(c.Id, v.VideoId, log), cfg.App.ParallelCollect);
}
}
public UserScrape(AzureContainers containers, RootCfg rootCfg, UserScrapeCfg cfg, SemVersion version, YtStore store)
{
Containers = containers;
RootCfg = rootCfg;
Cfg = cfg;
Version = version;
Store = store.Store;
}
