public YtCollect(YtStore store, ISimpleFileStore simpleFileStore, AppCfg cfg, ILogger log)
{
Yt = store;
Store = simpleFileStore;
Cfg = cfg;
Log = log;
}
public YtClient(AppCfg cfg, ILogger log)
{
Cfg = cfg;
Log = log;
YtService = new YouTubeService();
AvailableKeys = cfg.YTApiKeys?.ToList() ?? throw new InvalidOperationException("configuration requires YTApiKeys");
Start = DateTime.UtcNow;
}
public YtDataUpdater(YtStore store, AppCfg cfg, UpdateType updateType, Func <Task <DbConnection> > getConnection, ILogger log)
{
UpdateType = updateType;
GetConnection = getConnection;
Store = store;
Cfg = cfg;
Log = log;
Scraper = new YtScraper(cfg.Scraper);
Api = new YtClient(cfg.YTApiKeys, log);
}
public static async Task <ChannelConfig> LoadChannelConfig(this AppCfg cfg)
{
var channelCfg = new ChannelConfig();
var csv = await new WebClient().DownloadStringTaskAsync(cfg.YtReader.SeedsUrl);
var seedData = CsvExtensions.ReadFromCsv <SeedChannel>(csv);
channelCfg.Seeds.AddRange(cfg.LimitedToSeedChannels != null ? seedData.Where(s => cfg.LimitedToSeedChannels.Contains(s.Id)) : seedData);
//channelCfg.Excluded.AddRange(cfg.CrawlConfigDir.Combine("ChannelExclude.csv").ReadFromCsv<InfluencerOverride>());
return(channelCfg);
}
public YtClient(AppCfg cfg, ILogger log)
{
Cfg = cfg;
Log = log;
YtService = new YouTubeService();
var keys = cfg.YTApiKeys ?? throw new InvalidOperationException("configuration requires YTApiKeys");
AvailableKeys = new ConcurrentDictionary <string, string>(keys.Select(k => new KeyValuePair <string, string>(k, null)));
Start = DateTime.UtcNow;
}
public YtCollector(YtStore store, AppCfg cfg, SnowflakeConnectionProvider sf, IPipeCtx pipeCtx, WebScraper webScraper, ChromeScraper chromeScraper,
YtClient api)
{
Store = store;
Cfg = cfg;
Sf = sf;
PipeCtx = pipeCtx;
Scraper = webScraper;
ChromeScraper = chromeScraper;
Api = api;
}
static LoggerConfiguration ConfigureSeq(this LoggerConfiguration loggerCfg, AppCfg cfg)
{
var seqCfg = cfg?.Seq;
if (seqCfg?.SeqUrl == null)
{
return(loggerCfg);
}
var resCfg = loggerCfg.WriteTo.Seq(seqCfg.SeqUrl.OriginalString, cfg.LogLevel);
return(resCfg);
}
public static Logger CreateCliLogger(AppCfg cfg = null)
{
var c = new LoggerConfiguration()
.WriteTo.Console();
if (cfg != null)
{
c.WriteTo.ApplicationInsights(new TelemetryConfiguration(cfg.AppInsightsKey), TelemetryConverter.Traces);
}
return(c.CreateLogger());
}
public static Logger CreateLogger(AppCfg cfg = null)
{
var c = new LoggerConfiguration()
.WriteTo.Console(LogEventLevel.Information);
if (cfg?.SeqUrl.HasValue() == true)
{
c.WriteTo.Seq(cfg.SeqUrl, LogEventLevel.Debug);
}
if (cfg?.AppInsightsKey != null)
{
c.WriteTo.ApplicationInsights(new TelemetryConfiguration(cfg.AppInsightsKey), TelemetryConverter.Traces, LogEventLevel.Debug);
}
c.MinimumLevel.Debug();
return(c.CreateLogger());
}
public YtDataUpdater(YtStore store, AppCfg cfg, ILogger log)
{
Yt = store;
Cfg = cfg;
Log = log;
}
public StoreUpgrader(AppCfg cfg, ISimpleFileStore store, ILogger log)
{
Cfg = cfg;
Store = store;
Log = log;
}
public static Logger CreateLogger(string env, string app, VersionInfo version, AppCfg cfg = null)
{
var c = new LoggerConfiguration()
.WriteTo.Console(LogEventLevel.Information);
if (cfg?.AppInsightsKey != null)
{
c.WriteTo.ApplicationInsights(new TelemetryConfiguration(cfg.AppInsightsKey), TelemetryConverter.Traces, cfg.LogLevel);
}
if (cfg != null)
{
c = c.ConfigureSeq(cfg);
}
var log = c.YtEnrich(env, app, version.Version)
.MinimumLevel.ControlledBy(new LoggingLevelSwitch(cfg?.LogLevel ?? LogEventLevel.Debug))
.CreateLogger();
Log.Logger = log;
return(log);
}