public static IEnumerable <IJob>?CreateJobs(FeedResult feedResult, IJobBuilder jobBuilder, JobManager jobManager, CancellationToken cancellationToken)
{
if (!feedResult.Successful)
{
return(null);
}
if (feedResult.Songs.Count == 0)
{
Logger.log?.Info("No songs");
return(Array.Empty <IJob>());
}
List <IJob> jobs = new List <IJob>(feedResult.Count);
foreach (ScrapedSong song in feedResult.Songs.Values)
{
Job newJob = jobBuilder.CreateJob(song);
newJob.RegisterCancellationToken(cancellationToken);
jobManager.TryPostJob(newJob, out IJob? postedJob);
if (postedJob != null)
{
jobs.Add(postedJob);
}
else
{
Logger.log?.Info($"Posted job is null for {song}, this shouldn't happen.");
}
}
return(jobs);
}
public static IJobBuilder PrintEnvironment(this IJobBuilder job)
{
job.AddStep("Print Environment")
.Run("printenv")
.ShellBash();
return(job);
}
public void SetUp()
{
var bootstrapper = new IntegrationTestBootstrapper();
var container = bootstrapper.ConfigureContainer();
container.Options.AllowOverridingRegistrations = true;
_workflowFactory = container.GetInstance<IWorkflowFactory>();
_jobBuilder = container.GetInstance<IJobBuilder>();
_interactiveProfile = new ConversionProfile();
_interactiveProfile.Name = InteractivePrinterName;
_interactiveProfile.Guid = InteractiveProfileGuid;
_interactiveProfile.AutoSave.Enabled = false;
_interactiveProfileMapping = new PrinterMapping(InteractivePrinterName, InteractiveProfileGuid);
_autosaveProfile = new ConversionProfile();
_autosaveProfile.Name = AutosaveProfileName;
_autosaveProfile.Guid = AutosaveProfileGuid;
_autosaveProfile.AutoSave.Enabled = true;
_autosaveProfileMapping = new PrinterMapping(AutosavePrinterName, AutosaveProfileGuid);
_th = container.GetInstance<TestHelper>();
_th.InitTempFolder("ConversionWorklowTest");
_th.GenerateGsJob(PSfiles.ThreePDFCreatorTestpages, OutputFormat.Pdf);
_settings = new PdfCreatorSettings(null);
}
public static IJobBuilder Checkout(this IJobBuilder job, int fetchDepth = 0)
{
job.AddStep("Checkout")
.Uses("actions/checkout@v2")
.With("fetch-depth", fetchDepth.ToString());
return(job);
}
public FileSystemWatcherService(ILogger <FileSystemWatcherService> logger,
IOptionsMonitor <FolderSettings> folderSettings, IJobQueue jobQueue, IJobBuilder job)
{
_logger = logger;
_folderSettings = folderSettings;
_jobQueue = jobQueue;
_job = job;
}
public async Task <JobStats[]> RunAsync(BeatSyncConfig config, IJobBuilder jobBuilder, JobManager manager, CancellationToken cancellationToken)
{
Task <JobStats>[] downloadTasks = new Task <JobStats>[]
{
GetBeatSaverAsync(config, jobBuilder, manager, cancellationToken),
GetBeastSaberAsync(config, jobBuilder, manager, cancellationToken),
GetScoreSaberAsync(config, jobBuilder, manager, cancellationToken)
};
return(await Task.WhenAll(downloadTasks).ConfigureAwait(false));
}
protected async Task <JobStats> GetBeastSaberAsync(BeatSyncConfig config, IJobBuilder jobBuilder, JobManager jobManager, CancellationToken cancellationToken)
{
BeastSaberConfig sourceConfig = config.BeastSaber;
JobStats sourceStats = new JobStats();
if (!sourceConfig.Enabled)
{
return(sourceStats);
}
BeastSaberReader reader = new BeastSaberReader(sourceConfig.Username, sourceConfig.MaxConcurrentPageChecks);
FeedConfigBase[] feedConfigs = new FeedConfigBase[] { sourceConfig.Bookmarks, sourceConfig.Follows, sourceConfig.CuratorRecommended };
if (!feedConfigs.Any(f => f.Enabled))
{
Logger.log?.Info($"No feeds enabled for {reader.Name}");
return(sourceStats);
}
SourceStarted?.Invoke(this, "BeastSaber");
foreach (FeedConfigBase?feedConfig in feedConfigs.Where(c => c.Enabled))
{
//if (string.IsNullOrEmpty(sourceConfig.Username) && feedConfig.GetType() != typeof(BeastSaberCuratorRecommended))
//{
// Logger.log?.Warn($" {feedConfig.GetType().Name} feed not available without a valid username.");
// continue;
//}
Logger.log?.Info($" Starting {feedConfig.GetType().Name} feed...");
FeedResult results = await reader.GetSongsFromFeedAsync(feedConfig.ToFeedSettings()).ConfigureAwait(false);
if (results.Successful)
{
IEnumerable <IJob>?jobs = CreateJobs(results, jobBuilder, jobManager, cancellationToken);
JobResult[] jobResults = await Task.WhenAll(jobs.Select(j => j.JobTask).ToArray());
JobStats feedStats = new JobStats(jobResults);
ProcessFinishedJobs(jobs, jobBuilder.SongTargets, config, feedConfig);
Logger.log?.Info($" Finished {feedConfig.GetType().Name} feed: ({feedStats}).");
sourceStats += feedStats;
}
else
{
if (results.Exception != null)
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: {results.Exception.Message}");
Logger.log?.Debug(results.Exception);
}
else
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: Unknown error.");
}
}
}
Logger.log?.Info($" Finished BeastSaber reading: ({sourceStats}).");
return(sourceStats);
}
public JobInfoQueueManager(IManagePrintJobExceptionHandler managePrintJobExceptionHandler, IThreadManager threadManager, IWorkflowFactory workflowFactory, IJobInfoQueue jobInfoQueue, IJobBuilder jobBuilder, ISettingsProvider settingsProvider)
{
_managePrintJobExceptionHandler = managePrintJobExceptionHandler;
_threadManager = threadManager;
_workflowFactory = workflowFactory;
_jobInfoQueue = jobInfoQueue;
_jobBuilder = jobBuilder;
_settingsProvider = settingsProvider;
_jobInfoQueue.OnNewJobInfo += JobInfoQueue_OnNewJobInfo;
}
public static IJobBuilder UploadArtifacts(
this IJobBuilder job,
string name = "artifacts",
string path = "artifacts")
{
job.AddStep("Upload artifacts")
.Uses("actions/upload-artifact@v2")
.With("name", name)
.With("path", path);
return(job);
}
public IJobBuilder AddJob(string id)
{
// Are we currently building a job?
if (_jobBuilder != null)
{
// Add the current job to the jobs context
BuildCurrentJob();
}
// Clear the existing job builder and get ready to make a new job
_jobBuilder = new JobBuilderImpl(this);
_jobBuilder.SetId(id);
return(_jobBuilder);
}
protected async Task <JobStats> GetScoreSaberAsync(BeatSyncConfig config, IJobBuilder jobBuilder, JobManager jobManager, CancellationToken cancellationToken)
{
ScoreSaberConfig sourceConfig = config.ScoreSaber;
JobStats sourceStats = new JobStats();
if (!sourceConfig.Enabled)
{
return(sourceStats);
}
ScoreSaberReader reader = new ScoreSaberReader();
FeedConfigBase[] feedConfigs = new FeedConfigBase[] { sourceConfig.TopRanked, sourceConfig.LatestRanked, sourceConfig.Trending, sourceConfig.TopPlayed };
if (!feedConfigs.Any(f => f.Enabled))
{
Logger.log?.Info($"No feeds enabled for {reader.Name}");
return(sourceStats);
}
SourceStarted?.Invoke(this, "ScoreSaber");
foreach (FeedConfigBase?feedConfig in feedConfigs.Where(c => c.Enabled))
{
Logger.log?.Info($" Starting {feedConfig.GetType().Name} feed...");
FeedResult results = await reader.GetSongsFromFeedAsync(feedConfig.ToFeedSettings()).ConfigureAwait(false);
if (results.Successful)
{
IEnumerable <IJob>?jobs = CreateJobs(results, jobBuilder, jobManager, cancellationToken);
JobResult[] jobResults = await Task.WhenAll(jobs.Select(j => j.JobTask).ToArray());
JobStats feedStats = new JobStats(jobResults);
ProcessFinishedJobs(jobs, jobBuilder.SongTargets, config, feedConfig);
Logger.log?.Info($" Finished {feedConfig.GetType().Name} feed: ({feedStats}).");
sourceStats += feedStats;
}
else
{
if (results.Exception != null)
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: {results.Exception.Message}");
Logger.log?.Debug(results.Exception);
}
else
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: Unknown error.");
}
}
}
Logger.log?.Info($" Finished ScoreSaber reading: ({sourceStats}).");
return(sourceStats);
}
public async void Start()
{
Plugin.log?.Debug("BeatSync Start()");
IsRunning = true;
SetupComponents();
if (playlistManager != null)
{
var recentPlaylist = Plugin.config.RecentPlaylistDays > 0 ? playlistManager.GetOrAddPlaylist(BuiltInPlaylist.BeatSyncRecent) : null;
if (recentPlaylist != null && Plugin.config.RecentPlaylistDays > 0)
{
var minDate = DateTime.Now - new TimeSpan(Plugin.config.RecentPlaylistDays, 0, 0, 0);
int removedCount = recentPlaylist.RemoveAll(s => s.DateAdded < minDate);
if (removedCount > 0)
{
Plugin.log?.Info($"Removed {removedCount} old songs from the RecentPlaylist.");
recentPlaylist.RaisePlaylistChanged();
try
{
playlistManager.StorePlaylist(recentPlaylist);
}
catch (Exception ex)
{
Plugin.log?.Warn($"Unable to write {recentPlaylist.Filename}: {ex.Message}");
Plugin.log?.Debug(ex);
}
}
else
{
Plugin.log?.Info("Didn't remove any songs from RecentPlaylist.");
}
}
}
var syncInterval = new TimeSpan(Plugin.modConfig.TimeBetweenSyncs.Hours, Plugin.modConfig.TimeBetweenSyncs.Minutes, 0);
var nowTime = DateTime.Now;
if (Plugin.config.LastRun + syncInterval <= nowTime)
{
if (Plugin.config.LastRun != DateTime.MinValue)
{
Plugin.log?.Info($"BeatSync ran {TimeSpanToString(nowTime - Plugin.config.LastRun)} ago");
}
if (songHasher != null)
{
await songHasher.InitializeAsync().ConfigureAwait(false);
Plugin.log?.Info($"Hashed {songHasher.HashDictionary.Count} songs in {CustomLevelsDirectory}.");
}
else
{
Plugin.log?.Error($"SongHasher was null.");
}
// Start downloader
IJobBuilder jobBuilder = CreateJobBuilder(Plugin.config);
SongDownloader songDownloader = new SongDownloader();
JobManager JobManager = new JobManager(Plugin.config.MaxConcurrentDownloads);
JobManager.Start(CancelAllToken);
Stopwatch sw = new Stopwatch();
sw.Start();
if (jobBuilder.SongTargets.Count() == 0)
{
Plugin.log?.Error("jobBuilder has no SongTargets.");
}
JobStats[] sourceStats = await songDownloader.RunAsync(Plugin.config, jobBuilder, JobManager).ConfigureAwait(false); // TODO: CancellationToken
JobStats beatSyncStats = sourceStats.Aggregate((a, b) => a + b);
await JobManager.CompleteAsync();
int recentPlaylistDays = Plugin.config.RecentPlaylistDays;
DateTime cutoff = DateTime.Now - new TimeSpan(recentPlaylistDays, 0, 0, 0);
foreach (SongTarget target in jobBuilder.SongTargets)
{
if (target is ITargetWithPlaylists targetWithPlaylists)
{
PlaylistManager?targetPlaylistManager = targetWithPlaylists.PlaylistManager;
if (recentPlaylistDays > 0)
{
BeatSaberPlaylistsLib.Types.IPlaylist?recent = targetPlaylistManager?.GetOrAddPlaylist(BuiltInPlaylist.BeatSyncRecent);
if (recent != null && recent.Count > 0)
{
int songsRemoved = recent.RemoveAll(s => s.DateAdded < cutoff);
if (songsRemoved > 0)
{
recent.RaisePlaylistChanged();
}
}
}
try
{
targetPlaylistManager?.StoreAllPlaylists();
}
catch (AggregateException ex)
{
Plugin.log?.Error($"Error storing playlists: {ex.Message}");
foreach (var e in ex.InnerExceptions)
{
Plugin.log?.Debug(e);
}
}
catch (Exception ex)
{
Plugin.log?.Error($"Error storing playlists: {ex.Message}");
Plugin.log?.Debug(ex);
}
}
if (target is ITargetWithHistory targetWithHistory)
{
try
{
targetWithHistory.HistoryManager?.WriteToFile();
}
catch (Exception ex)
{
Plugin.log?.Info($"Unable to save history at '{targetWithHistory.HistoryManager?.HistoryPath}': {ex.Message}");
}
}
}
sw.Stop();
Plugin.log?.Info($"Finished after {sw.Elapsed.TotalSeconds}s: {beatSyncStats}");
Plugin.config.LastRun = DateTime.Now;
Plugin.ConfigManager.SaveConfig();
SongCore.Loader loader = SongCore.Loader.Instance;
SongCore.Loader.SongsLoadedEvent -= Loader_SongsLoadedEvent;
SongCore.Loader.SongsLoadedEvent += Loader_SongsLoadedEvent;
if (!SongCore.Loader.AreSongsLoading)
{
SongCore.Loader.SongsLoadedEvent -= Loader_SongsLoadedEvent;
if (SongCore.Loader.AreSongsLoaded)
{
loader.RefreshSongs();
}
}
}
else
{
Plugin.log?.Info($"BeatSync ran {TimeSpanToString(nowTime - Plugin.config.LastRun)} ago, skipping because TimeBetweenSyncs is {Plugin.modConfig.TimeBetweenSyncs}");
}
}
static async Task Main(string[] args)
{
try
{
ConsoleLogWriter?consoleLogger = SetupLogging();
string version = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "0.0.0.0";
Logger.log.Info($"Starting BeatSyncConsole v{version}");
await CheckVersion().ConfigureAwait(false);
ConfigManager = new ConfigManager(ConfigDirectory);
bool validConfig = await ConfigManager.InitializeConfigAsync().ConfigureAwait(false);
if (consoleLogger != null)
{
consoleLogger.LogLevel = ConfigManager.Config?.ConsoleLogLevel ?? BeatSyncLib.Logging.LogLevel.Info;
}
Config?config = ConfigManager.Config;
if (validConfig && config != null && config.BeatSyncConfig != null)
{
SongFeedReaders.WebUtils.Initialize(new WebUtilities.HttpClientWrapper.HttpClientWrapper());
SongFeedReaders.WebUtils.WebClient.SetUserAgent("BeatSyncConsole/" + version);
JobManager manager = new JobManager(config.BeatSyncConfig.MaxConcurrentDownloads);
manager.Start(CancellationToken.None);
IJobBuilder jobBuilder = await CreateJobBuilderAsync(config).ConfigureAwait(false);
SongDownloader songDownloader = new SongDownloader();
Stopwatch sw = new Stopwatch();
sw.Start();
JobStats[] sourceStats = await songDownloader.RunAsync(config.BeatSyncConfig, jobBuilder, manager).ConfigureAwait(false);
JobStats beatSyncStats = sourceStats.Aggregate((a, b) => a + b);
await manager.CompleteAsync().ConfigureAwait(false);
int recentPlaylistDays = config.BeatSyncConfig.RecentPlaylistDays;
DateTime cutoff = DateTime.Now - new TimeSpan(recentPlaylistDays, 0, 0, 0);
foreach (SongTarget?target in jobBuilder.SongTargets)
{
if (target is ITargetWithPlaylists targetWithPlaylists)
{
PlaylistManager?targetPlaylistManager = targetWithPlaylists.PlaylistManager;
if (recentPlaylistDays > 0)
{
IPlaylist?recent = targetPlaylistManager?.GetOrAddPlaylist(BuiltInPlaylist.BeatSyncRecent);
if (recent != null && recent.Count > 0)
{
int songsRemoved = recent.RemoveAll(s => s.DateAdded < cutoff);
if (songsRemoved > 0)
{
recent.RaisePlaylistChanged();
}
}
}
try
{
targetPlaylistManager?.StoreAllPlaylists();
}
catch (AggregateException ex)
{
Logger.log.Error($"Error storing playlists: {ex.Message}");
foreach (var e in ex.InnerExceptions)
{
Logger.log.Debug(e);
}
}
catch (Exception ex)
{
Logger.log.Error($"Error storing playlists: {ex.Message}");
Logger.log.Debug(ex);
}
}
if (target is ITargetWithHistory targetWithHistory)
{
try
{
targetWithHistory.HistoryManager?.WriteToFile();
}
catch (Exception ex)
{
Logger.log.Info($"Unable to save history at '{targetWithHistory.HistoryManager?.HistoryPath}': {ex.Message}");
}
}
}
sw.Stop();
Logger.log.Info($"Finished after {sw.Elapsed.TotalSeconds}s: {beatSyncStats}");
config.BeatSyncConfig.LastRun = DateTime.Now;
}
else
{
Logger.log.Info("BeatSyncConsole cannot run without a valid config, exiting.");
}
LogManager.Stop();
LogManager.Wait();
Console.WriteLine("Press Enter to continue...");
Console.Read();
}
catch (Exception ex)
{
string message = $"Fatal Error in BeatSyncConsole: {ex.Message}\n{ex.StackTrace}";
if (LogManager.IsAlive && LogManager.HasWriters && Logger.log != null)
{
Logger.log.Error(message);
}
else
{
ConsoleColor previousColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(message);
Console.ForegroundColor = previousColor;
}
LogManager.Stop();
LogManager.Wait();
Console.WriteLine("Press Enter to continue...");
Console.Read();
}
finally
{
LogManager.Abort();
}
}
public static IJobBuilder LogIntoGitHubContainerRegistry(this IJobBuilder job)
{
job.AddStep("Log into GitHub Container Registry")
.Run("echo \"${{secrets.GITHUB_TOKEN}}\" | docker login ghcr.io -u ${{github.actor}} --password-stdin");
return(job);
}
public JobSchedulerStartupConfig(IJobBuilder jobBuilder)
{
_jobBuilder = jobBuilder;
}
private void Flush()
{
_jobBuilder = null;
}
public Task <JobStats[]> RunAsync(BeatSyncConfig config, IJobBuilder jobBuilder, JobManager manager) => RunAsync(config, jobBuilder, manager, CancellationToken.None);
protected async Task <JobStats> GetBeatSaverAsync(BeatSyncConfig config, IJobBuilder jobBuilder, JobManager jobManager, CancellationToken cancellationToken)
{
BeatSaverConfig sourceConfig = config.BeatSaver;
JobStats sourceStats = new JobStats();
if (!sourceConfig.Enabled)
{
return(sourceStats);
}
BeatSaverReader reader = new BeatSaverReader();
FeedConfigBase[] feedConfigs = new FeedConfigBase[] { sourceConfig.Hot, sourceConfig.Downloads, sourceConfig.Latest };
if (!(feedConfigs.Any(f => f.Enabled) || sourceConfig.FavoriteMappers.Enabled))
{
Logger.log?.Info($"No feeds enabled for {reader.Name}");
return(sourceStats);
}
SourceStarted?.Invoke(this, "BeatSaver");
foreach (FeedConfigBase?feedConfig in feedConfigs.Where(c => c.Enabled))
{
Logger.log?.Info($" Starting {feedConfig.GetType().Name} feed...");
FeedResult results = await reader.GetSongsFromFeedAsync(feedConfig.ToFeedSettings(), cancellationToken).ConfigureAwait(false);
if (results.Successful)
{
IEnumerable <IJob>?jobs = CreateJobs(results, jobBuilder, jobManager, cancellationToken);
JobResult[] jobResults = await Task.WhenAll(jobs.Select(j => j.JobTask).ToArray());
JobStats feedStats = new JobStats(jobResults);
ProcessFinishedJobs(jobs, jobBuilder.SongTargets, config, feedConfig);
Logger.log?.Info($" Finished {feedConfig.GetType().Name} feed: ({feedStats}).");
sourceStats += feedStats;
}
else
{
if (results.Exception != null)
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}{results.Exception.Message}");
Logger.log?.Debug($"{results.Exception}");
}
else
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: Unknown error.");
}
}
}
string[] mappers = sourceConfig.FavoriteMappers.Mappers ?? Array.Empty <string>();
if (sourceConfig.FavoriteMappers.Enabled)
{
FeedConfigBase feedConfig = sourceConfig.FavoriteMappers;
if (mappers.Length > 0)
{
Logger.log?.Info(" Starting FavoriteMappers feed...");
List <IPlaylist> playlists = new List <IPlaylist>();
List <IPlaylist> feedPlaylists = new List <IPlaylist>();
List <IPlaylist> recentPlaylists = new List <IPlaylist>();
JobStats feedStats = new JobStats();
foreach (string?mapper in mappers)
{
Logger.log?.Info($" Getting songs by {mapper}...");
playlists.Clear();
FeedResult results = await reader.GetSongsFromFeedAsync(sourceConfig.FavoriteMappers.ToFeedSettings(mapper)).ConfigureAwait(false);
if (results.Successful)
{
foreach (ITargetWithPlaylists?targetWithPlaylist in jobBuilder.SongTargets.Where(t => t is ITargetWithPlaylists).Select(t => (ITargetWithPlaylists)t))
{
PlaylistManager?playlistManager = targetWithPlaylist.PlaylistManager;
if (playlistManager != null)
{
if (config.RecentPlaylistDays > 0)
{
recentPlaylists.Add(playlistManager.GetOrAddPlaylist(BuiltInPlaylist.BeatSyncRecent));
}
if (config.AllBeatSyncSongsPlaylist)
{
playlists.Add(playlistManager.GetOrAddPlaylist(BuiltInPlaylist.BeatSyncAll));
}
if (sourceConfig.FavoriteMappers.CreatePlaylist)
{
IPlaylist feedPlaylist;
try
{
if (sourceConfig.FavoriteMappers.SeparateMapperPlaylists)
{
feedPlaylist = playlistManager.GetOrCreateAuthorPlaylist(mapper);
}
else
{
feedPlaylist = playlistManager.GetOrAddPlaylist(BuiltInPlaylist.BeatSaverFavoriteMappers);
}
feedPlaylists.Add(feedPlaylist);
playlists.Add(feedPlaylist);
}
catch (ArgumentException ex)
{
Logger.log?.Error($"Error getting playlist for FavoriteMappers: {ex.Message}");
Logger.log?.Debug(ex);
}
}
}
}
IEnumerable <IJob> jobs = CreateJobs(results, jobBuilder, jobManager, cancellationToken) ?? Array.Empty <IJob>();
JobResult[] jobResults = await Task.WhenAll(jobs.Select(j => j.JobTask).ToArray());
JobStats mapperStats = new JobStats(jobResults);
feedStats += mapperStats;
if (jobs.Any(j => j.Result?.Successful ?? false) && feedConfig.PlaylistStyle == PlaylistStyle.Replace)
{
// TODO: This should only apply to successful targets.
foreach (IPlaylist?feedPlaylist in feedPlaylists)
{
feedPlaylist.Clear();
feedPlaylist.RaisePlaylistChanged();
}
}
ProcessFinishedJobs(jobs, playlists, recentPlaylists);
Logger.log?.Info($" Finished getting songs by {mapper}: ({mapperStats}).");
}
else
{
if (results.Exception != null)
{
Logger.log?.Error($"Error getting songs by {mapper}: {results.Exception.Message}");
Logger.log?.Debug(results.Exception);
}
else
{
Logger.log?.Error($"Error getting songs by {mapper}");
}
}
}
sourceStats += feedStats;
Logger.log?.Info($" Finished {feedConfig.GetType().Name} feed: ({feedStats}).");
}
else
{
Logger.log?.Warn(" No FavoriteMappers found, skipping...");
}
}
Logger.log?.Info($" Finished BeatSaver reading: ({sourceStats}).");
return(sourceStats);
}