public static async Task <List <ScrapedSong> > SearchAsync(string criteria, SearchType type, BeatSaverFeedSettings settings = null) { // TODO: Hits rate limit if (type == SearchType.key) { return(new List <ScrapedSong>() { await GetSongByKeyAsync(criteria).ConfigureAwait(false) }); } if (type == SearchType.user) { return(await GetSongsByUploaderIdAsync((await GetAuthorNamesByIDAsync(criteria).ConfigureAwait(false)).FirstOrDefault()).ConfigureAwait(false)); } if (type == SearchType.hash) { return(new List <ScrapedSong>() { await GetSongByHashAsync(criteria).ConfigureAwait(false) }); } StringBuilder url; int maxSongs = 0; int maxPages = 0; //int lastPage; //int nextPage; int pageIndex = 0; if (settings != null) { maxSongs = settings.MaxSongs; maxPages = settings.MaxPages; pageIndex = Math.Max(settings.StartingPage - 1, 0); } bool useMaxPages = maxPages > 0; bool useMaxSongs = maxSongs > 0; if (useMaxPages && pageIndex > 0) { maxPages = maxPages + pageIndex; } bool continueLooping = true; var songs = new List <ScrapedSong>(); List <ScrapedSong> newSongs; do { url = new StringBuilder(Feeds[BeatSaverFeed.Search].BaseUrl); url.Replace(SEARCHTYPEKEY, type.ToString()); url.Replace(SEARCHKEY, criteria); url.Replace(PAGEKEY, pageIndex.ToString()); var uri = new Uri(url.ToString()); string pageText = string.Empty; using (var response = await WebUtils.WebClient.GetAsync(uri).ConfigureAwait(false)) { Logger.Debug($"Checking {uri} for songs."); if (response.IsSuccessStatusCode) { pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false); } else { Logger.Error($"Error searching for song, {uri} responded with {response.StatusCode}:{response.ReasonPhrase}"); return(songs); } } newSongs = ParseSongsFromPage(pageText, uri); foreach (var song in newSongs) { if (!useMaxSongs || songs.Count < maxSongs) { songs.Add(song); } } pageIndex++; if (newSongs.Count == 0) { continueLooping = false; } if (useMaxPages && (pageIndex >= maxPages)) { continueLooping = false; } if (useMaxSongs && pageIndex * SONGS_PER_PAGE >= maxSongs) { continueLooping = false; } } while (continueLooping); return(songs); }
public static List <ScrapedSong> GetBeatSaverSongs(BeatSaverFeedSettings settings) { return(GetBeatSaverSongsAsync(settings).Result); }
public static async Task <List <ScrapedSong> > GetBeatSaverSongsAsync(BeatSaverFeedSettings settings) { if (settings == null) { throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeatSaverReader.GetBeatSaverSongsAsync"); } // TODO: double checks the first page int feedIndex = settings.FeedIndex; bool useMaxPages = settings.MaxPages != 0; bool useMaxSongs = settings.MaxSongs != 0; List <ScrapedSong> songs = new List <ScrapedSong>(); string pageText = string.Empty; using (var response = await WebUtils.WebClient.GetAsync(GetPageUrl(feedIndex)).ConfigureAwait(false)) { if (response.IsSuccessStatusCode) { pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false); } else { return(songs); } } JObject result = new JObject(); try { result = JObject.Parse(pageText); } catch (JsonReaderException ex) { Logger.Exception("Unable to parse JSON from text", ex); } int?numSongs = result["totalDocs"]?.Value <int>(); int?lastPage = result["lastPage"]?.Value <int>(); if (numSongs == null || lastPage == null || numSongs == 0) { Logger.Warning($"Error checking Beat Saver's {settings.FeedName} feed."); return(songs); } Logger.Info($"Checking Beat Saver's {settings.FeedName} feed, {numSongs} songs available"); int maxPages = settings.MaxPages; int pageNum = Math.Max(settings.StartingPage - 1, 0); if (pageNum > 0 && useMaxPages) { maxPages = maxPages + pageNum; // Add starting page to maxPages so we actually get songs if maxPages < starting page } List <Task <List <ScrapedSong> > > pageReadTasks = new List <Task <List <ScrapedSong> > >(); Uri uri = null; bool continueLooping = true; do { uri = GetPageUrl(feedIndex, pageNum); Logger.Debug($"Creating task for {uri.ToString()}"); pageReadTasks.Add(GetSongsFromPageAsync(uri)); pageNum++; if ((pageNum > lastPage)) { continueLooping = false; } if (useMaxPages && (pageNum >= maxPages)) { continueLooping = false; } if (useMaxSongs && pageNum * SONGS_PER_PAGE >= settings.MaxSongs) { continueLooping = false; } } while (continueLooping); try { await Task.WhenAll(pageReadTasks.ToArray()).ConfigureAwait(false); } #pragma warning disable CA1031 // Do not catch general exception types catch (Exception) // TODO: Better exception handling here, does it even throw here or in the for loop below? #pragma warning restore CA1031 // Do not catch general exception types { Logger.Error($"Error waiting for pageReadTasks"); } foreach (var job in pageReadTasks) { songs.AddRange(await job.ConfigureAwait(false)); foreach (var song in await job.ConfigureAwait(false)) { if (!useMaxSongs || songs.Count <= settings.MaxSongs) { songs.Add(song); } } } return(songs); }