public static IEnumerable <IJob>?CreateJobs(FeedResult feedResult, IJobBuilder jobBuilder, JobManager jobManager, CancellationToken cancellationToken)
{
if (!feedResult.Successful)
{
return(null);
}
if (feedResult.Songs.Count == 0)
{
Logger.log?.Info("No songs");
return(Array.Empty <IJob>());
}
List <IJob> jobs = new List <IJob>(feedResult.Count);
foreach (ScrapedSong song in feedResult.Songs.Values)
{
Job newJob = jobBuilder.CreateJob(song);
newJob.RegisterCancellationToken(cancellationToken);
jobManager.TryPostJob(newJob, out IJob? postedJob);
if (postedJob != null)
{
jobs.Add(postedJob);
}
else
{
Logger.log?.Info($"Posted job is null for {song}, this shouldn't happen.");
}
}
return(jobs);
}
public void Search_Default_LimitedSongs()
{
BeatSaverReader reader = new BeatSaverReader()
{
StoreRawData = true
};
int maxSongs = 10;
BeatSaverSearchType searchType = BeatSaverSearchType.name;
string criteria = "Believer";
BeatSaverSearchQuery query = new SearchQueryBuilder(searchType, criteria).GetQuery();
BeatSaverFeedSettings settings = new BeatSaverFeedSettings((int)BeatSaverFeedName.Search)
{
MaxSongs = maxSongs, SearchQuery = query
};
FeedResult result = reader.GetSongsFromFeed(settings);
Assert.IsTrue(result.Count > 0);
Assert.IsTrue(result.Count <= 10);
int expectedPages = ExpectedPagesForSongs(result.Count);
Assert.AreEqual(expectedPages, result.PagesChecked);
foreach (ScrapedSong song in result.Songs.Values)
{
Console.WriteLine($"{song.Name} by {song.LevelAuthorName}, {song.Hash}");
}
}
public FeedResult RSS2()
{
var data = _gameLog.GetAll();
var feedItems = data.OrderByDescending(gle => gle.Timestamp).Select(gle => {
var item = new SyndicationItem()
{
Title = new TextSyndicationContent(gle.Description)
};
item.Authors.Add(new SyndicationPerson(gle.User));
item.PublishDate = gle.Timestamp;
//var link = HtmlHelper.GenerateLink
//item.Links.Add()
return(item);
}
);
var feed = new SyndicationFeed("kMagic Activity Feed", "Match Results", null, feedItems);
var feedResult = new FeedResult(new Rss20FeedFormatter(feed));
feedResult.ContentType = "text/rss";
return(feedResult);
}
/// <summary>
/// Updates the <see cref="Device"/> cache. All new or changed <see cref="Device"/>s are obtained using a GetFeed() call. Then, <see cref="Device"/>s that are already in the cache are replaced and any <see cref="Device"/>s not in the cache are added to the cache.
/// </summary>
/// <returns></returns>
async Task UpdateDeviceCacheAsync()
{
ConsoleUtility.LogInfoStart("Updating Device cache...");
// Populate the deviceCache, adding new items and updating existing items with their changed counterparts from the database. Repeat execution of the GetFeedDeviceAsync method until no more results are returned to ensure that the cache is complete and up-to-date.
FeedResult <Device> feedResult = null;
bool keepGoing = true;
while (keepGoing == true)
{
feedResult = await api.GetFeedDeviceAsync(lastDeviceFeedVersion);
lastDeviceFeedVersion = feedResult.ToVersion;
foreach (Device feedResultDevice in feedResult.Data)
{
if (deviceCache.ContainsKey(feedResultDevice.Id))
{
deviceCache[feedResultDevice.Id] = feedResultDevice;
}
else
{
deviceCache.Add(feedResultDevice.Id, feedResultDevice);
}
}
if (feedResult.Data.Count < DefaultFeedResultsLimitDevice)
{
keepGoing = false;
}
}
ConsoleUtility.LogComplete(Common.ConsoleColorForUnchangedData);
ConsoleUtility.LogListItem($"Device cache records added/updated:", feedResult.Data.Count.ToString(), Common.ConsoleColorForListItems, (feedResult.Data.Count > 0) ? Common.ConsoleColorForChangedData : Common.ConsoleColorForUnchangedData);
}
private async void GetFeedBtn_Click(object sender, EventArgs e)
{
FeedResult <FaultData> FaultData = await GetFaultFeed();
DisplayCodes(FaultData);
GC.Collect();
GC.WaitForPendingFinalizers();
}
public void Success_Authors_LimitedSongs()
{
BeatSaverReader reader = new BeatSaverReader()
{
StoreRawData = true
};
string[] authorList = new string[] { "BlackBlazon", "greatyazer", "joetastic" };
Dictionary <string, ScrapedSong> songList = new Dictionary <string, ScrapedSong>();
int maxSongs = 59;
int maxPages = 10;
SearchQueryBuilder queryBuilder = new SearchQueryBuilder(BeatSaverSearchType.author, null);
foreach (string author in authorList)
{
queryBuilder.Criteria = author;
BeatSaverFeedSettings settings = new BeatSaverFeedSettings((int)BeatSaverFeedName.Author)
{
SearchQuery = queryBuilder.GetQuery(), MaxSongs = maxSongs, MaxPages = maxPages
};
FeedResult songsByAuthor = reader.GetSongsFromFeed(settings);
Assert.IsTrue(songsByAuthor.Count > 0);
Assert.IsTrue(songsByAuthor.Count <= maxSongs);
int expectedPages = ExpectedPagesForSongs(songsByAuthor.Count);
Assert.IsTrue(expectedPages <= songsByAuthor.PagesChecked);
foreach (KeyValuePair <string, ScrapedSong> song in songsByAuthor.Songs)
{
songList.TryAdd(song.Key, song.Value);
}
}
IEnumerable <string> detectedAuthors = songList.Values.Select(s => s.LevelAuthorName.ToLower()).Distinct();
foreach (KeyValuePair <string, ScrapedSong> song in songList)
{
Assert.IsTrue(song.Value.DownloadUri != null);
Assert.IsTrue(authorList.Any(a => a.ToLower() == song.Value.LevelAuthorName.ToLower()));
}
foreach (string author in authorList)
{
Assert.IsTrue(songList.Any(s => s.Value.LevelAuthorName.ToLower() == author.ToLower()));
}
// BlackBlazon check
string blazonHash = "58de2d709a45b68fdb1dbbfefb187f59f629bfc5".ToUpper();
ScrapedSong blazonSong = songList[blazonHash];
Assert.IsTrue(blazonSong != null);
Assert.IsTrue(blazonSong.DownloadUri != null);
// GreatYazer check
string songHash = "bf8c016dc6b9832ece3030f05277bbbe67db790d".ToUpper();
ScrapedSong yazerSong = songList[songHash];
Assert.IsTrue(yazerSong != null);
Assert.IsTrue(yazerSong.DownloadUri != null);
var checkedPages = songList.Values.Select(s => s.SourceUri.OriginalString).Distinct().ToList();
checkedPages.ForEach(p => Console.WriteLine(p));
}
public static async Task <List <string> > GetAuthorNamesByIDAsync(string mapperId, CancellationToken cancellationToken)
{
List <string> authorNames = new List <string>();
FeedResult result = await GetSongsByUploaderIdAsync(mapperId, cancellationToken).ConfigureAwait(false);
authorNames = result.Songs.Values.Select(s => s.MapperName).Distinct().ToList();
//authorNames.ForEach(n => Logger?.Warning($"Found authorName: {n}"));
return(authorNames);
}
protected async Task <JobStats> GetBeastSaberAsync(BeatSyncConfig config, IJobBuilder jobBuilder, JobManager jobManager, CancellationToken cancellationToken)
{
BeastSaberConfig sourceConfig = config.BeastSaber;
JobStats sourceStats = new JobStats();
if (!sourceConfig.Enabled)
{
return(sourceStats);
}
BeastSaberReader reader = new BeastSaberReader(sourceConfig.Username, sourceConfig.MaxConcurrentPageChecks);
FeedConfigBase[] feedConfigs = new FeedConfigBase[] { sourceConfig.Bookmarks, sourceConfig.Follows, sourceConfig.CuratorRecommended };
if (!feedConfigs.Any(f => f.Enabled))
{
Logger.log?.Info($"No feeds enabled for {reader.Name}");
return(sourceStats);
}
SourceStarted?.Invoke(this, "BeastSaber");
foreach (FeedConfigBase?feedConfig in feedConfigs.Where(c => c.Enabled))
{
//if (string.IsNullOrEmpty(sourceConfig.Username) && feedConfig.GetType() != typeof(BeastSaberCuratorRecommended))
//{
// Logger.log?.Warn($" {feedConfig.GetType().Name} feed not available without a valid username.");
// continue;
//}
Logger.log?.Info($" Starting {feedConfig.GetType().Name} feed...");
FeedResult results = await reader.GetSongsFromFeedAsync(feedConfig.ToFeedSettings()).ConfigureAwait(false);
if (results.Successful)
{
IEnumerable <IJob>?jobs = CreateJobs(results, jobBuilder, jobManager, cancellationToken);
JobResult[] jobResults = await Task.WhenAll(jobs.Select(j => j.JobTask).ToArray());
JobStats feedStats = new JobStats(jobResults);
ProcessFinishedJobs(jobs, jobBuilder.SongTargets, config, feedConfig);
Logger.log?.Info($" Finished {feedConfig.GetType().Name} feed: ({feedStats}).");
sourceStats += feedStats;
}
else
{
if (results.Exception != null)
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: {results.Exception.Message}");
Logger.log?.Debug(results.Exception);
}
else
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: Unknown error.");
}
}
}
Logger.log?.Info($" Finished BeastSaber reading: ({sourceStats}).");
return(sourceStats);
}
public void Success_Filtered360()
{
BeatSaverReader reader = new BeatSaverReader()
{
StoreRawData = true
};
int maxSongs = 20;
int maxPages = 30;
BeatSaverFeedSettings settings = new BeatSaverFeedSettings((int)BeatSaverFeedName.Latest)
{
MaxSongs = maxSongs, MaxPages = maxPages
};
Progress <ReaderProgress> progress = new Progress <ReaderProgress>(p =>
{
if (p.SongCount > 0)
{
Console.WriteLine($"Progress: Page {p.CurrentPage} found {p.SongCount} songs.");
}
else
{
Console.WriteLine($"Progress: Page {p.CurrentPage} did not have any songs.");
}
});
bool stopAfter(ScrapedSong song)
{
DateTime uploadDate = song.JsonData["uploaded"].Value <DateTime>();
bool shouldStop = uploadDate < (DateTime.Now - TimeSpan.FromDays(5));
if (shouldStop)
{
Console.WriteLine($"StopWhenAny reached with {song.Key} ({uploadDate.ToString()})");
}
return(shouldStop);
}
Func <ScrapedSong, bool> filter = SongFeedReaders.Filtering.BuiltInFilters.ThreeSixtyDegree;
settings.Filter = filter;
settings.StopWhenAny = stopAfter;
FeedResult result = reader.GetSongsFromFeedAsync(settings, progress, CancellationToken.None).Result;
Assert.IsTrue(result.Count > 0 && result.Count < maxPages * settings.SongsPerPage);
int expectedPages = ExpectedPagesForSongs(result.Count);
Assert.IsTrue(expectedPages <= result.PagesChecked);
Console.WriteLine($"----------------");
foreach (ScrapedSong song in result.Songs.Values)
{
Console.WriteLine($"{song.Name} by {song.LevelAuthorName}, {song.Hash}");
}
}
public void ExecuteResult_Should_Write_Atom()
{
var controllerContext = new ControllerContext(_httpContext.Object, new RouteData(), new Mock <ControllerBase>().Object);
_httpContext.HttpResponse.ExpectSet(r => r.ContentType).Verifiable();
_httpContext.HttpResponse.Expect(r => r.Write(It.IsAny <string>())).Verifiable();
FeedResult result = new FeedResult(feedViewData, "atom");
result.ExecuteResult(controllerContext);
_httpContext.Verify();
}
public void ExecuteResult_Should_Write_Rss()
{
var controllerContext = new ControllerContext(_httpContext.Object, new RouteData(), new Mock <ControllerBase>().Object);
_httpContext.HttpResponse.SetupSet(r => r.ContentType = "application/rss+xml").Verifiable();
_httpContext.HttpResponse.Setup(r => r.Write(It.IsAny <string>())).Verifiable();
FeedResult result = new FeedResult(feedViewData, "rss");
result.ExecuteResult(controllerContext);
_httpContext.Verify();
}
private async Task <FeedResult <FaultData> > GetFaultFeed()
{
var FaultData = Convert.ToInt64(Program.CONFIG.GeotabFeeds.Users[Program.CONFIG.UserConfig.Name].DataFeeds["FaultData"].Token);
FeedResult <FaultData> FeedResults = await GeotabAPI.Get <FeedResult <FaultData>, FaultData>("GetFeed", new
{
resultsLimit = 10000,
fromVersion = FaultData
});
Program.CONFIG.GeotabFeeds.Users[Program.CONFIG.UserConfig.Name].DataFeeds["FaultData"].Token = FeedResults.ToVersion.ToString();
Program.CONFIG.APP_CONFIG.Save();
return(FeedResults);
}
private void SetFeedCreatedPicture(FeedResult feed, IList <UserPhotoResult> createdByPictures = null)
{
if (createdByPictures == null || !createdByPictures.Any())
{
return;
}
var createdByUserPicture = createdByPictures.FirstOrDefault(x => x.UserId == feed.CreatedById);
if (createdByUserPicture != null)
{
feed.CreatedByPhotoCode = createdByUserPicture.Code;
}
}
protected async Task <JobStats> GetScoreSaberAsync(BeatSyncConfig config, IJobBuilder jobBuilder, JobManager jobManager, CancellationToken cancellationToken)
{
ScoreSaberConfig sourceConfig = config.ScoreSaber;
JobStats sourceStats = new JobStats();
if (!sourceConfig.Enabled)
{
return(sourceStats);
}
ScoreSaberReader reader = new ScoreSaberReader();
FeedConfigBase[] feedConfigs = new FeedConfigBase[] { sourceConfig.TopRanked, sourceConfig.LatestRanked, sourceConfig.Trending, sourceConfig.TopPlayed };
if (!feedConfigs.Any(f => f.Enabled))
{
Logger.log?.Info($"No feeds enabled for {reader.Name}");
return(sourceStats);
}
SourceStarted?.Invoke(this, "ScoreSaber");
foreach (FeedConfigBase?feedConfig in feedConfigs.Where(c => c.Enabled))
{
Logger.log?.Info($" Starting {feedConfig.GetType().Name} feed...");
FeedResult results = await reader.GetSongsFromFeedAsync(feedConfig.ToFeedSettings()).ConfigureAwait(false);
if (results.Successful)
{
IEnumerable <IJob>?jobs = CreateJobs(results, jobBuilder, jobManager, cancellationToken);
JobResult[] jobResults = await Task.WhenAll(jobs.Select(j => j.JobTask).ToArray());
JobStats feedStats = new JobStats(jobResults);
ProcessFinishedJobs(jobs, jobBuilder.SongTargets, config, feedConfig);
Logger.log?.Info($" Finished {feedConfig.GetType().Name} feed: ({feedStats}).");
sourceStats += feedStats;
}
else
{
if (results.Exception != null)
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: {results.Exception.Message}");
Logger.log?.Debug(results.Exception);
}
else
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: Unknown error.");
}
}
}
Logger.log?.Info($" Finished ScoreSaber reading: ({sourceStats}).");
return(sourceStats);
}
private void SetFeedCreatedByName(FeedResult feed, IEnumerable <UserResult> createdByUsers = null)
{
if (createdByUsers == null || !createdByUsers.Any())
{
return;
}
var user = createdByUsers.FirstOrDefault(x => x.Id == feed.CreatedById);
if (user != null)
{
feed.CreatedByName = user.DisplayName;
}
}
/// <summary>
/// Updates the <see cref="Diagnostic"/> cache. All new or changed <see cref="Diagnostic"/>s are obtained using a GetFeed() call. Then, the <see cref="Controller"/> and <see cref="UnitOfMeasure"/> child objects are "hydrated" with fully-populated cached counterparts. Finally, <see cref="Diagnostic"/>s that are already in the cache are replaced and any <see cref="Diagnostic"/>s not in the cache are added to the cache.
/// </summary>
async Task UpdateDiagnosticCacheAsync()
{
ConsoleUtility.LogInfoStart("Updating Diagnostic cache...");
// Populate the diagnosticCache, adding new items and updating existing items with their changed counterparts from the database. Repeat execution of the GetFeedDiagnosticAsync method until no more results are returned to ensure that the cache is complete and up-to-date.
FeedResult <Diagnostic> feedResult = null;
bool keepGoing = true;
while (keepGoing == true)
{
feedResult = await api.GetFeedDiagnosticAsync(lastDiagnosticFeedVersion);
lastDiagnosticFeedVersion = feedResult.ToVersion;
foreach (Diagnostic feedResultDiagnostic in feedResult.Data)
{
// Hydrate Controller and UnitOfMeasure objects.
Controller controller = feedResultDiagnostic.Controller;
if (controller == null)
{
feedResultDiagnostic.Controller = NoController.Value;
}
else if (!controller.Equals(NoController.Value))
{
feedResultDiagnostic.Controller = controllerCache[controller.Id];
}
UnitOfMeasure unitOfMeasure = feedResultDiagnostic.UnitOfMeasure;
if (unitOfMeasure != null)
{
feedResultDiagnostic.UnitOfMeasure = unitOfMeasureCache[unitOfMeasure.Id];
}
// Add or update.
if (diagnosticCache.ContainsKey(feedResultDiagnostic.Id))
{
diagnosticCache[feedResultDiagnostic.Id] = feedResultDiagnostic;
}
else
{
diagnosticCache.Add(feedResultDiagnostic.Id, feedResultDiagnostic);
}
}
if (feedResult.Data.Count < DefaultFeedResultsLimitDiagnostic)
{
keepGoing = false;
}
}
ConsoleUtility.LogComplete(Common.ConsoleColorForUnchangedData);
ConsoleUtility.LogListItem($"Diagnostics added/updated:", feedResult.Data.Count.ToString(), Common.ConsoleColorForListItems, (feedResult.Data.Count > 0) ? Common.ConsoleColorForChangedData : Common.ConsoleColorForUnchangedData);
}
// declare a method within that which converts the string to data. This will be unique for this website, hence why I’ve put that in the name.
// you’ll have to make a new method like this for each api you want to call.
public List <Event> GetInfoFromWholeFootballDataComJSONstring(String originalJsonData)
{
List <Event> events = new List <Event>();
// this line is using the Json.Net classes to convert to a list of Strings
FeedResult data = JsonConvert.DeserializeObject <FeedResult> (originalJsonData);
foreach (FeedFixture fixture in data.fixtures)
{
// build an event object, and add to our collection...
events.Add(this.BuildEvent(fixture));
}
return(events);
}
public override object ConvertSourceToObject(PublishedPropertyType propertyType, object source, bool preview)
{
if (source == null)
{
return(null);
}
var feedUrl = source.ToString();
var feedResult = new FeedResult()
{
HasFeedResults = false, FeedUrl = feedUrl
};
var feedContent = default(SyndicationFeed);
if (!String.IsNullOrEmpty(feedUrl) && feedUrl.ToLower().StartsWith("http"))
{
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(feedUrl);
request.UserAgent = "tooorangey.FeedReader";
request.Headers.Add("Accept-Encoding", "gzip,deflate");
request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
using (WebResponse response = request.GetResponse())
{
using (System.Xml.XmlReader reader = XmlReader.Create(response.GetResponseStream()))
{
feedContent = System.ServiceModel.Syndication.SyndicationFeed.Load(reader);
}
}
}
catch (Exception ex)
{
//think about maybe logging the error
LogHelper.Error(MethodBase.GetCurrentMethod().DeclaringType, "Error loading feed: " + feedUrl, ex);
feedResult.StatusMessage = "A bad error has occurred: " + ex.Message;
}
}
else
{
feedResult.StatusMessage = "Provide a Feed Url beginning with http...";
}
if (feedContent != null)
{
feedResult.HasFeedResults = true;
feedResult.SyndicationFeed = feedContent;
}
//what if requested dynamically...
return(feedResult);
}
public FeedResult GetData(DateTime startDate, DateTime endDate, string dataType)
{
//Initialize variables
cacheManager = new CacheManager();
values = new List <double>();
List <string> lines;
FeedResult result = new FeedResult();
//First text lines from cache for the range of requested dates
lines = cacheManager.GetDataFromCache(startDate, endDate, dataType);
//If any requested date isn't found, the cache manager throws
//an exception that bubbles up LakeStatsService.GetLakeStats()
//If we get here, process the lines of text
foreach (string line in lines)
{
//Get the 3rd item in the list, add to arraylist
char[] separators = new char[] { ' ' };
string[] words = line.Split(separators, StringSplitOptions.RemoveEmptyEntries);
values.Add(Convert.ToDouble(words[2]));
}
//Now calculate and package the results
//Start with average, rounded to 2 decimals.
result.mean = values.Average();
//Calculate the median value; first sort the values in ascending order
values.Sort();
//Get the median index as a double
double medianIndex = (double)(values.Count - 1) / 2;
//Check whether median index is fractional or whole, get the value
if (Math.Truncate(medianIndex) == medianIndex)
{
result.median = values[Convert.ToInt32(medianIndex)];
}
else
{
int lowIndex = (Int32)Math.Truncate(medianIndex);
int highIndex = lowIndex + 1;
result.median = ((values[lowIndex] + values[highIndex]) / 2);
}
return(result);
}
public void OnActionExecutedReplacesWithAtomResult()
{
AtomResultActionFilter filter = new AtomResultActionFilter();
ActionExecutedContext context = new ActionExecutedContext()
{
Result = new ViewResult()
};
filter.OnActionExecuted(context);
FeedResult result = context.Result as FeedResult;
Assert.NotNull(result);
Assert.Equal("ATOM", result.ViewName);
Assert.False(result.IsClientCached);
}
public void CancelledInProgress()
{
BeatSaverReader reader = new BeatSaverReader()
{
StoreRawData = true
};
var cts = new CancellationTokenSource(1000);
int maxSongs = 50;
BeatSaverFeedSettings settings = new BeatSaverFeedSettings((int)BeatSaverFeedName.Plays)
{
MaxSongs = maxSongs
};
FeedResult result = reader.GetSongsFromFeed(settings, cts.Token);
Assert.IsFalse(result.Successful);
Assert.IsTrue(result.Count > 0);
Assert.AreEqual(FeedResultError.Cancelled, result.ErrorCode);
cts.Dispose();
}
public Tokenizer(StreamReader stream, CompilerOptions options)
{
Options = options;
context = new Context()
{
Tcr = new TextCharReader(stream)
};
result = new FeedResult();
result.Token = null;
result.Type = FeedResultType.NotDone;
modeEvents.Add(Context.ModeType.New, new Action <CharType>(this.onMode_New));
modeEvents.Add(Context.ModeType.Number, new Action <CharType>(this.onMode_Number));
modeEvents.Add(Context.ModeType.String, new Action <CharType>(this.onMode_String));
modeEvents.Add(Context.ModeType.Identifier, new Action <CharType>(this.onMode_Identifier));
modeEvents.Add(Context.ModeType.CommentSingle, new Action <CharType>(this.onMode_CommentSingle));
modeEvents.Add(Context.ModeType.CommentRange, new Action <CharType>(this.onMode_CommentRange));
modeEvents.Add(Context.ModeType.Whitespace, new Action <CharType>(this.onMode_WhiteSpace));
}
public void GetLakeStats(string start, string end)
{
FeedManager mgr = new FeedManager();
//Set default date values
if (start.Length == 0)
{
start = "2014-01-01";
}
if (end.Length == 0)
{
end = "2014-01-03";
}
DateTime startDate;
DateTime endDate;
try
{
//Parse dates into DateTime values
startDate = DateTime.Parse(start);
endDate = DateTime.Parse(end);
//Get data from the cached web service responses
Dictionary <String, FeedResult> returnData = new Dictionary <string, FeedResult>();
FeedResult windResult = mgr.GetData(startDate, endDate, FeedManager.WIND_SPEED);
returnData.Add("windSpeed", windResult);
FeedResult tempResult = mgr.GetData(startDate, endDate, FeedManager.AIR_TEMP);
returnData.Add("airTemperature", tempResult);
FeedResult baroResult = mgr.GetData(startDate, endDate, FeedManager.BARO_PRESSURE);
returnData.Add("barometricPressure", baroResult);
//Serialize the results as JSON and return to client
var json = new JavaScriptSerializer().Serialize(returnData);
Context.Response.Write(json);
}
catch (Exception ex)
{
string exJson = GetExceptionAsJson(ex);
Context.Response.Write(exJson);
return;
}
}
/// <summary>
///
/// </summary>
/// <param name="_settings"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <exception cref="InvalidCastException">Throw when the provided settings object isn't a BeatSaverFeedSettings</exception>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="_settings"/> is null.</exception>
public async Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings _settings, CancellationToken cancellationToken)
{
PrepareReader();
if (_settings == null)
{
throw new ArgumentNullException(nameof(_settings), "Settings cannot be null for BeatSaverReader.GetSongsFromFeedAsync");
}
if (!(_settings is BeatSaverFeedSettings settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
FeedResult songs = null;
switch ((BeatSaverFeed)settings.FeedIndex)
{
// Author
case BeatSaverFeed.Author:
songs = await GetSongsByAuthorAsync(settings.Criteria, cancellationToken, CalcMaxSongs(settings.MaxPages, settings.MaxSongs)).ConfigureAwait(false);
break;
case BeatSaverFeed.Search:
songs = await SearchAsync(settings, cancellationToken).ConfigureAwait(false);
break;
// Latest/Hot/Plays/Downloads
default:
songs = await GetBeatSaverSongsAsync(settings, cancellationToken).ConfigureAwait(false);
break;
}
//Dictionary<string, ScrapedSong> retDict = new Dictionary<string, ScrapedSong>();
//foreach (var song in songs)
//{
// if (!retDict.ContainsKey(song.Hash))
// {
// retDict.Add(song.Hash, song);
// }
//}
return(songs);
}
private void DisplayCodes(FeedResult <FaultData> faultData)
{
DataTable dt = new DataTable();
dt.Columns.Add("Deivce Name");
dt.Columns.Add("Code");
dt.Columns.Add("Description");
dt.Columns.Add("Controller");
dt.Columns.Add("FMI");
dt.Columns.Add("Source");
dt.Columns.Add("Date");
MessageBox.Show($"{faultData.Data.Count}");
foreach (FaultData code in faultData.Data)
{
DataRow dr = dt.NewRow();
dr = FaultCodeRow(dr, code);
dt.Rows.Add(dr);
}
FaultCodeView.DataSource = dt;
}
public void Success_Newest()
{
BeatSaverReader reader = new BeatSaverReader()
{
StoreRawData = true
};
int maxSongs = 55;
BeatSaverFeedSettings settings = new BeatSaverFeedSettings((int)BeatSaverFeedName.Latest)
{
MaxSongs = maxSongs
};
FeedResult result = reader.GetSongsFromFeed(settings);
Assert.AreEqual(settings.MaxSongs, result.Count);
int expectedPages = ExpectedPagesForSongs(result.Count);
Assert.IsTrue(expectedPages <= result.PagesChecked);
foreach (ScrapedSong song in result.Songs.Values)
{
Console.WriteLine($"{song.Name} by {song.LevelAuthorName}, {song.Hash}");
}
}
public void OnExecutedSetsClientCachedToTrueForEmptyList()
{
RssResultActionFilter filter = new RssResultActionFilter();
ActionExecutedContext context = new ActionExecutedContext()
{
Result = new ViewResult(),
Controller = new FakeController()
{
ViewData = new ViewDataDictionary(new OxiteModelList <Post>()
{
List = Enumerable.Empty <Post>().ToList()
})
}
};
filter.OnActionExecuted(context);
FeedResult result = context.Result as FeedResult;
Assert.NotNull(result);
Assert.True(result.IsClientCached);
}
public void OnExecutedReplacesResultWithRss()
{
RssResultActionFilter filter = new RssResultActionFilter();
ActionExecutedContext context = new ActionExecutedContext()
{
Result = new ViewResult(),
Controller = new FakeController()
{
ViewData = new ViewDataDictionary(new OxiteModelList <Post>()
{
List = new List <Post>(new[] { new Post() })
})
}
};
filter.OnActionExecuted(context);
FeedResult result = context.Result as FeedResult;
Assert.NotNull(result);
Assert.True(string.Equals("Rss", result.ViewName, StringComparison.OrdinalIgnoreCase));
Assert.False(result.IsClientCached);
}
/// <summary>
/// Requests <see cref="LogRecord"/>, <see cref="FaultData"/> and <see cref="StatusData"/> records via data feeds. Then, updates local caches of "lookup" data. Finally, iterates through the returned objects, "hydrating" important child objects using their fully-populated counterparts in the caches.
/// </summary>
/// <param name="feedParameters">Contains the latest data tokens and collections to be used in the next set of data feed calls.</param>
/// <returns><see cref="FeedResultData"/></returns>
public async Task <FeedResultData> GetFeedDataAsync(FeedParameters feedParameters)
{
FeedResultData feedResults = new(new List <LogRecord>(), new List <StatusData>(), new List <FaultData>());
FeedResult <LogRecord> feedLogRecordData;
FeedResult <StatusData> feedStatusData = null;
FeedResult <FaultData> feedFaultData = null;
try
{
if (feedParameters.FeedStartOption == Common.FeedStartOption.CurrentTime || feedParameters.FeedStartOption == Common.FeedStartOption.SpecificTime)
{
// If the feeds are to be started at the current date/time or at a specific date/time, get the appropriate DateTime value and then use it to populate the fromDate parameter when making the GetFeed() calls.
DateTime feedStartTime = DateTime.UtcNow;
if (feedParameters.FeedStartOption == Common.FeedStartOption.SpecificTime)
{
feedStartTime = feedParameters.FeedStartSpecificTimeUTC;
}
feedLogRecordData = await api.GetFeedLogRecordAsync(feedStartTime);
ConsoleUtility.LogListItem("GPS log records received:", feedLogRecordData.Data.Count.ToString(), Common.ConsoleColorForListItems, (feedLogRecordData.Data.Count > 0) ? Common.ConsoleColorForChangedData : Common.ConsoleColorForUnchangedData);
if (UseStatusDataFeed == true)
{
feedStatusData = await api.GetFeedStatusDataAsync(feedStartTime);
ConsoleUtility.LogListItem("StatusData records received:", feedStatusData.Data.Count.ToString(), Common.ConsoleColorForListItems, (feedStatusData.Data.Count > 0) ? Common.ConsoleColorForChangedData : Common.ConsoleColorForUnchangedData);
}
if (UseFaultDataFeed == true)
{
feedFaultData = await api.GetFeedFaultDataAsync(feedStartTime);
ConsoleUtility.LogListItem("FaultData records received:", feedFaultData.Data.Count.ToString(), Common.ConsoleColorForListItems, (feedFaultData.Data.Count > 0) ? Common.ConsoleColorForChangedData : Common.ConsoleColorForUnchangedData);
}
// Switch to FeedResultToken for subsequent calls.
feedParameters.FeedStartOption = Common.FeedStartOption.FeedResultToken;
}
else
{
// If the feeds are to be called based on feed result token, use the tokens returned in the toVersion of previous GetFeed() calls (or loaded from file, if continuing where processing last left-off) to populate the fromVersion parameter when making the next GetFeed() calls.
feedLogRecordData = await api.GetFeedLogRecordAsync(feedParameters.LastGpsDataToken);
ConsoleUtility.LogListItem("GPS log records received:", feedLogRecordData.Data.Count.ToString(), Common.ConsoleColorForListItems, (feedLogRecordData.Data.Count > 0) ? Common.ConsoleColorForChangedData : Common.ConsoleColorForUnchangedData);
if (UseStatusDataFeed == true)
{
feedStatusData = await api.GetFeedStatusDataAsync(feedParameters.LastStatusDataToken);
ConsoleUtility.LogListItem("StatusData records received:", feedStatusData.Data.Count.ToString(), Common.ConsoleColorForListItems, (feedStatusData.Data.Count > 0) ? Common.ConsoleColorForChangedData : Common.ConsoleColorForUnchangedData);
}
if (UseFaultDataFeed == true)
{
feedFaultData = await api.GetFeedFaultDataAsync(feedParameters.LastFaultDataToken);
ConsoleUtility.LogListItem("FaultData records received:", feedFaultData.Data.Count.ToString(), Common.ConsoleColorForListItems, (feedFaultData.Data.Count > 0) ? Common.ConsoleColorForChangedData : Common.ConsoleColorForUnchangedData);
}
}
// Update local caches of "lookup" data.
if (DateTime.UtcNow > nextCacheRepopulationTime)
{
// "Feedless" caches are for object types not available via data feed in the MyGeotab API. In this case, it is necessary to updates all of the objects of each type with every update. Since these lookup data objects are not frequently-changing (if they were, they would be accessible via data feed), these caches are only updated on a specified interval instead of on every call to this GetFeedDataAsync() method in order to avoid unnecessary processing.
await UpdateAllFeedlessCachesAsync();
nextCacheRepopulationTime = DateTime.UtcNow.AddMinutes(CacheRepopulationIntervalMinutes);
}
// For object types supported by the MyGeotab API data feed, the local caches can be updated every time this GetFeedDataAsync() method is executed bacause only new or changed objects are returned.
await UpdateDeviceCacheAsync();
await UpdateDiagnosticCacheAsync();
// Use the local caches to "hydrate" child objects of objects returned via data feed.
feedParameters.LastGpsDataToken = feedLogRecordData.ToVersion;
foreach (LogRecord logRecord in feedLogRecordData.Data)
{
// Populate relevant LogRecord fields.
logRecord.Device = GetDevice(logRecord.Device);
feedResults.GpsRecords.Add(logRecord);
}
if (UseStatusDataFeed == true)
{
feedParameters.LastStatusDataToken = feedStatusData.ToVersion;
foreach (StatusData statusData in feedStatusData.Data)
{
// Populate relevant StatusData fields.
statusData.Device = GetDevice(statusData.Device);
statusData.Diagnostic = GetDiagnostic(statusData.Diagnostic);
feedResults.StatusData.Add(statusData);
}
}
if (UseFaultDataFeed == true)
{
feedParameters.LastFaultDataToken = feedFaultData.ToVersion;
foreach (FaultData faultData in feedFaultData.Data)
{
// Populate relevant FaultData fields.
faultData.Device = GetDevice(faultData.Device);
faultData.Diagnostic = GetDiagnostic(faultData.Diagnostic);
faultData.Controller = await GetControllerAsync(faultData.Controller);
faultData.FailureMode = await GetFailureModeAsync(faultData.FailureMode);
feedResults.FaultData.Add(faultData);
}
}
}
catch (Exception e)
{
Console.ForegroundColor = Common.ConsoleColorForErrors;
Console.WriteLine(e.Message);
Console.ForegroundColor = Common.ConsoleColorDefault;
if (e is HttpRequestException)
{
await Task.Delay(5000);
}
if (e is DbUnavailableException)
{
await Task.Delay(TimeSpan.FromMinutes(5));
}
}
return(feedResults);
}
protected async Task <JobStats> GetBeatSaverAsync(BeatSyncConfig config, IJobBuilder jobBuilder, JobManager jobManager, CancellationToken cancellationToken)
{
BeatSaverConfig sourceConfig = config.BeatSaver;
JobStats sourceStats = new JobStats();
if (!sourceConfig.Enabled)
{
return(sourceStats);
}
BeatSaverReader reader = new BeatSaverReader();
FeedConfigBase[] feedConfigs = new FeedConfigBase[] { sourceConfig.Hot, sourceConfig.Downloads, sourceConfig.Latest };
if (!(feedConfigs.Any(f => f.Enabled) || sourceConfig.FavoriteMappers.Enabled))
{
Logger.log?.Info($"No feeds enabled for {reader.Name}");
return(sourceStats);
}
SourceStarted?.Invoke(this, "BeatSaver");
foreach (FeedConfigBase?feedConfig in feedConfigs.Where(c => c.Enabled))
{
Logger.log?.Info($" Starting {feedConfig.GetType().Name} feed...");
FeedResult results = await reader.GetSongsFromFeedAsync(feedConfig.ToFeedSettings(), cancellationToken).ConfigureAwait(false);
if (results.Successful)
{
IEnumerable <IJob>?jobs = CreateJobs(results, jobBuilder, jobManager, cancellationToken);
JobResult[] jobResults = await Task.WhenAll(jobs.Select(j => j.JobTask).ToArray());
JobStats feedStats = new JobStats(jobResults);
ProcessFinishedJobs(jobs, jobBuilder.SongTargets, config, feedConfig);
Logger.log?.Info($" Finished {feedConfig.GetType().Name} feed: ({feedStats}).");
sourceStats += feedStats;
}
else
{
if (results.Exception != null)
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}{results.Exception.Message}");
Logger.log?.Debug($"{results.Exception}");
}
else
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: Unknown error.");
}
}
}
string[] mappers = sourceConfig.FavoriteMappers.Mappers ?? Array.Empty <string>();
if (sourceConfig.FavoriteMappers.Enabled)
{
FeedConfigBase feedConfig = sourceConfig.FavoriteMappers;
if (mappers.Length > 0)
{
Logger.log?.Info(" Starting FavoriteMappers feed...");
List <IPlaylist> playlists = new List <IPlaylist>();
List <IPlaylist> feedPlaylists = new List <IPlaylist>();
List <IPlaylist> recentPlaylists = new List <IPlaylist>();
JobStats feedStats = new JobStats();
foreach (string?mapper in mappers)
{
Logger.log?.Info($" Getting songs by {mapper}...");
playlists.Clear();
FeedResult results = await reader.GetSongsFromFeedAsync(sourceConfig.FavoriteMappers.ToFeedSettings(mapper)).ConfigureAwait(false);
if (results.Successful)
{
foreach (ITargetWithPlaylists?targetWithPlaylist in jobBuilder.SongTargets.Where(t => t is ITargetWithPlaylists).Select(t => (ITargetWithPlaylists)t))
{
PlaylistManager?playlistManager = targetWithPlaylist.PlaylistManager;
if (playlistManager != null)
{
if (config.RecentPlaylistDays > 0)
{
recentPlaylists.Add(playlistManager.GetOrAddPlaylist(BuiltInPlaylist.BeatSyncRecent));
}
if (config.AllBeatSyncSongsPlaylist)
{
playlists.Add(playlistManager.GetOrAddPlaylist(BuiltInPlaylist.BeatSyncAll));
}
if (sourceConfig.FavoriteMappers.CreatePlaylist)
{
IPlaylist feedPlaylist;
try
{
if (sourceConfig.FavoriteMappers.SeparateMapperPlaylists)
{
feedPlaylist = playlistManager.GetOrCreateAuthorPlaylist(mapper);
}
else
{
feedPlaylist = playlistManager.GetOrAddPlaylist(BuiltInPlaylist.BeatSaverFavoriteMappers);
}
feedPlaylists.Add(feedPlaylist);
playlists.Add(feedPlaylist);
}
catch (ArgumentException ex)
{
Logger.log?.Error($"Error getting playlist for FavoriteMappers: {ex.Message}");
Logger.log?.Debug(ex);
}
}
}
}
IEnumerable <IJob> jobs = CreateJobs(results, jobBuilder, jobManager, cancellationToken) ?? Array.Empty <IJob>();
JobResult[] jobResults = await Task.WhenAll(jobs.Select(j => j.JobTask).ToArray());
JobStats mapperStats = new JobStats(jobResults);
feedStats += mapperStats;
if (jobs.Any(j => j.Result?.Successful ?? false) && feedConfig.PlaylistStyle == PlaylistStyle.Replace)
{
// TODO: This should only apply to successful targets.
foreach (IPlaylist?feedPlaylist in feedPlaylists)
{
feedPlaylist.Clear();
feedPlaylist.RaisePlaylistChanged();
}
}
ProcessFinishedJobs(jobs, playlists, recentPlaylists);
Logger.log?.Info($" Finished getting songs by {mapper}: ({mapperStats}).");
}
else
{
if (results.Exception != null)
{
Logger.log?.Error($"Error getting songs by {mapper}: {results.Exception.Message}");
Logger.log?.Debug(results.Exception);
}
else
{
Logger.log?.Error($"Error getting songs by {mapper}");
}
}
}
sourceStats += feedStats;
Logger.log?.Info($" Finished {feedConfig.GetType().Name} feed: ({feedStats}).");
}
else
{
Logger.log?.Warn(" No FavoriteMappers found, skipping...");
}
}
Logger.log?.Info($" Finished BeatSaver reading: ({sourceStats}).");
return(sourceStats);
}
