public async Task TestParseFrontPage()
{
string s = await File.ReadAllTextAsync("resources/frontpage.json");
JObject frontpagejson = JObject.Parse(s);
RedditService RSS = new RedditService(logger);
string bsturl = RSS.FindBSTThreadUrl(frontpagejson);
Assert.Equal("https://old.reddit.com/r/ElectricSkateboarding/comments/93t8gd/buyselltrade_monthly_sticky_august/.json", bsturl);
}
public async Task TestLiveScan()
{
RedditService RSS = new RedditService(logger);
Esk8Service ESS = new Esk8Service(logger);
/* Get the BST Thread */
if (!(await RSS.GetRedditItem() is JObject frontPage))
{
/* Some kind of error ocurred, do not update */
return;
}
string BSTUrl = RSS.FindBSTThreadUrl(frontPage);
if (String.IsNullOrWhiteSpace(BSTUrl))
{
/* Some kind of error ocurred, do not update */
return;
}
if (!(await RSS.GetRedditItem(BSTUrl) is JArray BSTPage))
{
/* Some kind of error ocurred, do not update */
return;
}
List <Product> prods = await ESS.GetCommonBoards();
List <Company> comps = await ESS.GetCommonCompanies();
comps = CompanyParser.CombineCompanyLists(comps, prods.Select(x => x.Company));
List <RegexCategory <Company> > CompRs = ESS.GetCompanyRegexs(comps);
List <RegexCategory <Product> > ProdRs = ESS.GetProductRegexs(prods);
/* Parse the full thread for new posts */
List <BSTComment> comments = RSS.ParseComments(BSTPage, CompRs, ProdRs, DateTimeOffset.MinValue);
}
/// <summary>
/// The core function launched every hour.
/// </summary>
/// <param name="request"></param>
/// <param name="context"></param>
/// <returns></returns>
public async Task Scan(APIGatewayProxyRequest request, ILambdaContext context)
{
var logger = new Esk8LambdaLogger(context.Logger);
logger.Log("Scan initiated endpoint reached");
HttpClient Client = new HttpClient();
/* Set up our Services */
FirestoreService FSS = new FirestoreService(logger);
Esk8Service ESS = new Esk8Service(logger, Client);
RedditService RSS = new RedditService(logger, ESS, Client);
MailgunService MSS = new MailgunService(logger, Client);
MatchService LSS = new MatchService(logger);
/* Get our Firebase last-scanned time */
ScanData sd = await FSS.GetScanData();
if (sd == null)
{
return; /* Something happened, so we quit. */
}
/* Check Firebase Subscribers */
List <Subscriber> subscribers = await FSS.GetSubscribers();
if (subscribers == null || subscribers.Count == 0)
{
ScanData updatedScanData = new ScanData()
{
LastScanDate = DateTimeOffset.Now, MostRecentlySeen = DateTimeOffset.Now
};
await FSS.UpdateScanTime(updatedScanData);
return;
}
/* Get the BST Thread */
if (!(await RSS.GetRedditItem() is JObject frontPage))
{
/* Some kind of error ocurred, do not update */
return;
}
string BSTUrl = RSS.FindBSTThreadUrl(frontPage);
if (String.IsNullOrWhiteSpace(BSTUrl))
{
/* Some kind of error ocurred, do not update */
return;
}
if (!(await RSS.GetRedditItem(BSTUrl) is JArray BSTPage))
{
/* Some kind of error ocurred, do not update */
return;
}
/* Check if there are new posts, to save on Network requests fetching Company / Board information */
if (!RSS.AnyNewPosts(BSTPage, sd.MostRecentlySeen))
{
/* There have been no new posts since last time. */
ScanData updatedScanData = new ScanData()
{
LastScanDate = DateTimeOffset.Now, MostRecentlySeen = DateTimeOffset.Now
};
await FSS.UpdateScanTime(updatedScanData);
return;
}
/* Fetch Company and Product information from Esk8 servers */
List <Product> prods = await ESS.GetCommonBoards();
List <Company> comps = await ESS.GetCommonCompanies();
comps = CompanyParser.CombineCompanyLists(comps, prods.Select(x => x.Company));
List <RegexCategory <Company> > CompRs = ESS.GetCompanyRegexs(comps);
List <RegexCategory <Product> > ProdRs = ESS.GetProductRegexs(prods);
/* Parse the full thread for new posts */
List <BSTComment> comments = RSS.ParseComments(BSTPage, CompRs, ProdRs, sd.LastScanDate);
/* Line up potential Match Objects */
/* At this point we don't update until emails have been sent out. */
Dictionary <Subscriber, List <LambdaMatch> > lambdadelta = LSS.MakeMatches(subscribers, comments);
/* Assemble the emails to send */
IEnumerable <MailgunEmail> emails = lambdadelta.Select(x => MSS.MakeEmail(x.Key, x.Value));
/* Send emails */
bool SentSuccessfully = await MSS.BatchSend(emails);
if (SentSuccessfully)
{
ScanData updatedScanData = new ScanData()
{
LastScanDate = DateTimeOffset.Now, MostRecentlySeen = DateTimeOffset.Now
};
await FSS.UpdateScanTime(updatedScanData);
}
}
