public async Task TestParseProducts()
{
string s = await File.ReadAllTextAsync("resources/common_boards.json");
JArray jarrProduct = JArray.Parse(s);
Esk8Service ESS = new Esk8Service(logger);
List <Product> Products = ProductParser.ParseProducts(jarrProduct);
Assert.Equal(138, Products.Count);
return;
}
public async Task TestParseCompanies()
{
string s = await File.ReadAllTextAsync("resources/common_companies.json");
JArray jarrCompany = JArray.Parse(s);
Esk8Service ESS = new Esk8Service(logger);
List <Company> companies = CompanyParser.ParseCompanies(jarrCompany);
Assert.Equal(42, companies.Count);
return;
}
public async Task TestCombineProductCompanies()
{
Esk8Service ESS = new Esk8Service(logger);
string rawCompanies = await File.ReadAllTextAsync("resources/common_companies.json");
string rawProducts = await File.ReadAllTextAsync("resources/common_boards.json");
JArray jarrCompany = JArray.Parse(rawCompanies);
JArray jarrProduct = JArray.Parse(rawProducts);
List <Company> companies = CompanyParser.ParseCompanies(jarrCompany);
List <Product> products = ProductParser.ParseProducts(jarrProduct);
List <Company> combined = CompanyParser.CombineCompanyLists(companies, products.Select(x => x.Company));
Assert.Equal(42, combined.Count);
}
public async Task TestMakeEmailsFromMatches()
{
MailgunService MSS = new MailgunService(logger);
string bstthreadloc = await File.ReadAllTextAsync("resources/bstthread.json");
JArray BSTThreadJson = JArray.Parse(bstthreadloc);
RedditService RSS = new RedditService(logger);
// Get BST thread
string rawCompanies = await File.ReadAllTextAsync("resources/common_companies.json");
string rawProducts = await File.ReadAllTextAsync("resources/common_boards.json");
JArray jarrCompany = JArray.Parse(rawCompanies);
JArray jarrProduct = JArray.Parse(rawProducts);
List <Company> companies = CompanyParser.ParseCompanies(jarrCompany);
List <Product> products = ProductParser.ParseProducts(jarrProduct);
List <Company> combined = CompanyParser.CombineCompanyLists(companies, products.Select(x => x.Company));
Esk8Service ESS = new Esk8Service(logger);
List <RegexCategory <Company> > CompRs = ESS.GetCompanyRegexs(companies);
List <RegexCategory <Product> > ProdRs = ESS.GetProductRegexs(products);
List <BSTComment> comments = RSS.ParseComments(BSTThreadJson, CompRs, ProdRs);
// Get mock subscribers
string subloc = await File.ReadAllTextAsync("resources/Subscribers.json");
JArray subArray = JArray.Parse(subloc);
List <PostedSubscribeObject> psubs = SubscriberParser.ParseSubscribers(subArray);
List <Subscriber> subs = psubs.Select(x => Subscriber.FromPostedSubscriber(x)).ToList();
MatchService LSS = new MatchService(logger);
Dictionary <Subscriber, List <LambdaMatch> > matches = LSS.MakeMatches(subs, comments);
List <MailgunEmail> emails = matches.Select(x => MSS.MakeEmail(x.Key, x.Value)).ToList();
}
public async Task TestParseBSTThreadWithUntilDate()
{
string s = await File.ReadAllTextAsync("resources/bstthread.json");
JArray BSTThreadJson = JArray.Parse(s);
RedditService RSS = new RedditService(logger);
// Get Companies, Products, Regexes, etc
string rawCompanies = await File.ReadAllTextAsync("resources/common_companies.json");
string rawProducts = await File.ReadAllTextAsync("resources/common_boards.json");
JArray jarrCompany = JArray.Parse(rawCompanies);
JArray jarrProduct = JArray.Parse(rawProducts);
List <Company> companies = CompanyParser.ParseCompanies(jarrCompany);
List <Product> products = ProductParser.ParseProducts(jarrProduct);
List <Company> combined = CompanyParser.CombineCompanyLists(companies, products.Select(x => x.Company));
Esk8Service ESS = new Esk8Service(logger);
List <RegexCategory <Company> > CompRs = ESS.GetCompanyRegexs(companies);
List <RegexCategory <Product> > ProdRs = ESS.GetProductRegexs(products);
DateTimeOffset Aug42018 = new DateTimeOffset(2018, 8, 4, 0, 0, 0, TimeSpan.FromSeconds(0));
List <BSTComment> comments = RSS.ParseComments(BSTThreadJson, CompRs, ProdRs, Aug42018);
// Assert that there are 29 items in this json
Assert.Equal(18, comments.Count);
// Assert that there are X items for WTB
Assert.Equal(1, comments.Count(x => x.BuySellTradeStatus == BST.BUY));
// Assert that there are Y items for WTS
Assert.Equal(17, comments.Count(x => x.BuySellTradeStatus == BST.SELL));
// Assert that there are 0 items for Trade
Assert.Equal(0, comments.Count(x => x.BuySellTradeStatus == BST.TRADE));
}
public async Task TestLiveScan()
{
RedditService RSS = new RedditService(logger);
Esk8Service ESS = new Esk8Service(logger);
/* Get the BST Thread */
if (!(await RSS.GetRedditItem() is JObject frontPage))
{
/* Some kind of error ocurred, do not update */
return;
}
string BSTUrl = RSS.FindBSTThreadUrl(frontPage);
if (String.IsNullOrWhiteSpace(BSTUrl))
{
/* Some kind of error ocurred, do not update */
return;
}
if (!(await RSS.GetRedditItem(BSTUrl) is JArray BSTPage))
{
/* Some kind of error ocurred, do not update */
return;
}
List <Product> prods = await ESS.GetCommonBoards();
List <Company> comps = await ESS.GetCommonCompanies();
comps = CompanyParser.CombineCompanyLists(comps, prods.Select(x => x.Company));
List <RegexCategory <Company> > CompRs = ESS.GetCompanyRegexs(comps);
List <RegexCategory <Product> > ProdRs = ESS.GetProductRegexs(prods);
/* Parse the full thread for new posts */
List <BSTComment> comments = RSS.ParseComments(BSTPage, CompRs, ProdRs, DateTimeOffset.MinValue);
}
/// <summary>
/// The core function launched every hour.
/// </summary>
/// <param name="request"></param>
/// <param name="context"></param>
/// <returns></returns>
public async Task Scan(APIGatewayProxyRequest request, ILambdaContext context)
{
var logger = new Esk8LambdaLogger(context.Logger);
logger.Log("Scan initiated endpoint reached");
HttpClient Client = new HttpClient();
/* Set up our Services */
FirestoreService FSS = new FirestoreService(logger);
Esk8Service ESS = new Esk8Service(logger, Client);
RedditService RSS = new RedditService(logger, ESS, Client);
MailgunService MSS = new MailgunService(logger, Client);
MatchService LSS = new MatchService(logger);
/* Get our Firebase last-scanned time */
ScanData sd = await FSS.GetScanData();
if (sd == null)
{
return; /* Something happened, so we quit. */
}
/* Check Firebase Subscribers */
List <Subscriber> subscribers = await FSS.GetSubscribers();
if (subscribers == null || subscribers.Count == 0)
{
ScanData updatedScanData = new ScanData()
{
LastScanDate = DateTimeOffset.Now, MostRecentlySeen = DateTimeOffset.Now
};
await FSS.UpdateScanTime(updatedScanData);
return;
}
/* Get the BST Thread */
if (!(await RSS.GetRedditItem() is JObject frontPage))
{
/* Some kind of error ocurred, do not update */
return;
}
string BSTUrl = RSS.FindBSTThreadUrl(frontPage);
if (String.IsNullOrWhiteSpace(BSTUrl))
{
/* Some kind of error ocurred, do not update */
return;
}
if (!(await RSS.GetRedditItem(BSTUrl) is JArray BSTPage))
{
/* Some kind of error ocurred, do not update */
return;
}
/* Check if there are new posts, to save on Network requests fetching Company / Board information */
if (!RSS.AnyNewPosts(BSTPage, sd.MostRecentlySeen))
{
/* There have been no new posts since last time. */
ScanData updatedScanData = new ScanData()
{
LastScanDate = DateTimeOffset.Now, MostRecentlySeen = DateTimeOffset.Now
};
await FSS.UpdateScanTime(updatedScanData);
return;
}
/* Fetch Company and Product information from Esk8 servers */
List <Product> prods = await ESS.GetCommonBoards();
List <Company> comps = await ESS.GetCommonCompanies();
comps = CompanyParser.CombineCompanyLists(comps, prods.Select(x => x.Company));
List <RegexCategory <Company> > CompRs = ESS.GetCompanyRegexs(comps);
List <RegexCategory <Product> > ProdRs = ESS.GetProductRegexs(prods);
/* Parse the full thread for new posts */
List <BSTComment> comments = RSS.ParseComments(BSTPage, CompRs, ProdRs, sd.LastScanDate);
/* Line up potential Match Objects */
/* At this point we don't update until emails have been sent out. */
Dictionary <Subscriber, List <LambdaMatch> > lambdadelta = LSS.MakeMatches(subscribers, comments);
/* Assemble the emails to send */
IEnumerable <MailgunEmail> emails = lambdadelta.Select(x => MSS.MakeEmail(x.Key, x.Value));
/* Send emails */
bool SentSuccessfully = await MSS.BatchSend(emails);
if (SentSuccessfully)
{
ScanData updatedScanData = new ScanData()
{
LastScanDate = DateTimeOffset.Now, MostRecentlySeen = DateTimeOffset.Now
};
await FSS.UpdateScanTime(updatedScanData);
}
}
public async Task TestMakeLambdaDelta()
{
string bstthreadloc = await File.ReadAllTextAsync("resources/bstthread.json");
JArray BSTThreadJson = JArray.Parse(bstthreadloc);
RedditService RSS = new RedditService(logger);
// Get BST thread
string rawCompanies = await File.ReadAllTextAsync("resources/common_companies.json");
string rawProducts = await File.ReadAllTextAsync("resources/common_boards.json");
JArray jarrCompany = JArray.Parse(rawCompanies);
JArray jarrProduct = JArray.Parse(rawProducts);
List <Company> companies = CompanyParser.ParseCompanies(jarrCompany);
List <Product> products = ProductParser.ParseProducts(jarrProduct);
List <Company> combined = CompanyParser.CombineCompanyLists(companies, products.Select(x => x.Company));
Esk8Service ESS = new Esk8Service(logger);
List <RegexCategory <Company> > CompRs = ESS.GetCompanyRegexs(companies);
List <RegexCategory <Product> > ProdRs = ESS.GetProductRegexs(products);
List <BSTComment> comments = RSS.ParseComments(BSTThreadJson, CompRs, ProdRs);
// Get mock subscribers
string subloc = await File.ReadAllTextAsync("resources/Subscribers.json");
JArray subArray = JArray.Parse(subloc);
List <PostedSubscribeObject> psubs = SubscriberParser.ParseSubscribers(subArray);
List <Subscriber> subs = psubs.Select(x => Subscriber.FromPostedSubscriber(x)).ToList();
Subscriber BoostedMeepoWowGoEvolveGuy = subs.FirstOrDefault(x => x.DocumentId == "*****@*****.**".ToLower());
Subscriber LiterallyEverythingGuy = subs.FirstOrDefault(x => x.DocumentId == "*****@*****.**".ToLower());
Subscriber WTSABackfire = subs.FirstOrDefault(x => x.DocumentId == "*****@*****.**".ToLower());
MatchService LSS = new MatchService(logger);
Dictionary <Subscriber, List <LambdaMatch> > matches = LSS.MakeMatches(subs, comments);
// Assert that 3 people had matches on this thread
Assert.Equal(3, matches.Count);
// Assert that Guy#1 had 10 matches, all of which are either Boosted, Meepo, WowGo, or Evolve below $1000 USD
var bmwegMatches = matches[BoostedMeepoWowGoEvolveGuy];
Assert.Equal(10, bmwegMatches[0].Posts.Count);
Assert.True(bmwegMatches[0].Posts.All(x => new string[] { "boosted", "wowgo", "meepo", "evolve" }.Contains(x.Company.ToLowerInvariant())));
Assert.True(bmwegMatches[0].Posts.All(x => x.Price <= bmwegMatches[0].FbMatch.Price));
Assert.True(bmwegMatches[0].Posts.All(x => x.Currency == "USD"));
Assert.True(bmwegMatches[0].Posts.All(x => x.BuySellTradeStatus == BST.SELL));
// Assert that Guy#2 matched literally everything
var leMatches = matches[LiterallyEverythingGuy];
Assert.Equal(comments.Count, leMatches[0].Posts.Count); // Comment count and return count should always be equal - this guy just wants to see everything!
// Assert that Guy#3 has a single match for someome who wants to buy his Backfire
var bfMatches = matches[WTSABackfire];
Assert.Single(bfMatches[0].Posts);
Assert.True(bfMatches[0].Posts.All(x => x.Company.ToLower() == "backfire"));
Assert.True(bfMatches[0].Posts.All(x => x.BuySellTradeStatus == BST.BUY));
}
