public async Task TestParseCompanies() { string s = await File.ReadAllTextAsync("resources/common_companies.json"); JArray jarrCompany = JArray.Parse(s); Esk8Service ESS = new Esk8Service(logger); List <Company> companies = CompanyParser.ParseCompanies(jarrCompany); Assert.Equal(42, companies.Count); return; }
public async Task TestCombineProductCompanies() { Esk8Service ESS = new Esk8Service(logger); string rawCompanies = await File.ReadAllTextAsync("resources/common_companies.json"); string rawProducts = await File.ReadAllTextAsync("resources/common_boards.json"); JArray jarrCompany = JArray.Parse(rawCompanies); JArray jarrProduct = JArray.Parse(rawProducts); List <Company> companies = CompanyParser.ParseCompanies(jarrCompany); List <Product> products = ProductParser.ParseProducts(jarrProduct); List <Company> combined = CompanyParser.CombineCompanyLists(companies, products.Select(x => x.Company)); Assert.Equal(42, combined.Count); }
// Methods public async Task <List <Company> > GetCommonCompanies() { try { HttpResponseMessage m = await Client.GetAsync(COMMONCOMPANIESURL); if (!m.IsSuccessStatusCode) { Logger.Log($"Failed while trying to get Common Companies from Esk8 server. Code: {m.StatusCode}, Reason: {m.ReasonPhrase}"); return(null); } string companyjson = await m.Content.ReadAsStringAsync(); JArray comps = JArray.Parse(companyjson); return(CompanyParser.ParseCompanies(comps)); } catch (Exception e) { Logger.Log($"An unknown error ocurred while deserializing the Common Companies json file:\n{e.Message}"); return(null); } }
public async Task TestMakeEmailsFromMatches() { MailgunService MSS = new MailgunService(logger); string bstthreadloc = await File.ReadAllTextAsync("resources/bstthread.json"); JArray BSTThreadJson = JArray.Parse(bstthreadloc); RedditService RSS = new RedditService(logger); // Get BST thread string rawCompanies = await File.ReadAllTextAsync("resources/common_companies.json"); string rawProducts = await File.ReadAllTextAsync("resources/common_boards.json"); JArray jarrCompany = JArray.Parse(rawCompanies); JArray jarrProduct = JArray.Parse(rawProducts); List <Company> companies = CompanyParser.ParseCompanies(jarrCompany); List <Product> products = ProductParser.ParseProducts(jarrProduct); List <Company> combined = CompanyParser.CombineCompanyLists(companies, products.Select(x => x.Company)); Esk8Service ESS = new Esk8Service(logger); List <RegexCategory <Company> > CompRs = ESS.GetCompanyRegexs(companies); List <RegexCategory <Product> > ProdRs = ESS.GetProductRegexs(products); List <BSTComment> comments = RSS.ParseComments(BSTThreadJson, CompRs, ProdRs); // Get mock subscribers string subloc = await File.ReadAllTextAsync("resources/Subscribers.json"); JArray subArray = JArray.Parse(subloc); List <PostedSubscribeObject> psubs = SubscriberParser.ParseSubscribers(subArray); List <Subscriber> subs = psubs.Select(x => Subscriber.FromPostedSubscriber(x)).ToList(); MatchService LSS = new MatchService(logger); Dictionary <Subscriber, List <LambdaMatch> > matches = LSS.MakeMatches(subs, comments); List <MailgunEmail> emails = matches.Select(x => MSS.MakeEmail(x.Key, x.Value)).ToList(); }
public async Task TestParseBSTThreadWithUntilDate() { string s = await File.ReadAllTextAsync("resources/bstthread.json"); JArray BSTThreadJson = JArray.Parse(s); RedditService RSS = new RedditService(logger); // Get Companies, Products, Regexes, etc string rawCompanies = await File.ReadAllTextAsync("resources/common_companies.json"); string rawProducts = await File.ReadAllTextAsync("resources/common_boards.json"); JArray jarrCompany = JArray.Parse(rawCompanies); JArray jarrProduct = JArray.Parse(rawProducts); List <Company> companies = CompanyParser.ParseCompanies(jarrCompany); List <Product> products = ProductParser.ParseProducts(jarrProduct); List <Company> combined = CompanyParser.CombineCompanyLists(companies, products.Select(x => x.Company)); Esk8Service ESS = new Esk8Service(logger); List <RegexCategory <Company> > CompRs = ESS.GetCompanyRegexs(companies); List <RegexCategory <Product> > ProdRs = ESS.GetProductRegexs(products); DateTimeOffset Aug42018 = new DateTimeOffset(2018, 8, 4, 0, 0, 0, TimeSpan.FromSeconds(0)); List <BSTComment> comments = RSS.ParseComments(BSTThreadJson, CompRs, ProdRs, Aug42018); // Assert that there are 29 items in this json Assert.Equal(18, comments.Count); // Assert that there are X items for WTB Assert.Equal(1, comments.Count(x => x.BuySellTradeStatus == BST.BUY)); // Assert that there are Y items for WTS Assert.Equal(17, comments.Count(x => x.BuySellTradeStatus == BST.SELL)); // Assert that there are 0 items for Trade Assert.Equal(0, comments.Count(x => x.BuySellTradeStatus == BST.TRADE)); }
public static CompaniesUsersAddresses LoadFrom(ITableReader reader) { Company[] companies = null; User[] users = null; Address[] addresses = null; foreach (ITable table in reader.GetTables()) { switch (table.Name) { case "Companies": companies = new CompanyParser().GetAll(table).ToArray(); break; case "Users": users = new UserParser().GetAll(table).ToArray(); break; case "Addresses": addresses = new AddressParser().GetAll(table).ToArray(); break; } } return new CompaniesUsersAddresses(companies, users, addresses); }
public async Task TestLiveScan() { RedditService RSS = new RedditService(logger); Esk8Service ESS = new Esk8Service(logger); /* Get the BST Thread */ if (!(await RSS.GetRedditItem() is JObject frontPage)) { /* Some kind of error ocurred, do not update */ return; } string BSTUrl = RSS.FindBSTThreadUrl(frontPage); if (String.IsNullOrWhiteSpace(BSTUrl)) { /* Some kind of error ocurred, do not update */ return; } if (!(await RSS.GetRedditItem(BSTUrl) is JArray BSTPage)) { /* Some kind of error ocurred, do not update */ return; } List <Product> prods = await ESS.GetCommonBoards(); List <Company> comps = await ESS.GetCommonCompanies(); comps = CompanyParser.CombineCompanyLists(comps, prods.Select(x => x.Company)); List <RegexCategory <Company> > CompRs = ESS.GetCompanyRegexs(comps); List <RegexCategory <Product> > ProdRs = ESS.GetProductRegexs(prods); /* Parse the full thread for new posts */ List <BSTComment> comments = RSS.ParseComments(BSTPage, CompRs, ProdRs, DateTimeOffset.MinValue); }
/// <summary> /// The core function launched every hour. /// </summary> /// <param name="request"></param> /// <param name="context"></param> /// <returns></returns> public async Task Scan(APIGatewayProxyRequest request, ILambdaContext context) { var logger = new Esk8LambdaLogger(context.Logger); logger.Log("Scan initiated endpoint reached"); HttpClient Client = new HttpClient(); /* Set up our Services */ FirestoreService FSS = new FirestoreService(logger); Esk8Service ESS = new Esk8Service(logger, Client); RedditService RSS = new RedditService(logger, ESS, Client); MailgunService MSS = new MailgunService(logger, Client); MatchService LSS = new MatchService(logger); /* Get our Firebase last-scanned time */ ScanData sd = await FSS.GetScanData(); if (sd == null) { return; /* Something happened, so we quit. */ } /* Check Firebase Subscribers */ List <Subscriber> subscribers = await FSS.GetSubscribers(); if (subscribers == null || subscribers.Count == 0) { ScanData updatedScanData = new ScanData() { LastScanDate = DateTimeOffset.Now, MostRecentlySeen = DateTimeOffset.Now }; await FSS.UpdateScanTime(updatedScanData); return; } /* Get the BST Thread */ if (!(await RSS.GetRedditItem() is JObject frontPage)) { /* Some kind of error ocurred, do not update */ return; } string BSTUrl = RSS.FindBSTThreadUrl(frontPage); if (String.IsNullOrWhiteSpace(BSTUrl)) { /* Some kind of error ocurred, do not update */ return; } if (!(await RSS.GetRedditItem(BSTUrl) is JArray BSTPage)) { /* Some kind of error ocurred, do not update */ return; } /* Check if there are new posts, to save on Network requests fetching Company / Board information */ if (!RSS.AnyNewPosts(BSTPage, sd.MostRecentlySeen)) { /* There have been no new posts since last time. */ ScanData updatedScanData = new ScanData() { LastScanDate = DateTimeOffset.Now, MostRecentlySeen = DateTimeOffset.Now }; await FSS.UpdateScanTime(updatedScanData); return; } /* Fetch Company and Product information from Esk8 servers */ List <Product> prods = await ESS.GetCommonBoards(); List <Company> comps = await ESS.GetCommonCompanies(); comps = CompanyParser.CombineCompanyLists(comps, prods.Select(x => x.Company)); List <RegexCategory <Company> > CompRs = ESS.GetCompanyRegexs(comps); List <RegexCategory <Product> > ProdRs = ESS.GetProductRegexs(prods); /* Parse the full thread for new posts */ List <BSTComment> comments = RSS.ParseComments(BSTPage, CompRs, ProdRs, sd.LastScanDate); /* Line up potential Match Objects */ /* At this point we don't update until emails have been sent out. */ Dictionary <Subscriber, List <LambdaMatch> > lambdadelta = LSS.MakeMatches(subscribers, comments); /* Assemble the emails to send */ IEnumerable <MailgunEmail> emails = lambdadelta.Select(x => MSS.MakeEmail(x.Key, x.Value)); /* Send emails */ bool SentSuccessfully = await MSS.BatchSend(emails); if (SentSuccessfully) { ScanData updatedScanData = new ScanData() { LastScanDate = DateTimeOffset.Now, MostRecentlySeen = DateTimeOffset.Now }; await FSS.UpdateScanTime(updatedScanData); } }
public async Task TestMakeLambdaDelta() { string bstthreadloc = await File.ReadAllTextAsync("resources/bstthread.json"); JArray BSTThreadJson = JArray.Parse(bstthreadloc); RedditService RSS = new RedditService(logger); // Get BST thread string rawCompanies = await File.ReadAllTextAsync("resources/common_companies.json"); string rawProducts = await File.ReadAllTextAsync("resources/common_boards.json"); JArray jarrCompany = JArray.Parse(rawCompanies); JArray jarrProduct = JArray.Parse(rawProducts); List <Company> companies = CompanyParser.ParseCompanies(jarrCompany); List <Product> products = ProductParser.ParseProducts(jarrProduct); List <Company> combined = CompanyParser.CombineCompanyLists(companies, products.Select(x => x.Company)); Esk8Service ESS = new Esk8Service(logger); List <RegexCategory <Company> > CompRs = ESS.GetCompanyRegexs(companies); List <RegexCategory <Product> > ProdRs = ESS.GetProductRegexs(products); List <BSTComment> comments = RSS.ParseComments(BSTThreadJson, CompRs, ProdRs); // Get mock subscribers string subloc = await File.ReadAllTextAsync("resources/Subscribers.json"); JArray subArray = JArray.Parse(subloc); List <PostedSubscribeObject> psubs = SubscriberParser.ParseSubscribers(subArray); List <Subscriber> subs = psubs.Select(x => Subscriber.FromPostedSubscriber(x)).ToList(); Subscriber BoostedMeepoWowGoEvolveGuy = subs.FirstOrDefault(x => x.DocumentId == "*****@*****.**".ToLower()); Subscriber LiterallyEverythingGuy = subs.FirstOrDefault(x => x.DocumentId == "*****@*****.**".ToLower()); Subscriber WTSABackfire = subs.FirstOrDefault(x => x.DocumentId == "*****@*****.**".ToLower()); MatchService LSS = new MatchService(logger); Dictionary <Subscriber, List <LambdaMatch> > matches = LSS.MakeMatches(subs, comments); // Assert that 3 people had matches on this thread Assert.Equal(3, matches.Count); // Assert that Guy#1 had 10 matches, all of which are either Boosted, Meepo, WowGo, or Evolve below $1000 USD var bmwegMatches = matches[BoostedMeepoWowGoEvolveGuy]; Assert.Equal(10, bmwegMatches[0].Posts.Count); Assert.True(bmwegMatches[0].Posts.All(x => new string[] { "boosted", "wowgo", "meepo", "evolve" }.Contains(x.Company.ToLowerInvariant()))); Assert.True(bmwegMatches[0].Posts.All(x => x.Price <= bmwegMatches[0].FbMatch.Price)); Assert.True(bmwegMatches[0].Posts.All(x => x.Currency == "USD")); Assert.True(bmwegMatches[0].Posts.All(x => x.BuySellTradeStatus == BST.SELL)); // Assert that Guy#2 matched literally everything var leMatches = matches[LiterallyEverythingGuy]; Assert.Equal(comments.Count, leMatches[0].Posts.Count); // Comment count and return count should always be equal - this guy just wants to see everything! // Assert that Guy#3 has a single match for someome who wants to buy his Backfire var bfMatches = matches[WTSABackfire]; Assert.Single(bfMatches[0].Posts); Assert.True(bfMatches[0].Posts.All(x => x.Company.ToLower() == "backfire")); Assert.True(bfMatches[0].Posts.All(x => x.BuySellTradeStatus == BST.BUY)); }
public UserProcessor(CompanyParser companyParser) { this.companyParser = companyParser ?? throw new ArgumentNullException(nameof(companyParser)); }