public static async Task ScrapeAllSources()
{
//var dataSources = await SourceClient.GetSourcesAsync();
var areasOfConcern = await AreaClient.GetAreasAsync();
var allTemplates = await TemplateClient.GetTemplatesAsync();
var topLevelBrowser = new ScrapingBrowser();
// Dunno what these do but the tutorial had them.
//topLevelBrowser.AllowAutoRedirect = true;
//topLevelBrowser.AllowMetaRedirect = true;
foreach (var area in areasOfConcern)
{
foreach (var entryPoint in area.EntryPoints)
{
// Get a scraping Template for the Source associated with the current Entry Point.
var topLevelTemplate = allTemplates
.First(t => t.SourceId == entryPoint.SourceId && t.IsTopLevel);
// Create new scraping event and assign the area ID.
var scrapingEvent = new ScrapeEvent();
scrapingEvent.AreaId = area.AreaId;
// Create the record collection.
scrapingEvent.Records = new System.Collections.ObjectModel.ObservableCollection <ScrapeRecord>();
// Begin scraping from the top-level Template.
ScrapeFromTemplate(topLevelBrowser, scrapingEvent, entryPoint.EntryPointUri, topLevelTemplate, allTemplates);
// Send scraping event to the WebServices to be added to the database.
await TemplateClient.AddScrapeEventAsync(topLevelTemplate.TemplateId, scrapingEvent);
}
}
}
private static async Task EmailSubscribers()
{
var availableTemplates = await TemplateClient.GetTemplatesAsync();
var zillowTemplate = availableTemplates.First(t => t.IsTopLevel);
var recentZillowScrapeEvent = await TemplateClient.GetMostRecentScrapeEventAsync(zillowTemplate.TemplateId);
var recordTemplates = new List <string>();
foreach (var record in recentZillowScrapeEvent.Records)
{
var template = ZillowEmailRecordTemplate;
var recordData = record.TargetFieldIdToValueDictionary;
// Satisfy all "if" blocks first.
var ifSubstr = "<if present=\"";
var ifLocation = template.IndexOf(ifSubstr);
while (ifLocation != -1)
{
var ifClosing = template.IndexOf("</if>", ifLocation);
var openingLen = ifLocation + ifSubstr.Length;
var idQuoteClosing = template.IndexOf("\"", openingLen);
var targetIdOfConcern = template.Substring(openingLen, idQuoteClosing - openingLen);
var continueFrom = ifClosing;
// If the record does not contain the ID found within the "if" block, remove the block and its' contents.
if (!recordData.ContainsKey(targetIdOfConcern))
{
template = template.Substring(0, ifLocation) + template.Substring(ifClosing + 5);
continueFrom = ifLocation;
}
// Check for next "if" block.
ifLocation = template.IndexOf(ifSubstr, continueFrom);
}
// Replace all remaining "<targetId:x>" tags with the corresponding data value.
var targetTagSubstr = "<targetId:";
var tagLocation = template.IndexOf(targetTagSubstr);
while (tagLocation != -1)
{
var tagClosing = template.IndexOf(">", tagLocation);
var openingLen = tagLocation + targetTagSubstr.Length;
var targetIdOfConcern = template.Substring(openingLen, tagClosing - openingLen);
template =
template.Substring(0, tagLocation)
+ (recordData[targetIdOfConcern] ?? "").Trim()
+ template.Substring(tagClosing + 1);
// Check for next "targetId" tag.
tagLocation = template.IndexOf(targetTagSubstr);
}
recordTemplates.Add(template);
}
// Get the head template that will be the body of the email.
var masterTemplate = ZillowEmailTemplate;
masterTemplate = ReplaceSingleTagWithString(masterTemplate, "<timeOfDayGreeting>", ProduceTimeOfDayGreeting());
masterTemplate = ReplaceSingleTagWithString(masterTemplate, "<scapedArea>", recentZillowScrapeEvent.AreaName);
masterTemplate = ReplaceSingleTagWithString(masterTemplate, "<eventDateTime>", recentZillowScrapeEvent.TimeStamp.ToLongDateString() + " at " + recentZillowScrapeEvent.TimeStamp.ToLongTimeString());
var allRecords = recordTemplates.Aggregate((x, y) => x + "<br />" + y);
masterTemplate = ReplaceSingleTagWithString(masterTemplate, "<eventRecords>", allRecords);
// Always do last (prevent conflicts with string.Format(...)).
//masterTemplate = ReplaceSingleTagWithString(masterTemplate, "<templateStyles>", ZillowEmailStyles);
// Retrieve all active subscribers. Loop through all of them and send each subscriber an email.
var activeSubscribers = await UserClient.GetSubscribersAsync();
foreach (var subscriber in activeSubscribers)
{
// Now send out emails. Replace "<subscriberName>" tags with subscriber's names.
ReplaceSubscriberTagAndSendEmail(masterTemplate, subscriber);
}
}
