Пример #1
0
        public static async Task ScrapeAllSources()
        {
            //var dataSources = await SourceClient.GetSourcesAsync();
            var areasOfConcern = await AreaClient.GetAreasAsync();

            var allTemplates = await TemplateClient.GetTemplatesAsync();

            var topLevelBrowser = new ScrapingBrowser();

            // Dunno what these do but the tutorial had them.
            //topLevelBrowser.AllowAutoRedirect = true;
            //topLevelBrowser.AllowMetaRedirect = true;

            foreach (var area in areasOfConcern)
            {
                foreach (var entryPoint in area.EntryPoints)
                {
                    // Get a scraping Template for the Source associated with the current Entry Point.
                    var topLevelTemplate = allTemplates
                                           .First(t => t.SourceId == entryPoint.SourceId && t.IsTopLevel);

                    // Create new scraping event and assign the area ID.
                    var scrapingEvent = new ScrapeEvent();
                    scrapingEvent.AreaId = area.AreaId;

                    // Create the record collection.
                    scrapingEvent.Records = new System.Collections.ObjectModel.ObservableCollection <ScrapeRecord>();

                    // Begin scraping from the top-level Template.
                    ScrapeFromTemplate(topLevelBrowser, scrapingEvent, entryPoint.EntryPointUri, topLevelTemplate, allTemplates);

                    // Send scraping event to the WebServices to be added to the database.
                    await TemplateClient.AddScrapeEventAsync(topLevelTemplate.TemplateId, scrapingEvent);
                }
            }
        }
Пример #2
0
        private static async Task EmailSubscribers()
        {
            var availableTemplates = await TemplateClient.GetTemplatesAsync();

            var zillowTemplate = availableTemplates.First(t => t.IsTopLevel);

            var recentZillowScrapeEvent = await TemplateClient.GetMostRecentScrapeEventAsync(zillowTemplate.TemplateId);

            var recordTemplates = new List <string>();

            foreach (var record in recentZillowScrapeEvent.Records)
            {
                var template   = ZillowEmailRecordTemplate;
                var recordData = record.TargetFieldIdToValueDictionary;

                // Satisfy all "if" blocks first.
                var ifSubstr   = "<if present=\"";
                var ifLocation = template.IndexOf(ifSubstr);

                while (ifLocation != -1)
                {
                    var ifClosing  = template.IndexOf("</if>", ifLocation);
                    var openingLen = ifLocation + ifSubstr.Length;

                    var idQuoteClosing    = template.IndexOf("\"", openingLen);
                    var targetIdOfConcern = template.Substring(openingLen, idQuoteClosing - openingLen);

                    var continueFrom = ifClosing;

                    // If the record does not contain the ID found within the "if" block, remove the block and its' contents.
                    if (!recordData.ContainsKey(targetIdOfConcern))
                    {
                        template     = template.Substring(0, ifLocation) + template.Substring(ifClosing + 5);
                        continueFrom = ifLocation;
                    }

                    // Check for next "if" block.
                    ifLocation = template.IndexOf(ifSubstr, continueFrom);
                }

                // Replace all remaining "<targetId:x>" tags with the corresponding data value.
                var targetTagSubstr = "<targetId:";
                var tagLocation     = template.IndexOf(targetTagSubstr);

                while (tagLocation != -1)
                {
                    var tagClosing = template.IndexOf(">", tagLocation);
                    var openingLen = tagLocation + targetTagSubstr.Length;

                    var targetIdOfConcern = template.Substring(openingLen, tagClosing - openingLen);

                    template =
                        template.Substring(0, tagLocation)
                        + (recordData[targetIdOfConcern] ?? "").Trim()
                        + template.Substring(tagClosing + 1);

                    // Check for next "targetId" tag.
                    tagLocation = template.IndexOf(targetTagSubstr);
                }

                recordTemplates.Add(template);
            }

            // Get the head template that will be the body of the email.
            var masterTemplate = ZillowEmailTemplate;

            masterTemplate = ReplaceSingleTagWithString(masterTemplate, "<timeOfDayGreeting>", ProduceTimeOfDayGreeting());

            masterTemplate = ReplaceSingleTagWithString(masterTemplate, "<scapedArea>", recentZillowScrapeEvent.AreaName);

            masterTemplate = ReplaceSingleTagWithString(masterTemplate, "<eventDateTime>", recentZillowScrapeEvent.TimeStamp.ToLongDateString() + " at " + recentZillowScrapeEvent.TimeStamp.ToLongTimeString());

            var allRecords = recordTemplates.Aggregate((x, y) => x + "<br />" + y);

            masterTemplate = ReplaceSingleTagWithString(masterTemplate, "<eventRecords>", allRecords);

            // Always do last (prevent conflicts with string.Format(...)).
            //masterTemplate = ReplaceSingleTagWithString(masterTemplate, "<templateStyles>", ZillowEmailStyles);

            // Retrieve all active subscribers.  Loop through all of them and send each subscriber an email.
            var activeSubscribers = await UserClient.GetSubscribersAsync();

            foreach (var subscriber in activeSubscribers)
            {
                // Now send out emails. Replace "<subscriberName>" tags with subscriber's names.
                ReplaceSubscriberTagAndSendEmail(masterTemplate, subscriber);
            }
        }