public void DeserializeStatsYieldsExpectedValue()
{
SerializeStatsYieldsHttpCreated();
var dict = FeedRegistry.DeserializeIcalStatsFromJson(blobhost, containername, statsfile);
Assert.AreEqual(testscore, dict[testurl].score);
}
public void AfternoonTeaAt3PMLocal()
{
var example = "hillside";
var source = "hillside";
var fr = new FeedRegistry(berkeley_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_berkeley, fr, collector_berkeley, true);
HillsideExampleIsCorrectObj(zes, null, null);
}
public void InvestmentAndFinancialPlanningDoesNotRecurToPresent()
{
var example = "investment";
var source = "Ann Arbor City";
var fr = new FeedRegistry(keene_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_keene, fr, collector_keene, true);
Assert.That(zes.events.Count == 0);
}
public void LissaSchneckenburgerAt8PMLocal()
{
var example = "folklore";
var source = "Monadnock Folklore Society";
var fr = new FeedRegistry(keene_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_keene, fr, collector_keene, true);
Assert.That(zes.events.Count == 1);
var evt = zes.events[0];
Assert.That(evt.title.EndsWith("Lissa Schneckenburger | Nelson Town Hall"));
Assert.That(evt.dtstart.Hour == 20);
Assert.That(evt.dtstart.Minute == 0);
Assert.That(evt.dtstart.Second == 0);
}
public void HealthProfessionsProgramAt10AMLocal()
{
var example = "ucb";
var source = "UC Berkeley";
var fr = new FeedRegistry(berkeley_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_berkeley, fr, collector_berkeley, true);
Assert.That(zes.events.Count == 1);
var evt = zes.events[0];
Assert.That(evt.title.EndsWith("Post-Baccalaureate Health Professions Program"));
Assert.That(evt.dtstart.Hour == 10);
Assert.That(evt.dtstart.Minute == 0);
Assert.That(evt.dtstart.Second == 0);
}
public void KeeneChoraleAt4PMLocal()
{
var example = "chorale";
var source = "Keene Chorale";
var fr = new FeedRegistry(keene_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_keene, fr, collector_keene, true);
Assert.That(zes.events.Count == 1);
var evt = zes.events[0];
Assert.That(evt.title == "Keene Chorale Winter concert");
Assert.That(evt.dtstart.Hour == 16);
Assert.That(evt.dtstart.Minute == 0);
Assert.That(evt.dtstart.Second == 0);
}
public void BrowerYouthAwardsAt530PMLocal()
{
var example = "berkeleyside";
var source = "berkeleyside";
var fr = new FeedRegistry(berkeley_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_berkeley, fr, collector_berkeley, true);
Assert.That(zes.events.Count == 1);
var evt = zes.events[0];
Assert.That(evt.title.EndsWith("Brower Youth Awards 2011"));
Assert.That(evt.dtstart.Hour == 17);
Assert.That(evt.dtstart.Minute == 30);
Assert.That(evt.dtstart.Second == 0);
}
public void LifeRecoveryMinistryAt6PMLocal()
{
var example = "graceworship";
var source = "Grace Worship";
var fr = new FeedRegistry(keene_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_keene, fr, collector_keene, true);
Assert.That(zes.events.Count == 1);
var evt = zes.events[0];
Assert.That(evt.title.StartsWith("LRM"));
Assert.That(evt.dtstart.Hour == 18);
Assert.That(evt.dtstart.Minute == 0);
Assert.That(evt.dtstart.Second == 0);
}
public void CedarHillCommemoratesAt10AMLocal()
{
var example = "cedarhill";
var source = "Cedar Hill Commemorates";
var fr = new FeedRegistry(keene_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_keene, fr, collector_keene, true);
Assert.That(zes.events.Count == 1);
var evt = zes.events[0];
Assert.That(evt.title.StartsWith("Cedar Hill Commemorates"));
Assert.That(evt.dtstart.Hour == 10);
Assert.That(evt.dtstart.Minute == 0);
Assert.That(evt.dtstart.Second == 0);
}
private static bool _AfternoonTeaAt3PMLocal() // this version expects an altered feed
{
var example = "hillside";
var source = "hillside";
var fr = new FeedRegistry(berkeley_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_berkeley, fr, collector_berkeley, false);
try
{
HillsideExampleIsCorrectObj(zes, "Afternoon Tea", "Afternoon Coffee");
return(true);
}
catch
{
return(false);
}
}
public static bool __AfternoonTeaAt3PMLocal() // this version leaves cached ics and obj
{
var example = "hillside";
var source = "hillside";
var fr = new FeedRegistry(berkeley_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_berkeley, fr, collector_berkeley, false);
try
{
HillsideExampleIsCorrectObj(zes, null, null);
return(true);
}
catch
{
return(false);
}
}
public void SilkCityIsAllDay()
{
var example = "silkcity";
var source = "Silk City Flick Fest";
var fr = new FeedRegistry(keene_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_keene, fr, collector_keene, true);
Assert.That(zes.events.Count == 1);
var evt = zes.events[0];
Assert.That(evt.title.StartsWith("The Silk City Flick Fest"));
Assert.That(evt.allday == true);
Assert.That(evt.dtstart.Hour == 0);
Assert.That(evt.dtstart.Minute == 0);
Assert.That(evt.dtstart.Second == 0);
}
private void SetShowBadgesForHub()
{
var fr = new FeedRegistry(this.id);
fr.LoadFeedsFromAzure(FeedLoadOption.all);
try
{
this.show_eventbrite_badge = Utils.ShowEventBriteBadge(this, fr);
this.show_eventful_badge = Utils.ShowEventfulBadge(this);
this.show_meetup_badge = Utils.ShowMeetupBadge(this, fr);
this.show_facebook_badge = Utils.ShowFacebookBadge(fr);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "SetShowBadgesForHub: " + this.id, e.Message + e.StackTrace);
}
}
public void CollectIcal(FeedRegistry fr, ZonedEventStore es)
{
CollectIcal(fr, es, false);
}
public void CollectIcal(FeedRegistry fr, ZonedEventStore es, bool test)
{
this.LoadTags();
using (ical_ical)
{
Dictionary<string, string> feeds = fr.feeds;
DateTime utc_midnight_in_tz = Utils.MidnightInTz(this.calinfo.tzinfo).UniversalTime;
// enforce the limit. necessary because processing of icalendar sources can involve
// the unrolling of recurrence, and that can't go on forever
DateTime then = utc_midnight_in_tz.AddDays(calinfo.icalendar_horizon_days);
List<string> feedurls = test ? feeds.Keys.Take(test_feeds).ToList() : feeds.Keys.ToList();
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = Convert.ToInt32(this.settings["max_feed_processing_parallelism"]);
//options.MaxDegreeOfParallelism = 1; // for debugging
List<string> hubs_to_skip_date_only_recurrence = new List<string>();
try
{
hubs_to_skip_date_only_recurrence = settings["hubs_to_skip_date_only_recurrence"].Split(',').ToList();
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "hubs_to_skip_date_only_recurrence", e.Message);
}
GenUtils.LogMsg("info", id + " loading " + feedurls.Count() + " feeds", null);
var results_dict = new System.Collections.Concurrent.ConcurrentDictionary<string, string>();
try
{
//foreach (var feedurl in feedurls)
Parallel.ForEach(source: feedurls, parallelOptions: options, body: (feedurl) =>
{
if (settings["eventful_feeds_enabled"] == "False" && feedurl.StartsWith("http://eventful.com/") )
return;
if ( settings["eventbrite_feeds_enabled"] == "False" && feedurl.Contains("ics_from_eventbrite") )
return;
var tid = System.Threading.Thread.CurrentThread.ManagedThreadId;
results_dict.TryAdd(feedurl, null);
ZonedEventStore es_feed_cache = new ZonedEventStore(calinfo, SourceType.ical);
var eids_and_categories = new Dictionary<string, List<string>>(); // for augmenting eventful ics feeds with categories from corresponding atom feeds
// http://eventful.com/ical/annarbor/venues/michigan-theater-/V0-001-000827373-5
if ( feedurl.StartsWith("http://eventful.com/ical") )
{
var atom_url = feedurl.Replace("ical","atom");
eids_and_categories = Utils.CategoriesFromEventfulAtomFeed(atom_url);
}
iCalendar ical = new DDay.iCal.iCalendar();
string source_name = "source_name";
List<DDay.iCal.Event> events_to_include = new List<DDay.iCal.Event>();
Dictionary<DDay.iCal.Event, RecurrenceType> event_recurrence_types = new Dictionary<DDay.iCal.Event, RecurrenceType>();
List<DDay.iCal.Event> uniques = new List<DDay.iCal.Event>();
string feedtext = "";
try
{
source_name = FeedSetup(fr, feeds, feedurl, tid, source_name);
}
catch (Exception e0)
{
try
{
HandleE0(results_dict, feedurl, tid, e0);
return; // http://stackoverflow.com/questions/3765038/is-there-an-equivalent-to-continue-in-a-parallel-foreach
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE0", e.Message); }
}
bool changed = false;
try
{
feedtext = GetFeedTextFromFeedUrl(fr, this.calinfo, source_name, feedurl, this.wait_secs, this.max_retries, this.timeout_secs, this.settings, ref changed);
}
catch (Exception e1) // exception while loading feed
{
try
{
HandleE1(results_dict, feedurl, tid, source_name, e1);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE1", e.Message); return; }
}
try
{
var setting = "use_ics_cached_objects";
if (changed == false && Utils.CachedFeedObjExists(id, feedurl) && settings[setting].StartsWith("y") )
{
GenUtils.LogMsg("info", "CollectIcal: using cached obj for " + feedurl, null); // the test UnchangedFeedUsesCachedObj checks for this entry
AddEventsFromCachedIcalFeedObj(es, feedurl); // get events for the feedurl from the cached obj
TransferCachedResults(this.id, results_dict, feedurl); // and result for this feedurl from the cached results
TransferCachedStats(this.id, fr, feedurl); // and stats for this feedurl from cached stats
return; // done for this feedurl
}
else
{
ical = ParseTheFeed(feedtext);
}
}
catch (Exception e2)
{
try
{
HandleE2(fr, results_dict, feedurl, tid, source_name, e2);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE2", e.Message); return; }
}
if (ical == null || ical.Events.Count == 0)
{
try
{
HandleNoEvents(results_dict, feedurl, tid, source_name);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleNoEvents", e.Message); return; }
}
try
{
var skip_date_only_recurrence = hubs_to_skip_date_only_recurrence.Exists(x => x == this.id);
FeedGather(utc_midnight_in_tz, then, tid, ical, source_name, events_to_include, event_recurrence_types, skip_date_only_recurrence);
}
catch (Exception e3)
{
try
{
HandleE3(fr, results_dict, feedurl, tid, source_name, e3);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE3", e.Message); return; }
}
try
{
uniques = Utils.UniqueByTitleAndStart(events_to_include);
}
catch (Exception e4)
{
try
{
HandleE4(fr, results_dict, feedurl, tid, source_name, e4);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE4", e.Message); return; }
}
try
{
foreach (var unique in uniques)
{
// MaybeAugmentEventfulCategories(unique, eids_and_categories);
// sadly eventful categories can be haphazard so this augmentation has to be suppressed for now
lock (es)
{
AddIcalEvent(unique, fr, es, feedurl, source_name); // add to the all-feeds store
}
AddIcalEvent(unique, new FeedRegistry(id), es_feed_cache, feedurl, source_name); // and the per-feed cache
}
}
catch (Exception e5)
{
try
{
HandleE5(fr, results_dict, feedurl, tid, source_name, e5);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE5", e.Message); return; }
}
try
{
FeedStats(fr, feedurl, event_recurrence_types, uniques);
}
catch (Exception e6)
{
try
{
HandleE6(results_dict, feedurl, tid, source_name, e6);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE6", e.Message); return; }
}
try
{
Utils.SaveFeedObjToCache(id, feedurl, es_feed_cache); // cache the per-feed zoned eventstore
}
catch (Exception e) { GenUtils.LogMsg("exception", "SaveFeedObjToCache: " + this.id + ", " + feedurl, e.Message); return; }
try { results_dict[feedurl] = "ok: " + source_name; }
catch (Exception e)
{
GenUtils.LogMsg("exception", "update results_dict[feedurl]", e.Message);
return;
}
});
}
catch (AggregateException agg_ex)
{
var inners = agg_ex.InnerExceptions;
foreach (var inner in inners)
GenUtils.LogMsg("exception", "CollectIcal: parallel inner exception: " + inner.Message, inner.StackTrace);
}
var json = JsonConvert.SerializeObject(results_dict);
json = GenUtils.PrettifyJson(json);
bs.PutBlob(id, "feed_processing_results.json", json);
SerializeStatsAndIntermediateOutputs(fr, es, ical_ical, new NonIcalStats(), SourceType.ical);
}
}
public static string GetFeedTextFromFeedUrl(FeedRegistry fr, Calinfo calinfo, string source, string feedurl, int wait_secs, int max_retries, TimeSpan timeout_secs, Dictionary<string,string> settings, ref bool changed)
{
HttpWebRequest request;
try
{
var _feedurl = Utils.MaybeChangeWebcalToHttp(feedurl); // original is for feed registry and cache
request = (HttpWebRequest)WebRequest.Create(new Uri(_feedurl)); // only modify (if necessary) when actually fetching
}
catch (Exception e)
{
GenUtils.LogMsg("warning", "GetFeedTextFromFeedUrl", "cannot use " + feedurl + " : " + e.Message);
return String.Empty;
}
string feedtext = "";
string cached_feed_text = Utils.GetCachedFeedText(feedurl);
var response = HttpUtils.RetryHttpRequestExpectingStatus(request, HttpStatusCode.OK, data: null, wait_secs: wait_secs, max_tries: max_retries, timeout_secs: timeout_secs);
if (response.status != HttpStatusCode.OK)
{
var msg = "could not fetch " + source;
GenUtils.LogMsg("warning", msg, response.status.ToString());
feedtext = cached_feed_text;
}
else
{
feedtext = response.DataAsString();
}
if (cached_feed_text.Contains("X-QUOTA-THROTTLED"))
GenUtils.LogMsg("warning", "GetFeedTextFromFeed", "cached text contains X-QUOTA-THROTTLED");
if (feedtext != cached_feed_text && feedtext.Contains("X-QUOTA-THROTTLED") == false ) // don't cache if throttled
{
changed = true;
Utils.SaveFeedTextToCache(feedurl, feedtext);
}
feedtext = MassageFeedText(calinfo, feedurl, feedtext, settings);
if (fr != null)
EnsureProdId(fr, feedurl, feedtext);
return feedtext;
}
// put the event into a) the eventstore, and b) the per-type intermediate icalendar object
public void AddIcalEvent(DDay.iCal.Event evt, FeedRegistry fr, ZonedEventStore es, string feedurl, string source)
{
try
{
evt = NormalizeIcalEvt(evt, feedurl, source);
DateTimeWithZone dtstart;
DateTimeWithZone dtend;
var tzinfo = this.calinfo.tzinfo;
//dtstart = Utils.DtWithZoneFromICalDateTime(evt.Start.Value, tzinfo);
//dtend = (evt.DTEnd == null) ? new Utils.DateTimeWithZone(DateTime.MinValue, tzinfo) : Utils.DtWithZoneFromICalDateTime(evt.End.Value, tzinfo);
//dtstart = new Utils.DateTimeWithZone(evt.Start.Value,tzinfo);
//dtend = new Utils.DateTimeWithZone(evt.End.Value,tzinfo);
var localstart = evt.DTStart.IsUniversalTime ? TimeZoneInfo.ConvertTimeFromUtc(evt.Start.UTC, tzinfo) : evt.Start.Local;
dtstart = new DateTimeWithZone(localstart, tzinfo);
var localend = evt.DTEnd.IsUniversalTime ? TimeZoneInfo.ConvertTimeFromUtc(evt.End.UTC, tzinfo) : evt.End.Local;
dtend = new DateTimeWithZone(localend, tzinfo);
MakeGeo(this.calinfo, evt, this.calinfo.lat, this.calinfo.lon);
string categories = null;
if (evt.Categories != null && evt.Categories.Count() > 0)
categories = string.Join(",", evt.Categories.ToList().Select(cat => cat.ToString().ToLower()));
string description = this.calinfo.has_descriptions ? evt.Description : null;
string location = this.calinfo.has_locations ? evt.Location : null;
es.AddEvent(title: evt.Summary, url: evt.Url.ToString(), source: source, dtstart: dtstart, dtend: dtend, lat: this.calinfo.lat, lon: this.calinfo.lon, allday: evt.IsAllDay, categories: categories, description: description, location: location);
var evt_tmp = MakeTmpEvt(this.calinfo, dtstart: dtstart, dtend: dtend, title: evt.Summary, url: evt.Url.ToString(), location: evt.Location, description: source, lat: this.calinfo.lat, lon: this.calinfo.lon, allday: evt.IsAllDay);
AddEventToDDayIcal(ical_ical, evt_tmp);
if ( fr.stats.ContainsKey(feedurl) ) // won't be true when adding to the per-feed obj cache
fr.stats[feedurl].loaded++; // and this will have already been counted by the all-feeds AddIcalEvent
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "AddIcalEvent", source + ": " + e.Message + ": " + evt.Summary);
}
}
private void HandleE5(FeedRegistry fr, ConcurrentDictionary<string, string> results_dict, string feedurl, int tid, string source_name, Exception e5)
{
var msg = string.Format("exception adding events for {0} {1}, {2}, {3}", id, tid, source_name, e5.Message);
GenUtils.PriorityLogMsg("exception", msg, e5.StackTrace + "," + e5.Data.ToString() + "," + e5.InnerException.Message);
results_dict[feedurl] = msg;
fr.stats[feedurl].dday_error += " | " + msg;
}
private void SerializeStatsAndIntermediateOutputs(FeedRegistry fr, EventStore es, iCalendar ical, NonIcalStats stats, SourceType type)
{
BlobStorageResponse bsr;
HttpResponse tsr;
if (BlobStorage.ExistsContainer(this.id) == false)
bs.CreateContainer(this.id, is_public: true, headers: new Hashtable());
if (type == SourceType.ical) // NonIcalStats is null in this case, and not used
{
bsr = fr.SerializeIcalStatsToJson();
GenUtils.LogMsg("info", this.id + ": SerializeIcalStatsToJson", bsr.HttpResponse.status.ToString());
tsr = fr.SaveStatsToAzure();
GenUtils.LogMsg("info", this.id + ": FeedRegistry.SaveStatsToAzure", tsr.status.ToString());
}
else
{
bsr = Utils.SerializeObjectToJson(stats, this.id, stats.blobname + ".json");
GenUtils.LogMsg("info", this.id + ": Collector: SerializeObjectToJson: " + stats.blobname + ".json", bsr.HttpResponse.status.ToString());
tsr = this.SaveStatsToAzure(type);
GenUtils.LogMsg("info", this.id + ": Collector: SaveStatsToAzure", tsr.status.ToString());
}
bsr = this.SerializeIcalEventsToIcs(ical, type);
GenUtils.LogMsg("info", this.id + ": SerializeIcalStatsToIcs: " + id + "_" + type.ToString() + ".ics", bsr.HttpResponse.status.ToString());
bsr = es.Serialize();
GenUtils.LogMsg("info", this.id + ": EventStore.Serialize: " + es.objfile, bsr.HttpResponse.status.ToString());
}
private static ZonelessEventStore ProcessIcalExample(string example, string source, Calinfo calinfo, FeedRegistry fr, Collector collector)
{
DeleteZonedObjects(calinfo.id);
var feedurl = BlobStorage.MakeAzureBlobUri("admin", example + ".ics", false).ToString();
fr.AddFeed(feedurl, source);
var es = new ZonedEventStore(calinfo, SourceType.ical);
collector.CollectIcal(fr, es, false);
EventStore.CombineZonedEventStoresToZonelessEventStore(calinfo.id, settings);
var zes = new ZonelessEventStore(calinfo).Deserialize();
return zes;
}
// a feed without a PRODID property is actually invalid, but some homegrown feeds
// don't include it, try giving benefit of doubt
private static void EnsureProdId(FeedRegistry fr, string feedurl, string feedtext)
{
try
{
fr.stats[feedurl].prodid = GenUtils.RegexFindGroups(feedtext, "PRODID:(.+)")[1];
}
catch
{
fr.stats[feedurl].prodid = "unknown";
}
}
public void KeeneChoraleAt4PMLocal()
{
var example = "chorale";
var source = "Keene Chorale";
var fr = new FeedRegistry(keene_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_keene, fr, collector_keene);
Assert.That(zes.events.Count == 1);
var evt = zes.events[0];
Assert.That(evt.title == "Keene Chorale Winter concert");
Assert.That(evt.dtstart.Hour == 16);
Assert.That(evt.dtstart.Minute == 0);
Assert.That(evt.dtstart.Second == 0);
}
public void LissaSchneckenburgerAt8PMLocal()
{
var example = "folklore";
var source = "Monadnock Folklore Society";
var fr = new FeedRegistry(keene_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_keene, fr, collector_keene);
Assert.That(zes.events.Count == 1);
var evt = zes.events[0];
Assert.That(evt.title.EndsWith("Lissa Schneckenburger | Nelson Town Hall"));
Assert.That(evt.dtstart.Hour == 20);
Assert.That(evt.dtstart.Minute == 0);
Assert.That(evt.dtstart.Second == 0);
}
public void HealthProfessionsProgramAt10AMLocal()
{
var example = "ucb";
var source = "UC Berkeley";
var fr = new FeedRegistry(berkeley_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_berkeley, fr, collector_berkeley);
Assert.That(zes.events.Count == 1);
var evt = zes.events[0];
Assert.That(evt.title.EndsWith("Post-Baccalaureate Health Professions Program"));
Assert.That(evt.dtstart.Hour == 10);
Assert.That(evt.dtstart.Minute == 0);
Assert.That(evt.dtstart.Second == 0);
}
public void BrowerYouthAwardsAt530PMLocal()
{
var example = "berkeleyside";
var source = "berkeleyside";
var fr = new FeedRegistry(berkeley_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_berkeley, fr, collector_berkeley);
Assert.That(zes.events.Count == 1);
var evt = zes.events[0];
Assert.That(evt.title.EndsWith("Brower Youth Awards 2011"));
Assert.That(evt.dtstart.Hour == 17);
Assert.That(evt.dtstart.Minute == 30);
Assert.That(evt.dtstart.Second == 0);
}
public ActionResult view_calendar(string feedurl, string id)
{
var r = new ContentResult();
r.ContentType = "text/html";
if (id != String.Empty && Utils.CachedFeedObjExists(id, feedurl))
return view_cached_calendar(feedurl, id);
DDay.iCal.iCalendar ical;
var source = ""; // unneeded for this single-cal purpose
try
{
ical = Utils.iCalFromFeedUrl(feedurl, settings);
}
catch (Exception e)
{
r.Content = string.Format(@"Sorry, unable to parse {0} as an iCalendar feed. The error was: {1}", feedurl, e.Message + e.StackTrace);
return r;
}
var events_to_include = new List<DDay.iCal.Event>();
var event_recurrence_types = new Dictionary<DDay.iCal.Event, Collector.RecurrenceType>();
var calinfo = new Calinfo(id);
var collector = new Collector(calinfo, settings);
try
{
DateTime utc_midnight_in_tz = Utils.MidnightInTz(calinfo.tzinfo).UniversalTime;
DateTime then = utc_midnight_in_tz.AddDays(calinfo.icalendar_horizon_days);
collector.FeedGather(utc_midnight_in_tz, then, 0, ical, source, events_to_include, event_recurrence_types, false);
var es_zoned = new ZonedEventStore(calinfo, SourceType.ical);
var fr = new FeedRegistry(id);
foreach (var evt in events_to_include)
collector.AddIcalEvent(evt, fr, es_zoned, feedurl, source);
var es_zoneless = new ZonelessEventStore(calinfo);
r = view_calendar_helper(id, r, es_zoned, es_zoneless);
}
catch (Exception e)
{
r.Content = "Unable to view calendar. " + e.Message + e.StackTrace;
return r;
}
return r;
}
public FeedRegistryTest()
{
containername = this.id;
fr = new FeedRegistry(this.id);
fr.AddFeed(testurl, testsource);
}
private static ZonelessEventStore ProcessIcalExample(string example, string source, Calinfo calinfo, FeedRegistry fr, Collector collector, bool purge)
{
DeleteZonedObjects(calinfo.id);
if (purge)
{
Utils.PurgeFeedCacheForHub(calinfo.id);
}
var feedurl = BlobStorage.MakeAzureBlobUri("admin", example + ".ics", false).ToString();
fr.AddFeed(feedurl, source);
var es = new ZonedEventStore(calinfo, SourceType.ical);
collector.CollectIcal(fr, es, false);
EventStore.CombineZonedEventStoresToZonelessEventStore(calinfo.id, settings);
var zes = new ZonelessEventStore(calinfo).Deserialize();
return(zes);
}
private static void FeedStats(FeedRegistry fr, string feedurl, Dictionary<DDay.iCal.Event, RecurrenceType> event_recurrence_types, List<DDay.iCal.Event> uniques)
{
HashSet<string> recurring_uids = new HashSet<string>();
UpdateIcalStats(fr, feedurl, event_recurrence_types, uniques, recurring_uids);
fr.stats[feedurl].recurringcount = recurring_uids.Count; // count recurring events
}
private void SetShowBadgesForHub()
{
var fr = new FeedRegistry(this.id);
fr.LoadFeedsFromAzure(FeedLoadOption.all);
try
{
this.show_eventbrite_badge = Utils.ShowEventBriteBadge(this, fr);
this.show_eventful_badge = Utils.ShowEventfulBadge(this);
this.show_meetup_badge = Utils.ShowMeetupBadge(this, fr);
this.show_facebook_badge = Utils.ShowFacebookBadge(fr);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "SetShowBadgesForHub: " + this.id, e.Message + e.StackTrace);
}
}
public void AfternoonTeaAt3PMLocal()
{
var example = "hillside";
var source = "hillside";
var fr = new FeedRegistry(berkeley_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_berkeley, fr, collector_berkeley);
HillsideExampleIsCorrectObj(zes, null, null);
}
// when using cached obj, transfer (relevant parts of) the entry from the saved ical_stats.json into the live one
private static void TransferCachedStats(string id, FeedRegistry fr, string feedurl)
{
try
{
var uri = BlobStorage.MakeAzureBlobUri(id, "ical_stats.json");
var json = HttpUtils.FetchUrl(uri).DataAsString();
var dict = JsonConvert.DeserializeObject<Dictionary<string, Dictionary<string, string>>>(json);
var entry = dict[feedurl];
/* "prodid": "unknown",
"source": "Monadnock Regional High",
"valid": false,
"score": "0",
"loaded": 0,
"dday_error": "Object reference not set to an instance of an object.",
"contenttype": null,
"singlecount": 0,
"recurringcount": 0,
"recurringinstancecount": 0,
"futurecount": 0,
"whenchecked": "2013-02-26T12:42:59.1193517Z"
*/
lock (fr)
{
fr.stats[feedurl].futurecount = Convert.ToInt32(entry["futurecount"]);
fr.stats[feedurl].loaded = Convert.ToInt32(entry["loaded"]);
fr.stats[feedurl].singlecount = Convert.ToInt32(entry["singlecount"]);
fr.stats[feedurl].singlecount = Convert.ToInt32(entry["singlecount"]);
fr.stats[feedurl].recurringcount = Convert.ToInt32(entry["recurringcount"]);
fr.stats[feedurl].recurringinstancecount = Convert.ToInt32(entry["recurringinstancecount"]);
fr.stats[feedurl].futurecount = Convert.ToInt32(entry["futurecount"]);
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "TransferCachedStats", e.Message + e.StackTrace);
}
}
public void CedarHillCommemoratesAt10AMLocal()
{
var example = "cedarhill";
var source = "Cedar Hill Commemorates";
var fr = new FeedRegistry(keene_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_keene, fr, collector_keene);
Assert.That(zes.events.Count == 1);
var evt = zes.events[0];
Assert.That(evt.title.StartsWith("Cedar Hill Commemorates"));
Assert.That(evt.dtstart.Hour == 10);
Assert.That(evt.dtstart.Minute == 0);
Assert.That(evt.dtstart.Second == 0);
}
private static void UpdateIcalStats(FeedRegistry fr, string feedurl, Dictionary<DDay.iCal.Event, RecurrenceType> event_recurrence_types, List<DDay.iCal.Event> uniques, HashSet<string> recurring_uids)
{
foreach (var unique in uniques) // count as single event or instance of recurring
{
fr.stats[feedurl].futurecount++;
var recurrence_type = event_recurrence_types[unique];
if (recurrence_type == RecurrenceType.Recurring)
{
fr.stats[feedurl].recurringinstancecount++;
lock (recurring_uids)
{
recurring_uids.Add(unique.UID);
}
}
else
fr.stats[feedurl].singlecount++;
}
}
public void InvestmentAndFinancialPlanningDoesNotRecurToPresent()
{
var example = "investment";
var source = "Ann Arbor City";
var fr = new FeedRegistry(keene_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_keene, fr, collector_keene);
Assert.That(zes.events.Count == 0);
}
private string FeedSetup(FeedRegistry fr, Dictionary<string, string> feeds, string feedurl, int tid, string source_name)
{
source_name = feeds[feedurl];
fr.stats[feedurl].whenchecked = DateTime.Now.ToUniversalTime();
per_feed_metadata_cache[feedurl] = Metadata.LoadFeedMetadataFromAzureTableForFeedurlAndId(feedurl, id);
Metadata.TryLoadCatmapFromMetadict(per_feed_catmaps, per_feed_metadata_cache[feedurl]);
if ( per_feed_catmaps.ContainsKey(feedurl) )
foreach (var cat in per_feed_catmaps[feedurl].Values) // it was a curatorial decision
foreach ( var c in cat.Split(',') ) // (could be multiple)
{
var _c = c.ToLower().Trim();
if ( ! this.tags.HasItem(_c) )
this.tags.Add(_c); // so add to active taxonomy
}
return source_name;
}
public void LifeRecoveryMinistryAt6PMLocal()
{
var example = "graceworship";
var source = "Grace Worship";
var fr = new FeedRegistry(keene_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_keene, fr, collector_keene);
Assert.That(zes.events.Count == 1);
var evt = zes.events[0];
Assert.That(evt.title.StartsWith("LRM"));
Assert.That(evt.dtstart.Hour == 18);
Assert.That(evt.dtstart.Minute == 0);
Assert.That(evt.dtstart.Second == 0);
}
private void HandleE2(FeedRegistry fr, ConcurrentDictionary<string, string> results_dict, string feedurl, int tid, string source_name, Exception e2)
{
var msg = String.Format("exception loading calendar {0}, {1}, {2}, {3}", id, tid, source_name, e2.Message);
GenUtils.LogMsg("exception", msg, e2.StackTrace);
fr.stats[feedurl].dday_error = e2.Message;
results_dict[feedurl] = msg;
}
public void SilkCityIsAllDay()
{
var example = "silkcity";
var source = "Silk City Flick Fest";
var fr = new FeedRegistry(keene_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_keene, fr, collector_keene);
Assert.That(zes.events.Count == 1);
var evt = zes.events[0];
Assert.That(evt.title.StartsWith("The Silk City Flick Fest"));
Assert.That(evt.allday == true);
Assert.That(evt.dtstart.Hour == 0);
Assert.That(evt.dtstart.Minute == 0);
Assert.That(evt.dtstart.Second == 0);
}
private void HandleE3(FeedRegistry fr, ConcurrentDictionary<string, string> results_dict, string feedurl, int tid, string source_name, Exception e3)
{
var msg = String.Format("exception gathering future events for {0}, {1}, {2}, {3}, {4}", id, tid, source_name, e3.Message, e3.InnerException.Message);
GenUtils.PriorityLogMsg("exception", msg, e3.StackTrace);
results_dict[feedurl] = msg;
fr.stats[feedurl].dday_error += " | " + msg;
}
// this version expects an altered feed
private static bool _AfternoonTeaAt3PMLocal()
{
var example = "hillside";
var source = "hillside";
var fr = new FeedRegistry(berkeley_test_hub);
var zes = ProcessIcalExample(example, source, calinfo_berkeley, fr, collector_berkeley);
try
{
HillsideExampleIsCorrectObj(zes, "Afternoon Tea", "Afternoon Coffee");
return true;
}
catch
{
return false;
}
}
private void HandleE4(FeedRegistry fr, ConcurrentDictionary<string, string> results_dict, string feedurl, int tid, string source_name, Exception e4)
{
var msg = String.Format("exception deduplicating events for {0}, {1}, {2}, {3}", id, tid, source_name, e4.Message);
GenUtils.PriorityLogMsg("exception", msg, e4.StackTrace);
results_dict[feedurl] = msg;
fr.stats[feedurl].dday_error += " | " + msg;
}