public void AddZonedEventIncrementsCount()
{
var zoned = new ZonedEventStore(calinfo, SourceType.ical);
Assert.AreEqual(0, zoned.events.Count);
zoned.AddEvent(in_evt0_zoned.title, in_evt0_zoned.url, in_evt0_zoned.source, in_evt0_zoned.dtstart, in_evt0_zoned.dtend, test_lat, test_lon, false, in_evt0_zoned.categories, test_description, test_location);
Assert.AreEqual(1, zoned.events.Count);
}
public void TagsAndUrlsAreCoalesced() // 3 events with same title + start should coalesce tags and urls
{
DeleteZonedObjects(keene_test_hub);
var dtstart = new DateTimeWithZone(DateTime.Now, calinfo_keene.tzinfo);
var dtend = new DateTimeWithZone(dtstart.LocalTime + TimeSpan.FromHours(1), calinfo_keene.tzinfo);
var es1 = new ZonedEventStore(calinfo_keene, SourceType.ical);
es1.AddEvent(
"event",
"http://1",
"source1",
dtstart,
dtend,
"1",
"1",
false,
"cat1",
"first event",
"first location"
);
es1.Serialize();
Assert.IsTrue(calinfo_keene.eventful);
var es2 = new ZonedEventStore(calinfo_keene, SourceType.eventful);
es2.AddEvent(
"event",
"http://2",
"source2",
dtstart,
dtend,
"2",
"2",
false,
"cat2,cat2a",
"second event",
"second location"
);
es2.Serialize();
EventStore.CombineZonedEventStoresToZonelessEventStore(keene_test_hub, settings);
var es = new ZonelessEventStore(calinfo_keene).Deserialize();
Assert.That(es.events.Count == 1);
var evt = es.events.Find(e => e.title == "event");
Assert.That(evt.categories == "cat1,cat2,cat2a");
Assert.That(evt.urls_and_sources.Keys.Count == 2);
Assert.That(evt.urls_and_sources["http://1"] == "source1");
Assert.That(evt.urls_and_sources["http://2"] == "source2");
}
public void AddZonedEventIncrementsCount()
{
var zoned = new ZonedEventStore(calinfo, SourceType.ical);
Assert.AreEqual(0, zoned.events.Count);
zoned.AddEvent(in_evt0_zoned.title, in_evt0_zoned.url, in_evt0_zoned.source, in_evt0_zoned.dtstart, in_evt0_zoned.dtend, test_lat, test_lon, false, in_evt0_zoned.categories, test_description, test_location);
Assert.AreEqual(1, zoned.events.Count);
}
public void SerializeTwoZonedEvents()
{
var zoned = new ZonedEventStore(calinfo, SourceType.ical);
zoned.AddEvent(in_evt0_zoned.title, in_evt0_zoned.url, in_evt0_zoned.source, in_evt0_zoned.dtstart, in_evt0_zoned.dtend, test_lat, test_lon, in_evt0_zoned.allday, test_category, test_description, test_location);
zoned.AddEvent(in_evt1_zoned.title, in_evt1_zoned.url, in_evt1_zoned.source, in_evt1_zoned.dtstart, in_evt1_zoned.dtend, test_lat, test_lon, in_evt1_zoned.allday, test_category, test_description, test_location);
Assert.AreEqual(2, zoned.events.Count);
var response = bs.SerializeObjectToAzureBlob(zoned, test_container, zoned.objfile);
Assert.AreEqual(HttpStatusCode.Created, response.HttpResponse.status);
}
private static ZonelessEventStore ProcessEventBriteExample(string example, string source, Calinfo calinfo, Collector collector)
{
DeleteZonedObjects(calinfo.id);
var es = new ZonedEventStore(calinfo, SourceType.eventbrite);
collector.CollectEventBrite(es);
EventStore.CombineZonedEventStoresToZonelessEventStore(calinfo.id, settings);
var zes = new ZonelessEventStore(calinfo).Deserialize();
return(zes);
}
public static ZonelessEventStore ZonedToZoneless(string id, Calinfo calinfo, ZonedEventStore es_zoned)
{
var es_zoneless = new ZonelessEventStore(calinfo);
foreach (var evt in es_zoned.events)
{
es_zoneless.AddEvent(evt.title, evt.url, evt.source, evt.lat, evt.lon, evt.dtstart.LocalTime, evt.dtend.LocalTime, evt.allday, evt.categories, evt.description, evt.location);
}
ZonelessEventStore.Finalize(calinfo, es_zoneless);
return(es_zoneless);
}
public void DeserializeTwoZonedEvents()
{
SerializeTwoZonedEvents();
Utils.Wait(5);
var zoned = new ZonedEventStore(calinfo, SourceType.ical);
var uri = BlobStorage.MakeAzureBlobUri(test_container, zoned.objfile, false);
var obj = (ZonedEventStore)BlobStorage.DeserializeObjectFromUri(uri);
var out_evts = (List <ZonedEvent>)obj.events;
Assert.AreEqual(2, out_evts.Count);
var out_evt = (ZonedEvent)out_evts[0];
Assert.AreEqual(in_evt0_zoned.dtstart.UniversalTime.Ticks, out_evt.dtstart.UniversalTime.Ticks);
Assert.AreEqual(in_evt0_zoned.title, out_evt.title);
}
private static ZonelessEventStore ProcessIcalExample(string example, string source, Calinfo calinfo, FeedRegistry fr, Collector collector, bool purge)
{
DeleteZonedObjects(calinfo.id);
if (purge)
{
Utils.PurgeFeedCacheForHub(calinfo.id);
}
var feedurl = BlobStorage.MakeAzureBlobUri("admin", example + ".ics", false).ToString();
fr.AddFeed(feedurl, source);
var es = new ZonedEventStore(calinfo, SourceType.ical);
collector.CollectIcal(fr, es, false);
EventStore.CombineZonedEventStoresToZonelessEventStore(calinfo.id, settings);
var zes = new ZonelessEventStore(calinfo).Deserialize();
return(zes);
}
public void SerializeAndDeserializeZonedEventStoreYieldsExpectedEvents()
{
var es = new ZonedEventStore(calinfo, SourceType.ical);
es.AddEvent(title: title1, url: "http://foo", source: source1, dtstart: dt1_with_zone, dtend: min_with_zone, allday: false, lat: test_lat, lon: test_lon, categories: test_category, description: test_description, location: test_location);
es.AddEvent(title: title2, url: "http://bar", source: source2, dtstart: dt2_with_zone, dtend: min_with_zone, lat: test_lat, lon: test_lon, allday: false, categories: test_category, description: test_description, location: test_location);
var response = bs.SerializeObjectToAzureBlob(es, test_container, es.objfile);
Assert.AreEqual(HttpStatusCode.Created, response.HttpResponse.status);
var uri = BlobStorage.MakeAzureBlobUri(test_container, es.objfile, false);
var es2 = (ZonedEventStore)BlobStorage.DeserializeObjectFromUri(uri);
var evt1 = es2.events.First();
var evt2 = es2.events.Last();
Assert.AreEqual(evt1.dtstart.UniversalTime.Ticks, dt1_with_zone.UniversalTime.Ticks);
Assert.AreEqual(evt2.dtstart.UniversalTime.Ticks, dt2_with_zone.UniversalTime.Ticks);
Assert.AreEqual(evt1.title, title1);
Assert.AreEqual(evt2.title, title2);
}
public static void ClearObjFeedInCache(Calinfo calinfo, string cache_dir, string feedurl)
{
var blob_name = Utils.MakeCachedFeedObjName(calinfo.id, feedurl);
var es = new ZonedEventStore(calinfo, SourceType.ical);
bs.SerializeObjectToAzureBlob(es, cache_dir, blob_name);
}
public void EventfulQueryYieldsValidFirstEvent()
{
var collector = new Collector(test_calinfo, settings);
var events = (IEnumerable<XElement>)collector.EventfulIterator(1, test_eventful_args, "events/search", "event");
var es = new ZonedEventStore(test_calinfo, SourceType.ical);
collector.AddEventfulEvent(es, test_venue, events.First());
Assert.That(es.events[0].title != "");
}
public ActionResult view_calendar(string feedurl, string id)
{
var r = new ContentResult();
r.ContentType = "text/html";
if (id != String.Empty && Utils.CachedFeedObjExists(id, feedurl))
return view_cached_calendar(feedurl, id);
DDay.iCal.iCalendar ical;
var source = ""; // unneeded for this single-cal purpose
try
{
ical = Utils.iCalFromFeedUrl(feedurl, settings);
}
catch (Exception e)
{
r.Content = string.Format(@"Sorry, unable to parse {0} as an iCalendar feed. The error was: {1}", feedurl, e.Message + e.StackTrace);
return r;
}
var events_to_include = new List<DDay.iCal.Event>();
var event_recurrence_types = new Dictionary<DDay.iCal.Event, Collector.RecurrenceType>();
var calinfo = new Calinfo(id);
var collector = new Collector(calinfo, settings);
try
{
DateTime utc_midnight_in_tz = Utils.MidnightInTz(calinfo.tzinfo).UniversalTime;
DateTime then = utc_midnight_in_tz.AddDays(calinfo.icalendar_horizon_days);
collector.FeedGather(utc_midnight_in_tz, then, 0, ical, source, events_to_include, event_recurrence_types, false);
var es_zoned = new ZonedEventStore(calinfo, SourceType.ical);
var fr = new FeedRegistry(id);
foreach (var evt in events_to_include)
collector.AddIcalEvent(evt, fr, es_zoned, feedurl, source);
var es_zoneless = new ZonelessEventStore(calinfo);
r = view_calendar_helper(id, r, es_zoned, es_zoneless);
}
catch (Exception e)
{
r.Content = "Unable to view calendar. " + e.Message + e.StackTrace;
return r;
}
return r;
}
private static ContentResult view_calendar_helper(string id, ContentResult r, ZonedEventStore es_zoned, ZonelessEventStore es_zoneless)
{
foreach (var evt in es_zoned.events)
es_zoneless.AddEvent(evt.title, evt.url.ToString(), evt.source, "", "", evt.dtstart.LocalTime, evt.dtend.LocalTime, evt.allday, evt.categories, evt.description, evt.location);
es_zoneless.events = EventStore.UniqueByTitleAndStart(id, es_zoneless.events, save_tag_sources: false);
es_zoneless.ExcludePastEvents();
es_zoneless.SortEventList();
var cr = new CalendarRenderer(id);
r.Content = cr.RenderHtmlEventsOnly(es_zoneless, null, 0, DateTime.MinValue, DateTime.MinValue, new Dictionary<string, object>() { { "announce_time_of_day", false }, { "add_to_cal", false }, { "inline_descriptions", true }, {"taglist",false } } );
return r;
}
public void SerializeAndDeserializeZonedEventStoreYieldsExpectedEvents()
{
var es = new ZonedEventStore(calinfo, SourceType.ical);
es.AddEvent(title:title1, url:"http://foo", source:source1, dtstart:dt1_with_zone, dtend:min_with_zone, allday:false, lat:test_lat, lon:test_lon, categories:test_category, description:test_description, location: test_location);
es.AddEvent(title:title2, url:"http://bar", source:source2, dtstart:dt2_with_zone, dtend:min_with_zone, lat:test_lat, lon:test_lon, allday:false, categories:test_category, description:test_description, location: test_location);
var response = bs.SerializeObjectToAzureBlob(es, test_container, es.objfile);
Assert.AreEqual(HttpStatusCode.Created, response.HttpResponse.status);
var uri = BlobStorage.MakeAzureBlobUri(test_container, es.objfile,false);
var es2 = (ZonedEventStore)BlobStorage.DeserializeObjectFromUri(uri);
var evt1 = es2.events.First();
var evt2 = es2.events.Last();
Assert.AreEqual(evt1.dtstart.UniversalTime.Ticks, dt1_with_zone.UniversalTime.Ticks);
Assert.AreEqual(evt2.dtstart.UniversalTime.Ticks, dt2_with_zone.UniversalTime.Ticks);
Assert.AreEqual(evt1.title, title1);
Assert.AreEqual(evt2.title, title2);
}
public void CollectUpcoming(ZonedEventStore es, bool test)
{
using (upcoming_ical)
{
var page_size = test ? test_pagesize : 100;
var args = MakeUpcomingApiArgs(UpcomingSearchStyle.latlon);
var method = "event.search";
var xdoc = CallUpcomingApi(method, args);
int page_count = 1;
try
{
var result_count = GetUpcomingResultCount(xdoc);
if (result_count == 0) // try the other search style (upcoming seems flaky that way)
{
args = MakeUpcomingApiArgs(UpcomingSearchStyle.latlon);
xdoc = CallUpcomingApi(method, args);
GetUpcomingResultCount(xdoc);
}
page_count = result_count / page_size;
}
catch
{
GenUtils.LogMsg("warning", "CollectUpcoming", "resultcount unavailable");
return;
}
if (test == true && page_count > test_pagecount) page_count = test_pagecount;
if (page_count == 0) page_count = 1;
var msg = string.Format("{0}: loading {1} upcoming events", this.id, page_count * page_size);
GenUtils.LogMsg("info", msg, null);
Dictionary<string, int> event_count_by_venue = new Dictionary<string, int>();
int event_num = 0;
var uniques = new Dictionary<string, XElement>(); // dedupe by name + start
foreach (var evt in UpcomingIterator(page_count, method))
uniques.AddOrUpdateDictionary<string, XElement>(evt.Attribute("name").ToString() + evt.Attribute("start_date").ToString(), evt);
foreach (XElement evt in uniques.Values)
{
event_num += 1;
if (event_num > Configurator.upcoming_max_events)
break;
var dtstart = DateTimeWithZoneFromUpcomingXEvent(evt);
if (dtstart.UniversalTime < Utils.MidnightInTz(this.calinfo.tzinfo).UniversalTime)
continue;
var venue_name = evt.Attribute("venue_name").Value;
IncrementEventCountByVenue(event_count_by_venue, venue_name);
AddUpcomingEvent(es, venue_name, evt);
}
ustats.venuecount = event_count_by_venue.Keys.Count;
ustats.whenchecked = DateTime.Now.ToUniversalTime();
SerializeStatsAndIntermediateOutputs(es, upcoming_ical, ustats, SourceType.upcoming);
}
}
public void UpcomingUrlsAreNormalized()
{
DeleteZonedObjects(keene_test_hub);
var dtstart = new DateTimeWithZone(DateTime.Now, calinfo_keene.tzinfo);
var dtend = new DateTimeWithZone(dtstart.LocalTime + TimeSpan.FromHours(1), calinfo_keene.tzinfo);
var es1 = new ZonedEventStore(calinfo_keene, SourceType.ical);
es1.AddEvent(
"event",
"http://upcoming.yahoo.com/event/8504144/",
"Comedy Showcase",
dtstart,
dtend,
"1",
"1",
false,
"comedy",
"first event",
"first location"
);
es1.Serialize();
var es2 = new ZonedEventStore(calinfo_keene, SourceType.upcoming);
es2.AddEvent(
"event",
"http://upcoming.yahoo.com/event/8504144",
"Ann Arbor Comedy Showcase",
dtstart,
dtend,
"1",
"1",
false,
"upcoming",
"first event",
"first location"
);
es2.Serialize();
EventStore.CombineZonedEventStoresToZonelessEventStore(keene_test_hub, settings);
var es = new ZonelessEventStore(calinfo_keene).Deserialize();
Assert.That(es.events.Count == 1);
var evt = es.events.Find(e => e.title == "event");
var categories = evt.categories.Split(',').ToList();
categories.Sort();
Assert.That(categories.SequenceEqual(new List<string>() { "comedy", "upcoming" }));
Assert.That(evt.urls_and_sources.Keys.Count == 1);
}
public void AddEventBriteEvent(ZonedEventStore es, XElement evt, DateTimeWithZone dtstart, DateTimeWithZone dtend)
{
string title;
string event_url;
string source;
bool all_day;
string categories;
var evt_tmp = MakeDDayEventFromEventBriteEvent(evt, out title, out event_url, out source, out all_day, out categories);
AddEventToDDayIcal(eventbrite_ical, evt_tmp);
var min = DateTimeWithZone.MinValue(this.calinfo.tzinfo);
// todo: dig out location from http://developer.eventbrite.com/doc/events/event_search/
es.AddEvent(title: title, url: event_url, source: source, dtstart: dtstart, dtend: dtend, lat: null, lon: null, allday: all_day, categories: categories, description: null, location: null);
}
public void AddUpcomingEvent(ZonedEventStore es, string venue_name, XElement evt)
{
var title = evt.Attribute("name").Value;
var event_url = "http://upcoming.yahoo.com/event/" + evt.Attribute("id").Value;
var source = venue_name;
var venue_id = evt.Attribute("venue_id").Value;
var venue_url = "http://upcoming.yahoo.com/venue/" + venue_id;
string lat = this.calinfo.lat; // default to hub's lat/lon
string lon = this.calinfo.lon;
try
{
lat = evt.Attribute("latitude").Value;
lon = evt.Attribute("longitude").Value;
}
catch
{
GenUtils.LogMsg("warning", "AddUpcomingEvent", "cannot parse lat/lon");
}
string categories = "upcoming";
if (evt.Attribute("tags") != null)
{
var evt_tags = evt.Attribute("tags").Value.Split(',');
if (evt_tags.Length > 0)
{
var intersection = this.tags.Intersect(evt_tags);
categories += "," + String.Join(",", intersection.ToArray());
}
}
ustats.eventcount++;
bool all_day = false;
var dtstart = DateTimeWithZoneFromUpcomingXEvent(evt, ref all_day);
string location = venue_name;
string venue_address = evt.Attribute("venue_address").Value;
if (!String.IsNullOrEmpty(venue_address))
location += ", " + venue_address;
var evt_tmp = MakeTmpEvt(this.calinfo, dtstart, DateTimeWithZone.MinValue(this.calinfo.tzinfo), title, url: event_url, location: location, description: source, lat: lat, lon: lon, allday: all_day);
AddEventToDDayIcal(upcoming_ical, evt_tmp);
var min = DateTimeWithZone.MinValue(this.calinfo.tzinfo);
es.AddEvent(title: title, url: event_url, source: source, dtstart: dtstart, dtend: min, lat: lat, lon: lon, allday: all_day, categories: categories, description: null, location: location);
}
public void CollectEventBrite(ZonedEventStore es)
{
this.LoadTags();
using (eventbrite_ical)
{
string method = "event_search";
string args = MakeEventBriteArgs();
int page_count = GetEventBritePageCount(method, args);
var msg = string.Format("{0}: found about {1} pages of eventbrite events", this.id, page_count * eventbrite_page_size);
GenUtils.LogMsg("info", msg, null);
int event_num = 0;
foreach (XElement evt in EventBriteIterator(page_count, method, args))
{
if (event_num > Configurator.eventbrite_max_events)
break;
var dtstart_with_tz = Utils.ExtractEventBriteDateTime(evt, this.calinfo.tzinfo, "start_date");
var dtend_with_tz = Utils.ExtractEventBriteDateTime(evt, this.calinfo.tzinfo, "end_date");
if (dtstart_with_tz.UniversalTime < Utils.MidnightInTz(this.calinfo.tzinfo).UniversalTime)
continue;
AddEventBriteEvent(es, evt, dtstart_with_tz, dtend_with_tz);
}
ebstats.whenchecked = DateTime.Now.ToUniversalTime();
SerializeStatsAndIntermediateOutputs(es, eventbrite_ical, ebstats, SourceType.eventbrite);
bs.PutBlob(this.id, SourceType.eventbrite.ToString() + "_cats.txt", String.Join(",", this.eventbrite_tags.ToList()));
}
}
// put the event into a) the eventstore, and b) the per-type intermediate icalendar object
public void AddIcalEvent(DDay.iCal.Event evt, FeedRegistry fr, ZonedEventStore es, string feedurl, string source)
{
try
{
evt = NormalizeIcalEvt(evt, feedurl, source);
DateTimeWithZone dtstart;
DateTimeWithZone dtend;
var tzinfo = this.calinfo.tzinfo;
//dtstart = Utils.DtWithZoneFromICalDateTime(evt.Start.Value, tzinfo);
//dtend = (evt.DTEnd == null) ? new Utils.DateTimeWithZone(DateTime.MinValue, tzinfo) : Utils.DtWithZoneFromICalDateTime(evt.End.Value, tzinfo);
//dtstart = new Utils.DateTimeWithZone(evt.Start.Value,tzinfo);
//dtend = new Utils.DateTimeWithZone(evt.End.Value,tzinfo);
var localstart = evt.DTStart.IsUniversalTime ? TimeZoneInfo.ConvertTimeFromUtc(evt.Start.UTC, tzinfo) : evt.Start.Local;
dtstart = new DateTimeWithZone(localstart, tzinfo);
var localend = evt.DTEnd.IsUniversalTime ? TimeZoneInfo.ConvertTimeFromUtc(evt.End.UTC, tzinfo) : evt.End.Local;
dtend = new DateTimeWithZone(localend, tzinfo);
MakeGeo(this.calinfo, evt, this.calinfo.lat, this.calinfo.lon);
string categories = null;
if (evt.Categories != null && evt.Categories.Count() > 0)
categories = string.Join(",", evt.Categories.ToList().Select(cat => cat.ToString().ToLower()));
string description = this.calinfo.has_descriptions ? evt.Description : null;
string location = this.calinfo.has_locations ? evt.Location : null;
es.AddEvent(title: evt.Summary, url: evt.Url.ToString(), source: source, dtstart: dtstart, dtend: dtend, lat: this.calinfo.lat, lon: this.calinfo.lon, allday: evt.IsAllDay, categories: categories, description: description, location: location);
var evt_tmp = MakeTmpEvt(this.calinfo, dtstart: dtstart, dtend: dtend, title: evt.Summary, url: evt.Url.ToString(), location: evt.Location, description: source, lat: this.calinfo.lat, lon: this.calinfo.lon, allday: evt.IsAllDay);
AddEventToDDayIcal(ical_ical, evt_tmp);
if ( fr.stats.ContainsKey(feedurl) ) // won't be true when adding to the per-feed obj cache
fr.stats[feedurl].loaded++; // and this will have already been counted by the all-feeds AddIcalEvent
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "AddIcalEvent", source + ": " + e.Message + ": " + evt.Summary);
}
}
public void AddFacebookEvent(ZonedEventStore es, FacebookEvent fb_event, DateTimeWithZone dtstart)
{
var title = fb_event.name;
var event_url = "http://www.facebook.com/event.php?eid=" + fb_event.id;
var source = "facebook";
var location = fb_event.location;
var all_day = false;
fbstats.eventcount++;
var evt_tmp = MakeTmpEvt(this.calinfo, dtstart, DateTimeWithZone.MinValue(this.calinfo.tzinfo), title, url: event_url, location: location, description: source, lat: this.calinfo.lat, lon: this.calinfo.lon, allday: all_day);
AddEventToDDayIcal(facebook_ical, evt_tmp);
var min = DateTimeWithZone.MinValue(this.calinfo.tzinfo);
es.AddEvent(title: title, url: event_url, source: source, dtstart: dtstart, dtend: min, lat: null, lon: null, allday: all_day, categories: "facebook", description: null, location: location);
}
public void AddEventfulEvent(ZonedEventStore es, string venue_name, XElement evt)
{
var str_dtstart = XmlUtils.GetXeltValue(evt, ElmcityUtils.Configurator.no_ns, "start_time");
DateTime dtstart = Utils.LocalDateTimeFromLocalDateStr(str_dtstart);
var dtstart_with_tz = new DateTimeWithZone(dtstart, this.calinfo.tzinfo);
if (dtstart_with_tz.UniversalTime < Utils.MidnightInTz(this.calinfo.tzinfo).UniversalTime)
return;
var no_ns = ElmcityUtils.Configurator.no_ns;
var event_id = evt.Attribute("id").Value;
var event_owner = XmlUtils.GetXeltValue(evt, no_ns, "owner");
var title = XmlUtils.GetXeltValue(evt, no_ns, "title");
var venue_url = XmlUtils.GetXeltValue(evt, no_ns, "venue_url");
var all_day = XmlUtils.GetXeltValue(evt, no_ns, "all_day") == "1";
var venue_address = XmlUtils.GetXeltValue(evt, no_ns, "venue_address");
string lat = this.calinfo.lat; // default to hub lat/lon
string lon = this.calinfo.lon;
lat = XmlUtils.GetXeltValue(evt, no_ns, "latitude");
lon = XmlUtils.GetXeltValue(evt, no_ns, "longitude");
if (lat == null || lon == null)
{
GenUtils.LogMsg("warning", "AddEventfulEvent", "no lat/lon");
}
var category = XmlUtils.GetXeltValue(evt, ElmcityUtils.Configurator.no_ns, "category");
if (String.IsNullOrWhiteSpace(category))
category = SourceType.eventful.ToString();
else
category = SourceType.eventful.ToString() + "," + category;
string event_url = "http://eventful.com/events/" + event_id;
string source = venue_name;
if (String.IsNullOrEmpty(source))
source = "Unnamed Eventful Venue";
string location;
if (!String.IsNullOrEmpty(source))
{
location = venue_name;
if (!String.IsNullOrEmpty(venue_address))
location += ", " + venue_address;
}
else
{
location = event_url;
}
estats.eventcount++;
var evt_tmp = MakeTmpEvt(this.calinfo, dtstart_with_tz, DateTimeWithZone.MinValue(this.calinfo.tzinfo), title, url: event_url, location: location, description: source, lat: lat, lon: lon, allday: all_day);
AddEventToDDayIcal(eventful_ical, evt_tmp);
var min = DateTimeWithZone.MinValue(this.calinfo.tzinfo);
// suppress description to minimize amount of eventful info carried in the event packet
// allow location but it is subject to the per-hub locations setting (calinfo.has_locations)
es.AddEvent(title: title, url: event_url, source: source, dtstart: dtstart_with_tz, dtend: min, lat: lat, lon: lon, allday: all_day, categories: category, description: null, location: location);
}
public void EventfulUrlsAreNormalized()
{
DeleteZonedObjects(keene_test_hub);
var dtstart = new DateTimeWithZone(DateTime.Now, calinfo_keene.tzinfo);
var dtend = new DateTimeWithZone(dtstart.LocalTime + TimeSpan.FromHours(1), calinfo_keene.tzinfo);
var es1 = new ZonedEventStore(calinfo_keene, SourceType.ical);
es1.AddEvent(
"event",
"http://eventful.com/E0-001-039987477-3",
"The Blind Pig",
dtstart,
dtend,
"1",
"1",
false,
"music",
"first event",
"first location"
);
es1.Serialize();
var es2 = new ZonedEventStore(calinfo_keene, SourceType.eventful);
es2.AddEvent(
"event",
"http://eventful.com/events/E0-001-039987477-3",
"Blind Pig",
dtstart,
dtend,
"1",
"1",
false,
"eventful",
"first event",
"first location"
);
es2.Serialize();
EventStore.CombineZonedEventStoresToZonelessEventStore(keene_test_hub, settings);
var es = new ZonelessEventStore(calinfo_keene).Deserialize();
Assert.That(es.events.Count == 1);
var evt = es.events.Find(e => e.title == "event");
var categories = evt.categories.Split(',').ToList();
categories.Sort();
Assert.That(categories.SequenceEqual(new List<string>() { "eventful", "music" }));
Assert.That(evt.urls_and_sources.Keys.Count == 1);
}
private static ZonelessEventStore ProcessUpcomingExample(string example, string source, Calinfo calinfo, Collector collector)
{
DeleteZonedObjects(calinfo.id);
var es = new ZonedEventStore(calinfo, SourceType.upcoming);
collector.CollectUpcoming(es, test: true);
EventStore.CombineZonedEventStoresToZonelessEventStore(calinfo.id, settings);
var zes = new ZonelessEventStore(calinfo).Deserialize();
return zes;
}
public void TagsAndUrlsAreCoalesced()
{
DeleteZonedObjects(keene_test_hub);
var dtstart = new DateTimeWithZone(DateTime.Now, calinfo_keene.tzinfo);
var dtend = new DateTimeWithZone(dtstart.LocalTime + TimeSpan.FromHours(1), calinfo_keene.tzinfo);
var es1 = new ZonedEventStore(calinfo_keene, SourceType.ical);
es1.AddEvent(
"event",
"http://1",
"source1",
dtstart,
dtend,
"1",
"1",
false,
"cat1",
"first event",
"first location"
);
es1.Serialize();
Assert.IsTrue(calinfo_keene.eventful);
var es2 = new ZonedEventStore(calinfo_keene, SourceType.eventful);
es2.AddEvent(
"event",
"http://2",
"source2",
dtstart,
dtend,
"2",
"2",
false,
"cat2,cat2a",
"second event",
"second location"
);
es2.Serialize();
Assert.IsTrue(calinfo_keene.upcoming);
var es3 = new ZonedEventStore(calinfo_keene, SourceType.upcoming);
es3.AddEvent(
"event",
"http://3",
"source3",
dtstart,
dtend,
"3",
"3",
false,
"cat3,cat3a",
"third event",
"third location"
);
es3.AddEvent(
"another event",
"http://4",
"source4",
dtstart,
dtend,
"4",
"4",
false,
"cat4,cat4a",
"fourth event",
"fourth location"
);
es3.Serialize();
EventStore.CombineZonedEventStoresToZonelessEventStore(keene_test_hub, settings);
var es = new ZonelessEventStore(calinfo_keene).Deserialize();
Assert.That(es.events.Count == 2);
var evt = es.events.Find(e => e.title == "event");
Assert.That(evt.categories == "cat1,cat2,cat2a,cat3,cat3a");
Assert.That(evt.urls_and_sources.Keys.Count == 3);
Assert.That(evt.urls_and_sources["http://1"] == "source1");
Assert.That(evt.urls_and_sources["http://2"] == "source2");
Assert.That(evt.urls_and_sources["http://3"] == "source3");
//Assert.That(evt.list_of_urls_and_sources[0][0] == "http://1" && evt.list_of_urls_and_sources[0][1] == "source1");
//Assert.That(evt.list_of_urls_and_sources[1][0] == "http://2" && evt.list_of_urls_and_sources[1][1] == "source2");
//Assert.That(evt.list_of_urls_and_sources[2][0] == "http://3" && evt.list_of_urls_and_sources[2][1] == "source3");
}
private void AddEventsFromCachedIcalFeedObj(ZonedEventStore es, string feedurl)
{
ZonedEventStore es_saved = Utils.GetFeedObjFromCache(this.calinfo, feedurl);
foreach (ZonedEvent evt in es_saved.events)
{
es.events.Add(evt);
var evt_tmp = MakeTmpEvt(this.calinfo, dtstart: evt.dtstart, dtend: evt.dtend, title: evt.title, url: evt.url, location: evt.location, description: evt.description, lat: evt.lat, lon: evt.lon, allday: evt.allday);
AddEventToDDayIcal(this.ical_ical, evt_tmp);
}
}
private static ZonelessEventStore ProcessIcalExample(string example, string source, Calinfo calinfo, FeedRegistry fr, Collector collector)
{
DeleteZonedObjects(calinfo.id);
var feedurl = BlobStorage.MakeAzureBlobUri("admin", example + ".ics", false).ToString();
fr.AddFeed(feedurl, source);
var es = new ZonedEventStore(calinfo, SourceType.ical);
collector.CollectIcal(fr, es, false);
EventStore.CombineZonedEventStoresToZonelessEventStore(calinfo.id, settings);
var zes = new ZonelessEventStore(calinfo).Deserialize();
return zes;
}
public void CollectIcal(FeedRegistry fr, ZonedEventStore es)
{
CollectIcal(fr, es, false);
}
public void SerializeTwoZonedEvents()
{
var zoned = new ZonedEventStore(calinfo, SourceType.ical);
zoned.AddEvent(in_evt0_zoned.title, in_evt0_zoned.url, in_evt0_zoned.source, in_evt0_zoned.dtstart, in_evt0_zoned.dtend, test_lat, test_lon, in_evt0_zoned.allday, test_category, test_description, test_location);
zoned.AddEvent(in_evt1_zoned.title, in_evt1_zoned.url, in_evt1_zoned.source, in_evt1_zoned.dtstart, in_evt1_zoned.dtend, test_lat, test_lon, in_evt1_zoned.allday, test_category, test_description, test_location);
Assert.AreEqual(2, zoned.events.Count);
var response = bs.SerializeObjectToAzureBlob(zoned, test_container, zoned.objfile);
Assert.AreEqual(HttpStatusCode.Created, response.HttpResponse.status);
}
public void CollectIcal(FeedRegistry fr, ZonedEventStore es, bool test)
{
this.LoadTags();
using (ical_ical)
{
Dictionary<string, string> feeds = fr.feeds;
DateTime utc_midnight_in_tz = Utils.MidnightInTz(this.calinfo.tzinfo).UniversalTime;
// enforce the limit. necessary because processing of icalendar sources can involve
// the unrolling of recurrence, and that can't go on forever
DateTime then = utc_midnight_in_tz.AddDays(calinfo.icalendar_horizon_days);
List<string> feedurls = test ? feeds.Keys.Take(test_feeds).ToList() : feeds.Keys.ToList();
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = Convert.ToInt32(this.settings["max_feed_processing_parallelism"]);
//options.MaxDegreeOfParallelism = 1; // for debugging
List<string> hubs_to_skip_date_only_recurrence = new List<string>();
try
{
hubs_to_skip_date_only_recurrence = settings["hubs_to_skip_date_only_recurrence"].Split(',').ToList();
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "hubs_to_skip_date_only_recurrence", e.Message);
}
GenUtils.LogMsg("info", id + " loading " + feedurls.Count() + " feeds", null);
var results_dict = new System.Collections.Concurrent.ConcurrentDictionary<string, string>();
try
{
//foreach (var feedurl in feedurls)
Parallel.ForEach(source: feedurls, parallelOptions: options, body: (feedurl) =>
{
if (settings["eventful_feeds_enabled"] == "False" && feedurl.StartsWith("http://eventful.com/") )
return;
if ( settings["eventbrite_feeds_enabled"] == "False" && feedurl.Contains("ics_from_eventbrite") )
return;
var tid = System.Threading.Thread.CurrentThread.ManagedThreadId;
results_dict.TryAdd(feedurl, null);
ZonedEventStore es_feed_cache = new ZonedEventStore(calinfo, SourceType.ical);
var eids_and_categories = new Dictionary<string, List<string>>(); // for augmenting eventful ics feeds with categories from corresponding atom feeds
// http://eventful.com/ical/annarbor/venues/michigan-theater-/V0-001-000827373-5
if ( feedurl.StartsWith("http://eventful.com/ical") )
{
var atom_url = feedurl.Replace("ical","atom");
eids_and_categories = Utils.CategoriesFromEventfulAtomFeed(atom_url);
}
iCalendar ical = new DDay.iCal.iCalendar();
string source_name = "source_name";
List<DDay.iCal.Event> events_to_include = new List<DDay.iCal.Event>();
Dictionary<DDay.iCal.Event, RecurrenceType> event_recurrence_types = new Dictionary<DDay.iCal.Event, RecurrenceType>();
List<DDay.iCal.Event> uniques = new List<DDay.iCal.Event>();
string feedtext = "";
try
{
source_name = FeedSetup(fr, feeds, feedurl, tid, source_name);
}
catch (Exception e0)
{
try
{
HandleE0(results_dict, feedurl, tid, e0);
return; // http://stackoverflow.com/questions/3765038/is-there-an-equivalent-to-continue-in-a-parallel-foreach
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE0", e.Message); }
}
bool changed = false;
try
{
feedtext = GetFeedTextFromFeedUrl(fr, this.calinfo, source_name, feedurl, this.wait_secs, this.max_retries, this.timeout_secs, this.settings, ref changed);
}
catch (Exception e1) // exception while loading feed
{
try
{
HandleE1(results_dict, feedurl, tid, source_name, e1);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE1", e.Message); return; }
}
try
{
var setting = "use_ics_cached_objects";
if (changed == false && Utils.CachedFeedObjExists(id, feedurl) && settings[setting].StartsWith("y") )
{
GenUtils.LogMsg("info", "CollectIcal: using cached obj for " + feedurl, null); // the test UnchangedFeedUsesCachedObj checks for this entry
AddEventsFromCachedIcalFeedObj(es, feedurl); // get events for the feedurl from the cached obj
TransferCachedResults(this.id, results_dict, feedurl); // and result for this feedurl from the cached results
TransferCachedStats(this.id, fr, feedurl); // and stats for this feedurl from cached stats
return; // done for this feedurl
}
else
{
ical = ParseTheFeed(feedtext);
}
}
catch (Exception e2)
{
try
{
HandleE2(fr, results_dict, feedurl, tid, source_name, e2);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE2", e.Message); return; }
}
if (ical == null || ical.Events.Count == 0)
{
try
{
HandleNoEvents(results_dict, feedurl, tid, source_name);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleNoEvents", e.Message); return; }
}
try
{
var skip_date_only_recurrence = hubs_to_skip_date_only_recurrence.Exists(x => x == this.id);
FeedGather(utc_midnight_in_tz, then, tid, ical, source_name, events_to_include, event_recurrence_types, skip_date_only_recurrence);
}
catch (Exception e3)
{
try
{
HandleE3(fr, results_dict, feedurl, tid, source_name, e3);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE3", e.Message); return; }
}
try
{
uniques = Utils.UniqueByTitleAndStart(events_to_include);
}
catch (Exception e4)
{
try
{
HandleE4(fr, results_dict, feedurl, tid, source_name, e4);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE4", e.Message); return; }
}
try
{
foreach (var unique in uniques)
{
// MaybeAugmentEventfulCategories(unique, eids_and_categories);
// sadly eventful categories can be haphazard so this augmentation has to be suppressed for now
lock (es)
{
AddIcalEvent(unique, fr, es, feedurl, source_name); // add to the all-feeds store
}
AddIcalEvent(unique, new FeedRegistry(id), es_feed_cache, feedurl, source_name); // and the per-feed cache
}
}
catch (Exception e5)
{
try
{
HandleE5(fr, results_dict, feedurl, tid, source_name, e5);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE5", e.Message); return; }
}
try
{
FeedStats(fr, feedurl, event_recurrence_types, uniques);
}
catch (Exception e6)
{
try
{
HandleE6(results_dict, feedurl, tid, source_name, e6);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE6", e.Message); return; }
}
try
{
Utils.SaveFeedObjToCache(id, feedurl, es_feed_cache); // cache the per-feed zoned eventstore
}
catch (Exception e) { GenUtils.LogMsg("exception", "SaveFeedObjToCache: " + this.id + ", " + feedurl, e.Message); return; }
try { results_dict[feedurl] = "ok: " + source_name; }
catch (Exception e)
{
GenUtils.LogMsg("exception", "update results_dict[feedurl]", e.Message);
return;
}
});
}
catch (AggregateException agg_ex)
{
var inners = agg_ex.InnerExceptions;
foreach (var inner in inners)
GenUtils.LogMsg("exception", "CollectIcal: parallel inner exception: " + inner.Message, inner.StackTrace);
}
var json = JsonConvert.SerializeObject(results_dict);
json = GenUtils.PrettifyJson(json);
bs.PutBlob(id, "feed_processing_results.json", json);
SerializeStatsAndIntermediateOutputs(fr, es, ical_ical, new NonIcalStats(), SourceType.ical);
}
}
public void CollectEventful(ZonedEventStore es, bool test)
{
this.LoadTags();
using (eventful_ical)
{
string location = this.calinfo.where;
var page_size = test ? test_pagesize : 100;
string method = "events/search";
var uniques = new Dictionary<string, XElement>(); // dedupe by title + start
if (this.mock_eventful) // normally get tags from compiled object
this.tags = new List<string>() { eventful_cat_map.Keys.First() }; // but for testing, just use one that will pass the filter
foreach (var tag in this.tags) // do tagwise search of eventful
{
if ( eventful_cat_map.Keys.ToList().HasItem(tag) == false )
continue;
var msg = string.Format("{0}: loading eventful events for {1} ", this.id, tag);
GenUtils.LogMsg("info", msg, null);
string args = MakeEventfulArgs(location, page_size, this.eventful_cat_map[tag]);
var xdoc = CallEventfulApi(method, args);
var str_page_count = XmlUtils.GetXeltValue(xdoc.Root, ElmcityUtils.Configurator.no_ns, "page_count");
int page_count = test ? test_pagecount : Convert.ToInt16(str_page_count);
foreach (XElement evt in EventfulIterator(page_count, args, "events/search", "event"))
{
evt.Add(new XElement("category", tag));
var lat = XmlUtils.GetXeltValue(evt, ElmcityUtils.Configurator.no_ns, "latitude");
var lon = XmlUtils.GetXeltValue(evt, ElmcityUtils.Configurator.no_ns, "longitude");
if (lat != null && lon != null) // check if eventful's radius filter failed us
{
var distance = Utils.GeoCodeCalc.CalcDistance(lat, lon, this.calinfo.lat, this.calinfo.lon);
if (distance > this.calinfo.radius) // if so
continue; // skip this one
}
var ns = ElmcityUtils.Configurator.no_ns;
uniques.AddOrUpdateDictionary<string, XElement>(
XmlUtils.GetXeltValue(evt, ns, "title") +
XmlUtils.GetXeltValue(evt, ns, "start_time") +
XmlUtils.GetXeltValue(evt, ns, "category"),
evt);
}
}
Dictionary<string, int> event_count_by_venue = new Dictionary<string, int>();
int event_num = 0;
foreach (XElement evt in uniques.Values)
{
event_num += 1;
if (event_num > Configurator.eventful_max_events)
break;
var ns = ElmcityUtils.Configurator.no_ns;
var title = XmlUtils.GetXeltValue(evt, ns, "title");
var start_time = XmlUtils.GetXeltValue(evt, ns, "start_time");
var venue_name = XmlUtils.GetXeltValue(evt, ns, "venue_name");
var url = XmlUtils.GetXeltValue(evt, ns, "url");
IncrementEventCountByVenue(event_count_by_venue, venue_name);
AddEventfulEvent(es, venue_name, evt);
}
estats.venuecount = event_count_by_venue.Keys.Count;
estats.whenchecked = DateTime.Now.ToUniversalTime();
SerializeStatsAndIntermediateOutputs(es, eventful_ical, estats, SourceType.eventful);
//bs.PutBlob(this.id, SourceType.eventful.ToString() + "_cats.txt", String.Join(",", this.eventful_tags.ToList()));
}
}
public void DeserializeTwoZonedEvents()
{
SerializeTwoZonedEvents();
Utils.Wait(5);
var zoned = new ZonedEventStore(calinfo, SourceType.ical);
var uri = BlobStorage.MakeAzureBlobUri(test_container, zoned.objfile, false);
var obj = (ZonedEventStore)BlobStorage.DeserializeObjectFromUri(uri);
var out_evts = (List<ZonedEvent>)obj.events;
Assert.AreEqual(2, out_evts.Count);
var out_evt = (ZonedEvent)out_evts[0];
Assert.AreEqual(in_evt0_zoned.dtstart.UniversalTime.Ticks, out_evt.dtstart.UniversalTime.Ticks);
Assert.AreEqual(in_evt0_zoned.title, out_evt.title);
}
public void CollectFacebook(ZonedEventStore es, bool test)
{
using (facebook_ical)
{
var args = string.Format("q={0}&since=yesterday&limit=1000", this.calinfo.where);
var method = "search";
var msg = string.Format("{0}: loading facebook events", this.id);
GenUtils.LogMsg("info", msg, null);
var uniques = new Dictionary<string, FacebookEvent>(); // dedupe by title + start
foreach (var fb_event in FacebookIterator(method, args))
uniques.AddOrUpdateDictionary<string, FacebookEvent>(fb_event.name + fb_event.dt.ToString(), fb_event);
foreach (FacebookEvent fb_event in uniques.Values)
{
var dtstart_with_zone = new DateTimeWithZone(fb_event.dt, this.calinfo.tzinfo);
if (dtstart_with_zone.UniversalTime < Utils.MidnightInTz(this.calinfo.tzinfo).UniversalTime)
continue;
AddFacebookEvent(es, fb_event, dtstart_with_zone);
}
fbstats.whenchecked = DateTime.Now.ToUniversalTime();
SerializeStatsAndIntermediateOutputs(es, facebook_ical, fbstats, SourceType.facebook);
}
}
public static ZonelessEventStore ZonedToZoneless(string id, Calinfo calinfo, ZonedEventStore es_zoned)
{
var es_zoneless = new ZonelessEventStore(calinfo);
foreach (var evt in es_zoned.events)
es_zoneless.AddEvent(evt.title, evt.url, evt.source, evt.lat, evt.lon, evt.dtstart.LocalTime, evt.dtend.LocalTime, evt.allday, evt.categories, evt.description, evt.location);
EventStore.UniqueFilterSortSerialize(id, es_zoneless);
return es_zoneless;
}