public void AddZonedEventIncrementsCount() { var zoned = new ZonedEventStore(calinfo, SourceType.ical); Assert.AreEqual(0, zoned.events.Count); zoned.AddEvent(in_evt0_zoned.title, in_evt0_zoned.url, in_evt0_zoned.source, in_evt0_zoned.dtstart, in_evt0_zoned.dtend, test_lat, test_lon, false, in_evt0_zoned.categories, test_description, test_location); Assert.AreEqual(1, zoned.events.Count); }
public void TagsAndUrlsAreCoalesced() // 3 events with same title + start should coalesce tags and urls { DeleteZonedObjects(keene_test_hub); var dtstart = new DateTimeWithZone(DateTime.Now, calinfo_keene.tzinfo); var dtend = new DateTimeWithZone(dtstart.LocalTime + TimeSpan.FromHours(1), calinfo_keene.tzinfo); var es1 = new ZonedEventStore(calinfo_keene, SourceType.ical); es1.AddEvent( "event", "http://1", "source1", dtstart, dtend, "1", "1", false, "cat1", "first event", "first location" ); es1.Serialize(); Assert.IsTrue(calinfo_keene.eventful); var es2 = new ZonedEventStore(calinfo_keene, SourceType.eventful); es2.AddEvent( "event", "http://2", "source2", dtstart, dtend, "2", "2", false, "cat2,cat2a", "second event", "second location" ); es2.Serialize(); EventStore.CombineZonedEventStoresToZonelessEventStore(keene_test_hub, settings); var es = new ZonelessEventStore(calinfo_keene).Deserialize(); Assert.That(es.events.Count == 1); var evt = es.events.Find(e => e.title == "event"); Assert.That(evt.categories == "cat1,cat2,cat2a"); Assert.That(evt.urls_and_sources.Keys.Count == 2); Assert.That(evt.urls_and_sources["http://1"] == "source1"); Assert.That(evt.urls_and_sources["http://2"] == "source2"); }
public void SerializeTwoZonedEvents() { var zoned = new ZonedEventStore(calinfo, SourceType.ical); zoned.AddEvent(in_evt0_zoned.title, in_evt0_zoned.url, in_evt0_zoned.source, in_evt0_zoned.dtstart, in_evt0_zoned.dtend, test_lat, test_lon, in_evt0_zoned.allday, test_category, test_description, test_location); zoned.AddEvent(in_evt1_zoned.title, in_evt1_zoned.url, in_evt1_zoned.source, in_evt1_zoned.dtstart, in_evt1_zoned.dtend, test_lat, test_lon, in_evt1_zoned.allday, test_category, test_description, test_location); Assert.AreEqual(2, zoned.events.Count); var response = bs.SerializeObjectToAzureBlob(zoned, test_container, zoned.objfile); Assert.AreEqual(HttpStatusCode.Created, response.HttpResponse.status); }
private static ZonelessEventStore ProcessEventBriteExample(string example, string source, Calinfo calinfo, Collector collector) { DeleteZonedObjects(calinfo.id); var es = new ZonedEventStore(calinfo, SourceType.eventbrite); collector.CollectEventBrite(es); EventStore.CombineZonedEventStoresToZonelessEventStore(calinfo.id, settings); var zes = new ZonelessEventStore(calinfo).Deserialize(); return(zes); }
public static ZonelessEventStore ZonedToZoneless(string id, Calinfo calinfo, ZonedEventStore es_zoned) { var es_zoneless = new ZonelessEventStore(calinfo); foreach (var evt in es_zoned.events) { es_zoneless.AddEvent(evt.title, evt.url, evt.source, evt.lat, evt.lon, evt.dtstart.LocalTime, evt.dtend.LocalTime, evt.allday, evt.categories, evt.description, evt.location); } ZonelessEventStore.Finalize(calinfo, es_zoneless); return(es_zoneless); }
public void DeserializeTwoZonedEvents() { SerializeTwoZonedEvents(); Utils.Wait(5); var zoned = new ZonedEventStore(calinfo, SourceType.ical); var uri = BlobStorage.MakeAzureBlobUri(test_container, zoned.objfile, false); var obj = (ZonedEventStore)BlobStorage.DeserializeObjectFromUri(uri); var out_evts = (List <ZonedEvent>)obj.events; Assert.AreEqual(2, out_evts.Count); var out_evt = (ZonedEvent)out_evts[0]; Assert.AreEqual(in_evt0_zoned.dtstart.UniversalTime.Ticks, out_evt.dtstart.UniversalTime.Ticks); Assert.AreEqual(in_evt0_zoned.title, out_evt.title); }
private static ZonelessEventStore ProcessIcalExample(string example, string source, Calinfo calinfo, FeedRegistry fr, Collector collector, bool purge) { DeleteZonedObjects(calinfo.id); if (purge) { Utils.PurgeFeedCacheForHub(calinfo.id); } var feedurl = BlobStorage.MakeAzureBlobUri("admin", example + ".ics", false).ToString(); fr.AddFeed(feedurl, source); var es = new ZonedEventStore(calinfo, SourceType.ical); collector.CollectIcal(fr, es, false); EventStore.CombineZonedEventStoresToZonelessEventStore(calinfo.id, settings); var zes = new ZonelessEventStore(calinfo).Deserialize(); return(zes); }
public void SerializeAndDeserializeZonedEventStoreYieldsExpectedEvents() { var es = new ZonedEventStore(calinfo, SourceType.ical); es.AddEvent(title: title1, url: "http://foo", source: source1, dtstart: dt1_with_zone, dtend: min_with_zone, allday: false, lat: test_lat, lon: test_lon, categories: test_category, description: test_description, location: test_location); es.AddEvent(title: title2, url: "http://bar", source: source2, dtstart: dt2_with_zone, dtend: min_with_zone, lat: test_lat, lon: test_lon, allday: false, categories: test_category, description: test_description, location: test_location); var response = bs.SerializeObjectToAzureBlob(es, test_container, es.objfile); Assert.AreEqual(HttpStatusCode.Created, response.HttpResponse.status); var uri = BlobStorage.MakeAzureBlobUri(test_container, es.objfile, false); var es2 = (ZonedEventStore)BlobStorage.DeserializeObjectFromUri(uri); var evt1 = es2.events.First(); var evt2 = es2.events.Last(); Assert.AreEqual(evt1.dtstart.UniversalTime.Ticks, dt1_with_zone.UniversalTime.Ticks); Assert.AreEqual(evt2.dtstart.UniversalTime.Ticks, dt2_with_zone.UniversalTime.Ticks); Assert.AreEqual(evt1.title, title1); Assert.AreEqual(evt2.title, title2); }
public static void ClearObjFeedInCache(Calinfo calinfo, string cache_dir, string feedurl) { var blob_name = Utils.MakeCachedFeedObjName(calinfo.id, feedurl); var es = new ZonedEventStore(calinfo, SourceType.ical); bs.SerializeObjectToAzureBlob(es, cache_dir, blob_name); }
public void EventfulQueryYieldsValidFirstEvent() { var collector = new Collector(test_calinfo, settings); var events = (IEnumerable<XElement>)collector.EventfulIterator(1, test_eventful_args, "events/search", "event"); var es = new ZonedEventStore(test_calinfo, SourceType.ical); collector.AddEventfulEvent(es, test_venue, events.First()); Assert.That(es.events[0].title != ""); }
public ActionResult view_calendar(string feedurl, string id) { var r = new ContentResult(); r.ContentType = "text/html"; if (id != String.Empty && Utils.CachedFeedObjExists(id, feedurl)) return view_cached_calendar(feedurl, id); DDay.iCal.iCalendar ical; var source = ""; // unneeded for this single-cal purpose try { ical = Utils.iCalFromFeedUrl(feedurl, settings); } catch (Exception e) { r.Content = string.Format(@"Sorry, unable to parse {0} as an iCalendar feed. The error was: {1}", feedurl, e.Message + e.StackTrace); return r; } var events_to_include = new List<DDay.iCal.Event>(); var event_recurrence_types = new Dictionary<DDay.iCal.Event, Collector.RecurrenceType>(); var calinfo = new Calinfo(id); var collector = new Collector(calinfo, settings); try { DateTime utc_midnight_in_tz = Utils.MidnightInTz(calinfo.tzinfo).UniversalTime; DateTime then = utc_midnight_in_tz.AddDays(calinfo.icalendar_horizon_days); collector.FeedGather(utc_midnight_in_tz, then, 0, ical, source, events_to_include, event_recurrence_types, false); var es_zoned = new ZonedEventStore(calinfo, SourceType.ical); var fr = new FeedRegistry(id); foreach (var evt in events_to_include) collector.AddIcalEvent(evt, fr, es_zoned, feedurl, source); var es_zoneless = new ZonelessEventStore(calinfo); r = view_calendar_helper(id, r, es_zoned, es_zoneless); } catch (Exception e) { r.Content = "Unable to view calendar. " + e.Message + e.StackTrace; return r; } return r; }
private static ContentResult view_calendar_helper(string id, ContentResult r, ZonedEventStore es_zoned, ZonelessEventStore es_zoneless) { foreach (var evt in es_zoned.events) es_zoneless.AddEvent(evt.title, evt.url.ToString(), evt.source, "", "", evt.dtstart.LocalTime, evt.dtend.LocalTime, evt.allday, evt.categories, evt.description, evt.location); es_zoneless.events = EventStore.UniqueByTitleAndStart(id, es_zoneless.events, save_tag_sources: false); es_zoneless.ExcludePastEvents(); es_zoneless.SortEventList(); var cr = new CalendarRenderer(id); r.Content = cr.RenderHtmlEventsOnly(es_zoneless, null, 0, DateTime.MinValue, DateTime.MinValue, new Dictionary<string, object>() { { "announce_time_of_day", false }, { "add_to_cal", false }, { "inline_descriptions", true }, {"taglist",false } } ); return r; }
public void SerializeAndDeserializeZonedEventStoreYieldsExpectedEvents() { var es = new ZonedEventStore(calinfo, SourceType.ical); es.AddEvent(title:title1, url:"http://foo", source:source1, dtstart:dt1_with_zone, dtend:min_with_zone, allday:false, lat:test_lat, lon:test_lon, categories:test_category, description:test_description, location: test_location); es.AddEvent(title:title2, url:"http://bar", source:source2, dtstart:dt2_with_zone, dtend:min_with_zone, lat:test_lat, lon:test_lon, allday:false, categories:test_category, description:test_description, location: test_location); var response = bs.SerializeObjectToAzureBlob(es, test_container, es.objfile); Assert.AreEqual(HttpStatusCode.Created, response.HttpResponse.status); var uri = BlobStorage.MakeAzureBlobUri(test_container, es.objfile,false); var es2 = (ZonedEventStore)BlobStorage.DeserializeObjectFromUri(uri); var evt1 = es2.events.First(); var evt2 = es2.events.Last(); Assert.AreEqual(evt1.dtstart.UniversalTime.Ticks, dt1_with_zone.UniversalTime.Ticks); Assert.AreEqual(evt2.dtstart.UniversalTime.Ticks, dt2_with_zone.UniversalTime.Ticks); Assert.AreEqual(evt1.title, title1); Assert.AreEqual(evt2.title, title2); }
public void CollectUpcoming(ZonedEventStore es, bool test) { using (upcoming_ical) { var page_size = test ? test_pagesize : 100; var args = MakeUpcomingApiArgs(UpcomingSearchStyle.latlon); var method = "event.search"; var xdoc = CallUpcomingApi(method, args); int page_count = 1; try { var result_count = GetUpcomingResultCount(xdoc); if (result_count == 0) // try the other search style (upcoming seems flaky that way) { args = MakeUpcomingApiArgs(UpcomingSearchStyle.latlon); xdoc = CallUpcomingApi(method, args); GetUpcomingResultCount(xdoc); } page_count = result_count / page_size; } catch { GenUtils.LogMsg("warning", "CollectUpcoming", "resultcount unavailable"); return; } if (test == true && page_count > test_pagecount) page_count = test_pagecount; if (page_count == 0) page_count = 1; var msg = string.Format("{0}: loading {1} upcoming events", this.id, page_count * page_size); GenUtils.LogMsg("info", msg, null); Dictionary<string, int> event_count_by_venue = new Dictionary<string, int>(); int event_num = 0; var uniques = new Dictionary<string, XElement>(); // dedupe by name + start foreach (var evt in UpcomingIterator(page_count, method)) uniques.AddOrUpdateDictionary<string, XElement>(evt.Attribute("name").ToString() + evt.Attribute("start_date").ToString(), evt); foreach (XElement evt in uniques.Values) { event_num += 1; if (event_num > Configurator.upcoming_max_events) break; var dtstart = DateTimeWithZoneFromUpcomingXEvent(evt); if (dtstart.UniversalTime < Utils.MidnightInTz(this.calinfo.tzinfo).UniversalTime) continue; var venue_name = evt.Attribute("venue_name").Value; IncrementEventCountByVenue(event_count_by_venue, venue_name); AddUpcomingEvent(es, venue_name, evt); } ustats.venuecount = event_count_by_venue.Keys.Count; ustats.whenchecked = DateTime.Now.ToUniversalTime(); SerializeStatsAndIntermediateOutputs(es, upcoming_ical, ustats, SourceType.upcoming); } }
public void UpcomingUrlsAreNormalized() { DeleteZonedObjects(keene_test_hub); var dtstart = new DateTimeWithZone(DateTime.Now, calinfo_keene.tzinfo); var dtend = new DateTimeWithZone(dtstart.LocalTime + TimeSpan.FromHours(1), calinfo_keene.tzinfo); var es1 = new ZonedEventStore(calinfo_keene, SourceType.ical); es1.AddEvent( "event", "http://upcoming.yahoo.com/event/8504144/", "Comedy Showcase", dtstart, dtend, "1", "1", false, "comedy", "first event", "first location" ); es1.Serialize(); var es2 = new ZonedEventStore(calinfo_keene, SourceType.upcoming); es2.AddEvent( "event", "http://upcoming.yahoo.com/event/8504144", "Ann Arbor Comedy Showcase", dtstart, dtend, "1", "1", false, "upcoming", "first event", "first location" ); es2.Serialize(); EventStore.CombineZonedEventStoresToZonelessEventStore(keene_test_hub, settings); var es = new ZonelessEventStore(calinfo_keene).Deserialize(); Assert.That(es.events.Count == 1); var evt = es.events.Find(e => e.title == "event"); var categories = evt.categories.Split(',').ToList(); categories.Sort(); Assert.That(categories.SequenceEqual(new List<string>() { "comedy", "upcoming" })); Assert.That(evt.urls_and_sources.Keys.Count == 1); }
public void AddEventBriteEvent(ZonedEventStore es, XElement evt, DateTimeWithZone dtstart, DateTimeWithZone dtend) { string title; string event_url; string source; bool all_day; string categories; var evt_tmp = MakeDDayEventFromEventBriteEvent(evt, out title, out event_url, out source, out all_day, out categories); AddEventToDDayIcal(eventbrite_ical, evt_tmp); var min = DateTimeWithZone.MinValue(this.calinfo.tzinfo); // todo: dig out location from http://developer.eventbrite.com/doc/events/event_search/ es.AddEvent(title: title, url: event_url, source: source, dtstart: dtstart, dtend: dtend, lat: null, lon: null, allday: all_day, categories: categories, description: null, location: null); }
public void AddUpcomingEvent(ZonedEventStore es, string venue_name, XElement evt) { var title = evt.Attribute("name").Value; var event_url = "http://upcoming.yahoo.com/event/" + evt.Attribute("id").Value; var source = venue_name; var venue_id = evt.Attribute("venue_id").Value; var venue_url = "http://upcoming.yahoo.com/venue/" + venue_id; string lat = this.calinfo.lat; // default to hub's lat/lon string lon = this.calinfo.lon; try { lat = evt.Attribute("latitude").Value; lon = evt.Attribute("longitude").Value; } catch { GenUtils.LogMsg("warning", "AddUpcomingEvent", "cannot parse lat/lon"); } string categories = "upcoming"; if (evt.Attribute("tags") != null) { var evt_tags = evt.Attribute("tags").Value.Split(','); if (evt_tags.Length > 0) { var intersection = this.tags.Intersect(evt_tags); categories += "," + String.Join(",", intersection.ToArray()); } } ustats.eventcount++; bool all_day = false; var dtstart = DateTimeWithZoneFromUpcomingXEvent(evt, ref all_day); string location = venue_name; string venue_address = evt.Attribute("venue_address").Value; if (!String.IsNullOrEmpty(venue_address)) location += ", " + venue_address; var evt_tmp = MakeTmpEvt(this.calinfo, dtstart, DateTimeWithZone.MinValue(this.calinfo.tzinfo), title, url: event_url, location: location, description: source, lat: lat, lon: lon, allday: all_day); AddEventToDDayIcal(upcoming_ical, evt_tmp); var min = DateTimeWithZone.MinValue(this.calinfo.tzinfo); es.AddEvent(title: title, url: event_url, source: source, dtstart: dtstart, dtend: min, lat: lat, lon: lon, allday: all_day, categories: categories, description: null, location: location); }
public void CollectEventBrite(ZonedEventStore es) { this.LoadTags(); using (eventbrite_ical) { string method = "event_search"; string args = MakeEventBriteArgs(); int page_count = GetEventBritePageCount(method, args); var msg = string.Format("{0}: found about {1} pages of eventbrite events", this.id, page_count * eventbrite_page_size); GenUtils.LogMsg("info", msg, null); int event_num = 0; foreach (XElement evt in EventBriteIterator(page_count, method, args)) { if (event_num > Configurator.eventbrite_max_events) break; var dtstart_with_tz = Utils.ExtractEventBriteDateTime(evt, this.calinfo.tzinfo, "start_date"); var dtend_with_tz = Utils.ExtractEventBriteDateTime(evt, this.calinfo.tzinfo, "end_date"); if (dtstart_with_tz.UniversalTime < Utils.MidnightInTz(this.calinfo.tzinfo).UniversalTime) continue; AddEventBriteEvent(es, evt, dtstart_with_tz, dtend_with_tz); } ebstats.whenchecked = DateTime.Now.ToUniversalTime(); SerializeStatsAndIntermediateOutputs(es, eventbrite_ical, ebstats, SourceType.eventbrite); bs.PutBlob(this.id, SourceType.eventbrite.ToString() + "_cats.txt", String.Join(",", this.eventbrite_tags.ToList())); } }
// put the event into a) the eventstore, and b) the per-type intermediate icalendar object public void AddIcalEvent(DDay.iCal.Event evt, FeedRegistry fr, ZonedEventStore es, string feedurl, string source) { try { evt = NormalizeIcalEvt(evt, feedurl, source); DateTimeWithZone dtstart; DateTimeWithZone dtend; var tzinfo = this.calinfo.tzinfo; //dtstart = Utils.DtWithZoneFromICalDateTime(evt.Start.Value, tzinfo); //dtend = (evt.DTEnd == null) ? new Utils.DateTimeWithZone(DateTime.MinValue, tzinfo) : Utils.DtWithZoneFromICalDateTime(evt.End.Value, tzinfo); //dtstart = new Utils.DateTimeWithZone(evt.Start.Value,tzinfo); //dtend = new Utils.DateTimeWithZone(evt.End.Value,tzinfo); var localstart = evt.DTStart.IsUniversalTime ? TimeZoneInfo.ConvertTimeFromUtc(evt.Start.UTC, tzinfo) : evt.Start.Local; dtstart = new DateTimeWithZone(localstart, tzinfo); var localend = evt.DTEnd.IsUniversalTime ? TimeZoneInfo.ConvertTimeFromUtc(evt.End.UTC, tzinfo) : evt.End.Local; dtend = new DateTimeWithZone(localend, tzinfo); MakeGeo(this.calinfo, evt, this.calinfo.lat, this.calinfo.lon); string categories = null; if (evt.Categories != null && evt.Categories.Count() > 0) categories = string.Join(",", evt.Categories.ToList().Select(cat => cat.ToString().ToLower())); string description = this.calinfo.has_descriptions ? evt.Description : null; string location = this.calinfo.has_locations ? evt.Location : null; es.AddEvent(title: evt.Summary, url: evt.Url.ToString(), source: source, dtstart: dtstart, dtend: dtend, lat: this.calinfo.lat, lon: this.calinfo.lon, allday: evt.IsAllDay, categories: categories, description: description, location: location); var evt_tmp = MakeTmpEvt(this.calinfo, dtstart: dtstart, dtend: dtend, title: evt.Summary, url: evt.Url.ToString(), location: evt.Location, description: source, lat: this.calinfo.lat, lon: this.calinfo.lon, allday: evt.IsAllDay); AddEventToDDayIcal(ical_ical, evt_tmp); if ( fr.stats.ContainsKey(feedurl) ) // won't be true when adding to the per-feed obj cache fr.stats[feedurl].loaded++; // and this will have already been counted by the all-feeds AddIcalEvent } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "AddIcalEvent", source + ": " + e.Message + ": " + evt.Summary); } }
public void AddFacebookEvent(ZonedEventStore es, FacebookEvent fb_event, DateTimeWithZone dtstart) { var title = fb_event.name; var event_url = "http://www.facebook.com/event.php?eid=" + fb_event.id; var source = "facebook"; var location = fb_event.location; var all_day = false; fbstats.eventcount++; var evt_tmp = MakeTmpEvt(this.calinfo, dtstart, DateTimeWithZone.MinValue(this.calinfo.tzinfo), title, url: event_url, location: location, description: source, lat: this.calinfo.lat, lon: this.calinfo.lon, allday: all_day); AddEventToDDayIcal(facebook_ical, evt_tmp); var min = DateTimeWithZone.MinValue(this.calinfo.tzinfo); es.AddEvent(title: title, url: event_url, source: source, dtstart: dtstart, dtend: min, lat: null, lon: null, allday: all_day, categories: "facebook", description: null, location: location); }
public void AddEventfulEvent(ZonedEventStore es, string venue_name, XElement evt) { var str_dtstart = XmlUtils.GetXeltValue(evt, ElmcityUtils.Configurator.no_ns, "start_time"); DateTime dtstart = Utils.LocalDateTimeFromLocalDateStr(str_dtstart); var dtstart_with_tz = new DateTimeWithZone(dtstart, this.calinfo.tzinfo); if (dtstart_with_tz.UniversalTime < Utils.MidnightInTz(this.calinfo.tzinfo).UniversalTime) return; var no_ns = ElmcityUtils.Configurator.no_ns; var event_id = evt.Attribute("id").Value; var event_owner = XmlUtils.GetXeltValue(evt, no_ns, "owner"); var title = XmlUtils.GetXeltValue(evt, no_ns, "title"); var venue_url = XmlUtils.GetXeltValue(evt, no_ns, "venue_url"); var all_day = XmlUtils.GetXeltValue(evt, no_ns, "all_day") == "1"; var venue_address = XmlUtils.GetXeltValue(evt, no_ns, "venue_address"); string lat = this.calinfo.lat; // default to hub lat/lon string lon = this.calinfo.lon; lat = XmlUtils.GetXeltValue(evt, no_ns, "latitude"); lon = XmlUtils.GetXeltValue(evt, no_ns, "longitude"); if (lat == null || lon == null) { GenUtils.LogMsg("warning", "AddEventfulEvent", "no lat/lon"); } var category = XmlUtils.GetXeltValue(evt, ElmcityUtils.Configurator.no_ns, "category"); if (String.IsNullOrWhiteSpace(category)) category = SourceType.eventful.ToString(); else category = SourceType.eventful.ToString() + "," + category; string event_url = "http://eventful.com/events/" + event_id; string source = venue_name; if (String.IsNullOrEmpty(source)) source = "Unnamed Eventful Venue"; string location; if (!String.IsNullOrEmpty(source)) { location = venue_name; if (!String.IsNullOrEmpty(venue_address)) location += ", " + venue_address; } else { location = event_url; } estats.eventcount++; var evt_tmp = MakeTmpEvt(this.calinfo, dtstart_with_tz, DateTimeWithZone.MinValue(this.calinfo.tzinfo), title, url: event_url, location: location, description: source, lat: lat, lon: lon, allday: all_day); AddEventToDDayIcal(eventful_ical, evt_tmp); var min = DateTimeWithZone.MinValue(this.calinfo.tzinfo); // suppress description to minimize amount of eventful info carried in the event packet // allow location but it is subject to the per-hub locations setting (calinfo.has_locations) es.AddEvent(title: title, url: event_url, source: source, dtstart: dtstart_with_tz, dtend: min, lat: lat, lon: lon, allday: all_day, categories: category, description: null, location: location); }
public void EventfulUrlsAreNormalized() { DeleteZonedObjects(keene_test_hub); var dtstart = new DateTimeWithZone(DateTime.Now, calinfo_keene.tzinfo); var dtend = new DateTimeWithZone(dtstart.LocalTime + TimeSpan.FromHours(1), calinfo_keene.tzinfo); var es1 = new ZonedEventStore(calinfo_keene, SourceType.ical); es1.AddEvent( "event", "http://eventful.com/E0-001-039987477-3", "The Blind Pig", dtstart, dtend, "1", "1", false, "music", "first event", "first location" ); es1.Serialize(); var es2 = new ZonedEventStore(calinfo_keene, SourceType.eventful); es2.AddEvent( "event", "http://eventful.com/events/E0-001-039987477-3", "Blind Pig", dtstart, dtend, "1", "1", false, "eventful", "first event", "first location" ); es2.Serialize(); EventStore.CombineZonedEventStoresToZonelessEventStore(keene_test_hub, settings); var es = new ZonelessEventStore(calinfo_keene).Deserialize(); Assert.That(es.events.Count == 1); var evt = es.events.Find(e => e.title == "event"); var categories = evt.categories.Split(',').ToList(); categories.Sort(); Assert.That(categories.SequenceEqual(new List<string>() { "eventful", "music" })); Assert.That(evt.urls_and_sources.Keys.Count == 1); }
private static ZonelessEventStore ProcessUpcomingExample(string example, string source, Calinfo calinfo, Collector collector) { DeleteZonedObjects(calinfo.id); var es = new ZonedEventStore(calinfo, SourceType.upcoming); collector.CollectUpcoming(es, test: true); EventStore.CombineZonedEventStoresToZonelessEventStore(calinfo.id, settings); var zes = new ZonelessEventStore(calinfo).Deserialize(); return zes; }
public void TagsAndUrlsAreCoalesced() { DeleteZonedObjects(keene_test_hub); var dtstart = new DateTimeWithZone(DateTime.Now, calinfo_keene.tzinfo); var dtend = new DateTimeWithZone(dtstart.LocalTime + TimeSpan.FromHours(1), calinfo_keene.tzinfo); var es1 = new ZonedEventStore(calinfo_keene, SourceType.ical); es1.AddEvent( "event", "http://1", "source1", dtstart, dtend, "1", "1", false, "cat1", "first event", "first location" ); es1.Serialize(); Assert.IsTrue(calinfo_keene.eventful); var es2 = new ZonedEventStore(calinfo_keene, SourceType.eventful); es2.AddEvent( "event", "http://2", "source2", dtstart, dtend, "2", "2", false, "cat2,cat2a", "second event", "second location" ); es2.Serialize(); Assert.IsTrue(calinfo_keene.upcoming); var es3 = new ZonedEventStore(calinfo_keene, SourceType.upcoming); es3.AddEvent( "event", "http://3", "source3", dtstart, dtend, "3", "3", false, "cat3,cat3a", "third event", "third location" ); es3.AddEvent( "another event", "http://4", "source4", dtstart, dtend, "4", "4", false, "cat4,cat4a", "fourth event", "fourth location" ); es3.Serialize(); EventStore.CombineZonedEventStoresToZonelessEventStore(keene_test_hub, settings); var es = new ZonelessEventStore(calinfo_keene).Deserialize(); Assert.That(es.events.Count == 2); var evt = es.events.Find(e => e.title == "event"); Assert.That(evt.categories == "cat1,cat2,cat2a,cat3,cat3a"); Assert.That(evt.urls_and_sources.Keys.Count == 3); Assert.That(evt.urls_and_sources["http://1"] == "source1"); Assert.That(evt.urls_and_sources["http://2"] == "source2"); Assert.That(evt.urls_and_sources["http://3"] == "source3"); //Assert.That(evt.list_of_urls_and_sources[0][0] == "http://1" && evt.list_of_urls_and_sources[0][1] == "source1"); //Assert.That(evt.list_of_urls_and_sources[1][0] == "http://2" && evt.list_of_urls_and_sources[1][1] == "source2"); //Assert.That(evt.list_of_urls_and_sources[2][0] == "http://3" && evt.list_of_urls_and_sources[2][1] == "source3"); }
private void AddEventsFromCachedIcalFeedObj(ZonedEventStore es, string feedurl) { ZonedEventStore es_saved = Utils.GetFeedObjFromCache(this.calinfo, feedurl); foreach (ZonedEvent evt in es_saved.events) { es.events.Add(evt); var evt_tmp = MakeTmpEvt(this.calinfo, dtstart: evt.dtstart, dtend: evt.dtend, title: evt.title, url: evt.url, location: evt.location, description: evt.description, lat: evt.lat, lon: evt.lon, allday: evt.allday); AddEventToDDayIcal(this.ical_ical, evt_tmp); } }
private static ZonelessEventStore ProcessIcalExample(string example, string source, Calinfo calinfo, FeedRegistry fr, Collector collector) { DeleteZonedObjects(calinfo.id); var feedurl = BlobStorage.MakeAzureBlobUri("admin", example + ".ics", false).ToString(); fr.AddFeed(feedurl, source); var es = new ZonedEventStore(calinfo, SourceType.ical); collector.CollectIcal(fr, es, false); EventStore.CombineZonedEventStoresToZonelessEventStore(calinfo.id, settings); var zes = new ZonelessEventStore(calinfo).Deserialize(); return zes; }
public void CollectIcal(FeedRegistry fr, ZonedEventStore es) { CollectIcal(fr, es, false); }
public void CollectIcal(FeedRegistry fr, ZonedEventStore es, bool test) { this.LoadTags(); using (ical_ical) { Dictionary<string, string> feeds = fr.feeds; DateTime utc_midnight_in_tz = Utils.MidnightInTz(this.calinfo.tzinfo).UniversalTime; // enforce the limit. necessary because processing of icalendar sources can involve // the unrolling of recurrence, and that can't go on forever DateTime then = utc_midnight_in_tz.AddDays(calinfo.icalendar_horizon_days); List<string> feedurls = test ? feeds.Keys.Take(test_feeds).ToList() : feeds.Keys.ToList(); ParallelOptions options = new ParallelOptions(); options.MaxDegreeOfParallelism = Convert.ToInt32(this.settings["max_feed_processing_parallelism"]); //options.MaxDegreeOfParallelism = 1; // for debugging List<string> hubs_to_skip_date_only_recurrence = new List<string>(); try { hubs_to_skip_date_only_recurrence = settings["hubs_to_skip_date_only_recurrence"].Split(',').ToList(); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "hubs_to_skip_date_only_recurrence", e.Message); } GenUtils.LogMsg("info", id + " loading " + feedurls.Count() + " feeds", null); var results_dict = new System.Collections.Concurrent.ConcurrentDictionary<string, string>(); try { //foreach (var feedurl in feedurls) Parallel.ForEach(source: feedurls, parallelOptions: options, body: (feedurl) => { if (settings["eventful_feeds_enabled"] == "False" && feedurl.StartsWith("http://eventful.com/") ) return; if ( settings["eventbrite_feeds_enabled"] == "False" && feedurl.Contains("ics_from_eventbrite") ) return; var tid = System.Threading.Thread.CurrentThread.ManagedThreadId; results_dict.TryAdd(feedurl, null); ZonedEventStore es_feed_cache = new ZonedEventStore(calinfo, SourceType.ical); var eids_and_categories = new Dictionary<string, List<string>>(); // for augmenting eventful ics feeds with categories from corresponding atom feeds // http://eventful.com/ical/annarbor/venues/michigan-theater-/V0-001-000827373-5 if ( feedurl.StartsWith("http://eventful.com/ical") ) { var atom_url = feedurl.Replace("ical","atom"); eids_and_categories = Utils.CategoriesFromEventfulAtomFeed(atom_url); } iCalendar ical = new DDay.iCal.iCalendar(); string source_name = "source_name"; List<DDay.iCal.Event> events_to_include = new List<DDay.iCal.Event>(); Dictionary<DDay.iCal.Event, RecurrenceType> event_recurrence_types = new Dictionary<DDay.iCal.Event, RecurrenceType>(); List<DDay.iCal.Event> uniques = new List<DDay.iCal.Event>(); string feedtext = ""; try { source_name = FeedSetup(fr, feeds, feedurl, tid, source_name); } catch (Exception e0) { try { HandleE0(results_dict, feedurl, tid, e0); return; // http://stackoverflow.com/questions/3765038/is-there-an-equivalent-to-continue-in-a-parallel-foreach } catch (Exception e) { GenUtils.LogMsg("exception", "HandleE0", e.Message); } } bool changed = false; try { feedtext = GetFeedTextFromFeedUrl(fr, this.calinfo, source_name, feedurl, this.wait_secs, this.max_retries, this.timeout_secs, this.settings, ref changed); } catch (Exception e1) // exception while loading feed { try { HandleE1(results_dict, feedurl, tid, source_name, e1); return; } catch (Exception e) { GenUtils.LogMsg("exception", "HandleE1", e.Message); return; } } try { var setting = "use_ics_cached_objects"; if (changed == false && Utils.CachedFeedObjExists(id, feedurl) && settings[setting].StartsWith("y") ) { GenUtils.LogMsg("info", "CollectIcal: using cached obj for " + feedurl, null); // the test UnchangedFeedUsesCachedObj checks for this entry AddEventsFromCachedIcalFeedObj(es, feedurl); // get events for the feedurl from the cached obj TransferCachedResults(this.id, results_dict, feedurl); // and result for this feedurl from the cached results TransferCachedStats(this.id, fr, feedurl); // and stats for this feedurl from cached stats return; // done for this feedurl } else { ical = ParseTheFeed(feedtext); } } catch (Exception e2) { try { HandleE2(fr, results_dict, feedurl, tid, source_name, e2); return; } catch (Exception e) { GenUtils.LogMsg("exception", "HandleE2", e.Message); return; } } if (ical == null || ical.Events.Count == 0) { try { HandleNoEvents(results_dict, feedurl, tid, source_name); return; } catch (Exception e) { GenUtils.LogMsg("exception", "HandleNoEvents", e.Message); return; } } try { var skip_date_only_recurrence = hubs_to_skip_date_only_recurrence.Exists(x => x == this.id); FeedGather(utc_midnight_in_tz, then, tid, ical, source_name, events_to_include, event_recurrence_types, skip_date_only_recurrence); } catch (Exception e3) { try { HandleE3(fr, results_dict, feedurl, tid, source_name, e3); return; } catch (Exception e) { GenUtils.LogMsg("exception", "HandleE3", e.Message); return; } } try { uniques = Utils.UniqueByTitleAndStart(events_to_include); } catch (Exception e4) { try { HandleE4(fr, results_dict, feedurl, tid, source_name, e4); return; } catch (Exception e) { GenUtils.LogMsg("exception", "HandleE4", e.Message); return; } } try { foreach (var unique in uniques) { // MaybeAugmentEventfulCategories(unique, eids_and_categories); // sadly eventful categories can be haphazard so this augmentation has to be suppressed for now lock (es) { AddIcalEvent(unique, fr, es, feedurl, source_name); // add to the all-feeds store } AddIcalEvent(unique, new FeedRegistry(id), es_feed_cache, feedurl, source_name); // and the per-feed cache } } catch (Exception e5) { try { HandleE5(fr, results_dict, feedurl, tid, source_name, e5); return; } catch (Exception e) { GenUtils.LogMsg("exception", "HandleE5", e.Message); return; } } try { FeedStats(fr, feedurl, event_recurrence_types, uniques); } catch (Exception e6) { try { HandleE6(results_dict, feedurl, tid, source_name, e6); return; } catch (Exception e) { GenUtils.LogMsg("exception", "HandleE6", e.Message); return; } } try { Utils.SaveFeedObjToCache(id, feedurl, es_feed_cache); // cache the per-feed zoned eventstore } catch (Exception e) { GenUtils.LogMsg("exception", "SaveFeedObjToCache: " + this.id + ", " + feedurl, e.Message); return; } try { results_dict[feedurl] = "ok: " + source_name; } catch (Exception e) { GenUtils.LogMsg("exception", "update results_dict[feedurl]", e.Message); return; } }); } catch (AggregateException agg_ex) { var inners = agg_ex.InnerExceptions; foreach (var inner in inners) GenUtils.LogMsg("exception", "CollectIcal: parallel inner exception: " + inner.Message, inner.StackTrace); } var json = JsonConvert.SerializeObject(results_dict); json = GenUtils.PrettifyJson(json); bs.PutBlob(id, "feed_processing_results.json", json); SerializeStatsAndIntermediateOutputs(fr, es, ical_ical, new NonIcalStats(), SourceType.ical); } }
public void CollectEventful(ZonedEventStore es, bool test) { this.LoadTags(); using (eventful_ical) { string location = this.calinfo.where; var page_size = test ? test_pagesize : 100; string method = "events/search"; var uniques = new Dictionary<string, XElement>(); // dedupe by title + start if (this.mock_eventful) // normally get tags from compiled object this.tags = new List<string>() { eventful_cat_map.Keys.First() }; // but for testing, just use one that will pass the filter foreach (var tag in this.tags) // do tagwise search of eventful { if ( eventful_cat_map.Keys.ToList().HasItem(tag) == false ) continue; var msg = string.Format("{0}: loading eventful events for {1} ", this.id, tag); GenUtils.LogMsg("info", msg, null); string args = MakeEventfulArgs(location, page_size, this.eventful_cat_map[tag]); var xdoc = CallEventfulApi(method, args); var str_page_count = XmlUtils.GetXeltValue(xdoc.Root, ElmcityUtils.Configurator.no_ns, "page_count"); int page_count = test ? test_pagecount : Convert.ToInt16(str_page_count); foreach (XElement evt in EventfulIterator(page_count, args, "events/search", "event")) { evt.Add(new XElement("category", tag)); var lat = XmlUtils.GetXeltValue(evt, ElmcityUtils.Configurator.no_ns, "latitude"); var lon = XmlUtils.GetXeltValue(evt, ElmcityUtils.Configurator.no_ns, "longitude"); if (lat != null && lon != null) // check if eventful's radius filter failed us { var distance = Utils.GeoCodeCalc.CalcDistance(lat, lon, this.calinfo.lat, this.calinfo.lon); if (distance > this.calinfo.radius) // if so continue; // skip this one } var ns = ElmcityUtils.Configurator.no_ns; uniques.AddOrUpdateDictionary<string, XElement>( XmlUtils.GetXeltValue(evt, ns, "title") + XmlUtils.GetXeltValue(evt, ns, "start_time") + XmlUtils.GetXeltValue(evt, ns, "category"), evt); } } Dictionary<string, int> event_count_by_venue = new Dictionary<string, int>(); int event_num = 0; foreach (XElement evt in uniques.Values) { event_num += 1; if (event_num > Configurator.eventful_max_events) break; var ns = ElmcityUtils.Configurator.no_ns; var title = XmlUtils.GetXeltValue(evt, ns, "title"); var start_time = XmlUtils.GetXeltValue(evt, ns, "start_time"); var venue_name = XmlUtils.GetXeltValue(evt, ns, "venue_name"); var url = XmlUtils.GetXeltValue(evt, ns, "url"); IncrementEventCountByVenue(event_count_by_venue, venue_name); AddEventfulEvent(es, venue_name, evt); } estats.venuecount = event_count_by_venue.Keys.Count; estats.whenchecked = DateTime.Now.ToUniversalTime(); SerializeStatsAndIntermediateOutputs(es, eventful_ical, estats, SourceType.eventful); //bs.PutBlob(this.id, SourceType.eventful.ToString() + "_cats.txt", String.Join(",", this.eventful_tags.ToList())); } }
public void DeserializeTwoZonedEvents() { SerializeTwoZonedEvents(); Utils.Wait(5); var zoned = new ZonedEventStore(calinfo, SourceType.ical); var uri = BlobStorage.MakeAzureBlobUri(test_container, zoned.objfile, false); var obj = (ZonedEventStore)BlobStorage.DeserializeObjectFromUri(uri); var out_evts = (List<ZonedEvent>)obj.events; Assert.AreEqual(2, out_evts.Count); var out_evt = (ZonedEvent)out_evts[0]; Assert.AreEqual(in_evt0_zoned.dtstart.UniversalTime.Ticks, out_evt.dtstart.UniversalTime.Ticks); Assert.AreEqual(in_evt0_zoned.title, out_evt.title); }
public void CollectFacebook(ZonedEventStore es, bool test) { using (facebook_ical) { var args = string.Format("q={0}&since=yesterday&limit=1000", this.calinfo.where); var method = "search"; var msg = string.Format("{0}: loading facebook events", this.id); GenUtils.LogMsg("info", msg, null); var uniques = new Dictionary<string, FacebookEvent>(); // dedupe by title + start foreach (var fb_event in FacebookIterator(method, args)) uniques.AddOrUpdateDictionary<string, FacebookEvent>(fb_event.name + fb_event.dt.ToString(), fb_event); foreach (FacebookEvent fb_event in uniques.Values) { var dtstart_with_zone = new DateTimeWithZone(fb_event.dt, this.calinfo.tzinfo); if (dtstart_with_zone.UniversalTime < Utils.MidnightInTz(this.calinfo.tzinfo).UniversalTime) continue; AddFacebookEvent(es, fb_event, dtstart_with_zone); } fbstats.whenchecked = DateTime.Now.ToUniversalTime(); SerializeStatsAndIntermediateOutputs(es, facebook_ical, fbstats, SourceType.facebook); } }
public static ZonelessEventStore ZonedToZoneless(string id, Calinfo calinfo, ZonedEventStore es_zoned) { var es_zoneless = new ZonelessEventStore(calinfo); foreach (var evt in es_zoned.events) es_zoneless.AddEvent(evt.title, evt.url, evt.source, evt.lat, evt.lon, evt.dtstart.LocalTime, evt.dtend.LocalTime, evt.allday, evt.categories, evt.description, evt.location); EventStore.UniqueFilterSortSerialize(id, es_zoneless); return es_zoneless; }