public static string RenderTaggableLink(TaggableSource taggable, List <Dictionary <string, string> > feeds, string type) { var is_curated = feeds.Exists(feed => feed["feedurl"] == taggable.ical_url); var name = taggable.name.Replace("<title>", "").Replace("</title>", ""); if (type == "facebook") { name = name.Replace("'", " "); name = Regex.Replace(name, " = [^|]+ ", ""); name = Regex.Replace(name, "| Facebook ", ""); } if (!String.IsNullOrEmpty(taggable.city)) { name = name + " (" + taggable.city + ")"; } var extra = ""; if (!String.IsNullOrEmpty(taggable.extra_url)) { extra = string.Format(@" [<a href=""{0}"">{0}</a>]", taggable.extra_url); } var html = string.Format("<div><p><a href=\"{0}\">{1}</a>{2}</p></div>\n", taggable.home_url, name, extra ); return(html); }
public static List<TaggableSource> GetEventBriteOrganizers(Calinfo calinfo, Dictionary<string, string> settings) { var organizers = new List<TaggableSource>(); var name_and_pk = "eventbritesources"; var collector = new Collector(calinfo, settings); string method = "event_search"; string args = collector.MakeEventBriteArgs(); int page_count = collector.GetEventBritePageCount(method, args); var results = from evt in collector.EventBriteIterator(page_count, method, args) select new { id = evt.Descendants("organizer").Descendants("id").FirstOrDefault().Value, name = evt.Descendants("organizer").Descendants("name").FirstOrDefault().Value, city = evt.Descendants("venue").Descendants("city").FirstOrDefault().Value.ToLower() }; results = results.Distinct(); Parallel.ForEach(source: results, body: (id_name_city) => { if (id_name_city.city != calinfo.City) return; var organizer_id = id_name_city.id; var name = id_name_city.name; var home_url = "http://www.eventbrite.com/org/" + organizer_id; var escaped_name = Uri.EscapeDataString(name); var ical_url = string.Format("http://elmcity.cloudapp.net/ics_from_eventbrite_organizer?organizer={0}&elmcity_id={1}", escaped_name, calinfo.id); var taggable = new TaggableSource(name, calinfo.id, home_url, ical_url, id_name_city.city); RememberTaggable(name_and_pk, organizer_id, taggable); organizers.Add(taggable); }); return organizers; }
public static string RenderTaggableJson(TaggableSource taggable, List <Dictionary <string, string> > feeds, string type) { var json = string.Format("{{\"url\" : \"{0}\", \"feedurl\" : \"{1}\", \"source\" : \"{2}\", \"category\" : \"{3}\" }},\n", taggable.home_url, taggable.ical_url, taggable.name.Replace("<title>", "").Replace("</title>", "") + " (" + type + ")", type); return(json); }
public static List <TaggableSource> GetMeetupGroups(Calinfo calinfo, int delay, Dictionary <string, string> settings) { var meetup_key = settings["meetup_api_key"]; var template = "https://api.meetup.com/2/groups?key={0}&lat={1}&lon={2}&radius={3}&page=200"; var url = String.Format(template, meetup_key, calinfo.lat, calinfo.lon, calinfo.radius); var json = HttpUtils.SlowFetchUrl(new Uri(url), delay).DataAsString(); var obj = JsonConvert.DeserializeObject <Dictionary <string, object> >(json); var taggable_sources = new List <TaggableSource>(); string name_and_pk = "meetupsources"; foreach (var group in ((JArray)obj["results"]).ToList()) { try { var name = (string)group["name"]; var urlname = (string)group["urlname"]; var id = group["id"].ToString(); var home_url = "http://www.meetup.com/" + urlname; var ical_url = string.Format("http://www.meetup.com/{0}/events/ical/{1}/", urlname, Uri.EscapeDataString(name)); ical_url = ical_url.Replace("%20", "+"); // otherwise meetup weirdly reports a 505 error var taggable = new TaggableSource( name, calinfo.id, home_url, ical_url, calinfo.where); taggable_sources.Add(taggable); RememberTaggable(name_and_pk, id, taggable); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "FindMeetupGroups", e.Message + e.StackTrace); } } return(taggable_sources); }
public static List <TaggableSource> GetEventfulVenues(Calinfo calinfo, int min_per_venue, Dictionary <string, string> settings) { var collector = new Collector(calinfo, settings); string args = collector.MakeEventfulArgs(calinfo.where, 100, ""); string method = "venues/search"; var xdoc = collector.CallEventfulApi(method, args); var str_page_count = XmlUtils.GetXeltValue(xdoc.Root, ElmcityUtils.Configurator.no_ns, "page_count"); int page_count = Convert.ToInt16(str_page_count); var ns = ElmcityUtils.Configurator.no_ns; var results = from venue in collector.EventfulIterator(page_count, args, "venues/search", "venue") select new { id = venue.Attribute("id").Value, name = XmlUtils.GetXeltValue(venue, ns, "name"), city_name = XmlUtils.GetXeltValue(venue, ns, "city_name").ToLower(), count = Convert.ToInt32(XmlUtils.GetXeltValue(venue, ns, "event_count")), home_url = XmlUtils.GetXeltValue(venue, ns, "url") }; var venues = new List <TaggableSource>(); var name_and_pk = "eventfulsources"; Parallel.ForEach(source: results, body: (venue) => { //if (venue.city_name != calinfo.City) if (!calinfo.City.Contains(venue.city_name)) { return; } if (venue.count < min_per_venue) { return; } var home_url = Regex.Replace(venue.home_url, "\\?.+", ""); var ical_url = home_url.Replace("eventful.com/", "eventful.com/ical/"); var taggable = new TaggableSource(venue.name, calinfo.id, home_url, ical_url, venue.city_name); RememberTaggable(name_and_pk, venue.id, taggable); venues.Add(taggable); }); return(venues); }
public static List <TaggableSource> GetEventBriteOrganizers(Calinfo calinfo, Dictionary <string, string> settings) { var organizers = new List <TaggableSource>(); var name_and_pk = "eventbritesources"; var collector = new Collector(calinfo, settings); string method = "event_search"; string args = collector.MakeEventBriteArgs(); int page_count = collector.GetEventBritePageCount(method, args); var results = from evt in collector.EventBriteIterator(page_count, method, args) select new { id = evt.Descendants("organizer").Descendants("id").FirstOrDefault().Value, name = evt.Descendants("organizer").Descendants("name").FirstOrDefault().Value, city = evt.Descendants("venue").Descendants("city").FirstOrDefault().Value.ToLower() }; results = results.Distinct(); Parallel.ForEach(source: results, body: (id_name_city) => { if (id_name_city.city != calinfo.City) { return; } var organizer_id = id_name_city.id; var name = id_name_city.name; var home_url = "http://www.eventbrite.com/org/" + organizer_id; var escaped_name = Uri.EscapeDataString(name); var ical_url = string.Format("http://{0}/ics_from_eventbrite_organizer?organizer={1}&elmcity_id={2}", ElmcityUtils.Configurator.appdomain, escaped_name, calinfo.id); var taggable = new TaggableSource(name, calinfo.id, home_url, ical_url, id_name_city.city); RememberTaggable(name_and_pk, organizer_id, taggable); organizers.Add(taggable); }); return(organizers); }
public static string RenderTaggableLink(TaggableSource taggable, List<Dictionary<string, string>> feeds, string type) { var is_curated = feeds.Exists(feed => feed["feedurl"] == taggable.ical_url); var name = taggable.name.Replace("<title>", "").Replace("</title>", ""); if (type == "facebook") { name = name.Replace("'", " "); name = Regex.Replace(name, " = [^|]+ ", ""); name = Regex.Replace(name, "| Facebook ", ""); } if (!String.IsNullOrEmpty(taggable.city)) name = name + " (" + taggable.city + ")"; var extra = ""; if (!String.IsNullOrEmpty(taggable.extra_url)) extra = string.Format(@" [<a href=""{0}"">{0}</a>]", taggable.extra_url); var html = string.Format("<div><p><a href=\"{0}\">{1}</a>{2}</p></div>\n", taggable.home_url, name, extra ); return html; }
public static string RenderTaggableJson(TaggableSource taggable, List<Dictionary<string, string>> feeds, string type) { var json = string.Format("{{\"url\" : \"{0}\", \"feedurl\" : \"{1}\", \"source\" : \"{2}\", \"category\" : \"{3}\" }},\n", taggable.home_url, taggable.ical_url, taggable.name.Replace("<title>", "").Replace("</title>", "") + " (" + type + ")", type); return json; }
public static List<TaggableSource> GetUpcomingVenues(Calinfo calinfo, Dictionary<string, string> settings) { var collector = new Collector(calinfo, settings); var args = collector.MakeUpcomingApiArgs(Collector.UpcomingSearchStyle.latlon); var method = "event.search"; var xdoc = collector.CallUpcomingApi(method, args); int page_count = 1; var result_count = Collector.GetUpcomingResultCount(xdoc); page_count = (result_count / 100 == 0) ? 1 : result_count / 100; var events = collector.UpcomingIterator(page_count, "event.search"); var venues = new List<TaggableSource>(); var name_and_pk = "upcomingsources"; Parallel.ForEach(source: events, body: (xelt) => { var city = xelt.Attribute("venue_city"); if (city.Value.ToLower() != calinfo.City) return; var id = xelt.Attribute("venue_id").Value; var state = xelt.Attribute("venue_state_code"); var name = xelt.Attribute("venue_name"); // http://upcoming.yahoo.com/venue/863238/NH/Keene/The-Colonial-Theatre/ var home_url = string.Format("http://upcoming.yahoo.com/venue/{0}/{1}/{2}/{3}/", id, state.Value, city.Value, name.Value.Replace(" ", "-") ); var ical_url = "http://upcoming.yahoo.com/calendar/v2/venue/" + id; var taggable = new TaggableSource(name.Value, calinfo.id, home_url, ical_url, city.Value); venues.Add(taggable); RememberTaggable(name_and_pk, id, taggable); }); return venues; }
public static List<TaggableSource> GetMeetupGroups2(Calinfo calinfo, int delay, Dictionary<string, string> settings) { var meetup_key = settings["meetup_api_key"]; var template = "https://api.meetup.com/2/open_events?key={0}&lat={1}&lon={2}&radius={3}"; var url = String.Format(template, meetup_key, calinfo.lat, calinfo.lon, calinfo.radius); var json = HttpUtils.SlowFetchUrl(new Uri(url), delay).DataAsString(); var dict = JsonConvert.DeserializeObject<Dictionary<String, object>>(json); var results = (JArray)dict["results"]; var ids_and_cities = from result in results select new { id = result["group"]["id"].Value<string>(), city = GetMeetupCity(result) }; var unique_ids_and_cities = ids_and_cities.Distinct(); var taggable_sources = new List<TaggableSource>(); Parallel.ForEach(source: unique_ids_and_cities, body: (id_and_city) => { if (id_and_city.city != calinfo.City) return; string name_and_pk = "meetupsources"; try { template = "https://api.meetup.com/2/groups?key={0}&group_id={1}"; url = String.Format(template, meetup_key, id_and_city.id); json = HttpUtils.SlowFetchUrl(new Uri(url), delay).DataAsString(); dict = JsonConvert.DeserializeObject<Dictionary<String, object>>(json); results = (JArray)dict["results"]; var result = results.First(); var name = result["name"].Value<string>(); var urlname = result["urlname"].Value<string>(); var home_url = "http://www.meetup.com/" + urlname; var ical_url = string.Format("http://www.meetup.com/{0}/events/ical/{1}/", urlname, Uri.EscapeDataString(name)); ical_url = ical_url.Replace("%20", "+"); // otherwise meetup weirdly reports a 505 error var taggable = new TaggableSource( name, calinfo.id, home_url, ical_url, id_and_city.city); taggable_sources.Add(taggable); RememberTaggable(name_and_pk, id_and_city.id, taggable); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "FindMeetupGroups", e.Message + e.StackTrace); } }); return taggable_sources; }
public static List<TaggableSource> GetMeetupGroups(Calinfo calinfo, int delay, Dictionary<string, string> settings) { var meetup_key = settings["meetup_api_key"]; var template = "https://api.meetup.com/2/groups?key={0}&lat={1}&lon={2}&radius={3}&page=200"; var url = String.Format(template, meetup_key, calinfo.lat, calinfo.lon, calinfo.radius); var json = HttpUtils.SlowFetchUrl(new Uri(url), delay).DataAsString(); var obj = JsonConvert.DeserializeObject<Dictionary<string,object>>(json); var taggable_sources = new List<TaggableSource>(); string name_and_pk = "meetupsources"; foreach ( var group in ((JArray) obj["results"]).ToList() ) { try { var name = (string) group["name"]; var urlname = (string)group["urlname"]; var id = group["id"].ToString(); var home_url = "http://www.meetup.com/" + urlname; var ical_url = string.Format("http://www.meetup.com/{0}/events/ical/{1}/", urlname, Uri.EscapeDataString(name)); ical_url = ical_url.Replace("%20", "+"); // otherwise meetup weirdly reports a 505 error var taggable = new TaggableSource( name, calinfo.id, home_url, ical_url, calinfo.where); taggable_sources.Add(taggable); RememberTaggable(name_and_pk, id, taggable); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "FindMeetupGroups", e.Message + e.StackTrace); } } return taggable_sources; }
private static void RememberTaggable(string name_and_pk, string rowkey, TaggableSource taggable) { var entity = ObjectUtils.ObjToDictObj(taggable); TableStorage.UpmergeDictToTableStore(entity, name_and_pk, name_and_pk, rowkey); }
public static List <TaggableSource> GetMeetupGroups2(Calinfo calinfo, int delay, Dictionary <string, string> settings) { var meetup_key = settings["meetup_api_key"]; var template = "https://api.meetup.com/2/open_events?key={0}&lat={1}&lon={2}&radius={3}"; var url = String.Format(template, meetup_key, calinfo.lat, calinfo.lon, calinfo.radius); var json = HttpUtils.SlowFetchUrl(new Uri(url), delay).DataAsString(); var dict = JsonConvert.DeserializeObject <Dictionary <String, object> >(json); var results = (JArray)dict["results"]; var ids_and_cities = from result in results select new { id = result["group"]["id"].Value <string>(), city = GetMeetupCity(result) }; var unique_ids_and_cities = ids_and_cities.Distinct(); var taggable_sources = new List <TaggableSource>(); Parallel.ForEach(source: unique_ids_and_cities, body: (id_and_city) => { if (id_and_city.city != calinfo.City) { return; } string name_and_pk = "meetupsources"; try { template = "https://api.meetup.com/2/groups?key={0}&group_id={1}"; url = String.Format(template, meetup_key, id_and_city.id); json = HttpUtils.SlowFetchUrl(new Uri(url), delay).DataAsString(); dict = JsonConvert.DeserializeObject <Dictionary <String, object> >(json); results = (JArray)dict["results"]; var result = results.First(); var name = result["name"].Value <string>(); var urlname = result["urlname"].Value <string>(); var home_url = "http://www.meetup.com/" + urlname; var ical_url = string.Format("http://www.meetup.com/{0}/events/ical/{1}/", urlname, Uri.EscapeDataString(name)); ical_url = ical_url.Replace("%20", "+"); // otherwise meetup weirdly reports a 505 error var taggable = new TaggableSource( name, calinfo.id, home_url, ical_url, id_and_city.city); taggable_sources.Add(taggable); RememberTaggable(name_and_pk, id_and_city.id, taggable); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "FindMeetupGroups", e.Message + e.StackTrace); } }); return(taggable_sources); }
public static List <TaggableSource> GetFacebookPages(Calinfo calinfo, string location) { var search_template = String.Format("site:www.facebook.com/__TARGET__ \"{0}\"", location); var search_for_fan_pages = search_template.Replace("__TARGET__", "pages"); var search_for_groups = search_template.Replace("__TARGET__", "groups"); var stats = new Dictionary <string, object>(); var fan_page_results = Search.BingSearch(search_for_fan_pages, 1000, stats); // var group_results = Search.BingSearch(search_for_groups, 1000, stats); // doesn't work, location string won't usually appear var group_results = new List <SearchResult>(); // placeholder for now var bing_results = fan_page_results.Concat(group_results).ToList(); var taggable_sources = InitializeTaggables(calinfo, "facebook"); var seen_ids = new List <string>(); string name_and_pk = "facebooksources"; var settings = GenUtils.GetSettingsFromAzureTable(); var options = new ParallelOptions(); Parallel.ForEach(source: bing_results, parallelOptions: options, body: (result) => //foreach (var result in bing_results) { try { var url = Regex.Replace(result.url, @"\?.+", ""); // remove query string if any var name = Regex.Match(result.url, "facebook.com/(pages|groups)/([^/]+)").Groups[2].Value; name = name.Replace('-', ' '); var fb_id = Utils.id_from_fb_fanpage_or_group(url); if (seen_ids.Exists(x => x == fb_id)) { return; } else { seen_ids.Add(fb_id); } string slat = null; string slon = null; var ical = new DDay.iCal.iCalendar(); var facebook_access_token = settings["facebook_access_token"]; // todo: merge with code in collector var j_obj = Utils.GetFacebookEventsAsJsonObject(fb_id, facebook_access_token); var events = Utils.iCalendarizeJsonObjectFromFacebook(j_obj, calinfo, ical, slat, slon, settings); if (events.Count == 0) // no calendar on this page { return; } string page; if (FacebookPageMatchesLocation(url, location, settings, out page) == false) { return; } string origin_url = ""; if (!String.IsNullOrEmpty(page)) { origin_url = GetFacebookPageOrGroupOriginUrl(page); } var ical_url = string.Format("http://{0}/ics_from_fb_page?fb_id={1}&elmcity_id={2}", ElmcityUtils.Configurator.appdomain, fb_id, calinfo.id); var has_future_events = FacebookPageHasFutureEvents(events, calinfo); var taggable = new TaggableSource(name, calinfo.id, url + "?sk=events", ical_url, has_future_events, origin_url); taggable_sources.Add(taggable); RememberTaggable(name_and_pk, fb_id, taggable); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "GetFacebookPages", e.Message + e.StackTrace); return; } }); return(taggable_sources); }
public static List<TaggableSource> GetEventfulVenues(Calinfo calinfo, int min_per_venue, Dictionary<string, string> settings) { var collector = new Collector(calinfo, settings); string args = collector.MakeEventfulArgs(calinfo.where, 100, ""); string method = "venues/search"; var xdoc = collector.CallEventfulApi(method, args); var str_page_count = XmlUtils.GetXeltValue(xdoc.Root, ElmcityUtils.Configurator.no_ns, "page_count"); int page_count = Convert.ToInt16(str_page_count); var ns = ElmcityUtils.Configurator.no_ns; var results = from venue in collector.EventfulIterator(page_count, args, "venues/search", "venue") select new { id = venue.Attribute("id").Value, name = XmlUtils.GetXeltValue(venue, ns, "name"), city_name = XmlUtils.GetXeltValue(venue, ns, "city_name").ToLower(), count = Convert.ToInt32(XmlUtils.GetXeltValue(venue, ns, "event_count")), home_url = XmlUtils.GetXeltValue(venue, ns, "url") }; var venues = new List<TaggableSource>(); var name_and_pk = "eventfulsources"; Parallel.ForEach(source: results, body: (venue) => { if (venue.city_name != calinfo.City) return; if (venue.count < min_per_venue) return; var home_url = Regex.Replace(venue.home_url, "\\?.+", ""); var ical_url = home_url.Replace("eventful.com/", "eventful.com/ical/"); var taggable = new TaggableSource(venue.name, calinfo.id, home_url, ical_url, venue.city_name); RememberTaggable(name_and_pk, venue.id, taggable); venues.Add(taggable); }); return venues; }
public static List<TaggableSource> GetFacebookPages(Calinfo calinfo, string location) { var search_template = String.Format( "site:www.facebook.com/__TARGET__ \"{0}\"", location); var search_for_fan_pages = search_template.Replace("__TARGET__", "pages"); var search_for_groups = search_template.Replace("__TARGET__", "groups"); var stats = new Dictionary<string, object>(); var fan_page_results = Search.BingSearch(search_for_fan_pages, 1000, stats); // var group_results = Search.BingSearch(search_for_groups, 1000, stats); // doesn't work, location string won't usually appear var group_results = new List<SearchResult>(); // placeholder for now var bing_results = fan_page_results.Concat(group_results).ToList(); var taggable_sources = InitializeTaggables(calinfo, "facebook"); var seen_ids = new List<string>(); string name_and_pk = "facebooksources"; var settings = GenUtils.GetSettingsFromAzureTable(); var options = new ParallelOptions(); Parallel.ForEach(source: bing_results, parallelOptions: options, body: (result) => //foreach (var result in bing_results) { try { var url = Regex.Replace(result.url, @"\?.+", ""); // remove query string if any var name = Regex.Match(result.url, "facebook.com/(pages|groups)/([^/]+)").Groups[2].Value; name = name.Replace('-', ' '); var fb_id = Utils.id_from_fb_fanpage_or_group(url); if (seen_ids.Exists(x => x == fb_id)) return; else seen_ids.Add(fb_id); var facebook_access_token = settings["facebook_access_token"]; // todo: merge with code in collector // https://graph.facebook.com/https://graph.facebook.com/142525312427391/events?access_token=... var graph_uri_template = "https://graph.facebook.com/{0}/events?access_token={1}"; var graph_uri = new Uri(string.Format(graph_uri_template, fb_id, facebook_access_token)); var json = HttpUtils.FetchUrl(graph_uri).DataAsString(); var j_obj = (JObject)JsonConvert.DeserializeObject(json); var events = Utils.UnpackFacebookEventsFromJson(j_obj); if (events.Count == 0) // no calendar on this page return; string page; if (FacebookPageMatchesLocation(url, location, settings, out page) == false) return; string origin_url = ""; if ( ! String.IsNullOrEmpty(page) ) origin_url = GetFacebookPageOrGroupOriginUrl(page); var ical_url = string.Format("http://elmcity.cloudapp.net/ics_from_fb_page?fb_id={0}&elmcity_id={1}", fb_id, calinfo.id); var has_future_events = FacebookPageHasFutureEvents(events, calinfo); var taggable = new TaggableSource(name, calinfo.id, url + "?sk=events", ical_url, has_future_events, origin_url); taggable_sources.Add(taggable); RememberTaggable(name_and_pk, fb_id, taggable); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "GetFacebookPages", e.Message + e.StackTrace); return; } }); return taggable_sources; }