public static string RenderTaggableLink(TaggableSource taggable, List <Dictionary <string, string> > feeds, string type)
{
var is_curated = feeds.Exists(feed => feed["feedurl"] == taggable.ical_url);
var name = taggable.name.Replace("<title>", "").Replace("</title>", "");
if (type == "facebook")
{
name = name.Replace("'", " ");
name = Regex.Replace(name, " = [^|]+ ", "");
name = Regex.Replace(name, "| Facebook ", "");
}
if (!String.IsNullOrEmpty(taggable.city))
{
name = name + " (" + taggable.city + ")";
}
var extra = "";
if (!String.IsNullOrEmpty(taggable.extra_url))
{
extra = string.Format(@" [<a href=""{0}"">{0}</a>]", taggable.extra_url);
}
var html = string.Format("<div><p><a href=\"{0}\">{1}</a>{2}</p></div>\n",
taggable.home_url,
name,
extra
);
return(html);
}
public static List<TaggableSource> GetEventBriteOrganizers(Calinfo calinfo, Dictionary<string, string> settings)
{
var organizers = new List<TaggableSource>();
var name_and_pk = "eventbritesources";
var collector = new Collector(calinfo, settings);
string method = "event_search";
string args = collector.MakeEventBriteArgs();
int page_count = collector.GetEventBritePageCount(method, args);
var results = from evt in collector.EventBriteIterator(page_count, method, args)
select new
{
id = evt.Descendants("organizer").Descendants("id").FirstOrDefault().Value,
name = evt.Descendants("organizer").Descendants("name").FirstOrDefault().Value,
city = evt.Descendants("venue").Descendants("city").FirstOrDefault().Value.ToLower()
};
results = results.Distinct();
Parallel.ForEach(source: results, body: (id_name_city) =>
{
if (id_name_city.city != calinfo.City)
return;
var organizer_id = id_name_city.id;
var name = id_name_city.name;
var home_url = "http://www.eventbrite.com/org/" + organizer_id;
var escaped_name = Uri.EscapeDataString(name);
var ical_url = string.Format("http://elmcity.cloudapp.net/ics_from_eventbrite_organizer?organizer={0}&elmcity_id={1}", escaped_name, calinfo.id);
var taggable = new TaggableSource(name, calinfo.id, home_url, ical_url, id_name_city.city);
RememberTaggable(name_and_pk, organizer_id, taggable);
organizers.Add(taggable);
});
return organizers;
}
public static string RenderTaggableJson(TaggableSource taggable, List <Dictionary <string, string> > feeds, string type)
{
var json = string.Format("{{\"url\" : \"{0}\", \"feedurl\" : \"{1}\", \"source\" : \"{2}\", \"category\" : \"{3}\" }},\n",
taggable.home_url,
taggable.ical_url,
taggable.name.Replace("<title>", "").Replace("</title>", "") + " (" + type + ")",
type);
return(json);
}
public static List <TaggableSource> GetMeetupGroups(Calinfo calinfo, int delay, Dictionary <string, string> settings)
{
var meetup_key = settings["meetup_api_key"];
var template = "https://api.meetup.com/2/groups?key={0}&lat={1}&lon={2}&radius={3}&page=200";
var url = String.Format(template,
meetup_key,
calinfo.lat,
calinfo.lon,
calinfo.radius);
var json = HttpUtils.SlowFetchUrl(new Uri(url), delay).DataAsString();
var obj = JsonConvert.DeserializeObject <Dictionary <string, object> >(json);
var taggable_sources = new List <TaggableSource>();
string name_and_pk = "meetupsources";
foreach (var group in ((JArray)obj["results"]).ToList())
{
try
{
var name = (string)group["name"];
var urlname = (string)group["urlname"];
var id = group["id"].ToString();
var home_url = "http://www.meetup.com/" + urlname;
var ical_url = string.Format("http://www.meetup.com/{0}/events/ical/{1}/",
urlname,
Uri.EscapeDataString(name));
ical_url = ical_url.Replace("%20", "+"); // otherwise meetup weirdly reports a 505 error
var taggable = new TaggableSource(
name,
calinfo.id,
home_url,
ical_url,
calinfo.where);
taggable_sources.Add(taggable);
RememberTaggable(name_and_pk, id, taggable);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "FindMeetupGroups", e.Message + e.StackTrace);
}
}
return(taggable_sources);
}
public static List <TaggableSource> GetEventfulVenues(Calinfo calinfo, int min_per_venue, Dictionary <string, string> settings)
{
var collector = new Collector(calinfo, settings);
string args = collector.MakeEventfulArgs(calinfo.where, 100, "");
string method = "venues/search";
var xdoc = collector.CallEventfulApi(method, args);
var str_page_count = XmlUtils.GetXeltValue(xdoc.Root, ElmcityUtils.Configurator.no_ns, "page_count");
int page_count = Convert.ToInt16(str_page_count);
var ns = ElmcityUtils.Configurator.no_ns;
var results = from venue in collector.EventfulIterator(page_count, args, "venues/search", "venue")
select new
{
id = venue.Attribute("id").Value,
name = XmlUtils.GetXeltValue(venue, ns, "name"),
city_name = XmlUtils.GetXeltValue(venue, ns, "city_name").ToLower(),
count = Convert.ToInt32(XmlUtils.GetXeltValue(venue, ns, "event_count")),
home_url = XmlUtils.GetXeltValue(venue, ns, "url")
};
var venues = new List <TaggableSource>();
var name_and_pk = "eventfulsources";
Parallel.ForEach(source: results, body: (venue) =>
{
//if (venue.city_name != calinfo.City)
if (!calinfo.City.Contains(venue.city_name))
{
return;
}
if (venue.count < min_per_venue)
{
return;
}
var home_url = Regex.Replace(venue.home_url, "\\?.+", "");
var ical_url = home_url.Replace("eventful.com/", "eventful.com/ical/");
var taggable = new TaggableSource(venue.name, calinfo.id, home_url, ical_url, venue.city_name);
RememberTaggable(name_and_pk, venue.id, taggable);
venues.Add(taggable);
});
return(venues);
}
public static List <TaggableSource> GetEventBriteOrganizers(Calinfo calinfo, Dictionary <string, string> settings)
{
var organizers = new List <TaggableSource>();
var name_and_pk = "eventbritesources";
var collector = new Collector(calinfo, settings);
string method = "event_search";
string args = collector.MakeEventBriteArgs();
int page_count = collector.GetEventBritePageCount(method, args);
var results = from evt in collector.EventBriteIterator(page_count, method, args)
select new
{
id = evt.Descendants("organizer").Descendants("id").FirstOrDefault().Value,
name = evt.Descendants("organizer").Descendants("name").FirstOrDefault().Value,
city = evt.Descendants("venue").Descendants("city").FirstOrDefault().Value.ToLower()
};
results = results.Distinct();
Parallel.ForEach(source: results, body: (id_name_city) =>
{
if (id_name_city.city != calinfo.City)
{
return;
}
var organizer_id = id_name_city.id;
var name = id_name_city.name;
var home_url = "http://www.eventbrite.com/org/" + organizer_id;
var escaped_name = Uri.EscapeDataString(name);
var ical_url = string.Format("http://{0}/ics_from_eventbrite_organizer?organizer={1}&elmcity_id={2}", ElmcityUtils.Configurator.appdomain, escaped_name, calinfo.id);
var taggable = new TaggableSource(name, calinfo.id, home_url, ical_url, id_name_city.city);
RememberTaggable(name_and_pk, organizer_id, taggable);
organizers.Add(taggable);
});
return(organizers);
}
public static string RenderTaggableLink(TaggableSource taggable, List<Dictionary<string, string>> feeds, string type)
{
var is_curated = feeds.Exists(feed => feed["feedurl"] == taggable.ical_url);
var name = taggable.name.Replace("<title>", "").Replace("</title>", "");
if (type == "facebook")
{
name = name.Replace("'", " ");
name = Regex.Replace(name, " = [^|]+ ", "");
name = Regex.Replace(name, "| Facebook ", "");
}
if (!String.IsNullOrEmpty(taggable.city))
name = name + " (" + taggable.city + ")";
var extra = "";
if (!String.IsNullOrEmpty(taggable.extra_url))
extra = string.Format(@" [<a href=""{0}"">{0}</a>]", taggable.extra_url);
var html = string.Format("<div><p><a href=\"{0}\">{1}</a>{2}</p></div>\n",
taggable.home_url,
name,
extra
);
return html;
}
public static string RenderTaggableJson(TaggableSource taggable, List<Dictionary<string, string>> feeds, string type)
{
var json = string.Format("{{\"url\" : \"{0}\", \"feedurl\" : \"{1}\", \"source\" : \"{2}\", \"category\" : \"{3}\" }},\n",
taggable.home_url,
taggable.ical_url,
taggable.name.Replace("<title>", "").Replace("</title>", "") + " (" + type + ")",
type);
return json;
}
public static List<TaggableSource> GetUpcomingVenues(Calinfo calinfo, Dictionary<string, string> settings)
{
var collector = new Collector(calinfo, settings);
var args = collector.MakeUpcomingApiArgs(Collector.UpcomingSearchStyle.latlon);
var method = "event.search";
var xdoc = collector.CallUpcomingApi(method, args);
int page_count = 1;
var result_count = Collector.GetUpcomingResultCount(xdoc);
page_count = (result_count / 100 == 0) ? 1 : result_count / 100;
var events = collector.UpcomingIterator(page_count, "event.search");
var venues = new List<TaggableSource>();
var name_and_pk = "upcomingsources";
Parallel.ForEach(source: events, body: (xelt) =>
{
var city = xelt.Attribute("venue_city");
if (city.Value.ToLower() != calinfo.City)
return;
var id = xelt.Attribute("venue_id").Value;
var state = xelt.Attribute("venue_state_code");
var name = xelt.Attribute("venue_name");
// http://upcoming.yahoo.com/venue/863238/NH/Keene/The-Colonial-Theatre/
var home_url = string.Format("http://upcoming.yahoo.com/venue/{0}/{1}/{2}/{3}/",
id,
state.Value,
city.Value,
name.Value.Replace(" ", "-")
);
var ical_url = "http://upcoming.yahoo.com/calendar/v2/venue/" + id;
var taggable = new TaggableSource(name.Value, calinfo.id, home_url, ical_url, city.Value);
venues.Add(taggable);
RememberTaggable(name_and_pk, id, taggable);
});
return venues;
}
public static List<TaggableSource> GetMeetupGroups2(Calinfo calinfo, int delay, Dictionary<string, string> settings)
{
var meetup_key = settings["meetup_api_key"];
var template = "https://api.meetup.com/2/open_events?key={0}&lat={1}&lon={2}&radius={3}";
var url = String.Format(template,
meetup_key,
calinfo.lat,
calinfo.lon,
calinfo.radius);
var json = HttpUtils.SlowFetchUrl(new Uri(url), delay).DataAsString();
var dict = JsonConvert.DeserializeObject<Dictionary<String, object>>(json);
var results = (JArray)dict["results"];
var ids_and_cities = from result in results
select new
{
id = result["group"]["id"].Value<string>(),
city = GetMeetupCity(result)
};
var unique_ids_and_cities = ids_and_cities.Distinct();
var taggable_sources = new List<TaggableSource>();
Parallel.ForEach(source: unique_ids_and_cities, body: (id_and_city) =>
{
if (id_and_city.city != calinfo.City)
return;
string name_and_pk = "meetupsources";
try
{
template = "https://api.meetup.com/2/groups?key={0}&group_id={1}";
url = String.Format(template,
meetup_key,
id_and_city.id);
json = HttpUtils.SlowFetchUrl(new Uri(url), delay).DataAsString();
dict = JsonConvert.DeserializeObject<Dictionary<String, object>>(json);
results = (JArray)dict["results"];
var result = results.First();
var name = result["name"].Value<string>();
var urlname = result["urlname"].Value<string>();
var home_url = "http://www.meetup.com/" + urlname;
var ical_url = string.Format("http://www.meetup.com/{0}/events/ical/{1}/",
urlname,
Uri.EscapeDataString(name));
ical_url = ical_url.Replace("%20", "+"); // otherwise meetup weirdly reports a 505 error
var taggable = new TaggableSource(
name,
calinfo.id,
home_url,
ical_url,
id_and_city.city);
taggable_sources.Add(taggable);
RememberTaggable(name_and_pk, id_and_city.id, taggable);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "FindMeetupGroups", e.Message + e.StackTrace);
}
});
return taggable_sources;
}
public static List<TaggableSource> GetMeetupGroups(Calinfo calinfo, int delay, Dictionary<string, string> settings)
{
var meetup_key = settings["meetup_api_key"];
var template = "https://api.meetup.com/2/groups?key={0}&lat={1}&lon={2}&radius={3}&page=200";
var url = String.Format(template,
meetup_key,
calinfo.lat,
calinfo.lon,
calinfo.radius);
var json = HttpUtils.SlowFetchUrl(new Uri(url), delay).DataAsString();
var obj = JsonConvert.DeserializeObject<Dictionary<string,object>>(json);
var taggable_sources = new List<TaggableSource>();
string name_and_pk = "meetupsources";
foreach ( var group in ((JArray) obj["results"]).ToList() )
{
try
{
var name = (string) group["name"];
var urlname = (string)group["urlname"];
var id = group["id"].ToString();
var home_url = "http://www.meetup.com/" + urlname;
var ical_url = string.Format("http://www.meetup.com/{0}/events/ical/{1}/",
urlname,
Uri.EscapeDataString(name));
ical_url = ical_url.Replace("%20", "+"); // otherwise meetup weirdly reports a 505 error
var taggable = new TaggableSource(
name,
calinfo.id,
home_url,
ical_url,
calinfo.where);
taggable_sources.Add(taggable);
RememberTaggable(name_and_pk, id, taggable);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "FindMeetupGroups", e.Message + e.StackTrace);
}
}
return taggable_sources;
}
private static void RememberTaggable(string name_and_pk, string rowkey, TaggableSource taggable)
{
var entity = ObjectUtils.ObjToDictObj(taggable);
TableStorage.UpmergeDictToTableStore(entity, name_and_pk, name_and_pk, rowkey);
}
public static List <TaggableSource> GetMeetupGroups2(Calinfo calinfo, int delay, Dictionary <string, string> settings)
{
var meetup_key = settings["meetup_api_key"];
var template = "https://api.meetup.com/2/open_events?key={0}&lat={1}&lon={2}&radius={3}";
var url = String.Format(template,
meetup_key,
calinfo.lat,
calinfo.lon,
calinfo.radius);
var json = HttpUtils.SlowFetchUrl(new Uri(url), delay).DataAsString();
var dict = JsonConvert.DeserializeObject <Dictionary <String, object> >(json);
var results = (JArray)dict["results"];
var ids_and_cities = from result in results
select new
{
id = result["group"]["id"].Value <string>(),
city = GetMeetupCity(result)
};
var unique_ids_and_cities = ids_and_cities.Distinct();
var taggable_sources = new List <TaggableSource>();
Parallel.ForEach(source: unique_ids_and_cities, body: (id_and_city) =>
{
if (id_and_city.city != calinfo.City)
{
return;
}
string name_and_pk = "meetupsources";
try
{
template = "https://api.meetup.com/2/groups?key={0}&group_id={1}";
url = String.Format(template,
meetup_key,
id_and_city.id);
json = HttpUtils.SlowFetchUrl(new Uri(url), delay).DataAsString();
dict = JsonConvert.DeserializeObject <Dictionary <String, object> >(json);
results = (JArray)dict["results"];
var result = results.First();
var name = result["name"].Value <string>();
var urlname = result["urlname"].Value <string>();
var home_url = "http://www.meetup.com/" + urlname;
var ical_url = string.Format("http://www.meetup.com/{0}/events/ical/{1}/",
urlname,
Uri.EscapeDataString(name));
ical_url = ical_url.Replace("%20", "+"); // otherwise meetup weirdly reports a 505 error
var taggable = new TaggableSource(
name,
calinfo.id,
home_url,
ical_url,
id_and_city.city);
taggable_sources.Add(taggable);
RememberTaggable(name_and_pk, id_and_city.id, taggable);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "FindMeetupGroups", e.Message + e.StackTrace);
}
});
return(taggable_sources);
}
public static List <TaggableSource> GetFacebookPages(Calinfo calinfo, string location)
{
var search_template = String.Format("site:www.facebook.com/__TARGET__ \"{0}\"", location);
var search_for_fan_pages = search_template.Replace("__TARGET__", "pages");
var search_for_groups = search_template.Replace("__TARGET__", "groups");
var stats = new Dictionary <string, object>();
var fan_page_results = Search.BingSearch(search_for_fan_pages, 1000, stats);
// var group_results = Search.BingSearch(search_for_groups, 1000, stats); // doesn't work, location string won't usually appear
var group_results = new List <SearchResult>(); // placeholder for now
var bing_results = fan_page_results.Concat(group_results).ToList();
var taggable_sources = InitializeTaggables(calinfo, "facebook");
var seen_ids = new List <string>();
string name_and_pk = "facebooksources";
var settings = GenUtils.GetSettingsFromAzureTable();
var options = new ParallelOptions();
Parallel.ForEach(source: bing_results, parallelOptions: options, body: (result) =>
//foreach (var result in bing_results)
{
try
{
var url = Regex.Replace(result.url, @"\?.+", ""); // remove query string if any
var name = Regex.Match(result.url, "facebook.com/(pages|groups)/([^/]+)").Groups[2].Value;
name = name.Replace('-', ' ');
var fb_id = Utils.id_from_fb_fanpage_or_group(url);
if (seen_ids.Exists(x => x == fb_id))
{
return;
}
else
{
seen_ids.Add(fb_id);
}
string slat = null;
string slon = null;
var ical = new DDay.iCal.iCalendar();
var facebook_access_token = settings["facebook_access_token"]; // todo: merge with code in collector
var j_obj = Utils.GetFacebookEventsAsJsonObject(fb_id, facebook_access_token);
var events = Utils.iCalendarizeJsonObjectFromFacebook(j_obj, calinfo, ical, slat, slon, settings);
if (events.Count == 0) // no calendar on this page
{
return;
}
string page;
if (FacebookPageMatchesLocation(url, location, settings, out page) == false)
{
return;
}
string origin_url = "";
if (!String.IsNullOrEmpty(page))
{
origin_url = GetFacebookPageOrGroupOriginUrl(page);
}
var ical_url = string.Format("http://{0}/ics_from_fb_page?fb_id={1}&elmcity_id={2}",
ElmcityUtils.Configurator.appdomain,
fb_id,
calinfo.id);
var has_future_events = FacebookPageHasFutureEvents(events, calinfo);
var taggable = new TaggableSource(name, calinfo.id, url + "?sk=events", ical_url, has_future_events, origin_url);
taggable_sources.Add(taggable);
RememberTaggable(name_and_pk, fb_id, taggable);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "GetFacebookPages", e.Message + e.StackTrace);
return;
}
});
return(taggable_sources);
}
private static void RememberTaggable(string name_and_pk, string rowkey, TaggableSource taggable)
{
var entity = ObjectUtils.ObjToDictObj(taggable);
TableStorage.UpmergeDictToTableStore(entity, name_and_pk, name_and_pk, rowkey);
}
public static List<TaggableSource> GetEventfulVenues(Calinfo calinfo, int min_per_venue, Dictionary<string, string> settings)
{
var collector = new Collector(calinfo, settings);
string args = collector.MakeEventfulArgs(calinfo.where, 100, "");
string method = "venues/search";
var xdoc = collector.CallEventfulApi(method, args);
var str_page_count = XmlUtils.GetXeltValue(xdoc.Root, ElmcityUtils.Configurator.no_ns, "page_count");
int page_count = Convert.ToInt16(str_page_count);
var ns = ElmcityUtils.Configurator.no_ns;
var results = from venue in collector.EventfulIterator(page_count, args, "venues/search", "venue")
select new
{
id = venue.Attribute("id").Value,
name = XmlUtils.GetXeltValue(venue, ns, "name"),
city_name = XmlUtils.GetXeltValue(venue, ns, "city_name").ToLower(),
count = Convert.ToInt32(XmlUtils.GetXeltValue(venue, ns, "event_count")),
home_url = XmlUtils.GetXeltValue(venue, ns, "url")
};
var venues = new List<TaggableSource>();
var name_and_pk = "eventfulsources";
Parallel.ForEach(source: results, body: (venue) =>
{
if (venue.city_name != calinfo.City)
return;
if (venue.count < min_per_venue)
return;
var home_url = Regex.Replace(venue.home_url, "\\?.+", "");
var ical_url = home_url.Replace("eventful.com/", "eventful.com/ical/");
var taggable = new TaggableSource(venue.name, calinfo.id, home_url, ical_url, venue.city_name);
RememberTaggable(name_and_pk, venue.id, taggable);
venues.Add(taggable);
});
return venues;
}
public static List<TaggableSource> GetFacebookPages(Calinfo calinfo, string location)
{
var search_template = String.Format( "site:www.facebook.com/__TARGET__ \"{0}\"", location);
var search_for_fan_pages = search_template.Replace("__TARGET__", "pages");
var search_for_groups = search_template.Replace("__TARGET__", "groups");
var stats = new Dictionary<string, object>();
var fan_page_results = Search.BingSearch(search_for_fan_pages, 1000, stats);
// var group_results = Search.BingSearch(search_for_groups, 1000, stats); // doesn't work, location string won't usually appear
var group_results = new List<SearchResult>(); // placeholder for now
var bing_results = fan_page_results.Concat(group_results).ToList();
var taggable_sources = InitializeTaggables(calinfo, "facebook");
var seen_ids = new List<string>();
string name_and_pk = "facebooksources";
var settings = GenUtils.GetSettingsFromAzureTable();
var options = new ParallelOptions();
Parallel.ForEach(source: bing_results, parallelOptions: options, body: (result) =>
//foreach (var result in bing_results)
{
try
{
var url = Regex.Replace(result.url, @"\?.+", ""); // remove query string if any
var name = Regex.Match(result.url, "facebook.com/(pages|groups)/([^/]+)").Groups[2].Value;
name = name.Replace('-', ' ');
var fb_id = Utils.id_from_fb_fanpage_or_group(url);
if (seen_ids.Exists(x => x == fb_id))
return;
else
seen_ids.Add(fb_id);
var facebook_access_token = settings["facebook_access_token"]; // todo: merge with code in collector
// https://graph.facebook.com/https://graph.facebook.com/142525312427391/events?access_token=...
var graph_uri_template = "https://graph.facebook.com/{0}/events?access_token={1}";
var graph_uri = new Uri(string.Format(graph_uri_template, fb_id, facebook_access_token));
var json = HttpUtils.FetchUrl(graph_uri).DataAsString();
var j_obj = (JObject)JsonConvert.DeserializeObject(json);
var events = Utils.UnpackFacebookEventsFromJson(j_obj);
if (events.Count == 0) // no calendar on this page
return;
string page;
if (FacebookPageMatchesLocation(url, location, settings, out page) == false)
return;
string origin_url = "";
if ( ! String.IsNullOrEmpty(page) )
origin_url = GetFacebookPageOrGroupOriginUrl(page);
var ical_url = string.Format("http://elmcity.cloudapp.net/ics_from_fb_page?fb_id={0}&elmcity_id={1}",
fb_id,
calinfo.id);
var has_future_events = FacebookPageHasFutureEvents(events, calinfo);
var taggable = new TaggableSource(name, calinfo.id, url + "?sk=events", ical_url, has_future_events, origin_url);
taggable_sources.Add(taggable);
RememberTaggable(name_and_pk, fb_id, taggable);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "GetFacebookPages", e.Message + e.StackTrace);
return;
}
});
return taggable_sources;
}
