private static string CallSearchApi(Uri search_url)
{
var delay = 2;
try
{
delay = Convert.ToInt32(settings["search_engine_api_delay_secs"]);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "CallSearchApi", e.Message + e.StackTrace);
}
Utils.Wait(delay);
var r = HttpUtils.FetchUrl(search_url);
if (r.status != HttpStatusCode.OK)
{
GenUtils.LogMsg("warning", "CallSearchApi" + r.status.ToString(), search_url.ToString());
return(null);
}
else
{
return(r.DataAsString());
}
}
// populate feed registry from azure table
public void LoadFeedsFromAzure(FeedLoadOption option)
{
var dict = Metadata.LoadFeedsFromAzureTableForId(this.id, option);
var settings = GenUtils.GetSettingsFromAzureTable("settings");
foreach (var url in dict.Keys)
{
try
{
if (settings["eventful_feeds_enabled"] == "False" && url.StartsWith("http://eventful.com/"))
{
continue;
}
if (settings["eventbrite_feeds_enabled"] == "False" && url.Contains("ics_from_eventbrite"))
{
continue;
}
if (settings["lastfm_feeds_enabled"] == "False" && url.Contains("ics_from_lastfm_venue"))
{
continue;
}
if (url.StartsWith("http://upcoming.yahoo.com"))
{
continue;
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "CollectIcal", e.Message + e.StackTrace);
}
this.AddFeed(url, dict[url]);
}
}
private static void PerformSearches(string where, Dictionary <string, SearchResult> final_results_dict, Dictionary <string, object> stats_dict)
{
foreach (var qualifier in qualifiers)
{
foreach (var day in days)
{
string q;
try
{
q = string.Format(@" ""{0}"" ""{1} {2}"" ", where, qualifier, day);
var results = GoogleSearch(q, stats_dict);
DictifyResults(results, final_results_dict, stats_dict);
}
catch (Exception ex1)
{
GenUtils.PriorityLogMsg("exception", "search_location: google: " + where, ex1.Message);
}
try
{
q = string.Format(@" ""{0}"" near:100 ""{1} {2}"" ", where, qualifier, day);
var results = BingSearch(q, 500, stats_dict);
DictifyResults(results, final_results_dict, stats_dict);
}
catch (Exception ex2)
{
GenUtils.PriorityLogMsg("exception", "search_location: bing: " + where, ex2.Message);
}
}
}
}
//[OutputCache(Duration = ... // output cache not used here, iis cache is managed directly
public ActionResult GetEvents(string id, string type, string view, string jsonp, string count, string from, string to, string eventsonly, string mobile, string test, string raw, string raw_sentinel, string style, string theme, string taglist, string tags, string template, string jsurl, string days, string bare_events, string hub, string source, string first)
{
if (id == "a2cal")
{
id = "AnnArborChronicle";
}
ElmcityApp.logger.LogHttpRequest(this.ControllerContext);
if (view != null)
{
view = view.ToLower();
}
EventsResult r = null;
try
{
var cr = ElmcityApp.wrd.renderers[id];
r = new EventsResult(this, cr, id, type, view, jsonp, count, from, to, eventsonly, mobile, test, raw, raw_sentinel, style, theme, taglist, tags, template, jsurl, days, bare_events, hub, source, first);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "GetEvents: " + id, e.Message);
}
return(r);
}
public static TimeSpan IntervalFromType(TaskType type)
{
var ts = TimeSpan.FromHours(24);
switch (type)
{
case TaskType.nonicaltasks:
ts = Scheduler.nonical_interval;
break;
case TaskType.icaltasks:
ts = Scheduler.ical_interval;
break;
case TaskType.regiontasks:
ts = Scheduler.region_interval;
break;
case TaskType.none:
ts = TimeSpan.FromHours(24);
GenUtils.PriorityLogMsg("warning", "Scheduler.IntervalFromType", "unexpected TaskType.none");
break;
}
return(ts);
}
public static bool RegionIsStale(string region)
{
var stale = false;
try
{
var bs = BlobStorage.MakeDefaultBlobStorage();
var region_props = bs.GetBlobProperties(region.ToLower(), region + ".zoneless.obj");
var region_modified = DateTime.Parse(region_props.HttpResponse.headers["Last-Modified"]);
foreach (var hub in Utils.GetIdsForRegion(region))
{
var hub_props = bs.GetBlobProperties(hub.ToLower(), hub + ".zoneless.obj");
var hub_modified = DateTime.Parse(hub_props.HttpResponse.headers["Last-Modified"]);
if (hub_modified > region_modified)
{
stale = true;
break;
}
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "RegionIsStale: " + region, e.Message + e.StackTrace);
}
return stale;
}
public static void SaveWrd(WebRoleData wrd)
{
if (!wrd.IsConsistent())
{
GenUtils.PriorityLogMsg("warning", "inconsistent WebRoleData!", null);
return;
}
var bs = BlobStorage.MakeDefaultBlobStorage();
var lease_response = bs.RetryAcquireLease("admin", "wrd.obj");
if (lease_response.status == HttpStatusCode.Created)
{
var lease_id = lease_response.headers["x-ms-lease-id"];
var bytes = ObjectUtils.SerializeObject(wrd);
var headers = new Hashtable()
{
{ "x-ms-lease-id", lease_id }
};
var r = bs.PutBlob("admin", "wrd.obj", headers, bytes, "binary/octet-stream");
if (r.HttpResponse.status != HttpStatusCode.Created)
{
GenUtils.PriorityLogMsg("warning", "SaveWrd: cannot save", null);
}
SaveTimestampedWrd(wrd);
}
else
{
GenUtils.PriorityLogMsg("warning", "SaveWrd: cannot lease", null);
}
}
public void GeneralAdmin(object o, ElapsedEventArgs args)
{
try
{
GenUtils.PriorityLogMsg("status", "GeneralAdmin", null);
Utils.MakeWhereSummary(); // refresh http://elmcity.blob.core.windows.net/admin/where_summary.html
Utils.MakeWhatSummary(); // refresh http://elmcity.blob.core.windows.net/admin/what_summary.html
var ids = Metadata.LoadWhatWhereIdsFromAzureTable();
Parallel.ForEach(source: ids, body: (id) =>
{
var calinfo = Utils.AcquireCalinfo(id);
Utils.SaveIcalPerFeedLocations(calinfo, settings);
});
}
catch (Exception e)
{
GenUtils.LogMsg("exception", "GeneralAdmin", e.Message + e.StackTrace);
}
//Utils.MakeFeaturedHubs(); // idle for now
}
protected void Application_End(object sender, EventArgs e)
{
HttpRuntime runtime = (HttpRuntime)typeof(System.Web.HttpRuntime).InvokeMember("_theRuntime",
BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.GetField,
null,
null,
null);
if (runtime == null)
{
return;
}
string shutdown_message = (string)runtime.GetType().InvokeMember("_shutDownMessage",
BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.GetField,
null,
runtime,
null);
string shutdown_stack = (string)runtime.GetType().InvokeMember("_shutDownStack",
BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.GetField,
null,
runtime,
null);
var ts = TableStorage.MakeDefaultTableStorage();
GenUtils.PriorityLogMsg("Application_End", shutdown_message, shutdown_stack);
}
public static WebRoleData MakeWebRoleData()
{
WebRoleData wrd = null;
try // create WebRoleData structure and store as blob, available to webrole on next _reload
{
var sw = new Stopwatch();
sw.Start();
wrd = new WebRoleData();
sw.Stop();
GenUtils.LogMsg("info", "new wrd: " + sw.Elapsed.ToString(), null);
var info = String.Format("new wrd: where_ids: {0}, what_ids: {1}, region_ids {2}", wrd.where_ids.Count, wrd.what_ids.Count, wrd.region_ids.Count);
GenUtils.LogMsg("info", info, null);
GenUtils.LogMsg("info", "new wrd: " + wrd.str_ready_ids, null);
sw.Start();
if (wrd.IsConsistent())
{
SaveWrd(wrd);
}
else
{
GenUtils.PriorityLogMsg("warning", "MakeWebRoleData: inconsistent", null);
wrd = GetWrd(); // fall back to last known good
}
sw.Stop();
GenUtils.LogMsg("info", "save wrd: " + sw.Elapsed.ToString(), null);
}
catch (Exception e3)
{
GenUtils.PriorityLogMsg("exception", "MakeWebRoleData: creating wrd", e3.Message);
}
return(wrd);
}
public byte[] GetObjectFromCacheOrWeb(AspNetCache cache, string key, string url)
{
byte[] data;
try
{
var object_is_cached = cache[key] != null;
if (object_is_cached)
{
data = (byte[])cache[key];
}
else
{
var request = (HttpWebRequest)WebRequest.Create(new Uri(url));
var response = HttpUtils.RetryHttpRequestExpectingStatus(request, HttpStatusCode.OK, data: null, wait_secs: 3, max_tries: 3, timeout_secs: TimeSpan.FromSeconds(10));
if (response.status != HttpStatusCode.OK)
{
GenUtils.PriorityLogMsg("exception", "GetObjectFromCacheOrWeb: " + url, response.status.ToString());
}
data = CacheUtils.MaybeSuppressResponseBodyForView(this.ControllerContext, response.bytes);
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "GetObjectFromCacheOrWeb: " + key + ", " + url, e.Message + e.StackTrace);
data = Encoding.UTF8.GetBytes(e.Message);
}
return(data);
}
public static void TryLoadCatmapFromMetadict(ConcurrentDictionary <string, Dictionary <string, string> > per_feed_catmaps, Dictionary <string, string> metadict)
{
if (metadict.ContainsKey("catmap") && !String.IsNullOrEmpty(metadict["catmap"])) // found a catmap
{
try
{
var feedurl = metadict["feedurl"];
var catmap_value = metadict["catmap"].ToLower();
string json = "";
if (catmap_value.StartsWith("http:")) // try indirect
{
var uri = new Uri(metadict["catmap"]);
json = HttpUtils.FetchUrl(uri).DataAsString();
}
else
{
json = catmap_value; // try direct
}
per_feed_catmaps[feedurl] = JsonConvert.DeserializeObject <Dictionary <string, string> >(json);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "TryLoadCatmap: " + JsonConvert.SerializeObject(metadict), e.Message);
}
}
}
public override void OnStop()
{
var msg = "WebRole: OnStop";
GenUtils.PriorityLogMsg("info", msg, null);
base.OnStop();
}
public override void Run()
{
var msg = "WebRole: Run";
GenUtils.PriorityLogMsg("info", msg, null);
base.Run();
}
protected void Application_Error(object sender, EventArgs e)
{
var ts = TableStorage.MakeDefaultTableStorage();
Exception exception = HttpContext.Current.Server.GetLastError();
GenUtils.PriorityLogMsg("exception", "Application_Error", exception.Message);
// see http://azuremiscellany.blogspot.com/2010/05/web-role-crash-dumps.html for interesting possibility
}
public LogEntriesResult(string log, string conditions, string minutes, string from, string until, string include, string exclude)
{
this.log = String.IsNullOrEmpty(log) ? "log" : log;
this.conditions = conditions;
this.include = include;
this.exclude = exclude;
if (!String.IsNullOrEmpty(from)) // use from/until style
{
this.style = LogStyle.from_until;
try
{
var max_span = TimeSpan.FromHours(24);
this.from = DateTime.Parse(from).ToUniversalTime();
if (String.IsNullOrEmpty(until))
{
this.until = this.from + TimeSpan.FromHours(1);
}
else
{
this.until = DateTime.Parse(until).ToUniversalTime();
}
if (this.until > this.from + max_span)
{
this.until = this.from + max_span;
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "LogEntriesResult: From/Until", e.Message + e.StackTrace);
}
}
else // use recent minutes style
{
this.style = LogStyle.recent;
try
{
var max = Convert.ToInt32(settings["max_log_minutes"]);
var recent = Convert.ToInt32(settings["recent_log_minutes"]);
if (minutes != null)
{
this.minutes = Convert.ToInt32(minutes);
if (this.minutes > max)
{
this.minutes = max;
}
}
else
{
this.minutes = recent;
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "LogEntriesResult: Recent", e.Message + e.StackTrace);
}
}
}
private bool TryDetectSmartPhone(Dictionary <string, object> render_args)
{
var ua = this.controller.HttpContext.Request.UserAgent;
bool smartphone_detected = false;
try
{
SortedList <string, List <string> > props = new SortedList <string, List <string> >();
props = this.controller.HttpContext.Request.Browser.Capabilities["51Degrees.mobi"] as SortedList <string, List <string> >;
var key = "unknown";
string width = "unknown";
string height = "unknown";
try
{
width = props["ScreenPixelsWidth"][0];
height = props["ScreenPixelsHeight"][0];
if (width.ToLower() != "unknown" && height.ToLower() != "unknown")
{
var w = Convert.ToInt32(width);
var h = Convert.ToInt32(height);
var l = new List <int>()
{
w, h
};
l.Sort();
key = l[0].ToString() + "x" + l[1].ToString();
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "TryDetectSmartPhone: unexpected width or height: " + width + "," + height + ": " + ua, e.Message);
}
if (props["IsMobile"][0] == "True")
{
render_args["mobile_short"] = width;
render_args["mobile_long"] = height;
render_args["ua"] = ua;
smartphone_detected = smartphone_screen_dimensions.ContainsKey(key);
if (smartphone_detected)
{
ElmcityApp.logger.LogMsg("smartphone", "IsMobile true, smartphone detected: " + key, ua);
}
else
{
ElmcityApp.logger.LogMsg("smartphone", "IsMobile true, smartphone not detected: " + key, ua);
}
}
else
{
ElmcityApp.logger.LogMsg("smartphone", "IsMobile false: " + key, ua);
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "mobile detection: " + ua, e.Message + e.StackTrace);
}
return(smartphone_detected);
}
public static List <SearchResult> BingSearch(string search_expression, int max, Dictionary <string, object> stats_dict)
{
var url_template = "http://api.search.live.net/json.aspx?AppId=" + Configurator.bing_api_key + "&Market=en-US&Sources=Web&Adult=Strict&Query={0}&Web.Count=50";
var offset_template = "&Web.Offset={1}";
var results_list = new List <SearchResult>();
Uri search_url;
List <int> offsets = GenUtils.EveryNth(start: 0, step: 50, stop: max).ToList();
var options = new ParallelOptions();
Parallel.ForEach(source: offsets, parallelOptions: options, body: (offset) =>
// foreach (var offset in offsets)
{
if (offset == 0)
{
search_url = new Uri(string.Format(url_template, search_expression));
}
else
{
search_url = new Uri(string.Format(url_template + offset_template, search_expression, offset));
}
var page = CallSearchApi(search_url);
if (page == null)
{
//continue;
return;
}
try
{
JObject o = (JObject)JsonConvert.DeserializeObject(page);
var results_query =
from result in o["SearchResponse"]["Web"]["Results"].Children()
select new SearchResult
(
url: result.Value <string>("Url").ToString() ?? "NoUrl",
title: result.Value <string>("Title").ToString() ?? "NoTitle",
content: result.Value <string>("Description").ToString() ?? "NoDescription",
engine: SearchResult.FindingEngine.bing
);
foreach (var result in results_query)
{
results_list.Add(result);
}
}
catch
{
GenUtils.PriorityLogMsg("exception", "BingSearch", search_url.ToString());
}
}
);
return(results_list);
}
public override void OnStop()
{
var msg = "Worker: OnStop";
GenUtils.LogMsg("status", msg, null);
GenUtils.PriorityLogMsg("status", msg, null);
var snapshot = Counters.MakeSnapshot(Counters.GetCounters());
monitor.StoreSnapshot(snapshot);
base.OnStop();
}
public static void ProcessRegion(string region)
{
if ( ! Utils.IsRegion (region) )
throw new Exception(String.Format("ProcessRegion called erroneously for {0}", region));
GenUtils.LogMsg("status", "worker starting on region tasks for " + region, null);
var calinfo = Utils.AcquireCalinfo(region);
try
{
var es_region = new ZonelessEventStore(calinfo);
var ids = Utils.GetIdsForRegion(region);
Parallel.ForEach(source: ids, body: (id) =>
{
var uri = BlobStorage.MakeAzureBlobUri(id, id + ".zoneless.obj", false);
var es = (ZonelessEventStore)BlobStorage.DeserializeObjectFromUri(uri);
foreach (var evt in es.events)
{
evt.categories += "," + id.ToLower(); // so tag viewer can slice region by hub name
lock (es_region)
{
es_region.events.Add(evt);
}
}
}
);
Utils.UpdateFeedCountForId(region);
ZonelessEventStore.Finalize(calinfo, es_region);
CacheUtils.MarkBaseCacheEntryForRemoval(Utils.MakeBaseZonelessUrl(region), Convert.ToInt32(settings["webrole_instance_count"]));
RenderTags(region);
Utils.TagsByHub(region, authorized: true);
Utils.TagsByHub(region, authorized: false);
SaveRegionStats(region);
ImageSelection.BuildCategoryImagesForHub(region);
Utils.SaveQuickStatsForRegion(region);
}
catch (Exception e)
{
var msg = "process region: " + region;
var data = e.Message + e.StackTrace;
GenUtils.PriorityLogMsg("exception", msg, data);
}
GenUtils.LogMsg("status", "worker done processing region " + region, null);
}
public static HttpResponse StoreFeedAndMetadataToAzure(string id, string feedurl, Dictionary <string, object> metadict)
{
string rowkey = TableStorage.MakeSafeRowkeyFromUrl(feedurl);
var r = TableStorage.UpdateDictToTableStore(metadict, ts_table, id, rowkey);
if (r.http_response.status != System.Net.HttpStatusCode.Created)
{
GenUtils.PriorityLogMsg("warning", "StoreFeedAndMetadataToAzure", r.http_response.status.ToString());
}
return(r.http_response);
}
public BlobStorageResponse Serialize()
{
var bs = BlobStorage.MakeDefaultBlobStorage();
var bsr = bs.SerializeObjectToAzureBlobWithoutLease(this, this.id, this.objfile); // because the upload might take longer than 60 secs
if (bsr.HttpResponse.status != System.Net.HttpStatusCode.Created)
{
GenUtils.PriorityLogMsg("warning", "serializing zoneless obj for " + this.id, bsr.HttpResponse.status + " " + bsr.HttpResponse.message);
}
return(bsr);
}
public DateTimeWithZone(DateTime dt, TimeZoneInfo tzinfo)
{
this._tzinfo = tzinfo;
if (dt.Kind == DateTimeKind.Utc)
{
GenUtils.PriorityLogMsg("warning", "DateTimeWithZone: expecting DateTimeKind.Local or Unspecified, got " + dt.Kind.ToString(), null);
}
this._LocalTime = dt;
var _dt = new DateTime(dt.Year, dt.Month, dt.Day, dt.Hour, dt.Minute, dt.Second); // can't convert if kind is local
this._UniversalTime = TimeZoneInfo.ConvertTimeToUtc(_dt, tzinfo);
}
public static void PurgeCache(Object o, ElapsedEventArgs e)
{
try
{
var cache = new AspNetCache(ElmcityApp.home_controller.HttpContext.Cache);
ElmcityUtils.CacheUtils.MaybePurgeCache(cache);
}
catch (Exception ex)
{
GenUtils.PriorityLogMsg("exception", "PurgeCache", ex.Message);
}
}
public static void StoreEventBriteTaggables(string id, Dictionary <string, string> settings, Calinfo calinfo)
{
try
{
var eventbrite_taggables = GetEventBriteOrganizers(calinfo, settings);
bs.SerializeObjectToAzureBlob(eventbrite_taggables, id, "eventbrite.taggables.obj");
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "StoreTaggables: EventBrite", e.Message);
}
}
public static void StoreMeetupTaggables(string id, Dictionary <string, string> settings, Calinfo calinfo)
{
try
{
var meetup_taggables = GetMeetupGroups(calinfo, 1, settings);
bs.SerializeObjectToAzureBlob(meetup_taggables, id, "meetup.taggables.obj");
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "StoreTaggables: Meetup", e.Message);
}
}
public static void StoreEventfulTaggables(string id, Dictionary <string, string> settings, Calinfo calinfo)
{
try
{
var eventful_taggables = GetEventfulVenues(calinfo, min_per_venue: 1, settings: settings);
bs.SerializeObjectToAzureBlob(eventful_taggables, id, "eventful.taggables.obj");
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "StoreTaggables: Eventful", e.Message);
}
}
public static void StoreFacebookTaggables(string id, string location, Calinfo calinfo)
{
try
{
var facebook_taggables = GetFacebookPages(calinfo, location);
bs.SerializeObjectToAzureBlob(facebook_taggables, id, "facebook.taggables.obj");
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "StoreTaggables: Facebook", e.Message);
}
}
private static bool HandleFeedUpdates(string id, TableStorage ts, List <Dictionary <string, string> > list_metadict_str, string feed_entry_schema, List <string> current_feed_urls, List <string> deleted_feed_urls)
{
List <string> maybe_updated_feed_urls = (List <string>)current_feed_urls.Except(deleted_feed_urls).ToList();
var update_lock = new Object();
var updated_feeds = false;
Parallel.ForEach(source: maybe_updated_feed_urls, body: (maybe_updated_feed_url) =>
{
var rowkey = TableStorage.MakeSafeRowkeyFromUrl(maybe_updated_feed_url);
var query = string.Format("$filter=PartitionKey eq '{0}' and RowKey eq '{1}'", id, rowkey);
var table_record = TableStorage.QueryForSingleEntityAsDictStr(ts, "metadata", query);
var json_record = (Dictionary <string, string>)list_metadict_str.Find(feed => feed["feedurl"] == maybe_updated_feed_url);
bool updated_feed = false;
//foreach (var key in new List<string>() { "source", "category", "feedurl", "url", "catmap", "approved", "comments" })
foreach (var key in feed_entry_schema.Split(','))
{
try
{
// if (table_record.ContainsKey(key) == false)
// GenUtils.LogMsg("info", id, "UpdateFeedsForId: table record missing key: " + key + " (will be added)");
if (json_record.ContainsKey(key) == false)
{
// GenUtils.LogMsg("info", id, "UpdateFeedsForId: json record missing key: " + key + " (adding it now)");
json_record[key] = "";
}
if (table_record.ContainsKey(key) == false || table_record[key] != json_record[key])
{
lock (update_lock)
{
updated_feeds = updated_feed = true;
}
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("info", "UpdateFeedsForId: missing key: " + key, e.Message + e.StackTrace);
}
}
if (updated_feed)
{
UpmergeChangedMetadict(id, rowkey, json_record);
}
});
// foreach (string maybe_updated_feed_url in maybe_updated_feed_urls)
// {
// }
return(updated_feeds);
}
// last-resort exception handler
protected override void OnException(ExceptionContext filterContext)
{
var msg = String.Format("url: {0}, message: {1}", filterContext.HttpContext.Request.Url.ToString(), filterContext.Exception.Message);
GenUtils.PriorityLogMsg("exception", "last chance", msg + filterContext.Exception.StackTrace);
if (msg.Length > 140)
{
msg = msg.Substring(0, 140);
}
//TwitterApi.SendTwitterDirectMessage(CalendarAggregator.Configurator.twitter_account, "last chance: " + msg);
filterContext.ExceptionHandled = true;
this.View("FinalError").ExecuteResult(this.ControllerContext);
}