public static bool RegionIsStale(string region)
{
var stale = false;
try
{
var bs = BlobStorage.MakeDefaultBlobStorage();
var region_props = bs.GetBlobProperties(region.ToLower(), region + ".zoneless.obj");
var region_modified = DateTime.Parse(region_props.HttpResponse.headers["Last-Modified"]);
foreach (var hub in Utils.GetIdsForRegion(region))
{
var hub_props = bs.GetBlobProperties(hub.ToLower(), hub + ".zoneless.obj");
var hub_modified = DateTime.Parse(hub_props.HttpResponse.headers["Last-Modified"]);
if (hub_modified > region_modified)
{
stale = true;
break;
}
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "RegionIsStale: " + region, e.Message + e.StackTrace);
}
return stale;
}
// populate feed registry from azure table
public void LoadFeedsFromAzure(FeedLoadOption option)
{
var dict = Metadata.LoadFeedsFromAzureTableForId(this.id, option);
var settings = GenUtils.GetSettingsFromAzureTable("settings");
foreach (var url in dict.Keys)
{
try
{
if (settings["eventful_feeds_enabled"] == "False" && url.StartsWith("http://eventful.com/"))
{
continue;
}
if (settings["eventbrite_feeds_enabled"] == "False" && url.Contains("ics_from_eventbrite"))
{
continue;
}
if (settings["lastfm_feeds_enabled"] == "False" && url.Contains("ics_from_lastfm_venue"))
{
continue;
}
if (url.StartsWith("http://upcoming.yahoo.com"))
{
continue;
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "CollectIcal", e.Message + e.StackTrace);
}
this.AddFeed(url, dict[url]);
}
}
private static string CallSearchApi(Uri search_url)
{
var delay = 2;
try
{
delay = Convert.ToInt32(settings["search_engine_api_delay_secs"]);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "CallSearchApi", e.Message + e.StackTrace);
}
Utils.Wait(delay);
var r = HttpUtils.FetchUrl(search_url);
if (r.status != HttpStatusCode.OK)
{
GenUtils.LogMsg("warning", "CallSearchApi" + r.status.ToString(), search_url.ToString());
return(null);
}
else
{
return(r.DataAsString());
}
}
public void RegexFindKeysAndValuesFindsUrlAndCategory()
{
var url = "http://www.co.yavapai.az.us/_Events.aspx?id=32794&foo=bar#asdf%d3-abc";
var category = "music,arts";
var s = string.Format(@"
url={0}
sys.path.append(""c:\users\jonu\dev"")
import clr
category={1}
clr.AddReference(""System.Core"")
import System
",
url, category);
var keys = new List <string>()
{
"url", "category"
};
var metadict = GenUtils.RegexFindKeysAndValues(keys, s);
Assert.That(metadict["url"] == url);
Assert.That(metadict["category"] == category);
}
private static void PerformSearches(string where, Dictionary <string, SearchResult> final_results_dict, Dictionary <string, object> stats_dict)
{
foreach (var qualifier in qualifiers)
{
foreach (var day in days)
{
string q;
try
{
q = string.Format(@" ""{0}"" ""{1} {2}"" ", where, qualifier, day);
var results = GoogleSearch(q, stats_dict);
DictifyResults(results, final_results_dict, stats_dict);
}
catch (Exception ex1)
{
GenUtils.PriorityLogMsg("exception", "search_location: google: " + where, ex1.Message);
}
try
{
q = string.Format(@" ""{0}"" near:100 ""{1} {2}"" ", where, qualifier, day);
var results = BingSearch(q, 500, stats_dict);
DictifyResults(results, final_results_dict, stats_dict);
}
catch (Exception ex2)
{
GenUtils.PriorityLogMsg("exception", "search_location: bing: " + where, ex2.Message);
}
}
}
}
public static void ProcessIcal(string id)
{
var calinfo = Utils.AcquireCalinfo(id);
GenUtils.LogMsg("status", "worker starting on ical tasks for " + id, null);
var fr = new FeedRegistry(id);
DoIcal(fr, calinfo);
}
public EventCollectorTest()
{
test_calinfo = new Calinfo(ElmcityUtils.Configurator.azure_compute_account);
lookup_lat = test_calinfo.lat;
lookup_lon = test_calinfo.lon;
radius = Configurator.default_radius;
test_upcoming_args = string.Format("location={0},{1}&radius={2}&min_date={3}", lookup_lat, lookup_lon, radius, min_date);
settings = GenUtils.GetSettingsFromAzureTable();
basic_ics = BlobStorage.MakeDefaultBlobStorage().GetBlob("admin", "basic.ics").HttpResponse.DataAsString();
bs = BlobStorage.MakeDefaultBlobStorage();
calinfo_berkeley = new Calinfo(berkeley_test_hub);
calinfo_keene = new Calinfo(keene_test_hub);
collector_berkeley = new Collector(calinfo_berkeley, settings);
collector_keene = new Collector(calinfo_keene, settings);
foreach (var example in ics_examples)
{
UpdateYYYY(example, "ics");
}
foreach (var example in eventful_examples)
{
UpdateYYYY(example, "xml");
}
//foreach (var example in upcoming_examples)
// UpdateYYYY(example, "xml");
foreach (var example in eventbrite_examples)
{
UpdateYYYY(example, "xml");
}
}
public static void SaveWrd(WebRoleData wrd)
{
if (!wrd.IsConsistent())
{
GenUtils.PriorityLogMsg("warning", "inconsistent WebRoleData!", null);
return;
}
var bs = BlobStorage.MakeDefaultBlobStorage();
var lease_response = bs.RetryAcquireLease("admin", "wrd.obj");
if (lease_response.status == HttpStatusCode.Created)
{
var lease_id = lease_response.headers["x-ms-lease-id"];
var bytes = ObjectUtils.SerializeObject(wrd);
var headers = new Hashtable()
{
{ "x-ms-lease-id", lease_id }
};
var r = bs.PutBlob("admin", "wrd.obj", headers, bytes, "binary/octet-stream");
if (r.HttpResponse.status != HttpStatusCode.Created)
{
GenUtils.PriorityLogMsg("warning", "SaveWrd: cannot save", null);
}
SaveTimestampedWrd(wrd);
}
else
{
GenUtils.PriorityLogMsg("warning", "SaveWrd: cannot lease", null);
}
}
//[OutputCache(Duration = ... // output cache not used here, iis cache is managed directly
public ActionResult GetEvents(string id, string type, string view, string jsonp, string count, string from, string to, string eventsonly, string mobile, string test, string raw, string raw_sentinel, string style, string theme, string taglist, string tags, string template, string jsurl, string days, string bare_events, string hub, string source, string first)
{
if (id == "a2cal")
{
id = "AnnArborChronicle";
}
ElmcityApp.logger.LogHttpRequest(this.ControllerContext);
if (view != null)
{
view = view.ToLower();
}
EventsResult r = null;
try
{
var cr = ElmcityApp.wrd.renderers[id];
r = new EventsResult(this, cr, id, type, view, jsonp, count, from, to, eventsonly, mobile, test, raw, raw_sentinel, style, theme, taglist, tags, template, jsurl, days, bare_events, hub, source, first);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "GetEvents: " + id, e.Message);
}
return(r);
}
public static TimeSpan IntervalFromType(TaskType type)
{
var ts = TimeSpan.FromHours(24);
switch (type)
{
case TaskType.nonicaltasks:
ts = Scheduler.nonical_interval;
break;
case TaskType.icaltasks:
ts = Scheduler.ical_interval;
break;
case TaskType.regiontasks:
ts = Scheduler.region_interval;
break;
case TaskType.none:
ts = TimeSpan.FromHours(24);
GenUtils.PriorityLogMsg("warning", "Scheduler.IntervalFromType", "unexpected TaskType.none");
break;
}
return(ts);
}
public static WebRoleData MakeWebRoleData()
{
WebRoleData wrd = null;
try // create WebRoleData structure and store as blob, available to webrole on next _reload
{
var sw = new Stopwatch();
sw.Start();
wrd = new WebRoleData();
sw.Stop();
GenUtils.LogMsg("info", "new wrd: " + sw.Elapsed.ToString(), null);
var info = String.Format("new wrd: where_ids: {0}, what_ids: {1}, region_ids {2}", wrd.where_ids.Count, wrd.what_ids.Count, wrd.region_ids.Count);
GenUtils.LogMsg("info", info, null);
GenUtils.LogMsg("info", "new wrd: " + wrd.str_ready_ids, null);
sw.Start();
if (wrd.IsConsistent())
{
SaveWrd(wrd);
}
else
{
GenUtils.PriorityLogMsg("warning", "MakeWebRoleData: inconsistent", null);
wrd = GetWrd(); // fall back to last known good
}
sw.Stop();
GenUtils.LogMsg("info", "save wrd: " + sw.Elapsed.ToString(), null);
}
catch (Exception e3)
{
GenUtils.PriorityLogMsg("exception", "MakeWebRoleData: creating wrd", e3.Message);
}
return(wrd);
}
public override void OnDataReceived(byte[] data)
{
byte[] msgData = new byte[data.Length];
Array.Copy(data, msgData, data.Length);
System.Diagnostics.Debug.WriteLine($"Received: {GenUtils.Byte2String(data)}");
decoder.Enqueue(data);
}
public static void SaveRegionStats(string region)
{
GenUtils.LogMsg("status", "SaveRegionStats: " + region, null);
var ids = Utils.GetIdsForRegion(region);
var region_report = new StringBuilder();
var chunks = new Dictionary<string, string>();
Parallel.ForEach(source: ids, body: (id) =>
{
var hub_report = new StringBuilder();
hub_report.Append("<h1>" + id + "</h1>\n");
var hub_report_uri = BlobStorage.MakeAzureBlobUri(id, id + ".stats.html");
var hub_report_html = HttpUtils.FetchUrl(hub_report_uri).DataAsString();
var doc = new HtmlDocument();
doc.LoadHtml(hub_report_html);
hub_report.AppendLine("<table style=\"border-spacing:6px\" class=\"icalstats\">");
var table = doc.DocumentNode.SelectSingleNode("//table");
hub_report.AppendLine(table.WriteContentTo());
hub_report.AppendLine("</table>");
lock (chunks)
{
chunks[id] = hub_report.ToString(); // to enable sorted output
}
}
);
foreach (var id in ids)
region_report.AppendLine(chunks[id]);
var region_calinfo = Utils.AcquireCalinfo(region);
var report = Utils.EmbedHtmlSnippetInDefaultPageWrapper(region_calinfo, region_report.ToString());
bs.PutBlob(region, region + ".stats.html", report, "text/html");
}
public void GeneralAdmin(object o, ElapsedEventArgs args)
{
try
{
GenUtils.PriorityLogMsg("status", "GeneralAdmin", null);
Utils.MakeWhereSummary(); // refresh http://elmcity.blob.core.windows.net/admin/where_summary.html
Utils.MakeWhatSummary(); // refresh http://elmcity.blob.core.windows.net/admin/what_summary.html
var ids = Metadata.LoadWhatWhereIdsFromAzureTable();
Parallel.ForEach(source: ids, body: (id) =>
{
var calinfo = Utils.AcquireCalinfo(id);
Utils.SaveIcalPerFeedLocations(calinfo, settings);
});
}
catch (Exception e)
{
GenUtils.LogMsg("exception", "GeneralAdmin", e.Message + e.StackTrace);
}
//Utils.MakeFeaturedHubs(); // idle for now
}
/// <summary>
/// ThreadStart method. Will clone all of the EPR's in the EPR list back tot he Host.
/// This is a special case, set up by Loading a snapshot - - where the EPR List will
/// contain what the EPR's *SHOULD BE*, rather than what they actually are.
/// </summary>
private void clone()
{
try
{
eprHostChannel = RemOpWithTimedRetries(() => (new DeviceSimulatorChannel(HostName)), TimeSpan.FromSeconds(3), 3);
try
{
foreach (EPRInstance _epr in EPRs)
{
if (string.Compare(_epr.Profile, EPRInstance.NOPROFILE) != 0)
{
DeviceInstance _di = RemOpWithTimedRetries(() => eprHostChannel.CreateInstance(_epr.Profile, _epr.IPAddress, _epr.Protocols), TimeSpan.FromSeconds(3), 3);
_epr.ResetDevice(_di);
}
}
WorkStatus = HostVMStatus.Done;
}
catch (Exception _ex)
{
GenUtils.HangError("Encountered error while cloning EPR " + HostName + " :: " + _ex.Message);
WorkStatus = HostVMStatus.Error;
}
}
catch (Exception _ex)
{
GenUtils.HangError("Encountered error while connect (for cloning) to EPR " + HostName + " :: " + _ex.Message);
WorkStatus = HostVMStatus.Error;
}
}
public byte[] GetObjectFromCacheOrWeb(AspNetCache cache, string key, string url)
{
byte[] data;
try
{
var object_is_cached = cache[key] != null;
if (object_is_cached)
{
data = (byte[])cache[key];
}
else
{
var request = (HttpWebRequest)WebRequest.Create(new Uri(url));
var response = HttpUtils.RetryHttpRequestExpectingStatus(request, HttpStatusCode.OK, data: null, wait_secs: 3, max_tries: 3, timeout_secs: TimeSpan.FromSeconds(10));
if (response.status != HttpStatusCode.OK)
{
GenUtils.PriorityLogMsg("exception", "GetObjectFromCacheOrWeb: " + url, response.status.ToString());
}
data = CacheUtils.MaybeSuppressResponseBodyForView(this.ControllerContext, response.bytes);
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "GetObjectFromCacheOrWeb: " + key + ", " + url, e.Message + e.StackTrace);
data = Encoding.UTF8.GetBytes(e.Message);
}
return(data);
}
/// <summary>
/// ThreadStart method. manages starting one or more EPRs on this host. All needed parameters are
/// passed through member variables (old school) - vis numToStart, startProfile and startProtocols.
/// </summary>
private void startEPRs()
{
try
{
int _started = numToStart;
for (int _ndx = 0; (_ndx < EPRs.Count) && (_started > 0); _ndx++)
{
if (!EPRs[_ndx].InUse)
{
DeviceInstance _di = RemOpWithTimedRetries(() => eprHostChannel.CreateInstance(startProfile, EPRs[_ndx].IPAddress, startProtocols), TimeSpan.FromSeconds(3), 3);
--_started;
EPRs[_ndx].ResetDevice(_di);
}
else
{
GenUtils.HangError(EPRs[_ndx].HostName + " is already in use as " + EPRs[_ndx].Profile + ". Start command ignored for this EPR");
}
}
WorkStatus = HostVMStatus.Done;
}
catch (Exception _ex)
{
GenUtils.HangError("Encountered error whie starting EPRs on " + HostName + " :: " + _ex.Message);
WorkStatus = HostVMStatus.Error;
}
}
/// <summary>
/// ThreadStart method. Will manage the tearing down of one or more EPR instancs on this Host VM.
/// This is executed in a thread - even though the operation itself is fairly quick.
/// </summary>
private void destroy()
{
//-------------------------------------------------------------------------------------------------------
// Follows use rule - if no IP addresses have been specifically specified on the command line, then
// operation will affect ALL EPR IP's on this host.
try
{
if (specifiedIPs.Count == 0)
{
RemOpWithTimedRetries(() => eprHostChannel.DestroyAllInstances(), TimeSpan.FromSeconds(3), 3);
}
else
{
foreach (EPRInstance _epr in EPRs)
{
RemOpWithTimedRetries(() => eprHostChannel.DestroyInstance(_epr.IPAddress), TimeSpan.FromSeconds(3), 3);
}
}
//-----------------------------------------------------------------------------------------------------
// If IP's have been specified, it is probably only a few (< 5) so running this loop twice is not
// terrible. If no IPs were specified, we would have to do this anyhow.
foreach (EPRInstance _epr in EPRs)
{
_epr.ClearDevice();
}
WorkStatus = HostVMStatus.Done;
}
catch (Exception _ex)
{
GenUtils.HangError("Error when destroying EPRs : " + _ex.Message);
WorkStatus = HostVMStatus.Error;
}
}
protected void Application_End(object sender, EventArgs e)
{
HttpRuntime runtime = (HttpRuntime)typeof(System.Web.HttpRuntime).InvokeMember("_theRuntime",
BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.GetField,
null,
null,
null);
if (runtime == null)
{
return;
}
string shutdown_message = (string)runtime.GetType().InvokeMember("_shutDownMessage",
BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.GetField,
null,
runtime,
null);
string shutdown_stack = (string)runtime.GetType().InvokeMember("_shutDownStack",
BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.GetField,
null,
runtime,
null);
var ts = TableStorage.MakeDefaultTableStorage();
GenUtils.PriorityLogMsg("Application_End", shutdown_message, shutdown_stack);
}
public WebRoleData()
{
GenUtils.LogMsg("info", String.Format("WebRoleData: {0}, {1}, {2}, {3}", procname, procid, domain_name, thread_id), null);
MakeWhereAndWhatAndRegionIdLists();
var ids = Metadata.LoadHubIdsFromAzureTable();
Parallel.ForEach(ids, id =>
//foreach (var id in ids)
{
GenUtils.LogMsg("info", "GatherWebRoleData: readying: " + id, null);
var cr = Utils.AcquireRenderer(id);
lock (this.renderers)
{
this.renderers.Add(id, cr);
}
lock (this.ready_ids)
{
this.ready_ids.Add(id);
}
});
//}
// this pipe-delimited string defines allowed IDs in the /services/ID/... URL pattern
this.str_ready_ids = String.Join("|", this.ready_ids.ToArray());
GenUtils.LogMsg("info", "GatherWebRoleData: str_ready_ids: " + this.str_ready_ids, null);
}
public static void TryLoadCatmapFromMetadict(ConcurrentDictionary <string, Dictionary <string, string> > per_feed_catmaps, Dictionary <string, string> metadict)
{
if (metadict.ContainsKey("catmap") && !String.IsNullOrEmpty(metadict["catmap"])) // found a catmap
{
try
{
var feedurl = metadict["feedurl"];
var catmap_value = metadict["catmap"].ToLower();
string json = "";
if (catmap_value.StartsWith("http:")) // try indirect
{
var uri = new Uri(metadict["catmap"]);
json = HttpUtils.FetchUrl(uri).DataAsString();
}
else
{
json = catmap_value; // try direct
}
per_feed_catmaps[feedurl] = JsonConvert.DeserializeObject <Dictionary <string, string> >(json);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "TryLoadCatmap: " + JsonConvert.SerializeObject(metadict), e.Message);
}
}
}
public static void EnsureTaskRecord(string id, TaskType type)
{
if (Scheduler.ExistsTaskRecordForId(id, type) == false)
{
GenUtils.LogMsg("status", "MaybeCreateTaskRecord: creating task for " + id, null);
Scheduler.InitTaskForId(id, type);
}
}
public void RenderedHtmlEventCountMatchesStoredEventCount()
{
var es = MakeNewEventStore();
var html = cr.RenderHtml(es);
var html_count = GenUtils.RegexCountSubstrings(html, this.event_html_header);
Assert.AreEqual(es.events.Count, html_count);
}
public void RenderedHtmlViewMatchesExpectedCount()
{
var es = MakeNewEventStore();
var html = cr.RenderHtml(es, EventStoreTest.test_category, 0, from: DateTime.MinValue, to: DateTime.MinValue, source: null, args: null);
var html_count = GenUtils.RegexCountSubstrings(html, event_html_header);
Assert.AreEqual(1, html_count);
}
protected void Application_Error(object sender, EventArgs e)
{
var ts = TableStorage.MakeDefaultTableStorage();
Exception exception = HttpContext.Current.Server.GetLastError();
GenUtils.PriorityLogMsg("exception", "Application_Error", exception.Message);
// see http://azuremiscellany.blogspot.com/2010/05/web-role-crash-dumps.html for interesting possibility
}
public override void Run()
{
var msg = "WebRole: Run";
GenUtils.PriorityLogMsg("info", msg, null);
base.Run();
}
public override void OnStop()
{
var msg = "WebRole: OnStop";
GenUtils.PriorityLogMsg("info", msg, null);
base.OnStop();
}
private bool TryDetectSmartPhone(Dictionary <string, object> render_args)
{
var ua = this.controller.HttpContext.Request.UserAgent;
bool smartphone_detected = false;
try
{
SortedList <string, List <string> > props = new SortedList <string, List <string> >();
props = this.controller.HttpContext.Request.Browser.Capabilities["51Degrees.mobi"] as SortedList <string, List <string> >;
var key = "unknown";
string width = "unknown";
string height = "unknown";
try
{
width = props["ScreenPixelsWidth"][0];
height = props["ScreenPixelsHeight"][0];
if (width.ToLower() != "unknown" && height.ToLower() != "unknown")
{
var w = Convert.ToInt32(width);
var h = Convert.ToInt32(height);
var l = new List <int>()
{
w, h
};
l.Sort();
key = l[0].ToString() + "x" + l[1].ToString();
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "TryDetectSmartPhone: unexpected width or height: " + width + "," + height + ": " + ua, e.Message);
}
if (props["IsMobile"][0] == "True")
{
render_args["mobile_short"] = width;
render_args["mobile_long"] = height;
render_args["ua"] = ua;
smartphone_detected = smartphone_screen_dimensions.ContainsKey(key);
if (smartphone_detected)
{
ElmcityApp.logger.LogMsg("smartphone", "IsMobile true, smartphone detected: " + key, ua);
}
else
{
ElmcityApp.logger.LogMsg("smartphone", "IsMobile true, smartphone not detected: " + key, ua);
}
}
else
{
ElmcityApp.logger.LogMsg("smartphone", "IsMobile false: " + key, ua);
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "mobile detection: " + ua, e.Message + e.StackTrace);
}
return(smartphone_detected);
}
public LogEntriesResult(string log, string conditions, string minutes, string from, string until, string include, string exclude)
{
this.log = String.IsNullOrEmpty(log) ? "log" : log;
this.conditions = conditions;
this.include = include;
this.exclude = exclude;
if (!String.IsNullOrEmpty(from)) // use from/until style
{
this.style = LogStyle.from_until;
try
{
var max_span = TimeSpan.FromHours(24);
this.from = DateTime.Parse(from).ToUniversalTime();
if (String.IsNullOrEmpty(until))
{
this.until = this.from + TimeSpan.FromHours(1);
}
else
{
this.until = DateTime.Parse(until).ToUniversalTime();
}
if (this.until > this.from + max_span)
{
this.until = this.from + max_span;
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "LogEntriesResult: From/Until", e.Message + e.StackTrace);
}
}
else // use recent minutes style
{
this.style = LogStyle.recent;
try
{
var max = Convert.ToInt32(settings["max_log_minutes"]);
var recent = Convert.ToInt32(settings["recent_log_minutes"]);
if (minutes != null)
{
this.minutes = Convert.ToInt32(minutes);
if (this.minutes > max)
{
this.minutes = max;
}
}
else
{
this.minutes = recent;
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "LogEntriesResult: Recent", e.Message + e.StackTrace);
}
}
}
public static void StoreTaskForId(Task task, string id, TaskType type)
{
task.hostname = System.Net.Dns.GetHostName();
var dict_obj = ObjectUtils.ObjToDictObj(task);
var tasktable = type.ToString();
var ts_response = TableStorage.UpmergeDictToTableStore(dict_obj, table: tasktable, partkey: master_pk, rowkey: id);
GenUtils.LogMsg("status", "Scheduler.StoreTaskForId: " + id, null);
}
