public void StartMonitor()
{
GenUtils.LogMsg("status", "StartMonitor", "starting");
var ProcessMonitorThread = new Thread(new ThreadStart(ProcessMonitorThreadMethod));
ProcessMonitorThread.Start();
}
public static HttpResponse RetryStorageRequestExpectingServiceAvailable(string method, Hashtable headers, byte[] data, string content_type, Uri uri, int wait_secs, int max_tries, TimeSpan timeout_secs)
{
try
{
return(GenUtils.Actions.Retry <HttpResponse>(
delegate()
{
try
{
return StorageUtils.DoStorageRequest(method, headers, data, content_type, uri);
}
catch // (Exception e)
{
//GenUtils.PriorityLogMsg("exception", "RetryStorageRequest: " + uri, e.Message + e.StackTrace);
throw new Exception("RetryStorageRequestException");
}
},
CompletedIfStatusNotServiceUnavailable,
completed_delegate_object: null,
wait_secs: wait_secs,
max_tries: max_tries,
timeout_secs: timeout_secs));
}
catch (Exception e)
{
GenUtils.LogMsg("exception", "RetryStorageRequest: " + uri, e.Message + e.StackTrace);
return(new HttpResponse(HttpStatusCode.ServiceUnavailable, uri.ToString(), null, new Dictionary <string, string>()));
}
}
// scan the cacheurls table, compare uris with counts > 0 to uris in cache, if match then evict uri and decrement count
public static void MaybePurgeCache(ICache cache)
{
try
{
var purgeable_entities = FetchPurgeableCacheDicts();
GenUtils.LogMsg("status", String.Format("MaybePurgeCache: {0} purgeable entities", purgeable_entities.Count), null);
foreach (var purgeable_entity in purgeable_entities)
{
var purgeable_cache_url = (string)purgeable_entity["cached_uri"];
if (cache[purgeable_cache_url] != null)
{
GenUtils.LogMsg("status", "MaybePurgeCache", purgeable_cache_url);
cache.Remove(purgeable_cache_url);
var obj = HttpUtils.FetchUrl(new Uri(purgeable_cache_url)); // rewarm the cache
var count = (int)purgeable_entity["count"];
count -= 1;
if (count < 0)
{
count = 0;
GenUtils.LogMsg("warning", "CacheUtils.MaybePurgeCache", "count went subzero, reset to zero");
}
purgeable_entity["count"] = count;
var rowkey = TableStorage.MakeSafeRowkeyFromUrl(purgeable_cache_url);
ts.UpdateEntity(cache_control_tablename, cache_control_partkey, rowkey, purgeable_entity);
}
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "MaybePurgeCache", e.Message + e.StackTrace);
}
}
// called when there's a rule for an item in snapshot. eval the rule and if trigger fires log a priority message
// todo: simplify this, it's way too verbose
private void EvaluateTrigger(string key, object value, Dictionary <string, string> trigger_dict)
{
int snapshot_int;
float snapshot_float;
int trigger_int;
float trigger_float;
var type = trigger_dict["type"];
switch (type)
{
case "float":
snapshot_float = (float)value;
if (trigger_dict.ContainsKey("max"))
{
trigger_float = float.Parse(trigger_dict["max"]);
if (snapshot_float > trigger_float)
{
GenUtils.PriorityLogMsg("warning", key, String.Format("snapshot ({0}) > trigger ({1})", snapshot_float, trigger_float));
}
}
if (trigger_dict.ContainsKey("min"))
{
trigger_float = float.Parse(trigger_dict["min"]);
if (snapshot_float < trigger_float)
{
GenUtils.PriorityLogMsg("warning", key, String.Format("snapshot ({0}) < trigger ({1})", snapshot_float, trigger_float));
}
}
break;
case "int":
snapshot_int = Convert.ToInt32(value);
if (trigger_dict.ContainsKey("max"))
{
trigger_int = Convert.ToInt32(trigger_dict["max"]);
if (snapshot_int > trigger_int)
{
GenUtils.PriorityLogMsg("warning", key, String.Format("snapshot ({0}) > trigger ({1})", snapshot_int, trigger_int));
}
}
if (trigger_dict.ContainsKey("min"))
{
trigger_int = Convert.ToInt32(trigger_dict["min"]);
if (snapshot_int < trigger_int)
{
GenUtils.PriorityLogMsg("warning", key, String.Format("snapshot ({0}) < trigger ({1})", snapshot_int, trigger_int));
}
}
break;
default:
GenUtils.LogMsg("warning", "MaybeWritePriorityLog", "unexpected type: " + type);
break;
}
}
public static HttpResponse RetryStorageRequestExpectingServiceAvailable(string method, Hashtable headers, byte[] data, string content_type, Uri uri)
{
try
{
return(RetryStorageRequestExpectingServiceAvailable(method, headers, data, content_type, uri, 60, 3, TimeSpan.FromSeconds(500)));
}
catch (Exception e)
{
GenUtils.LogMsg("exception", "RetryStorageRequest: " + uri, e.Message + e.StackTrace);
return(new HttpResponse(HttpStatusCode.ServiceUnavailable, uri.ToString(), null, new Dictionary <string, string>()));
}
}
public Object Remove(string key)
{
Object value = null;
if (this.cache.ContainsKey(key))
{
value = this.cache[key];
this.cache.Remove(key);
GenUtils.LogMsg("status", "MockCache.Remove", key);
}
return(value);
}
public static string EscapeValueForCsv(this string text)
{
try
{
text = text.Replace("\"", "\"\"");
text = text.Replace("\n", "\\n");
return(text);
}
catch (Exception e)
{
GenUtils.LogMsg("warning", "EscapeValueForCsv", e.Message);
return("");
}
}
public static string UrlsToLinks(this string text)
{
try
{ // http://rickyrosario.com/blog/converting-a-url-into-a-link-in-csharp-using-regular-expressions/
string regex = @"((www\.|(http|https|ftp|news|file)+\:\/\/)[_a-z0-9-]+\.[_a-z0-9\/+:@=.+?,##%&~-]*[^.|\'|\# |!|\(|?|,| |>|<|;|\)])";
Regex r = new Regex(regex, RegexOptions.IgnoreCase);
return(r.Replace(text, "<a href=\"$1\" title=\"Click to open in a new window or tab\" target=\"_blank\">$1</a>").Replace("href=\"www", "href=\"http://www"));
}
catch (Exception e)
{
GenUtils.LogMsg("exception", "UrlsToLinks: " + text, e.Message);
return(text);
}
}
public static XmlDocument XmlDocumentFromXmlBytes(byte[] xml_bytes)
{
var sr = new MemoryStream(xml_bytes);
XmlDocument doc = new XmlDocument();
try
{
doc.Load(sr);
}
catch (Exception e)
{
GenUtils.LogMsg("exception", "XmlDocumentFromHttpResponse", e.Message + e.StackTrace);
}
return(doc);
}
public Object Remove(string key)
{
Object o = null;
try
{
o = this.cache.Remove(key);
GenUtils.LogMsg("status", "AspNetCache.Remove", key);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "AspNetCache.Remove: " + key, e.Message + e.StackTrace);
}
return(o);
}
private static JsonCompareResult NewJsonMatchesExistingJson(JsonSnapshotType type, Object new_obj, Uri existing_obj_uri)
{
var result = JsonCompareResult.different;
try
{
if (BlobStorage.ExistsBlob(existing_obj_uri))
{
if (type == JsonSnapshotType.DictStr)
{
var new_dict_str = (Dictionary <string, string>)new_obj;
if (new_dict_str.Keys.Count == 0)
{
return(JsonCompareResult.invalid);
}
var existing_dict_str = ObjectUtils.GetDictStrFromJsonUri(existing_obj_uri);
if (existing_dict_str.Keys.Count == 0) // this shouldn't happen, but...
{
return(JsonCompareResult.invalid);
}
var equal = ObjectUtils.DictStrEqualsDictStr((Dictionary <string, string>)existing_dict_str, new_dict_str);
result = equal ? JsonCompareResult.same : JsonCompareResult.different;
}
else // JsonSnapshotType.ListDictStr
{
var new_list_dict_str = (List <Dictionary <string, string> >)new_obj;
if (AllDictsHaveKeys(new_list_dict_str) == false)
{
return(JsonCompareResult.invalid);
}
var existing_list_dict_str = ObjectUtils.GetListDictStrFromJsonUri(existing_obj_uri);
if (AllDictsHaveKeys(existing_list_dict_str) == false)
{
return(JsonCompareResult.invalid);
}
var equal = ObjectUtils.ListDictStrEqualsListDictStr((List <Dictionary <string, string> >)existing_list_dict_str, new_list_dict_str);
result = equal ? JsonCompareResult.same : JsonCompareResult.different;
}
}
}
catch (Exception e)
{
GenUtils.LogMsg("exception", "NewJsonMatchesExistingJson", e.Message + e.StackTrace);
}
return(result);
}
public static void InstallPythonStandardLibrary(string directory, TableStorage ts)
{
GenUtils.PriorityLogMsg("info", "installing python standard library", null);
try
{
var zip_url = Configurator.pylib_zip_url;
FileUtils.UnzipFromUrlToDirectory(zip_url, directory);
var args = new List <string> {
"", "", ""
};
var script_url = Configurator.python_test_script_url;
var result = RunIronPython(directory, script_url, args);
GenUtils.LogMsg("status", "result of python standard lib install test", result);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "InstallPythonStandardLibrary", e.Message + e.StackTrace);
}
}
// this query expects to find just one matching entity
public static Dictionary <string, object> QueryForSingleEntityAsDictObj(TableStorage ts, string table, string q)
{
var ts_response = ts.QueryEntities(table, q);
var dicts = ts_response.list_dict_obj;
var dict = new Dictionary <string, object>();
if (dicts.Count > 0)
{
dict = dicts.FirstOrDefault();
}
if (dicts.Count > 1)
{
// should not happen, but...
GenUtils.LogMsg("warning", "QueryForSingleEntity: " + table, q + ": more than one matching entity");
}
return(dict);
}
public void ProcessMonitorThreadMethod()
{
while (true)
{
try
{
GenUtils.LogMsg("status", "ProcessMonitorThreadMethod", "snapshot");
this.ReloadCounters();
var snapshot = Counters.MakeSnapshot(counters);
this.StoreSnapshot(snapshot);
this.MaybeWritePriorityLog(snapshot);
Thread.Sleep(TimeSpan.FromMinutes(this.interval_minutes));
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "ProcessMonitorThreadMethod: snapshot", e.Message + e.StackTrace);
}
}
}
// todo: externalize test url as a setting
public static void InstallPythonElmcityLibrary(string directory, TableStorage ts)
{
{
GenUtils.LogMsg("status", "installing python elmcity library", null);
try
{
var zip_url = Configurator.elmcity_pylib_zip_url;
FileUtils.UnzipFromUrlToDirectory(zip_url, directory: directory);
var args = new List <string> {
"http://www.libraryinsight.com/calendar.asp?jx=ea", "", "eastern"
};
var script_url = Configurator.elmcity_python_test_script_url;
var result = RunIronPython(directory, script_url, args);
GenUtils.LogMsg("status", "result of python elmcity install test", result, ts);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "InstallPythonElmcityLibrary", e.Message + e.StackTrace);
}
}
}
// try to insert a dict<str,obj> into table store
// if conflict, try to merge or update
public static TableStorageListDictResponse DictObjToTableStore(Operation operation, Dictionary <string, object> dict, string table, string partkey, string rowkey)
{
TableStorage ts = MakeDefaultTableStorage();
var entity = new Dictionary <string, object>();
entity.Add("PartitionKey", partkey);
entity.Add("RowKey", rowkey);
foreach (var key in dict.Keys)
{
if (key != "PartitionKey" && key != "RowKey")
{
entity.Add(key, dict[key]);
}
}
var response = ts.InsertEntity(table, entity);
if (response.http_response.status != HttpStatusCode.Created)
{
switch (operation)
{
case Operation.update:
response = ts.UpdateEntity(table, partkey, rowkey, entity);
break;
case Operation.merge:
response = ts.MergeEntity(table, partkey, rowkey, entity);
break;
default:
GenUtils.LogMsg("warning", "DictToTableStore unexpected operation", operation.ToString());
break;
}
if (response.http_response.status != HttpStatusCode.NoContent)
{
GenUtils.PriorityLogMsg("error", "DictToTableStore: " + operation, response.http_response.status.ToString() + ", " + response.http_response.message);
}
}
return(response);
}
public static HttpResponse RetryHttpRequestExpectingStatus(HttpWebRequest request, HttpStatusCode expected_status, byte[] data, int wait_secs, int max_tries, TimeSpan timeout_secs)
{
try
{
return(GenUtils.Actions.Retry <HttpResponse>(
delegate()
{
try
{
return DoHttpWebRequest(request, data);
}
catch (Exception e)
{
string msg;
if (e.GetType() != typeof(NullReferenceException))
{
msg = "RetryHttpRequest " + e.ToString() + ": " + request.RequestUri;
GenUtils.LogMsg("warning", msg, null);
}
else
{
msg = request.RequestUri.ToString();
}
return new HttpResponse(HttpStatusCode.NotFound, msg, null, null);
}
},
CompletedIfStatusEqualsExpected,
completed_delegate_object: expected_status,
wait_secs: wait_secs,
max_tries: max_tries,
timeout_secs: timeout_secs));
}
catch (Exception e)
{
var msg = "RetryHttpRequest " + e.ToString() + ": " + request.RequestUri;
GenUtils.LogMsg("exception", msg, null);
return(new HttpResponse(HttpStatusCode.NotFound, msg, null, null));
}
}
public static string RunIronPython(string directory, string str_script_url, List <string> args)
{
GenUtils.LogMsg("status", "Utils.run_ironpython: " + str_script_url, args[0] + "," + args[1] + "," + args[2]);
//var app_domain_name = "ironpython";
string result = "";
try
{
/*
* string domain_id = app_domain_name;
* var setup = new AppDomainSetup();
* setup.ApplicationName = app_domain_name;
* setup.ApplicationBase = AppDomain.CurrentDomain.BaseDirectory;
* var python_domain = AppDomain.CreateDomain(domain_id, securityInfo: null, info: setup);
*/
var options = new Dictionary <string, object>();
options["LightweightScopes"] = true;
var python = Python.CreateEngine(options);
var paths = new List <string>();
paths.Add(directory + "Lib"); // standard python lib
paths.Add(directory + "Lib\\site-packages");
paths.Add(directory + "ElmcityLib"); // Elmcity python lib
GenUtils.LogMsg("status", "Utils.run_ironpython", String.Join(":", paths.ToArray()));
python.SetSearchPaths(paths);
var ipy_args = new IronPython.Runtime.List();
foreach (var item in args)
{
ipy_args.Add(item);
}
var s = HttpUtils.FetchUrl(new Uri(str_script_url)).DataAsString();
try
{
var common_script_url = BlobStorage.MakeAzureBlobUri("admin", "common.py", false);
var common_script = HttpUtils.FetchUrl(common_script_url).DataAsString();
s = s.Replace("#include common.py", common_script);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "RunIronPython: cannot #include common.py", e.Message);
}
var source = python.CreateScriptSourceFromString(s, SourceCodeKind.Statements);
var scope = python.CreateScope();
var sys = python.GetSysModule();
//sys.SetVariable("argv", new PythonArgs() { args = ipy_args } );
sys.SetVariable("argv", args);
source.Execute(scope);
try
{
result = scope.GetVariable("result").ToString();
}
catch
{
// GenUtils.LogMsg("info", "RunIronPython: " + str_script_url, "no result");
}
python.Runtime.Shutdown();
//AppDomain.Unload(python_domain);
}
catch (Exception e)
{
result = e.Message.ToString() + e.StackTrace.ToString();
}
return(result);
}
public static void LogRemovedItemToAzure(String key, Object o, CacheItemRemovedReason r)
{
GenUtils.LogMsg("status", "LogRemovedItemToAzure: " + key, r.ToString());
}
public string oAuthWebRequest(Method method, string url, string oauth_verifier, string post_data)
{
string outUrl = "";
string querystring = "";
string ret = "";
GenUtils.LogMsg("status", "oAuthWebRequest", url + ", (" + post_data + ")");
try
{
//Setup postData for signing.
//Add the postData to the querystring.
if (method == Method.POST || method == Method.DELETE)
{
if (post_data.Length > 0)
{
//Decode the parameters and re-encode using the oAuth UrlEncode method.
NameValueCollection qs = HttpUtility.ParseQueryString(post_data);
post_data = "";
foreach (string key in qs.AllKeys)
{
if (post_data.Length > 0)
{
post_data += "&";
}
qs[key] = HttpUtility.UrlDecode(qs[key]);
qs[key] = this.UrlEncode(qs[key]);
post_data += key + "=" + qs[key];
}
if (url.IndexOf("?") > 0)
{
url += "&";
}
else
{
url += "?";
}
url += post_data;
}
}
Uri uri = new Uri(url);
string nonce = this.GenerateNonce();
string timeStamp = this.GenerateTimeStamp();
//Generate Signature
string sig = this.GenerateSignature(
uri,
this.consumer_key,
this.consumer_secret,
this.token,
this.token_secret,
method.ToString(),
timeStamp,
nonce,
SignatureTypes.HMACSHA1,
out outUrl,
out querystring);
querystring += "&oauth_signature=" + this.UrlEncode(sig);
if (oauth_verifier != null)
{
querystring += "&oauth_verifier=" + oauth_verifier;
}
//Convert the querystring to postData
if (method == Method.POST || method == Method.DELETE)
{
post_data = querystring;
querystring = "";
}
if (querystring.Length > 0)
{
outUrl += "?";
}
GenUtils.LogMsg("status", "oAuthWebRequest calling helper with ", outUrl + ", " + querystring + ", (" + post_data + ")");
ret = OAuthWebRequestHelper(method, outUrl + querystring, post_data);
GenUtils.LogMsg("status", "oAuthWebRequestHelper got ", ret);
return(ret);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "oAuthWebRequest", e.Message + e.StackTrace);
return(String.Empty);
}
}
public void ReloadCounters()
{
GenUtils.LogMsg("status", "Monitor.ReloadCounters", null);
this.counters = Counters.GetCounters();
this.priority_log_triggers = GetPriorityLogTriggers(this.ts);
}
