public static HttpResponse DoStorageRequest(string method, Hashtable headers, byte[] data, string content_type, Uri uri)
{
System.Threading.Thread.Sleep(500);
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uri);
request.Method = method;
if (content_type != null)
{
request.ContentType = content_type;
}
if (data == null || data.Length == 0)
{
request.ContentLength = 0;
}
foreach (string key in headers.Keys)
{
request.Headers.Add(key, (string)headers[key]);
}
var response = HttpUtils.DoHttpWebRequest(request, data);
return(response);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "DoStorageRequest: " + uri.ToString(), e.Message + e.InnerException.Message + e.StackTrace);
return(new HttpResponse(HttpStatusCode.ServiceUnavailable, "PossibleAzureTimeout", null, new Dictionary <string, string>()));
}
}
public static HttpResponse RetryStorageRequestExpectingServiceAvailable(string method, Hashtable headers, byte[] data, string content_type, Uri uri, int wait_secs, int max_tries, TimeSpan timeout_secs)
{
try
{
return(GenUtils.Actions.Retry <HttpResponse>(
delegate()
{
try
{
return StorageUtils.DoStorageRequest(method, headers, data, content_type, uri);
}
catch // (Exception e)
{
//GenUtils.PriorityLogMsg("exception", "RetryStorageRequest: " + uri, e.Message + e.StackTrace);
throw new Exception("RetryStorageRequestException");
}
},
CompletedIfStatusNotServiceUnavailable,
completed_delegate_object: null,
wait_secs: wait_secs,
max_tries: max_tries,
timeout_secs: timeout_secs));
}
catch (Exception e)
{
GenUtils.LogMsg("exception", "RetryStorageRequest: " + uri, e.Message + e.StackTrace);
return(new HttpResponse(HttpStatusCode.ServiceUnavailable, uri.ToString(), null, new Dictionary <string, string>()));
}
}
public static bool CompletedIfContainerIsGone(BlobStorageResponse response, object o)
{
if (response == null)
{
return(false);
}
string container;
try
{
container = (String)o;
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "CompletedIfContainerIsGone", e.Message + e.StackTrace);
throw new Exception("CompletedObjectDelegateException");
}
if (!ExistsContainer(container))
{
return(true);
}
else
{
return(false);
}
}
public string GetAuthenticatedUserOrNull(HttpRequestBase request)
{
HttpCookie cookie;
string session_id;
try
{
cookie = request.Cookies[this.cookie_name.ToString()];
if (cookie == null)
{
return(null);
}
session_id = request.Cookies[cookie_name.ToString()].Value;
var q = String.Format("$filter=PartitionKey eq 'sessions' and RowKey eq '{0}'", session_id);
var results = ts.QueryAllEntitiesAsListDict("sessions", q, 0);
if (results.list_dict_obj.Count > 0)
{
return((string)results.list_dict_obj[0][this.trusted_field.ToString()]);
}
else
{
return(null);
}
}
catch (Exception e)
{
session_id = null;
GenUtils.PriorityLogMsg("exception", "GetAuthenticatedUserOrNull:" + cookie_name + ":" + session_id + ":" + this.trusted_field, e.Message + e.StackTrace);
return(null);
}
}
public void FindsThreeGroupsLiteral()
{
var pat = "a (b) c (d) e";
var groups = GenUtils.RegexFindGroups("a b c d e", pat);
Assert.AreEqual(3, groups.Count);
}
public static Object DictObjToObj(Dictionary <string, object> dict_obj, Type type)
{
var o = Activator.CreateInstance(type); // create object
if (type.GetProperties() == null)
{
GenUtils.PriorityLogMsg("exception", "DictObjToObj: " + type.Name,
"target type does not define properties");
return(o);
}
foreach (var key in dict_obj.Keys)
{
try // set properties
{
type.GetProperty(key).SetValue(o, dict_obj[key], index: null);
}
catch (NullReferenceException)
{
// this is normal since an azure table includes PartitionKey, RowKey,
// and Timestamp which will not map into the object
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "DictObjToObj: " + type.Name,
e.Message + e.StackTrace);
}
}
return(o);
}
public void StartMonitor()
{
GenUtils.LogMsg("status", "StartMonitor", "starting");
var ProcessMonitorThread = new Thread(new ThreadStart(ProcessMonitorThreadMethod));
ProcessMonitorThread.Start();
}
public void DoesNotFindThreeGroupsLiteral()
{
var pat = "a (b) c (d) e";
var groups = GenUtils.RegexFindGroups("a b c D e", pat);
Assert.AreNotEqual(3, groups.Count);
}
public void DoesNotFindThreeGroupsAbstract()
{
var pat = @"a (http://.+\s*) c (\d+) e";
var groups = GenUtils.RegexFindGroups("a ftp://foo.com?x=y c 19423 e", pat);
Assert.AreNotEqual(3, groups.Count);
}
// scan the cacheurls table, compare uris with counts > 0 to uris in cache, if match then evict uri and decrement count
public static void MaybePurgeCache(ICache cache)
{
try
{
var purgeable_entities = FetchPurgeableCacheDicts();
GenUtils.LogMsg("status", String.Format("MaybePurgeCache: {0} purgeable entities", purgeable_entities.Count), null);
foreach (var purgeable_entity in purgeable_entities)
{
var purgeable_cache_url = (string)purgeable_entity["cached_uri"];
if (cache[purgeable_cache_url] != null)
{
GenUtils.LogMsg("status", "MaybePurgeCache", purgeable_cache_url);
cache.Remove(purgeable_cache_url);
var obj = HttpUtils.FetchUrl(new Uri(purgeable_cache_url)); // rewarm the cache
var count = (int)purgeable_entity["count"];
count -= 1;
if (count < 0)
{
count = 0;
GenUtils.LogMsg("warning", "CacheUtils.MaybePurgeCache", "count went subzero, reset to zero");
}
purgeable_entity["count"] = count;
var rowkey = TableStorage.MakeSafeRowkeyFromUrl(purgeable_cache_url);
ts.UpdateEntity(cache_control_tablename, cache_control_partkey, rowkey, purgeable_entity);
}
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "MaybePurgeCache", e.Message + e.StackTrace);
}
}
// a snapshot combines a bunch of process-level things with a set of counters
public static Dictionary <string, object> MakeSnapshot(CounterResponse counters)
{
var p = Process.GetCurrentProcess();
var dict = new Dictionary <string, object>();
dict["PartitionKey"] = "ProcessMonitor";
dict["RowKey"] = DateTime.Now.ToUniversalTime().Ticks.ToString();
// add general info
dict["HostName"] = System.Net.Dns.GetHostName();
dict["ProcName"] = p.ProcessName;
dict["ProcId"] = System.Diagnostics.Process.GetCurrentProcess().Id;
// add process info
dict["ThreadCount"] = p.Threads.Count;
dict["TotalProcTime"] = p.TotalProcessorTime.Ticks;
dict["TotalUserTime"] = p.UserProcessorTime.Ticks;
dict["PagedMemSize"] = p.PagedMemorySize64;
dict["NonPagedMemSize"] = p.NonpagedSystemMemorySize64;
dict["PrivateMemSize"] = p.PrivateMemorySize64;
dict["VirtMemSize"] = p.VirtualMemorySize64;
dict["PeakWorkingSet"] = p.PeakWorkingSet64;
dict["MinWorkingSet"] = p.MinWorkingSet;
// add counter info
if (counters == null)
{
counters = Counters.GetCounters();
}
foreach (var key in counters.counter_objects.Keys) // prime the pump
{
var counter = counters.counter_objects[key];
try
{ counter.NextValue(); }
catch
{ }
}
HttpUtils.Wait(1);
foreach (var key in counters.counter_objects.Keys) // now read values
{
var counter = counters.counter_objects[key];
try
{
dict[key] = counter.NextValue();
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", string.Format("MakeSnapshotDict: {0}/{1}/{2}", key, counter.CategoryName, counter.CounterName), e.Message + e.StackTrace);
}
}
return(dict);
}
public Logger()
{
this.ts = TableStorage.MakeDefaultTableStorage();
this.settings = GenUtils.GetSettingsFromAzureTable();
this.loglevel = Convert.ToInt32(settings["loglevel"]);
this.Start();
}
public static string ListDictStrToJson(List <Dictionary <string, string> > list_dict_str)
{
var serializer = new System.Web.Script.Serialization.JavaScriptSerializer();
var json = serializer.Serialize(list_dict_str);
return(GenUtils.PrettifyJson(json));
}
// called when there's a rule for an item in snapshot. eval the rule and if trigger fires log a priority message
// todo: simplify this, it's way too verbose
private void EvaluateTrigger(string key, object value, Dictionary <string, string> trigger_dict)
{
int snapshot_int;
float snapshot_float;
int trigger_int;
float trigger_float;
var type = trigger_dict["type"];
switch (type)
{
case "float":
snapshot_float = (float)value;
if (trigger_dict.ContainsKey("max"))
{
trigger_float = float.Parse(trigger_dict["max"]);
if (snapshot_float > trigger_float)
{
GenUtils.PriorityLogMsg("warning", key, String.Format("snapshot ({0}) > trigger ({1})", snapshot_float, trigger_float));
}
}
if (trigger_dict.ContainsKey("min"))
{
trigger_float = float.Parse(trigger_dict["min"]);
if (snapshot_float < trigger_float)
{
GenUtils.PriorityLogMsg("warning", key, String.Format("snapshot ({0}) < trigger ({1})", snapshot_float, trigger_float));
}
}
break;
case "int":
snapshot_int = Convert.ToInt32(value);
if (trigger_dict.ContainsKey("max"))
{
trigger_int = Convert.ToInt32(trigger_dict["max"]);
if (snapshot_int > trigger_int)
{
GenUtils.PriorityLogMsg("warning", key, String.Format("snapshot ({0}) > trigger ({1})", snapshot_int, trigger_int));
}
}
if (trigger_dict.ContainsKey("min"))
{
trigger_int = Convert.ToInt32(trigger_dict["min"]);
if (snapshot_int < trigger_int)
{
GenUtils.PriorityLogMsg("warning", key, String.Format("snapshot ({0}) < trigger ({1})", snapshot_int, trigger_int));
}
}
break;
default:
GenUtils.LogMsg("warning", "MaybeWritePriorityLog", "unexpected type: " + type);
break;
}
}
public string OAuthWebRequestHelper(Method method, string url, string post_data)
{
HttpWebRequest request = null;
try
{
try
{
request = System.Net.WebRequest.Create(url) as HttpWebRequest;
System.Diagnostics.Debug.Assert(request != null);
request.Method = method.ToString();
request.ServicePoint.Expect100Continue = false;
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "OAuthWebRequestHelper", e.Message + e.StackTrace);
throw new Exception("OAuthWebRequestHelperNoUrl");
}
if (method == Method.POST || method == Method.DELETE)
{
request.ContentType = "application/x-www-form-urlencoded";
/*
* //POST the data.
* requestWriter = new StreamWriter(webRequest.GetRequestStream());
* try
* {
* requestWriter.Write(postData);
* }
* catch
* {
* throw;
* }
* finally
* {
* requestWriter.Close();
* requestWriter = null;
* }*/
}
var response = HttpUtils.DoHttpWebRequest(request, Encoding.UTF8.GetBytes(post_data));
request = null;
return(response.DataAsString());
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("OAuthWebRequestHelper", method + " " + url, e.Message + e.StackTrace);
return(null);
}
}
public Object Remove(string key)
{
Object value = null;
if (this.cache.ContainsKey(key))
{
value = this.cache[key];
this.cache.Remove(key);
GenUtils.LogMsg("status", "MockCache.Remove", key);
}
return(value);
}
public static HttpResponse RetryStorageRequestExpectingServiceAvailable(string method, Hashtable headers, byte[] data, string content_type, Uri uri)
{
try
{
return(RetryStorageRequestExpectingServiceAvailable(method, headers, data, content_type, uri, 60, 3, TimeSpan.FromSeconds(500)));
}
catch (Exception e)
{
GenUtils.LogMsg("exception", "RetryStorageRequest: " + uri, e.Message + e.StackTrace);
return(new HttpResponse(HttpStatusCode.ServiceUnavailable, uri.ToString(), null, new Dictionary <string, string>()));
}
}
public static string DisplaySnapshotAsText()
{
try
{
var snapshot = Counters.MakeSnapshot(counters: null);
return(FormatSnapshotAsText(snapshot));
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "Counters.DisplaySnapshotAsText", e.Message + e.StackTrace);
return(e.Message + e.StackTrace);
}
}
public static void FetchResponseBodyAndETagFromUri(Uri uri, Dictionary <string, object> dict)
{
try
{
var response = FetchUrl(uri);
dict["response_body"] = (byte[])response.bytes;
dict["ETag"] = response.headers["ETag"];
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "FetchResponseBodyAndETagFromUri: " + uri.ToString(), e.Message);
}
}
public static object DeserializeObjectFromUri(Uri uri)
{
try
{
var buffer = HttpUtils.FetchUrlNoCache(uri).bytes;
return(DeserializeObjectFromBytes(buffer));
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "DeserializeObjectFromUri: " + uri.ToString(), e.Message);
throw;
}
}
public void StoreSnapshot(Dictionary <string, object> snapshot)
{
try
{
snapshot["PartitionKey"] = this.tablename;
snapshot["RowKey"] = DateTime.Now.Ticks.ToString();
this.ts.InsertEntity(tablename, snapshot);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "StoreSnapshot", e.Message + e.StackTrace);
}
}
public BlobStorageResponse MaybeDeleteContainer(string containername)
{
try
{
HttpResponse http_response = DoBlobStoreRequest(containername, blobname: null, method: "DELETE", headers: new Hashtable(), data: null, content_type: null, query_string: "?restype=container");
return(new BlobStorageResponse(http_response));
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "MaybeDeleteContainer: " + containername, e.Message + e.StackTrace);
return(default(BlobStorageResponse));
}
}
public static string EscapeValueForCsv(this string text)
{
try
{
text = text.Replace("\"", "\"\"");
text = text.Replace("\n", "\\n");
return(text);
}
catch (Exception e)
{
GenUtils.LogMsg("warning", "EscapeValueForCsv", e.Message);
return("");
}
}
public bool ElmcityIdIsAuthorized(string id)
{
try
{
var q = String.Format("$filter=RowKey eq '{0}'", id);
var list = ts.QueryAllEntitiesAsListDict(this.trusted_table.ToString(), q, 0).list_dict_obj;
return(list.Count >= 1);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "ElmcityIdUsesForeignAuth:" + this.trusted_table.ToString() + ":" + id, e.Message);
return(false);
}
}
public static HttpResponse DoHttpWebRequest(HttpWebRequest request, byte[] data)
{
if (request.UserAgent == null)
{
request.UserAgent = elmcity_user_agent;
}
try
{
if (data != null && data.Length > 0)
{
request.ContentLength = data.Length;
var bw = new BinaryWriter(request.GetRequestStream());
bw.Write(data);
bw.Flush();
bw.Close();
}
}
catch (Exception ex_write)
{
GenUtils.PriorityLogMsg("exception", "DoHttpWebRequest: writing data", ex_write.Message + ex_write.InnerException.Message + ex_write.StackTrace);
return(new HttpResponse(HttpStatusCode.ServiceUnavailable, "DoHttpWebRequest cannot write", null, new Dictionary <string, string>()));
}
try
{
var response = (HttpWebResponse)request.GetResponse2();
var status = response.StatusCode;
var message = response.StatusDescription;
var headers = new Dictionary <string, string>();
foreach (string key in response.Headers.Keys)
{
headers[key] = response.Headers[key];
}
byte[] return_data = GetResponseData(response);
response.Close();
return(new HttpResponse(status, message, return_data, headers));
}
catch (Exception ex_read)
{
GenUtils.PriorityLogMsg("exception", "DoHttpWebRequest: reading data", ex_read.Message + ex_read.InnerException.Message + ex_read.StackTrace);
return(new HttpResponse(HttpStatusCode.ServiceUnavailable, "DoHttpWebRequest cannot read", null, new Dictionary <string, string>()));
}
}
public static XmlDocument XmlDocumentFromXmlBytes(byte[] xml_bytes)
{
var sr = new MemoryStream(xml_bytes);
XmlDocument doc = new XmlDocument();
try
{
doc.Load(sr);
}
catch (Exception e)
{
GenUtils.LogMsg("exception", "XmlDocumentFromHttpResponse", e.Message + e.StackTrace);
}
return(doc);
}
public static Monitor TryStartMonitor(int interval_minutes, string tablename)
{
Monitor monitor = null;
try
{
monitor = new Monitor(interval_minutes, tablename);
monitor.StartMonitor();
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "StartMonitor", e.Message + e.StackTrace);
}
return(monitor);
}
public static string UrlsToLinks(this string text)
{
try
{ // http://rickyrosario.com/blog/converting-a-url-into-a-link-in-csharp-using-regular-expressions/
string regex = @"((www\.|(http|https|ftp|news|file)+\:\/\/)[_a-z0-9-]+\.[_a-z0-9\/+:@=.+?,##%&~-]*[^.|\'|\# |!|\(|?|,| |>|<|;|\)])";
Regex r = new Regex(regex, RegexOptions.IgnoreCase);
return(r.Replace(text, "<a href=\"$1\" title=\"Click to open in a new window or tab\" target=\"_blank\">$1</a>").Replace("href=\"www", "href=\"http://www"));
}
catch (Exception e)
{
GenUtils.LogMsg("exception", "UrlsToLinks: " + text, e.Message);
return(text);
}
}
public List <string> AuthenticatedElmcityIds(string foreign_id)
{
var q = String.Format("$filter={0} eq '{1}'", this.trusted_field, foreign_id);
try
{
var list = ts.QueryAllEntitiesAsListDict(this.trusted_table.ToString(), q, 0).list_dict_obj;
return(list.Select(x => (string)x["RowKey"]).ToList());
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "AuthenticatedElmcityIds: " + this.trusted_table + ":" + foreign_id + ":" + this.trusted_field, e.Message);
return(null);
}
}
public Object Remove(string key)
{
Object o = null;
try
{
o = this.cache.Remove(key);
GenUtils.LogMsg("status", "AspNetCache.Remove", key);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "AspNetCache.Remove: " + key, e.Message + e.StackTrace);
}
return(o);
}
