// scan the cacheurls table, compare uris with counts > 0 to uris in cache, if match then evict uri and decrement count public static void MaybePurgeCache(ICache cache) { try { var purgeable_entities = FetchPurgeableCacheDicts(); GenUtils.LogMsg("status", String.Format("MaybePurgeCache: {0} purgeable entities", purgeable_entities.Count), null); foreach (var purgeable_entity in purgeable_entities) { var purgeable_cache_url = (string)purgeable_entity["cached_uri"]; if (cache[purgeable_cache_url] != null) { GenUtils.LogMsg("status", "MaybePurgeCache", purgeable_cache_url); cache.Remove(purgeable_cache_url); var obj = HttpUtils.FetchUrl(new Uri(purgeable_cache_url)); // rewarm the cache var count = (int)purgeable_entity["count"]; count -= 1; if (count < 0) { count = 0; GenUtils.LogMsg("warning", "CacheUtils.MaybePurgeCache", "count went subzero, reset to zero"); } purgeable_entity["count"] = count; var rowkey = TableStorage.MakeSafeRowkeyFromUrl(purgeable_cache_url); ts.UpdateEntity(cache_control_tablename, cache_control_partkey, rowkey, purgeable_entity); } } } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "MaybePurgeCache", e.Message + e.StackTrace); } }
// a snapshot combines a bunch of process-level things with a set of counters public static Dictionary <string, object> MakeSnapshot(CounterResponse counters) { var p = Process.GetCurrentProcess(); var dict = new Dictionary <string, object>(); dict["PartitionKey"] = "ProcessMonitor"; dict["RowKey"] = DateTime.Now.ToUniversalTime().Ticks.ToString(); // add general info dict["HostName"] = System.Net.Dns.GetHostName(); dict["ProcName"] = p.ProcessName; dict["ProcId"] = System.Diagnostics.Process.GetCurrentProcess().Id; // add process info dict["ThreadCount"] = p.Threads.Count; dict["TotalProcTime"] = p.TotalProcessorTime.Ticks; dict["TotalUserTime"] = p.UserProcessorTime.Ticks; dict["PagedMemSize"] = p.PagedMemorySize64; dict["NonPagedMemSize"] = p.NonpagedSystemMemorySize64; dict["PrivateMemSize"] = p.PrivateMemorySize64; dict["VirtMemSize"] = p.VirtualMemorySize64; dict["PeakWorkingSet"] = p.PeakWorkingSet64; dict["MinWorkingSet"] = p.MinWorkingSet; // add counter info if (counters == null) { counters = Counters.GetCounters(); } foreach (var key in counters.counter_objects.Keys) // prime the pump { var counter = counters.counter_objects[key]; try { counter.NextValue(); } catch { } } HttpUtils.Wait(1); foreach (var key in counters.counter_objects.Keys) // now read values { var counter = counters.counter_objects[key]; try { dict[key] = counter.NextValue(); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", string.Format("MakeSnapshotDict: {0}/{1}/{2}", key, counter.CategoryName, counter.CounterName), e.Message + e.StackTrace); } } return(dict); }
public static Object DictObjToObj(Dictionary <string, object> dict_obj, Type type) { var o = Activator.CreateInstance(type); // create object if (type.GetProperties() == null) { GenUtils.PriorityLogMsg("exception", "DictObjToObj: " + type.Name, "target type does not define properties"); return(o); } foreach (var key in dict_obj.Keys) { try // set properties { type.GetProperty(key).SetValue(o, dict_obj[key], index: null); } catch (NullReferenceException) { // this is normal since an azure table includes PartitionKey, RowKey, // and Timestamp which will not map into the object } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "DictObjToObj: " + type.Name, e.Message + e.StackTrace); } } return(o); }
public static bool CompletedIfContainerIsGone(BlobStorageResponse response, object o) { if (response == null) { return(false); } string container; try { container = (String)o; } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "CompletedIfContainerIsGone", e.Message + e.StackTrace); throw new Exception("CompletedObjectDelegateException"); } if (!ExistsContainer(container)) { return(true); } else { return(false); } }
public static HttpResponse DoStorageRequest(string method, Hashtable headers, byte[] data, string content_type, Uri uri) { System.Threading.Thread.Sleep(500); try { HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uri); request.Method = method; if (content_type != null) { request.ContentType = content_type; } if (data == null || data.Length == 0) { request.ContentLength = 0; } foreach (string key in headers.Keys) { request.Headers.Add(key, (string)headers[key]); } var response = HttpUtils.DoHttpWebRequest(request, data); return(response); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "DoStorageRequest: " + uri.ToString(), e.Message + e.InnerException.Message + e.StackTrace); return(new HttpResponse(HttpStatusCode.ServiceUnavailable, "PossibleAzureTimeout", null, new Dictionary <string, string>())); } }
public string GetAuthenticatedUserOrNull(HttpRequestBase request) { HttpCookie cookie; string session_id; try { cookie = request.Cookies[this.cookie_name.ToString()]; if (cookie == null) { return(null); } session_id = request.Cookies[cookie_name.ToString()].Value; var q = String.Format("$filter=PartitionKey eq 'sessions' and RowKey eq '{0}'", session_id); var results = ts.QueryAllEntitiesAsListDict("sessions", q, 0); if (results.list_dict_obj.Count > 0) { return((string)results.list_dict_obj[0][this.trusted_field.ToString()]); } else { return(null); } } catch (Exception e) { session_id = null; GenUtils.PriorityLogMsg("exception", "GetAuthenticatedUserOrNull:" + cookie_name + ":" + session_id + ":" + this.trusted_field, e.Message + e.StackTrace); return(null); } }
// called when there's a rule for an item in snapshot. eval the rule and if trigger fires log a priority message // todo: simplify this, it's way too verbose private void EvaluateTrigger(string key, object value, Dictionary <string, string> trigger_dict) { int snapshot_int; float snapshot_float; int trigger_int; float trigger_float; var type = trigger_dict["type"]; switch (type) { case "float": snapshot_float = (float)value; if (trigger_dict.ContainsKey("max")) { trigger_float = float.Parse(trigger_dict["max"]); if (snapshot_float > trigger_float) { GenUtils.PriorityLogMsg("warning", key, String.Format("snapshot ({0}) > trigger ({1})", snapshot_float, trigger_float)); } } if (trigger_dict.ContainsKey("min")) { trigger_float = float.Parse(trigger_dict["min"]); if (snapshot_float < trigger_float) { GenUtils.PriorityLogMsg("warning", key, String.Format("snapshot ({0}) < trigger ({1})", snapshot_float, trigger_float)); } } break; case "int": snapshot_int = Convert.ToInt32(value); if (trigger_dict.ContainsKey("max")) { trigger_int = Convert.ToInt32(trigger_dict["max"]); if (snapshot_int > trigger_int) { GenUtils.PriorityLogMsg("warning", key, String.Format("snapshot ({0}) > trigger ({1})", snapshot_int, trigger_int)); } } if (trigger_dict.ContainsKey("min")) { trigger_int = Convert.ToInt32(trigger_dict["min"]); if (snapshot_int < trigger_int) { GenUtils.PriorityLogMsg("warning", key, String.Format("snapshot ({0}) < trigger ({1})", snapshot_int, trigger_int)); } } break; default: GenUtils.LogMsg("warning", "MaybeWritePriorityLog", "unexpected type: " + type); break; } }
public string OAuthWebRequestHelper(Method method, string url, string post_data) { HttpWebRequest request = null; try { try { request = System.Net.WebRequest.Create(url) as HttpWebRequest; System.Diagnostics.Debug.Assert(request != null); request.Method = method.ToString(); request.ServicePoint.Expect100Continue = false; } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "OAuthWebRequestHelper", e.Message + e.StackTrace); throw new Exception("OAuthWebRequestHelperNoUrl"); } if (method == Method.POST || method == Method.DELETE) { request.ContentType = "application/x-www-form-urlencoded"; /* * //POST the data. * requestWriter = new StreamWriter(webRequest.GetRequestStream()); * try * { * requestWriter.Write(postData); * } * catch * { * throw; * } * finally * { * requestWriter.Close(); * requestWriter = null; * }*/ } var response = HttpUtils.DoHttpWebRequest(request, Encoding.UTF8.GetBytes(post_data)); request = null; return(response.DataAsString()); } catch (Exception e) { GenUtils.PriorityLogMsg("OAuthWebRequestHelper", method + " " + url, e.Message + e.StackTrace); return(null); } }
public static string DisplaySnapshotAsText() { try { var snapshot = Counters.MakeSnapshot(counters: null); return(FormatSnapshotAsText(snapshot)); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "Counters.DisplaySnapshotAsText", e.Message + e.StackTrace); return(e.Message + e.StackTrace); } }
public void StoreSnapshot(Dictionary <string, object> snapshot) { try { snapshot["PartitionKey"] = this.tablename; snapshot["RowKey"] = DateTime.Now.Ticks.ToString(); this.ts.InsertEntity(tablename, snapshot); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "StoreSnapshot", e.Message + e.StackTrace); } }
public static object DeserializeObjectFromUri(Uri uri) { try { var buffer = HttpUtils.FetchUrlNoCache(uri).bytes; return(DeserializeObjectFromBytes(buffer)); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "DeserializeObjectFromUri: " + uri.ToString(), e.Message); throw; } }
public static void FetchResponseBodyAndETagFromUri(Uri uri, Dictionary <string, object> dict) { try { var response = FetchUrl(uri); dict["response_body"] = (byte[])response.bytes; dict["ETag"] = response.headers["ETag"]; } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "FetchResponseBodyAndETagFromUri: " + uri.ToString(), e.Message); } }
public BlobStorageResponse MaybeDeleteContainer(string containername) { try { HttpResponse http_response = DoBlobStoreRequest(containername, blobname: null, method: "DELETE", headers: new Hashtable(), data: null, content_type: null, query_string: "?restype=container"); return(new BlobStorageResponse(http_response)); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "MaybeDeleteContainer: " + containername, e.Message + e.StackTrace); return(default(BlobStorageResponse)); } }
public bool ElmcityIdIsAuthorized(string id) { try { var q = String.Format("$filter=RowKey eq '{0}'", id); var list = ts.QueryAllEntitiesAsListDict(this.trusted_table.ToString(), q, 0).list_dict_obj; return(list.Count >= 1); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "ElmcityIdUsesForeignAuth:" + this.trusted_table.ToString() + ":" + id, e.Message); return(false); } }
public static HttpResponse DoHttpWebRequest(HttpWebRequest request, byte[] data) { if (request.UserAgent == null) { request.UserAgent = elmcity_user_agent; } try { if (data != null && data.Length > 0) { request.ContentLength = data.Length; var bw = new BinaryWriter(request.GetRequestStream()); bw.Write(data); bw.Flush(); bw.Close(); } } catch (Exception ex_write) { GenUtils.PriorityLogMsg("exception", "DoHttpWebRequest: writing data", ex_write.Message + ex_write.InnerException.Message + ex_write.StackTrace); return(new HttpResponse(HttpStatusCode.ServiceUnavailable, "DoHttpWebRequest cannot write", null, new Dictionary <string, string>())); } try { var response = (HttpWebResponse)request.GetResponse2(); var status = response.StatusCode; var message = response.StatusDescription; var headers = new Dictionary <string, string>(); foreach (string key in response.Headers.Keys) { headers[key] = response.Headers[key]; } byte[] return_data = GetResponseData(response); response.Close(); return(new HttpResponse(status, message, return_data, headers)); } catch (Exception ex_read) { GenUtils.PriorityLogMsg("exception", "DoHttpWebRequest: reading data", ex_read.Message + ex_read.InnerException.Message + ex_read.StackTrace); return(new HttpResponse(HttpStatusCode.ServiceUnavailable, "DoHttpWebRequest cannot read", null, new Dictionary <string, string>())); } }
public static Monitor TryStartMonitor(int interval_minutes, string tablename) { Monitor monitor = null; try { monitor = new Monitor(interval_minutes, tablename); monitor.StartMonitor(); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "StartMonitor", e.Message + e.StackTrace); } return(monitor); }
public Object Remove(string key) { Object o = null; try { o = this.cache.Remove(key); GenUtils.LogMsg("status", "AspNetCache.Remove", key); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "AspNetCache.Remove: " + key, e.Message + e.StackTrace); } return(o); }
public List <string> AuthenticatedElmcityIds(string foreign_id) { var q = String.Format("$filter={0} eq '{1}'", this.trusted_field, foreign_id); try { var list = ts.QueryAllEntitiesAsListDict(this.trusted_table.ToString(), q, 0).list_dict_obj; return(list.Select(x => (string)x["RowKey"]).ToList()); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "AuthenticatedElmcityIds: " + this.trusted_table + ":" + foreign_id + ":" + this.trusted_field, e.Message); return(null); } }
public string AuthorizationLinkGet() { string ret = null; string response; try { response = this.oAuthWebRequest(Method.GET, REQUEST_TOKEN, null, String.Empty); if (response.Length > 0) { //response contains token and token secret. We only need the token. NameValueCollection qs = HttpUtility.ParseQueryString(response); string oauth_verifier = qs["oauth_verifier"]; if (oauth_verifier == null) { var msg = REQUEST_TOKEN + " sent back null oauth_verifier"; GenUtils.PriorityLogMsg("AuthorizationLinkGet", msg, null); // throw new Exception(msg); is it optional } if (qs["oauth_callback_confirmed"] != null && qs["oauth_callback_confirmed"] != "true") { var msg = REQUEST_TOKEN + " auth_callback_confirmed not true"; GenUtils.PriorityLogMsg("AuthorizationLinkGet", msg, null); throw new Exception(msg); } if (qs["oauth_token"] != null) { ret = AUTHORIZE + "?oauth_token=" + qs["oauth_token"] + "&oauth_verifier=" + qs["oauth_verifier"]; } else { var msg = REQUEST_TOKEN + " oauth_token_is_null"; GenUtils.PriorityLogMsg("AuthorizationLinkGet", msg, null); throw new Exception(msg); } } } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "AuthorizationLinkGet", e.Message); } return(ret); }
public void ProcessMonitorThreadMethod() { while (true) { try { GenUtils.LogMsg("status", "ProcessMonitorThreadMethod", "snapshot"); this.ReloadCounters(); var snapshot = Counters.MakeSnapshot(counters); this.StoreSnapshot(snapshot); this.MaybeWritePriorityLog(snapshot); Thread.Sleep(TimeSpan.FromMinutes(this.interval_minutes)); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "ProcessMonitorThreadMethod: snapshot", e.Message + e.StackTrace); } } }
public static void InstallPythonStandardLibrary(string directory, TableStorage ts) { GenUtils.PriorityLogMsg("info", "installing python standard library", null); try { var zip_url = Configurator.pylib_zip_url; FileUtils.UnzipFromUrlToDirectory(zip_url, directory); var args = new List <string> { "", "", "" }; var script_url = Configurator.python_test_script_url; var result = RunIronPython(directory, script_url, args); GenUtils.LogMsg("status", "result of python standard lib install test", result); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "InstallPythonStandardLibrary", e.Message + e.StackTrace); } }
// check to see if any of the conditions in prioritylogtriggers is met by an entry in the snapshot public void MaybeWritePriorityLog(Dictionary <string, object> snapshot) { try { foreach (var trigger_dict in priority_log_triggers) { var key = trigger_dict["RowKey"]; var is_min_trigger = trigger_dict.ContainsKey("min"); var is_max_trigger = trigger_dict.ContainsKey("max"); if (snapshot.ContainsKey(key)) { EvaluateTrigger(key, snapshot[key], trigger_dict); } } } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "MaybeWritePriorityLog", e.Message + e.StackTrace); } }
public static bool SavedJsonSnapshot(string id, JsonSnapshotType type, string name, Object new_obj) { try { var json_blob_name = id + "." + name + ".json"; var existing_obj_uri = BlobStorage.MakeAzureBlobUri(id, json_blob_name, false); var exists = BlobStorage.ExistsBlob(existing_obj_uri); JsonCompareResult comparison = NewJsonMatchesExistingJson(type, new_obj, existing_obj_uri); if (!exists || comparison == JsonCompareResult.different) { var bs = BlobStorage.MakeDefaultBlobStorage(); var timestamped_json_blob_name = string.Format(id + "." + string.Format("{0:yyyy.MM.dd.HH.mm}" + "." + name + ".json", DateTime.UtcNow)); var timestamped_dict_uri = BlobStorage.MakeAzureBlobUri(id, timestamped_json_blob_name, false); string new_obj_as_json; if (type == JsonSnapshotType.DictStr) { new_obj_as_json = ObjectUtils.DictStrToJson((Dictionary <string, string>)new_obj); } else // JsonSnapshotType.ListDictStr { new_obj_as_json = ObjectUtils.ListDictStrToJson((List <Dictionary <string, string> >)new_obj); } //bs.PutBlob(id, json_blob_name, new_obj_as_json, "application/json"); bs.PutBlob(id, timestamped_json_blob_name, new_obj_as_json, "application/json"); } if (comparison == JsonCompareResult.same || comparison == JsonCompareResult.invalid) { return(false); // either the objects matched, or the comparison failed, either way a json snapshot was not saved } else { return(true); // the objects did not match, a json snapshot was saved } } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "SavedJsonSnapshot", e.Message + e.StackTrace); return(false); } }
public static XDocument XdocFromXmlBytes(byte[] xml) { var xdoc = new XDocument(); if (xml.Length == 0) { return(xdoc); } var ms = new MemoryStream(xml); var xr = XmlReader.Create(ms); try { xdoc = XDocument.Load(xr); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "XdocFromXmlBytes", e.Message + e.StackTrace); } return(xdoc); }
public static HttpWebResponse GetResponse2(this WebRequest request) { HttpWebResponse response; try { response = request.GetResponse() as HttpWebResponse; } catch (WebException ex) { response = ex.Response as HttpWebResponse; } catch (Exception e) { var msg = "GetResponse2" + " " + e.Message; GenUtils.PriorityLogMsg("exception", msg, e.StackTrace); throw new Exception(msg); } return(response); }
// todo: externalize test url as a setting public static void InstallPythonElmcityLibrary(string directory, TableStorage ts) { { GenUtils.LogMsg("status", "installing python elmcity library", null); try { var zip_url = Configurator.elmcity_pylib_zip_url; FileUtils.UnzipFromUrlToDirectory(zip_url, directory: directory); var args = new List <string> { "http://www.libraryinsight.com/calendar.asp?jx=ea", "", "eastern" }; var script_url = Configurator.elmcity_python_test_script_url; var result = RunIronPython(directory, script_url, args); GenUtils.LogMsg("status", "result of python elmcity install test", result, ts); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "InstallPythonElmcityLibrary", e.Message + e.StackTrace); } } }
private static byte[] GetResponseData(HttpWebResponse response) { try { //NUnit.Framework.Assert.IsNotNull(response); Stream response_stream = response.GetResponseStream(); if (response_stream.CanRead == false) { return(new byte[0]); } //NUnit.Framework.Assert.IsNotNull(response_stream); Encoding encoding; var charset = response.CharacterSet ?? ""; switch (charset.ToLower()) { case "utf-8": encoding = Encoding.UTF8; break; default: encoding = Encoding.Default; break; } if (response.ContentLength > 0) { var reader = new BinaryReader(response_stream); return(reader.ReadBytes((int)response.ContentLength)); } else // empty or unspecified, read zero or more bytes { var reader = new StreamReader(response_stream); return(encoding.GetBytes(reader.ReadToEnd())); } } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "HttpUtils.GetResponseData", e.Message + e.StackTrace); throw (e); } }
public static T Retry <T>(RetryDelegate <T> Action, CompletedDelegate <T, Object> Completed, Object completed_delegate_object, int wait_secs, int max_tries, TimeSpan timeout_secs) { T result = default(T); DateTime start = DateTime.UtcNow; int tries = 0; var method_name = Action.Method.Name; while (TimeToGiveUp(start, timeout_secs, tries, max_tries) == false) { try { tries++; result = Action.Invoke(); if (Completed(result, completed_delegate_object) == true) { return(result); } } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "RetryDelegate: " + method_name, e.Message + e.StackTrace); throw (e); } HttpUtils.Wait(wait_secs); } if (TimedOut(start, timeout_secs)) { throw RetryTimedOut; } if (ExceededTries(tries, max_tries)) { throw RetryExceededMaxTries; } return(result); // default(T) }
// try to insert a dict<str,obj> into table store // if conflict, try to merge or update public static TableStorageListDictResponse DictObjToTableStore(Operation operation, Dictionary <string, object> dict, string table, string partkey, string rowkey) { TableStorage ts = MakeDefaultTableStorage(); var entity = new Dictionary <string, object>(); entity.Add("PartitionKey", partkey); entity.Add("RowKey", rowkey); foreach (var key in dict.Keys) { if (key != "PartitionKey" && key != "RowKey") { entity.Add(key, dict[key]); } } var response = ts.InsertEntity(table, entity); if (response.http_response.status != HttpStatusCode.Created) { switch (operation) { case Operation.update: response = ts.UpdateEntity(table, partkey, rowkey, entity); break; case Operation.merge: response = ts.MergeEntity(table, partkey, rowkey, entity); break; default: GenUtils.LogMsg("warning", "DictToTableStore unexpected operation", operation.ToString()); break; } if (response.http_response.status != HttpStatusCode.NoContent) { GenUtils.PriorityLogMsg("error", "DictToTableStore: " + operation, response.http_response.status.ToString() + ", " + response.http_response.message); } } return(response); }
public BlobStorageResponse PutBlobWithLease(string container, string blobname, Hashtable headers, byte[] data, string content_type) { try { if (BlobStorage.ExistsBlob(container, blobname)) { var r = this.RetryAcquireLease(container, blobname); if (r.status == HttpStatusCode.Created) { headers.Add("x-ms-lease-id", r.headers["x-ms-lease-id"]); } else { GenUtils.PriorityLogMsg("warning", "PutBlobWithLease: Did not acquire lease for: ", container + ", " + blobname); } } } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "PutBlobWithLease", e.Message + e.StackTrace); } return(this.PutBlob(container, blobname, headers, data, content_type)); }