public Logger(int milliseconds, int max_messages, TableStorage ts)
{
this.ts = ts;
this.wait_milliseconds = milliseconds;
this.max_messages = max_messages;
this.Start();
}
public static void InstallPythonStandardLibrary(string directory, TableStorage ts)
{
GenUtils.PriorityLogMsg("info", "installing python standard library", null);
try
{
var zip_url = Configurator.pylib_zip_url;
FileUtils.UnzipFromUrlToDirectory(zip_url, directory);
var args = new List<string> { "", "", "" };
var script_url = Configurator.python_test_script_url;
var result = RunIronPython(directory, script_url, args);
GenUtils.LogMsg("status", "result of python standard lib install test", result);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "InstallPythonStandardLibrary", e.Message + e.StackTrace);
}
}
// this query expects to find just one matching entity
public static Dictionary <string, object> QueryForSingleEntityAsDictObj(TableStorage ts, string table, string q)
{
var ts_response = ts.QueryEntities(table, q);
var dicts = ts_response.list_dict_obj;
var dict = new Dictionary <string, object>();
if (dicts.Count > 0)
{
dict = dicts.FirstOrDefault();
}
if (dicts.Count > 1)
{
// should not happen, but...
GenUtils.LogMsg("warning", "QueryForSingleEntity: " + table, q + ": more than one matching entity");
}
return(dict);
}
public static void InstallPythonStandardLibrary(string directory, TableStorage ts)
{
GenUtils.PriorityLogMsg("info", "installing python standard library", null);
try
{
var zip_url = Configurator.pylib_zip_url;
FileUtils.UnzipFromUrlToDirectory(zip_url, directory);
var args = new List <string> {
"", "", ""
};
var script_url = Configurator.python_test_script_url;
var result = RunIronPython(directory, script_url, args);
GenUtils.LogMsg("status", "result of python standard lib install test", result);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "InstallPythonStandardLibrary", e.Message + e.StackTrace);
}
}
// todo: externalize test url as a setting
public static void InstallPythonElmcityLibrary(string directory, TableStorage ts)
{
{
GenUtils.LogMsg("status", "installing python elmcity library", null);
try
{
var zip_url = Configurator.elmcity_pylib_zip_url;
FileUtils.UnzipFromUrlToDirectory(zip_url, directory: directory);
var args = new List<string> { "http://www.libraryinsight.com/calendar.asp?jx=ea", "", "eastern" };
var script_url = Configurator.elmcity_python_test_script_url;
var result = RunIronPython(directory, script_url, args);
GenUtils.LogMsg("status", "result of python elmcity install test", result, ts);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "InstallPythonElmcityLibrary", e.Message + e.StackTrace);
}
}
}
// todo: externalize test url as a setting
public static void InstallPythonElmcityLibrary(string directory, TableStorage ts)
{
{
GenUtils.LogMsg("status", "installing python elmcity library", null);
try
{
var zip_url = Configurator.elmcity_pylib_zip_url;
FileUtils.UnzipFromUrlToDirectory(zip_url, directory: directory);
var args = new List <string> {
"http://www.libraryinsight.com/calendar.asp?jx=ea", "", "eastern"
};
var script_url = Configurator.elmcity_python_test_script_url;
var result = RunIronPython(directory, script_url, args);
GenUtils.LogMsg("status", "result of python elmcity install test", result, ts);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "InstallPythonElmcityLibrary", e.Message + e.StackTrace);
}
}
}
public TableStorageListDictResponse QueryAllEntitiesAsListDict(string table, string query, int max)
{
var list_dict_obj = new List <Dictionary <string, object> >();
HttpResponse last_http_response = default(HttpResponse);
foreach (HttpResponse http_response in QueryAll(table, query))
{
last_http_response = http_response;
var response_dicts = TableStorage.GetTsDicts(http_response);
foreach (var response_dict in response_dicts)
{
list_dict_obj.Add(response_dict);
if (max != 0 && list_dict_obj.Count >= max)
{
break;
}
}
}
return(new TableStorageListDictResponse(last_http_response, list_dict_obj));
}
public void QueryForSingleEntityIsSuccessful()
{
if (ts.ExistsEntity(test_table, test_query) == false)
{
CreateEntityIsSuccessful();
}
var ts_response = ts.QueryEntities(test_table, test_query);
var dicts = ts_response.list_dict_obj;
Assert.That(dicts.Count == 1);
if (ts.ExistsEntity(test_table, test_query) == false)
{
CreateEntityIsSuccessful();
}
var multi_response_dict = ObjectUtils.DictObjToDictStr(dicts[0]);
var single_response_dict = TableStorage.QueryForSingleEntityAsDictStr(ts, test_table, test_query);
Assert.That(multi_response_dict["TestInt32"] == single_response_dict["TestInt32"]);
Assert.That(multi_response_dict["TestInt64"] == single_response_dict["TestInt64"]);
Assert.That(multi_response_dict["TestBool"] == single_response_dict["TestBool"]);
}
// try to insert a dict<str,obj> into table store
// if conflict, try to merge or update
public static TableStorageListDictResponse DictObjToTableStore(Operation operation, Dictionary <string, object> dict, string table, string partkey, string rowkey)
{
TableStorage ts = MakeDefaultTableStorage();
var entity = new Dictionary <string, object>();
entity.Add("PartitionKey", partkey);
entity.Add("RowKey", rowkey);
foreach (var key in dict.Keys)
{
if (key != "PartitionKey" && key != "RowKey")
{
entity.Add(key, dict[key]);
}
}
var response = ts.InsertEntity(table, entity);
if (response.http_response.status != HttpStatusCode.Created)
{
switch (operation)
{
case Operation.update:
response = ts.UpdateEntity(table, partkey, rowkey, entity);
break;
case Operation.merge:
response = ts.MergeEntity(table, partkey, rowkey, entity);
break;
default:
GenUtils.LogMsg("warning", "DictToTableStore unexpected operation", operation.ToString());
break;
}
if (response.http_response.status != HttpStatusCode.NoContent)
{
GenUtils.PriorityLogMsg("error", "DictToTableStore: " + operation, response.http_response.status.ToString() + ", " + response.http_response.message);
}
}
return(response);
}
private static bool HandleFeedUpdates(string id, TableStorage ts, List<Dictionary<string, string>> list_metadict_str, string feed_entry_schema, List<string> current_feed_urls, List<string> deleted_feed_urls)
{
List<string> maybe_updated_feed_urls = (List<string>)current_feed_urls.Except(deleted_feed_urls).ToList();
var update_lock = new Object();
var updated_feeds = false;
Parallel.ForEach(source: maybe_updated_feed_urls, body: (maybe_updated_feed_url) =>
{
var rowkey = TableStorage.MakeSafeRowkeyFromUrl(maybe_updated_feed_url);
var query = string.Format("$filter=PartitionKey eq '{0}' and RowKey eq '{1}'", id, rowkey);
var table_record = TableStorage.QueryForSingleEntityAsDictStr(ts, "metadata", query);
var json_record = (Dictionary<string, string>)list_metadict_str.Find(feed => feed["feedurl"] == maybe_updated_feed_url);
bool updated_feed = false;
//foreach (var key in new List<string>() { "source", "category", "feedurl", "url", "catmap", "approved", "comments" })
foreach (var key in feed_entry_schema.Split(','))
{
try
{
// if (table_record.ContainsKey(key) == false)
// GenUtils.LogMsg("info", id, "UpdateFeedsForId: table record missing key: " + key + " (will be added)");
if (json_record.ContainsKey(key) == false)
{
// GenUtils.LogMsg("info", id, "UpdateFeedsForId: json record missing key: " + key + " (adding it now)");
json_record[key] = "";
}
if (table_record.ContainsKey(key) == false || table_record[key] != json_record[key])
lock (update_lock)
{
updated_feeds = updated_feed = true;
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("info", "UpdateFeedsForId: missing key: " + key, e.Message + e.StackTrace);
}
}
if (updated_feed)
{
UpmergeChangedMetadict(id, rowkey, json_record);
}
});
// foreach (string maybe_updated_feed_url in maybe_updated_feed_urls)
// {
// }
return updated_feeds;
}
private static List<string> HandleFeedDeletes(string id, TableStorage ts, List<string> current_feed_urls, List<string> existing_feed_urls)
{
var deleted_feed_urls = existing_feed_urls.Except(current_feed_urls);
foreach (string deleted_feed_url in deleted_feed_urls)
{
var rowkey = TableStorage.MakeSafeRowkeyFromUrl(deleted_feed_url);
ts.DeleteEntity("metadata", id, rowkey);
}
GenUtils.LogMsg("info", "UpdateFeedsForId: find updated and update", null);
return deleted_feed_urls.ToList();
}
public static void PriorityLogMsg(string type, string title, string blurb, TableStorage ts)
{
title = MakeLogMsgTitle(title);
ts.WritePriorityLogMessage(type, title, blurb);
}
static public string RunTests(string dll_name)
{
var tests_failed = 0;
var results = new StringBuilder();
try
{
LogMsg("info", "GenUtils.RunTests", "starting");
var ts = TableStorage.MakeDefaultTableStorage();
var a = System.Reflection.Assembly.Load(dll_name);
var types = a.GetExportedTypes().ToList();
var test_classes = types.FindAll(type => type.Name.EndsWith("Test")).ToList();
test_classes.Sort((x, y) => x.Name.CompareTo(y.Name));
foreach (Type test_class in test_classes) // e.g. DeliciousTest
{
object o = Activator.CreateInstance(test_class);
var members = test_class.GetMembers().ToList();
members.Sort((x, y) => x.Name.CompareTo(y.Name));
foreach (var member in members)
{
var attrs = member.GetCustomAttributes(false).ToList();
var is_test = attrs.Exists(attr => attr.GetType() == typeof(NUnit.Framework.TestAttribute));
if (is_test == false)
{
continue;
}
var entity = new Dictionary <string, object>();
var partition_key = test_class.FullName;
var row_key = member.Name;
entity["PartitionKey"] = partition_key;
entity["RowKey"] = row_key;
try
{
test_class.InvokeMember(member.Name, invokeAttr: BindingFlags.InvokeMethod, binder: null, target: o, args: null);
entity["outcome"] = "OK";
entity["reason"] = "";
}
catch (Exception e)
{
var msg = e.Message + e.StackTrace;
entity["outcome"] = "Fail";
entity["reason"] = e.InnerException.Message + e.InnerException.StackTrace;
tests_failed += 1;
results.AppendLine(row_key);
results.AppendLine((string)entity["reason"]);
results.AppendLine();
}
var tablename = Configurator.test_results_tablename;
if (ts.ExistsEntity(tablename, partition_key, row_key))
{
ts.MergeEntity(tablename, partition_key, row_key, entity);
}
else
{
ts.InsertEntity(tablename, entity);
}
}
}
LogMsg("info", "GenUtils.RunTests", "done");
}
catch (Exception e)
{
PriorityLogMsg("exception", "RunTests", e.Message + e.StackTrace);
tests_failed = 999;
}
return(tests_failed > 0 ? results.ToString() : "");
}
// the prioritylogtriggers table has rules for things in a Counters
private static List<Dictionary<string, string>> GetPriorityLogTriggers(TableStorage ts)
{
var query = "$filter=(PartitionKey eq 'prioritylogtriggers')";
var ts_response = ts.QueryAllEntitiesAsListDict("prioritylogtriggers", query);
var list_dict_obj = ts_response.list_dict_obj;
var list_dict_str = new List<Dictionary<string, string>>();
foreach (var dict_obj in list_dict_obj)
list_dict_str.Add(ObjectUtils.DictObjToDictStr(dict_obj));
return list_dict_str;
}
public static TableStorageListDictResponse QueryEntities(string tablename, string query, TableStorage ts)
{
var http_response = ts.DoTableStoreRequest(tablename, query_string: query, method: "GET", headers: new Hashtable(), data: null);
if (http_response.status == HttpStatusCode.ServiceUnavailable)
{
throw new Exception("TableServiceUnavailable");
}
else
{
var response = GetTsDicts(http_response);
return(new TableStorageListDictResponse(http_response, GetTsDicts(http_response)));
}
}
// this query expects to find just one matching entity
public static Dictionary<string, object> QueryForSingleEntityAsDictObj(TableStorage ts, string table, string q)
{
var ts_response = ts.QueryEntities(table, q);
var dicts = ts_response.list_dict_obj;
var dict = new Dictionary<string, object>();
if (dicts.Count > 0)
dict = dicts.FirstOrDefault();
if (dicts.Count > 1)
// should not happen, but...
GenUtils.LogMsg("info", "QueryForSingleEntity: " + table, q + ": more than one matching entity");
return dict;
}
public static TimeSpan IntervalFromType(TaskType type)
{
var ts = TimeSpan.FromHours(24);
switch (type)
{
case TaskType.nonicaltasks:
ts = Scheduler.nonical_interval;
break;
case TaskType.icaltasks:
ts = Scheduler.ical_interval;
break;
case TaskType.regiontasks:
ts = Scheduler.region_interval;
break;
case TaskType.none:
ts = TimeSpan.FromHours(24);
GenUtils.PriorityLogMsg("warning", "Scheduler.IntervalFromType", "unexpected TaskType.none");
break;
}
return ts;
}
public static TableStorageListDictResponse QueryEntities(string tablename, string query, TableStorage ts)
{
var http_response = ts.DoTableStoreRequest(tablename, query_string: query, method: "GET", headers: new Hashtable(), data: null);
if (http_response.status == HttpStatusCode.ServiceUnavailable)
throw new Exception("TableServiceUnavailable");
else
{
var response = GetTsDicts(http_response);
return new TableStorageListDictResponse(http_response, GetTsDicts(http_response));
}
}
public static void SetTs(TableStorage ts)
{
Counters.ts = ts;
}
public static void SetTs(TableStorage ts)
{
Counters.ts = ts;
}
public static TableStorageListDictResponse QueryEntities(string tablename, string query, TableStorage ts)
{
var http_response = ts.DoTableStoreRequest(tablename, query_string: query, method: "GET", headers: new Hashtable(), data: null);
return new TableStorageListDictResponse(http_response, GetTsDicts(http_response));
}
public Monitor(int interval_minutes, string tablename, TableStorage ts)
{
this.interval_minutes = interval_minutes;
this.tablename = tablename;
this.ts = ts;
priority_log_triggers = GetPriorityLogTriggers(ts);
}
public static Dictionary<string, string> QueryForSingleEntityAsDictStr(TableStorage ts, string table, string q)
{
var dict = QueryForSingleEntityAsDictObj(ts, table, q);
return ObjectUtils.DictObjToDictStr(dict);
}
public static Dictionary <string, string> QueryForSingleEntityAsDictStr(TableStorage ts, string table, string q)
{
var dict = QueryForSingleEntityAsDictObj(ts, table, q);
return(ObjectUtils.DictObjToDictStr(dict));
}