public EventCollectorTest()
{
test_calinfo = new Calinfo(ElmcityUtils.Configurator.azure_compute_account);
lookup_lat = test_calinfo.lat;
lookup_lon = test_calinfo.lon;
radius = Configurator.default_radius;
test_upcoming_args = string.Format("location={0},{1}&radius={2}&min_date={3}", lookup_lat, lookup_lon, radius, min_date);
settings = GenUtils.GetSettingsFromAzureTable();
basic_ics = BlobStorage.MakeDefaultBlobStorage().GetBlob("admin", "basic.ics").HttpResponse.DataAsString();
bs = BlobStorage.MakeDefaultBlobStorage();
calinfo_berkeley = new Calinfo(berkeley_test_hub);
calinfo_keene = new Calinfo(keene_test_hub);
collector_berkeley = new Collector(calinfo_berkeley, settings);
collector_keene = new Collector(calinfo_keene,settings);
foreach (var example in ics_examples)
UpdateYYYY(example, "ics");
foreach (var example in eventful_examples)
UpdateYYYY(example, "xml");
foreach (var example in upcoming_examples)
UpdateYYYY(example, "xml");
foreach (var example in eventbrite_examples)
UpdateYYYY(example, "xml");
}
public void PutBlobCreatesExpectedMetadata()
{
var blobname = CreateBlob();
blobmeta = new Hashtable();
blobmeta.Add(BlobStorage.PREFIX_METADATA + "metakey", "metavalue");
string content_type = "text/random";
var bs_response = bs.PutBlob(containername, blobname, blobmeta, blobcontent, content_type);
string domain = Configurator.azure_storage_account + "." + Configurator.azure_blob_domain;
string str_url = string.Format("http://{0}/{1}/{2}", domain, BlobStorage.LegalizeContainerName(containername), blobname);
var request = (HttpWebRequest)WebRequest.Create(new Uri(str_url));
var response = HttpUtils.DoHttpWebRequest(request, null);
Assert.AreEqual(blobcontent, response.bytes);
Assert.AreEqual("text/random", response.headers["Content-Type"]);
Assert.AreEqual("metavalue", response.headers[BlobStorage.PREFIX_METADATA + "metakey"]);
bs.DeleteBlob(containername, blobname);
}
public static bool SavedJsonSnapshot(string id, JsonSnapshotType type, string name, Object new_obj)
{
try
{
var json_blob_name = id + "." + name + ".json";
var existing_obj_uri = BlobStorage.MakeAzureBlobUri(id, json_blob_name, false);
var exists = BlobStorage.ExistsBlob(existing_obj_uri);
JsonCompareResult comparison = NewJsonMatchesExistingJson(type, new_obj, existing_obj_uri);
if (!exists || comparison == JsonCompareResult.different)
{
var bs = BlobStorage.MakeDefaultBlobStorage();
var timestamped_json_blob_name = string.Format(id + "." + string.Format("{0:yyyy.MM.dd.HH.mm}" + "." + name + ".json", DateTime.UtcNow));
var timestamped_dict_uri = BlobStorage.MakeAzureBlobUri(id, timestamped_json_blob_name, false);
string new_obj_as_json;
if (type == JsonSnapshotType.DictStr)
{
new_obj_as_json = ObjectUtils.DictStrToJson((Dictionary <string, string>)new_obj);
}
else // JsonSnapshotType.ListDictStr
{
new_obj_as_json = ObjectUtils.ListDictStrToJson((List <Dictionary <string, string> >)new_obj);
}
//bs.PutBlob(id, json_blob_name, new_obj_as_json, "application/json");
bs.PutBlob(id, timestamped_json_blob_name, new_obj_as_json, "application/json");
}
if (comparison == JsonCompareResult.same || comparison == JsonCompareResult.invalid)
{
return(false); // either the objects matched, or the comparison failed, either way a json snapshot was not saved
}
else
{
return(true); // the objects did not match, a json snapshot was saved
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "SavedJsonSnapshot", e.Message + e.StackTrace);
return(false);
}
}
public BlobStorageResponse PutBlobWithLease(string container, string blobname, Hashtable headers, byte[] data, string content_type)
{
try
{
if (BlobStorage.ExistsBlob(container, blobname))
{
var r = this.RetryAcquireLease(container, blobname);
if (r.status == HttpStatusCode.Created)
{
headers.Add("x-ms-lease-id", r.headers["x-ms-lease-id"]);
}
else
{
GenUtils.PriorityLogMsg("warning", "PutBlobWithLease: Did not acquire lease for: ", container + ", " + blobname);
}
}
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "PutBlobWithLease", e.Message + e.StackTrace);
}
return(this.PutBlob(container, blobname, headers, data, content_type));
}
public static BlobStorageResponse WriteToAzureBlob(BlobStorage bs, string containername, string blobname, string content_type, byte[] bytes)
{
if (BlobStorage.ExistsContainer(containername) == false)
bs.CreateContainer(containername, true, new Hashtable());
var headers = new Hashtable();
BlobStorageResponse bs_response;
bs_response = bs.PutBlob(containername, blobname, headers, bytes, content_type);
return bs_response;
}
// public methods used by worker to collect events from all source types
public Collector(Calinfo calinfo, Dictionary<string, string> settings)
{
this.calinfo = calinfo;
this.settings = settings;
this.mock_eventful = false;
this.mock_upcoming = false;
this.mock_eventbrite = false;
this.id = calinfo.id;
this.bs = BlobStorage.MakeDefaultBlobStorage();
// an instance of a DDay.iCal for each source type, used to collect intermediate ICS
// results which are saved, then combined to produce a merged ICS, e.g.:
// http://elmcity.blob.core.windows.net/a2cal/a2cal.ics
this.ical_ical = NewCalendarWithTimezone();
this.eventful_ical = NewCalendarWithTimezone();
this.upcoming_ical = NewCalendarWithTimezone();
this.eventbrite_ical = NewCalendarWithTimezone();
this.facebook_ical = NewCalendarWithTimezone();
this.estats = new NonIcalStats();
this.estats.blobname = "eventful_stats";
this.ustats = new NonIcalStats();
this.ustats.blobname = "upcoming_stats";
this.ebstats = new NonIcalStats();
this.ebstats.blobname = "eventbrite_stats";
this.fbstats = new NonIcalStats();
this.fbstats.blobname = "facebook_stats";
this.mustats = new NonIcalStats();
this.mustats.blobname = "meetup_stats";
}
public static string GetAzureBlobAsString(string container, string name, bool use_cdn)
{
var uri = BlobStorage.MakeAzureBlobUri(container, name, use_cdn);
return(HttpUtils.FetchUrl(uri).DataAsString());
}
public static T GetTypedObj <T>(string container, string name)
{
var uri = BlobStorage.MakeAzureBlobUri(container, name, false);
return((T)BlobStorage.DeserializeObjectFromUri(uri));
}
public static string RunIronPython(string directory, string str_script_url, List <string> args)
{
GenUtils.LogMsg("status", "Utils.run_ironpython: " + str_script_url, args[0] + "," + args[1] + "," + args[2]);
//var app_domain_name = "ironpython";
string result = "";
try
{
/*
* string domain_id = app_domain_name;
* var setup = new AppDomainSetup();
* setup.ApplicationName = app_domain_name;
* setup.ApplicationBase = AppDomain.CurrentDomain.BaseDirectory;
* var python_domain = AppDomain.CreateDomain(domain_id, securityInfo: null, info: setup);
*/
var options = new Dictionary <string, object>();
options["LightweightScopes"] = true;
var python = Python.CreateEngine(options);
var paths = new List <string>();
paths.Add(directory + "Lib"); // standard python lib
paths.Add(directory + "Lib\\site-packages");
paths.Add(directory + "ElmcityLib"); // Elmcity python lib
GenUtils.LogMsg("status", "Utils.run_ironpython", String.Join(":", paths.ToArray()));
python.SetSearchPaths(paths);
var ipy_args = new IronPython.Runtime.List();
foreach (var item in args)
{
ipy_args.Add(item);
}
var s = HttpUtils.FetchUrl(new Uri(str_script_url)).DataAsString();
try
{
var common_script_url = BlobStorage.MakeAzureBlobUri("admin", "common.py", false);
var common_script = HttpUtils.FetchUrl(common_script_url).DataAsString();
s = s.Replace("#include common.py", common_script);
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "RunIronPython: cannot #include common.py", e.Message);
}
var source = python.CreateScriptSourceFromString(s, SourceCodeKind.Statements);
var scope = python.CreateScope();
var sys = python.GetSysModule();
//sys.SetVariable("argv", new PythonArgs() { args = ipy_args } );
sys.SetVariable("argv", args);
source.Execute(scope);
try
{
result = scope.GetVariable("result").ToString();
}
catch
{
// GenUtils.LogMsg("info", "RunIronPython: " + str_script_url, "no result");
}
python.Runtime.Shutdown();
//AppDomain.Unload(python_domain);
}
catch (Exception e)
{
result = e.Message.ToString() + e.StackTrace.ToString();
}
return(result);
}
