public EventCollectorTest() { test_calinfo = new Calinfo(ElmcityUtils.Configurator.azure_compute_account); lookup_lat = test_calinfo.lat; lookup_lon = test_calinfo.lon; radius = Configurator.default_radius; test_upcoming_args = string.Format("location={0},{1}&radius={2}&min_date={3}", lookup_lat, lookup_lon, radius, min_date); settings = GenUtils.GetSettingsFromAzureTable(); basic_ics = BlobStorage.MakeDefaultBlobStorage().GetBlob("admin", "basic.ics").HttpResponse.DataAsString(); bs = BlobStorage.MakeDefaultBlobStorage(); calinfo_berkeley = new Calinfo(berkeley_test_hub); calinfo_keene = new Calinfo(keene_test_hub); collector_berkeley = new Collector(calinfo_berkeley, settings); collector_keene = new Collector(calinfo_keene,settings); foreach (var example in ics_examples) UpdateYYYY(example, "ics"); foreach (var example in eventful_examples) UpdateYYYY(example, "xml"); foreach (var example in upcoming_examples) UpdateYYYY(example, "xml"); foreach (var example in eventbrite_examples) UpdateYYYY(example, "xml"); }
public void PutBlobCreatesExpectedMetadata() { var blobname = CreateBlob(); blobmeta = new Hashtable(); blobmeta.Add(BlobStorage.PREFIX_METADATA + "metakey", "metavalue"); string content_type = "text/random"; var bs_response = bs.PutBlob(containername, blobname, blobmeta, blobcontent, content_type); string domain = Configurator.azure_storage_account + "." + Configurator.azure_blob_domain; string str_url = string.Format("http://{0}/{1}/{2}", domain, BlobStorage.LegalizeContainerName(containername), blobname); var request = (HttpWebRequest)WebRequest.Create(new Uri(str_url)); var response = HttpUtils.DoHttpWebRequest(request, null); Assert.AreEqual(blobcontent, response.bytes); Assert.AreEqual("text/random", response.headers["Content-Type"]); Assert.AreEqual("metavalue", response.headers[BlobStorage.PREFIX_METADATA + "metakey"]); bs.DeleteBlob(containername, blobname); }
public static bool SavedJsonSnapshot(string id, JsonSnapshotType type, string name, Object new_obj) { try { var json_blob_name = id + "." + name + ".json"; var existing_obj_uri = BlobStorage.MakeAzureBlobUri(id, json_blob_name, false); var exists = BlobStorage.ExistsBlob(existing_obj_uri); JsonCompareResult comparison = NewJsonMatchesExistingJson(type, new_obj, existing_obj_uri); if (!exists || comparison == JsonCompareResult.different) { var bs = BlobStorage.MakeDefaultBlobStorage(); var timestamped_json_blob_name = string.Format(id + "." + string.Format("{0:yyyy.MM.dd.HH.mm}" + "." + name + ".json", DateTime.UtcNow)); var timestamped_dict_uri = BlobStorage.MakeAzureBlobUri(id, timestamped_json_blob_name, false); string new_obj_as_json; if (type == JsonSnapshotType.DictStr) { new_obj_as_json = ObjectUtils.DictStrToJson((Dictionary <string, string>)new_obj); } else // JsonSnapshotType.ListDictStr { new_obj_as_json = ObjectUtils.ListDictStrToJson((List <Dictionary <string, string> >)new_obj); } //bs.PutBlob(id, json_blob_name, new_obj_as_json, "application/json"); bs.PutBlob(id, timestamped_json_blob_name, new_obj_as_json, "application/json"); } if (comparison == JsonCompareResult.same || comparison == JsonCompareResult.invalid) { return(false); // either the objects matched, or the comparison failed, either way a json snapshot was not saved } else { return(true); // the objects did not match, a json snapshot was saved } } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "SavedJsonSnapshot", e.Message + e.StackTrace); return(false); } }
public BlobStorageResponse PutBlobWithLease(string container, string blobname, Hashtable headers, byte[] data, string content_type) { try { if (BlobStorage.ExistsBlob(container, blobname)) { var r = this.RetryAcquireLease(container, blobname); if (r.status == HttpStatusCode.Created) { headers.Add("x-ms-lease-id", r.headers["x-ms-lease-id"]); } else { GenUtils.PriorityLogMsg("warning", "PutBlobWithLease: Did not acquire lease for: ", container + ", " + blobname); } } } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "PutBlobWithLease", e.Message + e.StackTrace); } return(this.PutBlob(container, blobname, headers, data, content_type)); }
public static BlobStorageResponse WriteToAzureBlob(BlobStorage bs, string containername, string blobname, string content_type, byte[] bytes) { if (BlobStorage.ExistsContainer(containername) == false) bs.CreateContainer(containername, true, new Hashtable()); var headers = new Hashtable(); BlobStorageResponse bs_response; bs_response = bs.PutBlob(containername, blobname, headers, bytes, content_type); return bs_response; }
// public methods used by worker to collect events from all source types public Collector(Calinfo calinfo, Dictionary<string, string> settings) { this.calinfo = calinfo; this.settings = settings; this.mock_eventful = false; this.mock_upcoming = false; this.mock_eventbrite = false; this.id = calinfo.id; this.bs = BlobStorage.MakeDefaultBlobStorage(); // an instance of a DDay.iCal for each source type, used to collect intermediate ICS // results which are saved, then combined to produce a merged ICS, e.g.: // http://elmcity.blob.core.windows.net/a2cal/a2cal.ics this.ical_ical = NewCalendarWithTimezone(); this.eventful_ical = NewCalendarWithTimezone(); this.upcoming_ical = NewCalendarWithTimezone(); this.eventbrite_ical = NewCalendarWithTimezone(); this.facebook_ical = NewCalendarWithTimezone(); this.estats = new NonIcalStats(); this.estats.blobname = "eventful_stats"; this.ustats = new NonIcalStats(); this.ustats.blobname = "upcoming_stats"; this.ebstats = new NonIcalStats(); this.ebstats.blobname = "eventbrite_stats"; this.fbstats = new NonIcalStats(); this.fbstats.blobname = "facebook_stats"; this.mustats = new NonIcalStats(); this.mustats.blobname = "meetup_stats"; }
public static string GetAzureBlobAsString(string container, string name, bool use_cdn) { var uri = BlobStorage.MakeAzureBlobUri(container, name, use_cdn); return(HttpUtils.FetchUrl(uri).DataAsString()); }
public static T GetTypedObj <T>(string container, string name) { var uri = BlobStorage.MakeAzureBlobUri(container, name, false); return((T)BlobStorage.DeserializeObjectFromUri(uri)); }
public static string RunIronPython(string directory, string str_script_url, List <string> args) { GenUtils.LogMsg("status", "Utils.run_ironpython: " + str_script_url, args[0] + "," + args[1] + "," + args[2]); //var app_domain_name = "ironpython"; string result = ""; try { /* * string domain_id = app_domain_name; * var setup = new AppDomainSetup(); * setup.ApplicationName = app_domain_name; * setup.ApplicationBase = AppDomain.CurrentDomain.BaseDirectory; * var python_domain = AppDomain.CreateDomain(domain_id, securityInfo: null, info: setup); */ var options = new Dictionary <string, object>(); options["LightweightScopes"] = true; var python = Python.CreateEngine(options); var paths = new List <string>(); paths.Add(directory + "Lib"); // standard python lib paths.Add(directory + "Lib\\site-packages"); paths.Add(directory + "ElmcityLib"); // Elmcity python lib GenUtils.LogMsg("status", "Utils.run_ironpython", String.Join(":", paths.ToArray())); python.SetSearchPaths(paths); var ipy_args = new IronPython.Runtime.List(); foreach (var item in args) { ipy_args.Add(item); } var s = HttpUtils.FetchUrl(new Uri(str_script_url)).DataAsString(); try { var common_script_url = BlobStorage.MakeAzureBlobUri("admin", "common.py", false); var common_script = HttpUtils.FetchUrl(common_script_url).DataAsString(); s = s.Replace("#include common.py", common_script); } catch (Exception e) { GenUtils.PriorityLogMsg("exception", "RunIronPython: cannot #include common.py", e.Message); } var source = python.CreateScriptSourceFromString(s, SourceCodeKind.Statements); var scope = python.CreateScope(); var sys = python.GetSysModule(); //sys.SetVariable("argv", new PythonArgs() { args = ipy_args } ); sys.SetVariable("argv", args); source.Execute(scope); try { result = scope.GetVariable("result").ToString(); } catch { // GenUtils.LogMsg("info", "RunIronPython: " + str_script_url, "no result"); } python.Runtime.Shutdown(); //AppDomain.Unload(python_domain); } catch (Exception e) { result = e.Message.ToString() + e.StackTrace.ToString(); } return(result); }