public void AzureBlobDataSetTest3() { //Microsoft.Research.Science.Data.Factory.DataSetFactory.Register(typeof(Microsoft.Research.Science.Data.NetCDF4.NetCDFDataSet)); string uri = @"msds:nc?file=air_8times.nc&openMode=readOnly"; var d = DataSet.Open(uri); var blobUri = @"msds:ab?UseDevelopmentStorage=true&Container=testcontainer767x9&Blob=testblob767x9"; var blobD = AzureBlobDataSet.ArrangeData(blobUri, d, new SerializableVariableSchema[0]); int[] airShapeBlob = blobD["air0"].GetShape(); int[] airShape = d["air0"].GetShape(); Assert.AreEqual(airShape.Length, airShapeBlob.Length); for (int i = 0; i < airShape.Length; ++i) { Assert.AreEqual(airShape[i], airShapeBlob[i]); } int[] ori = new int[] { 3, 0, 0 }; int[] shp = new int[] { 2, airShapeBlob[1], airShapeBlob[2] }; Single[, ,] airBlob = (Single[, , ])blobD["air0"].GetData(ori, shp); Assert.AreEqual(3, ori[0]); Assert.AreEqual(0, ori[1]); Assert.AreEqual(0, ori[2]); Assert.AreEqual(2, shp[0]); Assert.AreEqual(airShapeBlob[1], shp[1]); Assert.AreEqual(airShapeBlob[2], shp[2]); blobD["air0"].PutData(new int[] { 3, 0, 0 }, airBlob); Assert.AreEqual(3, ori[0]); Assert.AreEqual(0, ori[1]); Assert.AreEqual(0, ori[2]); Assert.AreEqual(2, shp[0]); Assert.AreEqual(airShapeBlob[1], shp[1]); Assert.AreEqual(airShapeBlob[2], shp[2]); int[] ori2 = new int[] { 3, 2, 2 }; int[] shp2 = new int[] { 2, 5, 7 }; Single[, ,] airBlob2 = (Single[, , ])blobD["air0"].GetData(ori2, shp2); Assert.AreEqual(3, ori2[0]); Assert.AreEqual(2, ori2[1]); Assert.AreEqual(2, ori2[2]); Assert.AreEqual(2, shp2[0]); Assert.AreEqual(5, shp2[1]); Assert.AreEqual(7, shp2[2]); int[] ori3 = new int[] { 25 }; int[] shp3 = new int[] { 50 }; var airBlob3 = blobD["lon"].GetData(ori3, shp3); Assert.AreEqual(25, ori3[0]); Assert.AreEqual(50, shp3[0]); }
static void Main(string[] args) { string uri = @"msds:nc?file=\\vienna.mslab.cs.msu.su\ClimateData\air.sig995.2007.nc&openMode=readOnly"; DataSet d = DataSet.Open(uri); string blobUri = @"msds:ab?UseDevelopmentStorage=true&Container=testcontainer&Blob=testblob"; DataSet blobD = /*new AzureBlobDataSet(blobUri);*/ AzureBlobDataSet.ArrangeData(blobUri, d, new SerializableVariableSchema[0]); AzureBlobDataSetUri azureUri = new AzureBlobDataSetUri(blobUri); CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve a reference to a container CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container); CloudPageBlob blob = container.GetPageBlobReference(azureUri.Blob); Int32 schemeSize; using (BinaryReader br = new BinaryReader(blob.OpenRead())) { //schemeSize = br.ReadInt32(); UTF8Encoding utf8 = new UTF8Encoding(); byte[] buffer = new byte[8192]; br.BaseStream.Read(buffer, 0, 8192); string sizeStr = utf8.GetString(buffer); Console.WriteLine(sizeStr); //schemeSize = Int32.Parse(sizeStr); //br.BaseStream.Seek(512, SeekOrigin.Begin); //DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(DataSetInfo)); //byte[] scheme = new byte[schemeSize]; //br.BaseStream.Read(scheme, 0, schemeSize); //info = (DataSetInfo)serializer.ReadObject(new MemoryStream(scheme)); } //foreach (var i in blobD.Variables) Console.WriteLine(i.Name); Single[] a = (Single[])blobD["lat"].GetData(); blobD["lat"].PutData(a); //foreach (var i in a) Console.WriteLine(i.ToString()); foreach (var i in blobD["lat"].Metadata) { Console.WriteLine(i.Key.ToString() + " = " + i.Value.ToString()); } foreach (var i in blobD.Metadata) { Console.WriteLine(i.Key.ToString() + " = " + i.Value.ToString()); } Console.ReadLine(); }
public void SingleDimVarUpdate512Aligned() { using (DataSet sourceDs = DataSet.Open("msds:memory")) { sourceDs.AddVariable <double>("b", Enumerable.Repeat(0, 1024).Select(a => (double)a).ToArray(), "i"); var svs = new SerializableVariableSchema("a", typeof(double), new string[] { "i" }, new System.Collections.Generic.Dictionary <string, object>()); using (AzureBlobDataSet ds = AzureBlobDataSet.ArrangeData(string.Format(@"msds:ab?Container=tests&Blob={0}&UseDevelopmentStorage=true", new Random().NextDouble()), sourceDs, new SerializableVariableSchema[] { svs })) { ds.Variables["a"].PutData(Enumerable.Repeat(1, 64).Select(a => (double)a).ToArray()); //64x8=512 bytes shape } } }
public void TestMethod1() { string uri = @"msds:nc?file=air_9times.nc&openMode=readOnly"; DataSet d = DataSet.Open(uri); string blobUri = @"msds:ab?UseDevelopmentStorage=true&Container=testcontainer767x9&Blob=testblob767x9"; var schema = new SerializableVariableSchema("testvar", typeof(Int32), new string[] { "lat" }, new System.Collections.Generic.Dictionary <string, object>()); schema.Metadata.Add("testData", new ObjectWithTypeString(true)); DataSet blobD = AzureBlobDataSet.ArrangeData(blobUri, d, new SerializableVariableSchema[] { schema }); foreach (var i in blobD.Variables) { Console.WriteLine(i.Name); } Single[] a = (Single[])blobD["lat"].GetData(); Single[] b = (Single[])d["lat"].GetData(); Assert.AreEqual(b.Length, a.Length); for (int i = 0; i < a.Length; ++i) { Assert.AreEqual(a[i], b[i]); } Assert.AreEqual(d["lat"].Metadata.Count, blobD["lat"].Metadata.Count); foreach (var i in blobD["lat"].Metadata) { Assert.IsTrue(d["lat"].Metadata.ContainsKey(i.Key)); if (i.Value is Array) { for (int j = 0; j < ((Array)i.Value).Length; ++j) { Assert.AreEqual(((Array)d["lat"].Metadata[i.Key]).GetValue(j), ((Array)i.Value).GetValue(j)); } } else { Assert.AreEqual(d["lat"].Metadata[i.Key], i.Value); } } foreach (var i in blobD.Metadata) { Assert.IsTrue(d.Metadata.ContainsKey(i.Key)); if (i.Value is Array) { for (int j = 0; j < ((Array)i.Value).Length; ++j) { Assert.AreEqual(((Array)d.Metadata[i.Key]).GetValue(j), ((Array)i.Value).GetValue(j)); } } else { Assert.AreEqual(d.Metadata[i.Key], i.Value); } } Assert.AreEqual(true, blobD["testvar"].Metadata["testData"]); uri = @"msds:nc?file=air_8times.nc&openMode=readOnly"; d = DataSet.Open(uri); blobUri = @"msds:ab?UseDevelopmentStorage=true&Container=testcontainer767x9&Blob=testblob767x9"; blobD = AzureBlobDataSet.ArrangeData(blobUri, d, new SerializableVariableSchema[0]); int[] airShapeBlob = blobD["air0"].GetShape(); int[] airShape = d["air0"].GetShape(); Assert.AreEqual(airShape.Length, airShapeBlob.Length); for (int i = 0; i < airShape.Length; ++i) { Assert.AreEqual(airShape[i], airShapeBlob[i]); } Single[, ,] airBlob = (Single[, , ])blobD["air0"].GetData(new int[] { 3, 0, 0 }, new int[] { 2, airShapeBlob[1], airShapeBlob[2] }); Single[, ,] air = (Single[, , ])d["air0"].GetData(new int[] { 3, 0, 0 }, new int[] { 2, airShape[1], airShape[2] }); for (int i = 0; i < 2; ++i) { for (int j = 0; j < airShapeBlob[1]; ++j) { for (int k = 0; k < airShapeBlob[2]; ++k) { Assert.AreEqual(air[i, j, k], airBlob[i, j, k]); } } } blobD["air0"].PutData(new int[] { 3, 0, 0 }, air); airBlob = (Single[, , ])blobD["air0"].GetData(new int[] { 3, 0, 0 }, new int[] { 2, airShapeBlob[1], airShapeBlob[2] }); for (int i = 0; i < 2; ++i) { for (int j = 0; j < airShapeBlob[1]; ++j) { for (int k = 0; k < airShapeBlob[2]; ++k) { Assert.AreEqual(air[i, j, k], airBlob[i, j, k]); } } } DateTime[] datesBlob = (DateTime[])blobD["_time_subset"].GetData(); DateTime[] dates = (DateTime[])d["_time_subset"].GetData(); Assert.AreEqual(dates.Length, datesBlob.Length); for (int i = 0; i < dates.Length; ++i) { Assert.AreEqual(dates[i], datesBlob[i]); } blobD["_time_subset"].PutData(dates); datesBlob = (DateTime[])blobD["_time_subset"].GetData(); Assert.AreEqual(dates.Length, datesBlob.Length); for (int i = 0; i < dates.Length; ++i) { Assert.AreEqual(dates[i], datesBlob[i]); } }
public void AzureBlobDataSetTest1() { string uri = @"msds:nc?file=air_9times.nc&openMode=readOnly"; DataSet d = DataSet.Open(uri); string blobUri = @"msds:ab?UseDevelopmentStorage=true&Container=testcontainer767x9&Blob=testblob767x9"; var schema = new SerializableVariableSchema("testvar", typeof(Int32), new string[] { "lat" }, new System.Collections.Generic.Dictionary <string, object>()); schema.Metadata.Add("testData", new ObjectWithTypeString(true)); DateTime dateTimeNow = DateTime.Now; schema.Metadata.Add("dateTime", new ObjectWithTypeString(dateTimeNow)); DataSet blobD = AzureBlobDataSet.ArrangeData(blobUri, d, new SerializableVariableSchema[] { schema }); var json1 = JsonConvert.SerializeObject(blobD, Formatting.Indented, new JsonSerializerSettings() { ReferenceLoopHandling = ReferenceLoopHandling.Ignore }); foreach (var i in blobD.Variables) { Console.WriteLine(i.Name); } Single[] a = (Single[])blobD["lat"].GetData(); Single[] b = (Single[])d["lat"].GetData(); var json2 = JsonConvert.SerializeObject(a, Formatting.Indented, new JsonSerializerSettings() { ReferenceLoopHandling = ReferenceLoopHandling.Ignore }); Assert.AreEqual(b.Length, a.Length); for (int i = 0; i < a.Length; ++i) { Assert.AreEqual(a[i], b[i]); } Assert.AreEqual(d["lat"].Metadata.Count, blobD["lat"].Metadata.Count); foreach (var i in blobD["lat"].Metadata) { Assert.IsTrue(d["lat"].Metadata.ContainsKey(i.Key)); if (i.Value is Array) { for (int j = 0; j < ((Array)i.Value).Length; ++j) { Assert.AreEqual(((Array)d["lat"].Metadata[i.Key]).GetValue(j), ((Array)i.Value).GetValue(j)); } } else { Assert.AreEqual(d["lat"].Metadata[i.Key], i.Value); } } foreach (var i in blobD.Metadata) { Assert.IsTrue(d.Metadata.ContainsKey(i.Key)); if (i.Value is Array) { for (int j = 0; j < ((Array)i.Value).Length; ++j) { Assert.AreEqual(((Array)d.Metadata[i.Key]).GetValue(j), ((Array)i.Value).GetValue(j)); } } else { Assert.AreEqual(d.Metadata[i.Key], i.Value); } } Assert.AreEqual(true, blobD["testvar"].Metadata["testData"]); double deltaTimes = (dateTimeNow - (DateTime)blobD["testvar"].Metadata["dateTime"]).TotalMilliseconds; Assert.IsTrue(Math.Abs(deltaTimes) <= 1.0);//JSON somewhat cuts DateTimes :( //Assert.AreEqual(dateTimeNow, blobD["testvar"].Metadata["dateTime"]); uri = @"msds:nc?file=air_8times.nc&openMode=readOnly"; d = DataSet.Open(uri); blobUri = @"msds:ab?UseDevelopmentStorage=true&Container=testcontainer767x9&Blob=testblob767x9"; blobD = AzureBlobDataSet.ArrangeData(blobUri, d, new SerializableVariableSchema[0]); int[] airShapeBlob = blobD["air0"].GetShape(); int[] airShape = d["air0"].GetShape(); Assert.AreEqual(airShape.Length, airShapeBlob.Length); for (int i = 0; i < airShape.Length; ++i) { Assert.AreEqual(airShape[i], airShapeBlob[i]); } Single[, ,] airBlob = (Single[, , ])blobD["air0"].GetData(new int[] { 3, 0, 0 }, new int[] { 2, airShapeBlob[1], airShapeBlob[2] }); Single[, ,] air = (Single[, , ])d["air0"].GetData(new int[] { 3, 0, 0 }, new int[] { 2, airShape[1], airShape[2] }); for (int i = 0; i < 2; ++i) { for (int j = 0; j < airShapeBlob[1]; ++j) { for (int k = 0; k < airShapeBlob[2]; ++k) { Assert.AreEqual(air[i, j, k], airBlob[i, j, k]); } } } blobD["air0"].PutData(new int[] { 3, 0, 0 }, air); airBlob = (Single[, , ])blobD["air0"].GetData(new int[] { 3, 0, 0 }, new int[] { 2, airShapeBlob[1], airShapeBlob[2] }); for (int i = 0; i < 2; ++i) { for (int j = 0; j < airShapeBlob[1]; ++j) { for (int k = 0; k < airShapeBlob[2]; ++k) { Assert.AreEqual(air[i, j, k], airBlob[i, j, k]); } } } DateTime[] datesBlob = (DateTime[])blobD["_time_subset"].GetData(); DateTime[] dates = (DateTime[])d["_time_subset"].GetData(); Assert.AreEqual(dates.Length, datesBlob.Length); for (int i = 0; i < dates.Length; ++i) { Assert.AreEqual(dates[i], datesBlob[i]); } blobD["_time_subset"].PutData(dates); datesBlob = (DateTime[])blobD["_time_subset"].GetData(); Assert.AreEqual(dates.Length, datesBlob.Length); for (int i = 0; i < dates.Length; ++i) { Assert.AreEqual(dates[i], datesBlob[i]); } Single[, ,] airBlob2 = (Single[, , ])blobD["air0"].GetData(new int[] { 3, 2, 2 }, new int[] { 2, 5, 7 }); Single[, ,] air2 = (Single[, , ])d["air0"].GetData(new int[] { 3, 2, 2 }, new int[] { 2, 5, 7 }); for (int i = 0; i < 2; ++i) { for (int j = 0; j < 5; ++j) { for (int k = 0; k < 7; ++k) { Assert.AreEqual(air2[i, j, k], airBlob2[i, j, k]); } } } blobUri = @"msds:ab?UseDevelopmentStorage=true&Container=testcontainer767x9&Blob=testblob767x10"; blobD = AzureBlobDataSet.CreateEmptySet(blobUri, d.GetSerializableSchema()); blobD["_time_subset"].PutData(dates); datesBlob = (DateTime[])blobD["_time_subset"].GetData(); Assert.AreEqual(dates.Length, datesBlob.Length); for (int i = 0; i < dates.Length; ++i) { Assert.AreEqual(dates[i], datesBlob[i]); } }