public static DataSet CreateRequestBlobDataSet(string dsUri, IFetchRequest request)
{
var sd = GetRequestSchemaAndData(request);
var ds = AzureBlobDataSet.CreateSetWithSmallData(dsUri, sd.Item1, sd.Item2);
return(ds);
}
public void AzureBlobDataSetTest4CreateSetWithSmallData()
{
string uri = @"msds:nc?file=air_9times.nc&openMode=readOnly";
DataSet d = DataSet.Open(uri);
string blobUri = @"msds:ab?UseDevelopmentStorage=true&Container=testcontainer767x9&Blob=testblob767x9";
var schema = new SerializableVariableSchema("testvar", typeof(Int32), new string[] { "lat" }, new System.Collections.Generic.Dictionary <string, object>());
schema.Metadata.Add("testData", new ObjectWithTypeString(true));
DateTime dateTimeNow = DateTime.Now;
schema.Metadata.Add("dateTime", new ObjectWithTypeString(dateTimeNow));
//DataSet blobD = AzureBlobDataSet.ArrangeData(blobUri, d, new SerializableVariableSchema[] { schema });
System.Collections.Generic.Dictionary <string, Array> dataToPut = new System.Collections.Generic.Dictionary <string, Array>();
foreach (var v in d.Variables)
{
dataToPut.Add(v.Name, v.GetData());
}
var dsscheme = d.GetSerializableSchema();
var varList = dsscheme.Variables.ToList();
varList.Add(schema);
dsscheme.Variables = varList.ToArray();
DataSet blobD = AzureBlobDataSet.CreateSetWithSmallData(blobUri, dsscheme, dataToPut);
var json1 = JsonConvert.SerializeObject(blobD, Formatting.Indented,
new JsonSerializerSettings()
{
ReferenceLoopHandling = ReferenceLoopHandling.Ignore
});
foreach (var i in blobD.Variables)
{
Console.WriteLine(i.Name);
}
Single[] a = (Single[])blobD["lat"].GetData();
Single[] b = (Single[])d["lat"].GetData();
var json2 = JsonConvert.SerializeObject(a, Formatting.Indented,
new JsonSerializerSettings()
{
ReferenceLoopHandling = ReferenceLoopHandling.Ignore
});
Assert.AreEqual(b.Length, a.Length);
for (int i = 0; i < a.Length; ++i)
{
Assert.AreEqual(a[i], b[i]);
}
Assert.AreEqual(d["lat"].Metadata.Count, blobD["lat"].Metadata.Count);
foreach (var i in blobD["lat"].Metadata)
{
Assert.IsTrue(d["lat"].Metadata.ContainsKey(i.Key));
if (i.Value is Array)
{
for (int j = 0; j < ((Array)i.Value).Length; ++j)
{
Assert.AreEqual(((Array)d["lat"].Metadata[i.Key]).GetValue(j), ((Array)i.Value).GetValue(j));
}
}
else
{
Assert.AreEqual(d["lat"].Metadata[i.Key], i.Value);
}
}
foreach (var i in blobD.Metadata)
{
Assert.IsTrue(d.Metadata.ContainsKey(i.Key));
if (i.Value is Array)
{
for (int j = 0; j < ((Array)i.Value).Length; ++j)
{
Assert.AreEqual(((Array)d.Metadata[i.Key]).GetValue(j), ((Array)i.Value).GetValue(j));
}
}
else
{
Assert.AreEqual(d.Metadata[i.Key], i.Value);
}
}
Assert.AreEqual(true, blobD["testvar"].Metadata["testData"]);
double deltaTimes = (dateTimeNow - (DateTime)blobD["testvar"].Metadata["dateTime"]).TotalMilliseconds;
Assert.IsTrue(Math.Abs(deltaTimes) <= 1.0);//JSON somewhat cuts DateTimes :(
//Assert.AreEqual(dateTimeNow, blobD["testvar"].Metadata["dateTime"]);
uri = @"msds:nc?file=air_8times.nc&openMode=readOnly";
d = DataSet.Open(uri);
blobUri = @"msds:ab?UseDevelopmentStorage=true&Container=testcontainer767x9&Blob=testblob767x9";
//blobD = AzureBlobDataSet.ArrangeData(blobUri, d, new SerializableVariableSchema[0]);
dataToPut = new System.Collections.Generic.Dictionary <string, Array>();
foreach (var v in d.Variables)
{
dataToPut.Add(v.Name, v.GetData());
}
dsscheme = d.GetSerializableSchema();
blobD = AzureBlobDataSet.CreateSetWithSmallData(blobUri, dsscheme, dataToPut);
int[] airShapeBlob = blobD["air0"].GetShape();
int[] airShape = d["air0"].GetShape();
Assert.AreEqual(airShape.Length, airShapeBlob.Length);
for (int i = 0; i < airShape.Length; ++i)
{
Assert.AreEqual(airShape[i], airShapeBlob[i]);
}
Single[, ,] airBlob = (Single[, , ])blobD["air0"].GetData(new int[] { 3, 0, 0 }, new int[] { 2, airShapeBlob[1], airShapeBlob[2] });
Single[, ,] air = (Single[, , ])d["air0"].GetData(new int[] { 3, 0, 0 }, new int[] { 2, airShape[1], airShape[2] });
for (int i = 0; i < 2; ++i)
{
for (int j = 0; j < airShapeBlob[1]; ++j)
{
for (int k = 0; k < airShapeBlob[2]; ++k)
{
Assert.AreEqual(air[i, j, k], airBlob[i, j, k]);
}
}
}
blobD["air0"].PutData(new int[] { 3, 0, 0 }, air);
airBlob = (Single[, , ])blobD["air0"].GetData(new int[] { 3, 0, 0 }, new int[] { 2, airShapeBlob[1], airShapeBlob[2] });
for (int i = 0; i < 2; ++i)
{
for (int j = 0; j < airShapeBlob[1]; ++j)
{
for (int k = 0; k < airShapeBlob[2]; ++k)
{
Assert.AreEqual(air[i, j, k], airBlob[i, j, k]);
}
}
}
DateTime[] datesBlob = (DateTime[])blobD["_time_subset"].GetData();
DateTime[] dates = (DateTime[])d["_time_subset"].GetData();
Assert.AreEqual(dates.Length, datesBlob.Length);
for (int i = 0; i < dates.Length; ++i)
{
Assert.AreEqual(dates[i], datesBlob[i]);
}
blobD["_time_subset"].PutData(dates);
datesBlob = (DateTime[])blobD["_time_subset"].GetData();
Assert.AreEqual(dates.Length, datesBlob.Length);
for (int i = 0; i < dates.Length; ++i)
{
Assert.AreEqual(dates[i], datesBlob[i]);
}
Single[, ,] airBlob2 = (Single[, , ])blobD["air0"].GetData(new int[] { 3, 2, 2 }, new int[] { 2, 5, 7 });
Single[, ,] air2 = (Single[, , ])d["air0"].GetData(new int[] { 3, 2, 2 }, new int[] { 2, 5, 7 });
for (int i = 0; i < 2; ++i)
{
for (int j = 0; j < 5; ++j)
{
for (int k = 0; k < 7; ++k)
{
Assert.AreEqual(air2[i, j, k], airBlob2[i, j, k]);
}
}
}
blobUri = @"msds:ab?UseDevelopmentStorage=true&Container=testcontainer767x9&Blob=testblob767x10";
blobD = AzureBlobDataSet.CreateEmptySet(blobUri, d.GetSerializableSchema());
blobD["_time_subset"].PutData(dates);
datesBlob = (DateTime[])blobD["_time_subset"].GetData();
Assert.AreEqual(dates.Length, datesBlob.Length);
for (int i = 0; i < dates.Length; ++i)
{
Assert.AreEqual(dates[i], datesBlob[i]);
}
}
