static void DoCopy(string uri, string uri2)
{
var ds = OpenDataSet(uri);
Console.Write("Copying . . . ");
Stopwatch sw = new Stopwatch();
sw.Start();
DataSetUri dstUri = DataSetUri.Create(uri2);
if (dstUri.ProviderName.StartsWith("memory"))
{
throw new NotSupportedException("Copying to memory is not supported by the utility.");
}
DataSet ds2 = Microsoft.Research.Science.Data.Utilities.DataSetCloning.Clone(ds, dstUri); //ds.Clone(args[2]);
ds2.Dispose();
sw.Stop();
Console.Write(String.Format("({0}) ", sw.Elapsed));
}
public LinearizingStorageContext(DataSet dataSet)
{
this.dataSet = dataSet;
this.definition = dataSet.GetStorageDefinition();
var uri = DataSetUri.Create(this.dataSet.URI);
if (uri.ContainsParameter("dimensions"))
{
string dimstr = uri.GetParameterValue("dimensions");
var dimpairs = dimstr.Split(',');
foreach (var p in dimpairs)
{
var pair = p.Split(':');
this.definition.DimensionsLengths[pair[0]] = int.Parse(pair[1]);
}
}
requestHandler.ObserveOn(new EventLoopScheduler()).Subscribe(v =>
{
Perform(v);
});
}
static void DoMerge(string[] URIs, string output)
{
DataSetUri dstUri = DataSetUri.Create(output);
if (dstUri.ProviderName.StartsWith("memory"))
{
throw new NotSupportedException("Copying to memory is not supported by the utility.");
}
DataSet[] dss = new DataSet[URIs.Length];
int i = 0;
try
{
for (i = 0; i < dss.Length; i++)
{
dss[i] = DataSet.Open(URIs[i]);
}
}
catch
{
for (; --i >= 0;)
{
dss[i].Dispose();
}
throw;
}
using (DataSet mds = DataSet.Open("msds:memory"))
{
mds.IsAutocommitEnabled = false;
IMetadataConflictResolver conflictResolver = new WarningConflictResolver();
foreach (var ds in dss)
{
// Global metadata
foreach (var attr in ds.Metadata)
{
object val;
if (mds.Metadata.ContainsKey(attr.Key, SchemaVersion.Recent))
{
val = conflictResolver.Resolve(attr.Key, mds.Metadata[attr.Key, SchemaVersion.Recent], attr.Value);
}
else
{
val = attr.Value;
}
mds.Metadata[attr.Key] = val;
}
// Variables
foreach (var var in ds.Variables)
{
mds.AddVariableByReference(var);
}
}
try
{
mds.Commit();
}
catch (DistributedCommitFailedException dex)
{
if (dex.InnerException is ConstraintsFailedException)
{
string error = String.Format("Input DataSets are incompatible: {0}", ((ConstraintsFailedException)(dex.InnerException)).Message);
WriteError(error);
return;
}
throw;
}
catch (ConstraintsFailedException cex)
{
string error = String.Format("Input DataSets are incompatible: {0}", cex.Message);
WriteError(error);
return;
}
Microsoft.Research.Science.Data.Utilities.DataSetCloning.Clone(mds, dstUri,
Microsoft.Research.Science.Data.Utilities.DataSetCloning.DefaultUpdater).Dispose();
}
}
static void DoSlice(string uri, Dictionary <string, Range> ranges, string outputUri)
{
DataSetUri dstUri = DataSetUri.Create(outputUri);
if (dstUri.ProviderName.StartsWith("memory"))
{
throw new NotSupportedException("Copying to memory is not supported by the utility.");
}
using (DataSet src = DataSet.Open(uri))
{
using (DataSet mds = DataSet.Open("msds:memory"))
using (DataSet mds2 = DataSet.Open("msds:memory"))
{
mds.IsAutocommitEnabled = false;
mds2.IsAutocommitEnabled = false;
foreach (var var in src.Variables)
{
bool doSlice = false;
foreach (var dim in var.Dimensions)
{
if (var.Dimensions.Contains(dim.Name))
{
doSlice = true;
break;
}
}
if (doSlice)
{
var refVar = mds.AddVariableByReference(var);
int rank = refVar.Rank;
int[] origin = new int[rank];
int[] stride = new int[rank];
int[] count = new int[rank];
for (int i = 0; i < rank; i++)
{
Range r;
if (ranges.TryGetValue(var.Dimensions[i].Name, out r))
{
if (r.IsReduced)
{
origin[i] = r.Origin;
stride[i] = 0;
count[i] = 1;
}
else
{
throw new NotSupportedException("sds slice supports only dimension fixing");
}
}
else
{
origin[i] = 0;
stride[i] = 1;
count[i] = 0;
}
}
Variable strVar = StrideVariable(refVar, origin, stride, count);
mds2.AddVariableByReference(strVar);
}
else
{
mds2.AddVariableByReference(var);
}
}
// mds2 is ready to be committed
try
{
mds2.Commit();
}
catch (DistributedCommitFailedException dex)
{
if (dex.InnerException is ConstraintsFailedException)
{
string error = String.Format("Input DataSets are incompatible: {0}", ((ConstraintsFailedException)(dex.InnerException)).Message);
WriteError(error);
return;
}
throw;
}
catch (ConstraintsFailedException cex)
{
string error = String.Format("Input DataSets are incompatible: {0}", cex.Message);
WriteError(error);
return;
}
// mds2 is ready to be cloned
Microsoft.Research.Science.Data.Utilities.DataSetCloning.Clone(mds2, dstUri,
Microsoft.Research.Science.Data.Utilities.DataSetCloning.DefaultUpdater).Dispose();
}
}
}
public void ReadTest()
{
Stopwatch sw = new Stopwatch();
var uri = DataSetUri.Create(NetCDFFileName);
uri.OpenMode = ResourceOpenMode.Open;
//sw.Start();
//DataSet ds = DataSet.Open(uri);
//var val = ds.Variables.First(v => v.Name == "values");
//var vals = ds.GetData<Int16[, ,]>(val.ID);
//ds.Dispose();
//sw.Stop();
//var ts = sw.Elapsed;
//sw.Reset();
sw.Start();
var ds = DataSet.Open(uri);
sw.Stop();
var ts2 = sw.Elapsed;
var val = ds.Variables.First(v => v.Name == "values");
var time = ds.Variables.First(t => t.Name == "T").GetData();
var vv = ds.GetData <Int16[, , ]>(val.ID, DataSet.Range(0), DataSet.Range(0), DataSet.Range(0, 1439));
for (int i = 0; i < 1400; i++)
{
vv[0, 0, i] = 299;
}
ds.PutData <Int16[, , ]>(val.ID, vv, DataSet.Range(1), DataSet.Range(1), DataSet.Range(0, 1439));
int r = 110;
int z = 100;
Int16[, ,] grid = new Int16[r, z, 1];
Random rnd = new Random();
for (int k = 0; k < r; k++)
{
for (int i = 0; i < z; i++)
{
grid[k, i, 0] = (Int16)rnd.Next(1024);
}
}
sw.Reset();
sw.Start();
ds.PutData <Int16[, , ]>(val.ID, grid, DataSet.Range(0, r - 1), DataSet.Range(0, z - 1), DataSet.Range(5000));
sw.Stop();
ts2 = sw.Elapsed;
}