internal static NetCDFUri FromFileName(string file)
{
string uri = DataSetUri.CreateFromPath(file,
Microsoft.Research.Science.Data.Factory.DataSetFactory.GetProviderNameByType(typeof(NetCDFDataSet)) ??
((DataSetProviderNameAttribute)typeof(NetCDFDataSet).GetCustomAttributes(typeof(DataSetProviderNameAttribute), false)[0]).Name,
"file");
return(new NetCDFUri(uri));
}
public void DataSourceUpdate(string name)
{
var localDs = db.GetDataSources(DateTime.MaxValue).ToArray();
if (localDs.All(x => x.Name != name))
{
throw new ArgumentException("Specified data source does not exist.");
}
var datasource = localDs.Single(x => x.Name == name);
if (datasource.RemoteID != null)
{
//federated data source
db.SetDataSourceProcessor(name, null, datasource.RemoteID, datasource.RemoteName);
}
else if (!String.IsNullOrEmpty(datasource.Uri))
{
//local datasource
var dsuri = new DataSetUri(datasource.Uri);
Dictionary <string, int> oldDims = new Dictionary <string, int>();
if (dsuri.ContainsParameter("dimensions"))
{
string dimstr = dsuri.GetParameterValue("dimensions");
var dimpairs = dimstr.Split(',');
foreach (var p in dimpairs)
{
var pair = p.Split(':');
oldDims.Add(pair[0], int.Parse(pair[1]));
}
}
DataSet ds;
try
{
ds = DataSet.Open(dsuri);
}
catch (Exception ex)
{
throw new Exception("Failed to open associated dataset: " + ex.Message);
}
var newDims = ds.Dimensions;
if (oldDims.All(d => newDims.Contains(d.Key) && newDims[d.Key].Length == d.Value) && newDims.All(d => oldDims.ContainsKey(d.Name) && oldDims[d.Name] == d.Length))
{
throw new Exception("Associated data set has not grown.");
}
string newDimString = String.Join(",", newDims.Select(d => d.Name + ":" + d.Length.ToString()));
ds.Dispose();
dsuri.SetParameterValue("dimensions", newDimString);
string newUri = dsuri.ToString();
db.SetDataSourceUri(name, newUri);
}
}
public ComputationJob(
Job job, JobsDBDataContext jobsDataContext, // Reference to Jobs database contents including query
SqlExtendedConfigurationProvider configProvider, // current FetchClimate configuration
JobSettings settings, DataSetUri resultDataSetUri
)
: base(job, jobsDataContext, settings)
{
this.configProvider = configProvider;
jobDataset = DataSet.Open(resultDataSetUri);
var wholeRequest = jobDataset.ToFetchRequest();
var subJob = JobManager.EvaluateSubrequestData(wholeRequest, job.PartsCount, job.PartNo);
resultOrigin = subJob.Item2;
Request = subJob.Item1;
}
/// <summary>
/// Copies given dataset into dataset determined by <paramref name="dstUri"/>.
/// </summary>
/// <param name="src">Original dataset to clone.</param>
/// <param name="dstUri">URI of the destination dataset.</param>
/// <param name="updater">Delegate accepting update progressm notifications.</param>
/// <returns>New instance of <see cref="DataSet"/> class.</returns>
/// <remarks>
/// This method splits the original dataser into parts and therefore is able
/// to clone very large datasets not fitting to memory.
/// </remarks>
public static DataSet Clone(DataSet src, DataSetUri dstUri, ProgressUpdater updater)
{
DataSet dst = null;
try
{
dst = DataSet.Open(dstUri);
return(Clone(src, dst, updater));
}
catch
{
if (dst != null)
{
dst.Dispose();
}
throw;
}
}
/// <summary>
/// Initializes an instance.
/// </summary>
/// <param name="uri"></param>
public MemoryDataSet(string uri)
{
if (DataSetUri.IsDataSetUri(uri))
{
this.uri = new MemoryUri(uri);
}
else
{
this.uri = MemoryUri.FromName(uri);
}
bool autocommit = IsAutocommitEnabled;
IsAutocommitEnabled = false;
Name = ((MemoryUri)this.uri).Name;
Commit();
IsAutocommitEnabled = autocommit;
}
static void DoCopy(string uri, string uri2)
{
var ds = OpenDataSet(uri);
Console.Write("Copying . . . ");
Stopwatch sw = new Stopwatch();
sw.Start();
DataSetUri dstUri = DataSetUri.Create(uri2);
if (dstUri.ProviderName.StartsWith("memory"))
{
throw new NotSupportedException("Copying to memory is not supported by the utility.");
}
DataSet ds2 = Microsoft.Research.Science.Data.Utilities.DataSetCloning.Clone(ds, dstUri); //ds.Clone(args[2]);
ds2.Dispose();
sw.Stop();
Console.Write(String.Format("({0}) ", sw.Elapsed));
}
public LinearizingStorageContext(DataSet dataSet)
{
this.dataSet = dataSet;
this.definition = dataSet.GetStorageDefinition();
var uri = DataSetUri.Create(this.dataSet.URI);
if (uri.ContainsParameter("dimensions"))
{
string dimstr = uri.GetParameterValue("dimensions");
var dimpairs = dimstr.Split(',');
foreach (var p in dimpairs)
{
var pair = p.Split(':');
this.definition.DimensionsLengths[pair[0]] = int.Parse(pair[1]);
}
}
requestHandler.ObserveOn(new EventLoopScheduler()).Subscribe(v =>
{
Perform(v);
});
}
public static DataSet Clone(DataSet src, DataSetUri dstUri)
{
if (src == null)
{
throw new ArgumentNullException("src");
}
// Maximum memory capacity in bytes
int N = 200 * 1024 * 1024;
// Estimated size of a single string in bytes
int sizeofString = 100 * 1024;
/***********************************************************************************
* Preparing output
***********************************************************************************/
DataSet dst = DataSet.Open(dstUri);
if (dst.IsReadOnly)
{
throw new NotSupportedException("Output DataSet is read-only");
}
dst.IsAutocommitEnabled = false;
DataSetSchema srcSchema = src.GetSchema();
Dictionary <int, int> IDs = new Dictionary <int, int>();
// Creating empty variables and copying global metadata and scalar variables
Console.Out.Write("\n\nCreating structure and copying global metadata and scalar variables... ");
foreach (VariableSchema v in srcSchema.Variables)
{
if (v.ID == DataSet.GlobalMetadataVariableID)
{
// Copying global metadata
var dstGlobalMetadata = dst.Metadata;
foreach (var attr in v.Metadata)
{
dstGlobalMetadata[attr.Key] = attr.Value;
}
continue;
}
Variable t = dst.AddVariable(v.TypeOfData, v.Name, null, v.Dimensions.AsNamesArray());
IDs.Add(v.ID, t.ID);
foreach (var attr in v.Metadata)
{
t.Metadata[attr.Key] = attr.Value;
}
if (t.Rank == 0) // scalar
{
t.PutData(src.Variables.GetByID(v.ID).GetData());
}
}
dst.Commit();
Console.Out.WriteLine("Done.\n");
/***********************************************************************************
* Adjusting dimensions deltas
***********************************************************************************/
Dimension[] srcDims = srcSchema.GetDimensions();
Dictionary <string, int> deltas = new Dictionary <string, int>(srcDims.Length);
foreach (var d in srcDims)
{
deltas[d.Name] = d.Length;
}
Console.Out.WriteLine("Total memory capacity: " + (N / 1024.0 / 1024.0).ToString("F2") + " Mb");
int totalSize;
do
{
totalSize = 0;
foreach (var var in srcSchema.Variables)
{
if (var.Rank == 0)
{
continue; // scalar
}
int typeSize = SizeOf(var.TypeOfData, sizeofString);
int count = 0;
foreach (var vdim in var.Dimensions)
{
int dimDelta = deltas[vdim.Name];
if (count == 0)
{
count = dimDelta;
}
else
{
count *= dimDelta;
}
}
totalSize += typeSize * count;
}
if (totalSize > N)
{
string maxDim = null;
int max = int.MinValue;
foreach (var dim in deltas)
{
if (dim.Value > max)
{
max = dim.Value;
maxDim = dim.Key;
}
}
if (maxDim == null || max <= 1)
{
throw new NotSupportedException("Cannot copy the DataSet: it is too large to be copied entirely by the utility for the provided memory capacity");
}
deltas[maxDim] = max >> 1;
}
} while (totalSize > N);
// Printing deltas
Console.Out.WriteLine("Deltas for the dimensions adjusted (max iteration capacity: " + (totalSize / 1024.0 / 1024.0).ToString("F2") + " Mb):");
foreach (var delta in deltas)
{
Console.Out.WriteLine(" Dimension " + delta.Key + ": " + delta.Value);
}
/***********************************************************************************
* Copying data
***********************************************************************************/
Console.WriteLine();
UpdateProgress(0);
Dictionary <int, int[]> origins = new Dictionary <int, int[]>(srcSchema.Variables.Length);
Dictionary <int, int[]> shapes = new Dictionary <int, int[]>(srcSchema.Variables.Length);
List <VariableSchema> copyVars = srcSchema.Variables.Where(vs =>
(vs.Rank > 0 && vs.ID != DataSet.GlobalMetadataVariableID)).ToList();
Dictionary <string, int> dimOrigin = new Dictionary <string, int>(srcDims.Length);
foreach (var d in srcDims)
{
dimOrigin[d.Name] = 0;
}
Array.Sort(srcDims, (d1, d2) => d1.Length - d2.Length);
int totalDims = srcDims.Length;
do
{
// for each variable:
for (int varIndex = copyVars.Count; --varIndex >= 0;)
{
VariableSchema var = copyVars[varIndex];
bool hasChanged = false;
// Getting its origin
int[] origin;
if (!origins.TryGetValue(var.ID, out origin))
{
origin = new int[var.Rank];
origins[var.ID] = origin;
hasChanged = true;
}
// Getting its shape
int[] shape;
if (!shapes.TryGetValue(var.ID, out shape))
{
shape = new int[var.Rank];
for (int i = 0; i < var.Dimensions.Count; i++)
{
shape[i] = deltas[var.Dimensions[i].Name];
}
shapes.Add(var.ID, shape);
}
// Updating origin for the variable:
if (!hasChanged)
{
for (int i = 0; i < shape.Length; i++)
{
int o = dimOrigin[var.Dimensions[i].Name];
if (origin[i] != o)
{
hasChanged = true;
origin[i] = o;
}
}
}
if (!hasChanged) // this block is already copied
{
continue;
}
bool doCopy = false;
bool shapeUpdated = false;
for (int i = 0; i < shape.Length; i++)
{
int s = origin[i] + shape[i];
int len = var.Dimensions[i].Length;
if (s > len)
{
if (!shapeUpdated)
{
shapeUpdated = true;
shape = (int[])shape.Clone();
}
shape[i] = len - origin[i];
}
if (shape[i] > 0)
{
doCopy = true;
}
}
if (doCopy)
{
Array data = src.Variables.GetByID(var.ID).GetData(origin, shape);
// Compute real size here for strings
dst.Variables.GetByID(IDs[var.ID]).PutData(origin, data);
}
else // variable is copied
{
copyVars.RemoveAt(varIndex);
}
}
dst.Commit();
// Updating dimensions origin
bool isOver = true;
for (int i = 0; i < totalDims; i++)
{
Dimension dim = srcDims[i];
int origin = dimOrigin[dim.Name] + deltas[dim.Name];
if (origin < dim.Length)
{
dimOrigin[dim.Name] = origin;
isOver = false;
// Progress indicator
if (i == totalDims - 1)
{
double perc = (double)origin / dim.Length * 100.0;
UpdateProgress(perc);
}
break;
}
dimOrigin[dim.Name] = 0;
}
if (isOver)
{
break;
}
} while (copyVars.Count > 0);
UpdateProgress(100.0);
Console.Out.WriteLine();
return(dst);
}
/// <summary>
/// Creates the <see cref="DataSetUri"/> instance for the specified <paramref name="uri"></paramref>.
/// </summary>
/// <param name="uri">Uri containing provider name and probably parameters.</param>
/// <returns><see cref="DataSetUri"/> instance.</returns>
/// <remarks>
/// Creates a class inhereted from DataSetUri. Method looks in registered providers for
/// attribute <see cref="DataSetProviderUriTypeAttribute"/> and returns an appropriate class instance.
/// If such class is not found, throws an exception.
/// <para>
/// See remarks for <see cref="DataSetUri.Create(string)"/> for details.</para>
/// </remarks>
/// <seealso cref="CreateUri(Type)"/>
/// <seealso cref="DataSetUri.Create(string)"/>
/// <seealso cref="DataSet.Open(DataSetUri)"/>
public static DataSetUri CreateUri(string uri)
{
Trace.WriteLineIf(dataSetTraceLevel == TraceLevel.Verbose, "DataSetFactory: creating " + uri + " . . . ");
if (String.IsNullOrEmpty(uri))
uri = "msds:memory";
if (uri.StartsWith(DataSetUri.DataSetUriScheme + ":"))
{
DataSetUri dsUri = new DataSetUri(uri);
string provider = dsUri.ProviderName;
if (!providersByName.ContainsKey(provider))
throw BuildProviderNotRegisteredException(uri, "Provider " + provider + " is not registered");
DataSetFactoryEntry entry = providersByName[provider];
DataSetProviderUriTypeAttribute[] v = (DataSetProviderUriTypeAttribute[])entry.DataSetType.GetCustomAttributes(typeof(DataSetProviderUriTypeAttribute), false);
if ((v != null) && (v.Length > 0))
{
return (DataSetUri)v[0].UriType.GetConstructor(new Type[] { typeof(string) }).Invoke(new object[] { uri });
}
throw new ArgumentException("DataSetUri can't be built from " + uri + " uri for related provider type has no proper attributes");
}
// Creating data set by extension or provider name
string providerName;
string tmp; List<string> param = new List<string>();
if (uri.Contains("?"))
{
param.AddRange(uri.Split('?'));
tmp = param[0];
param.RemoveAt(0);
}
else tmp = uri;
if (uri.Contains(".") || uri.Contains("?"))
{
providerName = GetProviderNameByExtention(tmp);
if (String.IsNullOrEmpty(providerName))
throw new ArgumentException("DataSetUri can't be built from path " + uri + ": extension is not registered");
string newUri = DataSetUri.DataSetUriScheme + ":" + providerName;
if ((uri.Contains(".") || uri.Contains("?")) && (!uri.StartsWith(".")))
newUri = newUri + "?file=" + tmp;
if (param.Count > 0)
newUri = newUri + "&" + param[0];
if (!providersByName.ContainsKey(providerName))
throw BuildProviderNotRegisteredException(uri, "Provider " + providerName + " is not registered");
DataSetFactoryEntry entry = providersByName[providerName];
DataSetProviderUriTypeAttribute[] v = (DataSetProviderUriTypeAttribute[])entry.DataSetType.GetCustomAttributes(typeof(DataSetProviderUriTypeAttribute), false);
if ((v != null) && (v.Length > 0))
{
return (DataSetUri)
v[0].UriType.GetConstructor(new Type[] { typeof(string) }).Invoke(new object[] { newUri });
}
throw new ArgumentException("DataSetUri can't be built from " + uri + " uri for provider type has no proper attributes");
}
throw new ArgumentException("DataSetUri can't be build from " + uri + " uri for it is incorrect");
}
/// <summary>
/// Creates the <see cref="DataSet"/> instance for the specified <paramref name="uri"/>.
/// </summary>
/// <param name="uri">URI containing provider name and parameters.</param>
/// <returns>New instance of the <see cref="DataSet"/>.</returns>
/// <remarks>
/// See remarks for <see cref="DataSet.Open(DataSetUri)"/>.</remarks>
/// <seealso cref="Create(string)"/>
/// <seealso cref="CreateUri(string)"/>
public static DataSet Create(DataSetUri uri)
{
if (uri == null)
throw new ArgumentNullException("uri");
Trace.WriteLineIf(dataSetTraceLevel == TraceLevel.Info, "DataSetFactory: creating " + uri + " . . . ");
string provider = uri.ProviderName;
if (String.IsNullOrEmpty(provider))
throw BuildDataSetCreateException(uri.ToString(), "Provider name is empty or null");
if (!providersByName.ContainsKey(provider))
throw BuildProviderNotRegisteredException(uri.ToString(), "Provider " + provider + " is not registered");
DataSetFactoryEntry entry = providersByName[provider];
DataSet result = entry.CreateDataSet(uri.ToString());
List<string> includedUris = uri.GetParameterValues("include");
try
{
if (includedUris.Count > 0)
{
foreach (string includedUri in includedUris)
result.IncludeDataSet(includedUri);
result.Commit();
}
}
catch (DataSetCreateException)
{
throw;
}
catch (System.Reflection.TargetInvocationException invEx)
{
throw BuildDataSetCreateException(uri.ToString(), invEx.InnerException == null ? invEx : invEx.InnerException);
}
catch (Exception ex)
{
throw BuildDataSetCreateException(uri.ToString(), ex);
}
return result;
}
/// <summary>
/// Creates the <see cref="DataSet"/> instance for the specified <paramref name="uri"/>.
/// </summary>
/// <param name="uri">URI describing the DataSet to create.</param>
/// <returns>New instance of the <see cref="DataSet"/>.</returns>
/// <remarks>
/// <para>The method creates a <see cref="DataSet"/> instance based on the <paramref name="uri"/>.
/// If the uri contains schema definition (i.e. starts with <c>"msds:"</c>) the factory
/// tries to find a corresponded provider type from the registered providers table
/// (see method <see cref="Register(Type)"/>). If the provider is not registered,
/// an exception will be thrown.</para>
/// <para>If the uri has no schema definition, the string is considered as a path and
/// the factory looks for provider by the extension of the given path
/// (see method <see cref="RegisterExtension(Type)"/>). If there are more than one
/// providers associated with the extension, they will be constructed one by one until
/// the data set is created successfully.
/// </para>
/// <para>
/// The path also may contain parameters specified at the end after '?' (if the
/// provider associated with the extensions supports this feature).
/// For example, the code:
/// <code>
/// DataSet ds = DataSet.Open(@"c:\data\air0.csv?openMode=open&fillUpMissingValues=true&inferInt=true");
/// </code>
/// does the same as the code:
/// <code>
/// DataSet ds = DataSet.Open(@"msds:csv?file=c:\data\air0.csv&openMode=open&fillUpMissingValues=true&inferInt=true");
/// </code>
/// </para>
/// <para>
/// If the <paramref name="uri"/> is null or an empty string, an instance of the
/// <see cref="T:Microsoft.Research.Science.Data.Memory.MemoryDataSet"/> is created.
/// </para>
/// <para>
/// See also remarks for the <see cref="DataSetFactory"/> class.
/// </para>
/// </remarks>
/// <seealso cref="Register(string, Type)"/>
/// <seealso cref="Register(Type)"/>
/// <seealso cref="RegisterExtension(Type)"/>
/// <seealso cref="RegisterExtension(string, Type)"/>
/// <seealso cref="RegisterAssembly(Assembly)"/>
/// <seealso cref="SearchFolder(string)"/>
/// <seealso cref="Microsoft.Research.Science.Data.DataSetUri"/>
/// <exception cref="ProviderNotRegisteredException">
/// DataSet is not registered in the factory.</exception>
/// <exception cref="DataSetCreateException">
/// Particular DataSet provider has thrown an exception at construction.</exception>
/// <exception cref="ArgumentException" />
/// <exception cref="InvalidOperationException" />
public static DataSet Create(string uri)
{
if (String.IsNullOrEmpty(uri))
uri = "msds:memory";
if (uri.StartsWith(DataSetUri.DataSetUriScheme + ":"))
{
DataSetUri dsUri = new DataSetUri(uri);
return Create(dsUri);
}
Trace.WriteLineIf(dataSetTraceLevel == TraceLevel.Info, "DataSetFactory: creating " + uri + " . . . ");
// Creating data set by extension
string ext = GetExtension(uri);
if (string.IsNullOrEmpty(ext))
throw BuildDataSetCreateException(uri, "Extension is empty or null");
if (!providersByExt.ContainsKey(ext))
throw BuildProviderNotRegisteredException(uri, "Extension " + ext + " is not registered in the factory");
var list = providersByExt[ext];
if (list.Count == 0)
throw BuildProviderNotRegisteredException(uri, "No providers associated with the extension " + ext);
Exception refuseEx = null;
foreach (var e in list)
{
try
{
return (DataSet)e.Constructor.Invoke(new object[] { uri });
}
catch (TargetInvocationException invEx)
{
refuseEx = invEx.InnerException == null ? invEx : invEx.InnerException;
Trace.WriteLineIf(dataSetTraceLevel == TraceLevel.Warning, "Provider " + e.DataSetType.Name + " refused the uri.");
}
catch (Exception ex)
{
refuseEx = ex;
Trace.WriteLineIf(dataSetTraceLevel == TraceLevel.Warning, "Provider " + e.DataSetType.Name + " refused the uri.");
}
}
throw BuildDataSetCreateException(uri, refuseEx);
}
/// <summary>
/// Gets an array of extensions (".ext") for the given provider name.
/// </summary>
/// <param name="name"></param>
/// <returns></returns>
public static string[] GetExtensionsForProvider(string name)
{
if (name == null)
throw new ArgumentNullException("name");
if (providersByName.Count == 0)
return new string[0];
if (DataSetUri.IsDataSetUri(name))
name = new DataSetUri(name).ProviderName;
if (providersByName.ContainsKey(name))
{
var entry = providersByName[name];
List<string> extensions = new List<string>();
Type dst = entry.DataSetType;
foreach (var v in providersByExt)
{
if (v.Value.Exists(p => p.DataSetType == dst))
{
extensions.Add(v.Key);
}
}
return extensions.ToArray();
}
return new string[0];
}
static void Main(string[] args)
{
var schema = new SerializableDataSetSchema(
new SerializableDimension[] {
new SerializableDimension("i", 30000),
new SerializableDimension("j", 30000)
},
new SerializableVariableSchema[] {
new SerializableVariableSchema("vals", typeof(double), new string[] { "i", "j" }, null)
},
null
);
string BlobConnectionAccountName = @"fetchclimate2";
string BlobConnectionAccountKey = @"vQpyUA7h5QFX6VlEH944gyv/h2Kx//WDy32brNip+YKDpsrN5/pxcSOnP2igQQ5pkA8lRXkmqmAYrgB29nwo/w==";
string uri = @"msds:ab?DefaultEndpointsProtocol=http&Container=testcontainer&Blob=testBlob30000x30000&AccountName=" + BlobConnectionAccountName + @"&AccountKey=" + BlobConnectionAccountKey;
try
{
var ds = AzureBlobDataSet.CreateEmptySet(uri, schema);
double[,] data = new double[1, 30000];
for (int i = 0; i < 30000; ++i)
{
data[0, i] = (double)i;
}
ds["vals"].PutData(new int[] { 29999, 0 }, data);
var recvData = (double[, ])ds["vals"].GetData(new int[] { 29999, 0 }, new int[] { 1, 30000 });
for (int i = 0; i < 30000; ++i)
{
if (data[0, i] != recvData[0, i])
{
throw new Exception("difference at " + i.ToString());
}
}
Console.WriteLine("Everything is successful!");
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
Console.ReadLine();
//delete test blob
try
{
AzureBlobDataSetUri azureUri = null;
if (DataSetUri.IsDataSetUri(uri))
{
azureUri = new AzureBlobDataSetUri(uri);
}
else
{
azureUri = AzureBlobDataSetUri.ToUri(uri);
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container);
CloudPageBlob blob = container.GetPageBlobReference(azureUri.Blob);
blob.DeleteIfExists();
Console.WriteLine("Deleted test blob successfully!");
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
Console.ReadLine();
}
static void DoMerge(string[] URIs, string output)
{
DataSetUri dstUri = DataSetUri.Create(output);
if (dstUri.ProviderName.StartsWith("memory"))
{
throw new NotSupportedException("Copying to memory is not supported by the utility.");
}
DataSet[] dss = new DataSet[URIs.Length];
int i = 0;
try
{
for (i = 0; i < dss.Length; i++)
{
dss[i] = DataSet.Open(URIs[i]);
}
}
catch
{
for (; --i >= 0;)
{
dss[i].Dispose();
}
throw;
}
using (DataSet mds = DataSet.Open("msds:memory"))
{
mds.IsAutocommitEnabled = false;
IMetadataConflictResolver conflictResolver = new WarningConflictResolver();
foreach (var ds in dss)
{
// Global metadata
foreach (var attr in ds.Metadata)
{
object val;
if (mds.Metadata.ContainsKey(attr.Key, SchemaVersion.Recent))
{
val = conflictResolver.Resolve(attr.Key, mds.Metadata[attr.Key, SchemaVersion.Recent], attr.Value);
}
else
{
val = attr.Value;
}
mds.Metadata[attr.Key] = val;
}
// Variables
foreach (var var in ds.Variables)
{
mds.AddVariableByReference(var);
}
}
try
{
mds.Commit();
}
catch (DistributedCommitFailedException dex)
{
if (dex.InnerException is ConstraintsFailedException)
{
string error = String.Format("Input DataSets are incompatible: {0}", ((ConstraintsFailedException)(dex.InnerException)).Message);
WriteError(error);
return;
}
throw;
}
catch (ConstraintsFailedException cex)
{
string error = String.Format("Input DataSets are incompatible: {0}", cex.Message);
WriteError(error);
return;
}
Microsoft.Research.Science.Data.Utilities.DataSetCloning.Clone(mds, dstUri,
Microsoft.Research.Science.Data.Utilities.DataSetCloning.DefaultUpdater).Dispose();
}
}
static void DoSlice(string uri, Dictionary <string, Range> ranges, string outputUri)
{
DataSetUri dstUri = DataSetUri.Create(outputUri);
if (dstUri.ProviderName.StartsWith("memory"))
{
throw new NotSupportedException("Copying to memory is not supported by the utility.");
}
using (DataSet src = DataSet.Open(uri))
{
using (DataSet mds = DataSet.Open("msds:memory"))
using (DataSet mds2 = DataSet.Open("msds:memory"))
{
mds.IsAutocommitEnabled = false;
mds2.IsAutocommitEnabled = false;
foreach (var var in src.Variables)
{
bool doSlice = false;
foreach (var dim in var.Dimensions)
{
if (var.Dimensions.Contains(dim.Name))
{
doSlice = true;
break;
}
}
if (doSlice)
{
var refVar = mds.AddVariableByReference(var);
int rank = refVar.Rank;
int[] origin = new int[rank];
int[] stride = new int[rank];
int[] count = new int[rank];
for (int i = 0; i < rank; i++)
{
Range r;
if (ranges.TryGetValue(var.Dimensions[i].Name, out r))
{
if (r.IsReduced)
{
origin[i] = r.Origin;
stride[i] = 0;
count[i] = 1;
}
else
{
throw new NotSupportedException("sds slice supports only dimension fixing");
}
}
else
{
origin[i] = 0;
stride[i] = 1;
count[i] = 0;
}
}
Variable strVar = StrideVariable(refVar, origin, stride, count);
mds2.AddVariableByReference(strVar);
}
else
{
mds2.AddVariableByReference(var);
}
}
// mds2 is ready to be committed
try
{
mds2.Commit();
}
catch (DistributedCommitFailedException dex)
{
if (dex.InnerException is ConstraintsFailedException)
{
string error = String.Format("Input DataSets are incompatible: {0}", ((ConstraintsFailedException)(dex.InnerException)).Message);
WriteError(error);
return;
}
throw;
}
catch (ConstraintsFailedException cex)
{
string error = String.Format("Input DataSets are incompatible: {0}", cex.Message);
WriteError(error);
return;
}
// mds2 is ready to be cloned
Microsoft.Research.Science.Data.Utilities.DataSetCloning.Clone(mds2, dstUri,
Microsoft.Research.Science.Data.Utilities.DataSetCloning.DefaultUpdater).Dispose();
}
}
}
public void ReadTest()
{
Stopwatch sw = new Stopwatch();
var uri = DataSetUri.Create(NetCDFFileName);
uri.OpenMode = ResourceOpenMode.Open;
//sw.Start();
//DataSet ds = DataSet.Open(uri);
//var val = ds.Variables.First(v => v.Name == "values");
//var vals = ds.GetData<Int16[, ,]>(val.ID);
//ds.Dispose();
//sw.Stop();
//var ts = sw.Elapsed;
//sw.Reset();
sw.Start();
var ds = DataSet.Open(uri);
sw.Stop();
var ts2 = sw.Elapsed;
var val = ds.Variables.First(v => v.Name == "values");
var time = ds.Variables.First(t => t.Name == "T").GetData();
var vv = ds.GetData <Int16[, , ]>(val.ID, DataSet.Range(0), DataSet.Range(0), DataSet.Range(0, 1439));
for (int i = 0; i < 1400; i++)
{
vv[0, 0, i] = 299;
}
ds.PutData <Int16[, , ]>(val.ID, vv, DataSet.Range(1), DataSet.Range(1), DataSet.Range(0, 1439));
int r = 110;
int z = 100;
Int16[, ,] grid = new Int16[r, z, 1];
Random rnd = new Random();
for (int k = 0; k < r; k++)
{
for (int i = 0; i < z; i++)
{
grid[k, i, 0] = (Int16)rnd.Next(1024);
}
}
sw.Reset();
sw.Start();
ds.PutData <Int16[, , ]>(val.ID, grid, DataSet.Range(0, r - 1), DataSet.Range(0, z - 1), DataSet.Range(5000));
sw.Stop();
ts2 = sw.Elapsed;
}
/// <summary>
/// Copies given dataset into dataset determined by <paramref name="dstUri"/> and prints progress into console.
/// </summary>
/// <param name="src">Original dataset to clone.</param>
/// <param name="dstUri">URI of the destination dataset.</param>
/// <returns>New instance of <see cref="DataSet"/> class.</returns>
/// <remarks><para>
/// This method splits the original dataser into parts and therefore is able
/// to clone very large datasets not fitting to memory.</para>
/// <para>Progress is printed out into the console window.</para>
/// </remarks>
/// <seealso cref="DataSetCloning.Clone(DataSet,DataSetUri,ProgressUpdater)"/>
public static DataSet Clone(DataSet src, DataSetUri dstUri)
{
return(Clone(src, dstUri, DefaultUpdater));
}
