internal static NetCDFUri FromFileName(string file) { string uri = DataSetUri.CreateFromPath(file, Microsoft.Research.Science.Data.Factory.DataSetFactory.GetProviderNameByType(typeof(NetCDFDataSet)) ?? ((DataSetProviderNameAttribute)typeof(NetCDFDataSet).GetCustomAttributes(typeof(DataSetProviderNameAttribute), false)[0]).Name, "file"); return(new NetCDFUri(uri)); }
public void DataSourceUpdate(string name) { var localDs = db.GetDataSources(DateTime.MaxValue).ToArray(); if (localDs.All(x => x.Name != name)) { throw new ArgumentException("Specified data source does not exist."); } var datasource = localDs.Single(x => x.Name == name); if (datasource.RemoteID != null) { //federated data source db.SetDataSourceProcessor(name, null, datasource.RemoteID, datasource.RemoteName); } else if (!String.IsNullOrEmpty(datasource.Uri)) { //local datasource var dsuri = new DataSetUri(datasource.Uri); Dictionary <string, int> oldDims = new Dictionary <string, int>(); if (dsuri.ContainsParameter("dimensions")) { string dimstr = dsuri.GetParameterValue("dimensions"); var dimpairs = dimstr.Split(','); foreach (var p in dimpairs) { var pair = p.Split(':'); oldDims.Add(pair[0], int.Parse(pair[1])); } } DataSet ds; try { ds = DataSet.Open(dsuri); } catch (Exception ex) { throw new Exception("Failed to open associated dataset: " + ex.Message); } var newDims = ds.Dimensions; if (oldDims.All(d => newDims.Contains(d.Key) && newDims[d.Key].Length == d.Value) && newDims.All(d => oldDims.ContainsKey(d.Name) && oldDims[d.Name] == d.Length)) { throw new Exception("Associated data set has not grown."); } string newDimString = String.Join(",", newDims.Select(d => d.Name + ":" + d.Length.ToString())); ds.Dispose(); dsuri.SetParameterValue("dimensions", newDimString); string newUri = dsuri.ToString(); db.SetDataSourceUri(name, newUri); } }
public ComputationJob( Job job, JobsDBDataContext jobsDataContext, // Reference to Jobs database contents including query SqlExtendedConfigurationProvider configProvider, // current FetchClimate configuration JobSettings settings, DataSetUri resultDataSetUri ) : base(job, jobsDataContext, settings) { this.configProvider = configProvider; jobDataset = DataSet.Open(resultDataSetUri); var wholeRequest = jobDataset.ToFetchRequest(); var subJob = JobManager.EvaluateSubrequestData(wholeRequest, job.PartsCount, job.PartNo); resultOrigin = subJob.Item2; Request = subJob.Item1; }
/// <summary> /// Copies given dataset into dataset determined by <paramref name="dstUri"/>. /// </summary> /// <param name="src">Original dataset to clone.</param> /// <param name="dstUri">URI of the destination dataset.</param> /// <param name="updater">Delegate accepting update progressm notifications.</param> /// <returns>New instance of <see cref="DataSet"/> class.</returns> /// <remarks> /// This method splits the original dataser into parts and therefore is able /// to clone very large datasets not fitting to memory. /// </remarks> public static DataSet Clone(DataSet src, DataSetUri dstUri, ProgressUpdater updater) { DataSet dst = null; try { dst = DataSet.Open(dstUri); return(Clone(src, dst, updater)); } catch { if (dst != null) { dst.Dispose(); } throw; } }
/// <summary> /// Initializes an instance. /// </summary> /// <param name="uri"></param> public MemoryDataSet(string uri) { if (DataSetUri.IsDataSetUri(uri)) { this.uri = new MemoryUri(uri); } else { this.uri = MemoryUri.FromName(uri); } bool autocommit = IsAutocommitEnabled; IsAutocommitEnabled = false; Name = ((MemoryUri)this.uri).Name; Commit(); IsAutocommitEnabled = autocommit; }
static void DoCopy(string uri, string uri2) { var ds = OpenDataSet(uri); Console.Write("Copying . . . "); Stopwatch sw = new Stopwatch(); sw.Start(); DataSetUri dstUri = DataSetUri.Create(uri2); if (dstUri.ProviderName.StartsWith("memory")) { throw new NotSupportedException("Copying to memory is not supported by the utility."); } DataSet ds2 = Microsoft.Research.Science.Data.Utilities.DataSetCloning.Clone(ds, dstUri); //ds.Clone(args[2]); ds2.Dispose(); sw.Stop(); Console.Write(String.Format("({0}) ", sw.Elapsed)); }
public LinearizingStorageContext(DataSet dataSet) { this.dataSet = dataSet; this.definition = dataSet.GetStorageDefinition(); var uri = DataSetUri.Create(this.dataSet.URI); if (uri.ContainsParameter("dimensions")) { string dimstr = uri.GetParameterValue("dimensions"); var dimpairs = dimstr.Split(','); foreach (var p in dimpairs) { var pair = p.Split(':'); this.definition.DimensionsLengths[pair[0]] = int.Parse(pair[1]); } } requestHandler.ObserveOn(new EventLoopScheduler()).Subscribe(v => { Perform(v); }); }
public static DataSet Clone(DataSet src, DataSetUri dstUri) { if (src == null) { throw new ArgumentNullException("src"); } // Maximum memory capacity in bytes int N = 200 * 1024 * 1024; // Estimated size of a single string in bytes int sizeofString = 100 * 1024; /*********************************************************************************** * Preparing output ***********************************************************************************/ DataSet dst = DataSet.Open(dstUri); if (dst.IsReadOnly) { throw new NotSupportedException("Output DataSet is read-only"); } dst.IsAutocommitEnabled = false; DataSetSchema srcSchema = src.GetSchema(); Dictionary <int, int> IDs = new Dictionary <int, int>(); // Creating empty variables and copying global metadata and scalar variables Console.Out.Write("\n\nCreating structure and copying global metadata and scalar variables... "); foreach (VariableSchema v in srcSchema.Variables) { if (v.ID == DataSet.GlobalMetadataVariableID) { // Copying global metadata var dstGlobalMetadata = dst.Metadata; foreach (var attr in v.Metadata) { dstGlobalMetadata[attr.Key] = attr.Value; } continue; } Variable t = dst.AddVariable(v.TypeOfData, v.Name, null, v.Dimensions.AsNamesArray()); IDs.Add(v.ID, t.ID); foreach (var attr in v.Metadata) { t.Metadata[attr.Key] = attr.Value; } if (t.Rank == 0) // scalar { t.PutData(src.Variables.GetByID(v.ID).GetData()); } } dst.Commit(); Console.Out.WriteLine("Done.\n"); /*********************************************************************************** * Adjusting dimensions deltas ***********************************************************************************/ Dimension[] srcDims = srcSchema.GetDimensions(); Dictionary <string, int> deltas = new Dictionary <string, int>(srcDims.Length); foreach (var d in srcDims) { deltas[d.Name] = d.Length; } Console.Out.WriteLine("Total memory capacity: " + (N / 1024.0 / 1024.0).ToString("F2") + " Mb"); int totalSize; do { totalSize = 0; foreach (var var in srcSchema.Variables) { if (var.Rank == 0) { continue; // scalar } int typeSize = SizeOf(var.TypeOfData, sizeofString); int count = 0; foreach (var vdim in var.Dimensions) { int dimDelta = deltas[vdim.Name]; if (count == 0) { count = dimDelta; } else { count *= dimDelta; } } totalSize += typeSize * count; } if (totalSize > N) { string maxDim = null; int max = int.MinValue; foreach (var dim in deltas) { if (dim.Value > max) { max = dim.Value; maxDim = dim.Key; } } if (maxDim == null || max <= 1) { throw new NotSupportedException("Cannot copy the DataSet: it is too large to be copied entirely by the utility for the provided memory capacity"); } deltas[maxDim] = max >> 1; } } while (totalSize > N); // Printing deltas Console.Out.WriteLine("Deltas for the dimensions adjusted (max iteration capacity: " + (totalSize / 1024.0 / 1024.0).ToString("F2") + " Mb):"); foreach (var delta in deltas) { Console.Out.WriteLine(" Dimension " + delta.Key + ": " + delta.Value); } /*********************************************************************************** * Copying data ***********************************************************************************/ Console.WriteLine(); UpdateProgress(0); Dictionary <int, int[]> origins = new Dictionary <int, int[]>(srcSchema.Variables.Length); Dictionary <int, int[]> shapes = new Dictionary <int, int[]>(srcSchema.Variables.Length); List <VariableSchema> copyVars = srcSchema.Variables.Where(vs => (vs.Rank > 0 && vs.ID != DataSet.GlobalMetadataVariableID)).ToList(); Dictionary <string, int> dimOrigin = new Dictionary <string, int>(srcDims.Length); foreach (var d in srcDims) { dimOrigin[d.Name] = 0; } Array.Sort(srcDims, (d1, d2) => d1.Length - d2.Length); int totalDims = srcDims.Length; do { // for each variable: for (int varIndex = copyVars.Count; --varIndex >= 0;) { VariableSchema var = copyVars[varIndex]; bool hasChanged = false; // Getting its origin int[] origin; if (!origins.TryGetValue(var.ID, out origin)) { origin = new int[var.Rank]; origins[var.ID] = origin; hasChanged = true; } // Getting its shape int[] shape; if (!shapes.TryGetValue(var.ID, out shape)) { shape = new int[var.Rank]; for (int i = 0; i < var.Dimensions.Count; i++) { shape[i] = deltas[var.Dimensions[i].Name]; } shapes.Add(var.ID, shape); } // Updating origin for the variable: if (!hasChanged) { for (int i = 0; i < shape.Length; i++) { int o = dimOrigin[var.Dimensions[i].Name]; if (origin[i] != o) { hasChanged = true; origin[i] = o; } } } if (!hasChanged) // this block is already copied { continue; } bool doCopy = false; bool shapeUpdated = false; for (int i = 0; i < shape.Length; i++) { int s = origin[i] + shape[i]; int len = var.Dimensions[i].Length; if (s > len) { if (!shapeUpdated) { shapeUpdated = true; shape = (int[])shape.Clone(); } shape[i] = len - origin[i]; } if (shape[i] > 0) { doCopy = true; } } if (doCopy) { Array data = src.Variables.GetByID(var.ID).GetData(origin, shape); // Compute real size here for strings dst.Variables.GetByID(IDs[var.ID]).PutData(origin, data); } else // variable is copied { copyVars.RemoveAt(varIndex); } } dst.Commit(); // Updating dimensions origin bool isOver = true; for (int i = 0; i < totalDims; i++) { Dimension dim = srcDims[i]; int origin = dimOrigin[dim.Name] + deltas[dim.Name]; if (origin < dim.Length) { dimOrigin[dim.Name] = origin; isOver = false; // Progress indicator if (i == totalDims - 1) { double perc = (double)origin / dim.Length * 100.0; UpdateProgress(perc); } break; } dimOrigin[dim.Name] = 0; } if (isOver) { break; } } while (copyVars.Count > 0); UpdateProgress(100.0); Console.Out.WriteLine(); return(dst); }
/// <summary> /// Creates the <see cref="DataSetUri"/> instance for the specified <paramref name="uri"></paramref>. /// </summary> /// <param name="uri">Uri containing provider name and probably parameters.</param> /// <returns><see cref="DataSetUri"/> instance.</returns> /// <remarks> /// Creates a class inhereted from DataSetUri. Method looks in registered providers for /// attribute <see cref="DataSetProviderUriTypeAttribute"/> and returns an appropriate class instance. /// If such class is not found, throws an exception. /// <para> /// See remarks for <see cref="DataSetUri.Create(string)"/> for details.</para> /// </remarks> /// <seealso cref="CreateUri(Type)"/> /// <seealso cref="DataSetUri.Create(string)"/> /// <seealso cref="DataSet.Open(DataSetUri)"/> public static DataSetUri CreateUri(string uri) { Trace.WriteLineIf(dataSetTraceLevel == TraceLevel.Verbose, "DataSetFactory: creating " + uri + " . . . "); if (String.IsNullOrEmpty(uri)) uri = "msds:memory"; if (uri.StartsWith(DataSetUri.DataSetUriScheme + ":")) { DataSetUri dsUri = new DataSetUri(uri); string provider = dsUri.ProviderName; if (!providersByName.ContainsKey(provider)) throw BuildProviderNotRegisteredException(uri, "Provider " + provider + " is not registered"); DataSetFactoryEntry entry = providersByName[provider]; DataSetProviderUriTypeAttribute[] v = (DataSetProviderUriTypeAttribute[])entry.DataSetType.GetCustomAttributes(typeof(DataSetProviderUriTypeAttribute), false); if ((v != null) && (v.Length > 0)) { return (DataSetUri)v[0].UriType.GetConstructor(new Type[] { typeof(string) }).Invoke(new object[] { uri }); } throw new ArgumentException("DataSetUri can't be built from " + uri + " uri for related provider type has no proper attributes"); } // Creating data set by extension or provider name string providerName; string tmp; List<string> param = new List<string>(); if (uri.Contains("?")) { param.AddRange(uri.Split('?')); tmp = param[0]; param.RemoveAt(0); } else tmp = uri; if (uri.Contains(".") || uri.Contains("?")) { providerName = GetProviderNameByExtention(tmp); if (String.IsNullOrEmpty(providerName)) throw new ArgumentException("DataSetUri can't be built from path " + uri + ": extension is not registered"); string newUri = DataSetUri.DataSetUriScheme + ":" + providerName; if ((uri.Contains(".") || uri.Contains("?")) && (!uri.StartsWith("."))) newUri = newUri + "?file=" + tmp; if (param.Count > 0) newUri = newUri + "&" + param[0]; if (!providersByName.ContainsKey(providerName)) throw BuildProviderNotRegisteredException(uri, "Provider " + providerName + " is not registered"); DataSetFactoryEntry entry = providersByName[providerName]; DataSetProviderUriTypeAttribute[] v = (DataSetProviderUriTypeAttribute[])entry.DataSetType.GetCustomAttributes(typeof(DataSetProviderUriTypeAttribute), false); if ((v != null) && (v.Length > 0)) { return (DataSetUri) v[0].UriType.GetConstructor(new Type[] { typeof(string) }).Invoke(new object[] { newUri }); } throw new ArgumentException("DataSetUri can't be built from " + uri + " uri for provider type has no proper attributes"); } throw new ArgumentException("DataSetUri can't be build from " + uri + " uri for it is incorrect"); }
/// <summary> /// Creates the <see cref="DataSet"/> instance for the specified <paramref name="uri"/>. /// </summary> /// <param name="uri">URI containing provider name and parameters.</param> /// <returns>New instance of the <see cref="DataSet"/>.</returns> /// <remarks> /// See remarks for <see cref="DataSet.Open(DataSetUri)"/>.</remarks> /// <seealso cref="Create(string)"/> /// <seealso cref="CreateUri(string)"/> public static DataSet Create(DataSetUri uri) { if (uri == null) throw new ArgumentNullException("uri"); Trace.WriteLineIf(dataSetTraceLevel == TraceLevel.Info, "DataSetFactory: creating " + uri + " . . . "); string provider = uri.ProviderName; if (String.IsNullOrEmpty(provider)) throw BuildDataSetCreateException(uri.ToString(), "Provider name is empty or null"); if (!providersByName.ContainsKey(provider)) throw BuildProviderNotRegisteredException(uri.ToString(), "Provider " + provider + " is not registered"); DataSetFactoryEntry entry = providersByName[provider]; DataSet result = entry.CreateDataSet(uri.ToString()); List<string> includedUris = uri.GetParameterValues("include"); try { if (includedUris.Count > 0) { foreach (string includedUri in includedUris) result.IncludeDataSet(includedUri); result.Commit(); } } catch (DataSetCreateException) { throw; } catch (System.Reflection.TargetInvocationException invEx) { throw BuildDataSetCreateException(uri.ToString(), invEx.InnerException == null ? invEx : invEx.InnerException); } catch (Exception ex) { throw BuildDataSetCreateException(uri.ToString(), ex); } return result; }
/// <summary> /// Creates the <see cref="DataSet"/> instance for the specified <paramref name="uri"/>. /// </summary> /// <param name="uri">URI describing the DataSet to create.</param> /// <returns>New instance of the <see cref="DataSet"/>.</returns> /// <remarks> /// <para>The method creates a <see cref="DataSet"/> instance based on the <paramref name="uri"/>. /// If the uri contains schema definition (i.e. starts with <c>"msds:"</c>) the factory /// tries to find a corresponded provider type from the registered providers table /// (see method <see cref="Register(Type)"/>). If the provider is not registered, /// an exception will be thrown.</para> /// <para>If the uri has no schema definition, the string is considered as a path and /// the factory looks for provider by the extension of the given path /// (see method <see cref="RegisterExtension(Type)"/>). If there are more than one /// providers associated with the extension, they will be constructed one by one until /// the data set is created successfully. /// </para> /// <para> /// The path also may contain parameters specified at the end after '?' (if the /// provider associated with the extensions supports this feature). /// For example, the code: /// <code> /// DataSet ds = DataSet.Open(@"c:\data\air0.csv?openMode=open&fillUpMissingValues=true&inferInt=true"); /// </code> /// does the same as the code: /// <code> /// DataSet ds = DataSet.Open(@"msds:csv?file=c:\data\air0.csv&openMode=open&fillUpMissingValues=true&inferInt=true"); /// </code> /// </para> /// <para> /// If the <paramref name="uri"/> is null or an empty string, an instance of the /// <see cref="T:Microsoft.Research.Science.Data.Memory.MemoryDataSet"/> is created. /// </para> /// <para> /// See also remarks for the <see cref="DataSetFactory"/> class. /// </para> /// </remarks> /// <seealso cref="Register(string, Type)"/> /// <seealso cref="Register(Type)"/> /// <seealso cref="RegisterExtension(Type)"/> /// <seealso cref="RegisterExtension(string, Type)"/> /// <seealso cref="RegisterAssembly(Assembly)"/> /// <seealso cref="SearchFolder(string)"/> /// <seealso cref="Microsoft.Research.Science.Data.DataSetUri"/> /// <exception cref="ProviderNotRegisteredException"> /// DataSet is not registered in the factory.</exception> /// <exception cref="DataSetCreateException"> /// Particular DataSet provider has thrown an exception at construction.</exception> /// <exception cref="ArgumentException" /> /// <exception cref="InvalidOperationException" /> public static DataSet Create(string uri) { if (String.IsNullOrEmpty(uri)) uri = "msds:memory"; if (uri.StartsWith(DataSetUri.DataSetUriScheme + ":")) { DataSetUri dsUri = new DataSetUri(uri); return Create(dsUri); } Trace.WriteLineIf(dataSetTraceLevel == TraceLevel.Info, "DataSetFactory: creating " + uri + " . . . "); // Creating data set by extension string ext = GetExtension(uri); if (string.IsNullOrEmpty(ext)) throw BuildDataSetCreateException(uri, "Extension is empty or null"); if (!providersByExt.ContainsKey(ext)) throw BuildProviderNotRegisteredException(uri, "Extension " + ext + " is not registered in the factory"); var list = providersByExt[ext]; if (list.Count == 0) throw BuildProviderNotRegisteredException(uri, "No providers associated with the extension " + ext); Exception refuseEx = null; foreach (var e in list) { try { return (DataSet)e.Constructor.Invoke(new object[] { uri }); } catch (TargetInvocationException invEx) { refuseEx = invEx.InnerException == null ? invEx : invEx.InnerException; Trace.WriteLineIf(dataSetTraceLevel == TraceLevel.Warning, "Provider " + e.DataSetType.Name + " refused the uri."); } catch (Exception ex) { refuseEx = ex; Trace.WriteLineIf(dataSetTraceLevel == TraceLevel.Warning, "Provider " + e.DataSetType.Name + " refused the uri."); } } throw BuildDataSetCreateException(uri, refuseEx); }
/// <summary> /// Gets an array of extensions (".ext") for the given provider name. /// </summary> /// <param name="name"></param> /// <returns></returns> public static string[] GetExtensionsForProvider(string name) { if (name == null) throw new ArgumentNullException("name"); if (providersByName.Count == 0) return new string[0]; if (DataSetUri.IsDataSetUri(name)) name = new DataSetUri(name).ProviderName; if (providersByName.ContainsKey(name)) { var entry = providersByName[name]; List<string> extensions = new List<string>(); Type dst = entry.DataSetType; foreach (var v in providersByExt) { if (v.Value.Exists(p => p.DataSetType == dst)) { extensions.Add(v.Key); } } return extensions.ToArray(); } return new string[0]; }
static void Main(string[] args) { var schema = new SerializableDataSetSchema( new SerializableDimension[] { new SerializableDimension("i", 30000), new SerializableDimension("j", 30000) }, new SerializableVariableSchema[] { new SerializableVariableSchema("vals", typeof(double), new string[] { "i", "j" }, null) }, null ); string BlobConnectionAccountName = @"fetchclimate2"; string BlobConnectionAccountKey = @"vQpyUA7h5QFX6VlEH944gyv/h2Kx//WDy32brNip+YKDpsrN5/pxcSOnP2igQQ5pkA8lRXkmqmAYrgB29nwo/w=="; string uri = @"msds:ab?DefaultEndpointsProtocol=http&Container=testcontainer&Blob=testBlob30000x30000&AccountName=" + BlobConnectionAccountName + @"&AccountKey=" + BlobConnectionAccountKey; try { var ds = AzureBlobDataSet.CreateEmptySet(uri, schema); double[,] data = new double[1, 30000]; for (int i = 0; i < 30000; ++i) { data[0, i] = (double)i; } ds["vals"].PutData(new int[] { 29999, 0 }, data); var recvData = (double[, ])ds["vals"].GetData(new int[] { 29999, 0 }, new int[] { 1, 30000 }); for (int i = 0; i < 30000; ++i) { if (data[0, i] != recvData[0, i]) { throw new Exception("difference at " + i.ToString()); } } Console.WriteLine("Everything is successful!"); } catch (Exception ex) { Console.WriteLine(ex.Message); } Console.ReadLine(); //delete test blob try { AzureBlobDataSetUri azureUri = null; if (DataSetUri.IsDataSetUri(uri)) { azureUri = new AzureBlobDataSetUri(uri); } else { azureUri = AzureBlobDataSetUri.ToUri(uri); } CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container); CloudPageBlob blob = container.GetPageBlobReference(azureUri.Blob); blob.DeleteIfExists(); Console.WriteLine("Deleted test blob successfully!"); } catch (Exception ex) { Console.WriteLine(ex.Message); } Console.ReadLine(); }
static void DoMerge(string[] URIs, string output) { DataSetUri dstUri = DataSetUri.Create(output); if (dstUri.ProviderName.StartsWith("memory")) { throw new NotSupportedException("Copying to memory is not supported by the utility."); } DataSet[] dss = new DataSet[URIs.Length]; int i = 0; try { for (i = 0; i < dss.Length; i++) { dss[i] = DataSet.Open(URIs[i]); } } catch { for (; --i >= 0;) { dss[i].Dispose(); } throw; } using (DataSet mds = DataSet.Open("msds:memory")) { mds.IsAutocommitEnabled = false; IMetadataConflictResolver conflictResolver = new WarningConflictResolver(); foreach (var ds in dss) { // Global metadata foreach (var attr in ds.Metadata) { object val; if (mds.Metadata.ContainsKey(attr.Key, SchemaVersion.Recent)) { val = conflictResolver.Resolve(attr.Key, mds.Metadata[attr.Key, SchemaVersion.Recent], attr.Value); } else { val = attr.Value; } mds.Metadata[attr.Key] = val; } // Variables foreach (var var in ds.Variables) { mds.AddVariableByReference(var); } } try { mds.Commit(); } catch (DistributedCommitFailedException dex) { if (dex.InnerException is ConstraintsFailedException) { string error = String.Format("Input DataSets are incompatible: {0}", ((ConstraintsFailedException)(dex.InnerException)).Message); WriteError(error); return; } throw; } catch (ConstraintsFailedException cex) { string error = String.Format("Input DataSets are incompatible: {0}", cex.Message); WriteError(error); return; } Microsoft.Research.Science.Data.Utilities.DataSetCloning.Clone(mds, dstUri, Microsoft.Research.Science.Data.Utilities.DataSetCloning.DefaultUpdater).Dispose(); } }
static void DoSlice(string uri, Dictionary <string, Range> ranges, string outputUri) { DataSetUri dstUri = DataSetUri.Create(outputUri); if (dstUri.ProviderName.StartsWith("memory")) { throw new NotSupportedException("Copying to memory is not supported by the utility."); } using (DataSet src = DataSet.Open(uri)) { using (DataSet mds = DataSet.Open("msds:memory")) using (DataSet mds2 = DataSet.Open("msds:memory")) { mds.IsAutocommitEnabled = false; mds2.IsAutocommitEnabled = false; foreach (var var in src.Variables) { bool doSlice = false; foreach (var dim in var.Dimensions) { if (var.Dimensions.Contains(dim.Name)) { doSlice = true; break; } } if (doSlice) { var refVar = mds.AddVariableByReference(var); int rank = refVar.Rank; int[] origin = new int[rank]; int[] stride = new int[rank]; int[] count = new int[rank]; for (int i = 0; i < rank; i++) { Range r; if (ranges.TryGetValue(var.Dimensions[i].Name, out r)) { if (r.IsReduced) { origin[i] = r.Origin; stride[i] = 0; count[i] = 1; } else { throw new NotSupportedException("sds slice supports only dimension fixing"); } } else { origin[i] = 0; stride[i] = 1; count[i] = 0; } } Variable strVar = StrideVariable(refVar, origin, stride, count); mds2.AddVariableByReference(strVar); } else { mds2.AddVariableByReference(var); } } // mds2 is ready to be committed try { mds2.Commit(); } catch (DistributedCommitFailedException dex) { if (dex.InnerException is ConstraintsFailedException) { string error = String.Format("Input DataSets are incompatible: {0}", ((ConstraintsFailedException)(dex.InnerException)).Message); WriteError(error); return; } throw; } catch (ConstraintsFailedException cex) { string error = String.Format("Input DataSets are incompatible: {0}", cex.Message); WriteError(error); return; } // mds2 is ready to be cloned Microsoft.Research.Science.Data.Utilities.DataSetCloning.Clone(mds2, dstUri, Microsoft.Research.Science.Data.Utilities.DataSetCloning.DefaultUpdater).Dispose(); } } }
public void ReadTest() { Stopwatch sw = new Stopwatch(); var uri = DataSetUri.Create(NetCDFFileName); uri.OpenMode = ResourceOpenMode.Open; //sw.Start(); //DataSet ds = DataSet.Open(uri); //var val = ds.Variables.First(v => v.Name == "values"); //var vals = ds.GetData<Int16[, ,]>(val.ID); //ds.Dispose(); //sw.Stop(); //var ts = sw.Elapsed; //sw.Reset(); sw.Start(); var ds = DataSet.Open(uri); sw.Stop(); var ts2 = sw.Elapsed; var val = ds.Variables.First(v => v.Name == "values"); var time = ds.Variables.First(t => t.Name == "T").GetData(); var vv = ds.GetData <Int16[, , ]>(val.ID, DataSet.Range(0), DataSet.Range(0), DataSet.Range(0, 1439)); for (int i = 0; i < 1400; i++) { vv[0, 0, i] = 299; } ds.PutData <Int16[, , ]>(val.ID, vv, DataSet.Range(1), DataSet.Range(1), DataSet.Range(0, 1439)); int r = 110; int z = 100; Int16[, ,] grid = new Int16[r, z, 1]; Random rnd = new Random(); for (int k = 0; k < r; k++) { for (int i = 0; i < z; i++) { grid[k, i, 0] = (Int16)rnd.Next(1024); } } sw.Reset(); sw.Start(); ds.PutData <Int16[, , ]>(val.ID, grid, DataSet.Range(0, r - 1), DataSet.Range(0, z - 1), DataSet.Range(5000)); sw.Stop(); ts2 = sw.Elapsed; }
/// <summary> /// Copies given dataset into dataset determined by <paramref name="dstUri"/> and prints progress into console. /// </summary> /// <param name="src">Original dataset to clone.</param> /// <param name="dstUri">URI of the destination dataset.</param> /// <returns>New instance of <see cref="DataSet"/> class.</returns> /// <remarks><para> /// This method splits the original dataser into parts and therefore is able /// to clone very large datasets not fitting to memory.</para> /// <para>Progress is printed out into the console window.</para> /// </remarks> /// <seealso cref="DataSetCloning.Clone(DataSet,DataSetUri,ProgressUpdater)"/> public static DataSet Clone(DataSet src, DataSetUri dstUri) { return(Clone(src, dstUri, DefaultUpdater)); }