Пример #1
0
        /// <summary>
        /// Adds a vector of double values to a one-dimensional variable in an SDS object
        /// </summary>
        /// <param name="dataToConvert">The vector of values to add</param>
        /// <param name="outputVariableName">The name of the variable to write to</param>
        /// <param name="dimensionName">The name of the dimension variable of the output variable</param>
        /// <param name="dimensionValues">The values of the dimension variable</param>
        /// <param name="missingValue">The value used for missing data</param>
        /// <param name="SDSObject">The SDS object to write to</param>
        public void VectorToSDS1D(double[] dataToConvert, string outputVariableName, string dimensionName, float[] dimensionValues, double missingValue, DataSet SDSObject)
        {
            // If not already contained in the SDS object, add the dimension variable
            if (SDSObject.Variables.Contains(dimensionName))
            {
            }
            else
            {
                SDSObject.AddVariable <float>(dimensionName, dimensionValues, dimensionName);
            }

            // If not already contained in the SDS object, add the output variable
            if (SDSObject.Variables.Contains(outputVariableName))
            {
                SDSObject.PutData <double[]>(outputVariableName, dataToConvert);
                SDSObject.Commit();
            }
            else
            {
                // Set up the dimensions and add the gridded data
                string[] dimensions = { dimensionName };
                var      DataGrid   = SDSObject.AddVariable <double>(outputVariableName, dataToConvert, dimensions);

                // Add appropriate metadata (including missing values)
                DataGrid.Metadata["DisplayName"]  = outputVariableName;
                DataGrid.Metadata["MissingValue"] = (double)missingValue;

                // Commit changes
                SDSObject.Commit();
            }
        }
Пример #2
0
        /// <summary>
        /// Adds a one-dimensional variable to the specified SDS object with string dimension data
        /// </summary>
        /// <param name="SDSObject">A reference to an SDS object</param>
        /// <param name="variableName">The name of the variable to create</param>
        /// <param name="numDimensions">The number of dimensions for the new variable</param>
        /// <param name="namesDimensions">A vector of names of the dimensions for the variable</param>
        /// <param name="missingValue">The missing value to apply to the new variable</param>
        /// <param name="dimension1Data">A string vector of values of the first dimension</param>
        public void AddVariable(DataSet SDSObject, string variableName, int numDimensions, string[] namesDimensions, double missingValue, string[] dimension1Data)
        {
            //If the wrong number of dimension names have been provided, then return error
            if (namesDimensions.Length != numDimensions)
            {
                Debug.Fail("Error: you have provided the wrong number of dimension names");
            }

            //Since this overload method deals with one-dimensional variables, return an error if this is not the case
            if (numDimensions != 1)
            {
                Debug.Fail("Error: you have provided data for the wrong number of dimensions");
            }

            //If the variable already exists in the SDS, then return, otherwise create new variable
            if (SDSObject.Variables.Contains(variableName))
            {
                Console.WriteLine("SDS object already contains a variable with that name. Skipping...");
            }
            else
            {
                //For each dimension, if it already exists in the SDS then no action, otherwise create a new variable and populate it with the provided data
                if (SDSObject.Variables.Contains(namesDimensions[0]))
                {
                }
                else
                {
                    SDSObject.AddVariable <string>(namesDimensions[0], dimension1Data, namesDimensions[0]);
                }

                //If the variable is the same as one of the entered dimensions, then take no action, otherwise create the new variable and populate it with missing values
                if (SDSObject.Variables.Contains(variableName))
                {
                }
                else
                {
                    //Create array of missing values of the correct dimensions
                    double[] tempOutData = new double[dimension1Data.Length];

                    for (int ii = 0; ii < dimension1Data.Length; ii++)
                    {
                        tempOutData[ii] = missingValue;
                    }


                    //Add variable to SDS
                    var testOut = SDSObject.AddVariable <double>(variableName, tempOutData, namesDimensions);

                    //Metadata required by SDS
                    testOut.Metadata["DisplayName"]  = variableName;
                    testOut.Metadata["MissingValue"] = missingValue;

                    //Commit changes
                    SDSObject.Commit();
                }
            }
        }
Пример #3
0
 public GhcnStorageContext(FetchRequest request)
 {
     this.request = request;
     data.AddVariable <double>("lat", lats, "i");
     data.AddVariable <double>("lon", lons, "i");
     data.AddVariable <double>("temp", tvals, "t", "i");
     data.AddVariable <double>("prate", pvals, "t", "i");
     data.AddVariable <DateTime>("time", times, "t");
 }
Пример #4
0
        public SimplePlaneStorageContext(double[] lats, double[] lons, double[] vals, FetchRequest request)
        {
            this.lats = lats;
            this.lons = lons;
            this.vals = vals;

            this.request = request;
            data.AddVariable <double>("lat", lats, "i");
            data.AddVariable <double>("lon", lons, "i");
            data.AddVariable <double>("val", vals, "i");
        }
        public MainWindow()
        {
            InitializeComponent();

            // Create memory dataset with one variable
            DataSet ds = DataSet.Open("msds:memory");

            ds.AddVariable <double>("values", "i", "j");

            // Compute nice-looking data
            const int Size = 100;

            double[,] values = new double[Size, Size];
            Random r = new Random();

            for (int i = 0; i < Size; i++)
            {
                for (int j = 0; j < Size; j++)
                {
                    values[i, j] = Math.Sin((Math.Sqrt((i - Size / 2) * (i - Size / 2) +
                                                       (j - Size / 2) * (j - Size / 2)) + r.NextDouble()) / Math.Sqrt(Size));
                }
            }

            // Put data to variable
            ds.PutData("values", values);
            ds.Commit();

            // Assign dataset to view
            dsvc.DataSet = ds;

            // Show colormap first
            colorMap.IsChecked = true;
        }
        public void Init(string constructionString)
        {
            dataSet = DataSet.Create(constructionString);

            //Инициализируем DataSet
            Variable X = dataSet.AddVariable <double>("X", "x");
            Variable Y = dataSet.AddVariable <double>("Y", "y");
            Variable Z = dataSet.AddVariable <double>("Z", "z");

            time = dataSet.AddVariable <double>("Time", "t");

            u   = dataSet.AddVariable <double>("U velocity", "x", "y", "z", "t");
            v   = dataSet.AddVariable <double>("V velocity", "x", "y", "z", "t");
            w   = dataSet.AddVariable <double>("W velocity", "x", "y", "z", "t");
            T   = dataSet.AddVariable <double>("Temperature", "x", "y", "z", "t");
            div = dataSet.AddVariable <double>("Divergence", "x", "y", "z", "t");

            double[] wArr = new double[modellingParams.Nx];
            for (int i = 0; i < modellingParams.Nx; i++)
            {
                wArr[i] = i * modellingParams.Dx;
            }
            X.PutData(wArr);
            wArr = new double[modellingParams.Ny];
            for (int i = 0; i < modellingParams.Ny; i++)
            {
                wArr[i] = i * modellingParams.Dy;
            }
            Y.PutData(wArr);
            wArr = new double[modellingParams.Nz];
            for (int i = 0; i < modellingParams.Nz; i++)
            {
                wArr[i] = i * modellingParams.Dz;
            }
            Z.PutData(wArr);

            //Инициализируем рассчетный модуль для слоя начальными условиями
            solver = new LayerSolver(prevData, modellingParams);
            u.Append(prevData.U.ToArray(), "t");
            v.Append(prevData.V.ToArray(), "t");
            w.Append(prevData.W.ToArray(), "t");
            T.Append(prevData.T.ToArray(), "t");
            div.Append(prevData.Div.ToArray(), "t");

            time.PutData(new double[1] {
                0
            });
            dataSet.Commit();
        }
        public static void FillDataSetWithRequest(DataSet ds, ClimateParameter parameter, double[] latmin, double[] latmax, double[] lonmin, double[] lonmax, int[] starthour, int[] stophour, int[] startday, int[] stopday, int[] startyear, int[] stopyear, FetchingOptions options)
        {
            ds.IsAutocommitEnabled = false;

            ds.Metadata[Namings.metadataNameParameter] = Namings.GetParameterName(parameter);
            ds.Metadata[Namings.metadataNameCoverage] = Namings.GetCoverageName(parameter);
            ds.Metadata[Namings.metadataNameProvenanceHint] = options.FetchClimateProvenanceControlStr;

            if (!ds.Variables.Contains(Namings.VarNameLatMax))
                ds.AddVariable<double>(Namings.VarNameLatMax, Namings.dimNameCells);
            if (!ds.Variables.Contains(Namings.VarNameLatMin))
                ds.AddVariable<double>(Namings.VarNameLatMin, Namings.dimNameCells);
            if (!ds.Variables.Contains(Namings.VarNameLonMax))
                ds.AddVariable<double>(Namings.VarNameLonMax, Namings.dimNameCells);
            if (!ds.Variables.Contains(Namings.VarNameLonMin))
                ds.AddVariable<double>(Namings.VarNameLonMin, Namings.dimNameCells);
            if (!ds.Variables.Contains(Namings.VarNameDayMax))
                ds.AddVariable<int>(Namings.VarNameDayMax, Namings.dimNameCells);
            if (!ds.Variables.Contains(Namings.VarNameDayMin))
                ds.AddVariable<int>(Namings.VarNameDayMin, Namings.dimNameCells);
            if (!ds.Variables.Contains(Namings.VarNameHourMax))
                ds.AddVariable<int>(Namings.VarNameHourMax, Namings.dimNameCells);
            if (!ds.Variables.Contains(Namings.VarNameHourMin))
                ds.AddVariable<int>(Namings.VarNameHourMin, Namings.dimNameCells);
            if (!ds.Variables.Contains(Namings.VarNameYearMax))
                ds.AddVariable<int>(Namings.VarNameYearMax, Namings.dimNameCells);
            if (!ds.Variables.Contains(Namings.VarNameYearMin))
                ds.AddVariable<int>(Namings.VarNameYearMin, Namings.dimNameCells);

            ds.Variables[Namings.VarNameDayMax].Append(stopday);
            ds.Variables[Namings.VarNameDayMin].Append(startday);
            ds.Variables[Namings.VarNameHourMax].Append(stophour);
            ds.Variables[Namings.VarNameHourMin].Append(starthour);
            ds.Variables[Namings.VarNameYearMax].Append(stopyear);
            ds.Variables[Namings.VarNameYearMin].Append(startyear);
            ds.Variables[Namings.VarNameLonMax].Append(lonmax);
            ds.Variables[Namings.VarNameLonMin].Append(lonmin);
            ds.Variables[Namings.VarNameLatMax].Append(latmax);
            ds.Variables[Namings.VarNameLatMin].Append(latmin);

            ds.Commit();
        }
        public static void CopyRequestedDataSet(DataSet inDs, DataSet outDs, bool commit)
        {
            foreach (var entry in inDs.Metadata)
                outDs.Metadata[entry.Key] = entry.Value;

            if (!outDs.Variables.Contains(Namings.VarNameLatMax))
                outDs.AddVariable<double>(Namings.VarNameLatMax, Namings.dimNameCells);
            if (!outDs.Variables.Contains(Namings.VarNameLatMin))
                outDs.AddVariable<double>(Namings.VarNameLatMin, Namings.dimNameCells);
            if (!outDs.Variables.Contains(Namings.VarNameLonMax))
                outDs.AddVariable<double>(Namings.VarNameLonMax, Namings.dimNameCells);
            if (!outDs.Variables.Contains(Namings.VarNameLonMin))
                outDs.AddVariable<double>(Namings.VarNameLonMin, Namings.dimNameCells);
            if (!outDs.Variables.Contains(Namings.VarNameDayMax))
                outDs.AddVariable<int>(Namings.VarNameDayMax, Namings.dimNameCells);
            if (!outDs.Variables.Contains(Namings.VarNameDayMin))
                outDs.AddVariable<int>(Namings.VarNameDayMin, Namings.dimNameCells);
            if (!outDs.Variables.Contains(Namings.VarNameHourMax))
                outDs.AddVariable<int>(Namings.VarNameHourMax, Namings.dimNameCells);
            if (!outDs.Variables.Contains(Namings.VarNameHourMin))
                outDs.AddVariable<int>(Namings.VarNameHourMin, Namings.dimNameCells);
            if (!outDs.Variables.Contains(Namings.VarNameYearMax))
                outDs.AddVariable<int>(Namings.VarNameYearMax, Namings.dimNameCells);
            if (!outDs.Variables.Contains(Namings.VarNameYearMin))
                outDs.AddVariable<int>(Namings.VarNameYearMin, Namings.dimNameCells);

            outDs.Variables[Namings.VarNameDayMax].Append(inDs.Variables[Namings.VarNameDayMax].GetData());
            outDs.Variables[Namings.VarNameDayMin].Append(inDs.Variables[Namings.VarNameDayMin].GetData());
            outDs.Variables[Namings.VarNameHourMax].Append(inDs.Variables[Namings.VarNameHourMax].GetData());
            outDs.Variables[Namings.VarNameHourMin].Append(inDs.Variables[Namings.VarNameHourMin].GetData());
            outDs.Variables[Namings.VarNameYearMax].Append(inDs.Variables[Namings.VarNameYearMax].GetData());
            outDs.Variables[Namings.VarNameYearMin].Append(inDs.Variables[Namings.VarNameYearMin].GetData());
            outDs.Variables[Namings.VarNameLonMax].Append(inDs.Variables[Namings.VarNameLonMax].GetData());
            outDs.Variables[Namings.VarNameLonMin].Append(inDs.Variables[Namings.VarNameLonMin].GetData());
            outDs.Variables[Namings.VarNameLatMax].Append(inDs.Variables[Namings.VarNameLatMax].GetData());
            outDs.Variables[Namings.VarNameLatMin].Append(inDs.Variables[Namings.VarNameLatMin].GetData());

            if (commit)
            {
                outDs.Commit();
            }
        }
Пример #9
0
        /// <summary>
        /// Adds a geographical array of values to a two-dimensional variable in an SDS object
        /// </summary>
        /// <param name="dataToConvert">The array of values to add</param>
        /// <param name="ouputVariableName">The name of the variable to write to</param>
        /// <param name="lats">The values of the latitude dimension variable</param>
        /// <param name="lons">The values of the longitude dimension variable</param>
        /// <param name="missingValue">The value used for missing data</param>
        /// <param name="SDSObject">The SDS object to write to</param>
        public void Array2DToSDS2D(double[,] dataToConvert, string ouputVariableName, float[] lats, float[] lons, double missingValue, DataSet SDSObject)
        {
            // If not already contained in the SDS, add the dimension information (geographic coordinates)
            if (SDSObject.Variables.Contains("Latitude"))
            {
            }
            else
            {
                SDSObject.AddVariable <float>("Latitude", lats, "Lat");
            }
            if (SDSObject.Variables.Contains("Longitude"))
            {
            }
            else
            {
                SDSObject.AddVariable <float>("Longitude", lons, "Lon");
            }

            // If the SDS object contains the variable to write to, then simply add the data, otherwise add a new variable and then add the data
            if (SDSObject.Variables.Contains(ouputVariableName))
            {
                // Add the data
                SDSObject.PutData <double[, ]>(ouputVariableName, dataToConvert);

                // Commit the changes
                SDSObject.Commit();
            }
            else
            {
                // Set up the dimensions and add the gridded data
                string[] dimensions = { "Lat", "Lon" };
                var      DataGrid   = SDSObject.AddVariable <double>(ouputVariableName, dataToConvert, dimensions);

                // Add appropriate metadata (including missing values)
                DataGrid.Metadata["DisplayName"]  = ouputVariableName;
                DataGrid.Metadata["MissingValue"] = (double)missingValue;

                // Commit changes to update data set
                SDSObject.Commit();
            }
        }
Пример #10
0
        private static DataSet GetDataSet()
        {
            DataSet data = DataSet.Open("msds:memory");

            double[] lats = new double[] { 0.0, 0.0, 90.0, 0.0, 0.0, -90.0 };
            double[] lons = new double[] { 0.0, 90.0, 0.0, 180.0, 270.0, 0.0 };
            double[,] tvals = new double[, ] {
                { 3.0, 6.0, 15.0, 9.0, 12.0, 18.0 },
                { 3.0, 6.0, 15.0, 9.0, 12.0, 18.0 }
            };
            DateTime[] times = new DateTime[] {
                new DateTime(1970, 3, 1),
                new DateTime(1970, 4, 1)
            };

            data.AddVariable <double>("lat", lats, "stations");
            data.AddVariable <double>("lon", lons, "stations");
            data.AddVariable <double>("temp", tvals, "time", "stations");
            data.AddVariable <DateTime>("time", times, "time");

            return(data);
        }
Пример #11
0
        public void SingleDimVarUpdate512Aligned()
        {
            using (DataSet sourceDs = DataSet.Open("msds:memory"))
            {
                sourceDs.AddVariable <double>("b", Enumerable.Repeat(0, 1024).Select(a => (double)a).ToArray(), "i");
                var svs = new SerializableVariableSchema("a", typeof(double), new string[] { "i" }, new System.Collections.Generic.Dictionary <string, object>());

                using (AzureBlobDataSet ds = AzureBlobDataSet.ArrangeData(string.Format(@"msds:ab?Container=tests&Blob={0}&UseDevelopmentStorage=true", new Random().NextDouble()), sourceDs, new SerializableVariableSchema[] { svs }))
                {
                    ds.Variables["a"].PutData(Enumerable.Repeat(1, 64).Select(a => (double)a).ToArray()); //64x8=512 bytes shape
                }
            }
        }
Пример #12
0
        /// <summary>Creates data set with replicas of specified variables</summary>
        /// <param name="sourceVars">Variables from source data set to replicate</param>
        /// <returns>New data set with replicated variables</returns>
        /// <remarks>This method is not threadsafe and should be called from Dispatcher thread</remarks>
        public DataSet CreateReplica(params Variable[] sourceVars)
        {
            if (isDisposed)
            {
                throw new ObjectDisposedException("DataSetReplicator");
            }
            if (source.IsDisposed)
            {
                throw new ObjectDisposedException("Source DataSet is already disposed");
            }

            // Check prerequisites
            foreach (var v in sourceVars)
            {
                if (v.DataSet != source)
                {
                    throw new ArgumentException("Cannot replicate variable from DataSet other than source");
                }
            }

            // Create data set with variables with same names and dimensions
            DataSet targetDataSet = DataSet.Open("msds:memory");

            targetDataSet.IsAutocommitEnabled = false;
            Dictionary <int, int> targetVariables = new Dictionary <int, int>();

            foreach (var sv in sourceVars)
            {
                if (targetVariables.ContainsKey(sv.ID))
                {
                    continue; // the variable is already added
                }
                Variable tv = targetDataSet.AddVariable(sv.TypeOfData, sv.Name, null, null, sv.Dimensions.Select(d => d.Name).ToArray());
                targetVariables.Add(sv.ID, tv.ID);
                foreach (var mr in sv.Metadata)
                {
                    tv.Metadata[mr.Key] = mr.Value;
                }
                AddDataRequest(sv.ID, null, null); // Request entire data
            }

            // Create replication target structure and ask for data
            replicas.Add(new ReplicationTarget(targetDataSet, targetVariables));
            if (activeRequests.Count == 0)
            {
                BeginDataRequest();
            }

            return(targetDataSet);
        }
Пример #13
0
        public static void SaveToNetCDF(this DFSBase dfs, string NetCDFFileName)
        {
            double[,,] grid = new double[360, 720, 10];
            double[] x = new double[360];
            double[] y = new double[720];
            double[] t = new double[10];

            for (int k = 0; k < 10; k++)
            {
                for (int i = 0; i < 360; i++)
                {
                    x[i] = i;
                    for (int j = 0; j < 720; j++)
                    {
                        y[j]          = j;
                        grid[i, j, k] = i ^ 2 + j ^ 2;
                    }
                }
            }
            // ... compute grid, x and y values
            DataSet ds = DataSet.Open(NetCDFFileName + "?openMode=create");

            // MemoryDataSet ds = new MemoryDataSet();



            int vid = ds.AddVariable <Int16>("values", "x", "y", "t").ID;



            //ds.Append("values", grid, 2);
            //ds.IsAutocommitEnabled = false;
            //for (int i = 0; i < 10; i++)
            //{
//        ds.Append("values", grid, 2);
            //  ds.Append("t", i*1.1,0);
            //}



            ds.Commit();

            //ds.Clone(NetCDFFileName + "?openMode=create").Dispose();

            ds.Dispose();
        }
Пример #14
0
        public string Get(string path, bool outfolder)
        {
            logger.Log(LogLevel.Info, "Entered AVISTEDNetCDFConverter GET()");
            try
            {
                string result  = "false";
                string content = File.ReadAllText(path);
                List <Dictionary <string, string> > data = JsonConvert.DeserializeObject <List <Dictionary <string, string> > >(content);

                string randomlyGeneratedFolderNamePart = Path.GetFileNameWithoutExtension(Path.GetRandomFileName());

                string timeRelatedFolderNamePart = DateTime.Now.Year.ToString()
                                                   + DateTime.Now.Month.ToString()
                                                   + DateTime.Now.Day.ToString()
                                                   + DateTime.Now.Hour.ToString()
                                                   + DateTime.Now.Minute.ToString()
                                                   + DateTime.Now.Second.ToString()
                                                   + DateTime.Now.Millisecond.ToString();

                string processRelatedFolderNamePart = System.Diagnostics.Process.GetCurrentProcess().Id.ToString();
                string copypath = "";
                if (outfolder)
                {
                    copypath = ConfigurationManager.AppSettings["Save_Downloads"].ToString();
                }
                else
                {
                    copypath = ConfigurationManager.AppSettings["Converters"].ToString();
                }
                string temporaryDirectoryName = Path.Combine(copypath
                                                             , timeRelatedFolderNamePart
                                                             + processRelatedFolderNamePart
                                                             + randomlyGeneratedFolderNamePart);
                System.IO.Directory.CreateDirectory(temporaryDirectoryName);

                logger.Log(LogLevel.Info, "Created Directory");
                string  uri    = Path.Combine(temporaryDirectoryName, "result" + ".nc") + "?openMode=create";
                DataSet dscopy = DataSet.Open(uri);

                string[] results = new string[data.Count + 1];
                int      i = 0, j = 0;
                Dictionary <string, string> resultdict = new Dictionary <string, string>();
                Dictionary <string, string> tempdict = data.First();

                string[] names  = tempdict.Keys.ToArray();
                string[] values = new string[names.Length];
                foreach (Dictionary <string, string> dict in data)
                {
                    var value = dict.Values.ToArray();

                    if (j == 0)
                    {
                        for (int k = 0; k < values.Length; k++)
                        {
                            values[k] = value[k];
                        }
                        j = 1;
                    }
                    else
                    {
                        for (int k = 0; k < values.Length; k++)
                        {
                            values[k] += "," + value[k];
                        }
                    }
                }
                int index = 0;
                foreach (string s in names)
                {
                    if (s.Equals("date"))
                    {
                        string[]   strings = values[index++].Split(',');
                        DateTime[] date    = new DateTime[strings.Length];
                        int        l       = 0;
                        foreach (string d in strings)
                        {
                            date[l++] = DateTime.Parse(d);
                        }
                        dscopy.AddVariable <DateTime>(s, date);

                        logger.Log(LogLevel.Info, "Created parameter {0}", s);
                    }
                    else
                    {
                        string[] strings = values[index++].Split(',');
                        float[]  vl      = new float[strings.Length];
                        int      l       = 0;
                        foreach (string d in strings)
                        {
                            vl[l++] = float.Parse(d);
                        }
                        dscopy.AddVariable <float>(s, vl);
                        logger.Log(LogLevel.Info, "Created parameter {0}", s);
                    }
                }
                dscopy.Commit();
                dscopy.Dispose();

                string SourceFolderPath = temporaryDirectoryName;
                return(SourceFolderPath);
            }
            catch (Exception ex)
            {
                logger.Error("AVISTEDNetCDFConverter:Failed with exception {0}", ex.Message);
            }
            return("Error");
        }
Пример #15
0
        public static void FillDataSetWithRequest(DataSet ds, ClimateParameter parameter, double[] latmin, double[] latmax, double[] lonmin, double[] lonmax, int[] starthour, int[] stophour, int[] startday, int[] stopday, int[] startyear, int[] stopyear, FetchingOptions options)
        {
            ds.IsAutocommitEnabled = false;

            ds.Metadata[Namings.metadataNameParameter]      = Namings.GetParameterName(parameter);
            ds.Metadata[Namings.metadataNameCoverage]       = Namings.GetCoverageName(parameter);
            ds.Metadata[Namings.metadataNameProvenanceHint] = options.FetchClimateProvenanceControlStr;

            if (!ds.Variables.Contains(Namings.VarNameLatMax))
            {
                ds.AddVariable <double>(Namings.VarNameLatMax, Namings.dimNameCells);
            }
            if (!ds.Variables.Contains(Namings.VarNameLatMin))
            {
                ds.AddVariable <double>(Namings.VarNameLatMin, Namings.dimNameCells);
            }
            if (!ds.Variables.Contains(Namings.VarNameLonMax))
            {
                ds.AddVariable <double>(Namings.VarNameLonMax, Namings.dimNameCells);
            }
            if (!ds.Variables.Contains(Namings.VarNameLonMin))
            {
                ds.AddVariable <double>(Namings.VarNameLonMin, Namings.dimNameCells);
            }
            if (!ds.Variables.Contains(Namings.VarNameDayMax))
            {
                ds.AddVariable <int>(Namings.VarNameDayMax, Namings.dimNameCells);
            }
            if (!ds.Variables.Contains(Namings.VarNameDayMin))
            {
                ds.AddVariable <int>(Namings.VarNameDayMin, Namings.dimNameCells);
            }
            if (!ds.Variables.Contains(Namings.VarNameHourMax))
            {
                ds.AddVariable <int>(Namings.VarNameHourMax, Namings.dimNameCells);
            }
            if (!ds.Variables.Contains(Namings.VarNameHourMin))
            {
                ds.AddVariable <int>(Namings.VarNameHourMin, Namings.dimNameCells);
            }
            if (!ds.Variables.Contains(Namings.VarNameYearMax))
            {
                ds.AddVariable <int>(Namings.VarNameYearMax, Namings.dimNameCells);
            }
            if (!ds.Variables.Contains(Namings.VarNameYearMin))
            {
                ds.AddVariable <int>(Namings.VarNameYearMin, Namings.dimNameCells);
            }

            ds.Variables[Namings.VarNameDayMax].Append(stopday);
            ds.Variables[Namings.VarNameDayMin].Append(startday);
            ds.Variables[Namings.VarNameHourMax].Append(stophour);
            ds.Variables[Namings.VarNameHourMin].Append(starthour);
            ds.Variables[Namings.VarNameYearMax].Append(stopyear);
            ds.Variables[Namings.VarNameYearMin].Append(startyear);
            ds.Variables[Namings.VarNameLonMax].Append(lonmax);
            ds.Variables[Namings.VarNameLonMin].Append(lonmin);
            ds.Variables[Namings.VarNameLatMax].Append(latmax);
            ds.Variables[Namings.VarNameLatMin].Append(latmin);

            ds.Commit();
        }
Пример #16
0
        public static void CopyRequestedDataSet(DataSet inDs, DataSet outDs, bool commit)
        {
            foreach (var entry in inDs.Metadata)
            {
                outDs.Metadata[entry.Key] = entry.Value;
            }

            if (!outDs.Variables.Contains(Namings.VarNameLatMax))
            {
                outDs.AddVariable <double>(Namings.VarNameLatMax, Namings.dimNameCells);
            }
            if (!outDs.Variables.Contains(Namings.VarNameLatMin))
            {
                outDs.AddVariable <double>(Namings.VarNameLatMin, Namings.dimNameCells);
            }
            if (!outDs.Variables.Contains(Namings.VarNameLonMax))
            {
                outDs.AddVariable <double>(Namings.VarNameLonMax, Namings.dimNameCells);
            }
            if (!outDs.Variables.Contains(Namings.VarNameLonMin))
            {
                outDs.AddVariable <double>(Namings.VarNameLonMin, Namings.dimNameCells);
            }
            if (!outDs.Variables.Contains(Namings.VarNameDayMax))
            {
                outDs.AddVariable <int>(Namings.VarNameDayMax, Namings.dimNameCells);
            }
            if (!outDs.Variables.Contains(Namings.VarNameDayMin))
            {
                outDs.AddVariable <int>(Namings.VarNameDayMin, Namings.dimNameCells);
            }
            if (!outDs.Variables.Contains(Namings.VarNameHourMax))
            {
                outDs.AddVariable <int>(Namings.VarNameHourMax, Namings.dimNameCells);
            }
            if (!outDs.Variables.Contains(Namings.VarNameHourMin))
            {
                outDs.AddVariable <int>(Namings.VarNameHourMin, Namings.dimNameCells);
            }
            if (!outDs.Variables.Contains(Namings.VarNameYearMax))
            {
                outDs.AddVariable <int>(Namings.VarNameYearMax, Namings.dimNameCells);
            }
            if (!outDs.Variables.Contains(Namings.VarNameYearMin))
            {
                outDs.AddVariable <int>(Namings.VarNameYearMin, Namings.dimNameCells);
            }

            outDs.Variables[Namings.VarNameDayMax].Append(inDs.Variables[Namings.VarNameDayMax].GetData());
            outDs.Variables[Namings.VarNameDayMin].Append(inDs.Variables[Namings.VarNameDayMin].GetData());
            outDs.Variables[Namings.VarNameHourMax].Append(inDs.Variables[Namings.VarNameHourMax].GetData());
            outDs.Variables[Namings.VarNameHourMin].Append(inDs.Variables[Namings.VarNameHourMin].GetData());
            outDs.Variables[Namings.VarNameYearMax].Append(inDs.Variables[Namings.VarNameYearMax].GetData());
            outDs.Variables[Namings.VarNameYearMin].Append(inDs.Variables[Namings.VarNameYearMin].GetData());
            outDs.Variables[Namings.VarNameLonMax].Append(inDs.Variables[Namings.VarNameLonMax].GetData());
            outDs.Variables[Namings.VarNameLonMin].Append(inDs.Variables[Namings.VarNameLonMin].GetData());
            outDs.Variables[Namings.VarNameLatMax].Append(inDs.Variables[Namings.VarNameLatMax].GetData());
            outDs.Variables[Namings.VarNameLatMin].Append(inDs.Variables[Namings.VarNameLatMin].GetData());

            if (commit)
            {
                outDs.Commit();
            }
        }
Пример #17
0
        public static void MergeTable(IFetchConfiguration config, DataSet dst, Tuple <DataSet, string[]>[] requests)
        {
            // For faster lookup of data source name from id
            var id2name = new Dictionary <int, string>();

            foreach (var dsd in config.DataSources)
            {
                id2name.Add(dsd.ID, dsd.Name);
            }

            var var2var          = new Dictionary <string, Variables>(); // Environment variable short name => data set variables
            var cell2row         = new Dictionary <Cell, int>();         // Space-time cell => row number
            var rowCount         = 0;
            var optionalRowCount = 0;
            var regionRowCount   = 0;

            var regions = dst.AddVariable <string>("region", "i");
            var lat = dst.AddVariable <double>("lat", "i");
            var lon = dst.AddVariable <double>("lon", "i");
            Variable <double> latmin = null, latmax = null, lonmin = null, lonmax = null;
            var start = dst.AddVariable <DateTime>("start", "i");
            var end   = dst.AddVariable <DateTime>("end", "i");

            for (var i = 0; i < requests.Length; i++)
            {
                using (var src = requests[i].Item1.Clone("msds:memory"))
                {
                    var name = src.Metadata[RequestDataSetFormat.EnvironmentVariableNameKey].ToString();

                    // Define data source name to use when not provenance is supplied
                    var noProvDataSource = "";
                    if (src.Metadata.ContainsKey(RequestDataSetFormat.DataSourceNameKey))
                    {
                        var requestedDataSources = (string[])src.Metadata[RequestDataSetFormat.DataSourceNameKey];
                        if (requestedDataSources.Length == 1)
                        {
                            noProvDataSource = requestedDataSources[0];
                        }
                    }

                    var       envVar = config.EnvironmentalVariables.Where(ev => ev.Name == name).First();
                    Variables variables;
                    if (!var2var.TryGetValue(name, out variables))
                    {
                        variables        = new Variables();
                        variables.Values = dst.AddVariable <double>(String.Concat(name, " (", envVar.Units, ")"), "i");
                        variables.Values.MissingValue      = Double.NaN;
                        variables.Uncertainty              = dst.AddVariable <double>(String.Concat(name, "_uncertainty"), "i");
                        variables.Uncertainty.MissingValue = Double.NaN;
                        variables.Provenance = dst.AddVariable <string>(String.Concat(name, "_provenance"), "i");
                        variables.Provenance.MissingValue = null;
                        var2var.Add(name, variables);
                    }

                    foreach (var t in Linearize(src, requests[i].Item2))
                    {
                        int row;
                        if (!cell2row.TryGetValue(t.Item1, out row))
                        {
                            row = rowCount++;
                            cell2row.Add(t.Item1, row);
                        }

                        if (t.Item5 != "")
                        {
                            regions[row]   = t.Item5;
                            regionRowCount = Math.Max(regionRowCount, row);
                        }
                        lat[row]   = t.Item1.Lat;
                        lon[row]   = t.Item1.Lon;
                        start[row] = t.Item1.Start;
                        end[row]   = t.Item1.End;
                        if (t.Item1.LatMin != t.Item1.Lat)
                        {
                            if (latmin == null)
                            {
                                latmin = dst.AddVariable <double>("latmin", "i");
                                latmax = dst.AddVariable <double>("latmax", "i");
                                lonmin = dst.AddVariable <double>("lonmin", "i");
                                lonmax = dst.AddVariable <double>("lonmax", "i");
                            }
                            latmin[row]      = t.Item1.LatMin;
                            latmax[row]      = t.Item1.LatMax;
                            lonmin[row]      = t.Item1.LonMin;
                            lonmax[row]      = t.Item1.LonMax;
                            optionalRowCount = Math.Max(optionalRowCount, row);
                        }
                        variables.Values[row]      = t.Item2;
                        variables.Uncertainty[row] = t.Item3 < Double.MaxValue ? t.Item3 : Double.NaN;
                        variables.Provenance[row]  = (t.Item4 == 65535) ? noProvDataSource : id2name[t.Item4];
                        variables.RowCount         = row;
                    }
                }
            }

            foreach (var v in var2var)
            {
                for (var j = v.Value.RowCount + 1; j < rowCount; j++)
                {
                    v.Value.Values[j]      = Double.NaN;
                    v.Value.Uncertainty[j] = Double.NaN;
                    v.Value.Provenance[j]  = null;
                }
            }

            if (latmin != null)
            {
                for (var j = optionalRowCount + 1; j < rowCount; j++)
                {
                    latmin[j] = Double.NaN;
                    latmax[j] = Double.NaN;
                    lonmin[j] = Double.NaN;
                    lonmax[j] = Double.NaN;
                }
            }

            for (var j = regionRowCount + 1; j < rowCount; j++)
            {
                regions[j] = "";
            }

            dst.Commit();
        }
        public void SolveAll(string ctorString)
        {
            //dataSet = ProxyDataSet.Open("msds:nc?file=../../../temp.nc");
            dataSet = ProxyDataSet.Open("msds:memory");

            //Инициализируем DataSet
            Variable X    = dataSet.AddVariable <double>("X", "x");
            Variable Y    = dataSet.AddVariable <double>("Y", "y");
            Variable Z    = dataSet.AddVariable <double>("Z", "z");
            Variable time = dataSet.AddVariable <double>("Time", "t");

            Variable u   = dataSet.AddVariable <double>("U velocity", "x", "y", "z", "t");
            Variable v   = dataSet.AddVariable <double>("V velocity", "x", "y", "z", "t");
            Variable w   = dataSet.AddVariable <double>("W velocity", "x", "y", "z", "t");
            Variable T   = dataSet.AddVariable <double>("Temperature", "x", "y", "z", "t");
            Variable div = dataSet.AddVariable <double>("Divergence", "x", "y", "z", "t");

            dataSet.Commit();

            double[] wArr = new double[modellingParams.Nx];
            for (int i = 0; i < modellingParams.Nx; i++)
            {
                wArr[i] = i * modellingParams.Dx;
            }
            X.PutData(wArr);
            wArr = new double[modellingParams.Ny];
            for (int i = 0; i < modellingParams.Ny; i++)
            {
                wArr[i] = i * modellingParams.Dy;
            }
            Y.PutData(wArr);
            wArr = new double[modellingParams.Nz];
            for (int i = 0; i < modellingParams.Nz; i++)
            {
                wArr[i] = i * modellingParams.Dz;
            }
            Z.PutData(wArr);

            //Инициализируем рассчетный модуль для слоя начальными условиями
            LayerSolver solver = new LayerSolver(prevData, modellingParams);

            u.Append(prevData.U.ToArray(), "t");
            v.Append(prevData.V.ToArray(), "t");
            w.Append(prevData.W.ToArray(), "t");
            T.Append(prevData.T.ToArray(), "t");
            div.Append(prevData.Div.ToArray(), "t");

            time.PutData(new double[1] {
                0
            });
            dataSet.Commit();

            //Основной рассчет
            for (int i = 1; i < modellingParams.Nt; i++)
            {
                LayerData result = solver.Solve(true);
                //Кладем данные в DataSet
                u.Append(result.U.ToArray(), "t");
                v.Append(result.V.ToArray(), "t");
                w.Append(result.W.ToArray(), "t");
                T.Append(result.T.ToArray(), "t");
                div.Append(result.Div.ToArray(), "t");
                time.Append(new double[1] {
                    (double)i / modellingParams.Nt
                });
                dataSet.Commit();
                //Переходим на следующий слой
                solver   = new LayerSolver(prevData, result, modellingParams);
                prevData = result;

                double temp  = 0;
                int    count = 0;
                for (int ii = 1; ii < result.Width; ii++)
                {
                    for (int jj = 1; jj < result.Height; jj++)
                    {
                        for (int kk = 1; kk < result.Thickness; kk++)
                        {
                            temp += result.Div[ii, jj, kk];
                            count++;
                        }
                    }
                }
                temp = temp / count * modellingParams.Dx * modellingParams.Dy * modellingParams.Dz;

                Console.WriteLine((double)i / modellingParams.Nt * 100 + "% Error = " + temp);
            }
            dataSet.Commit();
        }
Пример #19
0
        /// <summary>
        /// Extract an array of values from a state variable in a model grid and add to a two-dimensional variable in an SDS object
        /// </summary>
        /// <param name="ecosystemModelGrid">The model grid to extract data from</param>
        /// <param name="cellIndices">List of indices of active cells in the model grid</param>
        /// <param name="gridVariableName">The name of the state variable in the model grid</param>
        /// <param name="traitValue">The trait value of the functional groups to get data for</param>
        /// <param name="variableType">The type of the state variable: 'stock' or 'cohort'</param>
        /// <param name="outputVariableName">The name of the variable to write to</param>
        /// <param name="SDSObject">The SDS object to write to</param>
        /// <param name="functionalGroupHandler">The functional group handler corresponding to cohorts or stocks</param>
        /// <param name="initialisation">The Madingley Model initialisation</param>
        public void Array2DToSDS2D(ModelGrid ecosystemModelGrid, List <uint[]> cellIndices, string gridVariableName, string traitValue,
                                   string variableType, string outputVariableName, DataSet SDSObject, FunctionalGroupDefinitions functionalGroupHandler,
                                   MadingleyModelInitialisation initialisation)
        {
            // Get the missing value from the model grid
            double MissingValue = ecosystemModelGrid.GlobalMissingValue;

            // create an array to hold the data to output
            double[,] dataToConvert = new double[ecosystemModelGrid.NumLatCells, ecosystemModelGrid.NumLonCells];

            // generate arrays to hold latitudes and longitudes
            float[] lats = new float[ecosystemModelGrid.NumLatCells];
            float[] lons = new float[ecosystemModelGrid.NumLonCells];

            // Populate arrays of latitudes and longitudes, converting from bottom left cell references as used in the model grid
            // to cell centre references as required by SDS
            for (uint ii = 0; ii < ecosystemModelGrid.NumLatCells; ii++)
            {
                lats[ii] = ecosystemModelGrid.Lats[ii] + (ecosystemModelGrid.LatCellSize / 2);
            }

            for (uint jj = 0; jj < ecosystemModelGrid.NumLonCells; jj++)
            {
                lons[jj] = ecosystemModelGrid.Lons[jj] + (ecosystemModelGrid.LonCellSize / 2);
            }

            // Get the required data from the model grid
            dataToConvert = ecosystemModelGrid.GetStateVariableGrid(gridVariableName, traitValue, functionalGroupHandler.AllFunctionalGroupsIndex, cellIndices,
                                                                    variableType, initialisation);

            // If not already contained in the SDS, add the dimension information (geographic coordinates)
            if (SDSObject.Variables.Contains("Latitude"))
            {
            }
            else
            {
                SDSObject.AddVariable <float>("Latitude", lats, "Lat");
            }
            if (SDSObject.Variables.Contains("Longitude"))
            {
            }
            else
            {
                SDSObject.AddVariable <float>("Longitude", lons, "Lon");
            }

            // If the SDS object already contains the output variable, then add the data. Otherwise, define the variable and then add the data
            if (SDSObject.Variables.Contains(outputVariableName))
            {
                SDSObject.PutData <double[, ]>(outputVariableName, dataToConvert);
                // Commit the changes
                SDSObject.Commit();
            }
            else
            {
                // Set up the dimensions and add the gridded data
                string[] dimensions = { "Lat", "Lon" };
                var      DataGrid   = SDSObject.AddVariable <double>(outputVariableName, dataToConvert, dimensions);

                // Add appropriate metadata (including missing values)
                DataGrid.Metadata["DisplayName"]  = outputVariableName;
                DataGrid.Metadata["MissingValue"] = (double)MissingValue;

                // Commit changes to update data set
                SDSObject.Commit();
            }
        }
        public void WriteFeedingInteractions(uint timestep)
        {
            int nrows = FeedingInteractionsMatrixPredation.GetLength(0);
            int ncols = FeedingInteractionsMatrixPredation.GetLength(1);

            int MaxLengthPredation = 0;

            for (int i = 0; i < nrows; i++)
            {
                for (int j = 0; j < ncols; j++)
                {
                    if (FeedingInteractionsMatrixPredation[i, j].Count > MaxLengthPredation)
                    {
                        MaxLengthPredation = FeedingInteractionsMatrixPredation[i, j].Count;
                    }
                }
            }

            double[, ,] FGIndicesPredation = new double[nrows, ncols, MaxLengthPredation];
            double[, ,] FGIndicesHerbivory = new double[nrows, ncols, 2];

            double[, ,] CIndicesPredation = new double[nrows, ncols, MaxLengthPredation];
            double[, ,] CIndicesHerbivory = new double[nrows, ncols, 2];

            double[, ,] BiomassAssimilatedPredation = new double[nrows, ncols, MaxLengthPredation];
            double[, ,] BiomassAssimilatedHerbivory = new double[nrows, ncols, 2];

            double[, ,] BiomassIngestedPredation = new double[nrows, ncols, MaxLengthPredation];
            double[, ,] BiomassIngestedHerbivory = new double[nrows, ncols, 2];


            float[] FGIndices        = new float[nrows];
            float[] CIndices         = new float[ncols];
            float[] PredationIndices = new float[MaxLengthPredation];
            float[] HerbivoryIndices = new float[2] {
                0, 1
            };

            for (int i = 0; i < nrows; i++)
            {
                FGIndices[i] = i;
            }

            for (int i = 0; i < ncols; i++)
            {
                CIndices[i] = i;
            }

            for (int i = 0; i < MaxLengthPredation; i++)
            {
                PredationIndices[i] = i;
            }

            for (int f = 0; f < nrows; f++)
            {
                for (int c = 0; c < ncols; c++)
                {
                    //Populate output data matrices for predation feeding events
                    for (int i = 0; i < FeedingInteractionsMatrixPredation[f, c].Count; i++)
                    {
                        FGIndicesPredation[f, c, i]          = Convert.ToDouble(FeedingInteractionsMatrixPredation[f, c].ElementAt(i).Item1);
                        CIndicesPredation[f, c, i]           = Convert.ToDouble(FeedingInteractionsMatrixPredation[f, c].ElementAt(i).Item2);
                        BiomassAssimilatedPredation[f, c, i] = (FeedingInteractionsMatrixPredation[f, c].ElementAt(i).Item3);
                        BiomassIngestedPredation[f, c, i]    = (FeedingInteractionsMatrixPredation[f, c].ElementAt(i).Item4);
                    }

                    //Populate output data matrices for herbivory feeding events
                    for (int i = 0; i < FeedingInteractionsMatrixHerbivory[f, c].Count; i++)
                    {
                        FGIndicesHerbivory[f, c, i]          = Convert.ToDouble(FeedingInteractionsMatrixHerbivory[f, c].ElementAt(i).Item1);
                        CIndicesHerbivory[f, c, i]           = Convert.ToDouble(FeedingInteractionsMatrixHerbivory[f, c].ElementAt(i).Item2);
                        BiomassAssimilatedHerbivory[f, c, i] = (FeedingInteractionsMatrixHerbivory[f, c].ElementAt(i).Item3);
                        BiomassIngestedHerbivory[f, c, i]    = (FeedingInteractionsMatrixHerbivory[f, c].ElementAt(i).Item4);
                    }
                }
            }



            // Initialise the data converter
            DataConverter = new ArraySDSConvert();

            // Initialise the SDS object creator
            SDSCreator = new CreateSDSObject();

            // Create an SDS object to hold the predation tracker data
            _FeedingInteractionsDS = SDSCreator.CreateSDS("netCDF", _Filename + _OutputFileSuffix + timestep, _OutputPath);

            string[] PredationDimensions = new string[3] {
                "Functional Group Indices", "Cohort Indices", "Predation Interaction Dimension"
            };
            string[] HerbivoryDimensions = new string[3] {
                "Functional Group Indices", "Cohort Indices", "Herbivory Interaction Dimension"
            };

            string[] PredD1 = new string [1] {
                PredationDimensions[0]
            };
            string[] PredD2 = new string[1] {
                PredationDimensions[1]
            };
            string[] PredD3 = new string[1] {
                PredationDimensions[2]
            };
            string[] HerbD3 = new string[1] {
                HerbivoryDimensions[2]
            };

            DataConverter.AddVariable(_FeedingInteractionsDS, PredationDimensions[0], "index", 1, PredD1, _MV, FGIndices);
            DataConverter.AddVariable(_FeedingInteractionsDS, PredationDimensions[1], "index", 1, PredD2, _MV, CIndices);
            DataConverter.AddVariable(_FeedingInteractionsDS, PredationDimensions[2], "index", 1, PredD3, _MV, PredationIndices);
            DataConverter.AddVariable(_FeedingInteractionsDS, HerbivoryDimensions[2], "index", 1, HerbD3, _MV, HerbivoryIndices);

            /*DataConverter.AddVariable(_FeedingInteractionsDS, "Predation Interactions Functional Groups", 3, PredationDimensions, _MV, FGIndices, CIndices, PredationIndices);
             * DataConverter.AddVariable(_FeedingInteractionsDS, "Predation Interactions Cohort Index", 3, PredationDimensions, _MV, FGIndices, CIndices, PredationIndices);
             * DataConverter.AddVariable(_FeedingInteractionsDS, "Predation Interactions Biomass Assimilated", 3, PredationDimensions, _MV, FGIndices, CIndices, PredationIndices);
             * DataConverter.AddVariable(_FeedingInteractionsDS, "Predation Interactions Biomass Ingested", 3, PredationDimensions, _MV, FGIndices, CIndices, PredationIndices);
             *
             * DataConverter.AddVariable(_FeedingInteractionsDS, "Herbivory Interactions Functional Groups", 3, HerbivoryDimensions, _MV, FGIndices, CIndices, HerbivoryIndices);
             * DataConverter.AddVariable(_FeedingInteractionsDS, "Herbivory Interactions Stock Index", 3, HerbivoryDimensions, _MV, FGIndices, CIndices, HerbivoryIndices);
             * DataConverter.AddVariable(_FeedingInteractionsDS, "Herbivory Interactions Biomass Assimilated", 3, HerbivoryDimensions, _MV, FGIndices, CIndices, HerbivoryIndices);
             * DataConverter.AddVariable(_FeedingInteractionsDS, "Herbivory Interactions Biomass Ingested", 3, HerbivoryDimensions, _MV, FGIndices, CIndices, HerbivoryIndices);*/

            //Add variable to SDS
            var FGIPredOut = _FeedingInteractionsDS.AddVariable <double>("Predation Interactions Functional Groups", FGIndicesPredation, PredationDimensions);

            FGIPredOut.Metadata["DisplayName"]  = "Predation Interactions Functional Groups";
            FGIPredOut.Metadata["MissingValue"] = _MV;

            //Add variable to SDS
            var CIPredOut = _FeedingInteractionsDS.AddVariable <double>("Predation Interactions Cohort Index", CIndicesPredation, PredationDimensions);

            CIPredOut.Metadata["DisplayName"]  = "Predation Interactions Cohort Index";
            CIPredOut.Metadata["MissingValue"] = _MV;

            //Add variable to SDS
            var BAPredOut = _FeedingInteractionsDS.AddVariable <double>("Predation Interactions Biomass Assimilated", BiomassAssimilatedPredation, PredationDimensions);

            BAPredOut.Metadata["DisplayName"]  = "Predation Interactions Biomass Assimilated";
            BAPredOut.Metadata["MissingValue"] = _MV;

            //Add variable to SDS
            var BIPredOut = _FeedingInteractionsDS.AddVariable <double>("Predation Interactions Biomass Ingested", BiomassIngestedPredation, PredationDimensions);

            BIPredOut.Metadata["DisplayName"]  = "Predation Interactions Biomass Ingested";
            BIPredOut.Metadata["MissingValue"] = _MV;


            //Add variable to SDS
            var FGIHerbOut = _FeedingInteractionsDS.AddVariable <double>("Herbivory Interactions Functional Groups", FGIndicesHerbivory, HerbivoryDimensions);

            FGIHerbOut.Metadata["DisplayName"]  = "Herbivory Interactions Functional Groups";
            FGIHerbOut.Metadata["MissingValue"] = _MV;

            //Add variable to SDS
            var CIHerbOut = _FeedingInteractionsDS.AddVariable <double>("Herbivory Interactions Cohort Index", CIndicesHerbivory, HerbivoryDimensions);

            CIHerbOut.Metadata["DisplayName"]  = "Herbivory Interactions Cohort Index";
            CIHerbOut.Metadata["MissingValue"] = _MV;

            //Add variable to SDS
            var BAHerbOut = _FeedingInteractionsDS.AddVariable <double>("Herbivory Interactions Biomass Assimilated", BiomassAssimilatedHerbivory, HerbivoryDimensions);

            BAHerbOut.Metadata["DisplayName"]  = "Herbivory Interactions Biomass Assimilated";
            BAHerbOut.Metadata["MissingValue"] = _MV;

            //Add variable to SDS
            var BIHerbOut = _FeedingInteractionsDS.AddVariable <double>("Herbivory Interactions Biomass Ingested", BiomassIngestedHerbivory, HerbivoryDimensions);

            BIHerbOut.Metadata["DisplayName"]  = "Herbivory Interactions Biomass Ingested";
            BIHerbOut.Metadata["MissingValue"] = _MV;


            //Commit changes
            _FeedingInteractionsDS.Commit();
        }
        public static DataSet Clone(DataSet src, DataSetUri dstUri)
        {
            if (src == null)
            {
                throw new ArgumentNullException("src");
            }

            // Maximum memory capacity in bytes
            int N = 200 * 1024 * 1024;
            // Estimated size of a single string in bytes
            int sizeofString = 100 * 1024;

            /***********************************************************************************
            * Preparing output
            ***********************************************************************************/
            DataSet dst = DataSet.Open(dstUri);

            if (dst.IsReadOnly)
            {
                throw new NotSupportedException("Output DataSet is read-only");
            }
            dst.IsAutocommitEnabled = false;

            DataSetSchema         srcSchema = src.GetSchema();
            Dictionary <int, int> IDs       = new Dictionary <int, int>();

            // Creating empty variables and copying global metadata and scalar variables
            Console.Out.Write("\n\nCreating structure and copying global metadata and scalar variables... ");
            foreach (VariableSchema v in srcSchema.Variables)
            {
                if (v.ID == DataSet.GlobalMetadataVariableID)
                {
                    // Copying global metadata
                    var dstGlobalMetadata = dst.Metadata;
                    foreach (var attr in v.Metadata)
                    {
                        dstGlobalMetadata[attr.Key] = attr.Value;
                    }
                    continue;
                }

                Variable t = dst.AddVariable(v.TypeOfData, v.Name, null, v.Dimensions.AsNamesArray());
                IDs.Add(v.ID, t.ID);

                foreach (var attr in v.Metadata)
                {
                    t.Metadata[attr.Key] = attr.Value;
                }

                if (t.Rank == 0) // scalar
                {
                    t.PutData(src.Variables.GetByID(v.ID).GetData());
                }
            }
            dst.Commit();
            Console.Out.WriteLine("Done.\n");

            /***********************************************************************************
            * Adjusting dimensions deltas
            ***********************************************************************************/
            Dimension[] srcDims             = srcSchema.GetDimensions();
            Dictionary <string, int> deltas = new Dictionary <string, int>(srcDims.Length);

            foreach (var d in srcDims)
            {
                deltas[d.Name] = d.Length;
            }

            Console.Out.WriteLine("Total memory capacity: " + (N / 1024.0 / 1024.0).ToString("F2") + " Mb");
            int totalSize;

            do
            {
                totalSize = 0;
                foreach (var var in srcSchema.Variables)
                {
                    if (var.Rank == 0)
                    {
                        continue;                // scalar
                    }
                    int typeSize = SizeOf(var.TypeOfData, sizeofString);

                    int count = 0;
                    foreach (var vdim in var.Dimensions)
                    {
                        int dimDelta = deltas[vdim.Name];
                        if (count == 0)
                        {
                            count = dimDelta;
                        }
                        else
                        {
                            count *= dimDelta;
                        }
                    }
                    totalSize += typeSize * count;
                }
                if (totalSize > N)
                {
                    string maxDim = null;
                    int    max    = int.MinValue;
                    foreach (var dim in deltas)
                    {
                        if (dim.Value > max)
                        {
                            max    = dim.Value;
                            maxDim = dim.Key;
                        }
                    }
                    if (maxDim == null || max <= 1)
                    {
                        throw new NotSupportedException("Cannot copy the DataSet: it is too large to be copied entirely by the utility for the provided memory capacity");
                    }
                    deltas[maxDim] = max >> 1;
                }
            } while (totalSize > N);

            // Printing deltas
            Console.Out.WriteLine("Deltas for the dimensions adjusted (max iteration capacity: " + (totalSize / 1024.0 / 1024.0).ToString("F2") + " Mb):");
            foreach (var delta in deltas)
            {
                Console.Out.WriteLine(" Dimension " + delta.Key + ": " + delta.Value);
            }

            /***********************************************************************************
            * Copying data
            ***********************************************************************************/
            Console.WriteLine();
            UpdateProgress(0);
            Dictionary <int, int[]> origins  = new Dictionary <int, int[]>(srcSchema.Variables.Length);
            Dictionary <int, int[]> shapes   = new Dictionary <int, int[]>(srcSchema.Variables.Length);
            List <VariableSchema>   copyVars = srcSchema.Variables.Where(vs =>
                                                                         (vs.Rank > 0 && vs.ID != DataSet.GlobalMetadataVariableID)).ToList();

            Dictionary <string, int> dimOrigin = new Dictionary <string, int>(srcDims.Length);

            foreach (var d in srcDims)
            {
                dimOrigin[d.Name] = 0;
            }

            Array.Sort(srcDims, (d1, d2) => d1.Length - d2.Length);
            int totalDims = srcDims.Length;

            do
            {
                // for each variable:
                for (int varIndex = copyVars.Count; --varIndex >= 0;)
                {
                    VariableSchema var        = copyVars[varIndex];
                    bool           hasChanged = false;
                    // Getting its origin
                    int[] origin;
                    if (!origins.TryGetValue(var.ID, out origin))
                    {
                        origin          = new int[var.Rank];
                        origins[var.ID] = origin;
                        hasChanged      = true;
                    }
                    // Getting its shape
                    int[] shape;
                    if (!shapes.TryGetValue(var.ID, out shape))
                    {
                        shape = new int[var.Rank];
                        for (int i = 0; i < var.Dimensions.Count; i++)
                        {
                            shape[i] = deltas[var.Dimensions[i].Name];
                        }
                        shapes.Add(var.ID, shape);
                    }

                    // Updating origin for the variable:
                    if (!hasChanged)
                    {
                        for (int i = 0; i < shape.Length; i++)
                        {
                            int o = dimOrigin[var.Dimensions[i].Name];
                            if (origin[i] != o)
                            {
                                hasChanged = true;
                                origin[i]  = o;
                            }
                        }
                    }
                    if (!hasChanged) // this block is already copied
                    {
                        continue;
                    }

                    bool doCopy       = false;
                    bool shapeUpdated = false;
                    for (int i = 0; i < shape.Length; i++)
                    {
                        int s   = origin[i] + shape[i];
                        int len = var.Dimensions[i].Length;
                        if (s > len)
                        {
                            if (!shapeUpdated)
                            {
                                shapeUpdated = true;
                                shape        = (int[])shape.Clone();
                            }
                            shape[i] = len - origin[i];
                        }
                        if (shape[i] > 0)
                        {
                            doCopy = true;
                        }
                    }

                    if (doCopy)
                    {
                        Array data = src.Variables.GetByID(var.ID).GetData(origin, shape);
                        // Compute real size here for strings
                        dst.Variables.GetByID(IDs[var.ID]).PutData(origin, data);
                    }
                    else // variable is copied
                    {
                        copyVars.RemoveAt(varIndex);
                    }
                }
                dst.Commit();

                // Updating dimensions origin
                bool isOver = true;
                for (int i = 0; i < totalDims; i++)
                {
                    Dimension dim    = srcDims[i];
                    int       origin = dimOrigin[dim.Name] + deltas[dim.Name];
                    if (origin < dim.Length)
                    {
                        dimOrigin[dim.Name] = origin;
                        isOver = false;
                        // Progress indicator
                        if (i == totalDims - 1)
                        {
                            double perc = (double)origin / dim.Length * 100.0;
                            UpdateProgress(perc);
                        }
                        break;
                    }
                    dimOrigin[dim.Name] = 0;
                }
                if (isOver)
                {
                    break;
                }
            } while (copyVars.Count > 0);

            UpdateProgress(100.0);

            Console.Out.WriteLine();
            return(dst);
        }
Пример #22
0
        private void MergeCSV(DataSet dst, string[] hashes)
        {
            for (var i = 0; i < hashes.Length; i++)
            {
                using (var src = DataSet.Open(WebApiApplication.GetSharedJobManager(Request).ResultDataSetUri(hashes[i], false)))
                {
                    var name = src.Metadata[RequestDataSetFormat.EnvironmentVariableNameKey].ToString();
                    if (src.Metadata.ContainsKey(RequestDataSetFormat.DataSourceNameKey))
                    {
                        string[] dataSources = (string[])src.Metadata[RequestDataSetFormat.DataSourceNameKey];
                        var      config      = WebApiApplication.GetFetchConfiguration(DateTime.MaxValue);
                        dst.Metadata[name + "_dataSourceNames"] = dataSources;
                        dst.Metadata[name + "_dataSourceIDs"]   = dataSources.Select(ds =>
                                                                                     config.DataSources.Where(dsd => dsd.Name == ds).First().ID).ToArray();
                    }

                    bool   isPointSet = src.Variables[RequestDataSetFormat.ValuesVariableName].Dimensions.Count() == 1;
                    string visMethod  = isPointSet ? "Points" : "Colormap";

                    if (i == 0)
                    {
                        if (src.Variables.Contains("lat"))
                        {
                            dst.AddVariableByValue(src.Variables["lat"]);
                        }
                        if (src.Variables.Contains("latmax"))
                        {
                            dst.AddVariableByValue(src.Variables["latmax"]);
                        }
                        if (src.Variables.Contains("latmin"))
                        {
                            dst.AddVariableByValue(src.Variables["latmin"]);
                        }

                        if (src.Variables.Contains("lon"))
                        {
                            dst.AddVariableByValue(src.Variables["lon"]);
                        }
                        if (src.Variables.Contains("lonmax"))
                        {
                            dst.AddVariableByValue(src.Variables["lonmax"]);
                        }
                        if (src.Variables.Contains("lonmin"))
                        {
                            dst.AddVariableByValue(src.Variables["lonmin"]);
                        }

                        dst.AddVariableByValue(src.Variables["hours"]);
                        dst.AddVariableByValue(src.Variables["days"]);
                        dst.AddVariableByValue(src.Variables["years"]);

                        dst.Metadata["VisualHints"] = name + "_" + RequestDataSetFormat.ValuesVariableName + "Style: " + visMethod;
                    }


                    var valuesVar = src[RequestDataSetFormat.ValuesVariableName];
                    dst.AddVariable <double>(name + "_" + RequestDataSetFormat.ValuesVariableName,
                                             valuesVar.GetData(),
                                             valuesVar.Dimensions.Select(d => d.Name).ToArray()).Metadata["VisualHints"] = "Style: " + visMethod;

                    var sdVar = src[RequestDataSetFormat.UncertaintyVariableName];
                    dst.AddVariable <double>(name + "_" + RequestDataSetFormat.UncertaintyVariableName,
                                             sdVar.GetData(),
                                             sdVar.Dimensions.Select(d => d.Name).ToArray());

                    if (src.Variables.Contains(RequestDataSetFormat.ProvenanceVariableName))
                    {
                        var provVar = src[RequestDataSetFormat.ProvenanceVariableName];
                        dst.AddVariable <ushort>(name + "_" + RequestDataSetFormat.ProvenanceVariableName,
                                                 provVar.GetData(),
                                                 provVar.Dimensions.Select(d => d.Name).ToArray());
                    }
                }
            }
            dst.Commit();
        }