public void ExportAsCSVNoEmptyLines()
 {
     var reader = new CSVReader(Path.Combine(_inputFilePath));
     _data.Sensors = reader.ReadSensors();
     DatasetExporter.Export(_data, _outputFilePath, ExportFormat.CSV, false, false, false, ExportedPoints.AllPoints, DateColumnFormat.TwoDateColumn, loadInUnloadedValues: false);
     Assert.AreEqual(Tools.GenerateMD5HashFromFile(_outputFilePath), Tools.GenerateMD5HashFromFile(_inputFilePath));
 }
 public void ExportAsCSVEmptyLinesIncludedWithAverageEveryHour()
 {
     var reader = new CSVReader(Path.Combine(_inputFilePath));
     _data.Sensors = reader.ReadSensors();
     DatasetExporter.Export(_data, _outputFilePath, ExportFormat.CSV, true, false, false, ExportedPoints.HourlyPoints, DateColumnFormat.TwoDateColumn, loadInUnloadedValues: false);
     Assert.AreEqual(File.ReadLines(_outputFilePath).Count(), ((_data.ExpectedDataPointCount) / 4) + 1);
 }
        public void Setup()
        {
            _reader = new CSVReader(Path.Combine(Common.TestDataPath, "lakeTutira120120110648_extra_small.csv"));
            _ds = new Dataset(_s, _reader.ReadSensors());
            var sensorVariables = SensorVariable.CreateSensorVariablesFromSensors(_ds.Sensors);
            foreach (var sensor in _ds.Sensors)
            {
                sensor.Owner = _ds;
                sensor.Variable = sensorVariables.FirstOrDefault(x => x.Sensor == sensor);
            }

            _eval = new FormulaEvaluator(_ds.Sensors);
        }
Beispiel #4
0
        public void ReadsCSVWithYYYYMMDDTogetherAndmmhhTogetherSplitByColon()
        {
            var fileName = Path.GetTempFileName() + ".csv";

            File.WriteAllText(fileName, "YYYYMMDD,hh:mm,Test Sensor\r\n20110510,13:00,100");

            var reader = new CSVReader(fileName);
            var sensors = reader.ReadSensors();

            Assert.AreEqual(1, sensors.Count);
            Assert.AreEqual(100, sensors[0].CurrentState.Values[new DateTime(2011, 5, 10, 13, 0, 0)]);
            Assert.AreEqual(1, sensors[0].CurrentState.Values.Count);
        }
Beispiel #5
0
        public void ReadsCSVWithYYYYMMDDmmhhSeperately()
        {
            var fileName = Path.GetTempFileName() + ".csv";

            File.WriteAllText(fileName, "YYYY,MM,DD,hh,mm,Test Sensor\r\n2011,05,10,13,00,100");

            var reader = new CSVReader(fileName);
            var sensors = reader.ReadSensors();

            Assert.AreEqual(1, sensors.Count);
            Assert.AreEqual(100, sensors[0].CurrentState.Values[new DateTime(2011, 5, 10, 13, 0, 0)]);
            Assert.AreEqual(1, sensors[0].CurrentState.Values.Count);
        }
Beispiel #6
0
 public void DetectIntervalWithNoMissingValues()
 {
     _reader = new CSVReader(FifteenMinuteIntervalData);
     Dataset ds = new Dataset(_b, _reader.ReadSensors());
     Assert.AreEqual(15, ds.DataInterval);
 }
Beispiel #7
0
 public void CorrectActualDataPointCount()
 {
     _reader = new CSVReader(Path.Combine(Common.TestDataPath, "lakeTutira120120110648.csv"));
     Dataset ds = new Dataset(_b, _reader.ReadSensors());
     Assert.AreEqual(53873, ds.ActualDataPointCount);
 }
Beispiel #8
0
        public void Setup()
        {
            _reader = new CSVReader(Path.Combine(Common.TestDataPath, "lakeTutira120120110648_extra_small.csv"));
            _sensors = _reader.ReadSensors();
            _startTime1 = new DateTime(46, 4, 12, 6, 48, 20);
            _endTime1 = new DateTime(84, 2, 8, 2, 4, 20);
            _startTime2 = new DateTime(2009, 1, 9, 14, 45, 0);
            _endTime2 = new DateTime(2009, 1, 14, 19, 15, 0);

            _ds1 = new Dataset(_b);
            _ds2 = new Dataset(_b, _sensors);
        }
Beispiel #9
0
 public void FileNotFound()
 {
     _reader = new CSVReader("NotEvenAFile.csv");
 }
Beispiel #10
0
 public void ConstructorTest()
 {
     var reader = new CSVReader(TestDataFileName);
 }
Beispiel #11
0
 public void BadFileName()
 {
     _reader = new CSVReader("CannotTellIfIAmCSV");
 }
Beispiel #12
0
 public void Setup()
 {
     _reader = new CSVReader(TestDataFileName);
 }
Beispiel #13
0
        /// <summary>
        /// starts an import of data
        /// </summary>
        /// <param name="filename">filepath of data file</param>
        public void Import(string filename)
        {
            DisableFeatures();
            var result = new List<Sensor>();

            ShowProgressArea = true;
            ProgressIndeterminate = false;
            ProgressValue = 0;

            var reader = new CSVReader(filename);

            reader.ProgressChanged += (sender, args) =>
                                          {
                                              ProgressValue = args.Progress;
                                              WaitEventString =
                                                  string.Format("Importing from {0} {1}%",
                                                                filename,
                                                                ProgressValue);
                                          };
            try
            {
                result = reader.ReadSensors(null, CurrentDataset);
            }
            catch (Exception ex)
            {
                Common.ShowMessageBoxWithException("Failed Import", "Bad File Format", false, true, ex);
                result = null;
            }

            if (result == null)
            {
                ShowProgressArea = false;
                EnableFeatures();
                return;
            }

            var sensors = (List<Sensor>)result;

            if (CurrentDataset.Sensors == null || CurrentDataset.Sensors.Count == 0)
            {
                CurrentDataset.Sensors = sensors;
                CurrentDataset.HighestYearLoaded = NumberOfDataChunks() - 1;
            }
            else
            {
                var askUser =
                    _container.GetInstance(typeof(SpecifyValueViewModel),
                                           "SpecifyValueViewModel") as
                    SpecifyValueViewModel;

                if (askUser == null)
                {
                    Common.ShowMessageBox("EPIC FAIL", "RUN AROUND WITH NO REASON",
                                          false, true);
                    return;
                }

                askUser.ComboBoxItems = new List<string> { "Keep old values", "Keep new values" };
                askUser.Text = "Keep old values";
                askUser.ShowComboBox = true;
                askUser.Message = "How do you want to handle overlapping points. Note this will also overwrite empty or missing values within the overlaping period";
                askUser.CanEditComboBox = false;
                askUser.ComboBoxSelectedIndex = 0;
                askUser.Title = "Importing";

                _windowManager.ShowDialog(askUser);

                var keepOldValues = askUser.ComboBoxSelectedIndex == 0;

                var sensorMatchView =
                    _container.GetInstance(typeof(MatchToExistingSensorsViewModel),
                                           "MatchToExistingSensorsViewModel") as
                    MatchToExistingSensorsViewModel;

                if (sensorMatchView == null)
                    return;

                sensorMatchView.ExistingSensors = CurrentDataset.Sensors;
                sensorMatchView.NewSensors = sensors;

                _windowManager.ShowDialog(sensorMatchView);

                var orderedTimestamps = sensors.SelectMany(x => x.CurrentState.Values).Select(x => x.Key).Distinct().ToArray();
                var firstTimestamp = orderedTimestamps.Min();
                var lastTimestamp = orderedTimestamps.Max();
                orderedTimestamps = new DateTime[0];
                var highestYear = NumberOfDataChunks() - 1;

                var firstNewYear = firstTimestamp.Year - CurrentDataset.StartYear.Year;

                if (firstNewYear < CurrentDataset.LowestYearLoaded)
                {
                    CurrentDataset.LoadInSensorData(firstNewYear < 0 ?
                           Enumerable.Range(0, CurrentDataset.LowestYearLoaded).ToArray()
                         : Enumerable.Range(firstNewYear, CurrentDataset.LowestYearLoaded - firstNewYear).ToArray(), true);
                }

                var lastNewYear = lastTimestamp.Year - CurrentDataset.StartYear.Year;
                var reason = ChangeReason.AddNewChangeReason("[Importer] Imported new values on " + DateTime.Now);
                if (lastNewYear > CurrentDataset.HighestYearLoaded)
                {
                    CurrentDataset.LoadInSensorData(lastNewYear > highestYear ?
                        Enumerable.Range(CurrentDataset.HighestYearLoaded + 1, highestYear - CurrentDataset.HighestYearLoaded).ToArray()
                      : Enumerable.Range(CurrentDataset.HighestYearLoaded + 1, lastNewYear - CurrentDataset.HighestYearLoaded).ToArray(), true);
                }

                foreach (var newSensor in sensors)
                {
                    var match =
                        sensorMatchView.SensorLinks.FirstOrDefault(
                            x => x.MatchingSensor == newSensor);
                    if (match == null)
                    {
                        Debug.WriteLine("Adding new sensor");
                        CurrentDataset.Sensors.Add(newSensor);
                    }
                    else
                    {
                        var matchingSensor =
                            CurrentDataset.Sensors.FirstOrDefault(
                                x => x == match.ExistingSensor);

                        if (matchingSensor == null)
                        {
                            Debug.WriteLine(
                                "Failed to find the sensor again, embarrasing!");
                            continue;
                        }

                        Debug.WriteLine("Merging sensors");
                        //Otherwise clone the current state
                        var newState = matchingSensor.CurrentState.Clone();
                        //Check to see if values are inserted
                        var insertedValues = false;

                        //And add values for any new dates we want
                        foreach (var value in newSensor.CurrentState.Values.Where(value =>
                                    !keepOldValues || !(matchingSensor.CurrentState.Values.ContainsKey(value.Key) || matchingSensor.RawData.Values.ContainsKey(value.Key))))
                        {
                            newState.Values[value.Key] = value.Value;
                            if (matchingSensor.CurrentState.Values.ContainsKey(value.Key) || matchingSensor.RawData.Values.ContainsKey(value.Key))
                                newState.AddToChanges(value.Key, reason.ID);
                            matchingSensor.RawData.Values[value.Key] = value.Value;
                            insertedValues = true;
                        }

                        if (insertedValues)
                        {
                            //Give a reason
                            newState.Reason = reason;
                            //Insert new state
                            matchingSensor.AddState(newState);
                            matchingSensor.ClearUndoStates();
                            EventLogger.LogSensorInfo(CurrentDataset,
                                                      matchingSensor.Name,
                                                      "Added values from new import");
                        }
                    }
                }
                CurrentDataset.CalculateDataSetValues();
                if (CurrentDataset.HighestYearLoaded == highestYear)
                    CurrentDataset.HighestYearLoaded = NumberOfDataChunks() - 1;
            }

            UpdateGUI();

            if (Sensors.FirstOrDefault(x => x.Variable == null) != null)
            {
                var sensorVariables = SensorVariable.CreateSensorVariablesFromSensors(Sensors);
                foreach (var sensor in Sensors)
                {
                    sensor.Variable = sensorVariables.FirstOrDefault(x => x.Sensor == sensor);
                }
            }
            _evaluator = new FormulaEvaluator(Sensors);
            _dateAnnotator.DataInterval = CurrentDataset.DataInterval;

            ShowProgressArea = false;
            EnableFeatures();
            NotifyOfPropertyChange(() => LowestYearLoadedOptions);
            NotifyOfPropertyChange(() => HighestYearLoadedOptions);
            NotifyOfPropertyChange(() => LowestYearLoaded);
            NotifyOfPropertyChange(() => HighestYearLoaded);

            CurrentDataset.SaveToFile();
        }
Beispiel #14
0
 public void SetUp()
 {
     _reader = new CSVReader(Path.Combine(Common.TestDataPath, "calibrationTestData.csv"));
     _ds = new Dataset(_s, _reader.ReadSensors());
 }
Beispiel #15
0
        public void LogReferencesExportTest()
        {
            var reader = new CSVReader(_inputFilePath);
            _data.Sensors = reader.ReadSensors();
            _data.Sensors = new List<Sensor> { _data.Sensors[0] };
            using (var writer = File.CreateText(_outputFilePath + "ChangesTest.csv"))
            {
                writer.WriteLine("Change matrix for file: " + Path.GetFileName(_outputFilePath));
                writer.WriteLine("Date,Time,Temperature");
                for (var time = _data.StartTimeStamp; time <= _data.EndTimeStamp; time = time.AddMinutes(_data.DataInterval))
                {
                    writer.WriteLine(time.ToString("dd/MM/yyyy,HH:mm") + ",1");
                }
            }
            var ll = new LinkedList<int>();
            ll.AddFirst(1);

            for (var time = _data.StartTimeStamp; time <= _data.EndTimeStamp; time = time.AddMinutes(_data.DataInterval))
            {
                _data.Sensors[0].CurrentState.Changes.Add(time, ll);
            }

            DatasetExporter.Export(_data, _outputFilePath, ExportFormat.CSV, true, false, true, loadInUnloadedValues: false);
            Assert.AreEqual(Tools.GenerateMD5HashFromFile(_outputFilePath + "ChangesTest.csv"), Tools.GenerateMD5HashFromFile(_outputFilePath + " Changes Matrix.csv"));
        }
Beispiel #16
0
        public void ExportsRawDataWhenRequested()
        {
            var reader = new CSVReader(_inputFilePath);
            _data.Sensors = reader.ReadSensors();

            Assert.AreNotEqual(0, _data.Sensors[0].CurrentState.Values[new DateTime(2009, 1, 10, 7, 45, 0)]);

            // Make some changes to check the raw comes out
            var newState = _data.Sensors[0].CurrentState.Clone();
            newState.Values[new DateTime(2009, 1, 10, 7, 45, 0)] = 0;
            _data.Sensors[0].AddState(newState);

            DatasetExporter.Export(_data, _outputFilePath, ExportFormat.CSV, true, false, false, ExportedPoints.AllPoints, DateColumnFormat.TwoDateColumn, true, loadInUnloadedValues: false);

            reader = new CSVReader(_outputFilePath + " Raw.csv");
            _data.Sensors = reader.ReadSensors();

            Assert.AreNotEqual(0, _data.Sensors[0].CurrentState.Values[new DateTime(2009, 1, 10, 7, 45, 0)]);
        }