public void ExportAsCSVNoEmptyLines() { var reader = new CSVReader(Path.Combine(_inputFilePath)); _data.Sensors = reader.ReadSensors(); DatasetExporter.Export(_data, _outputFilePath, ExportFormat.CSV, false, false, false, ExportedPoints.AllPoints, DateColumnFormat.TwoDateColumn, loadInUnloadedValues: false); Assert.AreEqual(Tools.GenerateMD5HashFromFile(_outputFilePath), Tools.GenerateMD5HashFromFile(_inputFilePath)); }
public void ExportAsCSVEmptyLinesIncludedWithAverageEveryHour() { var reader = new CSVReader(Path.Combine(_inputFilePath)); _data.Sensors = reader.ReadSensors(); DatasetExporter.Export(_data, _outputFilePath, ExportFormat.CSV, true, false, false, ExportedPoints.HourlyPoints, DateColumnFormat.TwoDateColumn, loadInUnloadedValues: false); Assert.AreEqual(File.ReadLines(_outputFilePath).Count(), ((_data.ExpectedDataPointCount) / 4) + 1); }
public void Setup() { _reader = new CSVReader(Path.Combine(Common.TestDataPath, "lakeTutira120120110648_extra_small.csv")); _ds = new Dataset(_s, _reader.ReadSensors()); var sensorVariables = SensorVariable.CreateSensorVariablesFromSensors(_ds.Sensors); foreach (var sensor in _ds.Sensors) { sensor.Owner = _ds; sensor.Variable = sensorVariables.FirstOrDefault(x => x.Sensor == sensor); } _eval = new FormulaEvaluator(_ds.Sensors); }
public void ReadsCSVWithYYYYMMDDTogetherAndmmhhTogetherSplitByColon() { var fileName = Path.GetTempFileName() + ".csv"; File.WriteAllText(fileName, "YYYYMMDD,hh:mm,Test Sensor\r\n20110510,13:00,100"); var reader = new CSVReader(fileName); var sensors = reader.ReadSensors(); Assert.AreEqual(1, sensors.Count); Assert.AreEqual(100, sensors[0].CurrentState.Values[new DateTime(2011, 5, 10, 13, 0, 0)]); Assert.AreEqual(1, sensors[0].CurrentState.Values.Count); }
public void ReadsCSVWithYYYYMMDDmmhhSeperately() { var fileName = Path.GetTempFileName() + ".csv"; File.WriteAllText(fileName, "YYYY,MM,DD,hh,mm,Test Sensor\r\n2011,05,10,13,00,100"); var reader = new CSVReader(fileName); var sensors = reader.ReadSensors(); Assert.AreEqual(1, sensors.Count); Assert.AreEqual(100, sensors[0].CurrentState.Values[new DateTime(2011, 5, 10, 13, 0, 0)]); Assert.AreEqual(1, sensors[0].CurrentState.Values.Count); }
public void DetectIntervalWithNoMissingValues() { _reader = new CSVReader(FifteenMinuteIntervalData); Dataset ds = new Dataset(_b, _reader.ReadSensors()); Assert.AreEqual(15, ds.DataInterval); }
public void CorrectActualDataPointCount() { _reader = new CSVReader(Path.Combine(Common.TestDataPath, "lakeTutira120120110648.csv")); Dataset ds = new Dataset(_b, _reader.ReadSensors()); Assert.AreEqual(53873, ds.ActualDataPointCount); }
public void Setup() { _reader = new CSVReader(Path.Combine(Common.TestDataPath, "lakeTutira120120110648_extra_small.csv")); _sensors = _reader.ReadSensors(); _startTime1 = new DateTime(46, 4, 12, 6, 48, 20); _endTime1 = new DateTime(84, 2, 8, 2, 4, 20); _startTime2 = new DateTime(2009, 1, 9, 14, 45, 0); _endTime2 = new DateTime(2009, 1, 14, 19, 15, 0); _ds1 = new Dataset(_b); _ds2 = new Dataset(_b, _sensors); }
public void FileNotFound() { _reader = new CSVReader("NotEvenAFile.csv"); }
public void ConstructorTest() { var reader = new CSVReader(TestDataFileName); }
public void BadFileName() { _reader = new CSVReader("CannotTellIfIAmCSV"); }
public void Setup() { _reader = new CSVReader(TestDataFileName); }
/// <summary> /// starts an import of data /// </summary> /// <param name="filename">filepath of data file</param> public void Import(string filename) { DisableFeatures(); var result = new List<Sensor>(); ShowProgressArea = true; ProgressIndeterminate = false; ProgressValue = 0; var reader = new CSVReader(filename); reader.ProgressChanged += (sender, args) => { ProgressValue = args.Progress; WaitEventString = string.Format("Importing from {0} {1}%", filename, ProgressValue); }; try { result = reader.ReadSensors(null, CurrentDataset); } catch (Exception ex) { Common.ShowMessageBoxWithException("Failed Import", "Bad File Format", false, true, ex); result = null; } if (result == null) { ShowProgressArea = false; EnableFeatures(); return; } var sensors = (List<Sensor>)result; if (CurrentDataset.Sensors == null || CurrentDataset.Sensors.Count == 0) { CurrentDataset.Sensors = sensors; CurrentDataset.HighestYearLoaded = NumberOfDataChunks() - 1; } else { var askUser = _container.GetInstance(typeof(SpecifyValueViewModel), "SpecifyValueViewModel") as SpecifyValueViewModel; if (askUser == null) { Common.ShowMessageBox("EPIC FAIL", "RUN AROUND WITH NO REASON", false, true); return; } askUser.ComboBoxItems = new List<string> { "Keep old values", "Keep new values" }; askUser.Text = "Keep old values"; askUser.ShowComboBox = true; askUser.Message = "How do you want to handle overlapping points. Note this will also overwrite empty or missing values within the overlaping period"; askUser.CanEditComboBox = false; askUser.ComboBoxSelectedIndex = 0; askUser.Title = "Importing"; _windowManager.ShowDialog(askUser); var keepOldValues = askUser.ComboBoxSelectedIndex == 0; var sensorMatchView = _container.GetInstance(typeof(MatchToExistingSensorsViewModel), "MatchToExistingSensorsViewModel") as MatchToExistingSensorsViewModel; if (sensorMatchView == null) return; sensorMatchView.ExistingSensors = CurrentDataset.Sensors; sensorMatchView.NewSensors = sensors; _windowManager.ShowDialog(sensorMatchView); var orderedTimestamps = sensors.SelectMany(x => x.CurrentState.Values).Select(x => x.Key).Distinct().ToArray(); var firstTimestamp = orderedTimestamps.Min(); var lastTimestamp = orderedTimestamps.Max(); orderedTimestamps = new DateTime[0]; var highestYear = NumberOfDataChunks() - 1; var firstNewYear = firstTimestamp.Year - CurrentDataset.StartYear.Year; if (firstNewYear < CurrentDataset.LowestYearLoaded) { CurrentDataset.LoadInSensorData(firstNewYear < 0 ? Enumerable.Range(0, CurrentDataset.LowestYearLoaded).ToArray() : Enumerable.Range(firstNewYear, CurrentDataset.LowestYearLoaded - firstNewYear).ToArray(), true); } var lastNewYear = lastTimestamp.Year - CurrentDataset.StartYear.Year; var reason = ChangeReason.AddNewChangeReason("[Importer] Imported new values on " + DateTime.Now); if (lastNewYear > CurrentDataset.HighestYearLoaded) { CurrentDataset.LoadInSensorData(lastNewYear > highestYear ? Enumerable.Range(CurrentDataset.HighestYearLoaded + 1, highestYear - CurrentDataset.HighestYearLoaded).ToArray() : Enumerable.Range(CurrentDataset.HighestYearLoaded + 1, lastNewYear - CurrentDataset.HighestYearLoaded).ToArray(), true); } foreach (var newSensor in sensors) { var match = sensorMatchView.SensorLinks.FirstOrDefault( x => x.MatchingSensor == newSensor); if (match == null) { Debug.WriteLine("Adding new sensor"); CurrentDataset.Sensors.Add(newSensor); } else { var matchingSensor = CurrentDataset.Sensors.FirstOrDefault( x => x == match.ExistingSensor); if (matchingSensor == null) { Debug.WriteLine( "Failed to find the sensor again, embarrasing!"); continue; } Debug.WriteLine("Merging sensors"); //Otherwise clone the current state var newState = matchingSensor.CurrentState.Clone(); //Check to see if values are inserted var insertedValues = false; //And add values for any new dates we want foreach (var value in newSensor.CurrentState.Values.Where(value => !keepOldValues || !(matchingSensor.CurrentState.Values.ContainsKey(value.Key) || matchingSensor.RawData.Values.ContainsKey(value.Key)))) { newState.Values[value.Key] = value.Value; if (matchingSensor.CurrentState.Values.ContainsKey(value.Key) || matchingSensor.RawData.Values.ContainsKey(value.Key)) newState.AddToChanges(value.Key, reason.ID); matchingSensor.RawData.Values[value.Key] = value.Value; insertedValues = true; } if (insertedValues) { //Give a reason newState.Reason = reason; //Insert new state matchingSensor.AddState(newState); matchingSensor.ClearUndoStates(); EventLogger.LogSensorInfo(CurrentDataset, matchingSensor.Name, "Added values from new import"); } } } CurrentDataset.CalculateDataSetValues(); if (CurrentDataset.HighestYearLoaded == highestYear) CurrentDataset.HighestYearLoaded = NumberOfDataChunks() - 1; } UpdateGUI(); if (Sensors.FirstOrDefault(x => x.Variable == null) != null) { var sensorVariables = SensorVariable.CreateSensorVariablesFromSensors(Sensors); foreach (var sensor in Sensors) { sensor.Variable = sensorVariables.FirstOrDefault(x => x.Sensor == sensor); } } _evaluator = new FormulaEvaluator(Sensors); _dateAnnotator.DataInterval = CurrentDataset.DataInterval; ShowProgressArea = false; EnableFeatures(); NotifyOfPropertyChange(() => LowestYearLoadedOptions); NotifyOfPropertyChange(() => HighestYearLoadedOptions); NotifyOfPropertyChange(() => LowestYearLoaded); NotifyOfPropertyChange(() => HighestYearLoaded); CurrentDataset.SaveToFile(); }
public void SetUp() { _reader = new CSVReader(Path.Combine(Common.TestDataPath, "calibrationTestData.csv")); _ds = new Dataset(_s, _reader.ReadSensors()); }
public void LogReferencesExportTest() { var reader = new CSVReader(_inputFilePath); _data.Sensors = reader.ReadSensors(); _data.Sensors = new List<Sensor> { _data.Sensors[0] }; using (var writer = File.CreateText(_outputFilePath + "ChangesTest.csv")) { writer.WriteLine("Change matrix for file: " + Path.GetFileName(_outputFilePath)); writer.WriteLine("Date,Time,Temperature"); for (var time = _data.StartTimeStamp; time <= _data.EndTimeStamp; time = time.AddMinutes(_data.DataInterval)) { writer.WriteLine(time.ToString("dd/MM/yyyy,HH:mm") + ",1"); } } var ll = new LinkedList<int>(); ll.AddFirst(1); for (var time = _data.StartTimeStamp; time <= _data.EndTimeStamp; time = time.AddMinutes(_data.DataInterval)) { _data.Sensors[0].CurrentState.Changes.Add(time, ll); } DatasetExporter.Export(_data, _outputFilePath, ExportFormat.CSV, true, false, true, loadInUnloadedValues: false); Assert.AreEqual(Tools.GenerateMD5HashFromFile(_outputFilePath + "ChangesTest.csv"), Tools.GenerateMD5HashFromFile(_outputFilePath + " Changes Matrix.csv")); }
public void ExportsRawDataWhenRequested() { var reader = new CSVReader(_inputFilePath); _data.Sensors = reader.ReadSensors(); Assert.AreNotEqual(0, _data.Sensors[0].CurrentState.Values[new DateTime(2009, 1, 10, 7, 45, 0)]); // Make some changes to check the raw comes out var newState = _data.Sensors[0].CurrentState.Clone(); newState.Values[new DateTime(2009, 1, 10, 7, 45, 0)] = 0; _data.Sensors[0].AddState(newState); DatasetExporter.Export(_data, _outputFilePath, ExportFormat.CSV, true, false, false, ExportedPoints.AllPoints, DateColumnFormat.TwoDateColumn, true, loadInUnloadedValues: false); reader = new CSVReader(_outputFilePath + " Raw.csv"); _data.Sensors = reader.ReadSensors(); Assert.AreNotEqual(0, _data.Sensors[0].CurrentState.Values[new DateTime(2009, 1, 10, 7, 45, 0)]); }