public void ExportAsCSVNoEmptyLines()
{
var reader = new CSVReader(Path.Combine(_inputFilePath));
_data.Sensors = reader.ReadSensors();
DatasetExporter.Export(_data, _outputFilePath, ExportFormat.CSV, false, false, false, ExportedPoints.AllPoints, DateColumnFormat.TwoDateColumn, loadInUnloadedValues: false);
Assert.AreEqual(Tools.GenerateMD5HashFromFile(_outputFilePath), Tools.GenerateMD5HashFromFile(_inputFilePath));
}
public void ExportAsCSVEmptyLinesIncludedWithAverageEveryHour()
{
var reader = new CSVReader(Path.Combine(_inputFilePath));
_data.Sensors = reader.ReadSensors();
DatasetExporter.Export(_data, _outputFilePath, ExportFormat.CSV, true, false, false, ExportedPoints.HourlyPoints, DateColumnFormat.TwoDateColumn, loadInUnloadedValues: false);
Assert.AreEqual(File.ReadLines(_outputFilePath).Count(), ((_data.ExpectedDataPointCount) / 4) + 1);
}
public void Setup()
{
_reader = new CSVReader(Path.Combine(Common.TestDataPath, "lakeTutira120120110648_extra_small.csv"));
_ds = new Dataset(_s, _reader.ReadSensors());
var sensorVariables = SensorVariable.CreateSensorVariablesFromSensors(_ds.Sensors);
foreach (var sensor in _ds.Sensors)
{
sensor.Owner = _ds;
sensor.Variable = sensorVariables.FirstOrDefault(x => x.Sensor == sensor);
}
_eval = new FormulaEvaluator(_ds.Sensors);
}
public void ReadsCSVWithYYYYMMDDTogetherAndmmhhTogetherSplitByColon()
{
var fileName = Path.GetTempFileName() + ".csv";
File.WriteAllText(fileName, "YYYYMMDD,hh:mm,Test Sensor\r\n20110510,13:00,100");
var reader = new CSVReader(fileName);
var sensors = reader.ReadSensors();
Assert.AreEqual(1, sensors.Count);
Assert.AreEqual(100, sensors[0].CurrentState.Values[new DateTime(2011, 5, 10, 13, 0, 0)]);
Assert.AreEqual(1, sensors[0].CurrentState.Values.Count);
}
public void ReadsCSVWithYYYYMMDDmmhhSeperately()
{
var fileName = Path.GetTempFileName() + ".csv";
File.WriteAllText(fileName, "YYYY,MM,DD,hh,mm,Test Sensor\r\n2011,05,10,13,00,100");
var reader = new CSVReader(fileName);
var sensors = reader.ReadSensors();
Assert.AreEqual(1, sensors.Count);
Assert.AreEqual(100, sensors[0].CurrentState.Values[new DateTime(2011, 5, 10, 13, 0, 0)]);
Assert.AreEqual(1, sensors[0].CurrentState.Values.Count);
}
public void DetectIntervalWithNoMissingValues()
{
_reader = new CSVReader(FifteenMinuteIntervalData);
Dataset ds = new Dataset(_b, _reader.ReadSensors());
Assert.AreEqual(15, ds.DataInterval);
}
public void CorrectActualDataPointCount()
{
_reader = new CSVReader(Path.Combine(Common.TestDataPath, "lakeTutira120120110648.csv"));
Dataset ds = new Dataset(_b, _reader.ReadSensors());
Assert.AreEqual(53873, ds.ActualDataPointCount);
}
public void Setup()
{
_reader = new CSVReader(Path.Combine(Common.TestDataPath, "lakeTutira120120110648_extra_small.csv"));
_sensors = _reader.ReadSensors();
_startTime1 = new DateTime(46, 4, 12, 6, 48, 20);
_endTime1 = new DateTime(84, 2, 8, 2, 4, 20);
_startTime2 = new DateTime(2009, 1, 9, 14, 45, 0);
_endTime2 = new DateTime(2009, 1, 14, 19, 15, 0);
_ds1 = new Dataset(_b);
_ds2 = new Dataset(_b, _sensors);
}
public void FileNotFound()
{
_reader = new CSVReader("NotEvenAFile.csv");
}
public void ConstructorTest()
{
var reader = new CSVReader(TestDataFileName);
}
public void BadFileName()
{
_reader = new CSVReader("CannotTellIfIAmCSV");
}
public void Setup()
{
_reader = new CSVReader(TestDataFileName);
}
/// <summary>
/// starts an import of data
/// </summary>
/// <param name="filename">filepath of data file</param>
public void Import(string filename)
{
DisableFeatures();
var result = new List<Sensor>();
ShowProgressArea = true;
ProgressIndeterminate = false;
ProgressValue = 0;
var reader = new CSVReader(filename);
reader.ProgressChanged += (sender, args) =>
{
ProgressValue = args.Progress;
WaitEventString =
string.Format("Importing from {0} {1}%",
filename,
ProgressValue);
};
try
{
result = reader.ReadSensors(null, CurrentDataset);
}
catch (Exception ex)
{
Common.ShowMessageBoxWithException("Failed Import", "Bad File Format", false, true, ex);
result = null;
}
if (result == null)
{
ShowProgressArea = false;
EnableFeatures();
return;
}
var sensors = (List<Sensor>)result;
if (CurrentDataset.Sensors == null || CurrentDataset.Sensors.Count == 0)
{
CurrentDataset.Sensors = sensors;
CurrentDataset.HighestYearLoaded = NumberOfDataChunks() - 1;
}
else
{
var askUser =
_container.GetInstance(typeof(SpecifyValueViewModel),
"SpecifyValueViewModel") as
SpecifyValueViewModel;
if (askUser == null)
{
Common.ShowMessageBox("EPIC FAIL", "RUN AROUND WITH NO REASON",
false, true);
return;
}
askUser.ComboBoxItems = new List<string> { "Keep old values", "Keep new values" };
askUser.Text = "Keep old values";
askUser.ShowComboBox = true;
askUser.Message = "How do you want to handle overlapping points. Note this will also overwrite empty or missing values within the overlaping period";
askUser.CanEditComboBox = false;
askUser.ComboBoxSelectedIndex = 0;
askUser.Title = "Importing";
_windowManager.ShowDialog(askUser);
var keepOldValues = askUser.ComboBoxSelectedIndex == 0;
var sensorMatchView =
_container.GetInstance(typeof(MatchToExistingSensorsViewModel),
"MatchToExistingSensorsViewModel") as
MatchToExistingSensorsViewModel;
if (sensorMatchView == null)
return;
sensorMatchView.ExistingSensors = CurrentDataset.Sensors;
sensorMatchView.NewSensors = sensors;
_windowManager.ShowDialog(sensorMatchView);
var orderedTimestamps = sensors.SelectMany(x => x.CurrentState.Values).Select(x => x.Key).Distinct().ToArray();
var firstTimestamp = orderedTimestamps.Min();
var lastTimestamp = orderedTimestamps.Max();
orderedTimestamps = new DateTime[0];
var highestYear = NumberOfDataChunks() - 1;
var firstNewYear = firstTimestamp.Year - CurrentDataset.StartYear.Year;
if (firstNewYear < CurrentDataset.LowestYearLoaded)
{
CurrentDataset.LoadInSensorData(firstNewYear < 0 ?
Enumerable.Range(0, CurrentDataset.LowestYearLoaded).ToArray()
: Enumerable.Range(firstNewYear, CurrentDataset.LowestYearLoaded - firstNewYear).ToArray(), true);
}
var lastNewYear = lastTimestamp.Year - CurrentDataset.StartYear.Year;
var reason = ChangeReason.AddNewChangeReason("[Importer] Imported new values on " + DateTime.Now);
if (lastNewYear > CurrentDataset.HighestYearLoaded)
{
CurrentDataset.LoadInSensorData(lastNewYear > highestYear ?
Enumerable.Range(CurrentDataset.HighestYearLoaded + 1, highestYear - CurrentDataset.HighestYearLoaded).ToArray()
: Enumerable.Range(CurrentDataset.HighestYearLoaded + 1, lastNewYear - CurrentDataset.HighestYearLoaded).ToArray(), true);
}
foreach (var newSensor in sensors)
{
var match =
sensorMatchView.SensorLinks.FirstOrDefault(
x => x.MatchingSensor == newSensor);
if (match == null)
{
Debug.WriteLine("Adding new sensor");
CurrentDataset.Sensors.Add(newSensor);
}
else
{
var matchingSensor =
CurrentDataset.Sensors.FirstOrDefault(
x => x == match.ExistingSensor);
if (matchingSensor == null)
{
Debug.WriteLine(
"Failed to find the sensor again, embarrasing!");
continue;
}
Debug.WriteLine("Merging sensors");
//Otherwise clone the current state
var newState = matchingSensor.CurrentState.Clone();
//Check to see if values are inserted
var insertedValues = false;
//And add values for any new dates we want
foreach (var value in newSensor.CurrentState.Values.Where(value =>
!keepOldValues || !(matchingSensor.CurrentState.Values.ContainsKey(value.Key) || matchingSensor.RawData.Values.ContainsKey(value.Key))))
{
newState.Values[value.Key] = value.Value;
if (matchingSensor.CurrentState.Values.ContainsKey(value.Key) || matchingSensor.RawData.Values.ContainsKey(value.Key))
newState.AddToChanges(value.Key, reason.ID);
matchingSensor.RawData.Values[value.Key] = value.Value;
insertedValues = true;
}
if (insertedValues)
{
//Give a reason
newState.Reason = reason;
//Insert new state
matchingSensor.AddState(newState);
matchingSensor.ClearUndoStates();
EventLogger.LogSensorInfo(CurrentDataset,
matchingSensor.Name,
"Added values from new import");
}
}
}
CurrentDataset.CalculateDataSetValues();
if (CurrentDataset.HighestYearLoaded == highestYear)
CurrentDataset.HighestYearLoaded = NumberOfDataChunks() - 1;
}
UpdateGUI();
if (Sensors.FirstOrDefault(x => x.Variable == null) != null)
{
var sensorVariables = SensorVariable.CreateSensorVariablesFromSensors(Sensors);
foreach (var sensor in Sensors)
{
sensor.Variable = sensorVariables.FirstOrDefault(x => x.Sensor == sensor);
}
}
_evaluator = new FormulaEvaluator(Sensors);
_dateAnnotator.DataInterval = CurrentDataset.DataInterval;
ShowProgressArea = false;
EnableFeatures();
NotifyOfPropertyChange(() => LowestYearLoadedOptions);
NotifyOfPropertyChange(() => HighestYearLoadedOptions);
NotifyOfPropertyChange(() => LowestYearLoaded);
NotifyOfPropertyChange(() => HighestYearLoaded);
CurrentDataset.SaveToFile();
}
public void SetUp()
{
_reader = new CSVReader(Path.Combine(Common.TestDataPath, "calibrationTestData.csv"));
_ds = new Dataset(_s, _reader.ReadSensors());
}
public void LogReferencesExportTest()
{
var reader = new CSVReader(_inputFilePath);
_data.Sensors = reader.ReadSensors();
_data.Sensors = new List<Sensor> { _data.Sensors[0] };
using (var writer = File.CreateText(_outputFilePath + "ChangesTest.csv"))
{
writer.WriteLine("Change matrix for file: " + Path.GetFileName(_outputFilePath));
writer.WriteLine("Date,Time,Temperature");
for (var time = _data.StartTimeStamp; time <= _data.EndTimeStamp; time = time.AddMinutes(_data.DataInterval))
{
writer.WriteLine(time.ToString("dd/MM/yyyy,HH:mm") + ",1");
}
}
var ll = new LinkedList<int>();
ll.AddFirst(1);
for (var time = _data.StartTimeStamp; time <= _data.EndTimeStamp; time = time.AddMinutes(_data.DataInterval))
{
_data.Sensors[0].CurrentState.Changes.Add(time, ll);
}
DatasetExporter.Export(_data, _outputFilePath, ExportFormat.CSV, true, false, true, loadInUnloadedValues: false);
Assert.AreEqual(Tools.GenerateMD5HashFromFile(_outputFilePath + "ChangesTest.csv"), Tools.GenerateMD5HashFromFile(_outputFilePath + " Changes Matrix.csv"));
}
public void ExportsRawDataWhenRequested()
{
var reader = new CSVReader(_inputFilePath);
_data.Sensors = reader.ReadSensors();
Assert.AreNotEqual(0, _data.Sensors[0].CurrentState.Values[new DateTime(2009, 1, 10, 7, 45, 0)]);
// Make some changes to check the raw comes out
var newState = _data.Sensors[0].CurrentState.Clone();
newState.Values[new DateTime(2009, 1, 10, 7, 45, 0)] = 0;
_data.Sensors[0].AddState(newState);
DatasetExporter.Export(_data, _outputFilePath, ExportFormat.CSV, true, false, false, ExportedPoints.AllPoints, DateColumnFormat.TwoDateColumn, true, loadInUnloadedValues: false);
reader = new CSVReader(_outputFilePath + " Raw.csv");
_data.Sensors = reader.ReadSensors();
Assert.AreNotEqual(0, _data.Sensors[0].CurrentState.Values[new DateTime(2009, 1, 10, 7, 45, 0)]);
}
