private void _removeDataWrittenOut(List <DateTime> timeStampsInRange)
{
lock (_theResultsLock)
{
foreach (var st in timeStampsInRange)
{
if (_results.ContainsKey(st))
{
_results.Remove(st);
}
}
}
var lastTimePoint = timeStampsInRange.LastOrDefault();
foreach (var sig in Signals)
{
var timeDict = sig.Value;
var tss = timeDict.Keys.Where(x => x < lastTimePoint).ToList();
for (int i = tss.Count() - 1; i >= 0; i--)
{
var ts = tss[i];
if (timeDict.ContainsKey(ts) && !_doneDataWriterFlags.ContainsKey(ts))
{
lock (_theInputSignalsLock)
{
timeDict.Remove(ts);
}
lock (_theEndTimeStampsLock)
{
EndTimeStamps.Remove(ts);
}
}
}
}
}
public bool HasDataAfter(DateTime startT, DateTime endT)
{
bool result;
lock (_theEndTimeStampsLock)
{
EndTimeStamps.Sort();
result = endT <= EndTimeStamps.LastOrDefault();
}
return(result);
}
public void Clean()
{
Signals.Clear();
StartTimeStamps.Clear();
EndTimeStamps.Clear();
TimePairs.Clear();
FirstFile = true;
ResumedTask = false;
DataCompleted = false;
FinishedSignatures.Clear();
_results.Clear();
_doneDataWriterFlags.Clear();
//FirstFileRead = false;
}
public void AddData(List <Signal> e)
{
if (e != null && e.Count > 0)
{
var startT = e.FirstOrDefault().TimeStamps.FirstOrDefault();
if (FirstFile)
{
TimeZero = startT;
//FirstFileRead = true;
FirstFile = false;
}
//else if (TimeZero >= startT)
//{
// throw new Exception("Possible data source files time order problem.");
//}
var endT = e.FirstOrDefault().TimeStamps.LastOrDefault();
//Console.WriteLine("Added end timestamp: " + endT.ToString("yyyyMMdd_HHmmss.ffffff") + " in " + e.FirstOrDefault().TimeStamps.Count() + " timestamps.");
foreach (var sig in e)
{
var name = sig.PMUName + "_" + sig.SignalName;
lock (_theInputSignalsLock)
{
if (!Signals.ContainsKey(name))
{
Signals[name] = new Dictionary <DateTime, Signal>();
}
Signals[name][endT] = sig;
}
}
lock (_theDataWriterLock)
{
if (!_doneDataWriterFlags.ContainsKey(endT))
{
_doneDataWriterFlags[endT] = new List <bool>();
}
TimePairs[startT] = endT;
StartTimeStamps.Add(startT);
}
lock (_theEndTimeStampsLock)
{
EndTimeStamps.Add(endT);
}
}
}
public bool GetData(List <Signal> signals, DateTime startT, DateTime endT, int windowSize, List <string> signalNames)
{
bool foundStart = false;
List <DateTime> possibleTimeStamps = new List <DateTime>();
lock (_theEndTimeStampsLock)
{
EndTimeStamps.Sort();
int i1;
int i2 = 0;
for (i1 = 0; i1 < EndTimeStamps.Count; i1++)
{
var firstEnd = EndTimeStamps[i1];
//find the first fragment that the start time stamp lies in.
if (startT <= firstEnd)
{
foundStart = true;
//find the last fragment that the end time stamp lines in
var endIdx = _findEndTimeFrame(endT, i1, EndTimeStamps);
if (endIdx == null)
{
// can not find end, the end might have not been read yet.
return(false);
}
else
{
i2 = (int)endIdx;
}
break;
}
}
if (!foundStart)
{
return(false);
}
//else
//{
// //delete all time stamp before i1
// EndTimeStamps.RemoveRange(0, i1);
// //for (int i = i1 - 1; i >= 0; i--)
// //{
// // EndTimeStamps.RemoveAt(i);
// //}
//}
//find the continuous time stamps that will be concatenated
possibleTimeStamps = EndTimeStamps.GetRange(i1, i2 - i1 + 1);
}
foreach (var name in signalNames)
{
Dictionary <DateTime, Signal> sig = null;
Signal firstSig, lastSig;
lock (_theInputSignalsLock)
{
sig = Signals[name];
firstSig = sig[possibleTimeStamps[0]];
lastSig = sig[possibleTimeStamps.LastOrDefault()];
}
var thisSig = new Signal(firstSig.PMUName, firstSig.SignalName);
thisSig.SamplingRate = firstSig.SamplingRate;
var firstDataPoint = firstSig.TimeStamps.IndexOf(startT);
var lastDataPoint = lastSig.TimeStamps.IndexOf(endT);
if (firstDataPoint == -1 && lastDataPoint == -1)
{
return(false);
}
if (possibleTimeStamps.Count == 1)
{
// if here's only 1 fragment, find the start and end point and get data between that range
//var lastDataPoint = firstSig.TimeStamps.IndexOf(endT);
if (lastDataPoint == -1)
{
//thisSig.Data = firstSig.Data;
thisSig.Data = firstSig.Data.GetRange(firstDataPoint, firstSig.Data.Count - firstDataPoint);
thisSig.Flags = firstSig.Flags.GetRange(firstDataPoint, firstSig.Flags.Count - firstDataPoint);
thisSig.TimeStamps = firstSig.TimeStamps.GetRange(firstDataPoint, firstSig.TimeStamps.Count - firstDataPoint);
}
else if (firstDataPoint == -1)
{
thisSig.Data = firstSig.Data.GetRange(0, lastDataPoint + 1);
thisSig.Flags = firstSig.Flags.GetRange(0, lastDataPoint + 1);
thisSig.TimeStamps = firstSig.TimeStamps.GetRange(0, lastDataPoint + 1);
}
else
{
thisSig.Data = firstSig.Data.GetRange(firstDataPoint, lastDataPoint - firstDataPoint + 1);
thisSig.Flags = firstSig.Flags.GetRange(firstDataPoint, lastDataPoint - firstDataPoint + 1);
thisSig.TimeStamps = firstSig.TimeStamps.GetRange(firstDataPoint, lastDataPoint - firstDataPoint + 1);
}
}
else
{
if (lastDataPoint == -1)
{
thisSig.Data = firstSig.Data.GetRange(firstDataPoint, firstSig.Data.Count - firstDataPoint);
thisSig.Flags = firstSig.Flags.GetRange(firstDataPoint, firstSig.Flags.Count - firstDataPoint);
thisSig.TimeStamps = firstSig.TimeStamps.GetRange(firstDataPoint, firstSig.TimeStamps.Count - firstDataPoint);
lock (_theInputSignalsLock)
{
for (int ii = 1; ii < possibleTimeStamps.Count; ii++)
{
if (sig[possibleTimeStamps[ii]].TimeStamps.LastOrDefault() <= endT)
{
thisSig.Data.AddRange(sig[possibleTimeStamps[ii]].Data);
thisSig.Flags.AddRange(sig[possibleTimeStamps[ii]].Flags);
thisSig.TimeStamps.AddRange(sig[possibleTimeStamps[ii]].TimeStamps);
}
}
}
}
else if (firstDataPoint == -1)
{
lock (_theInputSignalsLock)
{
for (int ii = 0; ii < possibleTimeStamps.Count; ii++)
{
if (sig[possibleTimeStamps[ii]].TimeStamps.LastOrDefault() <= endT)
{
thisSig.Data.AddRange(sig[possibleTimeStamps[ii]].Data);
thisSig.Flags.AddRange(sig[possibleTimeStamps[ii]].Flags);
thisSig.TimeStamps.AddRange(sig[possibleTimeStamps[ii]].TimeStamps);
}
else
{
thisSig.Data.AddRange(sig[possibleTimeStamps[ii]].Data.GetRange(0, lastDataPoint + 1));
thisSig.Flags.AddRange(sig[possibleTimeStamps[ii]].Flags.GetRange(0, lastDataPoint + 1));
thisSig.TimeStamps.AddRange(sig[possibleTimeStamps[ii]].TimeStamps.GetRange(0, lastDataPoint + 1));
}
}
}
}
else
{
// if need to put several fragment together, find the first partial piece first
thisSig.Data = firstSig.Data.GetRange(firstDataPoint, firstSig.Data.Count - firstDataPoint);
thisSig.Flags = firstSig.Flags.GetRange(firstDataPoint, firstSig.Flags.Count - firstDataPoint);
thisSig.TimeStamps = firstSig.TimeStamps.GetRange(firstDataPoint, firstSig.TimeStamps.Count - firstDataPoint);
// if there are middle pieces, add the middle pieces which should be whole pieces
for (int ii = 1; ii < possibleTimeStamps.Count; ii++)
{
thisSig.Data.AddRange(sig[possibleTimeStamps[ii]].Data);
thisSig.Flags.AddRange(sig[possibleTimeStamps[ii]].Flags);
thisSig.TimeStamps.AddRange(sig[possibleTimeStamps[ii]].TimeStamps);
}
}
// then add the last piece, which could be a partial piece too
//var lastSig = sig[possibleTimeStamps[possibleTimeStamps.Count - 1]];
//var lastDatPoint = lastSig.TimeStamps.IndexOf(endT);
//if (lastDatPoint == -1)
//{
// thisSig.Data.AddRange(lastSig.Data);
//}
//else
//{
// thisSig.Data.AddRange(lastSig.Data.GetRange(0, lastDatPoint + 1));
//}
}
//if (thisSig.Data.Count < windowSize)
//{
// return false;
//}
//else
if (thisSig.Data.Count > windowSize)
{
thisSig.Data.RemoveRange(windowSize, thisSig.Data.Count - windowSize);
thisSig.Flags.RemoveRange(windowSize, thisSig.Flags.Count - windowSize);
thisSig.TimeStamps.RemoveRange(windowSize, thisSig.TimeStamps.Count - windowSize);
signals.Add(thisSig);
}
else if (thisSig.Data.Count == windowSize)
{
signals.Add(thisSig);
}
else
{
return(false);
}
}
//EndTimeStamps.RemoveRange(0, i1);
return(true);
}
