private void _removeDataWrittenOut(List <DateTime> timeStampsInRange) { lock (_theResultsLock) { foreach (var st in timeStampsInRange) { if (_results.ContainsKey(st)) { _results.Remove(st); } } } var lastTimePoint = timeStampsInRange.LastOrDefault(); foreach (var sig in Signals) { var timeDict = sig.Value; var tss = timeDict.Keys.Where(x => x < lastTimePoint).ToList(); for (int i = tss.Count() - 1; i >= 0; i--) { var ts = tss[i]; if (timeDict.ContainsKey(ts) && !_doneDataWriterFlags.ContainsKey(ts)) { lock (_theInputSignalsLock) { timeDict.Remove(ts); } lock (_theEndTimeStampsLock) { EndTimeStamps.Remove(ts); } } } } }
public bool HasDataAfter(DateTime startT, DateTime endT) { bool result; lock (_theEndTimeStampsLock) { EndTimeStamps.Sort(); result = endT <= EndTimeStamps.LastOrDefault(); } return(result); }
public void Clean() { Signals.Clear(); StartTimeStamps.Clear(); EndTimeStamps.Clear(); TimePairs.Clear(); FirstFile = true; ResumedTask = false; DataCompleted = false; FinishedSignatures.Clear(); _results.Clear(); _doneDataWriterFlags.Clear(); //FirstFileRead = false; }
public void AddData(List <Signal> e) { if (e != null && e.Count > 0) { var startT = e.FirstOrDefault().TimeStamps.FirstOrDefault(); if (FirstFile) { TimeZero = startT; //FirstFileRead = true; FirstFile = false; } //else if (TimeZero >= startT) //{ // throw new Exception("Possible data source files time order problem."); //} var endT = e.FirstOrDefault().TimeStamps.LastOrDefault(); //Console.WriteLine("Added end timestamp: " + endT.ToString("yyyyMMdd_HHmmss.ffffff") + " in " + e.FirstOrDefault().TimeStamps.Count() + " timestamps."); foreach (var sig in e) { var name = sig.PMUName + "_" + sig.SignalName; lock (_theInputSignalsLock) { if (!Signals.ContainsKey(name)) { Signals[name] = new Dictionary <DateTime, Signal>(); } Signals[name][endT] = sig; } } lock (_theDataWriterLock) { if (!_doneDataWriterFlags.ContainsKey(endT)) { _doneDataWriterFlags[endT] = new List <bool>(); } TimePairs[startT] = endT; StartTimeStamps.Add(startT); } lock (_theEndTimeStampsLock) { EndTimeStamps.Add(endT); } } }
public bool GetData(List <Signal> signals, DateTime startT, DateTime endT, int windowSize, List <string> signalNames) { bool foundStart = false; List <DateTime> possibleTimeStamps = new List <DateTime>(); lock (_theEndTimeStampsLock) { EndTimeStamps.Sort(); int i1; int i2 = 0; for (i1 = 0; i1 < EndTimeStamps.Count; i1++) { var firstEnd = EndTimeStamps[i1]; //find the first fragment that the start time stamp lies in. if (startT <= firstEnd) { foundStart = true; //find the last fragment that the end time stamp lines in var endIdx = _findEndTimeFrame(endT, i1, EndTimeStamps); if (endIdx == null) { // can not find end, the end might have not been read yet. return(false); } else { i2 = (int)endIdx; } break; } } if (!foundStart) { return(false); } //else //{ // //delete all time stamp before i1 // EndTimeStamps.RemoveRange(0, i1); // //for (int i = i1 - 1; i >= 0; i--) // //{ // // EndTimeStamps.RemoveAt(i); // //} //} //find the continuous time stamps that will be concatenated possibleTimeStamps = EndTimeStamps.GetRange(i1, i2 - i1 + 1); } foreach (var name in signalNames) { Dictionary <DateTime, Signal> sig = null; Signal firstSig, lastSig; lock (_theInputSignalsLock) { sig = Signals[name]; firstSig = sig[possibleTimeStamps[0]]; lastSig = sig[possibleTimeStamps.LastOrDefault()]; } var thisSig = new Signal(firstSig.PMUName, firstSig.SignalName); thisSig.SamplingRate = firstSig.SamplingRate; var firstDataPoint = firstSig.TimeStamps.IndexOf(startT); var lastDataPoint = lastSig.TimeStamps.IndexOf(endT); if (firstDataPoint == -1 && lastDataPoint == -1) { return(false); } if (possibleTimeStamps.Count == 1) { // if here's only 1 fragment, find the start and end point and get data between that range //var lastDataPoint = firstSig.TimeStamps.IndexOf(endT); if (lastDataPoint == -1) { //thisSig.Data = firstSig.Data; thisSig.Data = firstSig.Data.GetRange(firstDataPoint, firstSig.Data.Count - firstDataPoint); thisSig.Flags = firstSig.Flags.GetRange(firstDataPoint, firstSig.Flags.Count - firstDataPoint); thisSig.TimeStamps = firstSig.TimeStamps.GetRange(firstDataPoint, firstSig.TimeStamps.Count - firstDataPoint); } else if (firstDataPoint == -1) { thisSig.Data = firstSig.Data.GetRange(0, lastDataPoint + 1); thisSig.Flags = firstSig.Flags.GetRange(0, lastDataPoint + 1); thisSig.TimeStamps = firstSig.TimeStamps.GetRange(0, lastDataPoint + 1); } else { thisSig.Data = firstSig.Data.GetRange(firstDataPoint, lastDataPoint - firstDataPoint + 1); thisSig.Flags = firstSig.Flags.GetRange(firstDataPoint, lastDataPoint - firstDataPoint + 1); thisSig.TimeStamps = firstSig.TimeStamps.GetRange(firstDataPoint, lastDataPoint - firstDataPoint + 1); } } else { if (lastDataPoint == -1) { thisSig.Data = firstSig.Data.GetRange(firstDataPoint, firstSig.Data.Count - firstDataPoint); thisSig.Flags = firstSig.Flags.GetRange(firstDataPoint, firstSig.Flags.Count - firstDataPoint); thisSig.TimeStamps = firstSig.TimeStamps.GetRange(firstDataPoint, firstSig.TimeStamps.Count - firstDataPoint); lock (_theInputSignalsLock) { for (int ii = 1; ii < possibleTimeStamps.Count; ii++) { if (sig[possibleTimeStamps[ii]].TimeStamps.LastOrDefault() <= endT) { thisSig.Data.AddRange(sig[possibleTimeStamps[ii]].Data); thisSig.Flags.AddRange(sig[possibleTimeStamps[ii]].Flags); thisSig.TimeStamps.AddRange(sig[possibleTimeStamps[ii]].TimeStamps); } } } } else if (firstDataPoint == -1) { lock (_theInputSignalsLock) { for (int ii = 0; ii < possibleTimeStamps.Count; ii++) { if (sig[possibleTimeStamps[ii]].TimeStamps.LastOrDefault() <= endT) { thisSig.Data.AddRange(sig[possibleTimeStamps[ii]].Data); thisSig.Flags.AddRange(sig[possibleTimeStamps[ii]].Flags); thisSig.TimeStamps.AddRange(sig[possibleTimeStamps[ii]].TimeStamps); } else { thisSig.Data.AddRange(sig[possibleTimeStamps[ii]].Data.GetRange(0, lastDataPoint + 1)); thisSig.Flags.AddRange(sig[possibleTimeStamps[ii]].Flags.GetRange(0, lastDataPoint + 1)); thisSig.TimeStamps.AddRange(sig[possibleTimeStamps[ii]].TimeStamps.GetRange(0, lastDataPoint + 1)); } } } } else { // if need to put several fragment together, find the first partial piece first thisSig.Data = firstSig.Data.GetRange(firstDataPoint, firstSig.Data.Count - firstDataPoint); thisSig.Flags = firstSig.Flags.GetRange(firstDataPoint, firstSig.Flags.Count - firstDataPoint); thisSig.TimeStamps = firstSig.TimeStamps.GetRange(firstDataPoint, firstSig.TimeStamps.Count - firstDataPoint); // if there are middle pieces, add the middle pieces which should be whole pieces for (int ii = 1; ii < possibleTimeStamps.Count; ii++) { thisSig.Data.AddRange(sig[possibleTimeStamps[ii]].Data); thisSig.Flags.AddRange(sig[possibleTimeStamps[ii]].Flags); thisSig.TimeStamps.AddRange(sig[possibleTimeStamps[ii]].TimeStamps); } } // then add the last piece, which could be a partial piece too //var lastSig = sig[possibleTimeStamps[possibleTimeStamps.Count - 1]]; //var lastDatPoint = lastSig.TimeStamps.IndexOf(endT); //if (lastDatPoint == -1) //{ // thisSig.Data.AddRange(lastSig.Data); //} //else //{ // thisSig.Data.AddRange(lastSig.Data.GetRange(0, lastDatPoint + 1)); //} } //if (thisSig.Data.Count < windowSize) //{ // return false; //} //else if (thisSig.Data.Count > windowSize) { thisSig.Data.RemoveRange(windowSize, thisSig.Data.Count - windowSize); thisSig.Flags.RemoveRange(windowSize, thisSig.Flags.Count - windowSize); thisSig.TimeStamps.RemoveRange(windowSize, thisSig.TimeStamps.Count - windowSize); signals.Add(thisSig); } else if (thisSig.Data.Count == windowSize) { signals.Add(thisSig); } else { return(false); } } //EndTimeStamps.RemoveRange(0, i1); return(true); }