/// <summary>
/// Start the data rate calculation. Must be called prior to calling <see cref="Add"/>.
/// </summary>
/// <param name="startTimestamp">Start timestamp. If null, the current time will be used.</param>
public void Start(DateTimeOffset?startTimestamp = null)
{
lock (_locker)
{
_startTimestamp = startTimestamp ?? DateTimeOffset.UtcNow;
_sampleSize = _originalSampleSize;
_dataCount = 0;
_mostRecentTimestamp = null;
_dataPerSecond = null;
// store all data to start with. we'll compute a store/drop rate after a data sample has been taken.
_samplingModulus = 1;
_samplingModulusMatchAction = SamplingAction.Keep;
}
}
/// <summary>
/// Add the specified datum to the data rate calculation. You must call <see cref="Start"/> before calling this method.
/// </summary>
/// <returns><see cref="SamplingAction"/> indicating whether the <see cref="Datum"/> should be kept (<see cref="SamplingAction.Keep"/>)
/// or dropped (<see cref="SamplingAction.Drop"/>) to meet the <see cref="_maxSamplesToKeepPerSecond"/> requirement.</returns>
/// <param name="datum">Datum to add.</param>
public SamplingAction Add(Datum datum)
{
lock (_locker)
{
// we've seen race conditions when stopping probes in which the probe is stopped (and its
// data rate calculator is also stopped), but some final data come in (e.g., via
// accelerometer). drop any such data.
if (datum == null || _sampleStartTimestamp == null)
{
return(SamplingAction.Drop);
}
// data may arrive out of order. if the current datum precedes the sample start, we don't
// really know what to do with it because the sampling modulus and match action apply to the
// current sample and not any previous samples. for the sake of safety (to not lose data),
// just keep the datum. don't update the sample data count or timestamp, as the datum does
// not apply to the current sample.
else if (datum.Timestamp < _sampleStartTimestamp)
{
return(SamplingAction.Keep);
}
// the datum applies to the current sample. update the counts. the total count is used to
// track the data indices, and the sample count is used to track progress toward gathering
// a new full sample from which to calculate the data rate. we keep separate counts, as
// a full sample triggers recalculation of the sampling modulus and match action, and this
// recalculation should not reset the data index. previously, we only kept a single index
// and lost data when the data index was reset.
_sampleCount++;
_totalCount++;
// if no maximum is specified, use the maximum possible.
double maxSamplesToKeepPerSecond = _maxSamplesToKeepPerSecond.GetValueOrDefault(double.MaxValue);
// check whether the current datum should be kept as part of sampling. if a negligible data
// rate has been specified (e.g., 0 or something close to it), then we will never keep it.
// if no sampling rate has been specified, then we'll be using a match modulus of 1 (always
// match) and a match action of keep -- that is, we'll keep all data.
SamplingAction samplingAction = SamplingAction.Drop;
if (maxSamplesToKeepPerSecond > DATA_RATE_EPSILON)
{
bool isModulusMatch = (_totalCount % _samplingModulus) == 0;
if ((isModulusMatch && _samplingModulusMatchAction == SamplingAction.Keep) ||
(!isModulusMatch && _samplingModulusMatchAction == SamplingAction.Drop))
{
samplingAction = SamplingAction.Keep;
}
}
// update the most recent timestamp (samples might come out of order). we use this to
// calculate the data rate.
if (_mostRecentTimestamp == null)
{
_mostRecentTimestamp = datum.Timestamp;
}
else if (datum.Timestamp > _mostRecentTimestamp.Value)
{
_mostRecentTimestamp = datum.Timestamp;
}
// the sample is complete. update data per second and sampling parameters.
if (_sampleCount >= _sampleSize)
{
// recalculate data per second based on current count and most recent timestamp
_dataPerSecond = _sampleCount / (_mostRecentTimestamp.Value - _sampleStartTimestamp.Value).TotalSeconds;
#region recalculate the sampling modulus/action for the new sampling rate
// in theory, the following code should work fine if a data rate of 0. however, the sampling
// modulus would then be infinity, and we cannot represent this with an integer. so check for
// a data rate of 0 and don't recalculate.
if (maxSamplesToKeepPerSecond > DATA_RATE_EPSILON)
{
// if no data rate is specified, maxDataStoresPerSecond will be double.MaxValue, making
// overage always negative and keeping all data.
double overagePerSecond = _dataPerSecond.Value - maxSamplesToKeepPerSecond;
// if we're not over the limit then keep all future data until the next sample is complete.
if (overagePerSecond <= 0)
{
_samplingModulus = 1;
_samplingModulusMatchAction = SamplingAction.Keep;
}
// we've seen cases where the data and start timestamps are identical, resulting in an infinite data per second.
// the creates a sampling modulus of zero below and subsequent divide-by-zero errors above. drop all data until
// a sample with more reasonable data timestamps comes in.
else if (double.IsInfinity(overagePerSecond))
{
SensusServiceHelper.Get().Logger.Log("Data and start timestamps are equal. Dropping all data.", LoggingLevel.Normal, GetType());
_samplingModulus = 1;
_samplingModulusMatchAction = SamplingAction.Drop;
}
// otherwise calculate a modulus that will get as close as possible to the desired rate given the empirical rate
else
{
double samplingModulusMatchRate = overagePerSecond / _dataPerSecond.Value;
_samplingModulusMatchAction = SamplingAction.Drop;
if (samplingModulusMatchRate > 0.5)
{
samplingModulusMatchRate = 1 - samplingModulusMatchRate;
_samplingModulusMatchAction = SamplingAction.Keep;
}
if (_samplingModulusMatchAction == SamplingAction.Keep)
{
// round the (store) modulus down to oversample -- more is better, right?
_samplingModulus = (long)Math.Floor(1 / samplingModulusMatchRate);
}
else
{
// round the (drop) modulus up to oversample -- more is better, right?
_samplingModulus = (long)Math.Ceiling(1 / samplingModulusMatchRate);
}
}
}
#endregion
// the sample size must be at least as large as the sampling modulus, in order to hit the modulus
// at some point in the future. if the sampling modulus is smaller than the original, then use
// the original as requested.
_sampleSize = Math.Max(_originalSampleSize, _samplingModulus);
// start a new sample from the most recent timestamp
_sampleCount = 0;
_sampleStartTimestamp = _mostRecentTimestamp.Value;
}
return(samplingAction);
}
}
/// <summary>
/// Add the specified datum to the data rate calculation. You must call <see cref="Start"/> before calling this method.
/// </summary>
/// <returns><see cref="SamplingAction"/> indicating whether the <see cref="Datum"/> should be kept (<see cref="SamplingAction.Keep"/>)
/// or dropped (<see cref="SamplingAction.Drop"/>) to meet the <see cref="_maxSamplesToKeepPerSecond"/> requirement.</returns>
/// <param name="datum">Datum to add.</param>
public SamplingAction Add(Datum datum)
{
lock (_locker)
{
if (_startTimestamp == null)
{
throw SensusException.Report("Data rate calculator has not been started.");
}
SamplingAction samplingAction = SamplingAction.Drop;
if (datum != null && datum.Timestamp >= _startTimestamp.Value)
{
_dataCount++;
// update the most recent timestamp (samples might come out of order)
if (_mostRecentTimestamp == null)
{
_mostRecentTimestamp = datum.Timestamp;
}
else if (datum.Timestamp > _mostRecentTimestamp.Value)
{
_mostRecentTimestamp = datum.Timestamp;
}
double maxSamplesToKeepPerSecond = _maxSamplesToKeepPerSecond.GetValueOrDefault(double.MaxValue);
// check whether the current datum should be kept as part of sampling. if a negligible data
// rate has been specified (e.g., 0 or something close to it), then we will never keep it.
if (maxSamplesToKeepPerSecond > DATA_RATE_EPSILON)
{
bool isModulusMatch = (_dataCount % _samplingModulus) == 0;
if ((isModulusMatch && _samplingModulusMatchAction == SamplingAction.Keep) ||
(!isModulusMatch && _samplingModulusMatchAction == SamplingAction.Drop))
{
samplingAction = SamplingAction.Keep;
}
}
// update data per second and sampling parameters for each new sample
if (_dataCount >= _sampleSize)
{
// recalculate data per second based on current count and most recent timestamp
_dataPerSecond = _dataCount / (_mostRecentTimestamp.Value - _startTimestamp.Value).TotalSeconds;
#region recalculate the sampling modulus/action for the new sampling rate
// in theory, the following code should work fine if a data rate of 0. however, the sampling
// modulus would then be infinity, and we cannot represent this with an integer. so check for
// a data rate of 0 and don't recalculate.
if (maxSamplesToKeepPerSecond > DATA_RATE_EPSILON)
{
// if no data rate is specified, maxDataStoresPerSecond will be double.MaxValue, making
// overage always negative and keeping all data.
double overagePerSecond = _dataPerSecond.Value - maxSamplesToKeepPerSecond;
// if we're not over the limit then keep all samples
if (overagePerSecond <= 0)
{
_samplingModulus = 1;
_samplingModulusMatchAction = SamplingAction.Keep;
}
// we've seen cases where the data and start timestamps are identical, resulting in an infinite data per second.
// the creates a sampling modulus of zero below and subsequent divide-by-zero errors above. drop all data until
// a sample with more reasonable data timestamps comes in.
else if (double.IsInfinity(overagePerSecond))
{
_samplingModulus = 1;
_samplingModulusMatchAction = SamplingAction.Drop;
}
// otherwise calculate a modulus that will get as close as possible to the desired rate given the empirical rate
else
{
double samplingModulusMatchRate = overagePerSecond / _dataPerSecond.Value;
_samplingModulusMatchAction = SamplingAction.Drop;
if (samplingModulusMatchRate > 0.5)
{
samplingModulusMatchRate = 1 - samplingModulusMatchRate;
_samplingModulusMatchAction = SamplingAction.Keep;
}
if (_samplingModulusMatchAction == SamplingAction.Keep)
{
// round the (store) modulus down to oversample -- more is better, right?
_samplingModulus = (long)Math.Floor(1 / samplingModulusMatchRate);
}
else
{
// round the (drop) modulus up to oversample -- more is better, right?
_samplingModulus = (long)Math.Ceiling(1 / samplingModulusMatchRate);
}
}
}
#endregion
// the sample size must be at least as large as the sampling modulus, in order to hit the modulus
// at some point in the future. if the sampling modulus is smaller than the original, then use
// the original as requested.
_sampleSize = Math.Max(_originalSampleSize, _samplingModulus);
// start a new sample
_dataCount = 0;
_startTimestamp = _mostRecentTimestamp.Value;
}
}
return(samplingAction);
}
}