public bool Serialize(DateTime startTime, DateTime endTime, Stream destination)
{
// When serializing, if asked to send data for multiple buckets, we simply have them all serialize one-by-
// one into the stream. The reader fully supports this behavior. We could potentially save wire size by
// condensing all buckets together, but the caller may not want that and the CPU cost would be quite
// substantial to do so.
startTime = startTime.ToUniversalTime();
endTime = endTime.ToUniversalTime();
var serializeBuckets = new List <DataBucket <TInternal> >();
using (SharedLock.OpenShared(this.dataLock))
{
foreach (var bucketPair in this.data)
{
var bucketTimestamp = bucketPair.Key;
if (bucketTimestamp < startTime || bucketTimestamp >= endTime)
{
continue;
}
serializeBuckets.Add(bucketPair.Value);
}
}
foreach (var bucket in serializeBuckets)
{
bucket.Serialize(this.Name, destination);
}
return(serializeBuckets.Count > 0);
}
private IEnumerable <DataSample> PopulateBucketedDataSamples(DimensionSpecification filterDims,
Action <DataSample, TInternal, QuerySpecification>
sampleAction, QuerySpecification querySpec)
{
using (SharedLock.OpenShared(this.dataLock))
{
var bucketQuery = new BucketQuery(this, filterDims);
foreach (var bucket in bucketQuery)
{
foreach (var match in (querySpec.IsCrossQuery
? bucket.GetMatchesSplitByDimension(filterDims,
querySpec.CrossQueryDimension)
: bucket.GetMatches(filterDims)))
{
if (match.DataCount == 0)
{
continue;
}
var sample = new DataSample
{
Name = this.Name,
Dimensions = match.DimensionValues.Data,
StartTime = bucket.StartTime.ToMillisecondTimestamp(),
EndTime = bucket.EndTime.ToMillisecondTimestamp(),
};
sampleAction(sample, match.Data, querySpec);
yield return(sample);
}
}
}
Events.Write.EndQueryData(this);
}
public bool ReleaseOldestData(bool releaseLatest)
{
DataBucket <TInternal> releaseBucket = null;
// Under heavy stress we want to unblock even if we can't find data to release.
using (SharedLock.OpenShared(this.dataLock))
{
for (var i = this.data.Count - 1; i >= (releaseLatest ? 0 : 1); --i)
{
var bucket = this.data.Values[i];
if (bucket.Loaded)
{
releaseBucket = bucket;
break;
}
}
}
if (releaseBucket != null)
{
releaseBucket.ReleaseData();
return(true);
}
return(false);
}
public void UpdateFromAggregator(IPersistedDataAggregator aggregator, DateTimeOffset start, DateTimeOffset end)
{
DataBucket <TInternal> updateBucket = null;
using (SharedLock.OpenShared(this.dataLock))
{
foreach (var bucket in this.data.Values)
{
if (bucket.StartTime == start && bucket.EndTime == end)
{
updateBucket = bucket;
break;
}
}
}
if (updateBucket == null)
{
Events.Write.UnknownBucketCannotBeUpdated(this.Name, start, end);
return;
}
if (updateBucket.Sealed)
{
Events.Write.SealedBucketCannotBeUpdated(this.Name, start, end);
return;
}
var agg = aggregator as PersistedDataAggregator <TInternal>;
var availableSources = new List <string>();
foreach (var source in agg.Sources)
{
switch (source.Status)
{
case PersistedDataSourceStatus.Unavailable:
updateBucket.SetSourceUnavailable(source.Name);
break;
case PersistedDataSourceStatus.Available:
availableSources.Add(source.Name);
break;
case PersistedDataSourceStatus.Unknown:
break;
default:
throw new ArgumentException("Unexpected source status " + source.Status, "aggregator");
}
}
if (availableSources.Count > 0)
{
var aggregateData = agg.AcquireData();
updateBucket.UpdateDataFromSources(availableSources, agg.DimensionSet, aggregateData);
}
// XXX: Dump data back to disk for now (eases memory pressure)
updateBucket.ReleaseData();
}
private IEnumerable <string> GetTimestampValues(GetTimestamp getter)
{
using (SharedLock.OpenShared(this.dataLock))
{
foreach (var bucket in this.data.Values)
{
yield return(getter(bucket).ToString(Protocol.TimestampStringFormat));
}
}
}
public void Flush()
{
using (SharedLock.OpenShared(this.dataLock))
{
foreach (var b in this.data.Values)
{
b.Flush();
}
}
}
public PendingData GetNextPendingData(DateTimeOffset previousStartTime)
{
DataBucket <TInternal> pendingBucket = null;
IList <string> pendingSources = null;
using (SharedLock.OpenShared(this.dataLock))
{
for (var i = this.data.Count - 1; i >= 0; --i)
{
var bucket = this.data.Values[i];
if (bucket.Sealed || bucket.StartTime <= previousStartTime)
{
continue;
}
var sources = bucket.GetPendingSources();
if (sources.Count == 0)
{
continue;
}
if (pendingBucket == null)
{
pendingBucket = bucket;
pendingSources = sources;
}
else if (sources.Count > pendingSources.Count)
{
pendingBucket = bucket;
pendingSources = sources;
}
}
}
if (pendingBucket != null)
{
return(new PendingData
{
// These are guaranteed UTC.
StartTime = new DateTimeOffset(pendingBucket.StartTime, TimeSpan.Zero),
EndTime = new DateTimeOffset(pendingBucket.EndTime, TimeSpan.Zero),
Sources = pendingSources,
});
}
// It may be that we have older pending data for them to work on, if we couldn't find anything try again
// with the minimum start time.
return(previousStartTime != DateTimeOffset.MinValue
? this.GetNextPendingData(DateTimeOffset.MinValue)
: null);
}
public void SharedLockReleasesOnUnhandledException()
{
using (var rwSlim = new ReaderWriterLockSlim())
{
try
{
using (var testMe = SharedLock.OpenShared(rwSlim))
{
Assert.AreEqual(1, rwSlim.CurrentReadCount);
throw new InvalidOperationException("Tacos must be delicious");
}
}
catch (InvalidOperationException) { }
Assert.AreEqual(0, rwSlim.CurrentReadCount);
}
}
public IEnumerable <string> GetDimensionValues(string dimensionName, DimensionSpecification filterDims)
{
if (ReservedDimensions.StartTimeDimension.Equals(dimensionName, StringComparison.OrdinalIgnoreCase))
{
return(this.GetTimestampValues(b => b.StartTime));
}
if (ReservedDimensions.EndTimeDimension.Equals(dimensionName, StringComparison.OrdinalIgnoreCase))
{
return(this.GetTimestampValues(b => b.EndTime));
}
var values = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
if (!this.HaveDimension(dimensionName))
{
throw new KeyNotFoundException(dimensionName);
}
// If all dimensions are provided we'll end up just adding the single value back for the given dimension
// and pushing that out -- is this a neat hack to test if a dimension value exists, or is it ridiculous?
// Going with ridiculous today.
int matched = (from dim in filterDims.Keys where this.HaveDimension(dim) select dim).Count();
if (matched == this.DimensionSet.dimensions.Length)
{
throw new ArgumentException("All known dimensions were supplied in filter.", "filterDims");
}
using (SharedLock.OpenShared(this.dataLock))
{
var bucketQuery = new BucketQuery(this, filterDims);
foreach (var bucket in bucketQuery)
{
foreach (var value in bucket.GetDimensionValues(dimensionName, filterDims))
{
values.Add(value);
}
}
}
return(values);
}
private void DoCompaction()
{
var compactionQueue = new Dictionary <DateTime, CompactionSet <TInternal> >();
using (SharedLock.OpenShared(this.dataLock))
{
if (!this.properties.CompactionConfiguration.IsCompactionEnabled || this.data.Count == 0)
{
return;
}
var bucketGroups =
this.properties.CompactionConfiguration.GetEarliestTimestampsPerBucket(this.latestStartTime);
if (bucketGroups.Count < 2)
{
return; // no possible work.
}
for (var i = this.data.Count - 1; i >= 0; --i)
{
var bucket = this.data.Values[i];
if (!bucket.Sealed)
{
break;
}
DataIntervalConfiguration correctBucketConfig = null;
foreach (var kvp in bucketGroups)
{
var minTimestamp = kvp.Key;
var bucketConfig = kvp.Value;
correctBucketConfig = bucketConfig;
if (minTimestamp <= bucket.StartTime)
{
break; // previously set configuration is correct.
}
}
if (bucket.TimeSpan < correctBucketConfig.Interval)
{
CompactionSet <TInternal> compactionSet;
var targetTimestamp =
correctBucketConfig.ConvertToAlternateBucketDuration(bucket.StartTime);
if (!compactionQueue.TryGetValue(targetTimestamp, out compactionSet))
{
compactionSet = new CompactionSet <TInternal>
{
SourceBucketDuration = bucket.TimeSpan,
TargetTimestamp = targetTimestamp,
TargetIntervalConfiguration = correctBucketConfig,
};
compactionQueue.Add(targetTimestamp, compactionSet);
}
compactionSet.BucketSet.Add(bucket);
}
}
}
foreach (var compactionSet in compactionQueue.Values)
{
if (this.properties.ShuttingDown)
{
break;
}
Events.Write.BeginCompactingData(this, compactionSet.TargetIntervalConfiguration.Interval,
compactionSet.TargetTimestamp.Ticks);
this.CompactBuckets(compactionSet.BucketSet, compactionSet.TargetTimestamp,
compactionSet.TargetIntervalConfiguration.Interval.Ticks);
Events.Write.EndCompactingData(this);
}
}
private IEnumerable <DataSample> PopulateCombinedDataSamples(DimensionSpecification filterDims,
Action <DataSample, TInternal, QuerySpecification> sampleAction, QuerySpecification querySpec)
{
var combinedData = new Dictionary <string, CombinedSample>();
long start = long.MaxValue;
long end = long.MinValue;
using (SharedLock.OpenShared(this.dataLock))
{
var bucketQuery = new BucketQuery(this, filterDims);
foreach (var bucket in bucketQuery)
{
foreach (var match in (querySpec.IsCrossQuery
? bucket.GetMatchesSplitByDimension(filterDims,
querySpec.CrossQueryDimension)
: bucket.GetMatches(filterDims)))
{
if (match.DataCount == 0)
{
continue;
}
CombinedSample value;
if (!combinedData.TryGetValue(match.SplitDimensionValue, out value))
{
value = new CombinedSample
{
Data = match.Data,
Dimensions = match.DimensionValues,
};
combinedData[match.SplitDimensionValue] = value;
}
else
{
value.Data.MergeFrom(match.Data);
}
if (bucket.StartTicks < start)
{
start = bucket.StartTicks;
}
if (bucket.EndTicks > end)
{
end = bucket.EndTicks;
}
}
}
}
foreach (var value in combinedData.Values)
{
var sample =
new DataSample
{
Name = this.Name,
StartTime =
new DateTime(start, DateTimeKind.Utc).ToMillisecondTimestamp(),
EndTime =
new DateTime(end, DateTimeKind.Utc).ToMillisecondTimestamp(),
Dimensions = value.Dimensions.Data,
};
sampleAction(sample, value.Data, querySpec);
yield return(sample);
}
Events.Write.EndQueryData(this);
}
public void SharedLockValidatesParameters()
{
Assert.Throws <ArgumentNullException>(() => SharedLock.OpenShared(null));
}