public bool ReleaseOldestData(bool releaseLatest)
{
DataBucket <TInternal> releaseBucket = null;
// Under heavy stress we want to unblock even if we can't find data to release.
using (SharedLock.OpenShared(this.dataLock))
{
for (var i = this.data.Count - 1; i >= (releaseLatest ? 0 : 1); --i)
{
var bucket = this.data.Values[i];
if (bucket.Loaded)
{
releaseBucket = bucket;
break;
}
}
}
if (releaseBucket != null)
{
releaseBucket.ReleaseData();
return(true);
}
return(false);
}
private DataBucket <TInternal> GetOrCreateDataBucket(DateTime timestamp, bool addSourceData)
{
if (timestamp < earliestUnsealedBucketTime)
{
return(null);
}
var existingBucket = this.GetDataBucket(timestamp);
if (existingBucket != null)
{
return(existingBucket);
}
var newBucket = new DataBucket <TInternal>(this.CreateOptimizedDimensionSet(), timestamp,
this.properties.CompactionConfiguration
.DefaultBucketTicks,
this.storagePath,
this.properties.MemoryStreamManager);
Events.Write.CreatingDataBucket(this.Name, newBucket.StartTime, newBucket.EndTime);
this.AddBucket(newBucket);
if (addSourceData)
{
newBucket.SetSourceAvailable(this.properties.LocalSourceName);
}
return(newBucket);
}
public void UpdateFromAggregator(IPersistedDataAggregator aggregator, DateTimeOffset start, DateTimeOffset end)
{
DataBucket <TInternal> updateBucket = null;
using (SharedLock.OpenShared(this.dataLock))
{
foreach (var bucket in this.data.Values)
{
if (bucket.StartTime == start && bucket.EndTime == end)
{
updateBucket = bucket;
break;
}
}
}
if (updateBucket == null)
{
Events.Write.UnknownBucketCannotBeUpdated(this.Name, start, end);
return;
}
if (updateBucket.Sealed)
{
Events.Write.SealedBucketCannotBeUpdated(this.Name, start, end);
return;
}
var agg = aggregator as PersistedDataAggregator <TInternal>;
var availableSources = new List <string>();
foreach (var source in agg.Sources)
{
switch (source.Status)
{
case PersistedDataSourceStatus.Unavailable:
updateBucket.SetSourceUnavailable(source.Name);
break;
case PersistedDataSourceStatus.Available:
availableSources.Add(source.Name);
break;
case PersistedDataSourceStatus.Unknown:
break;
default:
throw new ArgumentException("Unexpected source status " + source.Status, "aggregator");
}
}
if (availableSources.Count > 0)
{
var aggregateData = agg.AcquireData();
updateBucket.UpdateDataFromSources(availableSources, agg.DimensionSet, aggregateData);
}
// XXX: Dump data back to disk for now (eases memory pressure)
updateBucket.ReleaseData();
}
public bool Equals(DataBucket <TInternal> other)
{
if (other != null && other.StartTicks == this.StartTicks)
{
return(true); // given usage elsewhere this really is all we need to check!
}
return(false);
}
public PendingData GetNextPendingData(DateTimeOffset previousStartTime)
{
DataBucket <TInternal> pendingBucket = null;
IList <string> pendingSources = null;
using (SharedLock.OpenShared(this.dataLock))
{
for (var i = this.data.Count - 1; i >= 0; --i)
{
var bucket = this.data.Values[i];
if (bucket.Sealed || bucket.StartTime <= previousStartTime)
{
continue;
}
var sources = bucket.GetPendingSources();
if (sources.Count == 0)
{
continue;
}
if (pendingBucket == null)
{
pendingBucket = bucket;
pendingSources = sources;
}
else if (sources.Count > pendingSources.Count)
{
pendingBucket = bucket;
pendingSources = sources;
}
}
}
if (pendingBucket != null)
{
return(new PendingData
{
// These are guaranteed UTC.
StartTime = new DateTimeOffset(pendingBucket.StartTime, TimeSpan.Zero),
EndTime = new DateTimeOffset(pendingBucket.EndTime, TimeSpan.Zero),
Sources = pendingSources,
});
}
// It may be that we have older pending data for them to work on, if we couldn't find anything try again
// with the minimum start time.
return(previousStartTime != DateTimeOffset.MinValue
? this.GetNextPendingData(DateTimeOffset.MinValue)
: null);
}
public void LoadStoredData()
{
if (this.storagePath == null)
{
return;
}
DateTime start, end;
var files = (from filename in Directory.EnumerateFiles(this.storagePath)
where DataBucket <TInternal> .GetTimestampsFromFullFilename(filename, out start, out end)
orderby filename.ToLowerInvariant() ascending
select filename).ToList();
foreach (var file in files)
{
var bucket = new DataBucket <TInternal>(new DimensionSet(this.DimensionSet), file,
this.properties.MemoryStreamManager, null);
// If we crash or are killed during compaction it is possible that both the uncompacted and
// compacted data files are still on the disk. In this case we prefer sticking with the compacted
// files.
if (this.GetDataBucket(bucket.StartTime) != null)
{
bucket.PermanentDelete();
bucket.Dispose();
continue;
}
this.AddBucket(bucket);
}
// Load the last bucket into memory (as it is most likely to be next used, and can help inform us of how
// to optimize future additions of data).
// For all unsealed buckets load their sources into memory as well.
if (this.data.Count > 0)
{
this.data.Values[0].LoadData();
foreach (var b in from bucket in this.data.Values where !bucket.Sealed select bucket)
{
b.LoadSources();
}
}
}
public void UpdateFromDataBucket(DataBucket <TInternal> otherBucket)
{
// If the other bucket isn't loaded and is file backed we can just read their file data (this helps
// to avoid creating additional unused data)
// We don't need to lock anything in that case either since we are guaranteed a sealed data bucket
// and can infer its file contents will not change.
otherBucket.Pin();
try
{
// Note that we don't mark the other bucket as dirty, so it should be reasonable to simply take
// its data once it's loaded (and then allow it to unload).
this.data.TakeData(otherBucket.data);
otherBucket.data.Dispose();
otherBucket.data = null;
this.MergeSourceStatus(otherBucket.sources.Values);
}
finally
{
otherBucket.Unpin();
}
}
private void CompactBuckets(IList <DataBucket <TInternal> > buckets, DateTime newBucketTimeStamp,
long rolledUpTimeSpanInTicks)
{
bool shouldRelease = true;
var rolledUpBucket = new DataBucket <TInternal>(buckets, this.CreateOptimizedDimensionSet(),
newBucketTimeStamp.ToLocalTime(),
rolledUpTimeSpanInTicks,
this.storagePath,
this.properties.MemoryStreamManager);
rolledUpBucket.Seal();
using (SharedLock.OpenExclusive(this.dataLock))
{
foreach (var dataBucket in buckets)
{
shouldRelease &= !dataBucket.Loaded;
if (!this.data.Remove(dataBucket.StartTime))
{
throw new InvalidOperationException("Double compaction attempted on same bucket: " + this.Name + " " + dataBucket.StartTime.ToString());
}
}
this.data.Add(rolledUpBucket.StartTime, rolledUpBucket);
}
foreach (var dataBucket in buckets)
{
dataBucket.PermanentDelete();
dataBucket.Dispose();
}
if (shouldRelease)
{
rolledUpBucket.ReleaseData();
}
}
private void AddBucket(DataBucket <TInternal> newBucket)
{
this.data.Add(newBucket.StartTime, newBucket);
if (newBucket.StartTime > this.latestStartTime)
{
this.latestStartTime = newBucket.StartTime;
}
// Every time we add a new bucket we re-scan our buckets to see if some data should be sealed or deleted
// the current bucket being added may (generally isn't, but may) not be for the newest time.
// Because we expect a low overall bucket count (topping out in the thousands) with relatively infrequent
// bucket addition, this scan is deemed reasonable. Additionally we make sure to stop the scan once
// no more buckets could be impacted. The expectation is that we will generally scan only the last bucket
// before bailing out after some relatively lightweight arithmetic.
var maxScanTime = DateTime.MinValue;
var maxBucketAge = DateTime.MinValue;
var maxUnsealedAge = DateTime.MinValue;
if (this.properties.MaximumDataAge > TimeSpan.Zero)
{
maxBucketAge = this.data.Values[0].StartTime - this.properties.MaximumDataAge;
maxScanTime = maxBucketAge;
}
if (this.properties.SealTime > TimeSpan.Zero)
{
maxUnsealedAge = this.data.Values[0].StartTime - this.properties.SealTime;
maxScanTime = new DateTime(Math.Max(maxScanTime.Ticks, maxUnsealedAge.Ticks), DateTimeKind.Utc);
}
this.earliestUnsealedBucketTime = maxScanTime;
if (maxScanTime == DateTime.MinValue)
{
return;
}
for (var i = this.data.Count - 1; i >= 0; --i)
{
var bucket = this.data.Values[i];
if (bucket.EndTime > maxScanTime)
{
break;
}
if (bucket.EndTime <= maxBucketAge)
{
this.data.Remove(bucket.StartTime);
bucket.PermanentDelete();
bucket.Dispose();
continue;
}
if (bucket.EndTime <= maxUnsealedAge && !bucket.Sealed)
{
bucket.Seal();
bucket.Persist(); // Write out sealed data immediately as well. TODO: Remove this pending serialization speed improvements.
if (this.OnDataSealed != null)
{
this.OnDataSealed(bucket.StartTime, bucket.EndTime);
}
}
}
}
