public void SetLatestTimeForDataSources(SortedList <DateTimeOffset, List <string> > sourcesByTime)
{
Events.Write.BeginSetDataSourceTimes(this.Name);
using (SharedLock.OpenExclusive(this.dataLock))
{
foreach (var kvp in sourcesByTime)
{
var latestTime = kvp.Key;
var serverList = kvp.Value;
if (latestTime < this.earliestUnsealedBucketTime)
{
continue;
}
DateTimeOffset bucketTimestamp =
this.earliestUnsealedBucketTime == DateTime.MinValue
? latestTime - this.properties.SealTime
: this.earliestUnsealedBucketTime;
while (bucketTimestamp < latestTime)
{
// we ensure above that we won't ask for a time that is too early, so bucket is guaranteed to be
// non-null.
var bucket = this.GetOrCreateDataBucket(bucketTimestamp.UtcDateTime, false);
bucketTimestamp += this.properties.CompactionConfiguration.Default.Interval;
foreach (var server in serverList)
{
bucket.AddDataSource(server);
}
}
}
}
Events.Write.EndSetDataSourceTimes(this.Name);
}
/// <summary>
/// Seal data (preventing further modification). Does not write to permanent storage.
/// </summary>
public void Seal()
{
using (SharedLock.OpenExclusive(this.dataAccessLock))
{
this.Flush();
this.Sealed = true;
}
}
public void SharedLockProtectsAgainstDoubleDispose()
{
using (var rwSlim = new ReaderWriterLockSlim())
{
var testMe = SharedLock.OpenExclusive(rwSlim);
testMe.Dispose();
Assert.Throws <ObjectDisposedException>(testMe.Dispose);
}
}
public void Dispose()
{
if (this.dataLock != null)
{
using (SharedLock.OpenExclusive(this.dataLock))
{
foreach (var bucket in this.data.Values)
{
bucket.Dispose();
}
this.data.Clear();
}
this.dataLock.Dispose();
this.dataLock = null;
}
}
public void UpdateDataFromSources(IList <string> sourceList, DimensionSet sourceDimensions,
KeyedDataStore <TInternal> sourceData)
{
if (this.Sealed)
{
throw new InvalidOperationException("Attempt to write to sealed bucket.");
}
foreach (var s in sourceList)
{
// Below we do some sanity checking to make sure that we're not ingesting data we were already given,
// or ingesting data from a source that wasn't pre-declared as an input. Either of these would indicate
// an upstream logic fault.
var source = this.FindSource(s);
if (source == null)
{
throw new InvalidOperationException("Adding data from previously unknown source " + s);
}
if (source.Status != PersistedDataSourceStatus.Unknown)
{
throw new InvalidOperationException("Double adding data from source " + s);
}
source.Status = PersistedDataSourceStatus.Available;
}
using (SharedLock.OpenExclusive(this.dataAccessLock))
{
this.Load();
if (this.data == null || this.data.Empty)
{
this.DimensionSet = sourceDimensions;
this.data = sourceData;
}
else
{
this.data.TakeData(sourceData);
sourceData.Dispose();
}
this.dirty = true;
}
}
public void AddValue(long value, DimensionSpecification dims, DateTime timestamp)
{
timestamp = timestamp.ToUniversalTime();
DataBucket <TInternal> bucket;
using (SharedLock.OpenExclusive(this.dataLock))
{
bucket = this.GetOrCreateDataBucket(timestamp, true);
if (bucket == null)
{
Events.Write.RejectedAttemptToWriteAncientData(this.Name, this.earliestUnsealedBucketTime,
timestamp);
return;
}
}
bucket.AddValue(dims, value);
}
private void CompactBuckets(IList <DataBucket <TInternal> > buckets, DateTime newBucketTimeStamp,
long rolledUpTimeSpanInTicks)
{
bool shouldRelease = true;
var rolledUpBucket = new DataBucket <TInternal>(buckets, this.CreateOptimizedDimensionSet(),
newBucketTimeStamp.ToLocalTime(),
rolledUpTimeSpanInTicks,
this.storagePath,
this.properties.MemoryStreamManager);
rolledUpBucket.Seal();
using (SharedLock.OpenExclusive(this.dataLock))
{
foreach (var dataBucket in buckets)
{
shouldRelease &= !dataBucket.Loaded;
if (!this.data.Remove(dataBucket.StartTime))
{
throw new InvalidOperationException("Double compaction attempted on same bucket: " + this.Name + " " + dataBucket.StartTime.ToString());
}
}
this.data.Add(rolledUpBucket.StartTime, rolledUpBucket);
}
foreach (var dataBucket in buckets)
{
dataBucket.PermanentDelete();
dataBucket.Dispose();
}
if (shouldRelease)
{
rolledUpBucket.ReleaseData();
}
}
/// <summary>
/// Release data from memory and write it to persisted storage if possible. This call will block until all users
/// are finished accessing the data.
/// </summary>
public void ReleaseData()
{
using (SharedLock.OpenExclusive(this.dataAccessLock))
{
this.Persist();
if (this.data != null)
{
this.data.Dispose();
// force unloading our (probably populous) dimension set by doing a shallow copy.
// NOTE: This also gets rid of potentially large arrays whereas clearing them might not do so. Other
// work could be done to free up this memory but there's not a ton of value in this.
this.DimensionSet = new DimensionSet(this.DimensionSet);
}
// Once sealed we can guarantee no more source updates.
if (this.Sealed)
{
this.sources.Clear();
}
this.data = null;
}
}