/// <summary>
/// Detach the aggregated data from the aggregator.
/// </summary>
/// <returns>The aggregated data (null if it was already detached).</returns>
public KeyedDataStore <TInternal> AcquireData()
{
var ret = this.data;
this.data = null;
return(ret);
}
public void WriteData <TInternal>(string name, DateTime start, DateTime end, uint dataCount,
IEnumerable <PersistedDataSource> sources, KeyedDataStore <TInternal> data)
where TInternal : class, IInternalData, new()
{
var header = new PersistedDataHeader(name, start, end,
PersistedDataProtocol.GetPersistedTypeCodeFromType(typeof(TInternal)),
sources, this.dimensionSet, dataCount);
this.WriteDataWithLengthAndCRC32(ms =>
{
header.Write(new BufferWriter(ms));
}, header.SerializedSize, true);
// The raw data does not compress particularly well. There is some overlap in the keys, but particularly
// for histograms they come pre-compressed. We also use VLE heavily in KeyedDataStore.Serialize which
// makes for pretty compact data.
this.WriteDataWithLengthAndCRC32(data.Serialize, data.SerializedSize, false);
// We can now determine what our block length really is and back-fill our data.
var currentPosition = this.sourceStream.Position;
var blockLength = this.sourceStream.Position - this.blockStartOffset;
this.sourceStream.Position = this.blockLengthOffset;
this.sourceStreamWriter.WriteUInt64((ulong)blockLength);
this.sourceStream.Position = currentPosition;
}
public KeyedDataStore <TInternal> LoadData <TInternal>()
where TInternal : class, IInternalData, new()
{
PersistedDataType dataType = this.Header.DataType;
KeyedDataStore <TInternal> store =
this.LoadAndValidateData((ms, buffer, length) =>
{
var newStore = new KeyedDataStore <TInternal>(
this.DimensionSet, this.memoryStreamManager,
ms, (int)this.pendingObjects, dataType, "new");
this.pendingObjects -= (uint)newStore.Count;
return(newStore);
});
return(store);
}
public DataBucket(DimensionSet dimensionSet, DateTime timestamp, long timeSpanInTicks, string storagePath,
RecyclableMemoryStreamManager memoryStreamManager)
{
timestamp = timestamp.ToUniversalTime();
this.DimensionSet = dimensionSet;
this.TimeSpan = TimeSpan.FromTicks(timeSpanInTicks);
this.StartTime = RoundTimeStampToBucketKey(timestamp, timeSpanInTicks);
this.memoryStreamManager = memoryStreamManager;
if (storagePath != null)
{
this.Filename = Path.Combine(storagePath, GenerateFilename(this.StartTicks, this.EndTicks));
}
this.Blobname = GenerateBlobname(this.StartTicks, this.EndTicks);
this.data = new KeyedDataStore <TInternal>(this.DimensionSet, this.memoryStreamManager, "db ctor");
}
public void UpdateDataFromSources(IList <string> sourceList, DimensionSet sourceDimensions,
KeyedDataStore <TInternal> sourceData)
{
if (this.Sealed)
{
throw new InvalidOperationException("Attempt to write to sealed bucket.");
}
foreach (var s in sourceList)
{
// Below we do some sanity checking to make sure that we're not ingesting data we were already given,
// or ingesting data from a source that wasn't pre-declared as an input. Either of these would indicate
// an upstream logic fault.
var source = this.FindSource(s);
if (source == null)
{
throw new InvalidOperationException("Adding data from previously unknown source " + s);
}
if (source.Status != PersistedDataSourceStatus.Unknown)
{
throw new InvalidOperationException("Double adding data from source " + s);
}
source.Status = PersistedDataSourceStatus.Available;
}
using (SharedLock.OpenExclusive(this.dataAccessLock))
{
this.Load();
if (this.data == null || this.data.Empty)
{
this.DimensionSet = sourceDimensions;
this.data = sourceData;
}
else
{
this.data.TakeData(sourceData);
sourceData.Dispose();
}
this.dirty = true;
}
}
public void TakeData(KeyedDataStore <TInternal> otherData)
{
lock (this)
{
if (otherData.mergedData != null)
{
this.unmergedData.Add(otherData.mergedData);
otherData.mergedData = null;
}
if (otherData.pendingDataStream != null)
{
otherData.SealWriteBuffer(null);
}
this.unmergedData.AddRange(otherData.unmergedData);
otherData.unmergedData.Clear();
}
}
/// <summary>
/// Release data from memory and write it to persisted storage if possible. This call will block until all users
/// are finished accessing the data.
/// </summary>
public void ReleaseData()
{
using (SharedLock.OpenExclusive(this.dataAccessLock))
{
this.Persist();
if (this.data != null)
{
this.data.Dispose();
// force unloading our (probably populous) dimension set by doing a shallow copy.
// NOTE: This also gets rid of potentially large arrays whereas clearing them might not do so. Other
// work could be done to free up this memory but there's not a ton of value in this.
this.DimensionSet = new DimensionSet(this.DimensionSet);
}
// Once sealed we can guarantee no more source updates.
if (this.Sealed)
{
this.sources.Clear();
}
this.data = null;
}
}
private void UnpackResponse(PersistedDataReader dataReader)
{
while (dataReader.ReadDataHeader())
{
lock (this)
{
if (this.DimensionSet == null)
{
this.DimensionSet = dataReader.DimensionSet;
}
var remoteData = dataReader.LoadData <TInternal>();
if (!remoteData.Validate())
{
remoteData.Dispose();
throw new PersistedDataException("Remote data is invalid.");
}
if (this.data == null && this.DimensionSet.Equals(dataReader.DimensionSet))
{
this.DimensionSet = dataReader.DimensionSet;
this.data = remoteData;
}
else
{
if (this.data == null)
{
this.data = new KeyedDataStore <TInternal>(this.DimensionSet, this.memoryStreamManager);
}
this.data.TakeData(remoteData);
remoteData.Dispose();
}
}
foreach (var responseSource in dataReader.Header.Sources)
{
// NOTE: We use 'StartsWith' below because lots of data is currently marked with
// the non-FQDN machine name. We can change it to 'Equals' in the future.
var localSource =
this.Sources.FirstOrDefault(s =>
s.Name.StartsWith(responseSource.Name,
StringComparison.OrdinalIgnoreCase));
if (localSource == null)
{
// XXX: here for testing because we have to query 'localhost' but won't ever get something good
// back. Yes, this is STUPID.
if (this.Sources.Count == 1 && this.Sources[0].Name == "localhost")
{
localSource = this.Sources[0];
}
else
{
throw new PersistedDataException(
string.Format("Source {0} returned by server is unknown", responseSource.Name));
}
}
localSource.Status = responseSource.Status;
}
}
}
private void ReadFromStorage(string filename, bool sourcesOnly)
{
KeyedDataStore <TInternal> readData = null;
try
{
Events.Write.BeginLoadingData(filename);
using (var stream = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
{
using (var reader = new PersistedDataReader(stream, this.memoryStreamManager, this.DimensionSet))
{
// We'll only ever write one set of data, so we only try to read one.
if (reader.ReadDataHeader())
{
this.MergeSourceStatus(reader.Header.Sources);
if (sourcesOnly)
{
return;
}
if (typeof(TInternal) != reader.DataType)
{
throw new InvalidDataException(
string.Format(
"this.start={0},this.end={1}, hdr.start={2}, hdr.end={3}, this.type={4}, hdr.type={5}",
this.StartTicks, this.EndTicks, reader.StartTime.Ticks,
reader.EndTime.Ticks,
typeof(TInternal), reader.DataType));
}
readData = reader.LoadData <TInternal>();
if (this.DimensionSet.Equals(reader.DimensionSet) &&
(this.data == null || this.data.Empty))
{
this.DimensionSet = reader.DimensionSet;
if (this.data != null)
{
this.data.Dispose();
}
this.data = readData;
readData = null;
}
else
{
if (this.data == null)
{
this.data = new KeyedDataStore <TInternal>(this.DimensionSet,
this.memoryStreamManager,
"read data");
}
this.data.TakeData(readData);
this.data.Merge();
}
}
}
}
}
catch (FileNotFoundException)
{
Events.Write.SealedDataFileMissing(filename);
}
catch (PersistedDataException ex)
{
Events.Write.PersistedDataException(ex);
Events.Write.DiscardingIncompleteData(filename);
if (this.data != null)
{
this.data.Dispose();
}
this.data = null;
}
catch (OutOfMemoryException)
{
// TODO: this code is here to deal with file corruption issues, once checksums have been added and
// are fully available it must be removed.
File.Delete(filename);
throw;
}
finally
{
if (readData != null)
{
readData.Dispose();
}
}
// if data is still null, some error happened loading the file (empty, missing, corrupt).
// Just start over with clean data.
if (this.data == null)
{
if (File.Exists(this.Filename))
{
File.Delete(this.Filename);
}
this.data = new KeyedDataStore <TInternal>(this.DimensionSet, this.memoryStreamManager,
"placeholder for invalid data.");
}
Events.Write.EndLoadingData(filename);
}
