internal unsafe Dimension(byte *buffer, long bufferSize)
{
string value;
var offset = PersistedDataReader.ReadLegacyStringValue(buffer, bufferSize, out value);
this.Name = value;
PersistedDataReader.CheckRemainingBufferSize(bufferSize, offset, sizeof(int));
var valueCount = *(int *)(buffer + offset);
offset += sizeof(int);
this.indexToValues = new List <string>(valueCount);
while (valueCount > 0)
{
offset += PersistedDataReader.ReadLegacyStringValue((buffer + offset), bufferSize - offset,
out value);
if (this.valuesToIndex.ContainsKey(value))
{
throw new PersistedDataException(string.Format("Duplicate dimension value {0} in {1}", value,
this.Name));
}
this.valuesToIndex.Add(value, (uint)this.indexToValues.Count);
this.indexToValues.Add(value);
--valueCount;
}
this.SerializedSize = offset;
}
internal unsafe DimensionSet(byte *buffer, uint bufferSize)
{
long offset = 0;
PersistedDataReader.CheckRemainingBufferSize(bufferSize, 0, sizeof(int));
var dimensionCount = *(int *)buffer;
offset += sizeof(int);
this.dimensions = new Dimension[dimensionCount];
for (var i = 0; i < dimensionCount; ++i)
{
this.dimensions[i] = new Dimension((buffer + offset), bufferSize - offset);
offset += this.dimensions[i].SerializedSize;
}
this.SerializedSize = offset;
}
private void UnpackResponse(PersistedDataReader dataReader)
{
while (dataReader.ReadDataHeader())
{
lock (this)
{
if (this.DimensionSet == null)
{
this.DimensionSet = dataReader.DimensionSet;
}
var remoteData = dataReader.LoadData <TInternal>();
if (!remoteData.Validate())
{
remoteData.Dispose();
throw new PersistedDataException("Remote data is invalid.");
}
if (this.data == null && this.DimensionSet.Equals(dataReader.DimensionSet))
{
this.DimensionSet = dataReader.DimensionSet;
this.data = remoteData;
}
else
{
if (this.data == null)
{
this.data = new KeyedDataStore <TInternal>(this.DimensionSet, this.memoryStreamManager);
}
this.data.TakeData(remoteData);
remoteData.Dispose();
}
}
foreach (var responseSource in dataReader.Header.Sources)
{
// NOTE: We use 'StartsWith' below because lots of data is currently marked with
// the non-FQDN machine name. We can change it to 'Equals' in the future.
var localSource =
this.Sources.FirstOrDefault(s =>
s.Name.StartsWith(responseSource.Name,
StringComparison.OrdinalIgnoreCase));
if (localSource == null)
{
// XXX: here for testing because we have to query 'localhost' but won't ever get something good
// back. Yes, this is STUPID.
if (this.Sources.Count == 1 && this.Sources[0].Name == "localhost")
{
localSource = this.Sources[0];
}
else
{
throw new PersistedDataException(
string.Format("Source {0} returned by server is unknown", responseSource.Name));
}
}
localSource.Status = responseSource.Status;
}
}
}
private async Task <bool> QuerySources(IList <PersistedDataSource> sources)
{
Events.Write.BeginQuerySources(sources);
var success = false;
var transferRequest =
new TransferRequest
{
DataType = PersistedDataProtocol.GetPersistedTypeCodeFromType(typeof(TInternal)),
Timeout = (this.Timeout.Seconds * 9) / 10, // 90% of timeout goes to the child
MaxFanout = this.MaxFanout,
Sources = new List <string>()
};
foreach (var source in sources)
{
source.Status = PersistedDataSourceStatus.Unknown;
transferRequest.Sources.Add(source.Name);
}
var serverOffset = this.randomNumberGenerator.Next(sources.Count);
var selectedSource = sources[serverOffset];
var request = new HttpRequestMessage(HttpMethod.Post, this.CreateRequestUri(selectedSource.Name));
using (var ms = (this.memoryStreamManager.GetStream()))
using (var writeStream = new WriterStream(ms, this.memoryStreamManager))
{
var writer = writeStream.CreateCompactBinaryWriter();
writer.Write(transferRequest);
request.Content = new ByteArrayContent(ms.ToArray());
}
try
{
Events.Write.BeginSendSourceQuery(selectedSource, request.RequestUri);
var response =
await this.httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
Events.Write.EndSendSourceQuery(selectedSource, (int)response.StatusCode, response.ReasonPhrase);
if (!response.IsSuccessStatusCode)
{
switch (response.StatusCode)
{
case HttpStatusCode.NotFound:
foreach (var source in sources)
{
source.Status = PersistedDataSourceStatus.Unavailable;
}
return(true);
default:
return(false);
}
}
Events.Write.BeginReceiveSourceResponseBody(selectedSource);
var length = response.Content.Headers.ContentLength ?? 0;
using (var dataStream = this.memoryStreamManager.GetStream("PersistedDataAggregator/Http/Read",
(int)length, true))
using (var responseStream = await response.Content.ReadAsStreamAsync())
{
responseStream.CopyTo(dataStream);
dataStream.Position = 0;
using (var dataReader =
new PersistedDataReader(dataStream, this.memoryStreamManager, this.DimensionSet))
{
Events.Write.EndReceiveSourceResponseBody(selectedSource, (int)dataStream.Length);
success = true;
if (dataStream.Length == 0)
{
Events.Write.EmptyResponseReceivedFromSource(selectedSource);
// Over the line. Mark it zero. Or just available, but empty (which is okay!)
foreach (var source in sources)
{
source.Status = PersistedDataSourceStatus.Available;
}
}
else
{
try
{
this.UnpackResponse(dataReader);
}
catch (PersistedDataException ex)
{
Events.Write.PersistedDataExceptionFromSource(selectedSource, ex);
success = false;
}
}
}
}
}
catch (OperationCanceledException) { }
catch (Exception ex)
{
if (ex is HttpRequestException || ex is IOException || ex is WebException || ex is ObjectDisposedException)
{
Events.Write.HttpExceptionFromSource(selectedSource, ex);
}
else
{
throw;
}
}
Events.Write.EndQuerySources(success, sources);
return(success);
}
private void ReadFromStorage(string filename, bool sourcesOnly)
{
KeyedDataStore <TInternal> readData = null;
try
{
Events.Write.BeginLoadingData(filename);
using (var stream = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
{
using (var reader = new PersistedDataReader(stream, this.memoryStreamManager, this.DimensionSet))
{
// We'll only ever write one set of data, so we only try to read one.
if (reader.ReadDataHeader())
{
this.MergeSourceStatus(reader.Header.Sources);
if (sourcesOnly)
{
return;
}
if (typeof(TInternal) != reader.DataType)
{
throw new InvalidDataException(
string.Format(
"this.start={0},this.end={1}, hdr.start={2}, hdr.end={3}, this.type={4}, hdr.type={5}",
this.StartTicks, this.EndTicks, reader.StartTime.Ticks,
reader.EndTime.Ticks,
typeof(TInternal), reader.DataType));
}
readData = reader.LoadData <TInternal>();
if (this.DimensionSet.Equals(reader.DimensionSet) &&
(this.data == null || this.data.Empty))
{
this.DimensionSet = reader.DimensionSet;
if (this.data != null)
{
this.data.Dispose();
}
this.data = readData;
readData = null;
}
else
{
if (this.data == null)
{
this.data = new KeyedDataStore <TInternal>(this.DimensionSet,
this.memoryStreamManager,
"read data");
}
this.data.TakeData(readData);
this.data.Merge();
}
}
}
}
}
catch (FileNotFoundException)
{
Events.Write.SealedDataFileMissing(filename);
}
catch (PersistedDataException ex)
{
Events.Write.PersistedDataException(ex);
Events.Write.DiscardingIncompleteData(filename);
if (this.data != null)
{
this.data.Dispose();
}
this.data = null;
}
catch (OutOfMemoryException)
{
// TODO: this code is here to deal with file corruption issues, once checksums have been added and
// are fully available it must be removed.
File.Delete(filename);
throw;
}
finally
{
if (readData != null)
{
readData.Dispose();
}
}
// if data is still null, some error happened loading the file (empty, missing, corrupt).
// Just start over with clean data.
if (this.data == null)
{
if (File.Exists(this.Filename))
{
File.Delete(this.Filename);
}
this.data = new KeyedDataStore <TInternal>(this.DimensionSet, this.memoryStreamManager,
"placeholder for invalid data.");
}
Events.Write.EndLoadingData(filename);
}
