public void WriteData <TInternal>(string name, DateTime start, DateTime end, uint dataCount,
IEnumerable <PersistedDataSource> sources, KeyedDataStore <TInternal> data)
where TInternal : class, IInternalData, new()
{
var header = new PersistedDataHeader(name, start, end,
PersistedDataProtocol.GetPersistedTypeCodeFromType(typeof(TInternal)),
sources, this.dimensionSet, dataCount);
this.WriteDataWithLengthAndCRC32(ms =>
{
header.Write(new BufferWriter(ms));
}, header.SerializedSize, true);
// The raw data does not compress particularly well. There is some overlap in the keys, but particularly
// for histograms they come pre-compressed. We also use VLE heavily in KeyedDataStore.Serialize which
// makes for pretty compact data.
this.WriteDataWithLengthAndCRC32(data.Serialize, data.SerializedSize, false);
// We can now determine what our block length really is and back-fill our data.
var currentPosition = this.sourceStream.Position;
var blockLength = this.sourceStream.Position - this.blockStartOffset;
this.sourceStream.Position = this.blockLengthOffset;
this.sourceStreamWriter.WriteUInt64((ulong)blockLength);
this.sourceStream.Position = currentPosition;
}
private async Task <bool> QuerySources(IList <PersistedDataSource> sources)
{
Events.Write.BeginQuerySources(sources);
var success = false;
var transferRequest =
new TransferRequest
{
DataType = PersistedDataProtocol.GetPersistedTypeCodeFromType(typeof(TInternal)),
Timeout = (this.Timeout.Seconds * 9) / 10, // 90% of timeout goes to the child
MaxFanout = this.MaxFanout,
Sources = new List <string>()
};
foreach (var source in sources)
{
source.Status = PersistedDataSourceStatus.Unknown;
transferRequest.Sources.Add(source.Name);
}
var serverOffset = this.randomNumberGenerator.Next(sources.Count);
var selectedSource = sources[serverOffset];
var request = new HttpRequestMessage(HttpMethod.Post, this.CreateRequestUri(selectedSource.Name));
using (var ms = (this.memoryStreamManager.GetStream()))
using (var writeStream = new WriterStream(ms, this.memoryStreamManager))
{
var writer = writeStream.CreateCompactBinaryWriter();
writer.Write(transferRequest);
request.Content = new ByteArrayContent(ms.ToArray());
}
try
{
Events.Write.BeginSendSourceQuery(selectedSource, request.RequestUri);
var response =
await this.httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
Events.Write.EndSendSourceQuery(selectedSource, (int)response.StatusCode, response.ReasonPhrase);
if (!response.IsSuccessStatusCode)
{
switch (response.StatusCode)
{
case HttpStatusCode.NotFound:
foreach (var source in sources)
{
source.Status = PersistedDataSourceStatus.Unavailable;
}
return(true);
default:
return(false);
}
}
Events.Write.BeginReceiveSourceResponseBody(selectedSource);
var length = response.Content.Headers.ContentLength ?? 0;
using (var dataStream = this.memoryStreamManager.GetStream("PersistedDataAggregator/Http/Read",
(int)length, true))
using (var responseStream = await response.Content.ReadAsStreamAsync())
{
responseStream.CopyTo(dataStream);
dataStream.Position = 0;
using (var dataReader =
new PersistedDataReader(dataStream, this.memoryStreamManager, this.DimensionSet))
{
Events.Write.EndReceiveSourceResponseBody(selectedSource, (int)dataStream.Length);
success = true;
if (dataStream.Length == 0)
{
Events.Write.EmptyResponseReceivedFromSource(selectedSource);
// Over the line. Mark it zero. Or just available, but empty (which is okay!)
foreach (var source in sources)
{
source.Status = PersistedDataSourceStatus.Available;
}
}
else
{
try
{
this.UnpackResponse(dataReader);
}
catch (PersistedDataException ex)
{
Events.Write.PersistedDataExceptionFromSource(selectedSource, ex);
success = false;
}
}
}
}
}
catch (OperationCanceledException) { }
catch (Exception ex)
{
if (ex is HttpRequestException || ex is IOException || ex is WebException || ex is ObjectDisposedException)
{
Events.Write.HttpExceptionFromSource(selectedSource, ex);
}
else
{
throw;
}
}
Events.Write.EndQuerySources(success, sources);
return(success);
}