/// <summary>
/// Streams the channel data.
/// </summary>
/// <param name="contextList">The context list.</param>
/// <param name="token">The token.</param>
/// <returns></returns>
protected virtual async Task StreamChannelData(IList <ChannelStreamingContext> contextList, CancellationToken token)
{
_channelStreamingContextLists.Add(contextList);
// These values can be set outside of our processing loop as the won't chnage
//... as context is processed and completed.
var firstContext = contextList.First();
var channelStreamingType = firstContext.ChannelStreamingType;
var parentUri = firstContext.ParentUri;
var indexes = firstContext.ChannelMetadata.Indexes.Cast <IIndexMetadataRecord>().ToList();
var primaryIndex = indexes[0];
var isTimeIndex = indexes.Select(i => i.IndexKind == (int)ChannelIndexTypes.Time).ToArray();
var requestLatestValues =
channelStreamingType == ChannelStreamingTypes.IndexCount
? firstContext.IndexCount
: channelStreamingType == ChannelStreamingTypes.LatestValue
? 1
: (int?)null;
var increasing = primaryIndex.Direction == (int)IndexDirections.Increasing;
bool?firstStart = null;
// Loop until there is a cancellation or all channals have been removed
while (!IsStreamingStopped(contextList, ref token))
{
firstStart = !firstStart.HasValue;
var channelIds = contextList.Select(i => i.ChannelId).Distinct().ToArray();
Logger.Debug($"Streaming data for parentUri {parentUri.Uri} and channelIds {string.Join(",", channelIds)}");
// We only need a start index value for IndexValue and RangeRequest or if we're streaming
//... IndexCount or LatestValue and requestLatestValues is no longer set.
var minStart =
(channelStreamingType == ChannelStreamingTypes.IndexValue || channelStreamingType == ChannelStreamingTypes.RangeRequest) ||
((channelStreamingType == ChannelStreamingTypes.IndexCount || channelStreamingType == ChannelStreamingTypes.LatestValue) &&
!requestLatestValues.HasValue)
? contextList.Min(x => Convert.ToInt64(x.StartIndex))
: (long?)null;
// Only need and end index value for range request
var maxEnd = channelStreamingType == ChannelStreamingTypes.RangeRequest
? contextList.Max(x => Convert.ToInt64(x.EndIndex))
: (long?)null;
//var isTimeIndex = primaryIndex.IndexType == ChannelIndexTypes.Time;
var rangeSize = WitsmlSettings.GetRangeSize(isTimeIndex[0]);
// Convert indexes from scaled values
var minStartIndex = minStart?.IndexFromScale(primaryIndex.Scale, isTimeIndex[0]);
var maxEndIndex = channelStreamingType == ChannelStreamingTypes.IndexValue
? (increasing ? minStartIndex + rangeSize : minStartIndex - rangeSize)
: maxEnd?.IndexFromScale(primaryIndex.Scale, isTimeIndex[0]);
// Get channel data
var mnemonics = contextList.Select(c => c.ChannelMetadata.ChannelName).ToList();
var dataProvider = GetDataProvider(parentUri);
var optimiseStart = channelStreamingType == ChannelStreamingTypes.IndexValue;
var channelData = dataProvider.GetChannelData(parentUri, new Range <double?>(minStartIndex, maxEndIndex), mnemonics, requestLatestValues, optimiseStart);
// Stream the channel data
await StreamChannelData(contextList, channelData, mnemonics.ToArray(), increasing, isTimeIndex, primaryIndex.Scale, firstStart.Value, token);
// If we have processed an IndexCount or LatestValue query clear requestLatestValues so we can
//... keep streaming new data as long as the channel is active.
if (channelStreamingType == ChannelStreamingTypes.IndexCount ||
channelStreamingType == ChannelStreamingTypes.LatestValue)
{
requestLatestValues = null;
}
// Check each context to see of all the data has streamed.
var completedContexts = contextList
.Where(
c =>
(c.ChannelStreamingType != ChannelStreamingTypes.RangeRequest &&
c.ChannelMetadata.Status != (int)ChannelStatuses.Active && c.ChannelMetadata.EndIndex.HasValue &&
c.StartIndex >= c.ChannelMetadata.EndIndex.Value) ||
(c.ChannelStreamingType == ChannelStreamingTypes.RangeRequest &&
c.StartIndex >= c.EndIndex))
.ToArray();
// Remove any contexts from the list that have completed returning all data
completedContexts.ForEach(c =>
{
// Notify consumer if the ReceiveChangeNotification field is true
if (c.ChannelMetadata.Status != (int)ChannelStatuses.Active && c.ReceiveChangeNotification)
{
// TODO: Decide which message shoud be sent...
// ChannelStatusChange(c.ChannelId, c.ChannelMetadata.Status);
// ChannelRemove(c.ChannelId);
}
contextList.Remove(c);
});
// Delay to prevent CPU overhead
await Task.Delay(WitsmlSettings.StreamChannelDataDelayMilliseconds, token);
}
}
/// <summary>
/// Merges <see cref="ChannelDataChunk" /> data for updates.
/// </summary>
/// <param name="reader">The reader.</param>
/// <exception cref="WitsmlException"></exception>
public void Merge(ChannelDataReader reader)
{
if (reader == null || reader.RecordsAffected <= 0)
{
return;
}
Logger.Debug("Merging records in ChannelDataReader.");
try
{
// Get the full range of the reader.
//... This is the range that we need to select existing ChannelDataChunks from the database to update
var updateRange = reader.GetIndexRange();
// Make sure we have a valid index range; otherwise, nothing to do
if (!updateRange.Start.HasValue || !updateRange.End.HasValue)
{
return;
}
var indexChannel = reader.GetIndex();
var increasing = indexChannel.Increasing;
var rangeSize = WitsmlSettings.GetRangeSize(indexChannel.IsTimeIndex);
// Based on the range of the updates, compute the range of the data chunk(s)
//... so we can merge updates with existing data.
var existingRange = new Range <double?>(
Range.ComputeRange(updateRange.Start.Value, rangeSize, increasing).Start,
Range.ComputeRange(updateRange.End.Value, rangeSize, increasing).End
);
// Get DataChannelChunk list from database for the computed range and URI
var filter = BuildDataFilter(reader.Uri, indexChannel.Mnemonic, existingRange, increasing);
var results = GetData(filter, increasing);
// Backup existing chunks for the transaction
AttachChunks(results);
try
{
BulkWriteChunks(
ToChunks(
MergeSequence(results.GetRecords(), reader.AsEnumerable(), updateRange, rangeSize)),
reader.Uri,
string.Join(",", reader.Mnemonics),
string.Join(",", reader.Units),
string.Join(",", reader.NullValues)
);
CreateChannelDataChunkIndex();
}
catch (FormatException ex)
{
Logger.ErrorFormat("Error when merging data: {0}", ex);
throw new WitsmlException(ErrorCodes.ErrorMaxDocumentSizeExceeded, ex);
}
}
catch (MongoException ex)
{
Logger.ErrorFormat("Error when merging data: {0}", ex);
throw new WitsmlException(ErrorCodes.ErrorUpdatingInDataStore, ex);
}
}
/// <summary>
/// Combines <see cref="IEnumerable{IChannelDataRecord}"/> data into RangeSize chunks for storage into the database
/// </summary>
/// <param name="records">The <see cref="IEnumerable{ChannelDataChunk}"/> records to be chunked.</param>
/// <returns>An <see cref="IEnumerable{ChannelDataChunk}"/> of channel data.</returns>
private IEnumerable <ChannelDataChunk> ToChunks(IEnumerable <IChannelDataRecord> records)
{
Logger.Debug("Converting ChannelDataRecords to ChannelDataChunks.");
var data = new List <string>();
var id = string.Empty;
List <ChannelIndexInfo> indexes = null;
ChannelIndexInfo indexChannel = null;
Range <long>? plannedRange = null;
double startIndex = 0;
double endIndex = 0;
double?previousIndex = null;
string[] chunkMnemonics = null;
string[] chunkUnits = null;
string[] chunkNullValues = null;
var chunkSettingsSet = false;
foreach (var record in records)
{
indexChannel = record.GetIndex();
indexes = record.Indices.Select(x => x.Clone()).ToList();
var increasing = indexChannel.Increasing;
var index = record.GetIndexValue();
var rangeSize = WitsmlSettings.GetRangeSize(indexChannel.IsTimeIndex);
if (previousIndex.HasValue)
{
if (previousIndex.Value == index)
{
Logger.ErrorFormat("Data node index repeated for uri: {0}; channel {1}; index: {2}", record.Uri, indexChannel.Mnemonic, index);
throw new WitsmlException(ErrorCodes.NodesWithSameIndex);
}
if (increasing && previousIndex.Value > index || !increasing && previousIndex.Value < index)
{
var error = $"Data node index not in sequence for uri: {record.Uri}: channel: {indexChannel.Mnemonic}; index: {index}";
Logger.Error(error);
throw new InvalidOperationException(error);
}
}
previousIndex = index;
if (!plannedRange.HasValue)
{
plannedRange = Range.ComputeRange(index, rangeSize, increasing);
id = record.Id;
startIndex = index;
}
// TODO: Can we use this instead? plannedRange.Value.Contains(index, increasing) or a new method?
if (WithinRange(index, plannedRange.Value.End, increasing, false))
{
id = string.IsNullOrEmpty(id) ? record.Id : id;
data.Add(record.GetJson());
endIndex = index;
if (chunkSettingsSet)
{
continue;
}
chunkMnemonics = record.Mnemonics;
chunkUnits = record.Units;
chunkNullValues = record.NullValues;
chunkSettingsSet = true;
}
else
{
//var newIndex = indexChannel.Clone();
//newIndex.Start = startIndex;
//newIndex.End = endIndex;
indexes[0].Start = startIndex;
indexes[0].End = endIndex;
Logger.DebugFormat("ChannelDataChunk created with id '{0}', startIndex '{1}' and endIndex '{2}'.", id, startIndex, endIndex);
yield return(new ChannelDataChunk()
{
Uid = id,
Data = "[" + String.Join(",", data) + "]",
//Indices = new List<ChannelIndexInfo> { newIndex },
Indices = indexes,
RecordCount = data.Count,
MnemonicList = string.Join(",", chunkMnemonics),
UnitList = string.Join(",", chunkUnits),
NullValueList = string.Join(",", chunkNullValues)
});
plannedRange = Range.ComputeRange(index, rangeSize, increasing);
data = new List <string>()
{
record.GetJson()
};
startIndex = index;
endIndex = index;
id = record.Id;
chunkMnemonics = record.Mnemonics;
chunkUnits = record.Units;
chunkNullValues = record.NullValues;
}
}
if (data.Count > 0 && indexes != null)
{
//var newIndex = indexChannel.Clone();
//newIndex.Start = startIndex;
//newIndex.End = endIndex;
indexes = indexes.Select(x => x.Clone()).ToList();
indexes[0].Start = startIndex;
indexes[0].End = endIndex;
Logger.DebugFormat("ChannelDataChunk created with id '{0}', startIndex '{1}' and endIndex '{2}'.", id, startIndex, endIndex);
var chunk = new ChannelDataChunk()
{
Uid = id,
Data = "[" + string.Join(",", data) + "]",
//Indices = new List<ChannelIndexInfo> { newIndex },
Indices = indexes,
RecordCount = data.Count
};
if (chunkMnemonics != null)
{
chunk.MnemonicList = string.Join(",", chunkMnemonics);
chunk.UnitList = string.Join(",", chunkUnits);
chunk.NullValueList = string.Join(",", chunkNullValues);
}
yield return(chunk);
}
}
/// <summary>
/// Merges <see cref="ChannelDataChunk" /> data for updates.
/// </summary>
/// <param name="reader">The reader.</param>
/// <exception cref="WitsmlException"></exception>
public void Merge(ChannelDataReader reader)
{
if (reader == null || reader.RecordsAffected <= 0)
{
return;
}
Logger.Debug("Merging records in ChannelDataReader.");
try
{
// Get the full range of the reader.
//... This is the range that we need to select existing ChannelDataChunks from the database to update
var updateRange = reader.GetIndexRange();
// Make sure we have a valid index range; otherwise, nothing to do
if (!updateRange.Start.HasValue || !updateRange.End.HasValue)
{
return;
}
var indexChannel = reader.GetIndex();
var increasing = indexChannel.Increasing;
var rangeSize = WitsmlSettings.GetRangeSize(indexChannel.IsTimeIndex);
// Based on the range of the updates, compute the range of the data chunk(s)
//... so we can merge updates with existing data.
var chunkRange = new Range <double?>(
Range.ComputeRange(updateRange.Start.Value, rangeSize, increasing).Start,
Range.ComputeRange(updateRange.End.Value, rangeSize, increasing).End
);
// Get DataChannelChunk list from database for the computed range and URI
//specifically using a chunk limiter that will seek until the end of range is found regardless of the default read limit in the config file
var results = GetData(reader.Uri, indexChannel.Mnemonic, chunkRange, increasing, false, GetDataSearchUntilFoundOrEndChunkLimiter);
// Backup existing chunks for the transaction
AttachChunks(results);
// Check if reader overlaps existing data
var hasOverlap = false;
var existingRange = new Range <double?>();
var existingMnemonics = results.Count > 0 ? results[0]?.MnemonicList.Split(',') : new string[0];
if (results.Count > 0)
{
existingRange = new Range <double?>(results.Min(x => x.Indices[0].Start), results.Max(x => x.Indices[0].End));
hasOverlap = updateRange.Overlaps(existingRange, increasing);
}
try
{
if (hasOverlap)
{
WriteRecordsToChunks(reader, MergeSequence(results.GetRecords(), reader.AsEnumerable(), updateRange, rangeSize));
}
else
{
// If there is no existing data add reader records only
if (results.Count == 0)
{
WriteRecordsToChunks(reader, reader.AsEnumerable());
}
// If there is only one chunk and the mnemonics match
else if (existingMnemonics != null && existingMnemonics.OrderBy(t => t).SequenceEqual(reader.Mnemonics.OrderBy(t => t)) && results.Count == 1)
{
// If the update is before the existing range
if (updateRange.EndsBefore(existingRange.Start.GetValueOrDefault(), increasing, true))
{
WriteRecordsToChunks(reader, reader.AsEnumerable().Concat(results.GetRecords()));
}
// If the update is after the existing range
else if (updateRange.StartsAfter(existingRange.End.GetValueOrDefault(), increasing, true))
{
WriteRecordsToChunks(reader, results.GetRecords().Concat(reader.AsEnumerable()));
}
}
// Resort to merging the records
else
{
WriteRecordsToChunks(reader, MergeSequence(results.GetRecords(), reader.AsEnumerable(), updateRange, rangeSize));
}
}
CreateChannelDataChunkIndex();
}
catch (FormatException ex)
{
Logger.ErrorFormat("Error when merging data: {0}", ex);
throw new WitsmlException(ErrorCodes.ErrorMaxDocumentSizeExceeded, ex);
}
}
catch (MongoException ex)
{
Logger.ErrorFormat("Error when merging data: {0}", ex);
throw new WitsmlException(ErrorCodes.ErrorUpdatingInDataStore, ex);
}
}