private async Task CopyModelAsCsvToStreamAsync(NameValueCollection requestParameters, Stream responseStream, CancellationToken cancellationToken)
{
const int DefaultFrameRate = 30;
SecurityProviderCache.ValidateCurrentProvider();
string dateTimeFormat = Program.Host.Model.Global.DateTimeFormat;
// TODO: Improve operation for large point lists:
// Pick-up "POST"ed parameters with a "genurl" param, then cache parameters
// in a memory cache and return the unique URL (a string instead of a file)
// with a "download" param and unique ID associated with cached parameters.
// Then extract params based on unique ID and follow normal steps...
string pointIDsParam = requestParameters["PointIDs"];
string startTimeParam = requestParameters["StartTime"];
string endTimeParam = requestParameters["EndTime"];
string frameRateParam = requestParameters["FrameRate"];
string alignTimestampsParam = requestParameters["AlignTimestamps"];
string missingAsNaNParam = requestParameters["MissingAsNaN"];
string fillMissingTimestampsParam = requestParameters["FillMissingTimestamps"];
string instanceName = requestParameters["InstanceName"];
ulong[] pointIDs;
string headers;
if (string.IsNullOrEmpty(pointIDsParam))
{
throw new ArgumentNullException("PointIDs", "Cannot export data: no values were provided in \"PointIDs\" parameter.");
}
try
{
pointIDs = pointIDsParam.Split(',').Select(ulong.Parse).ToArray();
Array.Sort(pointIDs);
}
catch (Exception ex)
{
throw new ArgumentNullException("PointIDs", $"Cannot export data: failed to parse \"PointIDs\" parameter value \"{pointIDsParam}\": {ex.Message}");
}
if (string.IsNullOrEmpty(startTimeParam))
{
throw new ArgumentNullException("StartTime", "Cannot export data: no \"StartTime\" parameter value was specified.");
}
if (string.IsNullOrEmpty(pointIDsParam))
{
throw new ArgumentNullException("EndTime", "Cannot export data: no \"EndTime\" parameter value was specified.");
}
DateTime startTime, endTime;
try
{
startTime = DateTime.ParseExact(startTimeParam, dateTimeFormat, null, DateTimeStyles.AdjustToUniversal);
}
catch (Exception ex)
{
throw new ArgumentException($"Cannot export data: failed to parse \"StartTime\" parameter value \"{startTimeParam}\". Expected format is \"{dateTimeFormat}\". Error message: {ex.Message}", "StartTime", ex);
}
try
{
endTime = DateTime.ParseExact(endTimeParam, dateTimeFormat, null, DateTimeStyles.AdjustToUniversal);
}
catch (Exception ex)
{
throw new ArgumentException($"Cannot export data: failed to parse \"EndTime\" parameter value \"{endTimeParam}\". Expected format is \"{dateTimeFormat}\". Error message: {ex.Message}", "EndTime", ex);
}
if (startTime > endTime)
{
throw new ArgumentOutOfRangeException("StartTime", "Cannot export data: start time exceeds end time.");
}
using (DataContext dataContext = new DataContext())
{
// Validate current user has access to requested data
if (!dataContext.UserIsInRole(s_minimumRequiredRoles))
{
throw new SecurityException($"Cannot export data: access is denied for user \"{Thread.CurrentPrincipal.Identity?.Name ?? "Undefined"}\", minimum required roles = {s_minimumRequiredRoles.ToDelimitedString(", ")}.");
}
headers = GetHeaders(dataContext, pointIDs.Select(id => (int)id));
}
int frameRate;
if (!int.TryParse(frameRateParam, out frameRate))
{
frameRate = DefaultFrameRate;
}
bool alignTimestamps = alignTimestampsParam?.ParseBoolean() ?? true;
bool missingAsNaN = missingAsNaNParam?.ParseBoolean() ?? true;
bool fillMissingTimestamps = alignTimestamps && (fillMissingTimestampsParam?.ParseBoolean() ?? false);
if (string.IsNullOrEmpty(instanceName))
{
instanceName = TrendValueAPI.DefaultInstanceName;
}
LocalOutputAdapter adapter;
LocalOutputAdapter.Instances.TryGetValue(instanceName, out adapter);
HistorianServer serverInstance = adapter?.Server;
if ((object)serverInstance == null)
{
throw new InvalidOperationException($"Cannot export data: failed to access internal historian server instance \"{instanceName}\".");
}
const int TargetBufferSize = 524288;
StringBuilder readBuffer = new StringBuilder(TargetBufferSize * 2);
ManualResetEventSlim bufferReady = new ManualResetEventSlim(false);
List <string> writeBuffer = new List <string>();
object writeBufferLock = new object();
bool readComplete = false;
Task readTask = Task.Factory.StartNew(() =>
{
try
{
using (SnapClient connection = SnapClient.Connect(serverInstance.Host))
{
Dictionary <ulong, int> pointIDIndex = new Dictionary <ulong, int>(pointIDs.Length);
float[] values = new float[pointIDs.Length];
for (int i = 0; i < pointIDs.Length; i++)
{
pointIDIndex.Add(pointIDs[i], i);
}
for (int i = 0; i < values.Length; i++)
{
values[i] = float.NaN;
}
Ticks[] subseconds = Ticks.SubsecondDistribution(frameRate);
ulong interval = (ulong)(subseconds.Length > 1 ? subseconds[1].Value : Ticks.PerSecond);
ulong lastTimestamp = 0;
// Write data pages
SeekFilterBase <HistorianKey> timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, endTime);
MatchFilterBase <HistorianKey, HistorianValue> pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(pointIDs);
HistorianKey historianKey = new HistorianKey();
HistorianValue historianValue = new HistorianValue();
// Write row values function
Action bufferValues = () =>
{
readBuffer.Append(missingAsNaN ? string.Join(",", values) : string.Join(",", values.Select(val => float.IsNaN(val) ? "" : $"{val}")));
if (readBuffer.Length < TargetBufferSize)
{
return;
}
lock (writeBufferLock)
writeBuffer.Add(readBuffer.ToString());
readBuffer.Clear();
bufferReady.Set();
};
using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(instanceName))
{
// Start stream reader for the provided time window and selected points
TreeStream <HistorianKey, HistorianValue> stream = database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter);
ulong timestamp = 0;
while (stream.Read(historianKey, historianValue) && !cancellationToken.IsCancellationRequested)
{
if (alignTimestamps)
{
timestamp = (ulong)Ticks.RoundToSubsecondDistribution((long)historianKey.Timestamp, frameRate).Value;
}
else
{
timestamp = historianKey.Timestamp;
}
// Start a new row for each encountered new timestamp
if (timestamp != lastTimestamp)
{
if (lastTimestamp > 0)
{
bufferValues();
}
for (int i = 0; i < values.Length; i++)
{
values[i] = float.NaN;
}
if (fillMissingTimestamps && lastTimestamp > 0 && timestamp > lastTimestamp)
{
ulong difference = timestamp - lastTimestamp;
if (difference > interval)
{
ulong interpolated = lastTimestamp;
for (ulong i = 1; i < difference / interval; i++)
{
interpolated = (ulong)Ticks.RoundToSubsecondDistribution((long)(interpolated + interval), frameRate).Value;
readBuffer.Append($"{Environment.NewLine}{new DateTime((long)interpolated, DateTimeKind.Utc).ToString(dateTimeFormat)},");
bufferValues();
}
}
}
readBuffer.Append($"{Environment.NewLine}{new DateTime((long)timestamp, DateTimeKind.Utc).ToString(dateTimeFormat)},");
lastTimestamp = timestamp;
}
// Save value to its column
values[pointIDIndex[historianKey.PointID]] = historianValue.AsSingle;
}
if (timestamp > 0)
{
bufferValues();
}
if (readBuffer.Length > 0)
{
lock (writeBufferLock)
writeBuffer.Add(readBuffer.ToString());
}
}
}
}
finally
{
readComplete = true;
bufferReady.Set();
}
}, cancellationToken);
Task writeTask = Task.Factory.StartNew(() =>
{
using (StreamWriter writer = new StreamWriter(responseStream))
{
//Ticks exportStart = DateTime.UtcNow.Ticks;
string[] localBuffer;
// Write column headers
writer.Write(headers);
while ((writeBuffer.Count > 0 || !readComplete) && !cancellationToken.IsCancellationRequested)
{
bufferReady.Wait(cancellationToken);
bufferReady.Reset();
lock (writeBufferLock)
{
localBuffer = writeBuffer.ToArray();
writeBuffer.Clear();
}
foreach (string buffer in localBuffer)
{
writer.Write(buffer);
}
}
// Flush stream
writer.Flush();
//Debug.WriteLine("Export time: " + (DateTime.UtcNow.Ticks - exportStart).ToElapsedTimeString(3));
}
}, cancellationToken);
await readTask;
await writeTask;
}
private async Task CopyModelAsCsvToStreamAsync(SecurityPrincipal securityPrincipal, NameValueCollection requestParameters, Stream responseStream, CancellationToken cancellationToken)
{
const double DefaultFrameRate = 30;
const int DefaultTimestampSnap = 0;
string dateTimeFormat = Program.Host.Model.Global.DateTimeFormat;
string cacheIDParam = requestParameters["CacheID"];
string pointIDsParam = requestParameters["PointIDs"];
string startTimeParam = requestParameters["StartTime"];
string endTimeParam = requestParameters["EndTime"];
string timestampSnapParam = requestParameters["TSSnap"];
string frameRateParam = requestParameters["FrameRate"];
string alignTimestampsParam = requestParameters["AlignTimestamps"];
string missingAsNaNParam = requestParameters["MissingAsNaN"];
string fillMissingTimestampsParam = requestParameters["FillMissingTimestamps"];
string instanceName = requestParameters["InstanceName"];
string toleranceParam = requestParameters["TSTolerance"]; // In milliseconds
ulong[] pointIDs;
string headers;
if (string.IsNullOrEmpty(cacheIDParam) && string.IsNullOrEmpty(pointIDsParam))
{
throw new ArgumentException("Cannot export data: no point ID values can be accessed. Neither the \"CacheID\" nor the \"PointIDs\" parameter was provided.");
}
if (string.IsNullOrEmpty(pointIDsParam))
{
pointIDs = GetCachedPointIDs(cacheIDParam);
if (pointIDs is null)
{
throw new ArgumentNullException("CacheID", $"Cannot export data: failed to load cached point ID list referenced by \"CacheID\" parameter value \"{cacheIDParam}\".");
}
}
else
{
try
{
pointIDs = pointIDsParam.Split(',').Select(ulong.Parse).ToArray();
Array.Sort(pointIDs);
}
catch (Exception ex)
{
throw new ArgumentNullException("PointIDs", $"Cannot export data: failed to parse \"PointIDs\" parameter value \"{pointIDsParam}\": {ex.Message}");
}
}
if (string.IsNullOrEmpty(startTimeParam))
{
throw new ArgumentNullException("StartTime", "Cannot export data: no \"StartTime\" parameter value was specified.");
}
if (string.IsNullOrEmpty(endTimeParam))
{
throw new ArgumentNullException("EndTime", "Cannot export data: no \"EndTime\" parameter value was specified.");
}
DateTime startTime, endTime;
try
{
startTime = DateTime.ParseExact(startTimeParam, dateTimeFormat, null, DateTimeStyles.AdjustToUniversal);
}
catch (Exception ex)
{
throw new ArgumentException($"Cannot export data: failed to parse \"StartTime\" parameter value \"{startTimeParam}\". Expected format is \"{dateTimeFormat}\". Error message: {ex.Message}", "StartTime", ex);
}
try
{
endTime = DateTime.ParseExact(endTimeParam, dateTimeFormat, null, DateTimeStyles.AdjustToUniversal);
}
catch (Exception ex)
{
throw new ArgumentException($"Cannot export data: failed to parse \"EndTime\" parameter value \"{endTimeParam}\". Expected format is \"{dateTimeFormat}\". Error message: {ex.Message}", "EndTime", ex);
}
if (startTime > endTime)
{
throw new ArgumentOutOfRangeException("StartTime", "Cannot export data: start time exceeds end time.");
}
using (DataContext dataContext = new DataContext())
{
// Validate current user has access to requested data
if (!dataContext.UserIsInRole(securityPrincipal, s_minimumRequiredRoles))
{
throw new SecurityException($"Cannot export data: access is denied for user \"{Thread.CurrentPrincipal.Identity?.Name ?? "Undefined"}\", minimum required roles = {s_minimumRequiredRoles.ToDelimitedString(", ")}.");
}
headers = GetHeaders(dataContext, pointIDs.Select(id => (int)id));
}
if (!double.TryParse(frameRateParam, out double frameRate))
{
frameRate = DefaultFrameRate;
}
if (!int.TryParse(timestampSnapParam, out int timestampSnap))
{
timestampSnap = DefaultTimestampSnap;
}
if (!double.TryParse(toleranceParam, out double tolerance))
{
tolerance = 0.5;
}
int toleranceTicks = (int)Math.Ceiling(tolerance * Ticks.PerMillisecond);
bool alignTimestamps = alignTimestampsParam?.ParseBoolean() ?? true;
bool missingAsNaN = missingAsNaNParam?.ParseBoolean() ?? true;
bool fillMissingTimestamps = alignTimestamps && (fillMissingTimestampsParam?.ParseBoolean() ?? false);
if (string.IsNullOrEmpty(instanceName))
{
instanceName = TrendValueAPI.DefaultInstanceName;
}
LocalOutputAdapter.Instances.TryGetValue(instanceName, out LocalOutputAdapter adapter);
HistorianServer serverInstance = adapter?.Server;
if (serverInstance is null)
{
throw new InvalidOperationException($"Cannot export data: failed to access internal historian server instance \"{instanceName}\".");
}
const int TargetBufferSize = 524288;
StringBuilder readBuffer = new StringBuilder(TargetBufferSize * 2);
ManualResetEventSlim bufferReady = new ManualResetEventSlim(false);
List <string> writeBuffer = new List <string>();
object writeBufferLock = new object();
bool readComplete = false;
Task readTask = Task.Factory.StartNew(() =>
{
try
{
using SnapClient connection = SnapClient.Connect(serverInstance.Host);
Dictionary <ulong, int> pointIDIndex = new Dictionary <ulong, int>(pointIDs.Length);
float[] values = new float[pointIDs.Length];
for (int i = 0; i < pointIDs.Length; i++)
{
pointIDIndex.Add(pointIDs[i], i);
}
for (int i = 0; i < values.Length; i++)
{
values[i] = float.NaN;
}
ulong interval;
if (Math.Abs(frameRate % 1) <= double.Epsilon * 100)
{
Ticks[] subseconds = Ticks.SubsecondDistribution((int)frameRate);
interval = (ulong)(subseconds.Length > 1 ? subseconds[1].Value : Ticks.PerSecond);
}
else
{
interval = (ulong)(Math.Floor(1.0d / frameRate) * Ticks.PerSecond);
}
ulong lastTimestamp = 0;
// Write data pages
SeekFilterBase <HistorianKey> timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, endTime);
MatchFilterBase <HistorianKey, HistorianValue> pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(pointIDs);
HistorianKey historianKey = new HistorianKey();
HistorianValue historianValue = new HistorianValue();
// Write row values function
void bufferValues()
{
readBuffer.Append(missingAsNaN ? string.Join(",", values) : string.Join(",", values.Select(val => float.IsNaN(val) ? "" : $"{val}")));
if (readBuffer.Length < TargetBufferSize)
{
return;
}
lock (writeBufferLock)
writeBuffer.Add(readBuffer.ToString());
readBuffer.Clear();
bufferReady.Set();
}
// Start stream reader for the provided time window and selected points
using ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(instanceName);
TreeStream <HistorianKey, HistorianValue> stream = database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter);
ulong timestamp = 0;
// Adjust timestamp to use first timestamp as base
bool adjustTimeStamp = timestampSnap switch
{
0 => false,
1 => true,
2 => false,
_ => true
};
long baseTime = timestampSnap switch
{
0 => Ticks.RoundToSecondDistribution(startTime.Ticks, frameRate, startTime.Ticks - startTime.Ticks % Ticks.PerSecond),
_ => startTime.Ticks
};
while (stream.Read(historianKey, historianValue) && !cancellationToken.IsCancellationRequested)
{
if (alignTimestamps)
{
if (adjustTimeStamp)
{
adjustTimeStamp = false;
baseTime = (long)historianKey.Timestamp;
}
// Make sure the timestamp is actually close enough to the distribution
Ticks ticks = Ticks.ToSecondDistribution((long)historianKey.Timestamp, frameRate, baseTime, toleranceTicks);
if (ticks == Ticks.MinValue)
{
continue;
}
timestamp = (ulong)ticks.Value;
}
else
{
timestamp = historianKey.Timestamp;
}
// Start a new row for each encountered new timestamp
if (timestamp != lastTimestamp)
{
if (lastTimestamp > 0)
{
bufferValues();
}
for (int i = 0; i < values.Length; i++)
{
values[i] = float.NaN;
}
if (fillMissingTimestamps && lastTimestamp > 0 && timestamp > lastTimestamp)
{
ulong difference = timestamp - lastTimestamp;
if (difference > interval)
{
ulong interpolated = lastTimestamp;
for (ulong i = 1; i < difference / interval; i++)
{
interpolated = (ulong)Ticks.RoundToSecondDistribution((long)(interpolated + interval), frameRate, startTime.Ticks).Value;
readBuffer.Append($"{Environment.NewLine}{new DateTime((long)interpolated, DateTimeKind.Utc).ToString(dateTimeFormat)},");
bufferValues();
}
}
}
readBuffer.Append($"{Environment.NewLine}{new DateTime((long)timestamp, DateTimeKind.Utc).ToString(dateTimeFormat)},");
lastTimestamp = timestamp;
}
// Save value to its column
values[pointIDIndex[historianKey.PointID]] = historianValue.AsSingle;
}
if (timestamp > 0)
{
bufferValues();
}
if (readBuffer.Length > 0)
{
lock (writeBufferLock)
writeBuffer.Add(readBuffer.ToString());
}
}
finally
{
readComplete = true;
bufferReady.Set();
}
},
cancellationToken);
Task writeTask = Task.Factory.StartNew(() =>
{
using StreamWriter writer = new StreamWriter(responseStream);
// Write column headers
writer.Write(headers);
while ((writeBuffer.Count > 0 || !readComplete) && !cancellationToken.IsCancellationRequested)
{
string[] localBuffer;
bufferReady.Wait(cancellationToken);
bufferReady.Reset();
lock (writeBufferLock)
{
localBuffer = writeBuffer.ToArray();
writeBuffer.Clear();
}
foreach (string buffer in localBuffer)
{
writer.Write(buffer);
}
}
// Flush stream
writer.Flush();
//Debug.WriteLine("Export time: " + (DateTime.UtcNow.Ticks - exportStart).ToElapsedTimeString(3));
},
cancellationToken);
await readTask;
await writeTask;
}
