private BundledRows GetResult(string result)
{
var resultBundle = new BundledRows();
var resultRow = new Row();
resultRow.Duals.Add(new Dual {
StrData = result, NumData = 0
});
resultBundle.Rows.Add(resultRow);
return(resultBundle);
}
private static BundledRows GetResult(object result)
{
var resultBundle = new BundledRows();
var resultRow = new Row();
var settings = new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore
};
resultRow.Duals.Add(new Dual {
StrData = JsonConvert.SerializeObject(result, settings), NumData = 0
});
resultBundle.Rows.Add(resultRow);
return(resultBundle);
}
private async Task GenerateResult(Sexp RResult, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, ServerCallContext context,
bool failIfWrongDataTypeInFirstCol = false, DataType expectedFirstDataType = DataType.Numeric, bool cacheResultInQlik = true)
{
int nrOfCols = 0;
int nrOfRows = 0;
ResultDataColumn[] resultDataColumns = null;
var names = RResult.Names;
if (names != null)
{
logger.Debug($"Rserve result column names: {String.Join(", ", names)}");
}
if (RResult is SexpList)
{
// Indicating this is a data.frame/matrix response structure. Figure out how many columns, names and data types
nrOfCols = RResult.Count;
logger.Debug($"Rserve result nrOfColumns: {nrOfCols}");
if (RResult.Attributes != null && RResult.Attributes.Count > 0)
{
Sexp resObjectNames;
if ((names == null || names.Length == 0) && RResult.Attributes.TryGetValue("names", out resObjectNames))
{
names = resObjectNames.AsStrings;
logger.Debug($"Rserve result column names: {String.Join(", ", names)}");
}
Sexp resObjectClass;
if (RResult.Attributes.TryGetValue("class", out resObjectClass))
{
logger.Debug($"Rserve result object class: {resObjectClass.ToString()}");
}
}
if (nrOfCols > 0)
{
var columns = RResult.AsList;
resultDataColumns = GetResultDataColumns(ref nrOfRows, names, columns);
}
}
else if (RResult is SexpArrayBool || RResult is SexpArrayDouble || RResult is SexpArrayInt)
{
nrOfCols = 1;
var bundledRows = new BundledRows();
var numerics = RResult.AsDoubles;
nrOfRows = numerics.Length;
var c = new ResultDataColumn();
c.Name = "";
c.DataType = DataType.Numeric;
c.Numerics = numerics;
resultDataColumns = new ResultDataColumn[1];
resultDataColumns[0] = c;
if (logger.IsTraceEnabled)
{
var logNumerics = String.Join(", ", numerics);
logger.Trace("Numeric result column data[0]: {0}", logNumerics);
}
}
else if (RResult is SexpArrayString)
{
nrOfCols = 1;
var bundledRows = new BundledRows();
var strings = RResult.AsStrings;
nrOfRows = strings.Length;
var c = new ResultDataColumn();
c.Name = "";
c.DataType = DataType.String;
c.Strings = strings;
resultDataColumns = new ResultDataColumn[1];
resultDataColumns[0] = c;
if (logger.IsTraceEnabled)
{
var logStrings = String.Join(", ", strings);
logger.Trace("String result column data[0]: {0}", logStrings);
}
}
else
{
logger.Warn($"Rserve result, column data type not recognized: {RResult.GetType().ToString()}");
throw new NotImplementedException();
}
if (resultDataColumns != null)
{
if (failIfWrongDataTypeInFirstCol && expectedFirstDataType != resultDataColumns[0].DataType)
{
string msg = $"Rserve result datatype mismatch in first column, expected {expectedFirstDataType}, got {resultDataColumns[0].DataType}";
logger.Warn($"{msg}");
throw new RpcException(new Status(StatusCode.InvalidArgument, $"{msg}"));
}
//Send TableDescription header
TableDescription tableDesc = new TableDescription
{
NumberOfRows = nrOfRows
};
for (int col = 0; col < nrOfCols; col++)
{
if (String.IsNullOrEmpty(resultDataColumns[col].Name))
{
tableDesc.Fields.Add(new FieldDescription
{
DataType = resultDataColumns[col].DataType
});
}
else
{
tableDesc.Fields.Add(new FieldDescription
{
DataType = resultDataColumns[col].DataType,
Name = resultDataColumns[col].Name
});
}
}
var tableMetadata = new Metadata
{
{ new Metadata.Entry("qlik-tabledescription-bin", MessageExtensions.ToByteArray(tableDesc)) }
};
if (!cacheResultInQlik)
{
tableMetadata.Add("qlik-cache", "no-store");
}
await context.WriteResponseHeadersAsync(tableMetadata);
// Send data
var bundledRows = new BundledRows();
for (int i = 0; i < nrOfRows; i++)
{
var row = new Row();
for (int col = 0; col < nrOfCols; col++)
{
if (resultDataColumns[col].DataType == DataType.Numeric)
{
row.Duals.Add(new Dual()
{
NumData = resultDataColumns[col].Numerics[i]
});
}
else if (resultDataColumns[col].DataType == DataType.String)
{
row.Duals.Add(new Dual()
{
StrData = resultDataColumns[col].Strings[i] ?? ""
});
}
}
bundledRows.Rows.Add(row);
if (((i + 1) % 2000) == 0)
{
// Send a bundle
await responseStream.WriteAsync(bundledRows);
bundledRows = new BundledRows();
}
}
if (bundledRows.Rows.Count() > 0)
{
// Send last bundle
await responseStream.WriteAsync(bundledRows);
}
}
}
public override async Task ExecuteFunction(IAsyncStreamReader <BundledRows> requestStream, IServerStreamWriter <BundledRows> responseStream, ServerCallContext context)
{
Logger.Trace("-- ExecuteFunction --");
if (logType == "file")
{
logConnector = new FileAuditLogConnector();
}
Dictionary <String, String> headerInfo = TraceServerCallContext(context);
var functionRequestHeaderStream = context.RequestHeaders.SingleOrDefault(header => header.Key == "qlik-functionrequestheader-bin");
if (functionRequestHeaderStream == null)
{
throw new Exception("ExecuteFunction called without Function Request Header in Request Headers.");
}
var functionRequestHeader = new FunctionRequestHeader();
functionRequestHeader.MergeFrom(new CodedInputStream(functionRequestHeaderStream.ValueBytes));
//Logger.Trace($"FunctionRequestHeader.FunctionId String : {(FunctionConstant)functionRequestHeader.FunctionId}");
Guid g = LogRequest(logConnector, headerInfo["AppId"], headerInfo["UserId"]);
switch (functionRequestHeader.FunctionId)
{
case (int)FunctionConstant.LogAsSeenStrCheck:
{
while (await requestStream.MoveNext())
{
var resultBundle = new BundledRows();
var cacheMetadata = new Metadata
{
{ new Metadata.Entry("qlik-cache", "no-store") }
};
await context.WriteResponseHeadersAsync(cacheMetadata);
foreach (var row in requestStream.Current.Rows)
{
Row resultRow = LogRow(logConnector, g, headerInfo["AppId"], headerInfo["UserId"], row.Duals[0].StrData, "√");
resultBundle.Rows.Add(resultRow);
}
await responseStream.WriteAsync(resultBundle);
}
break;
}
case (int)FunctionConstant.LogAsSeenStrEcho:
{
while (await requestStream.MoveNext())
{
var resultBundle = new BundledRows();
var cacheMetadata = new Metadata
{
{ new Metadata.Entry("qlik-cache", "no-store") }
};
await context.WriteResponseHeadersAsync(cacheMetadata);
foreach (var row in requestStream.Current.Rows)
{
Row resultRow = LogRow(logConnector, g, headerInfo["AppId"], headerInfo["UserId"], row.Duals[0].StrData, row.Duals[0].StrData);
resultBundle.Rows.Add(resultRow);
}
await responseStream.WriteAsync(resultBundle);
}
break;
}
case (int)FunctionConstant.LogAsSeenCheck:
{
while (await requestStream.MoveNext())
{
var resultBundle = new BundledRows();
var cacheMetadata = new Metadata
{
{ new Metadata.Entry("qlik-cache", "no-store") }
};
await context.WriteResponseHeadersAsync(cacheMetadata);
foreach (var row in requestStream.Current.Rows)
{
Row resultRow = LogRow(logConnector, g, headerInfo["AppId"], headerInfo["UserId"], row.Duals[0].NumData, "√");
resultBundle.Rows.Add(resultRow);
}
await responseStream.WriteAsync(resultBundle);
}
break;
}
case (int)FunctionConstant.LogAsSeenEcho:
{
while (await requestStream.MoveNext())
{
var resultBundle = new BundledRows();
var cacheMetadata = new Metadata
{
{ new Metadata.Entry("qlik-cache", "no-store") }
};
await context.WriteResponseHeadersAsync(cacheMetadata);
foreach (var row in requestStream.Current.Rows)
{
Row resultRow = LogRow(logConnector, g, headerInfo["AppId"], headerInfo["UserId"], row.Duals[0].NumData, row.Duals[0].NumData);
resultBundle.Rows.Add(resultRow);
}
await responseStream.WriteAsync(resultBundle);
}
break;
}
default:
break;
}
logConnector.EndLogData();
Logger.Trace("-- (ExecuteFunction) --");
}
/// <summary>
/// Return the results from connector to Qlik Engine
/// </summary>
private async Task GenerateResult(string request_type, MemoryStream returnedData, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, ServerCallContext context, int reqHash,
bool failIfWrongDataTypeInFirstCol = false, DataType expectedFirstDataType = DataType.Numeric, bool cacheResultInQlik = true, ResultDataColumn keyField = null, string keyname = null, bool includeDetail = false, bool rawExplain = false)
{
int nrOfCols = 0;
int nrOfRows = 0;
List <ResultDataColumn> resultDataColumns = new List <ResultDataColumn>();
Logger.Info($"{reqHash} - Generate Results");
if (true)
{
Logger.Debug($"{reqHash} - Extract JSON");
//Convert the stream (json) to dictionary
Logger.Info($"{reqHash} - Returned Datasize: {returnedData.Length}");
Logger.Info($"{reqHash} - Request Type: {request_type}");
StreamReader sr = new StreamReader(returnedData);
returnedData.Position = 0;
var data = sr.ReadToEnd();
Dictionary <string, dynamic> response = JsonConvert.DeserializeObject <Dictionary <string, dynamic> >(data);
Logger.Trace($"{reqHash} - Returned Data: {data}");
if (response.ContainsKey("data"))
{
//Prediction Column
var a = new ResultDataColumn();
a.Name = "Prediction";
a.DataType = DataType.String;
a.Strings = new List <string>();
//Time Series Columns
var seriesId = new ResultDataColumn();
var forecastPoint = new ResultDataColumn();
var rowId = new ResultDataColumn();
var timestamp = new ResultDataColumn();
var forecastDistance = new ResultDataColumn();
if (request_type == "timeseries")
{
seriesId.Name = "seriesId";
seriesId.DataType = DataType.String;
seriesId.Strings = new List <string>();
forecastPoint.Name = "forecastPoint";
forecastPoint.DataType = DataType.String;
forecastPoint.Strings = new List <string>();
rowId.Name = "rowId";
rowId.DataType = DataType.String;
rowId.Strings = new List <string>();
timestamp.Name = "timestamp";
timestamp.DataType = DataType.String;
timestamp.Strings = new List <string>();
forecastDistance.Name = "forecastDistance";
forecastDistance.DataType = DataType.String;
forecastDistance.Strings = new List <string>();
}
var pe = new ResultDataColumn();
if (rawExplain == true)
{
pe.Name = $"Prediction Explanations";
pe.DataType = DataType.String;
pe.Strings = new List <string>();
}
//The first row will determine which fields to return in table for prediction values
bool fieldListAgreed = false;
//Loop through each response in array (one for each row of input data)
foreach (dynamic p in response["data"])
{
// **** For debug to display array data on console ****
Logger.Info($"{reqHash} {p}");
a.Strings.Add(Convert.ToString(p["prediction"]));
// Add Time Series Column Data
if (request_type == "timeseries")
{
seriesId.Strings.Add(Convert.ToString(p["seriesId"]));
forecastPoint.Strings.Add(Convert.ToString(p["forecastPoint"]));
rowId.Strings.Add(Convert.ToString(p["rowId"]));
timestamp.Strings.Add(Convert.ToString(p["timestamp"]));
forecastDistance.Strings.Add(Convert.ToString(p["forecastDistance"]));
}
if (includeDetail == true)
{
if (fieldListAgreed == false)
{
foreach (dynamic pv in p["predictionValues"])
{
var pvi = new ResultDataColumn();
pvi.Name = $"Prediction value for label: {pv["label"]}";
pvi.DataType = DataType.String;
pvi.Strings = new List <string>();
resultDataColumns.Add(pvi);
}
fieldListAgreed = true;
Logger.Trace($"{reqHash} - Columns: {resultDataColumns.Count}");
}
//Loop through each predicted value and insert the row values to the column
int index = 0;
foreach (dynamic pv in p["predictionValues"])
{
resultDataColumns[index].Strings.Add(Convert.ToString(pv["value"]));
index++;
}
}
if (rawExplain == true)
{
pe.Strings.Add(Convert.ToString(p["predictionExplanations"]));
}
}
if (keyname != null)
{
resultDataColumns.Add(keyField);
}
if (rawExplain == true)
{
resultDataColumns.Add(pe);
}
// Add Time Series Columns to resultData
if (request_type == "timeseries")
{
resultDataColumns.Add(seriesId);
resultDataColumns.Add(forecastPoint);
resultDataColumns.Add(rowId);
resultDataColumns.Add(timestamp);
resultDataColumns.Add(forecastDistance);
}
resultDataColumns.Add(a);
}
else if (response.ContainsKey("response"))
{
var a = new ResultDataColumn();
a.Name = "Result";
a.DataType = DataType.String;
a.Strings = new List <string>();
a.Strings.Add(Convert.ToString(response["response"]["id"]));
resultDataColumns.Add(a);
}
else
{
if (response.ContainsKey("message"))
{
throw new Exception($"The following error message was returned from DataRobot: {response["message"]}");
}
else
{
throw new Exception($"An Unknown Error Occured: {data}");
}
}
nrOfRows = resultDataColumns[0].DataType == DataType.String ? resultDataColumns[0].Strings.Count : resultDataColumns[0].Numerics.Count;
nrOfCols = resultDataColumns.Count;
Logger.Debug($"{reqHash} - Result Number of Columns: {nrOfCols}");
}
if (resultDataColumns != null)
{
if (failIfWrongDataTypeInFirstCol && expectedFirstDataType != resultDataColumns[0].DataType)
{
string msg = $"Result datatype mismatch in first column, expected {expectedFirstDataType}, got {resultDataColumns[0].DataType}";
Logger.Warn($"{reqHash} - {msg}");
throw new RpcException(new Status(StatusCode.InvalidArgument, $"{msg}"));
}
//Send TableDescription header
TableDescription tableDesc = new TableDescription
{
NumberOfRows = nrOfRows
};
for (int col = 0; col < nrOfCols; col++)
{
if (String.IsNullOrEmpty(resultDataColumns[col].Name))
{
tableDesc.Fields.Add(new FieldDescription
{
DataType = resultDataColumns[col].DataType
});
}
else
{
tableDesc.Fields.Add(new FieldDescription
{
DataType = resultDataColumns[col].DataType,
Name = resultDataColumns[col].Name
});
}
}
var tableMetadata = new Metadata
{
{ new Metadata.Entry("qlik-tabledescription-bin", MessageExtensions.ToByteArray(tableDesc)) }
};
if (!cacheResultInQlik)
{
tableMetadata.Add("qlik-cache", "no-store");
}
await context.WriteResponseHeadersAsync(tableMetadata);
// Send data
var bundledRows = new BundledRows();
for (int i = 0; i < nrOfRows; i++)
{
var row = new Row();
for (int col = 0; col < nrOfCols; col++)
{
if (resultDataColumns[col].DataType == DataType.Numeric)
{
row.Duals.Add(new Dual()
{
NumData = resultDataColumns[col].Numerics[i]
});
}
else if (resultDataColumns[col].DataType == DataType.String)
{
row.Duals.Add(new Dual()
{
StrData = resultDataColumns[col].Strings[i] ?? ""
});
}
}
bundledRows.Rows.Add(row);
if (((i + 1) % 2000) == 0)
{
// Send a bundle
await responseStream.WriteAsync(bundledRows);
bundledRows = null;
bundledRows = new BundledRows();
}
}
if (bundledRows.Rows.Count() > 0)
{
// Send last bundle
await responseStream.WriteAsync(bundledRows);
bundledRows = null;
}
}
}
public override async Task ExecuteFunction(IAsyncStreamReader <BundledRows> requestStream, IServerStreamWriter <BundledRows> responseStream, ServerCallContext context)
{
if (Logger.IsTraceEnabled)
{
Logger.Trace("-- ExecuteFunction --");
TraceServerCallContext(context);
}
else
{
Logger.Debug("ExecuteFunction called");
}
var functionRequestHeaderStream = context.RequestHeaders.SingleOrDefault(header => header.Key == "qlik-functionrequestheader-bin");
if (functionRequestHeaderStream == null)
{
throw new Exception("ExecuteFunction called without Function Request Header in Request Headers.");
}
var functionRequestHeader = new FunctionRequestHeader();
functionRequestHeader.MergeFrom(new CodedInputStream(functionRequestHeaderStream.ValueBytes));
Logger.Trace($"FunctionRequestHeader.FunctionId String : {(FunctionConstant)functionRequestHeader.FunctionId}");
switch (functionRequestHeader.FunctionId)
{
case (int)FunctionConstant.Add42:
{
while (await requestStream.MoveNext())
{
var resultBundle = new BundledRows();
foreach (var row in requestStream.Current.Rows)
{
var resultRow = new Row();
resultRow.Duals.Add(new Dual {
NumData = row.Duals[0].NumData + 42.0
});
resultBundle.Rows.Add(resultRow);
}
await responseStream.WriteAsync(resultBundle);
}
break;
}
case (int)FunctionConstant.SumOfAllNumbers:
{
double sum = 0.0;
while (await requestStream.MoveNext())
{
foreach (var row in requestStream.Current.Rows)
{
sum = sum + row.Duals.Select(d => d.NumData).Sum();
}
}
var resultBundle = new BundledRows();
var resultRow = new Row();
resultRow.Duals.Add(new Dual {
NumData = sum
});
resultBundle.Rows.Add(resultRow);
await responseStream.WriteAsync(resultBundle);
break;
}
case (int)FunctionConstant.Concatenate:
{
var requestAsList = await requestStream.ToListAsync();
var concatenatedStrings = String.Join(", ",
requestAsList.SelectMany(bundledRows => bundledRows.Rows).SelectMany(row => row.Duals)
.Select(dual => dual.StrData));
var resultBundle = new BundledRows();
var resultRow = new Row();
resultRow.Duals.Add(new Dual {
StrData = concatenatedStrings
});
resultBundle.Rows.Add(resultRow);
await responseStream.WriteAsync(resultBundle);
break;
}
case (int)FunctionConstant.CallCounter:
{
var currentIncrement = Interlocked.Increment(ref _callCounter);
while (await requestStream.MoveNext())
{
var resultBundle = new BundledRows();
foreach (var row in requestStream.Current.Rows)
{
var resultRow = new Row();
resultRow.Duals.Add(new Dual {
NumData = currentIncrement
});
resultBundle.Rows.Add(resultRow);
}
await responseStream.WriteAsync(resultBundle);
}
break;
}
case (int)FunctionConstant.CallCounterNoCache:
{
context.ResponseTrailers.Add("qlik-cache", "no-store");
var currentIncrement = Interlocked.Increment(ref _callCounter);
while (await requestStream.MoveNext())
{
var resultBundle = new BundledRows();
foreach (var row in requestStream.Current.Rows)
{
var resultRow = new Row();
resultRow.Duals.Add(new Dual {
NumData = currentIncrement
});
resultBundle.Rows.Add(resultRow);
}
await responseStream.WriteAsync(resultBundle);
}
break;
}
default:
break;
}
Logger.Trace("-- (ExecuteFunction) --");
}
private async Task GenerateResult(Sexp RResult, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, RserveConnection rserveConn)
{
if (RResult is SexpArrayBool || RResult is SexpArrayDouble || RResult is SexpArrayInt)
{
var bundledRows = new BundledRows();
var numerics = RResult.AsDoubles;
if (numerics.Length == 0)
{
HandleZeroRowsFromRserve(rserveConn);
}
for (int i = 0; i < numerics.Length; i++)
{
var row = new Row();
row.Duals.Add(new Dual()
{
NumData = numerics[i]
});
bundledRows.Rows.Add(row);
if ((i % 2000) == 0)
{
// Send a bundle
await responseStream.WriteAsync(bundledRows);
bundledRows = new BundledRows();
}
}
if (bundledRows.Rows.Count() > 0)
{
// Send last bundle
await responseStream.WriteAsync(bundledRows);
}
if (logger.IsTraceEnabled)
{
var logNumerics = String.Join(", ", numerics);
logger.Trace("Numeric result column data: {0}", logNumerics);
}
}
else if (RResult is SexpArrayString)
{
var bundledRows = new BundledRows();
var strings = RResult.AsStrings;
if (strings.Length == 0)
{
HandleZeroRowsFromRserve(rserveConn);
}
for (int i = 0; i < strings.Length; i++)
{
var row = new Row();
row.Duals.Add(new Dual()
{
StrData = strings[i] ?? ""
});
bundledRows.Rows.Add(row);
if ((i % 2000) == 0)
{
// Send a bundle
await responseStream.WriteAsync(bundledRows);
bundledRows = new BundledRows();
}
}
if (bundledRows.Rows.Count() > 0)
{
// Send last bundle
await responseStream.WriteAsync(bundledRows);
}
if (logger.IsTraceEnabled)
{
var logStrings = String.Join(", ", strings);
logger.Trace("String result column data: {0}", logStrings);
}
}
else
{
throw new NotImplementedException();
}
}
/// <summary>
/// Return the results from connector to Qlik Engine
/// </summary>
private async Task GenerateResult(string request_type, MemoryStream returnedData, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, ServerCallContext context, int reqHash,
bool failIfWrongDataTypeInFirstCol = false, DataType expectedFirstDataType = DataType.Numeric, bool cacheResultInQlik = true, ResultDataColumn keyField = null, string keyname = null, bool includeDetail = false, bool shouldExplain = false, bool rawExplain = false, int explain_max = 0)
{
int nrOfCols = 0;
int nrOfRows = 0;
List <ResultDataColumn> resultDataColumns = new List <ResultDataColumn>();
Logger.Info($"{reqHash} - Generate Results");
if (true)
{
Logger.Debug($"{reqHash} - Extract JSON");
//Convert the stream (json) to dictionary
Logger.Info($"{reqHash} - Returned Datasize: {returnedData.Length}");
Logger.Info($"{reqHash} - Request Type: {request_type}");
StreamReader sr = new StreamReader(returnedData);
returnedData.Position = 0;
var data = sr.ReadToEnd();
//Dictionary<string, dynamic> response = JsonConvert.DeserializeObject<Dictionary<string, dynamic>>(data);
ResponseSpecification response = JsonConvert.DeserializeObject <ResponseSpecification>(data);
Logger.Trace($"{reqHash} - Returned Data: {data}");
if (response.data != null)
{
Logger.Trace($"{reqHash} - Response Data: {response.data}");
//Sort the response by RowId
List <DataSpecification> sortedData = response.data.OrderBy(o => o.rowId).ToList();
//Return Raw Explain First So Works In Chart Expression
if (request_type != "timeseries")
{
if (shouldExplain && rawExplain)
{
var pe = new ResultDataColumn();
pe.Name = $"Prediction Explanations";
pe.DataType = DataType.String;
resultDataColumns.Add(pe);
}
}
//Prediction Column
var a = new ResultDataColumn();
a.Name = "Prediction";
a.DataType = DataType.String;
//a.Strings = new List<string>();
resultDataColumns.Add(a);
//Include Keyfield
if (keyname != null)
{
resultDataColumns.Add(keyField);
}
//The first row will determine which fields to return in table for prediction values
if (includeDetail == true)
{
foreach (PredictionValueSpecification pv in sortedData[0].predictionValues)
{
var pvi = new ResultDataColumn();
pvi.Name = $"Prediction value for label: {pv.label}";
pvi.DataType = DataType.String;
resultDataColumns.Add(pvi);
}
}
// Add Time Series Columns to resultData
if (request_type == "timeseries")
{
//Row Id
var rowId = new ResultDataColumn();
rowId.Name = "rowId";
rowId.DataType = DataType.String;
//Time Series Columns
var seriesId = new ResultDataColumn();
seriesId.Name = "seriesId";
seriesId.DataType = DataType.String;
var forecastPoint = new ResultDataColumn();
forecastPoint.Name = "forecastPoint";
forecastPoint.DataType = DataType.String;
var timestamp = new ResultDataColumn();
timestamp.Name = "timestamp";
timestamp.DataType = DataType.String;
var forecastDistance = new ResultDataColumn();
forecastDistance.Name = "forecastDistance";
forecastDistance.DataType = DataType.String;
var originalFormatTimestamp = new ResultDataColumn();
originalFormatTimestamp.Name = "originalFormatTimestamp";
originalFormatTimestamp.DataType = DataType.String;
resultDataColumns.Add(rowId);
resultDataColumns.Add(seriesId);
resultDataColumns.Add(forecastPoint);
resultDataColumns.Add(timestamp);
resultDataColumns.Add(forecastDistance);
resultDataColumns.Add(originalFormatTimestamp);
}
if (request_type != "timeseries")
{
if (shouldExplain && !rawExplain)
{
for (int j = 0; j < explain_max; j++)
{
foreach (string field in new[] { "label", "feature", "featureValue", "strength", "qualitativeStrength" })
{
var pe = new ResultDataColumn();
pe.Name = $"PE_{j + 1}_{field}";
pe.DataType = DataType.String;
resultDataColumns.Add(pe);
}
}
}
}
nrOfRows = sortedData.Count;
nrOfCols = resultDataColumns.Count;
Logger.Debug($"{reqHash} - Result Number of Columns: {nrOfCols}");
await GenerateAndSendHeadersAsync(context, nrOfRows, nrOfCols, resultDataColumns, cacheResultInQlik);
Logger.Trace($"{reqHash} - Start Loop");
// Send data
var bundledRows = new BundledRows();
//Loop through each response in array (one for each row of input data)
int i = 0;
foreach (DataSpecification p in sortedData)
{
var row = new Row();
if (request_type != "timeseries")
{
if (shouldExplain && rawExplain)
{
if (p.predictionExplanations != null)
{
row.Duals.Add(new Dual()
{
StrData = JsonConvert.SerializeObject(p.predictionExplanations) ?? ""
});
}
else
{
row.Duals.Add(new Dual()
{
StrData = "[]"
});
}
}
}
//Prediction Column
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.prediction) ?? ""
});
Logger.Trace($"{reqHash} - In Loop RowId: {p.rowId}");
//KeyField Column
if (p.passthroughValues != null)
{
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.passthroughValues[keyField.Name]) ?? ""
});
}
//Include Details
if (includeDetail == true)
{
//Loop through each predicted value and insert the row values to the column
foreach (PredictionValueSpecification pv in p.predictionValues)
{
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(pv.value) ?? ""
});
}
}
//Timeseries field
if (request_type == "timeseries")
{
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.rowId) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.seriesId) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.forecastPoint) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.timestamp) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.forecastDistance) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.originalFormatTimestamp) ?? ""
});
}
//Include Prediction Explanations
if (shouldExplain && !rawExplain)
{
foreach (PredictionExplanationSpecification pe in p.predictionExplanations)
{
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(pe.label) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(pe.feature) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(pe.featureValue) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(pe.strength) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(pe.qualitativeStrength) ?? ""
});
}
for (int j = p.predictionExplanations.Count; j < explain_max; j++)
{
row.Duals.Add(new Dual()
{
});
row.Duals.Add(new Dual()
{
});
row.Duals.Add(new Dual()
{
});
row.Duals.Add(new Dual()
{
});
row.Duals.Add(new Dual()
{
});
}
}
//Add the row the bundle
bundledRows.Rows.Add(row);
//When we get a batch of 2000 rows, send the response
if (((i + 1) % 2000) == 0)
{
// Send a bundle
await responseStream.WriteAsync(bundledRows);
//Reset the bundle
bundledRows = null;
bundledRows = new BundledRows();
}
i++;
}
//Send any left over rows after the final loop
if (bundledRows.Rows.Count() > 0)
{
// Send last bundle
await responseStream.WriteAsync(bundledRows);
bundledRows = null;
}
}
else if (response.response != null)
{
Logger.Trace($"{reqHash} - Processing Status Response for Project ID: {response.response.id}");
var a = new ResultDataColumn();
a.Name = "Result";
a.DataType = DataType.String;
resultDataColumns.Add(a);
await GenerateAndSendHeadersAsync(context, 1, 1, resultDataColumns, cacheResultInQlik);
var bundledRows = new BundledRows();
var row = new Row();
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(response.response.id) ?? ""
});
bundledRows.Rows.Add(row);
await responseStream.WriteAsync(bundledRows);
bundledRows = null;
}
else
{
if (response.message != null)
{
throw new Exception($"The following error message was returned from DataRobot: {response.message}");
}
else
{
throw new Exception($"An Unknown Error Occured: {data}");
}
}
}
}