private ResultDataColumn[] GetResultDataColumns(ref int nrOfRows, string[] names, IList <object> columns)
{
int nRows = nrOfRows;
ResultDataColumn[] resultDataColumns = columns
.Select((col, index) =>
{
var c = new ResultDataColumn();
if (names != null && names.Length > 0)
{
c.Name = names[index] ?? "";
}
if (col is SexpArrayBool || col is SexpArrayDouble || col is SexpArrayInt)
{
c.DataType = DataType.Numeric;
Sexp colAsSexp = (Sexp)col;
c.Numerics = colAsSexp.AsDoubles;
if (index == 0)
{
nRows = c.Numerics.Length;
}
else if (nRows != c.Numerics.Length)
{
logger.Warn($"Rserve result, different length in columns: {nRows} vs {c.Numerics.Length}");
throw new NotImplementedException();
}
if (logger.IsTraceEnabled)
{
var logNumerics = String.Join(", ", c.Numerics);
logger.Trace($"Numeric result column data[{index}]: {logNumerics}");
}
}
else if (col is SexpArrayString)
{
c.DataType = DataType.String;
Sexp colAsSexp = (Sexp)col;
c.Strings = colAsSexp.AsStrings;
if (index == 0)
{
nRows = c.Strings.Length;
}
else if (nRows != c.Strings.Length)
{
logger.Warn($"Rserve result, different length in columns: {nRows} vs {c.Strings.Length}");
throw new NotImplementedException();
}
if (logger.IsTraceEnabled)
{
var logStrings = String.Join(", ", c.Strings);
logger.Trace($"String result column data[{index}]: {logStrings}");
}
}
else
{
logger.Warn($"Rserve result, column data type not recognized: {col.GetType().ToString()}");
throw new NotImplementedException();
}
return(c);
})
.ToArray();
nrOfRows = nRows;
return(resultDataColumns);
}
private async Task GenerateResult(Sexp RResult, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, ServerCallContext context,
bool failIfWrongDataTypeInFirstCol = false, DataType expectedFirstDataType = DataType.Numeric, bool cacheResultInQlik = true)
{
int nrOfCols = 0;
int nrOfRows = 0;
ResultDataColumn[] resultDataColumns = null;
var names = RResult.Names;
if (names != null)
{
logger.Debug($"Rserve result column names: {String.Join(", ", names)}");
}
if (RResult is SexpList)
{
// Indicating this is a data.frame/matrix response structure. Figure out how many columns, names and data types
nrOfCols = RResult.Count;
logger.Debug($"Rserve result nrOfColumns: {nrOfCols}");
if (RResult.Attributes != null && RResult.Attributes.Count > 0)
{
Sexp resObjectNames;
if ((names == null || names.Length == 0) && RResult.Attributes.TryGetValue("names", out resObjectNames))
{
names = resObjectNames.AsStrings;
logger.Debug($"Rserve result column names: {String.Join(", ", names)}");
}
Sexp resObjectClass;
if (RResult.Attributes.TryGetValue("class", out resObjectClass))
{
logger.Debug($"Rserve result object class: {resObjectClass.ToString()}");
}
}
if (nrOfCols > 0)
{
var columns = RResult.AsList;
resultDataColumns = GetResultDataColumns(ref nrOfRows, names, columns);
}
}
else if (RResult is SexpArrayBool || RResult is SexpArrayDouble || RResult is SexpArrayInt)
{
nrOfCols = 1;
var bundledRows = new BundledRows();
var numerics = RResult.AsDoubles;
nrOfRows = numerics.Length;
var c = new ResultDataColumn();
c.Name = "";
c.DataType = DataType.Numeric;
c.Numerics = numerics;
resultDataColumns = new ResultDataColumn[1];
resultDataColumns[0] = c;
if (logger.IsTraceEnabled)
{
var logNumerics = String.Join(", ", numerics);
logger.Trace("Numeric result column data[0]: {0}", logNumerics);
}
}
else if (RResult is SexpArrayString)
{
nrOfCols = 1;
var bundledRows = new BundledRows();
var strings = RResult.AsStrings;
nrOfRows = strings.Length;
var c = new ResultDataColumn();
c.Name = "";
c.DataType = DataType.String;
c.Strings = strings;
resultDataColumns = new ResultDataColumn[1];
resultDataColumns[0] = c;
if (logger.IsTraceEnabled)
{
var logStrings = String.Join(", ", strings);
logger.Trace("String result column data[0]: {0}", logStrings);
}
}
else
{
logger.Warn($"Rserve result, column data type not recognized: {RResult.GetType().ToString()}");
throw new NotImplementedException();
}
if (resultDataColumns != null)
{
if (failIfWrongDataTypeInFirstCol && expectedFirstDataType != resultDataColumns[0].DataType)
{
string msg = $"Rserve result datatype mismatch in first column, expected {expectedFirstDataType}, got {resultDataColumns[0].DataType}";
logger.Warn($"{msg}");
throw new RpcException(new Status(StatusCode.InvalidArgument, $"{msg}"));
}
//Send TableDescription header
TableDescription tableDesc = new TableDescription
{
NumberOfRows = nrOfRows
};
for (int col = 0; col < nrOfCols; col++)
{
if (String.IsNullOrEmpty(resultDataColumns[col].Name))
{
tableDesc.Fields.Add(new FieldDescription
{
DataType = resultDataColumns[col].DataType
});
}
else
{
tableDesc.Fields.Add(new FieldDescription
{
DataType = resultDataColumns[col].DataType,
Name = resultDataColumns[col].Name
});
}
}
var tableMetadata = new Metadata
{
{ new Metadata.Entry("qlik-tabledescription-bin", MessageExtensions.ToByteArray(tableDesc)) }
};
if (!cacheResultInQlik)
{
tableMetadata.Add("qlik-cache", "no-store");
}
await context.WriteResponseHeadersAsync(tableMetadata);
// Send data
var bundledRows = new BundledRows();
for (int i = 0; i < nrOfRows; i++)
{
var row = new Row();
for (int col = 0; col < nrOfCols; col++)
{
if (resultDataColumns[col].DataType == DataType.Numeric)
{
row.Duals.Add(new Dual()
{
NumData = resultDataColumns[col].Numerics[i]
});
}
else if (resultDataColumns[col].DataType == DataType.String)
{
row.Duals.Add(new Dual()
{
StrData = resultDataColumns[col].Strings[i] ?? ""
});
}
}
bundledRows.Rows.Add(row);
if (((i + 1) % 2000) == 0)
{
// Send a bundle
await responseStream.WriteAsync(bundledRows);
bundledRows = new BundledRows();
}
}
if (bundledRows.Rows.Count() > 0)
{
// Send last bundle
await responseStream.WriteAsync(bundledRows);
}
}
}
/// <summary>
/// All requests are processed through evaluate script, however in the context of this connector, the script is a JSON notation string which contains the metadata required to correctly process the attached data.
/// </summary>
public override async Task EvaluateScript(IAsyncStreamReader <global::Qlik.Sse.BundledRows> requestStream, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, ServerCallContext context)
{
ScriptRequestHeader scriptHeader;
CommonRequestHeader commonHeader;
Qlik2DataRobotMetrics.RequestCounter.Inc();
int reqHash = requestStream.GetHashCode();
try
{
var header = GetHeader(context.RequestHeaders, "qlik-scriptrequestheader-bin");
scriptHeader = ScriptRequestHeader.Parser.ParseFrom(header);
var commonRequestHeader = GetHeader(context.RequestHeaders, "qlik-commonrequestheader-bin");
commonHeader = CommonRequestHeader.Parser.ParseFrom(commonRequestHeader);
Logger.Info($"{reqHash} - EvaluateScript called from client ({context.Peer}), hashid ({reqHash})");
Logger.Debug($"{reqHash} - EvaluateScript header info: AppId ({commonHeader.AppId}), UserId ({commonHeader.UserId}), Cardinality ({commonHeader.Cardinality} rows)");
}
catch (Exception e)
{
Logger.Error($"EvaluateScript with hashid ({reqHash}) failed: {e.Message}");
throw new RpcException(new Status(StatusCode.DataLoss, e.Message));
}
try
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var paramnames = $"{reqHash} - EvaluateScript call with hashid({reqHash}) got Param names: ";
foreach (var param in scriptHeader.Params)
{
paramnames += $" {param.Name}";
}
Logger.Trace("{0}", paramnames);
Logger.Trace(scriptHeader.Script);
Dictionary <string, dynamic> config = JsonConvert.DeserializeObject <Dictionary <string, dynamic> >(scriptHeader.Script);
var Params = GetParams(scriptHeader.Params.ToArray());
string keyname = null;
if (config.ContainsKey("keyfield"))
{
keyname = Convert.ToString(config["keyfield"]);
}
ResultDataColumn keyField = new ResultDataColumn();
var rowdatastream = await ConvertBundledRowsToCSV(Params, requestStream, context, keyField, keyname);
Logger.Debug($"{reqHash} - Input Data Size: {rowdatastream.Length}");
var outData = await SelectFunction(config, rowdatastream, reqHash);
rowdatastream = null;
bool shouldCache = false;
if (config.ContainsKey("should_cache"))
{
shouldCache = config["should_cache"];
}
bool inc_details = false;
bool rawExplain = false;
if (config.ContainsKey("inc_details"))
{
inc_details = config["inc_details"];
}
if (config.ContainsKey("explain"))
{
rawExplain = config["explain"]["return_raw"];
}
string request_type = config["request_type"];
await GenerateResult(request_type, outData, responseStream, context, reqHash, cacheResultInQlik : shouldCache, keyField : keyField, keyname : keyname, includeDetail : inc_details, rawExplain : rawExplain);
outData = null;
stopwatch.Stop();
Logger.Debug($"{reqHash} - Took {stopwatch.ElapsedMilliseconds} ms, hashid ({reqHash})");
Qlik2DataRobotMetrics.DurHist.Observe(stopwatch.ElapsedMilliseconds / 1000);
}
catch (Exception e)
{
Logger.Error($"{reqHash} - ERROR: {e.Message}");
throw new RpcException(new Status(StatusCode.InvalidArgument, $"{e.Message}"));
}
finally
{
}
GC.Collect();
}
/// <summary>
/// Return the results from connector to Qlik Engine
/// </summary>
private async Task GenerateResult(string request_type, MemoryStream returnedData, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, ServerCallContext context, int reqHash,
bool failIfWrongDataTypeInFirstCol = false, DataType expectedFirstDataType = DataType.Numeric, bool cacheResultInQlik = true, ResultDataColumn keyField = null, string keyname = null, bool includeDetail = false, bool rawExplain = false)
{
int nrOfCols = 0;
int nrOfRows = 0;
List <ResultDataColumn> resultDataColumns = new List <ResultDataColumn>();
Logger.Info($"{reqHash} - Generate Results");
if (true)
{
Logger.Debug($"{reqHash} - Extract JSON");
//Convert the stream (json) to dictionary
Logger.Info($"{reqHash} - Returned Datasize: {returnedData.Length}");
Logger.Info($"{reqHash} - Request Type: {request_type}");
StreamReader sr = new StreamReader(returnedData);
returnedData.Position = 0;
var data = sr.ReadToEnd();
Dictionary <string, dynamic> response = JsonConvert.DeserializeObject <Dictionary <string, dynamic> >(data);
Logger.Trace($"{reqHash} - Returned Data: {data}");
if (response.ContainsKey("data"))
{
//Prediction Column
var a = new ResultDataColumn();
a.Name = "Prediction";
a.DataType = DataType.String;
a.Strings = new List <string>();
//Time Series Columns
var seriesId = new ResultDataColumn();
var forecastPoint = new ResultDataColumn();
var rowId = new ResultDataColumn();
var timestamp = new ResultDataColumn();
var forecastDistance = new ResultDataColumn();
if (request_type == "timeseries")
{
seriesId.Name = "seriesId";
seriesId.DataType = DataType.String;
seriesId.Strings = new List <string>();
forecastPoint.Name = "forecastPoint";
forecastPoint.DataType = DataType.String;
forecastPoint.Strings = new List <string>();
rowId.Name = "rowId";
rowId.DataType = DataType.String;
rowId.Strings = new List <string>();
timestamp.Name = "timestamp";
timestamp.DataType = DataType.String;
timestamp.Strings = new List <string>();
forecastDistance.Name = "forecastDistance";
forecastDistance.DataType = DataType.String;
forecastDistance.Strings = new List <string>();
}
var pe = new ResultDataColumn();
if (rawExplain == true)
{
pe.Name = $"Prediction Explanations";
pe.DataType = DataType.String;
pe.Strings = new List <string>();
}
//The first row will determine which fields to return in table for prediction values
bool fieldListAgreed = false;
//Loop through each response in array (one for each row of input data)
foreach (dynamic p in response["data"])
{
// **** For debug to display array data on console ****
Logger.Info($"{reqHash} {p}");
a.Strings.Add(Convert.ToString(p["prediction"]));
// Add Time Series Column Data
if (request_type == "timeseries")
{
seriesId.Strings.Add(Convert.ToString(p["seriesId"]));
forecastPoint.Strings.Add(Convert.ToString(p["forecastPoint"]));
rowId.Strings.Add(Convert.ToString(p["rowId"]));
timestamp.Strings.Add(Convert.ToString(p["timestamp"]));
forecastDistance.Strings.Add(Convert.ToString(p["forecastDistance"]));
}
if (includeDetail == true)
{
if (fieldListAgreed == false)
{
foreach (dynamic pv in p["predictionValues"])
{
var pvi = new ResultDataColumn();
pvi.Name = $"Prediction value for label: {pv["label"]}";
pvi.DataType = DataType.String;
pvi.Strings = new List <string>();
resultDataColumns.Add(pvi);
}
fieldListAgreed = true;
Logger.Trace($"{reqHash} - Columns: {resultDataColumns.Count}");
}
//Loop through each predicted value and insert the row values to the column
int index = 0;
foreach (dynamic pv in p["predictionValues"])
{
resultDataColumns[index].Strings.Add(Convert.ToString(pv["value"]));
index++;
}
}
if (rawExplain == true)
{
pe.Strings.Add(Convert.ToString(p["predictionExplanations"]));
}
}
if (keyname != null)
{
resultDataColumns.Add(keyField);
}
if (rawExplain == true)
{
resultDataColumns.Add(pe);
}
// Add Time Series Columns to resultData
if (request_type == "timeseries")
{
resultDataColumns.Add(seriesId);
resultDataColumns.Add(forecastPoint);
resultDataColumns.Add(rowId);
resultDataColumns.Add(timestamp);
resultDataColumns.Add(forecastDistance);
}
resultDataColumns.Add(a);
}
else if (response.ContainsKey("response"))
{
var a = new ResultDataColumn();
a.Name = "Result";
a.DataType = DataType.String;
a.Strings = new List <string>();
a.Strings.Add(Convert.ToString(response["response"]["id"]));
resultDataColumns.Add(a);
}
else
{
if (response.ContainsKey("message"))
{
throw new Exception($"The following error message was returned from DataRobot: {response["message"]}");
}
else
{
throw new Exception($"An Unknown Error Occured: {data}");
}
}
nrOfRows = resultDataColumns[0].DataType == DataType.String ? resultDataColumns[0].Strings.Count : resultDataColumns[0].Numerics.Count;
nrOfCols = resultDataColumns.Count;
Logger.Debug($"{reqHash} - Result Number of Columns: {nrOfCols}");
}
if (resultDataColumns != null)
{
if (failIfWrongDataTypeInFirstCol && expectedFirstDataType != resultDataColumns[0].DataType)
{
string msg = $"Result datatype mismatch in first column, expected {expectedFirstDataType}, got {resultDataColumns[0].DataType}";
Logger.Warn($"{reqHash} - {msg}");
throw new RpcException(new Status(StatusCode.InvalidArgument, $"{msg}"));
}
//Send TableDescription header
TableDescription tableDesc = new TableDescription
{
NumberOfRows = nrOfRows
};
for (int col = 0; col < nrOfCols; col++)
{
if (String.IsNullOrEmpty(resultDataColumns[col].Name))
{
tableDesc.Fields.Add(new FieldDescription
{
DataType = resultDataColumns[col].DataType
});
}
else
{
tableDesc.Fields.Add(new FieldDescription
{
DataType = resultDataColumns[col].DataType,
Name = resultDataColumns[col].Name
});
}
}
var tableMetadata = new Metadata
{
{ new Metadata.Entry("qlik-tabledescription-bin", MessageExtensions.ToByteArray(tableDesc)) }
};
if (!cacheResultInQlik)
{
tableMetadata.Add("qlik-cache", "no-store");
}
await context.WriteResponseHeadersAsync(tableMetadata);
// Send data
var bundledRows = new BundledRows();
for (int i = 0; i < nrOfRows; i++)
{
var row = new Row();
for (int col = 0; col < nrOfCols; col++)
{
if (resultDataColumns[col].DataType == DataType.Numeric)
{
row.Duals.Add(new Dual()
{
NumData = resultDataColumns[col].Numerics[i]
});
}
else if (resultDataColumns[col].DataType == DataType.String)
{
row.Duals.Add(new Dual()
{
StrData = resultDataColumns[col].Strings[i] ?? ""
});
}
}
bundledRows.Rows.Add(row);
if (((i + 1) % 2000) == 0)
{
// Send a bundle
await responseStream.WriteAsync(bundledRows);
bundledRows = null;
bundledRows = new BundledRows();
}
}
if (bundledRows.Rows.Count() > 0)
{
// Send last bundle
await responseStream.WriteAsync(bundledRows);
bundledRows = null;
}
}
}
/// <summary>
/// Convert the input data into a CSV file within memory stream
/// </summary>
private async Task <MemoryStream> ConvertBundledRowsToCSV(ParameterData[] Parameters, IAsyncStreamReader <global::Qlik.Sse.BundledRows> requestStream, ServerCallContext context, ResultDataColumn keyField, string keyname)
{
int reqHash = requestStream.GetHashCode();
Logger.Debug($"{reqHash} - Start Create CSV");
var memStream = new MemoryStream();
var streamWriter = new StreamWriter(memStream);
var tw = TextWriter.Synchronized(streamWriter);
var csv = new CsvWriter(tw);
var keyindex = 0;
for (int i = 0; i < Parameters.Length; i++)
{
var param = Parameters[i];
if (keyname != null)
{
if (param.ParamName == keyname)
{
keyindex = i;
keyField.Name = param.ParamName;
keyField.DataType = param.DataType;
switch (param.DataType)
{
case DataType.Numeric:
case DataType.Dual:
keyField.Numerics = new List <double>();
break;
case DataType.String:
keyField.Strings = new List <string>();
break;
}
}
}
csv.WriteField(param.ParamName);
}
if (keyField.Name == null && keyname != null)
{
throw new Exception("The keyfield was not found in the source data, please ensure you are including this field in the dataset sent from Qlik.");
}
csv.NextRecord();
Logger.Debug($"{reqHash} - Finished Header");
int a = 0;
while (await requestStream.MoveNext())
{
foreach (var Row in requestStream.Current.Rows)
{
for (int i = 0; i < Parameters.Length; i++)
{
var param = Parameters[i];
var dual = Row.Duals[i];
switch (param.DataType)
{
case DataType.Numeric:
if (keyindex == i && keyname != null)
{
keyField.Numerics.Add(dual.NumData);
}
csv.WriteField(dual.NumData.ToString());
break;
case DataType.String:
if (keyindex == i && keyname != null)
{
keyField.Strings.Add(dual.StrData);
}
csv.WriteField(dual.StrData);
break;
case DataType.Dual:
if (keyindex == i && keyname != null)
{
keyField.Numerics.Add(dual.NumData);
}
csv.WriteField(dual.NumData.ToString());
break;
}
}
a++;
csv.NextRecord();
}
}
csv.Flush();
tw.Flush();
streamWriter.Flush();
memStream.Flush();
memStream.Position = 0;
Logger.Debug($"{reqHash} - Rows" + a);
if (a == 0)
{
throw new Exception("There were no rows in the table sent from Qlik. Check that the table has at least 1 row of data.");
}
return(await Task.FromResult(memStream));
}
/// <summary>
/// Return the results from connector to Qlik Engine
/// </summary>
private async Task GenerateResult(string request_type, MemoryStream returnedData, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, ServerCallContext context, int reqHash,
bool failIfWrongDataTypeInFirstCol = false, DataType expectedFirstDataType = DataType.Numeric, bool cacheResultInQlik = true, ResultDataColumn keyField = null, string keyname = null, bool includeDetail = false, bool shouldExplain = false, bool rawExplain = false, int explain_max = 0)
{
int nrOfCols = 0;
int nrOfRows = 0;
List <ResultDataColumn> resultDataColumns = new List <ResultDataColumn>();
Logger.Info($"{reqHash} - Generate Results");
if (true)
{
Logger.Debug($"{reqHash} - Extract JSON");
//Convert the stream (json) to dictionary
Logger.Info($"{reqHash} - Returned Datasize: {returnedData.Length}");
Logger.Info($"{reqHash} - Request Type: {request_type}");
StreamReader sr = new StreamReader(returnedData);
returnedData.Position = 0;
var data = sr.ReadToEnd();
//Dictionary<string, dynamic> response = JsonConvert.DeserializeObject<Dictionary<string, dynamic>>(data);
ResponseSpecification response = JsonConvert.DeserializeObject <ResponseSpecification>(data);
Logger.Trace($"{reqHash} - Returned Data: {data}");
if (response.data != null)
{
Logger.Trace($"{reqHash} - Response Data: {response.data}");
//Sort the response by RowId
List <DataSpecification> sortedData = response.data.OrderBy(o => o.rowId).ToList();
//Return Raw Explain First So Works In Chart Expression
if (request_type != "timeseries")
{
if (shouldExplain && rawExplain)
{
var pe = new ResultDataColumn();
pe.Name = $"Prediction Explanations";
pe.DataType = DataType.String;
resultDataColumns.Add(pe);
}
}
//Prediction Column
var a = new ResultDataColumn();
a.Name = "Prediction";
a.DataType = DataType.String;
//a.Strings = new List<string>();
resultDataColumns.Add(a);
//Include Keyfield
if (keyname != null)
{
resultDataColumns.Add(keyField);
}
//The first row will determine which fields to return in table for prediction values
if (includeDetail == true)
{
foreach (PredictionValueSpecification pv in sortedData[0].predictionValues)
{
var pvi = new ResultDataColumn();
pvi.Name = $"Prediction value for label: {pv.label}";
pvi.DataType = DataType.String;
resultDataColumns.Add(pvi);
}
}
// Add Time Series Columns to resultData
if (request_type == "timeseries")
{
//Row Id
var rowId = new ResultDataColumn();
rowId.Name = "rowId";
rowId.DataType = DataType.String;
//Time Series Columns
var seriesId = new ResultDataColumn();
seriesId.Name = "seriesId";
seriesId.DataType = DataType.String;
var forecastPoint = new ResultDataColumn();
forecastPoint.Name = "forecastPoint";
forecastPoint.DataType = DataType.String;
var timestamp = new ResultDataColumn();
timestamp.Name = "timestamp";
timestamp.DataType = DataType.String;
var forecastDistance = new ResultDataColumn();
forecastDistance.Name = "forecastDistance";
forecastDistance.DataType = DataType.String;
var originalFormatTimestamp = new ResultDataColumn();
originalFormatTimestamp.Name = "originalFormatTimestamp";
originalFormatTimestamp.DataType = DataType.String;
resultDataColumns.Add(rowId);
resultDataColumns.Add(seriesId);
resultDataColumns.Add(forecastPoint);
resultDataColumns.Add(timestamp);
resultDataColumns.Add(forecastDistance);
resultDataColumns.Add(originalFormatTimestamp);
}
if (request_type != "timeseries")
{
if (shouldExplain && !rawExplain)
{
for (int j = 0; j < explain_max; j++)
{
foreach (string field in new[] { "label", "feature", "featureValue", "strength", "qualitativeStrength" })
{
var pe = new ResultDataColumn();
pe.Name = $"PE_{j + 1}_{field}";
pe.DataType = DataType.String;
resultDataColumns.Add(pe);
}
}
}
}
nrOfRows = sortedData.Count;
nrOfCols = resultDataColumns.Count;
Logger.Debug($"{reqHash} - Result Number of Columns: {nrOfCols}");
await GenerateAndSendHeadersAsync(context, nrOfRows, nrOfCols, resultDataColumns, cacheResultInQlik);
Logger.Trace($"{reqHash} - Start Loop");
// Send data
var bundledRows = new BundledRows();
//Loop through each response in array (one for each row of input data)
int i = 0;
foreach (DataSpecification p in sortedData)
{
var row = new Row();
if (request_type != "timeseries")
{
if (shouldExplain && rawExplain)
{
if (p.predictionExplanations != null)
{
row.Duals.Add(new Dual()
{
StrData = JsonConvert.SerializeObject(p.predictionExplanations) ?? ""
});
}
else
{
row.Duals.Add(new Dual()
{
StrData = "[]"
});
}
}
}
//Prediction Column
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.prediction) ?? ""
});
Logger.Trace($"{reqHash} - In Loop RowId: {p.rowId}");
//KeyField Column
if (p.passthroughValues != null)
{
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.passthroughValues[keyField.Name]) ?? ""
});
}
//Include Details
if (includeDetail == true)
{
//Loop through each predicted value and insert the row values to the column
foreach (PredictionValueSpecification pv in p.predictionValues)
{
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(pv.value) ?? ""
});
}
}
//Timeseries field
if (request_type == "timeseries")
{
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.rowId) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.seriesId) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.forecastPoint) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.timestamp) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.forecastDistance) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(p.originalFormatTimestamp) ?? ""
});
}
//Include Prediction Explanations
if (shouldExplain && !rawExplain)
{
foreach (PredictionExplanationSpecification pe in p.predictionExplanations)
{
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(pe.label) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(pe.feature) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(pe.featureValue) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(pe.strength) ?? ""
});
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(pe.qualitativeStrength) ?? ""
});
}
for (int j = p.predictionExplanations.Count; j < explain_max; j++)
{
row.Duals.Add(new Dual()
{
});
row.Duals.Add(new Dual()
{
});
row.Duals.Add(new Dual()
{
});
row.Duals.Add(new Dual()
{
});
row.Duals.Add(new Dual()
{
});
}
}
//Add the row the bundle
bundledRows.Rows.Add(row);
//When we get a batch of 2000 rows, send the response
if (((i + 1) % 2000) == 0)
{
// Send a bundle
await responseStream.WriteAsync(bundledRows);
//Reset the bundle
bundledRows = null;
bundledRows = new BundledRows();
}
i++;
}
//Send any left over rows after the final loop
if (bundledRows.Rows.Count() > 0)
{
// Send last bundle
await responseStream.WriteAsync(bundledRows);
bundledRows = null;
}
}
else if (response.response != null)
{
Logger.Trace($"{reqHash} - Processing Status Response for Project ID: {response.response.id}");
var a = new ResultDataColumn();
a.Name = "Result";
a.DataType = DataType.String;
resultDataColumns.Add(a);
await GenerateAndSendHeadersAsync(context, 1, 1, resultDataColumns, cacheResultInQlik);
var bundledRows = new BundledRows();
var row = new Row();
row.Duals.Add(new Dual()
{
StrData = Convert.ToString(response.response.id) ?? ""
});
bundledRows.Rows.Add(row);
await responseStream.WriteAsync(bundledRows);
bundledRows = null;
}
else
{
if (response.message != null)
{
throw new Exception($"The following error message was returned from DataRobot: {response.message}");
}
else
{
throw new Exception($"An Unknown Error Occured: {data}");
}
}
}
}
