private BundledRows GetResult(string result) { var resultBundle = new BundledRows(); var resultRow = new Row(); resultRow.Duals.Add(new Dual { StrData = result, NumData = 0 }); resultBundle.Rows.Add(resultRow); return(resultBundle); }
private static BundledRows GetResult(object result) { var resultBundle = new BundledRows(); var resultRow = new Row(); var settings = new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }; resultRow.Duals.Add(new Dual { StrData = JsonConvert.SerializeObject(result, settings), NumData = 0 }); resultBundle.Rows.Add(resultRow); return(resultBundle); }
private async Task GenerateResult(Sexp RResult, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, ServerCallContext context, bool failIfWrongDataTypeInFirstCol = false, DataType expectedFirstDataType = DataType.Numeric, bool cacheResultInQlik = true) { int nrOfCols = 0; int nrOfRows = 0; ResultDataColumn[] resultDataColumns = null; var names = RResult.Names; if (names != null) { logger.Debug($"Rserve result column names: {String.Join(", ", names)}"); } if (RResult is SexpList) { // Indicating this is a data.frame/matrix response structure. Figure out how many columns, names and data types nrOfCols = RResult.Count; logger.Debug($"Rserve result nrOfColumns: {nrOfCols}"); if (RResult.Attributes != null && RResult.Attributes.Count > 0) { Sexp resObjectNames; if ((names == null || names.Length == 0) && RResult.Attributes.TryGetValue("names", out resObjectNames)) { names = resObjectNames.AsStrings; logger.Debug($"Rserve result column names: {String.Join(", ", names)}"); } Sexp resObjectClass; if (RResult.Attributes.TryGetValue("class", out resObjectClass)) { logger.Debug($"Rserve result object class: {resObjectClass.ToString()}"); } } if (nrOfCols > 0) { var columns = RResult.AsList; resultDataColumns = GetResultDataColumns(ref nrOfRows, names, columns); } } else if (RResult is SexpArrayBool || RResult is SexpArrayDouble || RResult is SexpArrayInt) { nrOfCols = 1; var bundledRows = new BundledRows(); var numerics = RResult.AsDoubles; nrOfRows = numerics.Length; var c = new ResultDataColumn(); c.Name = ""; c.DataType = DataType.Numeric; c.Numerics = numerics; resultDataColumns = new ResultDataColumn[1]; resultDataColumns[0] = c; if (logger.IsTraceEnabled) { var logNumerics = String.Join(", ", numerics); logger.Trace("Numeric result column data[0]: {0}", logNumerics); } } else if (RResult is SexpArrayString) { nrOfCols = 1; var bundledRows = new BundledRows(); var strings = RResult.AsStrings; nrOfRows = strings.Length; var c = new ResultDataColumn(); c.Name = ""; c.DataType = DataType.String; c.Strings = strings; resultDataColumns = new ResultDataColumn[1]; resultDataColumns[0] = c; if (logger.IsTraceEnabled) { var logStrings = String.Join(", ", strings); logger.Trace("String result column data[0]: {0}", logStrings); } } else { logger.Warn($"Rserve result, column data type not recognized: {RResult.GetType().ToString()}"); throw new NotImplementedException(); } if (resultDataColumns != null) { if (failIfWrongDataTypeInFirstCol && expectedFirstDataType != resultDataColumns[0].DataType) { string msg = $"Rserve result datatype mismatch in first column, expected {expectedFirstDataType}, got {resultDataColumns[0].DataType}"; logger.Warn($"{msg}"); throw new RpcException(new Status(StatusCode.InvalidArgument, $"{msg}")); } //Send TableDescription header TableDescription tableDesc = new TableDescription { NumberOfRows = nrOfRows }; for (int col = 0; col < nrOfCols; col++) { if (String.IsNullOrEmpty(resultDataColumns[col].Name)) { tableDesc.Fields.Add(new FieldDescription { DataType = resultDataColumns[col].DataType }); } else { tableDesc.Fields.Add(new FieldDescription { DataType = resultDataColumns[col].DataType, Name = resultDataColumns[col].Name }); } } var tableMetadata = new Metadata { { new Metadata.Entry("qlik-tabledescription-bin", MessageExtensions.ToByteArray(tableDesc)) } }; if (!cacheResultInQlik) { tableMetadata.Add("qlik-cache", "no-store"); } await context.WriteResponseHeadersAsync(tableMetadata); // Send data var bundledRows = new BundledRows(); for (int i = 0; i < nrOfRows; i++) { var row = new Row(); for (int col = 0; col < nrOfCols; col++) { if (resultDataColumns[col].DataType == DataType.Numeric) { row.Duals.Add(new Dual() { NumData = resultDataColumns[col].Numerics[i] }); } else if (resultDataColumns[col].DataType == DataType.String) { row.Duals.Add(new Dual() { StrData = resultDataColumns[col].Strings[i] ?? "" }); } } bundledRows.Rows.Add(row); if (((i + 1) % 2000) == 0) { // Send a bundle await responseStream.WriteAsync(bundledRows); bundledRows = new BundledRows(); } } if (bundledRows.Rows.Count() > 0) { // Send last bundle await responseStream.WriteAsync(bundledRows); } } }
public override async Task ExecuteFunction(IAsyncStreamReader <BundledRows> requestStream, IServerStreamWriter <BundledRows> responseStream, ServerCallContext context) { Logger.Trace("-- ExecuteFunction --"); if (logType == "file") { logConnector = new FileAuditLogConnector(); } Dictionary <String, String> headerInfo = TraceServerCallContext(context); var functionRequestHeaderStream = context.RequestHeaders.SingleOrDefault(header => header.Key == "qlik-functionrequestheader-bin"); if (functionRequestHeaderStream == null) { throw new Exception("ExecuteFunction called without Function Request Header in Request Headers."); } var functionRequestHeader = new FunctionRequestHeader(); functionRequestHeader.MergeFrom(new CodedInputStream(functionRequestHeaderStream.ValueBytes)); //Logger.Trace($"FunctionRequestHeader.FunctionId String : {(FunctionConstant)functionRequestHeader.FunctionId}"); Guid g = LogRequest(logConnector, headerInfo["AppId"], headerInfo["UserId"]); switch (functionRequestHeader.FunctionId) { case (int)FunctionConstant.LogAsSeenStrCheck: { while (await requestStream.MoveNext()) { var resultBundle = new BundledRows(); var cacheMetadata = new Metadata { { new Metadata.Entry("qlik-cache", "no-store") } }; await context.WriteResponseHeadersAsync(cacheMetadata); foreach (var row in requestStream.Current.Rows) { Row resultRow = LogRow(logConnector, g, headerInfo["AppId"], headerInfo["UserId"], row.Duals[0].StrData, "√"); resultBundle.Rows.Add(resultRow); } await responseStream.WriteAsync(resultBundle); } break; } case (int)FunctionConstant.LogAsSeenStrEcho: { while (await requestStream.MoveNext()) { var resultBundle = new BundledRows(); var cacheMetadata = new Metadata { { new Metadata.Entry("qlik-cache", "no-store") } }; await context.WriteResponseHeadersAsync(cacheMetadata); foreach (var row in requestStream.Current.Rows) { Row resultRow = LogRow(logConnector, g, headerInfo["AppId"], headerInfo["UserId"], row.Duals[0].StrData, row.Duals[0].StrData); resultBundle.Rows.Add(resultRow); } await responseStream.WriteAsync(resultBundle); } break; } case (int)FunctionConstant.LogAsSeenCheck: { while (await requestStream.MoveNext()) { var resultBundle = new BundledRows(); var cacheMetadata = new Metadata { { new Metadata.Entry("qlik-cache", "no-store") } }; await context.WriteResponseHeadersAsync(cacheMetadata); foreach (var row in requestStream.Current.Rows) { Row resultRow = LogRow(logConnector, g, headerInfo["AppId"], headerInfo["UserId"], row.Duals[0].NumData, "√"); resultBundle.Rows.Add(resultRow); } await responseStream.WriteAsync(resultBundle); } break; } case (int)FunctionConstant.LogAsSeenEcho: { while (await requestStream.MoveNext()) { var resultBundle = new BundledRows(); var cacheMetadata = new Metadata { { new Metadata.Entry("qlik-cache", "no-store") } }; await context.WriteResponseHeadersAsync(cacheMetadata); foreach (var row in requestStream.Current.Rows) { Row resultRow = LogRow(logConnector, g, headerInfo["AppId"], headerInfo["UserId"], row.Duals[0].NumData, row.Duals[0].NumData); resultBundle.Rows.Add(resultRow); } await responseStream.WriteAsync(resultBundle); } break; } default: break; } logConnector.EndLogData(); Logger.Trace("-- (ExecuteFunction) --"); }
/// <summary> /// Return the results from connector to Qlik Engine /// </summary> private async Task GenerateResult(string request_type, MemoryStream returnedData, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, ServerCallContext context, int reqHash, bool failIfWrongDataTypeInFirstCol = false, DataType expectedFirstDataType = DataType.Numeric, bool cacheResultInQlik = true, ResultDataColumn keyField = null, string keyname = null, bool includeDetail = false, bool rawExplain = false) { int nrOfCols = 0; int nrOfRows = 0; List <ResultDataColumn> resultDataColumns = new List <ResultDataColumn>(); Logger.Info($"{reqHash} - Generate Results"); if (true) { Logger.Debug($"{reqHash} - Extract JSON"); //Convert the stream (json) to dictionary Logger.Info($"{reqHash} - Returned Datasize: {returnedData.Length}"); Logger.Info($"{reqHash} - Request Type: {request_type}"); StreamReader sr = new StreamReader(returnedData); returnedData.Position = 0; var data = sr.ReadToEnd(); Dictionary <string, dynamic> response = JsonConvert.DeserializeObject <Dictionary <string, dynamic> >(data); Logger.Trace($"{reqHash} - Returned Data: {data}"); if (response.ContainsKey("data")) { //Prediction Column var a = new ResultDataColumn(); a.Name = "Prediction"; a.DataType = DataType.String; a.Strings = new List <string>(); //Time Series Columns var seriesId = new ResultDataColumn(); var forecastPoint = new ResultDataColumn(); var rowId = new ResultDataColumn(); var timestamp = new ResultDataColumn(); var forecastDistance = new ResultDataColumn(); if (request_type == "timeseries") { seriesId.Name = "seriesId"; seriesId.DataType = DataType.String; seriesId.Strings = new List <string>(); forecastPoint.Name = "forecastPoint"; forecastPoint.DataType = DataType.String; forecastPoint.Strings = new List <string>(); rowId.Name = "rowId"; rowId.DataType = DataType.String; rowId.Strings = new List <string>(); timestamp.Name = "timestamp"; timestamp.DataType = DataType.String; timestamp.Strings = new List <string>(); forecastDistance.Name = "forecastDistance"; forecastDistance.DataType = DataType.String; forecastDistance.Strings = new List <string>(); } var pe = new ResultDataColumn(); if (rawExplain == true) { pe.Name = $"Prediction Explanations"; pe.DataType = DataType.String; pe.Strings = new List <string>(); } //The first row will determine which fields to return in table for prediction values bool fieldListAgreed = false; //Loop through each response in array (one for each row of input data) foreach (dynamic p in response["data"]) { // **** For debug to display array data on console **** Logger.Info($"{reqHash} {p}"); a.Strings.Add(Convert.ToString(p["prediction"])); // Add Time Series Column Data if (request_type == "timeseries") { seriesId.Strings.Add(Convert.ToString(p["seriesId"])); forecastPoint.Strings.Add(Convert.ToString(p["forecastPoint"])); rowId.Strings.Add(Convert.ToString(p["rowId"])); timestamp.Strings.Add(Convert.ToString(p["timestamp"])); forecastDistance.Strings.Add(Convert.ToString(p["forecastDistance"])); } if (includeDetail == true) { if (fieldListAgreed == false) { foreach (dynamic pv in p["predictionValues"]) { var pvi = new ResultDataColumn(); pvi.Name = $"Prediction value for label: {pv["label"]}"; pvi.DataType = DataType.String; pvi.Strings = new List <string>(); resultDataColumns.Add(pvi); } fieldListAgreed = true; Logger.Trace($"{reqHash} - Columns: {resultDataColumns.Count}"); } //Loop through each predicted value and insert the row values to the column int index = 0; foreach (dynamic pv in p["predictionValues"]) { resultDataColumns[index].Strings.Add(Convert.ToString(pv["value"])); index++; } } if (rawExplain == true) { pe.Strings.Add(Convert.ToString(p["predictionExplanations"])); } } if (keyname != null) { resultDataColumns.Add(keyField); } if (rawExplain == true) { resultDataColumns.Add(pe); } // Add Time Series Columns to resultData if (request_type == "timeseries") { resultDataColumns.Add(seriesId); resultDataColumns.Add(forecastPoint); resultDataColumns.Add(rowId); resultDataColumns.Add(timestamp); resultDataColumns.Add(forecastDistance); } resultDataColumns.Add(a); } else if (response.ContainsKey("response")) { var a = new ResultDataColumn(); a.Name = "Result"; a.DataType = DataType.String; a.Strings = new List <string>(); a.Strings.Add(Convert.ToString(response["response"]["id"])); resultDataColumns.Add(a); } else { if (response.ContainsKey("message")) { throw new Exception($"The following error message was returned from DataRobot: {response["message"]}"); } else { throw new Exception($"An Unknown Error Occured: {data}"); } } nrOfRows = resultDataColumns[0].DataType == DataType.String ? resultDataColumns[0].Strings.Count : resultDataColumns[0].Numerics.Count; nrOfCols = resultDataColumns.Count; Logger.Debug($"{reqHash} - Result Number of Columns: {nrOfCols}"); } if (resultDataColumns != null) { if (failIfWrongDataTypeInFirstCol && expectedFirstDataType != resultDataColumns[0].DataType) { string msg = $"Result datatype mismatch in first column, expected {expectedFirstDataType}, got {resultDataColumns[0].DataType}"; Logger.Warn($"{reqHash} - {msg}"); throw new RpcException(new Status(StatusCode.InvalidArgument, $"{msg}")); } //Send TableDescription header TableDescription tableDesc = new TableDescription { NumberOfRows = nrOfRows }; for (int col = 0; col < nrOfCols; col++) { if (String.IsNullOrEmpty(resultDataColumns[col].Name)) { tableDesc.Fields.Add(new FieldDescription { DataType = resultDataColumns[col].DataType }); } else { tableDesc.Fields.Add(new FieldDescription { DataType = resultDataColumns[col].DataType, Name = resultDataColumns[col].Name }); } } var tableMetadata = new Metadata { { new Metadata.Entry("qlik-tabledescription-bin", MessageExtensions.ToByteArray(tableDesc)) } }; if (!cacheResultInQlik) { tableMetadata.Add("qlik-cache", "no-store"); } await context.WriteResponseHeadersAsync(tableMetadata); // Send data var bundledRows = new BundledRows(); for (int i = 0; i < nrOfRows; i++) { var row = new Row(); for (int col = 0; col < nrOfCols; col++) { if (resultDataColumns[col].DataType == DataType.Numeric) { row.Duals.Add(new Dual() { NumData = resultDataColumns[col].Numerics[i] }); } else if (resultDataColumns[col].DataType == DataType.String) { row.Duals.Add(new Dual() { StrData = resultDataColumns[col].Strings[i] ?? "" }); } } bundledRows.Rows.Add(row); if (((i + 1) % 2000) == 0) { // Send a bundle await responseStream.WriteAsync(bundledRows); bundledRows = null; bundledRows = new BundledRows(); } } if (bundledRows.Rows.Count() > 0) { // Send last bundle await responseStream.WriteAsync(bundledRows); bundledRows = null; } } }
public override async Task ExecuteFunction(IAsyncStreamReader <BundledRows> requestStream, IServerStreamWriter <BundledRows> responseStream, ServerCallContext context) { if (Logger.IsTraceEnabled) { Logger.Trace("-- ExecuteFunction --"); TraceServerCallContext(context); } else { Logger.Debug("ExecuteFunction called"); } var functionRequestHeaderStream = context.RequestHeaders.SingleOrDefault(header => header.Key == "qlik-functionrequestheader-bin"); if (functionRequestHeaderStream == null) { throw new Exception("ExecuteFunction called without Function Request Header in Request Headers."); } var functionRequestHeader = new FunctionRequestHeader(); functionRequestHeader.MergeFrom(new CodedInputStream(functionRequestHeaderStream.ValueBytes)); Logger.Trace($"FunctionRequestHeader.FunctionId String : {(FunctionConstant)functionRequestHeader.FunctionId}"); switch (functionRequestHeader.FunctionId) { case (int)FunctionConstant.Add42: { while (await requestStream.MoveNext()) { var resultBundle = new BundledRows(); foreach (var row in requestStream.Current.Rows) { var resultRow = new Row(); resultRow.Duals.Add(new Dual { NumData = row.Duals[0].NumData + 42.0 }); resultBundle.Rows.Add(resultRow); } await responseStream.WriteAsync(resultBundle); } break; } case (int)FunctionConstant.SumOfAllNumbers: { double sum = 0.0; while (await requestStream.MoveNext()) { foreach (var row in requestStream.Current.Rows) { sum = sum + row.Duals.Select(d => d.NumData).Sum(); } } var resultBundle = new BundledRows(); var resultRow = new Row(); resultRow.Duals.Add(new Dual { NumData = sum }); resultBundle.Rows.Add(resultRow); await responseStream.WriteAsync(resultBundle); break; } case (int)FunctionConstant.Concatenate: { var requestAsList = await requestStream.ToListAsync(); var concatenatedStrings = String.Join(", ", requestAsList.SelectMany(bundledRows => bundledRows.Rows).SelectMany(row => row.Duals) .Select(dual => dual.StrData)); var resultBundle = new BundledRows(); var resultRow = new Row(); resultRow.Duals.Add(new Dual { StrData = concatenatedStrings }); resultBundle.Rows.Add(resultRow); await responseStream.WriteAsync(resultBundle); break; } case (int)FunctionConstant.CallCounter: { var currentIncrement = Interlocked.Increment(ref _callCounter); while (await requestStream.MoveNext()) { var resultBundle = new BundledRows(); foreach (var row in requestStream.Current.Rows) { var resultRow = new Row(); resultRow.Duals.Add(new Dual { NumData = currentIncrement }); resultBundle.Rows.Add(resultRow); } await responseStream.WriteAsync(resultBundle); } break; } case (int)FunctionConstant.CallCounterNoCache: { context.ResponseTrailers.Add("qlik-cache", "no-store"); var currentIncrement = Interlocked.Increment(ref _callCounter); while (await requestStream.MoveNext()) { var resultBundle = new BundledRows(); foreach (var row in requestStream.Current.Rows) { var resultRow = new Row(); resultRow.Duals.Add(new Dual { NumData = currentIncrement }); resultBundle.Rows.Add(resultRow); } await responseStream.WriteAsync(resultBundle); } break; } default: break; } Logger.Trace("-- (ExecuteFunction) --"); }
private async Task GenerateResult(Sexp RResult, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, RserveConnection rserveConn) { if (RResult is SexpArrayBool || RResult is SexpArrayDouble || RResult is SexpArrayInt) { var bundledRows = new BundledRows(); var numerics = RResult.AsDoubles; if (numerics.Length == 0) { HandleZeroRowsFromRserve(rserveConn); } for (int i = 0; i < numerics.Length; i++) { var row = new Row(); row.Duals.Add(new Dual() { NumData = numerics[i] }); bundledRows.Rows.Add(row); if ((i % 2000) == 0) { // Send a bundle await responseStream.WriteAsync(bundledRows); bundledRows = new BundledRows(); } } if (bundledRows.Rows.Count() > 0) { // Send last bundle await responseStream.WriteAsync(bundledRows); } if (logger.IsTraceEnabled) { var logNumerics = String.Join(", ", numerics); logger.Trace("Numeric result column data: {0}", logNumerics); } } else if (RResult is SexpArrayString) { var bundledRows = new BundledRows(); var strings = RResult.AsStrings; if (strings.Length == 0) { HandleZeroRowsFromRserve(rserveConn); } for (int i = 0; i < strings.Length; i++) { var row = new Row(); row.Duals.Add(new Dual() { StrData = strings[i] ?? "" }); bundledRows.Rows.Add(row); if ((i % 2000) == 0) { // Send a bundle await responseStream.WriteAsync(bundledRows); bundledRows = new BundledRows(); } } if (bundledRows.Rows.Count() > 0) { // Send last bundle await responseStream.WriteAsync(bundledRows); } if (logger.IsTraceEnabled) { var logStrings = String.Join(", ", strings); logger.Trace("String result column data: {0}", logStrings); } } else { throw new NotImplementedException(); } }
/// <summary> /// Return the results from connector to Qlik Engine /// </summary> private async Task GenerateResult(string request_type, MemoryStream returnedData, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, ServerCallContext context, int reqHash, bool failIfWrongDataTypeInFirstCol = false, DataType expectedFirstDataType = DataType.Numeric, bool cacheResultInQlik = true, ResultDataColumn keyField = null, string keyname = null, bool includeDetail = false, bool shouldExplain = false, bool rawExplain = false, int explain_max = 0) { int nrOfCols = 0; int nrOfRows = 0; List <ResultDataColumn> resultDataColumns = new List <ResultDataColumn>(); Logger.Info($"{reqHash} - Generate Results"); if (true) { Logger.Debug($"{reqHash} - Extract JSON"); //Convert the stream (json) to dictionary Logger.Info($"{reqHash} - Returned Datasize: {returnedData.Length}"); Logger.Info($"{reqHash} - Request Type: {request_type}"); StreamReader sr = new StreamReader(returnedData); returnedData.Position = 0; var data = sr.ReadToEnd(); //Dictionary<string, dynamic> response = JsonConvert.DeserializeObject<Dictionary<string, dynamic>>(data); ResponseSpecification response = JsonConvert.DeserializeObject <ResponseSpecification>(data); Logger.Trace($"{reqHash} - Returned Data: {data}"); if (response.data != null) { Logger.Trace($"{reqHash} - Response Data: {response.data}"); //Sort the response by RowId List <DataSpecification> sortedData = response.data.OrderBy(o => o.rowId).ToList(); //Return Raw Explain First So Works In Chart Expression if (request_type != "timeseries") { if (shouldExplain && rawExplain) { var pe = new ResultDataColumn(); pe.Name = $"Prediction Explanations"; pe.DataType = DataType.String; resultDataColumns.Add(pe); } } //Prediction Column var a = new ResultDataColumn(); a.Name = "Prediction"; a.DataType = DataType.String; //a.Strings = new List<string>(); resultDataColumns.Add(a); //Include Keyfield if (keyname != null) { resultDataColumns.Add(keyField); } //The first row will determine which fields to return in table for prediction values if (includeDetail == true) { foreach (PredictionValueSpecification pv in sortedData[0].predictionValues) { var pvi = new ResultDataColumn(); pvi.Name = $"Prediction value for label: {pv.label}"; pvi.DataType = DataType.String; resultDataColumns.Add(pvi); } } // Add Time Series Columns to resultData if (request_type == "timeseries") { //Row Id var rowId = new ResultDataColumn(); rowId.Name = "rowId"; rowId.DataType = DataType.String; //Time Series Columns var seriesId = new ResultDataColumn(); seriesId.Name = "seriesId"; seriesId.DataType = DataType.String; var forecastPoint = new ResultDataColumn(); forecastPoint.Name = "forecastPoint"; forecastPoint.DataType = DataType.String; var timestamp = new ResultDataColumn(); timestamp.Name = "timestamp"; timestamp.DataType = DataType.String; var forecastDistance = new ResultDataColumn(); forecastDistance.Name = "forecastDistance"; forecastDistance.DataType = DataType.String; var originalFormatTimestamp = new ResultDataColumn(); originalFormatTimestamp.Name = "originalFormatTimestamp"; originalFormatTimestamp.DataType = DataType.String; resultDataColumns.Add(rowId); resultDataColumns.Add(seriesId); resultDataColumns.Add(forecastPoint); resultDataColumns.Add(timestamp); resultDataColumns.Add(forecastDistance); resultDataColumns.Add(originalFormatTimestamp); } if (request_type != "timeseries") { if (shouldExplain && !rawExplain) { for (int j = 0; j < explain_max; j++) { foreach (string field in new[] { "label", "feature", "featureValue", "strength", "qualitativeStrength" }) { var pe = new ResultDataColumn(); pe.Name = $"PE_{j + 1}_{field}"; pe.DataType = DataType.String; resultDataColumns.Add(pe); } } } } nrOfRows = sortedData.Count; nrOfCols = resultDataColumns.Count; Logger.Debug($"{reqHash} - Result Number of Columns: {nrOfCols}"); await GenerateAndSendHeadersAsync(context, nrOfRows, nrOfCols, resultDataColumns, cacheResultInQlik); Logger.Trace($"{reqHash} - Start Loop"); // Send data var bundledRows = new BundledRows(); //Loop through each response in array (one for each row of input data) int i = 0; foreach (DataSpecification p in sortedData) { var row = new Row(); if (request_type != "timeseries") { if (shouldExplain && rawExplain) { if (p.predictionExplanations != null) { row.Duals.Add(new Dual() { StrData = JsonConvert.SerializeObject(p.predictionExplanations) ?? "" }); } else { row.Duals.Add(new Dual() { StrData = "[]" }); } } } //Prediction Column row.Duals.Add(new Dual() { StrData = Convert.ToString(p.prediction) ?? "" }); Logger.Trace($"{reqHash} - In Loop RowId: {p.rowId}"); //KeyField Column if (p.passthroughValues != null) { row.Duals.Add(new Dual() { StrData = Convert.ToString(p.passthroughValues[keyField.Name]) ?? "" }); } //Include Details if (includeDetail == true) { //Loop through each predicted value and insert the row values to the column foreach (PredictionValueSpecification pv in p.predictionValues) { row.Duals.Add(new Dual() { StrData = Convert.ToString(pv.value) ?? "" }); } } //Timeseries field if (request_type == "timeseries") { row.Duals.Add(new Dual() { StrData = Convert.ToString(p.rowId) ?? "" }); row.Duals.Add(new Dual() { StrData = Convert.ToString(p.seriesId) ?? "" }); row.Duals.Add(new Dual() { StrData = Convert.ToString(p.forecastPoint) ?? "" }); row.Duals.Add(new Dual() { StrData = Convert.ToString(p.timestamp) ?? "" }); row.Duals.Add(new Dual() { StrData = Convert.ToString(p.forecastDistance) ?? "" }); row.Duals.Add(new Dual() { StrData = Convert.ToString(p.originalFormatTimestamp) ?? "" }); } //Include Prediction Explanations if (shouldExplain && !rawExplain) { foreach (PredictionExplanationSpecification pe in p.predictionExplanations) { row.Duals.Add(new Dual() { StrData = Convert.ToString(pe.label) ?? "" }); row.Duals.Add(new Dual() { StrData = Convert.ToString(pe.feature) ?? "" }); row.Duals.Add(new Dual() { StrData = Convert.ToString(pe.featureValue) ?? "" }); row.Duals.Add(new Dual() { StrData = Convert.ToString(pe.strength) ?? "" }); row.Duals.Add(new Dual() { StrData = Convert.ToString(pe.qualitativeStrength) ?? "" }); } for (int j = p.predictionExplanations.Count; j < explain_max; j++) { row.Duals.Add(new Dual() { }); row.Duals.Add(new Dual() { }); row.Duals.Add(new Dual() { }); row.Duals.Add(new Dual() { }); row.Duals.Add(new Dual() { }); } } //Add the row the bundle bundledRows.Rows.Add(row); //When we get a batch of 2000 rows, send the response if (((i + 1) % 2000) == 0) { // Send a bundle await responseStream.WriteAsync(bundledRows); //Reset the bundle bundledRows = null; bundledRows = new BundledRows(); } i++; } //Send any left over rows after the final loop if (bundledRows.Rows.Count() > 0) { // Send last bundle await responseStream.WriteAsync(bundledRows); bundledRows = null; } } else if (response.response != null) { Logger.Trace($"{reqHash} - Processing Status Response for Project ID: {response.response.id}"); var a = new ResultDataColumn(); a.Name = "Result"; a.DataType = DataType.String; resultDataColumns.Add(a); await GenerateAndSendHeadersAsync(context, 1, 1, resultDataColumns, cacheResultInQlik); var bundledRows = new BundledRows(); var row = new Row(); row.Duals.Add(new Dual() { StrData = Convert.ToString(response.response.id) ?? "" }); bundledRows.Rows.Add(row); await responseStream.WriteAsync(bundledRows); bundledRows = null; } else { if (response.message != null) { throw new Exception($"The following error message was returned from DataRobot: {response.message}"); } else { throw new Exception($"An Unknown Error Occured: {data}"); } } } }