/// <summary> /// All requests are processed through evaluate script, however in the context of this connector, the script is a JSON notation string which contains the metadata required to correctly process the attached data. /// </summary> public override async Task EvaluateScript(IAsyncStreamReader <global::Qlik.Sse.BundledRows> requestStream, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, ServerCallContext context) { ScriptRequestHeader scriptHeader; CommonRequestHeader commonHeader; Qlik2DataRobotMetrics.RequestCounter.Inc(); int reqHash = requestStream.GetHashCode(); try { var header = GetHeader(context.RequestHeaders, "qlik-scriptrequestheader-bin"); scriptHeader = ScriptRequestHeader.Parser.ParseFrom(header); var commonRequestHeader = GetHeader(context.RequestHeaders, "qlik-commonrequestheader-bin"); commonHeader = CommonRequestHeader.Parser.ParseFrom(commonRequestHeader); Logger.Info($"{reqHash} - EvaluateScript called from client ({context.Peer}), hashid ({reqHash})"); Logger.Debug($"{reqHash} - EvaluateScript header info: AppId ({commonHeader.AppId}), UserId ({commonHeader.UserId}), Cardinality ({commonHeader.Cardinality} rows)"); } catch (Exception e) { Logger.Error($"EvaluateScript with hashid ({reqHash}) failed: {e.Message}"); throw new RpcException(new Status(StatusCode.DataLoss, e.Message)); } try { var stopwatch = new Stopwatch(); stopwatch.Start(); var paramnames = $"{reqHash} - EvaluateScript call with hashid({reqHash}) got Param names: "; foreach (var param in scriptHeader.Params) { paramnames += $" {param.Name}"; } Logger.Trace("{0}", paramnames); Logger.Trace(scriptHeader.Script); RequestSpecification config = JsonConvert.DeserializeObject <RequestSpecification>(scriptHeader.Script); var Params = GetParams(scriptHeader.Params.ToArray()); string keyname = null; if (config.keyfield != null) { keyname = config.keyfield; } ResultDataColumn keyField = new ResultDataColumn(); var rowdatastream = await ConvertBundledRowsToCSV(Params, requestStream, context, keyField, keyname, config.timestamp_field, config.timestamp_format); Logger.Debug($"{reqHash} - Input Data Size: {rowdatastream.Length}"); var outData = await SelectFunction(config, rowdatastream, reqHash); rowdatastream = null; bool shouldCache = config.should_cache; bool inc_details = config.inc_details; bool rawExplain = false; bool shouldExplain = false; int max_codes = 0; if (config.explain != null) { shouldExplain = true; rawExplain = config.explain.return_raw; max_codes = config.explain.max_codes; } string request_type = config.request_type; await GenerateResult(request_type, outData, responseStream, context, reqHash, cacheResultInQlik : shouldCache, keyField : keyField, keyname : keyname, includeDetail : inc_details, shouldExplain : shouldExplain, rawExplain : rawExplain, explain_max : max_codes); outData = null; stopwatch.Stop(); Logger.Debug($"{reqHash} - Took {stopwatch.ElapsedMilliseconds} ms, hashid ({reqHash})"); Qlik2DataRobotMetrics.DurHist.Observe(stopwatch.ElapsedMilliseconds / 1000); } catch (Exception e) { Logger.Error($"{reqHash} - ERROR: {e.Message}"); throw new RpcException(new Status(StatusCode.InvalidArgument, $"{e.Message}")); } finally { } GC.Collect(); }
/// <summary> /// Select the functiona based upon the request specification /// </summary> private async Task <MemoryStream> SelectFunction(RequestSpecification config, MemoryStream rowdatastream, int reqHash) { Logger.Info($"{reqHash} - Start DataRobot"); DataRobotRestRequest dr = new DataRobotRestRequest(reqHash); string api_token = Convert.ToString(config.auth_config.api_token); string datarobot_key = config.auth_config.datarobot_key; string host = config.auth_config.endpoint; string project_id = config.project_id; string model_id = config.model_id; string deployment_id = config.deployment_id; string keyField = config.keyfield; MemoryStream result = new MemoryStream(); switch (config.request_type) { case "createproject": Logger.Info($"{reqHash} - Create Project"); string project_name = Convert.ToString(config.project_name); var zippedstream = await CompressStream(rowdatastream, project_name, reqHash); Logger.Info($"{reqHash} - Zipped Data Size: {zippedstream.Length}"); string endpoint = Convert.ToString(config.auth_config.endpoint); if (endpoint.Substring(endpoint.Length - 2) != "/") { endpoint = endpoint + "/"; } result = await dr.CreateProjectsAsync(endpoint, api_token, zippedstream, project_name, project_name + ".zip"); break; case "predictapi": Logger.Info($"{reqHash} - Predict API"); int maxCodes = 0; double thresholdHigh = 0; double thresholdLow = 0; bool explain = false; if (config.explain != null) { maxCodes = config.explain.max_codes; thresholdHigh = config.explain.threshold_high; thresholdLow = config.explain.threshold_low; explain = true; } result = await dr.PredictApiAsync(rowdatastream, api_token, datarobot_key, host, deployment_id : deployment_id, project_id : project_id, model_id : model_id, keyField : keyField, explain : explain, maxCodes : maxCodes, thresholdHigh : thresholdHigh, thresholdLow : thresholdLow); break; case "timeseries": Logger.Info($"{reqHash} - Time Series Prediction API"); string forecast_point = null; if (config.forecast_point != null) { forecast_point = Convert.ToString(config.forecast_point); //forecast_point = config.forecast_point.ToString("s"); } result = await dr.TimeSeriesAsync(rowdatastream, api_token, datarobot_key, host, deployment_id : deployment_id, project_id : project_id, model_id : model_id, forecast_point : forecast_point); break; default: break; } Logger.Info($"{reqHash} - DataRobot Finish"); return(result); }