public override async Task EvaluateScript(IAsyncStreamReader <global::Qlik.Sse.BundledRows> requestStream, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, ServerCallContext context)
{
ScriptRequestHeader scriptHeader;
CommonRequestHeader commonHeader;
RserveConnection rserveConn;
int reqHash = requestStream.GetHashCode();
if (!(capabilities.AllowScript))
{
throw new RpcException(new Status(StatusCode.PermissionDenied, $"Script evaluations disabled"));
}
try
{
rserveConn = connPool.GetConnection(rservePara);
var header = GetHeader(context.RequestHeaders, "qlik-scriptrequestheader-bin");
scriptHeader = ScriptRequestHeader.Parser.ParseFrom(header);
var commonRequestHeader = GetHeader(context.RequestHeaders, "qlik-commonrequestheader-bin");
commonHeader = CommonRequestHeader.Parser.ParseFrom(commonRequestHeader);
logger.Info($"EvaluateScript called from client ({context.Peer}), hashid ({reqHash})");
logger.Debug($"EvaluateScript header info: AppId ({commonHeader.AppId}), UserId ({commonHeader.UserId}), Cardinality ({commonHeader.Cardinality} rows)");
}
catch (Exception e)
{
logger.Error($"EvaluateScript with hashid ({reqHash}) failed: {e.Message}");
throw new RpcException(new Status(StatusCode.DataLoss, e.Message));
}
try
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var paramnames = $"EvaluateScript call with hashid({reqHash}) got Param names: ";
foreach (var param in scriptHeader.Params)
{
paramnames += $" {param.Name}";
}
logger.Info("{0}", paramnames);
SexpList inputDataFrame = null;
if (scriptHeader.Params != null && scriptHeader.Params.Count > 0)
{
inputDataFrame = await AddInputData(scriptHeader.Params.ToArray(), requestStream);
}
var rResult = await EvaluateScriptInRserve(inputDataFrame, reqHash, scriptHeader.Script, rserveConn);
// Disable caching (uncomment line below and comment next line if you do not want the results sent to Qlik to be cached in Qlik)
//await GenerateResult(rResult, responseStream, context, cacheResultInQlik: false);
await GenerateResult(rResult, responseStream, context);
stopwatch.Stop();
logger.Debug($"Took {stopwatch.ElapsedMilliseconds} ms, hashid ({reqHash})");
}
catch (Exception e)
{
throw new RpcException(new Status(StatusCode.InvalidArgument, $"{e.Message}"));
}
finally
{
//
}
}
/// <summary>
/// All requests are processed through evaluate script, however in the context of this connector, the script is a JSON notation string which contains the metadata required to correctly process the attached data.
/// </summary>
public override async Task EvaluateScript(IAsyncStreamReader <global::Qlik.Sse.BundledRows> requestStream, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, ServerCallContext context)
{
ScriptRequestHeader scriptHeader;
CommonRequestHeader commonHeader;
Qlik2DataRobotMetrics.RequestCounter.Inc();
int reqHash = requestStream.GetHashCode();
try
{
var header = GetHeader(context.RequestHeaders, "qlik-scriptrequestheader-bin");
scriptHeader = ScriptRequestHeader.Parser.ParseFrom(header);
var commonRequestHeader = GetHeader(context.RequestHeaders, "qlik-commonrequestheader-bin");
commonHeader = CommonRequestHeader.Parser.ParseFrom(commonRequestHeader);
Logger.Info($"{reqHash} - EvaluateScript called from client ({context.Peer}), hashid ({reqHash})");
Logger.Debug($"{reqHash} - EvaluateScript header info: AppId ({commonHeader.AppId}), UserId ({commonHeader.UserId}), Cardinality ({commonHeader.Cardinality} rows)");
}
catch (Exception e)
{
Logger.Error($"EvaluateScript with hashid ({reqHash}) failed: {e.Message}");
throw new RpcException(new Status(StatusCode.DataLoss, e.Message));
}
try
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var paramnames = $"{reqHash} - EvaluateScript call with hashid({reqHash}) got Param names: ";
foreach (var param in scriptHeader.Params)
{
paramnames += $" {param.Name}";
}
Logger.Trace("{0}", paramnames);
Logger.Trace(scriptHeader.Script);
Dictionary <string, dynamic> config = JsonConvert.DeserializeObject <Dictionary <string, dynamic> >(scriptHeader.Script);
var Params = GetParams(scriptHeader.Params.ToArray());
string keyname = null;
if (config.ContainsKey("keyfield"))
{
keyname = Convert.ToString(config["keyfield"]);
}
ResultDataColumn keyField = new ResultDataColumn();
var rowdatastream = await ConvertBundledRowsToCSV(Params, requestStream, context, keyField, keyname);
Logger.Debug($"{reqHash} - Input Data Size: {rowdatastream.Length}");
var outData = await SelectFunction(config, rowdatastream, reqHash);
rowdatastream = null;
bool shouldCache = false;
if (config.ContainsKey("should_cache"))
{
shouldCache = config["should_cache"];
}
bool inc_details = false;
bool rawExplain = false;
if (config.ContainsKey("inc_details"))
{
inc_details = config["inc_details"];
}
if (config.ContainsKey("explain"))
{
rawExplain = config["explain"]["return_raw"];
}
string request_type = config["request_type"];
await GenerateResult(request_type, outData, responseStream, context, reqHash, cacheResultInQlik : shouldCache, keyField : keyField, keyname : keyname, includeDetail : inc_details, rawExplain : rawExplain);
outData = null;
stopwatch.Stop();
Logger.Debug($"{reqHash} - Took {stopwatch.ElapsedMilliseconds} ms, hashid ({reqHash})");
Qlik2DataRobotMetrics.DurHist.Observe(stopwatch.ElapsedMilliseconds / 1000);
}
catch (Exception e)
{
Logger.Error($"{reqHash} - ERROR: {e.Message}");
throw new RpcException(new Status(StatusCode.InvalidArgument, $"{e.Message}"));
}
finally
{
}
GC.Collect();
}
public override async Task ExecuteFunction(IAsyncStreamReader <BundledRows> requestStream, IServerStreamWriter <BundledRows> responseStream, ServerCallContext context)
{
FunctionRequestHeader functionHeader;
CommonRequestHeader commonHeader;
RserveConnection rserveConn;
int reqHash = requestStream.GetHashCode();
Qlik.Sse.FunctionDefinition sseFunction;
DefinedFunctions.Function internalFunction;
if (nrOfDefinedFunctions == 0)
{
throw new RpcException(new Status(StatusCode.Unimplemented, $"No functions defined"));
}
try
{
rserveConn = connPool.GetConnection(rservePara);
var header = GetHeader(context.RequestHeaders, "qlik-functionrequestheader-bin");
functionHeader = FunctionRequestHeader.Parser.ParseFrom(header);
var commonRequestHeader = GetHeader(context.RequestHeaders, "qlik-commonrequestheader-bin");
commonHeader = CommonRequestHeader.Parser.ParseFrom(commonRequestHeader);
logger.Info($"ExecuteFunction: FunctionId ({functionHeader.FunctionId}), from client ({context.Peer}), hashid ({reqHash})");
logger.Debug($"ExecuteFunction header info: AppId ({commonHeader.AppId}), UserId ({commonHeader.UserId}), Cardinality ({commonHeader.Cardinality} rows)");
int funcIndex = definedFunctions.GetIndexOfFuncId(functionHeader.FunctionId);
if (funcIndex < 0)
{
throw new Exception($"FunctionId ({functionHeader.FunctionId}) is not a defined function");
}
sseFunction = definedFunctions.sseFunctions[funcIndex];
internalFunction = definedFunctions.funcDefs.functions[funcIndex];
}
catch (Exception e)
{
logger.Error($"ExecuteFunction with hashid ({reqHash}) failed: {e.Message}");
throw new RpcException(new Status(StatusCode.DataLoss, e.Message));
}
try
{
var stopwatch = new Stopwatch();
stopwatch.Start();
SexpList inputDataFrame = null;
if (sseFunction.Params.Count > 0)
{
inputDataFrame = await AddInputData(sseFunction.Params.ToArray(), requestStream);
}
var rResult = await EvaluateScriptInRserve(inputDataFrame, reqHash, internalFunction.FunctionRScript.Replace("\r", " "), rserveConn);
await GenerateResult(rResult, responseStream, context, true, sseFunction.ReturnType, internalFunction.CacheResultInQlik);
stopwatch.Stop();
logger.Debug($"Took {stopwatch.ElapsedMilliseconds} ms, hashid ({reqHash})");
}
catch (Exception e)
{
throw new RpcException(new Status(StatusCode.InvalidArgument, $"{e.Message}"));
}
finally
{
//
}
}
/// <summary>
/// Convert the input data into a CSV file within memory stream
/// </summary>
private async Task <MemoryStream> ConvertBundledRowsToCSV(ParameterData[] Parameters, IAsyncStreamReader <global::Qlik.Sse.BundledRows> requestStream, ServerCallContext context, ResultDataColumn keyField, string keyname)
{
int reqHash = requestStream.GetHashCode();
Logger.Debug($"{reqHash} - Start Create CSV");
var memStream = new MemoryStream();
var streamWriter = new StreamWriter(memStream);
var tw = TextWriter.Synchronized(streamWriter);
var csv = new CsvWriter(tw);
var keyindex = 0;
for (int i = 0; i < Parameters.Length; i++)
{
var param = Parameters[i];
if (keyname != null)
{
if (param.ParamName == keyname)
{
keyindex = i;
keyField.Name = param.ParamName;
keyField.DataType = param.DataType;
switch (param.DataType)
{
case DataType.Numeric:
case DataType.Dual:
keyField.Numerics = new List <double>();
break;
case DataType.String:
keyField.Strings = new List <string>();
break;
}
}
}
csv.WriteField(param.ParamName);
}
if (keyField.Name == null && keyname != null)
{
throw new Exception("The keyfield was not found in the source data, please ensure you are including this field in the dataset sent from Qlik.");
}
csv.NextRecord();
Logger.Debug($"{reqHash} - Finished Header");
int a = 0;
while (await requestStream.MoveNext())
{
foreach (var Row in requestStream.Current.Rows)
{
for (int i = 0; i < Parameters.Length; i++)
{
var param = Parameters[i];
var dual = Row.Duals[i];
switch (param.DataType)
{
case DataType.Numeric:
if (keyindex == i && keyname != null)
{
keyField.Numerics.Add(dual.NumData);
}
csv.WriteField(dual.NumData.ToString());
break;
case DataType.String:
if (keyindex == i && keyname != null)
{
keyField.Strings.Add(dual.StrData);
}
csv.WriteField(dual.StrData);
break;
case DataType.Dual:
if (keyindex == i && keyname != null)
{
keyField.Numerics.Add(dual.NumData);
}
csv.WriteField(dual.NumData.ToString());
break;
}
}
a++;
csv.NextRecord();
}
}
csv.Flush();
tw.Flush();
streamWriter.Flush();
memStream.Flush();
memStream.Position = 0;
Logger.Debug($"{reqHash} - Rows" + a);
if (a == 0)
{
throw new Exception("There were no rows in the table sent from Qlik. Check that the table has at least 1 row of data.");
}
return(await Task.FromResult(memStream));
}