async Task <Sexp> EvaluateScriptInRserve(SexpList inputDataFrame, int reqHash, string rScript, RserveConnection rserveConn)
{
await semaphoreRserve.WaitAsync();
try
{
if (inputDataFrame != null && inputDataFrame.Count > 0)
{
rserveConn.Connection["q"] = inputDataFrame;
}
logger.Debug($"Evaluating R script, hashid ({reqHash}): {rScript}");
var res = rserveConn.Connection.Eval(rScript);
logger.Info($"Rserve result: {res.Count} rows, hashid ({reqHash})");
if (res.Count == 0)
{
HandleZeroRowsFromRserve(rserveConn);
}
return(res);
}
catch (Exception e)
{
HandleError(e, rserveConn);
return(null);
}
finally
{
semaphoreRserve.Release();
}
}
async Task <Sexp> EvaluateScriptInRserve(SexpList inputDataFrame, int reqHash, string rScript, RserveConnection rserveConn)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
await rserveConn.semaphoreRserve.WaitAsync();
stopwatch.Stop();
logger.Trace($"Wait For Connection Took {stopwatch.ElapsedMilliseconds} ms, hashid ({reqHash})");
try
{
if (inputDataFrame != null && inputDataFrame.Count > 0)
{
rserveConn.Connection["q"] = inputDataFrame;
}
logger.Debug($"Evaluating R script, hashid ({reqHash}): {rScript}");
var res = rserveConn.Connection.Eval(rScript);
logger.Info($"Rserve result: {res.Count} rows, hashid ({reqHash})");
if (res.Count == 0)
{
HandleZeroRowsFromRserve(rserveConn);
}
return(res);
}
catch (Exception e)
{
HandleError(e, rserveConn);
return(null);
}
finally
{
rserveConn.semaphoreRserve.Release();
}
}
public override async Task EvaluateScript(IAsyncStreamReader <global::Qlik.Sse.BundledRows> requestStream, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, ServerCallContext context)
{
ScriptRequestHeader scriptHeader;
CommonRequestHeader commonHeader;
RserveConnection rserveConn;
int reqHash = requestStream.GetHashCode();
if (!(capabilities.AllowScript))
{
throw new RpcException(new Status(StatusCode.PermissionDenied, $"Script evaluations disabled"));
}
try
{
rserveConn = connPool.GetConnection(rservePara);
var header = GetHeader(context.RequestHeaders, "qlik-scriptrequestheader-bin");
scriptHeader = ScriptRequestHeader.Parser.ParseFrom(header);
var commonRequestHeader = GetHeader(context.RequestHeaders, "qlik-commonrequestheader-bin");
commonHeader = CommonRequestHeader.Parser.ParseFrom(commonRequestHeader);
logger.Info($"EvaluateScript called from client ({context.Peer}), hashid ({reqHash})");
logger.Debug($"EvaluateScript header info: AppId ({commonHeader.AppId}), UserId ({commonHeader.UserId}), Cardinality ({commonHeader.Cardinality} rows)");
}
catch (Exception e)
{
logger.Error($"EvaluateScript with hashid ({reqHash}) failed: {e.Message}");
throw new RpcException(new Status(StatusCode.DataLoss, e.Message));
}
try
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var paramnames = $"EvaluateScript call with hashid({reqHash}) got Param names: ";
foreach (var param in scriptHeader.Params)
{
paramnames += $" {param.Name}";
}
logger.Info("{0}", paramnames);
SexpList inputDataFrame = null;
if (scriptHeader.Params != null && scriptHeader.Params.Count > 0)
{
inputDataFrame = await AddInputData(scriptHeader.Params.ToArray(), requestStream);
}
var rResult = await EvaluateScriptInRserve(inputDataFrame, reqHash, scriptHeader.Script, rserveConn);
// Disable caching (uncomment line below and comment next line if you do not want the results sent to Qlik to be cached in Qlik)
//await GenerateResult(rResult, responseStream, context, cacheResultInQlik: false);
await GenerateResult(rResult, responseStream, context);
stopwatch.Stop();
logger.Debug($"Took {stopwatch.ElapsedMilliseconds} ms, hashid ({reqHash})");
}
catch (Exception e)
{
throw new RpcException(new Status(StatusCode.InvalidArgument, $"{e.Message}"));
}
finally
{
//
}
}
public override async Task ExecuteFunction(IAsyncStreamReader <BundledRows> requestStream, IServerStreamWriter <BundledRows> responseStream, ServerCallContext context)
{
FunctionRequestHeader functionHeader;
CommonRequestHeader commonHeader;
RserveConnection rserveConn;
int reqHash = requestStream.GetHashCode();
Qlik.Sse.FunctionDefinition sseFunction;
DefinedFunctions.Function internalFunction;
if (nrOfDefinedFunctions == 0)
{
throw new RpcException(new Status(StatusCode.Unimplemented, $"No functions defined"));
}
try
{
rserveConn = connPool.GetConnection(rservePara);
var header = GetHeader(context.RequestHeaders, "qlik-functionrequestheader-bin");
functionHeader = FunctionRequestHeader.Parser.ParseFrom(header);
var commonRequestHeader = GetHeader(context.RequestHeaders, "qlik-commonrequestheader-bin");
commonHeader = CommonRequestHeader.Parser.ParseFrom(commonRequestHeader);
logger.Info($"ExecuteFunction: FunctionId ({functionHeader.FunctionId}), from client ({context.Peer}), hashid ({reqHash})");
logger.Debug($"ExecuteFunction header info: AppId ({commonHeader.AppId}), UserId ({commonHeader.UserId}), Cardinality ({commonHeader.Cardinality} rows)");
int funcIndex = definedFunctions.GetIndexOfFuncId(functionHeader.FunctionId);
if (funcIndex < 0)
{
throw new Exception($"FunctionId ({functionHeader.FunctionId}) is not a defined function");
}
sseFunction = definedFunctions.sseFunctions[funcIndex];
internalFunction = definedFunctions.funcDefs.functions[funcIndex];
}
catch (Exception e)
{
logger.Error($"ExecuteFunction with hashid ({reqHash}) failed: {e.Message}");
throw new RpcException(new Status(StatusCode.DataLoss, e.Message));
}
try
{
var stopwatch = new Stopwatch();
stopwatch.Start();
SexpList inputDataFrame = null;
if (sseFunction.Params.Count > 0)
{
inputDataFrame = await AddInputData(sseFunction.Params.ToArray(), requestStream);
}
var rResult = await EvaluateScriptInRserve(inputDataFrame, reqHash, internalFunction.FunctionRScript.Replace("\r", " "), rserveConn);
await GenerateResult(rResult, responseStream, context, true, sseFunction.ReturnType, internalFunction.CacheResultInQlik);
stopwatch.Stop();
logger.Debug($"Took {stopwatch.ElapsedMilliseconds} ms, hashid ({reqHash})");
}
catch (Exception e)
{
throw new RpcException(new Status(StatusCode.InvalidArgument, $"{e.Message}"));
}
finally
{
//
}
}
/// <summary>
/// Decode a Qap1-encoded Sexp
/// </summary>
/// <param name="data">The byte stream in which the Sexp is encoded</param>
/// <param name="start">At which index of data does the Sexp begin?</param>
/// <returns>The decoded Sexp.</returns>
private static Sexp DecodeSexp( byte[] data , ref long start )
{
// pull sexp type
byte xt = data[ start ];
// calculate length of payload
var lengthBuf = new byte[ 8 ];
Array.Copy( data , start + 1 , lengthBuf , 0 , 3 );
start += 4;
if ( ( xt & XtLarge ) == XtLarge )
{
Array.Copy( data , start , lengthBuf , 3 , 4 );
start += 4;
xt -= XtLarge;
}
var length = ( long )BitConverter.ToUInt64( lengthBuf , 0 );
// has attributes? process first
SexpTaggedList attrs = null;
if ( ( xt & XtHasAttr ) == XtHasAttr )
{
xt -= XtHasAttr;
long oldstart = start;
attrs = ( SexpTaggedList )DecodeSexp( data , ref start );
length -= start - oldstart;
}
long end = start + length;
Sexp result;
switch ( xt )
{
case XtNull:
{
if ( length != 0 )
{
throw new RserveException( "Attempting to decode an SexpNull, but it is followed by data when it shouldn't be." );
}
result = new SexpNull();
}
break;
case XtSymName:
{
// keep all characters up to the first null
var symnNamBuf = new byte[ length ];
Array.Copy( data , start , symnNamBuf , 0 , length );
string res = Encoding.UTF8.GetString( symnNamBuf );
result = new SexpSymname( res.Split( '\x00' )[ 0 ] );
}
break;
case XtArrayInt:
{
var res = new int[ length / 4 ];
var intBuf = new byte[ 4 ];
for ( long i = 0 ; i < length ; i += 4 )
{
Array.Copy( data , start + i , intBuf , 0 , 4 );
res[ i / 4 ] = BitConverter.ToInt32( intBuf , 0 );
}
// is date or just an integer?
if ( ( attrs != null ) && ( attrs.ContainsKey( "class" ) && attrs[ "class" ].AsStrings.Contains( "Date" ) ) )
{
result = new SexpArrayDate( res );
}
else
{
result = new SexpArrayInt( res );
}
}
break;
case XtArrayBool:
{
if ( length < 4 )
{
throw new RserveException( "Decoding an SexpArrayBool where data doesn't seem to contain a data length field." );
}
var boolLengthBuf = new byte[ 4 ];
Array.Copy( data , start , boolLengthBuf , 0 , 4 );
var datalength = BitConverter.ToInt32( boolLengthBuf , 0 );
if ( datalength > length - 4 )
{
throw new RserveException( "Decoding an SexpArrayBool where transmitted data field too short for number of entries." );
}
var res = new bool?[ datalength ];
for ( int i = 0 ; i < datalength ; i++ )
{
// R logical is false if 0, true if 1, and NA if 2
switch ( data[ start + i + 4 ] )
{
case 0:
res[ i ] = false;
break;
case 1:
res[ i ] = true;
break;
case 2:
res[ i ] = null;
break;
default:
throw new RserveException( "Decoding an SexpArrayBool and found an element in the array that is not an R bool: " + data[ start + i + 4 ] );
}
}
result = new SexpArrayBool( res );
}
break;
case XtArrayDouble:
{
var res = new double[ length / 8 ];
var doubleBuf = new byte[ 8 ];
for ( long i = 0 ; i < length ; i += 8 )
{
Array.Copy( data , start + i , doubleBuf , 0 , 8 );
res[ i / 8 ] = BitConverter.ToDouble( doubleBuf , 0 );
}
// is date or just a double?
if ( ( attrs != null ) && ( attrs.ContainsKey( "class" ) && attrs[ "class" ].AsStrings.Contains( "Date" ) ) )
{
result = new SexpArrayDate( res.Select( Convert.ToInt32 ) );
}
else
{
result = new SexpArrayDouble( res );
}
}
break;
case XtArrayString:
{
var res = new List<string>();
long i = 0;
for ( long j = 0 ; j < length ; j++ )
{
if ( data[ start + j ] != 0 )
{
continue;
}
if ( ( j == i + 1 ) && ( data[ start + i ] == 255 ) )
{
res.Add( null );
}
else
{
if ( data[ start + i ] == 255 )
{
i++;
}
var stringBuf = new byte[ j - i ];
Array.Copy( data , start + i , stringBuf , 0 , j - i );
res.Add( Encoding.UTF8.GetString( stringBuf ) );
}
i = j + 1;
}
result = new SexpArrayString( res );
}
break;
case XtListNoTag:
case XtLangNoTag:
case XtVector:
result = new SexpList();
while ( start < end )
{
result.Add( DecodeSexp( data , ref start ) );
}
break;
case XtLangTag:
case XtListTag:
result = new SexpTaggedList();
while ( start < end )
{
Sexp val = DecodeSexp( data , ref start );
Sexp key = DecodeSexp( data , ref start );
result.Add( key.IsNull ? String.Empty : key.AsString , val );
}
break;
case XtRaw:
{
var d = new byte[ length ];
Array.Copy( data , start , d , 0 , length );
result = new SexpQap1Raw( xt , d );
}
break;
default:
throw new RserveException( "Cannot decode an Sexp because the type is not recognized: " + xt );
}
if ( start > end )
{
throw new RserveException( "When decoding an Sexp, more data consumed than provided." );
}
start = end;
if ( attrs != null )
{
foreach ( var a in attrs.AsSexpDictionary )
{
result.Attributes.Add( a.Key , a.Value );
}
}
return result;
}
