public RserveConnection GetConnection(RserveParameter rserveParams)
{
Interlocked.Increment(ref poolcounter);
int poolnodeindex = poolcounter % (rserveParams.PoolNodes.Count);
PoolNode node = rserveParams.PoolNodes.ElementAt(poolnodeindex);
var hash = $"{node.rserveParameter.GetHashCode().ToString()}-{node.getNextConnectionNumber().ToString()}";
logger.Info($"Connecting to node {poolnodeindex} with hash: {hash}");
RserveConnection conn = null;
if (rserveConns.ContainsKey(hash))
{
rserveConns.TryGetValue(hash, out conn);
var dateTime = DateTime.MinValue;
if (lastUsed.TryGetValue(hash, out dateTime))
{
lastUsed.TryUpdate(hash, DateTime.Now, dateTime);
}
}
else
{
conn = new RserveConnection(node.rserveParameter);
Thread.Sleep(500);
rserveConns.TryAdd(hash, conn);
lastUsed.TryAdd(hash, DateTime.Now);
}
return(conn);
}
private void HandleError(Exception ex, RserveConnection rserveConn)
{
String msg;
try
{
msg = rserveConn.Connection.Eval("geterrmessage()").AsString;
if (String.IsNullOrWhiteSpace(msg))
{
msg = ex.Message;
}
logger.Warn("Rserve error: {0}", msg);
}
catch
{
logger.Warn("Rserve error: {0}", ex.Message);
throw new RpcException(new Status(StatusCode.Unavailable, $"Rserve error: {ex.Message}"));
}
// Try to get the stack trace
// It's possible that geterrmessage() succeeds and traceback() fails.
try
{
var tracebacks = rserveConn.Connection.Eval("traceback()").AsStrings;
var traceback = String.Join("\r\n", tracebacks);
logger.Warn("Rserve Traceback: {0}", traceback);
}
catch
{
// Error msg already logged
}
throw new RpcException(new Status(StatusCode.InvalidArgument, $"Rserve error: {msg}"));
}
private void HandleZeroRowsFromRserve(RserveConnection rserveConn)
{
String msg = $"No data returned from R script execution. Possible error in script: ";
try
{
String errMsg = rserveConn.Connection.Eval("geterrmessage()").AsString;
if (!String.IsNullOrWhiteSpace(errMsg))
{
msg = msg + errMsg;
}
logger.Warn("{0}", msg);
}
catch
{
throw new RpcException(new Status(StatusCode.Unknown, $"{msg}"));
}
// Try to get the stack trace
// It's possible that geterrmessage() succeeds and traceback() fails.
try
{
var tracebacks = rserveConn.Connection.Eval("traceback()").AsStrings;
var traceback = String.Join("\r\n", tracebacks);
logger.Warn("Rserve Traceback when no data returned from R script execution: {0}", traceback);
}
catch
{
// Error msg already logged
}
throw new RpcException(new Status(StatusCode.InvalidArgument, $"{msg}"));
}
public RServeEvaluator(RserveParameter para)
{
connPool = new RserveConnectionPool();
rservePara = para;
RserveConnection rserveConnInitial = connPool.GetConnection(rservePara);
logger.Trace($"Rserve connection initiated: {rserveConnInitial!=null}");
}
public RServeEvaluator(RserveParameter para, bool enableScript, string functionDefinitionsFile)
{
allowScript = enableScript;
if (!String.IsNullOrEmpty(functionDefinitionsFile))
{
definedFunctions = new DefinedFunctions(functionDefinitionsFile);
}
CreateCapabilities();
connPool = new RserveConnectionPool();
rservePara = para;
RserveConnection rserveConnInitial = connPool.GetConnection(rservePara);
logger.Trace($"Rserve connection initiated: {rserveConnInitial != null}");
}
public RserveConnection GetConnection(RserveParameter rserveParams)
{
var hash = rserveParams.GetHashCode().ToString();
RserveConnection conn = null;
if (rserveConns.ContainsKey(hash))
{
rserveConns.TryGetValue(hash, out conn);
var dateTime = DateTime.MinValue;
if (lastUsed.TryGetValue(hash, out dateTime))
{
lastUsed.TryUpdate(hash, DateTime.Now, dateTime);
}
}
else
{
conn = new RserveConnection(rserveParams);
Thread.Sleep(500);
rserveConns.TryAdd(hash, conn);
lastUsed.TryAdd(hash, DateTime.Now);
}
return(conn);
}
async Task <Sexp> EvaluateScriptInRserve(SexpList inputDataFrame, int reqHash, string rScript, RserveConnection rserveConn)
{
await semaphoreRserve.WaitAsync();
try
{
if (inputDataFrame != null && inputDataFrame.Count > 0)
{
rserveConn.Connection["q"] = inputDataFrame;
}
logger.Debug($"Evaluating R script, hashid ({reqHash}): {rScript}");
var res = rserveConn.Connection.Eval(rScript);
logger.Info($"Rserve result: {res.Count} rows, hashid ({reqHash})");
if (res.Count == 0)
{
HandleZeroRowsFromRserve(rserveConn);
}
return(res);
}
catch (Exception e)
{
HandleError(e, rserveConn);
return(null);
}
finally
{
semaphoreRserve.Release();
}
}
async Task <Sexp> EvaluateScriptInRserve(SexpList inputDataFrame, int reqHash, string rScript, RserveConnection rserveConn)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
await rserveConn.semaphoreRserve.WaitAsync();
stopwatch.Stop();
logger.Trace($"Wait For Connection Took {stopwatch.ElapsedMilliseconds} ms, hashid ({reqHash})");
try
{
if (inputDataFrame != null && inputDataFrame.Count > 0)
{
rserveConn.Connection["q"] = inputDataFrame;
}
logger.Debug($"Evaluating R script, hashid ({reqHash}): {rScript}");
var res = rserveConn.Connection.Eval(rScript);
logger.Info($"Rserve result: {res.Count} rows, hashid ({reqHash})");
if (res.Count == 0)
{
HandleZeroRowsFromRserve(rserveConn);
}
return(res);
}
catch (Exception e)
{
HandleError(e, rserveConn);
return(null);
}
finally
{
rserveConn.semaphoreRserve.Release();
}
}
private async Task GenerateResult(Sexp RResult, IServerStreamWriter <global::Qlik.Sse.BundledRows> responseStream, RserveConnection rserveConn)
{
if (RResult is SexpArrayBool || RResult is SexpArrayDouble || RResult is SexpArrayInt)
{
var bundledRows = new BundledRows();
var numerics = RResult.AsDoubles;
if (numerics.Length == 0)
{
HandleZeroRowsFromRserve(rserveConn);
}
for (int i = 0; i < numerics.Length; i++)
{
var row = new Row();
row.Duals.Add(new Dual()
{
NumData = numerics[i]
});
bundledRows.Rows.Add(row);
if ((i % 2000) == 0)
{
// Send a bundle
await responseStream.WriteAsync(bundledRows);
bundledRows = new BundledRows();
}
}
if (bundledRows.Rows.Count() > 0)
{
// Send last bundle
await responseStream.WriteAsync(bundledRows);
}
if (logger.IsTraceEnabled)
{
var logNumerics = String.Join(", ", numerics);
logger.Trace("Numeric result column data: {0}", logNumerics);
}
}
else if (RResult is SexpArrayString)
{
var bundledRows = new BundledRows();
var strings = RResult.AsStrings;
if (strings.Length == 0)
{
HandleZeroRowsFromRserve(rserveConn);
}
for (int i = 0; i < strings.Length; i++)
{
var row = new Row();
row.Duals.Add(new Dual()
{
StrData = strings[i] ?? ""
});
bundledRows.Rows.Add(row);
if ((i % 2000) == 0)
{
// Send a bundle
await responseStream.WriteAsync(bundledRows);
bundledRows = new BundledRows();
}
}
if (bundledRows.Rows.Count() > 0)
{
// Send last bundle
await responseStream.WriteAsync(bundledRows);
}
if (logger.IsTraceEnabled)
{
var logStrings = String.Join(", ", strings);
logger.Trace("String result column data: {0}", logStrings);
}
}
else
{
throw new NotImplementedException();
}
}
async Task AddInputData(Parameter[] Parameters, IAsyncStreamReader <global::Qlik.Sse.BundledRows> requestStream, RserveConnection rserveConn)
{
var Params = GetParams(Parameters);
await ConvertToColumnar(Params, requestStream);
var data = new List <KeyValuePair <string, object> >();
for (int i = 0; i < Params.Length; i++)
{
var s = GenerateData(Params[i]);
data.Add(new KeyValuePair <string, object>(Parameters[i].Name, s));
}
rserveConn.Connection["q"] = Sexp.MakeDataFrame(data);
}