private void StopManagedTask(ManagedTask task)
{
task.InternalStatus = InternalTaskStatus.STOPSTART;
var stopRequest = new QlikRequest
{
ManagedTaskId = task.Id.ToString()
};
var stopFunction = new StopFunction(runtimeOptions);
stopFunction.StopReportJobs(stopRequest, true);
}
public void StopReportJobs(QlikRequest request, bool isTimeout = false)
{
Task.Run(() =>
{
try
{
if (request.ManagedTaskId == "all")
{
logger.Debug("All tasks will be stopped...");
if (Options.TaskPool.ManagedTasks.IsEmpty)
{
logger.Warn("No stopping jobs.");
}
var managedTasks = Options.TaskPool.ManagedTasks?.Values?.ToList() ?? new List <ManagedTask>();
foreach (var managedTask in managedTasks)
{
if (managedTask.Status >= 0 && managedTask.Status <= 2)
{
StopReportJob(managedTask.Id, isTimeout);
}
else
{
logger.Warn($"The task '{managedTask.Id}' has a status that cannot be stopped.");
}
}
}
else if (request.ManagedTaskId != null)
{
logger.Debug($"Managed task '{request.ManagedTaskId}' will be stopped...");
StopReportJob(new Guid(request.ManagedTaskId), isTimeout);
}
else
{
logger.Warn($"The task id of the task to be stopped is empty.");
}
}
catch (Exception ex)
{
logger.Error(ex, "The stop function has an unknown error.");
}
}, Options.Cancellation.Token);
}
public QlikResponse FormatJobResult(QlikRequest request)
{
var result = new QlikResponse();
try
{
if (request.ManagedTaskId != null)
{
logger.Debug($"Formatting for Qlik will be created by Task '{request.ManagedTaskId}'.");
var managedTaskId = new Guid(request.ManagedTaskId);
var resultQlikTask = Options.TaskPool.ManagedTasks.Values.ToList().FirstOrDefault(t => t.Id == managedTaskId);
if (resultQlikTask != null)
{
var distibuteJson = resultQlikTask.DistributeResult;
if (distibuteJson != null)
{
result.FormatedResult = GetFormatedJsonForQlik(distibuteJson);
logger.Debug("The delivery result was successfully formatted.");
return(result);
}
else
{
logger.Warn("The delivery result is null.");
}
}
else
{
logger.Warn($"No managed task id '{managedTaskId}' for formated Qlik result found.");
}
}
else
{
logger.Warn("No task id for formated Qlik result found.");
}
}
catch (Exception ex)
{
result.SetErrorMessage(ex);
logger.Error(ex, "The result function has an unknown error.");
}
return(null);
}
public SessionInfo GetSession(SerConnection connection, QlikRequest request)
{
try
{
lock (threadObject)
{
var uri = connection.ServerUri;
var oldSession = Sessions?.FirstOrDefault(u => u.ConnectUri.OriginalString == uri.OriginalString &&
u.User.ToString() == request.QlikUser.ToString() &&
u.AppId == request.AppId) ?? null;
if (oldSession != null)
{
logger.Debug("Old session found...");
var result = ValidateSession(oldSession);
if (result)
{
logger.Debug("Old session is working...");
return(oldSession);
}
logger.Debug("Old session not working, i remove it...");
Sessions.Remove(oldSession);
}
}
logger.Debug("Create new Qlik Session...");
var sessionInfo = Manager.CreateNewSession(connection, request.QlikUser, request.AppId);
if (sessionInfo != null)
{
Sessions.Add(sessionInfo);
}
return(sessionInfo);
}
catch (Exception ex)
{
logger.Error(ex, "The session could not recreated or reused.");
return(null);
}
}
private JArray GetAllowedTasks(List <ManagedTask> activeTasks, QlikRequest request)
{
var results = new JArray();
try
{
Options.Analyser?.SetCheckPoint("GetAllowedTasks", $"Start - Find all running tasks");
var session = Options.SessionHelper.GetSession(Options.Config.Connection, request);
var qrsHub = new QlikQrsHub(session.ConnectUri, session.Cookie);
foreach (var activeTask in activeTasks)
{
if (activeTask.Status == 4)
{
continue;
}
var appOwner = request.GetAppOwner(qrsHub, activeTask.Session.AppId);
if (appOwner != null && appOwner.ToString() == activeTask.Session.User.ToString())
{
logger.Debug($"The app owner '{appOwner}' was found.");
results.Add(CreateTaskResult(activeTask));
}
else
{
if (HasReadRights(qrsHub, activeTask))
{
results.Add(CreateTaskResult(activeTask));
}
}
}
Options.Analyser?.SetCheckPoint("GetAllowedTasks", $"End - Find all running tasks");
}
catch (Exception ex)
{
logger.Error(ex, "The list of tasks could not be determined.");
}
return(results);
}
private SerConfig CreateEngineConfig(QlikRequest request, SessionInfo session)
{
logger.Debug($"Resolve script '{request.JsonScript}' for engine config...");
var jsonStr = request.JsonScript;
//Make full JSON for Engine
logger.Debug("Auto replacement to normal json structure...");
if (!jsonStr.ToLowerInvariant().Contains("\"reports\":"))
{
jsonStr = $"\"reports\":[{jsonStr}]";
}
if (!jsonStr.ToLowerInvariant().Contains("\"tasks\":"))
{
jsonStr = $"\"tasks\":[{{{jsonStr}}}]";
}
if (!jsonStr.Trim().StartsWith("{"))
{
jsonStr = $"{{{jsonStr}";
}
if (!jsonStr.Trim().EndsWith("}"))
{
jsonStr = $"{jsonStr}}}";
}
logger.Debug("Search for connections for the engine config...");
dynamic serJsonConfig = JObject.Parse(jsonStr);
var tasks = serJsonConfig?.tasks ?? new JArray();
foreach (var task in tasks)
{
var reports = task?.reports ?? new JArray();
foreach (var report in reports)
{
var userConnections = new JArray();
JToken connections = report?.connections;
if (connections.Type == JTokenType.Object)
{
userConnections.Add(report?.connections);
}
else if (connections.Type == JTokenType.Array)
{
userConnections = report?.connections;
}
else
{
logger.Error($"No valid connection type '{connections.Type}'.");
}
var newUserConnections = new List <JToken>();
for (int i = 0; i < userConnections.Count; i++)
{
var mergeConnection = userConnections[i] as JObject;
var credType = Options.Config?.Connection?.Credentials?.Type ?? QlikCredentialType.JWT;
var connectorConnection = JObject.FromObject(CreateConnection(credType, session));
connectorConnection.Merge(mergeConnection, new JsonMergeSettings()
{
MergeNullValueHandling = MergeNullValueHandling.Ignore
});
newUserConnections.Add(connectorConnection);
}
report.connections = new JArray(newUserConnections);
if (report.distribute is JObject distribute)
{
var children = distribute?.Children().Children()?.ToList() ?? new List <JToken>();
foreach (dynamic child in children)
{
var connection = child.connections ?? null;
if (connection == null)
{
child.connections = new JArray(newUserConnections);
}
else if (connection?.ToString() == "@CONFIGCONNECTION@")
{
child.connections = new JArray(newUserConnections);
}
var childProp = (child as JObject).Parent as JProperty;
if (childProp?.Name?.StartsWith("hub") ?? false)
{
if (child?.owner == null)
{
child.owner = session.User.ToString();
}
}
else if (childProp?.Name?.StartsWith("mail") ?? false)
{
if (child?.mailServer != null)
{
if (child?.mailServer?.privateKey == null)
{
child.mailServer.privateKey = Options?.Config?.Connection?.Credentials?.PrivateKey;
}
}
}
}
}
var privateKey = Options.Config?.Connection?.Credentials?.PrivateKey ?? null;
if (!String.IsNullOrEmpty(privateKey))
{
// For file access
lock (threadObject)
{
var path = HelperUtilities.GetFullPathFromApp(privateKey);
var crypter = new TextCrypter(path);
var value = report?.template?.outputPassword ?? null;
if (value == null)
{
value = report?.template?.outputpassword ?? null;
}
if (value != null)
{
string password = value.ToString();
if (value.Type == JTokenType.Boolean)
{
password = password.ToLowerInvariant();
}
bool useBase64Password = report?.template?.useBase64Password ?? false;
if (useBase64Password == true)
{
report.template.outputPassword = crypter.DecryptText(password);
}
else
{
report.template.outputPassword = password;
}
}
}
}
}
}
//Resolve @PROPERTYNAME@ and @("
var jsonResolver = new JsonConfigResolver(serJsonConfig.ToString());
var jsonResult = jsonResolver.Resolve();
var serConfiguration = JsonConvert.DeserializeObject <SerConfig>(jsonResult);
return(serConfiguration);
}
public void StartReportJob(QlikRequest request, ManagedTask newManagedTask)
{
Task.Run(() =>
{
try
{
newManagedTask.InternalStatus = InternalTaskStatus.CREATEREPORTJOBSTART;
logger.Debug("Create new job...");
logger.Info($"Memory usage: {GC.GetTotalMemory(true)}");
Options.Analyser?.ClearCheckPoints();
Options.Analyser?.Start();
Options.Analyser?.SetCheckPoint("StartReportJob", "Start report generation");
MappedDiagnosticsLogicalContext.Set("jobId", newManagedTask.Id.ToString());
//Get Qlik session over jwt
logger.Debug("Get cookie over JWT session...");
newManagedTask.Session = Options.SessionHelper.GetSession(Options.Config.Connection, request);
if (newManagedTask.Session == null)
{
throw new Exception("No session cookie generated (check qmc settings or connector config).");
}
//Connect to Qlik app
logger.Debug("Connecting to Qlik via websocket...");
Options.Analyser?.SetCheckPoint("StartReportJob", "Connect to Qlik");
var fullConnectionConfig = new SerConnection
{
App = request.AppId,
ServerUri = Options.Config.Connection.ServerUri,
Credentials = new SerCredentials()
{
Type = QlikCredentialType.SESSION,
Key = newManagedTask.Session.Cookie.Name,
Value = newManagedTask.Session.Cookie.Value
}
};
var qlikConnection = ConnectionManager.NewConnection(fullConnectionConfig, true);
newManagedTask.Session.QlikConn = qlikConnection ?? throw new Exception("The web socket connection to qlik could not be established (Connector).");
//Create full engine config
logger.Debug("Create configuration for the engine...");
Options.Analyser?.SetCheckPoint("StartReportJob", "Gernerate Config Json");
var newEngineConfig = CreateEngineConfig(request, newManagedTask.Session);
//Remove emtpy Tasks without report infos
newEngineConfig.Tasks.RemoveAll(t => t.Reports.Count == 0);
foreach (var configTask in newEngineConfig.Tasks)
{
if (configTask.Id == Guid.Empty)
{
configTask.Id = Guid.NewGuid();
}
else
{
//Check the task is of unique
if (newEngineConfig.Tasks.Count(t => t.Id == configTask.Id) > 1)
{
throw new Exception("The task id is used twice. Please change the task id. This must always be unique.");
}
}
foreach (var configReport in configTask.Reports)
{
//Important: Add bearer connection as last connection item.
var firstConnection = configReport?.Connections?.FirstOrDefault() ?? null;
if (firstConnection != null)
{
logger.Debug("Create bearer connection.");
var newBearerConnection = CreateConnection(QlikCredentialType.JWT, newManagedTask.Session, firstConnection.App);
configReport.Connections.Add(newBearerConnection);
}
//Check app Id
var appList = Q2g.HelperQlik.Connection.PossibleApps;
var activeApp = appList.FirstOrDefault(a => a.qDocId == firstConnection.App);
if (activeApp == null)
{
throw new Exception($"The app id {firstConnection.App} was not found. Please check the app id or the security rules.");
}
//Read content from lib and content libary
logger.Debug("Get template data from qlik.");
if (configReport.Template != null)
{
var uploadData = FindTemplatePath(newManagedTask.Session, configReport.Template);
logger.Debug("Upload template data to rest service.");
var uploadId = Options.RestClient.UploadData(uploadData, configReport.Template.Input);
logger.Debug($"Upload with id {uploadId} was successfully.");
newManagedTask.FileUploadIds.Add(uploadId);
}
else
{
logger.Debug("No Template found. - Use alternative mode.");
}
// Perfomance analyser for the engine
configReport.General.UsePerfomanceAnalyzer = Options.Config.UsePerfomanceAnalyzer;
//Add all server uris
foreach (var connection in configReport.Connections)
{
connection.LicenseServers.Add(new SerServer()
{
ServerUri = new Uri("https://license.analyticsgate.com"), Location = "de", Priority = 1
});
connection.LicenseServers.AddRange(Options.Config.Connection.LicenseServers);
connection.RendererServers.Add(new SerServer()
{
ServerUri = new Uri("https://localhost:40271"), Location = "default", Priority = 100
});
connection.RendererServers.AddRange(Options.Config.Connection.RendererServers);
}
}
}
//Append upload ids on config
var jobJson = JObject.FromObject(newEngineConfig);
jobJson = AppendUploadGuids(jobJson, newManagedTask.FileUploadIds);
newManagedTask.JobScript = jobJson;
//Use the connector in the same App, than wait for data reload
Options.Analyser?.SetCheckPoint("StartReportJob", "Start connector reporting task");
var scriptConnection = newEngineConfig?.Tasks?.SelectMany(s => s.Reports)
?.SelectMany(r => r.Connections)
?.FirstOrDefault(c => c.App == newManagedTask.Session.AppId) ?? null;
//Wait for data load in single App mode
WaitForDataLoad(newManagedTask, scriptConnection);
Options.Analyser?.SetCheckPoint("StartReportJob", "End report generation");
newManagedTask.InternalStatus = InternalTaskStatus.CREATEREPORTJOBEND;
}
catch (Exception ex)
{
logger.Error(ex, "The reporting order could not be executed properly.");
newManagedTask.Endtime = DateTime.Now;
newManagedTask.Status = -1;
newManagedTask.Error = ex;
newManagedTask.InternalStatus = InternalTaskStatus.ERROR;
Options.SessionHelper.Manager.MakeSocketFree(newManagedTask?.Session ?? null);
}
}, newManagedTask.Cancellation.Token);
}
public QlikResponse GetStatusResponse(QlikRequest request)
{
var result = new QlikResponse();
try
{
if (request.VersionMode == "all")
{
logger.Debug("Status - Read main versions...");
result.Version = Options.Config.PackageVersion;
result.Status = 100;
return(result);
}
if (request.VersionMode == "versions")
{
logger.Debug("Status - Read all versions...");
result.Version = String.Join("|", Options.Config.PackageVersions);
result.Status = 100;
return(result);
}
if (request.VersionMode == "packages")
{
logger.Debug("Status - Read external package versions...");
result.ExternalPackagesInfo = Options.Config.ExternalPackageJson;
return(result);
}
if (request.ManagedTaskId == "all")
{
logger.Debug("Status - Get all managed tasks.");
var activeTasks = Options.TaskPool.ManagedTasks.Values?.ToList() ?? new List <ManagedTask>();
result.ManagedTasks = GetAllowedTasks(activeTasks, request);
}
else if (request.ManagedTaskId != null)
{
logger.Debug($"Status - Get managed task status from id '{request.ManagedTaskId}'.");
var managedTaskId = new Guid(request.ManagedTaskId);
var currentTask = Options.TaskPool.ManagedTasks.Values.ToList().FirstOrDefault(t => t.Id == managedTaskId);
if (currentTask != null)
{
result.Distribute = currentTask.DistributeResult;
result.Status = currentTask.Status;
if (currentTask.Error != null)
{
result.SetErrorMessage(currentTask.Error);
}
else
{
result.Log = currentTask.Message;
}
result.JobResults = JArray.FromObject(currentTask.JobResults);
currentTask.LastQlikFunctionCall = DateTime.Now;
}
else
{
logger.Warn($"Status - No managed task id '{request.ManagedTaskId}' in pool found.");
result.Log = "Status information is not available.";
result.Status = -1;
}
}
else
{
logger.Warn("Status - No managed tasks with 'all' or 'id' found.");
result.Log = "Status information is not available.";
result.Status = -1;
if (QlikRequest.Error != null)
{
result.SetErrorMessage(QlikRequest.Error);
}
}
}
catch (Exception ex)
{
logger.Error(ex, "The status function has an unknown error.");
}
return(result);
}
public override async Task ExecuteFunction(IAsyncStreamReader <BundledRows> requestStream,
IServerStreamWriter <BundledRows> responseStream,
ServerCallContext context)
{
var response = new QlikResponse();
try
{
logger.Debug("The method 'ExecuteFunction' is called...");
//Read function header
var functionHeader = context.RequestHeaders.ParseIMessageFirstOrDefault <FunctionRequestHeader>();
//Read common header
var commonHeader = context.RequestHeaders.ParseIMessageFirstOrDefault <CommonRequestHeader>();
//Set appid
logger.Info($"The Qlik app id '{commonHeader?.AppId}' in header found.");
var qlikAppId = commonHeader?.AppId;
//Set qlik user
logger.Info($"The Qlik user '{commonHeader?.UserId}' in header found.");
var domainUser = new DomainUser(commonHeader?.UserId);
//Very important code line
await context.WriteResponseHeadersAsync(new Metadata { { "qlik-cache", "no-store" } });
//Read parameter from qlik
var row = GetParameter(requestStream);
var userJson = GetParameterValue(0, row);
//Parse request from qlik script
logger.Debug("Parse user request...");
var request = QlikRequest.Parse(domainUser, qlikAppId, userJson);
var functionCall = (ConnectorFunction)functionHeader.FunctionId;
logger.Debug($"Call Function id: '{functionCall}' from client '{context?.Peer}'.");
if (functionCall == ConnectorFunction.START)
{
#region Switch qlik user to app owner
if (domainUser?.UserId == "sa_scheduler" && domainUser?.UserDirectory == "INTERNAL")
{
try
{
var oldUser = domainUser.ToString();
domainUser = new DomainUser("INTERNAL\\ser_scheduler");
logger.Debug($"Change Qlik user '{oldUser}' to task service user '{domainUser}'.");
var connection = RuntimeOptions.Config.Connection;
var tmpsession = RuntimeOptions.SessionHelper.Manager.CreateNewSession(connection, domainUser, qlikAppId);
if (tmpsession == null)
{
throw new Exception("No session cookie generated. (Qlik Task)");
}
var qrsHub = new QlikQrsHub(RuntimeOptions.Config.Connection.ServerUri, tmpsession.Cookie);
qrsHub.ServerCertificateValidationCallback += (sender, certificate, chain, sslPolicyErrors) => true;
domainUser = request.GetAppOwner(qrsHub, qlikAppId);
if (domainUser == null)
{
throw new Exception("The owner of the App could not found.");
}
logger.Debug($"App owner '{domainUser}' found.");
request.QlikUser = domainUser;
}
catch (Exception ex)
{
logger.Error(ex, "Could not switch the task user to real qlik user.");
}
}
#endregion
#region Function call SER.START
logger.Debug("Function call SER.START...");
var newManagedTask = new ManagedTask()
{
StartTime = DateTime.Now,
Message = "Create new report job...",
Cancellation = new CancellationTokenSource(),
Status = 0
};
RuntimeOptions.TaskPool.ManagedTasks.TryAdd(newManagedTask.Id, newManagedTask);
var startFunction = new StartFunction(RuntimeOptions);
startFunction.StartReportJob(request, newManagedTask);
response.TaskId = newManagedTask.Id.ToString();
#endregion
}
else if (functionCall == ConnectorFunction.STOP)
{
#region Function call SER.STOP
logger.Debug("Function call SER.STOP...");
var stopFunction = new StopFunction(RuntimeOptions);
stopFunction.StopReportJobs(request);
if (request.ManagedTaskId == "all")
{
response.Log = "All report jobs is stopping...";
}
else
{
response.Log = $"Report job '{request.ManagedTaskId}' is stopping...";
}
response.Status = 4;
#endregion
}
else if (functionCall == ConnectorFunction.RESULT)
{
#region Function call SER.RESULT
logger.Debug("Function call SER.RESULT...");
var resultFunction = new ResultFunction(RuntimeOptions);
response = resultFunction.FormatJobResult(request);
#endregion
}
else if (functionCall == ConnectorFunction.STATUS)
{
#region Function call SER.STATUS
logger.Debug("Function call SER.STATUS...");
var statusFunction = new StatusFunction(RuntimeOptions);
response = statusFunction.GetStatusResponse(request);
#endregion
}
else
{
throw new Exception($"The id '{functionCall}' of the function call was unknown.");
}
}
catch (Exception ex)
{
logger.Error(ex, $"The method 'ExecuteFunction' failed with error '{ex.Message}'.");
response.Status = -1;
response.SetErrorMessage(ex);
}
finally
{
logger.Trace($"Qlik status result: {JsonConvert.SerializeObject(response)}");
await responseStream.WriteAsync(GetResult(response));
LogManager.Flush();
}
}
