private void StopManagedTask(ManagedTask task) { task.InternalStatus = InternalTaskStatus.STOPSTART; var stopRequest = new QlikRequest { ManagedTaskId = task.Id.ToString() }; var stopFunction = new StopFunction(runtimeOptions); stopFunction.StopReportJobs(stopRequest, true); }
public void StopReportJobs(QlikRequest request, bool isTimeout = false) { Task.Run(() => { try { if (request.ManagedTaskId == "all") { logger.Debug("All tasks will be stopped..."); if (Options.TaskPool.ManagedTasks.IsEmpty) { logger.Warn("No stopping jobs."); } var managedTasks = Options.TaskPool.ManagedTasks?.Values?.ToList() ?? new List <ManagedTask>(); foreach (var managedTask in managedTasks) { if (managedTask.Status >= 0 && managedTask.Status <= 2) { StopReportJob(managedTask.Id, isTimeout); } else { logger.Warn($"The task '{managedTask.Id}' has a status that cannot be stopped."); } } } else if (request.ManagedTaskId != null) { logger.Debug($"Managed task '{request.ManagedTaskId}' will be stopped..."); StopReportJob(new Guid(request.ManagedTaskId), isTimeout); } else { logger.Warn($"The task id of the task to be stopped is empty."); } } catch (Exception ex) { logger.Error(ex, "The stop function has an unknown error."); } }, Options.Cancellation.Token); }
public QlikResponse FormatJobResult(QlikRequest request) { var result = new QlikResponse(); try { if (request.ManagedTaskId != null) { logger.Debug($"Formatting for Qlik will be created by Task '{request.ManagedTaskId}'."); var managedTaskId = new Guid(request.ManagedTaskId); var resultQlikTask = Options.TaskPool.ManagedTasks.Values.ToList().FirstOrDefault(t => t.Id == managedTaskId); if (resultQlikTask != null) { var distibuteJson = resultQlikTask.DistributeResult; if (distibuteJson != null) { result.FormatedResult = GetFormatedJsonForQlik(distibuteJson); logger.Debug("The delivery result was successfully formatted."); return(result); } else { logger.Warn("The delivery result is null."); } } else { logger.Warn($"No managed task id '{managedTaskId}' for formated Qlik result found."); } } else { logger.Warn("No task id for formated Qlik result found."); } } catch (Exception ex) { result.SetErrorMessage(ex); logger.Error(ex, "The result function has an unknown error."); } return(null); }
public SessionInfo GetSession(SerConnection connection, QlikRequest request) { try { lock (threadObject) { var uri = connection.ServerUri; var oldSession = Sessions?.FirstOrDefault(u => u.ConnectUri.OriginalString == uri.OriginalString && u.User.ToString() == request.QlikUser.ToString() && u.AppId == request.AppId) ?? null; if (oldSession != null) { logger.Debug("Old session found..."); var result = ValidateSession(oldSession); if (result) { logger.Debug("Old session is working..."); return(oldSession); } logger.Debug("Old session not working, i remove it..."); Sessions.Remove(oldSession); } } logger.Debug("Create new Qlik Session..."); var sessionInfo = Manager.CreateNewSession(connection, request.QlikUser, request.AppId); if (sessionInfo != null) { Sessions.Add(sessionInfo); } return(sessionInfo); } catch (Exception ex) { logger.Error(ex, "The session could not recreated or reused."); return(null); } }
private JArray GetAllowedTasks(List <ManagedTask> activeTasks, QlikRequest request) { var results = new JArray(); try { Options.Analyser?.SetCheckPoint("GetAllowedTasks", $"Start - Find all running tasks"); var session = Options.SessionHelper.GetSession(Options.Config.Connection, request); var qrsHub = new QlikQrsHub(session.ConnectUri, session.Cookie); foreach (var activeTask in activeTasks) { if (activeTask.Status == 4) { continue; } var appOwner = request.GetAppOwner(qrsHub, activeTask.Session.AppId); if (appOwner != null && appOwner.ToString() == activeTask.Session.User.ToString()) { logger.Debug($"The app owner '{appOwner}' was found."); results.Add(CreateTaskResult(activeTask)); } else { if (HasReadRights(qrsHub, activeTask)) { results.Add(CreateTaskResult(activeTask)); } } } Options.Analyser?.SetCheckPoint("GetAllowedTasks", $"End - Find all running tasks"); } catch (Exception ex) { logger.Error(ex, "The list of tasks could not be determined."); } return(results); }
private SerConfig CreateEngineConfig(QlikRequest request, SessionInfo session) { logger.Debug($"Resolve script '{request.JsonScript}' for engine config..."); var jsonStr = request.JsonScript; //Make full JSON for Engine logger.Debug("Auto replacement to normal json structure..."); if (!jsonStr.ToLowerInvariant().Contains("\"reports\":")) { jsonStr = $"\"reports\":[{jsonStr}]"; } if (!jsonStr.ToLowerInvariant().Contains("\"tasks\":")) { jsonStr = $"\"tasks\":[{{{jsonStr}}}]"; } if (!jsonStr.Trim().StartsWith("{")) { jsonStr = $"{{{jsonStr}"; } if (!jsonStr.Trim().EndsWith("}")) { jsonStr = $"{jsonStr}}}"; } logger.Debug("Search for connections for the engine config..."); dynamic serJsonConfig = JObject.Parse(jsonStr); var tasks = serJsonConfig?.tasks ?? new JArray(); foreach (var task in tasks) { var reports = task?.reports ?? new JArray(); foreach (var report in reports) { var userConnections = new JArray(); JToken connections = report?.connections; if (connections.Type == JTokenType.Object) { userConnections.Add(report?.connections); } else if (connections.Type == JTokenType.Array) { userConnections = report?.connections; } else { logger.Error($"No valid connection type '{connections.Type}'."); } var newUserConnections = new List <JToken>(); for (int i = 0; i < userConnections.Count; i++) { var mergeConnection = userConnections[i] as JObject; var credType = Options.Config?.Connection?.Credentials?.Type ?? QlikCredentialType.JWT; var connectorConnection = JObject.FromObject(CreateConnection(credType, session)); connectorConnection.Merge(mergeConnection, new JsonMergeSettings() { MergeNullValueHandling = MergeNullValueHandling.Ignore }); newUserConnections.Add(connectorConnection); } report.connections = new JArray(newUserConnections); if (report.distribute is JObject distribute) { var children = distribute?.Children().Children()?.ToList() ?? new List <JToken>(); foreach (dynamic child in children) { var connection = child.connections ?? null; if (connection == null) { child.connections = new JArray(newUserConnections); } else if (connection?.ToString() == "@CONFIGCONNECTION@") { child.connections = new JArray(newUserConnections); } var childProp = (child as JObject).Parent as JProperty; if (childProp?.Name?.StartsWith("hub") ?? false) { if (child?.owner == null) { child.owner = session.User.ToString(); } } else if (childProp?.Name?.StartsWith("mail") ?? false) { if (child?.mailServer != null) { if (child?.mailServer?.privateKey == null) { child.mailServer.privateKey = Options?.Config?.Connection?.Credentials?.PrivateKey; } } } } } var privateKey = Options.Config?.Connection?.Credentials?.PrivateKey ?? null; if (!String.IsNullOrEmpty(privateKey)) { // For file access lock (threadObject) { var path = HelperUtilities.GetFullPathFromApp(privateKey); var crypter = new TextCrypter(path); var value = report?.template?.outputPassword ?? null; if (value == null) { value = report?.template?.outputpassword ?? null; } if (value != null) { string password = value.ToString(); if (value.Type == JTokenType.Boolean) { password = password.ToLowerInvariant(); } bool useBase64Password = report?.template?.useBase64Password ?? false; if (useBase64Password == true) { report.template.outputPassword = crypter.DecryptText(password); } else { report.template.outputPassword = password; } } } } } } //Resolve @PROPERTYNAME@ and @(" var jsonResolver = new JsonConfigResolver(serJsonConfig.ToString()); var jsonResult = jsonResolver.Resolve(); var serConfiguration = JsonConvert.DeserializeObject <SerConfig>(jsonResult); return(serConfiguration); }
public void StartReportJob(QlikRequest request, ManagedTask newManagedTask) { Task.Run(() => { try { newManagedTask.InternalStatus = InternalTaskStatus.CREATEREPORTJOBSTART; logger.Debug("Create new job..."); logger.Info($"Memory usage: {GC.GetTotalMemory(true)}"); Options.Analyser?.ClearCheckPoints(); Options.Analyser?.Start(); Options.Analyser?.SetCheckPoint("StartReportJob", "Start report generation"); MappedDiagnosticsLogicalContext.Set("jobId", newManagedTask.Id.ToString()); //Get Qlik session over jwt logger.Debug("Get cookie over JWT session..."); newManagedTask.Session = Options.SessionHelper.GetSession(Options.Config.Connection, request); if (newManagedTask.Session == null) { throw new Exception("No session cookie generated (check qmc settings or connector config)."); } //Connect to Qlik app logger.Debug("Connecting to Qlik via websocket..."); Options.Analyser?.SetCheckPoint("StartReportJob", "Connect to Qlik"); var fullConnectionConfig = new SerConnection { App = request.AppId, ServerUri = Options.Config.Connection.ServerUri, Credentials = new SerCredentials() { Type = QlikCredentialType.SESSION, Key = newManagedTask.Session.Cookie.Name, Value = newManagedTask.Session.Cookie.Value } }; var qlikConnection = ConnectionManager.NewConnection(fullConnectionConfig, true); newManagedTask.Session.QlikConn = qlikConnection ?? throw new Exception("The web socket connection to qlik could not be established (Connector)."); //Create full engine config logger.Debug("Create configuration for the engine..."); Options.Analyser?.SetCheckPoint("StartReportJob", "Gernerate Config Json"); var newEngineConfig = CreateEngineConfig(request, newManagedTask.Session); //Remove emtpy Tasks without report infos newEngineConfig.Tasks.RemoveAll(t => t.Reports.Count == 0); foreach (var configTask in newEngineConfig.Tasks) { if (configTask.Id == Guid.Empty) { configTask.Id = Guid.NewGuid(); } else { //Check the task is of unique if (newEngineConfig.Tasks.Count(t => t.Id == configTask.Id) > 1) { throw new Exception("The task id is used twice. Please change the task id. This must always be unique."); } } foreach (var configReport in configTask.Reports) { //Important: Add bearer connection as last connection item. var firstConnection = configReport?.Connections?.FirstOrDefault() ?? null; if (firstConnection != null) { logger.Debug("Create bearer connection."); var newBearerConnection = CreateConnection(QlikCredentialType.JWT, newManagedTask.Session, firstConnection.App); configReport.Connections.Add(newBearerConnection); } //Check app Id var appList = Q2g.HelperQlik.Connection.PossibleApps; var activeApp = appList.FirstOrDefault(a => a.qDocId == firstConnection.App); if (activeApp == null) { throw new Exception($"The app id {firstConnection.App} was not found. Please check the app id or the security rules."); } //Read content from lib and content libary logger.Debug("Get template data from qlik."); if (configReport.Template != null) { var uploadData = FindTemplatePath(newManagedTask.Session, configReport.Template); logger.Debug("Upload template data to rest service."); var uploadId = Options.RestClient.UploadData(uploadData, configReport.Template.Input); logger.Debug($"Upload with id {uploadId} was successfully."); newManagedTask.FileUploadIds.Add(uploadId); } else { logger.Debug("No Template found. - Use alternative mode."); } // Perfomance analyser for the engine configReport.General.UsePerfomanceAnalyzer = Options.Config.UsePerfomanceAnalyzer; //Add all server uris foreach (var connection in configReport.Connections) { connection.LicenseServers.Add(new SerServer() { ServerUri = new Uri("https://license.analyticsgate.com"), Location = "de", Priority = 1 }); connection.LicenseServers.AddRange(Options.Config.Connection.LicenseServers); connection.RendererServers.Add(new SerServer() { ServerUri = new Uri("https://localhost:40271"), Location = "default", Priority = 100 }); connection.RendererServers.AddRange(Options.Config.Connection.RendererServers); } } } //Append upload ids on config var jobJson = JObject.FromObject(newEngineConfig); jobJson = AppendUploadGuids(jobJson, newManagedTask.FileUploadIds); newManagedTask.JobScript = jobJson; //Use the connector in the same App, than wait for data reload Options.Analyser?.SetCheckPoint("StartReportJob", "Start connector reporting task"); var scriptConnection = newEngineConfig?.Tasks?.SelectMany(s => s.Reports) ?.SelectMany(r => r.Connections) ?.FirstOrDefault(c => c.App == newManagedTask.Session.AppId) ?? null; //Wait for data load in single App mode WaitForDataLoad(newManagedTask, scriptConnection); Options.Analyser?.SetCheckPoint("StartReportJob", "End report generation"); newManagedTask.InternalStatus = InternalTaskStatus.CREATEREPORTJOBEND; } catch (Exception ex) { logger.Error(ex, "The reporting order could not be executed properly."); newManagedTask.Endtime = DateTime.Now; newManagedTask.Status = -1; newManagedTask.Error = ex; newManagedTask.InternalStatus = InternalTaskStatus.ERROR; Options.SessionHelper.Manager.MakeSocketFree(newManagedTask?.Session ?? null); } }, newManagedTask.Cancellation.Token); }
public QlikResponse GetStatusResponse(QlikRequest request) { var result = new QlikResponse(); try { if (request.VersionMode == "all") { logger.Debug("Status - Read main versions..."); result.Version = Options.Config.PackageVersion; result.Status = 100; return(result); } if (request.VersionMode == "versions") { logger.Debug("Status - Read all versions..."); result.Version = String.Join("|", Options.Config.PackageVersions); result.Status = 100; return(result); } if (request.VersionMode == "packages") { logger.Debug("Status - Read external package versions..."); result.ExternalPackagesInfo = Options.Config.ExternalPackageJson; return(result); } if (request.ManagedTaskId == "all") { logger.Debug("Status - Get all managed tasks."); var activeTasks = Options.TaskPool.ManagedTasks.Values?.ToList() ?? new List <ManagedTask>(); result.ManagedTasks = GetAllowedTasks(activeTasks, request); } else if (request.ManagedTaskId != null) { logger.Debug($"Status - Get managed task status from id '{request.ManagedTaskId}'."); var managedTaskId = new Guid(request.ManagedTaskId); var currentTask = Options.TaskPool.ManagedTasks.Values.ToList().FirstOrDefault(t => t.Id == managedTaskId); if (currentTask != null) { result.Distribute = currentTask.DistributeResult; result.Status = currentTask.Status; if (currentTask.Error != null) { result.SetErrorMessage(currentTask.Error); } else { result.Log = currentTask.Message; } result.JobResults = JArray.FromObject(currentTask.JobResults); currentTask.LastQlikFunctionCall = DateTime.Now; } else { logger.Warn($"Status - No managed task id '{request.ManagedTaskId}' in pool found."); result.Log = "Status information is not available."; result.Status = -1; } } else { logger.Warn("Status - No managed tasks with 'all' or 'id' found."); result.Log = "Status information is not available."; result.Status = -1; if (QlikRequest.Error != null) { result.SetErrorMessage(QlikRequest.Error); } } } catch (Exception ex) { logger.Error(ex, "The status function has an unknown error."); } return(result); }
public override async Task ExecuteFunction(IAsyncStreamReader <BundledRows> requestStream, IServerStreamWriter <BundledRows> responseStream, ServerCallContext context) { var response = new QlikResponse(); try { logger.Debug("The method 'ExecuteFunction' is called..."); //Read function header var functionHeader = context.RequestHeaders.ParseIMessageFirstOrDefault <FunctionRequestHeader>(); //Read common header var commonHeader = context.RequestHeaders.ParseIMessageFirstOrDefault <CommonRequestHeader>(); //Set appid logger.Info($"The Qlik app id '{commonHeader?.AppId}' in header found."); var qlikAppId = commonHeader?.AppId; //Set qlik user logger.Info($"The Qlik user '{commonHeader?.UserId}' in header found."); var domainUser = new DomainUser(commonHeader?.UserId); //Very important code line await context.WriteResponseHeadersAsync(new Metadata { { "qlik-cache", "no-store" } }); //Read parameter from qlik var row = GetParameter(requestStream); var userJson = GetParameterValue(0, row); //Parse request from qlik script logger.Debug("Parse user request..."); var request = QlikRequest.Parse(domainUser, qlikAppId, userJson); var functionCall = (ConnectorFunction)functionHeader.FunctionId; logger.Debug($"Call Function id: '{functionCall}' from client '{context?.Peer}'."); if (functionCall == ConnectorFunction.START) { #region Switch qlik user to app owner if (domainUser?.UserId == "sa_scheduler" && domainUser?.UserDirectory == "INTERNAL") { try { var oldUser = domainUser.ToString(); domainUser = new DomainUser("INTERNAL\\ser_scheduler"); logger.Debug($"Change Qlik user '{oldUser}' to task service user '{domainUser}'."); var connection = RuntimeOptions.Config.Connection; var tmpsession = RuntimeOptions.SessionHelper.Manager.CreateNewSession(connection, domainUser, qlikAppId); if (tmpsession == null) { throw new Exception("No session cookie generated. (Qlik Task)"); } var qrsHub = new QlikQrsHub(RuntimeOptions.Config.Connection.ServerUri, tmpsession.Cookie); qrsHub.ServerCertificateValidationCallback += (sender, certificate, chain, sslPolicyErrors) => true; domainUser = request.GetAppOwner(qrsHub, qlikAppId); if (domainUser == null) { throw new Exception("The owner of the App could not found."); } logger.Debug($"App owner '{domainUser}' found."); request.QlikUser = domainUser; } catch (Exception ex) { logger.Error(ex, "Could not switch the task user to real qlik user."); } } #endregion #region Function call SER.START logger.Debug("Function call SER.START..."); var newManagedTask = new ManagedTask() { StartTime = DateTime.Now, Message = "Create new report job...", Cancellation = new CancellationTokenSource(), Status = 0 }; RuntimeOptions.TaskPool.ManagedTasks.TryAdd(newManagedTask.Id, newManagedTask); var startFunction = new StartFunction(RuntimeOptions); startFunction.StartReportJob(request, newManagedTask); response.TaskId = newManagedTask.Id.ToString(); #endregion } else if (functionCall == ConnectorFunction.STOP) { #region Function call SER.STOP logger.Debug("Function call SER.STOP..."); var stopFunction = new StopFunction(RuntimeOptions); stopFunction.StopReportJobs(request); if (request.ManagedTaskId == "all") { response.Log = "All report jobs is stopping..."; } else { response.Log = $"Report job '{request.ManagedTaskId}' is stopping..."; } response.Status = 4; #endregion } else if (functionCall == ConnectorFunction.RESULT) { #region Function call SER.RESULT logger.Debug("Function call SER.RESULT..."); var resultFunction = new ResultFunction(RuntimeOptions); response = resultFunction.FormatJobResult(request); #endregion } else if (functionCall == ConnectorFunction.STATUS) { #region Function call SER.STATUS logger.Debug("Function call SER.STATUS..."); var statusFunction = new StatusFunction(RuntimeOptions); response = statusFunction.GetStatusResponse(request); #endregion } else { throw new Exception($"The id '{functionCall}' of the function call was unknown."); } } catch (Exception ex) { logger.Error(ex, $"The method 'ExecuteFunction' failed with error '{ex.Message}'."); response.Status = -1; response.SetErrorMessage(ex); } finally { logger.Trace($"Qlik status result: {JsonConvert.SerializeObject(response)}"); await responseStream.WriteAsync(GetResult(response)); LogManager.Flush(); } }