public async Task Test_ManagedTask_CancelAsync() { var managedTasks = new ManagedTasks.ManagedTasks(); // simple task that can be cancelled var managedObject = new ProgressTask(10000, 1); // add the simple task 500 times. _cancelCounter = 0; managedTasks.OnStatus += CancelResult; var tasks = new ManagedTask[100]; for (var i = 0; i < 100; i++) { tasks[i] = managedTasks.Add("123", "task3", "test", managedObject, null); } for (var i = 0; i < 100; i++) { await tasks[i].CancelAsync(); } var cts = new CancellationTokenSource(); cts.CancelAfter(30000); await managedTasks.WhenAll(cts.Token); // counter should equal the number of tasks Assert.Equal(100, _cancelCounter); Assert.Empty(managedTasks); }
/// <summary> /// Start this worker /// </summary> public void Start(LineBasedTextWriter Log) { // Make sure there's not already a managed task in process if (ActiveInstance != null) { throw new Exception("Cannot start a new worker while a managed process is already active"); } // Create the new instance if (Type == SequenceProbeType.Optimize) { // Update our dependencies UpdateDependencies(Log); // Write the task to disk Worker.Serialize(TaskStateFile.FullName); // Construct the new managed task, and start it. If the task is already complete, create a dummy thread which just returns immediately. ActiveInstance = new ManagedTaskThread(Writer => FindNextDependency(TaskStateFile.FullName, Writer)); } else if (Type == SequenceProbeType.Verify) { // Write the task to disk Worker.Serialize(TaskStateFile.FullName); // Construct the new managed task, and start it. If the task is already complete, create a dummy thread which just returns immediately. ActiveInstance = new ManagedTaskThread(Writer => Verify(TaskStateFile.FullName, Writer)); } else { throw new NotImplementedException(); } ActiveInstance.Start(); }
public async Task ParallelManagedTaskHandlerConcurrentAsync(int taskCount) { var managedTasks = new ManagedTasks.ManagedTasks(); var oneSecondTask = new ProgressTask(1, 1); for (var i = 0; i < taskCount; i++) { var task = new ManagedTask() { TaskId = Guid.NewGuid().ToString(), CategoryKey = i, Name = "task", Category = "123", ManagedObject = oneSecondTask }; managedTasks.Add(task); } var cts = new CancellationTokenSource(); cts.CancelAfter(30000); await managedTasks.WhenAll(cts.Token); PrintManagedTasksCounters(managedTasks); Assert.Equal(taskCount, managedTasks.CompletedCount); }
private static bool HasReadRights(QlikQrsHub qrsHub, ManagedTask task) { try { logger.Debug($"Search for read right of user {task.Session.User}"); var parameter = CreateAuditMatrixParameters(task.Session.User); var contentData = Encoding.UTF8.GetBytes(parameter.ToString()); var data = new ContentData() { ContentType = "application/json", FileData = contentData, }; var matrixResponse = qrsHub.SendRequestAsync("systemrule/security/audit/matrix", HttpMethod.Post, data).Result; var responseObject = JObject.Parse(matrixResponse); var matrix = responseObject["matrix"]?.FirstOrDefault(m => m["resourceId"]?.ToString() == task.Session.AppId) ?? null; if (matrix != null) { var privileges = responseObject["resources"][matrix["resourceId"]?.ToObject <string>()]["privileges"].ToList() ?? new List <JToken>(); var canAppRead = privileges?.FirstOrDefault(p => p?.ToObject <string>() == "read") ?? null; if (canAppRead != null) { return(true); } } return(false); } catch (Exception ex) { logger.Error(ex, "The read rights of the user could not found."); return(false); } }
/// <summary> /// Dispose of any executing task object /// </summary> public void Dispose() { if (ActiveInstance != null) { ActiveInstance.Dispose(); ActiveInstance = null; } }
private void TaskProgressChange(ManagedTask value, ManagedTaskProgressItem progressItem) { // run as a separate task to minimise delays to core processes. if (_sendDatalinkProgress == null || _sendDatalinkProgress.IsCompleted) { _sendDatalinkProgress = SendDatalinkProgress(); } }
public int RegisterNewTask() { var newManagedTask = new ManagedTask(_nextAvailableTaskId); _nextAvailableTaskId += 1; _managedTasks.Add(newManagedTask.TaskId, newManagedTask); UpdateStallState(); return(newManagedTask.TaskId); }
private static JObject CreateTaskResult(ManagedTask task) { return(JObject.FromObject(new { startTime = task.StartTime, status = task.Status, taskId = task.Id, appId = task.Session.AppId, userId = task.Session.User.ToString() })); }
public MainWindow(string[] strArgs) { InitializeComponent(); //-----参数的处理----- var compareKey = string.Empty; foreach (var strArg in strArgs) { compareKey = "-Mouse="; if (strArg.StartsWith(compareKey)) { //指定Mouse历史文件路径 var fileName = strArg.Remove(0, compareKey.Length); var mouseHis = CustomEventLogger.GetHookMouseEventArgses(fileName); MouseHisDataList = new ObservableCollection <HookMouseEventArgs>(mouseHis); } else { compareKey = "-Key="; if (strArg.StartsWith(compareKey)) { //指定Key历史文件路径 var fileName = strArg.Remove(0, compareKey.Length); var keyHis = CustomEventLogger.GetHookKeyEventArgses(fileName); KeyHisDataList = new ObservableCollection <HookKeyEventArgs>(keyHis); } else { //设定关键词 var keyValueIndex = strArg.IndexOf('='); //获取关键词分隔符 var key = strArg.Substring(1, keyValueIndex - 1); //取得关键词 var value = strArg.Substring(keyValueIndex + 1); //取得替换值 foreach (var hookKeyEventArgse in KeyHisDataList) { if (hookKeyEventArgse.InputString == key) { hookKeyEventArgse.InputString = value; } } } } } //-----直接执行----- argAction = () => { ManagedTask.FactoryStart(() => { SimulateAll(); UIThread.Invoke(Close);//执行完关闭 }); }; }
public async Task Test_Add_SameTask_ActionsAsync(EConcurrentTaskAction concurrentTaskAction) { var managedTasks = new ManagedTasks.ManagedTasks(); var task1 = new ManagedTask { TaskId = Guid.NewGuid().ToString(), OriginatorId = "task", Name = "test", Category = "category", CategoryKey = 1, ReferenceKey = 1, ReferenceId = "id", ManagedObject = new ProgressTask(20, 1), Triggers = null, FileWatchers = null, DependentTaskIds = null, ConcurrentTaskAction = EConcurrentTaskAction.Sequence }; var task2 = new ManagedTask { TaskId = Guid.NewGuid().ToString(), OriginatorId = "task", Name = "test", Category = "category", CategoryKey = 1, ReferenceKey = 2, ReferenceId = "id", ManagedObject = new ProgressTask(0, 1), Triggers = null, FileWatchers = null, DependentTaskIds = null, ConcurrentTaskAction = concurrentTaskAction }; managedTasks.Add(task1); managedTasks.Add(task2); await managedTasks.WhenAll(); if (concurrentTaskAction == EConcurrentTaskAction.Parallel) { Assert.True(task1.EndTime > task2.EndTime); } else { Assert.True(task1.EndTime < task2.EndTime); } Assert.Equal(2, managedTasks.ErrorCount + managedTasks.CompletedCount); }
private void TaskStatusChange(ManagedTask value, EManagedTaskStatus managedTaskStatus) { if (managedTaskStatus == EManagedTaskStatus.Error) { _logger.LogWarning(value.Exception, $"The task {value.Name} with referenceId {value.ReferenceId} reported an error: {value.Message} "); } // run as a separate task to minimise delays to core processes. if (_sendDatalinkProgress == null || _sendDatalinkProgress.IsCompleted) { _sendDatalinkProgress = SendDatalinkProgress(); } }
private void WaitForDataLoad(ManagedTask task, SerConnection configConn) { var scriptApp = configConn?.App; var timeout = configConn?.RetryTimeout ?? 0; var dataloadCheck = ScriptCheck.DataLoadCheck(Options.Config.Connection.ServerUri, scriptApp, task, timeout); if (dataloadCheck) { logger.Debug("Transfer script to the rest service..."); var jobJson = task.JobScript.ToString(); logger.Trace($"JSON-Script: {jobJson}"); var taskId = Options.RestClient.RunTask(jobJson, task.Id); logger.Info($"The reporting request was successfully transferred and run with id '{taskId}' to the rest service..."); } else { logger.Debug("Dataload check failed."); } }
/// <summary> /// 执行模拟操作 /// </summary> private void ExcuteSimulateAll() { ManagedTask.FactoryStart(() => { UIThread.Invoke(() => { StatusTip = "开始执行!"; }); if (IsRepeat) { while (IsRepeat && RepeatTimes > 0) { SimulateAll(); UIThread.Invoke(() => { RepeatTimes--; }); } } else { SimulateAll(); } UIThread.Invoke(() => { StatusTip = "执行完毕!"; }); }); }
public void StartReportJob(QlikRequest request, ManagedTask newManagedTask) { Task.Run(() => { try { newManagedTask.InternalStatus = InternalTaskStatus.CREATEREPORTJOBSTART; logger.Debug("Create new job..."); logger.Info($"Memory usage: {GC.GetTotalMemory(true)}"); Options.Analyser?.ClearCheckPoints(); Options.Analyser?.Start(); Options.Analyser?.SetCheckPoint("StartReportJob", "Start report generation"); MappedDiagnosticsLogicalContext.Set("jobId", newManagedTask.Id.ToString()); //Get Qlik session over jwt logger.Debug("Get cookie over JWT session..."); newManagedTask.Session = Options.SessionHelper.GetSession(Options.Config.Connection, request); if (newManagedTask.Session == null) { throw new Exception("No session cookie generated (check qmc settings or connector config)."); } //Connect to Qlik app logger.Debug("Connecting to Qlik via websocket..."); Options.Analyser?.SetCheckPoint("StartReportJob", "Connect to Qlik"); var fullConnectionConfig = new SerConnection { App = request.AppId, ServerUri = Options.Config.Connection.ServerUri, Credentials = new SerCredentials() { Type = QlikCredentialType.SESSION, Key = newManagedTask.Session.Cookie.Name, Value = newManagedTask.Session.Cookie.Value } }; var qlikConnection = ConnectionManager.NewConnection(fullConnectionConfig, true); newManagedTask.Session.QlikConn = qlikConnection ?? throw new Exception("The web socket connection to qlik could not be established (Connector)."); //Create full engine config logger.Debug("Create configuration for the engine..."); Options.Analyser?.SetCheckPoint("StartReportJob", "Gernerate Config Json"); var newEngineConfig = CreateEngineConfig(request, newManagedTask.Session); //Remove emtpy Tasks without report infos newEngineConfig.Tasks.RemoveAll(t => t.Reports.Count == 0); foreach (var configTask in newEngineConfig.Tasks) { if (configTask.Id == Guid.Empty) { configTask.Id = Guid.NewGuid(); } else { //Check the task is of unique if (newEngineConfig.Tasks.Count(t => t.Id == configTask.Id) > 1) { throw new Exception("The task id is used twice. Please change the task id. This must always be unique."); } } foreach (var configReport in configTask.Reports) { //Important: Add bearer connection as last connection item. var firstConnection = configReport?.Connections?.FirstOrDefault() ?? null; if (firstConnection != null) { logger.Debug("Create bearer connection."); var newBearerConnection = CreateConnection(QlikCredentialType.JWT, newManagedTask.Session, firstConnection.App); configReport.Connections.Add(newBearerConnection); } //Check app Id var appList = Q2g.HelperQlik.Connection.PossibleApps; var activeApp = appList.FirstOrDefault(a => a.qDocId == firstConnection.App); if (activeApp == null) { throw new Exception($"The app id {firstConnection.App} was not found. Please check the app id or the security rules."); } //Read content from lib and content libary logger.Debug("Get template data from qlik."); if (configReport.Template != null) { var uploadData = FindTemplatePath(newManagedTask.Session, configReport.Template); logger.Debug("Upload template data to rest service."); var uploadId = Options.RestClient.UploadData(uploadData, configReport.Template.Input); logger.Debug($"Upload with id {uploadId} was successfully."); newManagedTask.FileUploadIds.Add(uploadId); } else { logger.Debug("No Template found. - Use alternative mode."); } // Perfomance analyser for the engine configReport.General.UsePerfomanceAnalyzer = Options.Config.UsePerfomanceAnalyzer; //Add all server uris foreach (var connection in configReport.Connections) { connection.LicenseServers.Add(new SerServer() { ServerUri = new Uri("https://license.analyticsgate.com"), Location = "de", Priority = 1 }); connection.LicenseServers.AddRange(Options.Config.Connection.LicenseServers); connection.RendererServers.Add(new SerServer() { ServerUri = new Uri("https://localhost:40271"), Location = "default", Priority = 100 }); connection.RendererServers.AddRange(Options.Config.Connection.RendererServers); } } } //Append upload ids on config var jobJson = JObject.FromObject(newEngineConfig); jobJson = AppendUploadGuids(jobJson, newManagedTask.FileUploadIds); newManagedTask.JobScript = jobJson; //Use the connector in the same App, than wait for data reload Options.Analyser?.SetCheckPoint("StartReportJob", "Start connector reporting task"); var scriptConnection = newEngineConfig?.Tasks?.SelectMany(s => s.Reports) ?.SelectMany(r => r.Connections) ?.FirstOrDefault(c => c.App == newManagedTask.Session.AppId) ?? null; //Wait for data load in single App mode WaitForDataLoad(newManagedTask, scriptConnection); Options.Analyser?.SetCheckPoint("StartReportJob", "End report generation"); newManagedTask.InternalStatus = InternalTaskStatus.CREATEREPORTJOBEND; } catch (Exception ex) { logger.Error(ex, "The reporting order could not be executed properly."); newManagedTask.Endtime = DateTime.Now; newManagedTask.Status = -1; newManagedTask.Error = ex; newManagedTask.InternalStatus = InternalTaskStatus.ERROR; Options.SessionHelper.Manager.MakeSocketFree(newManagedTask?.Session ?? null); } }, newManagedTask.Cancellation.Token); }
private void TaskProgressChange(ManagedTask value, ManagedTaskProgressItem progressItem) { Console.WriteLine($"Task Progress: {value.Name}, Step: {progressItem.StepName}"); }
private void TaskStatusChange(ManagedTask value, EManagedTaskStatus managedTaskStatus) { Console.WriteLine($"Task Status: {value.Name}, Status: {managedTaskStatus}"); }
static void Progress(ManagedTask managedTask, ManagedTaskProgressItem progressItem) { Console.WriteLine($"Task:{managedTask.Name}, Progress: {progressItem.StepName} "); }
public static bool DataLoadCheck(Uri serverUri, string scriptAppId, ManagedTask task, int timeout) { try { if (scriptAppId == null) { return(true); } if (timeout <= 0) { return(true); } var reloadTime = GetLastReloadTime(serverUri, task.Session.Cookie, scriptAppId); var tsConn = new CancellationTokenSource(timeout); var session = GetConnection(task.Session, tsConn.Token); var ts = new CancellationTokenSource(timeout); if (session != null) { logger.Debug("Reload - Attached session found."); var result = TestCalc(session.QlikConn.CurrentApp, ts.Token); return(true); } if (reloadTime == null) { return(false); } if (reloadTime != null) { task.Message = "Wait for data load..."; logger.Debug("Reload - Wait for finish scripts."); while (true) { Thread.Sleep(1000); if (ts.Token.IsCancellationRequested) { return(false); } var taskStatus = GetTaskStatus(serverUri, task.Session.Cookie, scriptAppId); if (taskStatus != 2) { return(true); } var tempLoad = GetLastReloadTime(serverUri, task.Session.Cookie, scriptAppId); if (tempLoad == null) { return(false); } if (reloadTime.Value.Ticks < tempLoad.Value.Ticks) { return(true); } } } return(false); } catch (Exception ex) { logger.Error(ex, "Check reload script failed."); return(false); } }
/// <summary> /// 模拟鼠标 /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private void MouseSimulateMenuItem_OnClick(object sender, RoutedEventArgs e) { ManagedTask.FactoryStart(SimulateMouseBehavior); }
/// <summary> /// Wait for this worker to complete /// </summary> /// <param name="Writer">Writer for log output</param> /// <returns>Return code from the thread</returns> public SequenceProbeResult Join(LineBasedTextWriter Writer) { // Finish the task instance BufferedTextWriter BufferedWriter = new BufferedTextWriter(); int ExitCode = ActiveInstance.Join(BufferedWriter); ActiveInstance.Dispose(); ActiveInstance = null; // Read the new state FileReference OutputFile = new FileReference(TaskStateFile.FullName + ".out"); SequenceWorker NewWorker = SequenceWorker.Deserialize(OutputFile.FullName); OutputFile.Delete(); // Make sure the exit code reflects the failure state. XGE can sometimes fail transferring back. if (ExitCode == 0 && !NewWorker.bResult) { ExitCode = -1; } // If it's a hard failure, print the compile log to the regular log if (ExitCode == -1) { Writer.WriteLine("Failed to compile {0}, exit code {1}:", UniqueName, ExitCode); foreach (string Line in NewWorker.CompileLog) { Writer.WriteLine(" > {0}", Line.Replace("error:", "err:")); } } // Annotate the log data if this is from a failed attempt. It still may be useful for debugging purposes. if (ExitCode != 0) { NewWorker.SummaryLog.Insert(0, String.Format("ExitCode={0}", ExitCode)); NewWorker.SummaryLog = NewWorker.SummaryLog.Select(x => "FAIL > " + x).ToList(); NewWorker.CompileLog.Insert(0, String.Format("ExitCode={0}", ExitCode)); NewWorker.CompileLog = NewWorker.CompileLog.Select(x => "FAIL > " + x).ToList(); } // Append the log data back to the local output File.AppendAllLines(SummaryLogFile.FullName, NewWorker.SummaryLog); NewWorker.SummaryLog.Clear(); File.AppendAllLines(CompileLogFile.FullName, NewWorker.CompileLog); NewWorker.CompileLog.Clear(); // If we failed, return the if (ExitCode != 0) { if (ExitCode == -1) { Writer.WriteLine("Warning: Failed to compile {0}, exit code {1}. Aborting.", UniqueName, ExitCode); return(SequenceProbeResult.Failed); } else { Writer.WriteLine("Warning: Failed to compile {0}; exit code {1}. Will retry.", UniqueName, ExitCode); return(SequenceProbeResult.FailedAllowRetry); } } // Update the task Worker = NewWorker; // Check if this is just an incremental update if (Type == SequenceProbeType.Verify) { // Save the task Worker.Serialize(TaskStateFile.FullName); // Return that we're done return(SequenceProbeResult.Completed); } else if (Type == SequenceProbeType.Optimize) { if (Worker.RemainingFragmentCount > 0) { // Get the top-most fragment - the one we've just established is a dependency for this leaf node - and add it to the list of known dependencies SourceFragment NextFragment = Fragments[Worker.RemainingFragmentCount - 1]; AddDependency(Worker, Fragments, Worker.RemainingFragmentCount - 1); Worker.SummaryLog.Add(String.Format(" [Added {0}: {1}]", Worker.RemainingFragmentCount - 1, Fragments[Worker.RemainingFragmentCount - 1].Location)); // Save the task Worker.Serialize(TaskStateFile.FullName); // Otherwise, return that we've just updated return(SequenceProbeResult.Updated); } else { // Save the task Worker.Serialize(TaskStateFile.FullName); // Return that we're done SetCompletedDependencies(); return(SequenceProbeResult.Completed); } } else { throw new NotImplementedException(); } }
public override async Task ExecuteFunction(IAsyncStreamReader <BundledRows> requestStream, IServerStreamWriter <BundledRows> responseStream, ServerCallContext context) { var response = new QlikResponse(); try { logger.Debug("The method 'ExecuteFunction' is called..."); //Read function header var functionHeader = context.RequestHeaders.ParseIMessageFirstOrDefault <FunctionRequestHeader>(); //Read common header var commonHeader = context.RequestHeaders.ParseIMessageFirstOrDefault <CommonRequestHeader>(); //Set appid logger.Info($"The Qlik app id '{commonHeader?.AppId}' in header found."); var qlikAppId = commonHeader?.AppId; //Set qlik user logger.Info($"The Qlik user '{commonHeader?.UserId}' in header found."); var domainUser = new DomainUser(commonHeader?.UserId); //Very important code line await context.WriteResponseHeadersAsync(new Metadata { { "qlik-cache", "no-store" } }); //Read parameter from qlik var row = GetParameter(requestStream); var userJson = GetParameterValue(0, row); //Parse request from qlik script logger.Debug("Parse user request..."); var request = QlikRequest.Parse(domainUser, qlikAppId, userJson); var functionCall = (ConnectorFunction)functionHeader.FunctionId; logger.Debug($"Call Function id: '{functionCall}' from client '{context?.Peer}'."); if (functionCall == ConnectorFunction.START) { #region Switch qlik user to app owner if (domainUser?.UserId == "sa_scheduler" && domainUser?.UserDirectory == "INTERNAL") { try { var oldUser = domainUser.ToString(); domainUser = new DomainUser("INTERNAL\\ser_scheduler"); logger.Debug($"Change Qlik user '{oldUser}' to task service user '{domainUser}'."); var connection = RuntimeOptions.Config.Connection; var tmpsession = RuntimeOptions.SessionHelper.Manager.CreateNewSession(connection, domainUser, qlikAppId); if (tmpsession == null) { throw new Exception("No session cookie generated. (Qlik Task)"); } var qrsHub = new QlikQrsHub(RuntimeOptions.Config.Connection.ServerUri, tmpsession.Cookie); qrsHub.ServerCertificateValidationCallback += (sender, certificate, chain, sslPolicyErrors) => true; domainUser = request.GetAppOwner(qrsHub, qlikAppId); if (domainUser == null) { throw new Exception("The owner of the App could not found."); } logger.Debug($"App owner '{domainUser}' found."); request.QlikUser = domainUser; } catch (Exception ex) { logger.Error(ex, "Could not switch the task user to real qlik user."); } } #endregion #region Function call SER.START logger.Debug("Function call SER.START..."); var newManagedTask = new ManagedTask() { StartTime = DateTime.Now, Message = "Create new report job...", Cancellation = new CancellationTokenSource(), Status = 0 }; RuntimeOptions.TaskPool.ManagedTasks.TryAdd(newManagedTask.Id, newManagedTask); var startFunction = new StartFunction(RuntimeOptions); startFunction.StartReportJob(request, newManagedTask); response.TaskId = newManagedTask.Id.ToString(); #endregion } else if (functionCall == ConnectorFunction.STOP) { #region Function call SER.STOP logger.Debug("Function call SER.STOP..."); var stopFunction = new StopFunction(RuntimeOptions); stopFunction.StopReportJobs(request); if (request.ManagedTaskId == "all") { response.Log = "All report jobs is stopping..."; } else { response.Log = $"Report job '{request.ManagedTaskId}' is stopping..."; } response.Status = 4; #endregion } else if (functionCall == ConnectorFunction.RESULT) { #region Function call SER.RESULT logger.Debug("Function call SER.RESULT..."); var resultFunction = new ResultFunction(RuntimeOptions); response = resultFunction.FormatJobResult(request); #endregion } else if (functionCall == ConnectorFunction.STATUS) { #region Function call SER.STATUS logger.Debug("Function call SER.STATUS..."); var statusFunction = new StatusFunction(RuntimeOptions); response = statusFunction.GetStatusResponse(request); #endregion } else { throw new Exception($"The id '{functionCall}' of the function call was unknown."); } } catch (Exception ex) { logger.Error(ex, $"The method 'ExecuteFunction' failed with error '{ex.Message}'."); response.Status = -1; response.SetErrorMessage(ex); } finally { logger.Trace($"Qlik status result: {JsonConvert.SerializeObject(response)}"); await responseStream.WriteAsync(GetResult(response)); LogManager.Flush(); } }