private void RetryRecurringJob( IStorageConnection connection, string recurringJobId, RecurringJobEntity recurringJob, Exception error) { IReadOnlyDictionary <string, string> changedFields; DateTime?nextExecution; var errorString = error.ToStringWithOriginalStackTrace(States.FailedState.MaxLinesInExceptionDetails); if (recurringJob.RetryAttempt < MaxRetryAttemptCount) { var delay = _pollingDelay > TimeSpan.Zero ? _pollingDelay : TimeSpan.FromMinutes(1); _logger.WarnException( $"Recurring job '{recurringJobId}' can't be scheduled due to an error and will be retried in {delay}.", error); recurringJob.ScheduleRetry(delay, errorString, out changedFields, out nextExecution); } else { _logger.ErrorException( $"Recurring job '{recurringJobId}' can't be scheduled due to an error and will be disabled.", error); recurringJob.Disable(errorString, out changedFields, out nextExecution); } using (var transaction = connection.CreateWriteTransaction()) { transaction.UpdateRecurringJob(recurringJob, changedFields, nextExecution, _logger); transaction.Commit(); } }
private void ScheduleRecurringJob(BackgroundProcessContext context, IStorageConnection connection, string recurringJobId, RecurringJobEntity recurringJob, DateTime now) { // We always start a transaction, regardless our recurring job was updated or not, // to prevent from infinite loop, when there's an old processing server (pre-1.7.0) // in our environment that doesn't know it should modify the score for entries in // the recurring jobs set. using (var transaction = connection.CreateWriteTransaction()) { try { // We can't handle recurring job with unsupported versions - there may be additional // features. we don't know about. We also shouldn't stop the whole scheduler as // there may be jobs with lower versions. Instead, we'll re-schedule such a job and // emit a warning message to the log. if (recurringJob.Version.HasValue && recurringJob.Version > MaxSupportedVersion) { throw new NotSupportedException($"Server '{context.ServerId}' can't process recurring job '{recurringJobId}' of version '{recurringJob.Version ?? 1}'. Max supported version of this server is '{MaxSupportedVersion}'."); } BackgroundJob backgroundJob = null; IReadOnlyDictionary <string, string> changedFields; if (recurringJob.TrySchedule(out var nextExecution, out var error)) { if (nextExecution.HasValue && nextExecution <= now) { backgroundJob = _factory.TriggerRecurringJob(context.Storage, connection, _profiler, recurringJob, now); if (String.IsNullOrEmpty(backgroundJob?.Id)) { _logger.Debug($"Recurring job '{recurringJobId}' execution at '{nextExecution}' has been canceled."); } } recurringJob.IsChanged(out changedFields, out nextExecution); } else { throw new InvalidOperationException("Recurring job can't be scheduled, see inner exception for details.", error); } if (backgroundJob != null) { _factory.StateMachine.EnqueueBackgroundJob( context.Storage, connection, transaction, recurringJob, backgroundJob, "Triggered by recurring job scheduler", _profiler); } transaction.UpdateRecurringJob(recurringJob, changedFields, nextExecution, _logger); } catch (Exception ex) { throw new BackgroundJobClientException(ex.Message, ex); } // We should commit transaction outside of the internal try/catch block, because these // exceptions are always due to network issues, and in case of a timeout exception we // can't determine whether it was actually succeeded or not, and can't perform an // idempotent retry in this case. transaction.Commit(); } }
public override void Execute() { WriteLiteral("\r\n"); #line 11 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Layout = new LayoutPage(Strings.RecurringJobsPage_Title); List <RecurringJobDto> recurringJobs; int from, perPage; int.TryParse(Query("from"), out from); int.TryParse(Query("count"), out perPage); Pager pager = null; using (var connection = Storage.GetConnection()) { var storageConnection = connection as JobStorageConnection; if (storageConnection != null) { pager = new Pager(from, perPage, storageConnection.GetRecurringJobCount()); recurringJobs = storageConnection.GetRecurringJobs(pager.FromRecord, pager.FromRecord + pager.RecordsPerPage - 1); } else { recurringJobs = connection.GetRecurringJobs(); } } #line default #line hidden WriteLiteral("\r\n<div class=\"row\">\r\n <div class=\"col-md-12\">\r\n <h1 class=\"page-header\"" + ">"); #line 39 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.RecurringJobsPage_Title); #line default #line hidden WriteLiteral("</h1>\r\n"); #line 40 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (recurringJobs.Count == 0) { #line default #line hidden WriteLiteral(" <div class=\"alert alert-info\">\r\n "); #line 43 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.RecurringJobsPage_NoJobs); #line default #line hidden WriteLiteral("\r\n </div>\r\n"); #line 45 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } else { #line default #line hidden WriteLiteral(" <div class=\"js-jobs-list\">\r\n <div class=\"btn-toolbar b" + "tn-toolbar-top\">\r\n"); #line 50 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (!IsReadOnly) { #line default #line hidden WriteLiteral(" <button class=\"js-jobs-list-command btn btn-sm btn-primar" + "y\"\r\n data-url=\""); #line 53 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Url.To("/recurring/trigger")); #line default #line hidden WriteLiteral("\"\r\n data-loading-text=\""); #line 54 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.RecurringJobsPage_Triggering); #line default #line hidden WriteLiteral("\"\r\n disabled=\"disabled\">\r\n " + " <span class=\"glyphicon glyphicon-play-circle\"></span>\r\n " + " "); #line 57 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.RecurringJobsPage_TriggerNow); #line default #line hidden WriteLiteral("\r\n </button>\r\n"); #line 59 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden #line 60 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (!IsReadOnly) { #line default #line hidden WriteLiteral(" <button class=\"js-jobs-list-command btn btn-sm btn-defaul" + "t\"\r\n data-url=\""); #line 63 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Url.To("/recurring/remove")); #line default #line hidden WriteLiteral("\"\r\n data-loading-text=\""); #line 64 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.Common_Deleting); #line default #line hidden WriteLiteral("\"\r\n data-confirm=\""); #line 65 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.Common_DeleteConfirm); #line default #line hidden WriteLiteral("\"\r\n disabled=\"disabled\">\r\n " + " <span class=\"glyphicon glyphicon-remove\"></span>\r\n " + " "); #line 68 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.Common_Delete); #line default #line hidden WriteLiteral("\r\n </button>\r\n"); #line 70 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden #line 71 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (!IsReadOnly) { #line default #line hidden WriteLiteral(" <button class=\"js-jobs-list-command btn btn-sm btn-defaul" + "t\"\r\n data-url=\""); #line 74 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Url.To("/recurring/start")); #line default #line hidden WriteLiteral("\"\r\n data-loading-text=\""); #line 75 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.Start); #line default #line hidden WriteLiteral("\"\r\n disabled=\"disabled\">\r\n " + " <span class=\"glyphicon glyphicon-play\"></span>\r\n " + ""); #line 78 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.Start); #line default #line hidden WriteLiteral("\r\n </button>\r\n"); #line 80 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden #line 81 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (!IsReadOnly) { #line default #line hidden WriteLiteral(" <button class=\"js-jobs-list-command btn btn-sm btn-defaul" + "t\"\r\n data-url=\""); #line 84 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Url.To("/recurring/stop")); #line default #line hidden WriteLiteral("\"\r\n data-loading-text=\""); #line 85 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.Stop); #line default #line hidden WriteLiteral("\"\r\n disabled=\"disabled\">\r\n " + " <span class=\"glyphicon glyphicon-pause\"></span>\r\n " + " "); #line 88 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.Stop); #line default #line hidden WriteLiteral("\r\n </button>\r\n"); #line 90 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden #line 91 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (pager != null) { #line default #line hidden WriteLiteral(" "); WriteLiteral(" "); #line 93 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Html.PerPageSelector(pager)); #line default #line hidden WriteLiteral("\r\n"); #line 94 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden WriteLiteral(" </div>\r\n\r\n <div class=\"table-responsive\">\r\n " + " <table class=\"table\">\r\n <thead>\r\n " + " <tr>\r\n"); #line 101 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (!IsReadOnly) { #line default #line hidden WriteLiteral(" <th class=\"min-width\">\r\n " + " <input type=\"checkbox\" class=\"js-jobs-list-select-all\" />\r\n " + " </th>\r\n"); #line 106 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden WriteLiteral(" <th class=\"min-width\">"); #line 107 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.Common_Id); #line default #line hidden WriteLiteral("</th>\r\n <th class=\"min-width\">"); #line 108 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.RecurringJobsPage_Table_Cron); #line default #line hidden WriteLiteral("</th>\r\n <th class=\"min-width\">"); #line 109 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.RecurringJobsPage_Table_TimeZone); #line default #line hidden WriteLiteral("</th>\r\n <th>"); #line 110 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.Common_Job); #line default #line hidden WriteLiteral("</th>\r\n <th class=\"align-right min-width\">"); #line 111 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.RecurringJobsPage_Table_NextExecution); #line default #line hidden WriteLiteral("</th>\r\n <th class=\"align-right min-width\">"); #line 112 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.RecurringJobsPage_Table_LastExecution); #line default #line hidden WriteLiteral("</th>\r\n <th class=\"align-right min-width\">"); #line 113 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.Common_Created); #line default #line hidden WriteLiteral("</th>\r\n <th class=\"align-right min-width\">"); #line 114 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.Status); #line default #line hidden WriteLiteral("</th>\r\n </tr>\r\n </thead>\r\n " + " <tbody>\r\n"); #line 118 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" foreach (var job in recurringJobs) { #line default #line hidden WriteLiteral(" <tr class=\"js-jobs-list-row hover\">\r\n"); #line 121 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (!IsReadOnly) { #line default #line hidden WriteLiteral(" <td>\r\n " + " <input type=\"checkbox\" class=\"js-jobs-list-checkbox\" name=\"jobs[]\" valu" + "e=\""); #line 124 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(job.Id); #line default #line hidden WriteLiteral("\" />\r\n </td>\r\n"); #line 126 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden WriteLiteral(" <td class=\"min-width\">"); #line 127 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(job.Id); #line default #line hidden WriteLiteral("</td>\r\n <td class=\"min-width\">\r\n " + " "); WriteLiteral("\r\n"); #line 130 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" string cronDescription = null; bool cronError = false; if (!String.IsNullOrEmpty(job.Cron)) { try { RecurringJobEntity.ParseCronExpression(job.Cron); } catch (Exception ex) { cronDescription = ex.Message; cronError = true; } if (cronDescription == null) { #if FEATURE_CRONDESCRIPTOR try { cronDescription = CronExpressionDescriptor.ExpressionDescriptor.GetDescription(job.Cron); } catch (FormatException) { } #endif } } #line default #line hidden WriteLiteral("\r\n"); #line 161 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (cronDescription != null) { #line default #line hidden WriteLiteral(" <code title=\""); #line 163 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(cronDescription); #line default #line hidden WriteLiteral("\">\r\n"); #line 164 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (cronError) { #line default #line hidden WriteLiteral(" <span class=\"glyphicon glyphi" + "con-exclamation-sign\"></span>\r\n"); #line 167 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden WriteLiteral(" "); #line 168 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(job.Cron); #line default #line hidden WriteLiteral("\r\n </code>\r\n"); #line 170 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } else { #line default #line hidden WriteLiteral(" <code>"); #line 173 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(job.Cron); #line default #line hidden WriteLiteral("</code>\r\n"); #line 174 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden WriteLiteral(" </td>\r\n <t" + "d class=\"min-width\">\r\n"); #line 177 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (!String.IsNullOrWhiteSpace(job.TimeZoneId)) { string displayName; Exception exception = null; try { var resolver = DashboardOptions.TimeZoneResolver ?? new DefaultTimeZoneResolver(); displayName = resolver.GetTimeZoneById(job.TimeZoneId).DisplayName; } catch (Exception ex) { displayName = null; exception = ex; } #line default #line hidden WriteLiteral(" <span title=\""); #line 193 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(displayName); #line default #line hidden WriteLiteral("\" data-container=\"body\">\r\n "); #line 194 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(job.TimeZoneId); #line default #line hidden WriteLiteral("\r\n"); #line 195 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (exception != null) { #line default #line hidden WriteLiteral(" <span class=\"glyphicon glyphi" + "con-exclamation-sign\" title=\""); #line 197 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(exception.Message); #line default #line hidden WriteLiteral("\"></span>\r\n"); #line 198 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden WriteLiteral(" </span>\r\n"); #line 200 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } else { #line default #line hidden WriteLiteral(" "); WriteLiteral(" UTC\r\n"); #line 204 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden WriteLiteral(" </td>\r\n <t" + "d class=\"word-break\">\r\n"); #line 207 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (job.Job != null) { #line default #line hidden WriteLiteral(" "); WriteLiteral(" "); #line 209 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Html.JobName(job.Job)); #line default #line hidden WriteLiteral("\r\n"); #line 210 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } else if (job.LoadException != null && job.LoadException.InnerException != null) { #line default #line hidden WriteLiteral(" <em>"); #line 213 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(job.LoadException.InnerException.Message); #line default #line hidden WriteLiteral("</em>\r\n"); #line 214 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } else if (job.LoadException != null) { #line default #line hidden WriteLiteral(" <em>"); #line 217 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(job.LoadException.Message); #line default #line hidden WriteLiteral("</em>\r\n"); #line 218 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } else { #line default #line hidden WriteLiteral(" <em>"); #line 221 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.Common_NotAvailable); #line default #line hidden WriteLiteral("</em>\r\n"); #line 222 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden WriteLiteral(" </td>\r\n <t" + "d class=\"align-right min-width\">\r\n"); #line 225 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (job.NextExecution != null) { #line default #line hidden #line 227 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Html.RelativeTime(job.NextExecution.Value)); #line default #line hidden #line 227 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } else { if (!String.IsNullOrEmpty(job.Error)) { #line default #line hidden WriteLiteral(" <span class=\"label label-danger t" + "ext-uppercase\" title=\""); #line 233 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(job.Error); #line default #line hidden WriteLiteral("\">"); #line 233 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.Common_Error); #line default #line hidden WriteLiteral("</span>\r\n"); #line 234 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } else { #line default #line hidden WriteLiteral(" <span class=\"label label-warning " + "text-uppercase\" title=\""); #line 237 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.RecurringJobsPage_RecurringJobDisabled_Tooltip); #line default #line hidden WriteLiteral("\">"); #line 237 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.Common_Disabled); #line default #line hidden WriteLiteral("</span>\r\n"); #line 238 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } } #line default #line hidden WriteLiteral(" </td>\r\n <t" + "d class=\"align-right min-width\">\r\n"); #line 242 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (job.LastExecution != null) { if (!String.IsNullOrEmpty(job.LastJobId)) { #line default #line hidden WriteLiteral(" <a href=\""); #line 246 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Url.JobDetails(job.LastJobId)); #line default #line hidden WriteLiteral("\" style=\"text-decoration: none\">\r\n " + " <span class=\"label label-default label-hover\" style=\""); #line 247 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write($"background-color: {JobHistoryRenderer.GetForegroundStateColor(job.LastJobState ?? EnqueuedState.StateName)};"); #line default #line hidden WriteLiteral("\">\r\n "); #line 248 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Html.RelativeTime(job.LastExecution.Value)); #line default #line hidden WriteLiteral("\r\n </span>\r\n " + " </a>\r\n"); #line 251 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } else { #line default #line hidden WriteLiteral(" <em>\r\n " + " "); #line 255 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.RecurringJobsPage_Canceled); #line default #line hidden WriteLiteral(" "); #line 255 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Html.RelativeTime(job.LastExecution.Value)); #line default #line hidden WriteLiteral("\r\n </em>\r\n"); #line 257 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } } else { #line default #line hidden WriteLiteral(" <em>"); #line 261 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Strings.Common_NotAvailable); #line default #line hidden WriteLiteral("</em>\r\n"); #line 262 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden WriteLiteral(" </td>\r\n <t" + "d class=\"align-right min-width\">\r\n"); #line 265 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (job.CreatedAt != null) { #line default #line hidden #line 267 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Html.RelativeTime(job.CreatedAt.Value)); #line default #line hidden #line 267 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } else { #line default #line hidden WriteLiteral(" <em>N/A</em>\r\n"); #line 272 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden WriteLiteral(" </td>\r\n <t" + "d class=\"align-right min-width\">\r\n "); #line 275 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(job.IsActive ? "Active" : "Passive"); #line default #line hidden WriteLiteral("\r\n </td>\r\n </tr" + ">\r\n"); #line 278 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden WriteLiteral(" </tbody>\r\n </table>\r\n <" + "/div>\r\n\r\n"); #line 283 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" if (pager != null) { #line default #line hidden WriteLiteral(" "); WriteLiteral(" "); #line 285 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" Write(Html.Paginator(pager)); #line default #line hidden WriteLiteral("\r\n"); #line 286 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden WriteLiteral(" </div>\r\n"); #line 288 "..\..\Dashboard\Pages\RecurringJobsPage.cshtml" } #line default #line hidden WriteLiteral(" </div>\r\n</div> "); }