public List <WorkItem> GetWorkItems(int count, WorkItemPriorityEnum priority) { _nowRunningWorkItems.Clear(); using (var context = new WorkItemContext()) { var now = DateTime.Now; var workItems = (from row in context.WorkItems where row.ScheduledTime < now && row.Priority == priority && (row.Status == WorkItemStatusEnum.Pending || row.Status == WorkItemStatusEnum.Idle) orderby row.Priority, row.ScheduledTime select row).Take(count); foreach (var item in workItems) { // 标记任务处于正在运行状态,避免任务的多次执行 item.Status = WorkItemStatusEnum.InProgress; _nowRunningWorkItems.Add(item); } return(new List <WorkItem>(_nowRunningWorkItems)); } }
public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData<RemoveProjectWorkItem>(); _logger.Info("Received remove project work item for: {0} Reset Data: {1}", workItem.ProjectId, workItem.Reset); await context.ReportProgressAsync(0, "Starting deletion...").AnyContext(); var project = await _projectRepository.GetByIdAsync(workItem.ProjectId).AnyContext(); if (project == null) { await context.ReportProgressAsync(100, workItem.Reset ? "Project data reset" : "Project deleted").AnyContext(); return; } if (!workItem.Reset) { await context.ReportProgressAsync(20, "Removing tokens").AnyContext(); await _tokenRepository.RemoveAllByOrganizationIdsAsync(new[] { project.Id }).AnyContext(); await context.ReportProgressAsync(40, "Removing web hooks").AnyContext(); await _webHookRepository.RemoveAllByOrganizationIdsAsync(new[] { project.Id }).AnyContext(); } await context.ReportProgressAsync(60, "Resetting project data").AnyContext(); await _eventRepository.RemoveAllByProjectIdsAsync(new[] { project.Id }).AnyContext(); await _stackRepository.RemoveAllByProjectIdsAsync(new[] { project.Id }).AnyContext(); if (!workItem.Reset) { await context.ReportProgressAsync(80, "Removing project").AnyContext(); await _projectRepository.RemoveAsync(project.Id).AnyContext(); } await context.ReportProgressAsync(100, workItem.Reset ? "Project data reset" : "Project deleted").AnyContext(); }
public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData<SetLocationFromGeoWorkItem>(); GeoResult result; if (!GeoResult.TryParse(workItem.Geo, out result)) return; var location = await _cacheClient.GetAsync<Location>(workItem.Geo, null).AnyContext(); if (location == null) { try { result = await _geocodeService.ReverseGeocodeAsync(result.Latitude.GetValueOrDefault(), result.Longitude.GetValueOrDefault()).AnyContext(); location = result.ToLocation(); await _metricsClient.CounterAsync(MetricNames.UsageGeocodingApi).AnyContext(); } catch (Exception ex) { Log.Error(ex, "Error occurred looking up reverse geocode: {0}", workItem.Geo); } } if (location == null) return; await _cacheClient.SetAsync(workItem.Geo, location, TimeSpan.FromDays(3)).AnyContext(); var ev = await _eventRepository.GetByIdAsync(workItem.EventId).AnyContext(); if (ev == null) return; ev.SetLocation(location); await _eventRepository.SaveAsync(ev).AnyContext(); }
public Task HandleItem(WorkItemContext context) { var workItem = context.GetData<ReindexWorkItem>(); Log.Info().Message("Received reindex work item for new index {0}", workItem.NewIndex).Write(); var startTime = DateTime.UtcNow.AddSeconds(-1); context.ReportProgress(0, "Starting reindex..."); var result = Reindex(workItem, context, 0, 90, workItem.StartUtc); context.ReportProgress(90, String.Format("Total: {0} Completed: {1}", result.Total, result.Completed)); // TODO: Check to make sure the docs have been added to the new index before changing alias if (!String.IsNullOrEmpty(workItem.Alias)) { _client.Alias(x => x.Remove(a => a.Alias(workItem.Alias).Index(workItem.OldIndex)).Add(a => a.Alias(workItem.Alias).Index(workItem.NewIndex))); context.ReportProgress(98, String.Format("Updated alias: {0} Remove: {1} Add: {2}", workItem.Alias, workItem.OldIndex, workItem.NewIndex)); } _client.Refresh(); var secondPassResult = Reindex(workItem, context, 90, 98, startTime); context.ReportProgress(98, String.Format("Total: {0} Completed: {1}", secondPassResult.Total, secondPassResult.Completed)); if (workItem.DeleteOld) { _client.Refresh(); long newDocCount = _client.Count(d => d.Index(workItem.OldIndex)).Count; long oldDocCount = _client.Count(d => d.Index(workItem.OldIndex)).Count; context.ReportProgress(98, String.Format("Old Docs: {0} New Docs: {1}", oldDocCount, newDocCount)); if (newDocCount >= oldDocCount) _client.DeleteIndex(d => d.Index(workItem.OldIndex)); context.ReportProgress(98, String.Format("Deleted index: {0}", workItem.OldIndex)); } context.ReportProgress(100); return TaskHelper.Completed(); }
public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData<ReindexWorkItem>(); Logger.Info().Message("Received reindex work item for new index {0}", workItem.NewIndex).Write(); var startTime = DateTime.UtcNow.AddSeconds(-1); await context.ReportProgressAsync(0, "Starting reindex...").AnyContext(); var result = await ReindexAsync(workItem, context, 0, 90, workItem.StartUtc).AnyContext(); await context.ReportProgressAsync(90, $"Total: {result.Total} Completed: {result.Completed}").AnyContext(); // TODO: Check to make sure the docs have been added to the new index before changing alias if (!String.IsNullOrEmpty(workItem.Alias)) { await _client.AliasAsync(x => x.Remove(a => a.Alias(workItem.Alias).Index(workItem.OldIndex)).Add(a => a.Alias(workItem.Alias).Index(workItem.NewIndex))).AnyContext(); await context.ReportProgressAsync(98, $"Updated alias: {workItem.Alias} Remove: {workItem.OldIndex} Add: {workItem.NewIndex}").AnyContext(); } await _client.RefreshAsync().AnyContext(); var secondPassResult = await ReindexAsync(workItem, context, 90, 98, startTime).AnyContext(); await context.ReportProgressAsync(98, $"Total: {secondPassResult.Total} Completed: {secondPassResult.Completed}").AnyContext(); if (workItem.DeleteOld) { await _client.RefreshAsync().AnyContext(); long newDocCount = (await _client.CountAsync(d => d.Index(workItem.OldIndex)).AnyContext()).Count; long oldDocCount = (await _client.CountAsync(d => d.Index(workItem.OldIndex)).AnyContext()).Count; await context.ReportProgressAsync(98, $"Old Docs: {oldDocCount} New Docs: {newDocCount}").AnyContext(); if (newDocCount >= oldDocCount) await _client.DeleteIndexAsync(d => d.Index(workItem.OldIndex)).AnyContext(); await context.ReportProgressAsync(98, $"Deleted index: {workItem.OldIndex}").AnyContext(); } await context.ReportProgressAsync(100).AnyContext(); }
public void ResetInProgressWorkItems() { using (var context = new WorkItemContext()) { var query = from row in context.WorkItems where row.Status == WorkItemStatusEnum.InProgress || row.Status == WorkItemStatusEnum.DeleteInProgress || row.Status == WorkItemStatusEnum.Canceling orderby row.Priority select row; foreach (var item in query.ToList()) { switch (item.Status) { case WorkItemStatusEnum.InProgress: item.Status = WorkItemStatusEnum.Pending; break; case WorkItemStatusEnum.DeleteInProgress: item.Status = WorkItemStatusEnum.Deleted; break; case WorkItemStatusEnum.Canceling: item.Status = WorkItemStatusEnum.Canceled; break; } } context.SaveChanges(); } }
public override async Task HandleItemAsync(WorkItemContext context) { var wi = context.GetData <StackWorkItem>(); using (Log.BeginScope(new ExceptionlessState().Organization(wi.OrganizationId).Project(wi.ProjectId))) { if (wi.Delete) { await _eventRepository.RemoveAllByStackIdAsync(wi.OrganizationId, wi.ProjectId, wi.StackId).AnyContext(); await _stackRepository.RemoveAsync(wi.StackId).AnyContext(); return; } if (wi.UpdateIsFixed) { await _eventRepository.UpdateFixedByStackAsync(wi.OrganizationId, wi.ProjectId, wi.StackId, wi.IsFixed).AnyContext(); } if (wi.UpdateIsHidden) { await _eventRepository.UpdateHiddenByStackAsync(wi.OrganizationId, wi.ProjectId, wi.StackId, wi.IsHidden).AnyContext(); } } }
// NOTE: Uncomment to ensure only one work item handler is called at a time. //public override Task<ILock> GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = new CancellationToken()) { // return _lockProvider.AcquireAsync(nameof(DeleteValueWorkItemHandler), cancellationToken: cancellationToken); //} public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData <DeleteValueWorkItem>(); await context.ReportProgressAsync(0, $"Starting to delete item: {workItem.Id}."); await context.ReportProgressAsync(1, "If you are seeing multiple progresses. Please uncomment the lock in the DeleteValueWorkItemHandler."); await Task.Delay(TimeSpan.FromSeconds(2.5)); await context.ReportProgressAsync(50, "Deleting"); await Task.Delay(TimeSpan.FromSeconds(.5)); await context.ReportProgressAsync(70, "Deleting."); await Task.Delay(TimeSpan.FromSeconds(.5)); await context.ReportProgressAsync(90, "Deleting.."); await Task.Delay(TimeSpan.FromSeconds(.5)); await _cacheClient.RemoveAsync(workItem.Id); await _publisher.PublishAsync(new EntityChanged { ChangeType = ChangeType.Removed, Id = workItem.Id }); await context.ReportProgressAsync(100); }
public override async Task HandleItemAsync(WorkItemContext context) { const int LIMIT = 100; var workItem = context.GetData<ProjectMaintenanceWorkItem>(); Log.Info("Received upgrade projects work item. Update Default Bot List: {0} IncrementConfigurationVersion: {1}", workItem.UpdateDefaultBotList, workItem.IncrementConfigurationVersion); var results = await _projectRepository.GetAllAsync(paging: new PagingOptions().WithLimit(LIMIT)).AnyContext(); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { foreach (var project in results.Documents) { if (workItem.UpdateDefaultBotList) project.SetDefaultUserAgentBotPatterns(); if (workItem.IncrementConfigurationVersion) project.Configuration.IncrementVersion(); } if (workItem.UpdateDefaultBotList) await _projectRepository.SaveAsync(results.Documents).AnyContext(); // Sleep so we are not hammering the backend. await Task.Delay(TimeSpan.FromSeconds(2.5)).AnyContext(); await results.NextPageAsync().AnyContext(); if (results.Documents.Count > 0) await context.RenewLockAsync().AnyContext(); } }
public override async Task HandleItemAsync(WorkItemContext context) { const int LIMIT = 100; var workItem = context.GetData <ProjectMaintenanceWorkItem>(); Log.LogInformation("Received upgrade projects work item. Update Default Bot List: {UpdateDefaultBotList} IncrementConfigurationVersion: {IncrementConfigurationVersion}", workItem.UpdateDefaultBotList, workItem.IncrementConfigurationVersion); var results = await _projectRepository.GetAllAsync(o => o.PageLimit(LIMIT)).AnyContext(); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { foreach (var project in results.Documents) { if (workItem.UpdateDefaultBotList) { project.SetDefaultUserAgentBotPatterns(); } if (workItem.IncrementConfigurationVersion) { project.Configuration.IncrementVersion(); } if (workItem.RemoveOldUsageStats) { foreach (var usage in project.OverageHours.Where(u => u.Date < SystemClock.UtcNow.Subtract(TimeSpan.FromDays(3))).ToList()) { project.OverageHours.Remove(usage); } foreach (var usage in project.Usage.Where(u => u.Date < SystemClock.UtcNow.Subtract(TimeSpan.FromDays(366))).ToList()) { project.Usage.Remove(usage); } } } if (workItem.UpdateDefaultBotList || workItem.IncrementConfigurationVersion || workItem.RemoveOldUsageStats) { await _projectRepository.SaveAsync(results.Documents).AnyContext(); } // Sleep so we are not hammering the backend. await SystemClock.SleepAsync(TimeSpan.FromSeconds(2.5)).AnyContext(); if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync().AnyContext()) { break; } if (results.Documents.Count > 0) { await context.RenewLockAsync().AnyContext(); } } }
public override void Postpone(TimeSpan delay) { using (var ctx = new WorkItemContext()) { var result = from row in ctx.WorkItems where row.Oid == Item.Oid select row; } }
public List <int> GetWorkItemIds(StoredQuery query, FieldList fields) { WorkItemContext context = new WorkItemContext { ProjectId = query.ProjectId, Me = WorkItemsContext.WhoAmI }; var invoker = new SoapInvoker(this); var envelope = invoker.CreateEnvelope("QueryWorkitems", headerName); envelope.Header.Add(GetHeaderElement()); XNamespace queryNs = XNamespace.Get(""); envelope.Body.Add(new XElement(MessageNs + "psQuery", new XElement(queryNs + "Query", new XAttribute("Product", this.Url.ToString()), query.GetQueryXml(context, fields)))); XElement sorting = query.GetSortingXml(); foreach (var items in sorting.DescendantsAndSelf()) { items.Name = MessageNs + items.Name.LocalName; } envelope.Body.Add(sorting); envelope.Body.Add(new XElement(MessageNs + "useMaster", "false")); var response = invoker.InvokeResponse(); var queryIds = response.Element(MessageNs + "resultIds").Element("QueryIds"); if (queryIds == null) { return(new List <int>()); } var list = new List <int>(); foreach (var item in queryIds.Elements("id")) { var startId = item.Attribute("s"); if (startId == null) { continue; } var s = Convert.ToInt32(startId.Value); var endId = item.Attribute("e"); if (endId != null) { var e = Convert.ToInt32(endId.Value); var range = Enumerable.Range(s, e - s + 1); list.AddRange(range); } else { list.Add(s); } } return(list); }
public bool AddWorkItem(WorkItem item) { using (var ctx = new WorkItemContext()) { ctx.WorkItems.Add(item); ctx.SaveChanges(); } return(true); }
public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData<SetProjectIsConfiguredWorkItem>(); Logger.Info().Message($"Setting Is Configured for project: {workItem.ProjectId}").Write(); var project = await _projectRepository.GetByIdAsync(workItem.ProjectId).AnyContext(); if (project == null || project.IsConfigured.GetValueOrDefault()) return; project.IsConfigured = workItem.IsConfigured || await _eventRepository.GetCountByProjectIdAsync(project.Id).AnyContext() > 0; await _projectRepository.SaveAsync(project, true).AnyContext(); }
public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData<OrganizationNotificationWorkItem>(); _logger.Info("Received organization notification work item for: {0} IsOverHourlyLimit: {1} IsOverMonthlyLimit: {2}", workItem.OrganizationId, workItem.IsOverHourlyLimit, workItem.IsOverMonthlyLimit); var organization = await _organizationRepository.GetByIdAsync(workItem.OrganizationId, true).AnyContext(); if (organization == null) return; if (workItem.IsOverMonthlyLimit) await SendOverageNotificationsAsync(organization, workItem.IsOverHourlyLimit, workItem.IsOverMonthlyLimit).AnyContext(); }
public WorkItem GetWorkItem(long oid) { using (var ctx = new WorkItemContext()) { var obj = (from row in ctx.WorkItems where row.Oid == oid select row).FirstOrDefault(); return(obj); } }
public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData<OrganizationNotificationWorkItem>(); Log.Info("Received organization notification work item for: {0} IsOverHourlyLimit: {1} IsOverMonthlyLimit: {2}", workItem.OrganizationId, workItem.IsOverHourlyLimit, workItem.IsOverMonthlyLimit); var organization = await _organizationRepository.GetByIdAsync(workItem.OrganizationId, true).AnyContext(); if (organization == null) return; if (workItem.IsOverMonthlyLimit) await SendOverageNotificationsAsync(organization, workItem.IsOverHourlyLimit, workItem.IsOverMonthlyLimit).AnyContext(); }
private async Task<ReindexResult> ReindexAsync(ReindexWorkItem workItem, WorkItemContext context, int startProgress = 0, int endProgress = 100, DateTime? startTime = null) { const int pageSize = 100; const string scroll = "10s"; var scanResults = await _client.SearchAsync<JObject>(s => s.Index(workItem.OldIndex).AllTypes().Filter(f => startTime.HasValue ? f.Range(r => r.OnField(workItem.TimestampField ?? "_timestamp").Greater(startTime.Value)) : f.MatchAll()).From(0).Take(pageSize).SearchType(SearchType.Scan).Scroll(scroll)).AnyContext(); if (!scanResults.IsValid || scanResults.ScrollId == null) { Logger.Error().Message("Invalid search result: message={0}", scanResults.GetErrorMessage()).Write(); return new ReindexResult(); } long totalHits = scanResults.Total; long completed = 0; int page = 0; var results = await _client.ScrollAsync<JObject>(scroll, scanResults.ScrollId).AnyContext(); while (results.Documents.Any()) { var bulkDescriptor = new BulkDescriptor(); foreach (var hit in results.Hits) { var h = hit; // TODO: Add support for doing JObject based schema migrations bulkDescriptor.Index<JObject>(idx => idx.Index(workItem.NewIndex).Type(h.Type).Id(h.Id).Document(h.Source)); } var bulkResponse = await _client.BulkAsync(bulkDescriptor).AnyContext(); if (!bulkResponse.IsValid) { string message = $"Reindex bulk error: old={workItem.OldIndex} new={workItem.NewIndex} page={page} message={bulkResponse.GetErrorMessage()}"; Logger.Warn().Message(message).Write(); // try each doc individually so we can see which doc is breaking us foreach (var hit in results.Hits) { var h = hit; var response = await _client.IndexAsync(h.Source, d => d.Index(workItem.NewIndex).Type(h.Type).Id(h.Id)).AnyContext(); if (response.IsValid) continue; message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={hit.Id} page={page} message={response.GetErrorMessage()}"; Logger.Error().Message(message).Write(); throw new ReindexException(response.ConnectionStatus, message); } } completed += bulkResponse.Items.Count(); await context.ReportProgressAsync(CalculateProgress(totalHits, completed, startProgress, endProgress), $"Total: {totalHits} Completed: {completed}").AnyContext(); results = await _client.ScrollAsync<JObject>(scroll, results.ScrollId).AnyContext(); page++; } return new ReindexResult { Total = totalHits, Completed = completed }; }
public override async Task HandleItemAsync(WorkItemContext context) { var jobData = context.GetData <MyWorkItem>(); Assert.Equal("Test", jobData.SomeData); for (int i = 1; i < 10; i++) { await SystemClock.SleepAsync(100); await context.ReportProgressAsync(10 *i); } }
public override async Task HandleItemAsync(WorkItemContext context) { var wi = context.GetData <RemoveStacksWorkItem>(); using (Log.BeginScope(new ExceptionlessState().Organization(wi.OrganizationId).Project(wi.ProjectId))) { Log.LogInformation("Received remove stacks work item for project: {ProjectId}", wi.ProjectId); await context.ReportProgressAsync(0, "Starting soft deleting of stacks...").AnyContext(); long deleted = await _stackRepository.SoftDeleteByProjectIdAsync(wi.OrganizationId, wi.ProjectId).AnyContext(); await context.ReportProgressAsync(100, $"Stacks soft deleted: {deleted}").AnyContext(); } }
public override async Task HandleItemAsync(WorkItemContext context) { var wi = context.GetData <RemoveBotEventsWorkItem>(); using var _ = Log.BeginScope(new ExceptionlessState().Organization(wi.OrganizationId).Project(wi.ProjectId)); Log.LogInformation("Received remove bot events work item OrganizationId={OrganizationId} ProjectId={ProjectId}, ClientIpAddress={ClientIpAddress}, UtcStartDate={UtcStartDate}, UtcEndDate={UtcEndDate}", wi.OrganizationId, wi.ProjectId, wi.ClientIpAddress, wi.UtcStartDate, wi.UtcEndDate); await context.ReportProgressAsync(0, $"Starting deleting of bot events... OrganizationId={wi.OrganizationId}").AnyContext(); long deleted = await _eventRepository.RemoveAllAsync(wi.OrganizationId, wi.ClientIpAddress, wi.UtcStartDate, wi.UtcEndDate).AnyContext(); await context.ReportProgressAsync(100, $"Bot events deleted: {deleted} OrganizationId={wi.OrganizationId}").AnyContext(); }
public override async Task HandleItemAsync(WorkItemContext context) { const int LIMIT = 100; var wi = context.GetData <OrganizationMaintenanceWorkItem>(); Log.LogInformation("Received upgrade organizations work item. Upgrade Plans: {UpgradePlans}", wi.UpgradePlans); var results = await _organizationRepository.GetAllAsync(o => o.PageLimit(LIMIT)).AnyContext(); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { foreach (var organization in results.Documents) { if (wi.UpgradePlans) { UpgradePlan(organization); } if (wi.RemoveOldUsageStats) { foreach (var usage in organization.OverageHours.Where(u => u.Date < SystemClock.UtcNow.Subtract(TimeSpan.FromDays(3))).ToList()) { organization.OverageHours.Remove(usage); } foreach (var usage in organization.Usage.Where(u => u.Date < SystemClock.UtcNow.Subtract(TimeSpan.FromDays(366))).ToList()) { organization.Usage.Remove(usage); } } } if (wi.UpgradePlans || wi.RemoveOldUsageStats) { await _organizationRepository.SaveAsync(results.Documents).AnyContext(); } // Sleep so we are not hammering the backend. await SystemClock.SleepAsync(TimeSpan.FromSeconds(2.5)).AnyContext(); if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync().AnyContext()) { break; } if (results.Documents.Count > 0) { await context.RenewLockAsync().AnyContext(); } } }
public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData<StackWorkItem>(); if (workItem.Delete) { await _eventRepository.RemoveAllByStackIdsAsync(new[] { workItem.StackId }).AnyContext(); await _stackRepository.RemoveAsync(workItem.StackId).AnyContext(); return; } if (workItem.UpdateIsFixed) await _eventRepository.UpdateFixedByStackAsync(workItem.OrganizationId, workItem.StackId, workItem.IsFixed).AnyContext(); if (workItem.UpdateIsHidden) await _eventRepository.UpdateHiddenByStackAsync(workItem.OrganizationId, workItem.StackId, workItem.IsHidden).AnyContext(); }
public override async Task HandleItemAsync(WorkItemContext context) { Assert.NotNull(Dependency); var jobData = context.GetData <MyWorkItem>(); Assert.Equal("Test", jobData.SomeData); for (int i = 1; i < 10; i++) { await Task.Delay(100); await context.ReportProgressAsync(10 *i); } }
public Task HandleItem(WorkItemContext context) { Assert.NotNull(Dependency); var jobData = context.GetData <MyWorkItem>(); Assert.Equal("Test", jobData.SomeData); for (int i = 0; i < 10; i++) { Thread.Sleep(100); context.ReportProgress(10 * i); } return(TaskHelper.Completed()); }
public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData <SetProjectIsConfiguredWorkItem>(); Logger.Info().Message($"Setting Is Configured for project: {workItem.ProjectId}").Write(); var project = await _projectRepository.GetByIdAsync(workItem.ProjectId).AnyContext(); if (project == null || project.IsConfigured.GetValueOrDefault()) { return; } project.IsConfigured = workItem.IsConfigured || await _eventRepository.GetCountByProjectIdAsync(project.Id).AnyContext() > 0; await _projectRepository.SaveAsync(project, true).AnyContext(); }
public void SaveWorkItem(WorkItem item) { using (var ctx = new WorkItemContext()) { var obj = (from row in ctx.WorkItems where row.Oid == item.Oid select row).FirstOrDefault(); if (obj != null) { obj.Status = item.Status; obj.Request = item.Request; } ctx.SaveChanges(); } }
public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData <SetProjectIsConfiguredWorkItem>(); Log.LogInformation("Setting Is Configured for project: {project}", workItem.ProjectId); var project = await _projectRepository.GetByIdAsync(workItem.ProjectId).AnyContext(); if (project == null || project.IsConfigured.GetValueOrDefault()) { return; } project.IsConfigured = workItem.IsConfigured || await _eventRepository.CountAsync(q => q.Project(project.Id)).AnyContext() > 0; await _projectRepository.SaveAsync(project, o => o.Cache()).AnyContext(); }
public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData <ReindexWorkItem>(); long existingDocCount = (await _client.CountAsync(d => d.Index(workItem.NewIndex)).AnyContext()).Count; Log.Info().Message("Received reindex work item for new index {0}", workItem.NewIndex).Write(); var startTime = DateTime.UtcNow.AddSeconds(-1); await context.ReportProgressAsync(0, "Starting reindex...").AnyContext(); var result = await ReindexAsync(workItem, context, 0, 90, workItem.StartUtc).AnyContext(); await context.ReportProgressAsync(90, $"Total: {result.Total} Completed: {result.Completed}").AnyContext(); // TODO: Check to make sure the docs have been added to the new index before changing alias if (!String.IsNullOrEmpty(workItem.Alias)) { await _client.AliasAsync(x => x .Remove(a => a.Alias(workItem.Alias).Index(workItem.OldIndex)) .Add(a => a.Alias(workItem.Alias).Index(workItem.NewIndex))).AnyContext(); await context.ReportProgressAsync(98, $"Updated alias: {workItem.Alias} Remove: {workItem.OldIndex} Add: {workItem.NewIndex}").AnyContext(); } await _client.RefreshAsync().AnyContext(); var secondPassResult = await ReindexAsync(workItem, context, 90, 98, startTime).AnyContext(); await context.ReportProgressAsync(98, $"Total: {secondPassResult.Total} Completed: {secondPassResult.Completed}").AnyContext(); if (workItem.DeleteOld) { await _client.RefreshAsync().AnyContext(); long newDocCount = (await _client.CountAsync(d => d.Index(workItem.NewIndex)).AnyContext()).Count - existingDocCount; long oldDocCount = (await _client.CountAsync(d => d.Index(workItem.OldIndex)).AnyContext()).Count; await context.ReportProgressAsync(98, $"Old Docs: {oldDocCount} New Docs: {newDocCount}").AnyContext(); if (newDocCount >= oldDocCount) { await _client.DeleteIndexAsync(d => d.Index(workItem.OldIndex)).AnyContext(); } await context.ReportProgressAsync(98, $"Deleted index: {workItem.OldIndex}").AnyContext(); } await context.ReportProgressAsync(100).AnyContext(); }
public override async Task HandleItemAsync(WorkItemContext context) { var wi = context.GetData <RemoveProjectWorkItem>(); using (Log.BeginScope(new ExceptionlessState().Organization(wi.OrganizationId).Project(wi.ProjectId))) { Log.LogInformation("Received remove project work item for: {0} Reset Data: {1}", wi.ProjectId, wi.Reset); await context.ReportProgressAsync(0, "Starting deletion...").AnyContext(); var project = await _projectRepository.GetByIdAsync(wi.ProjectId).AnyContext(); if (project == null) { await context.ReportProgressAsync(100, wi.Reset? "Project data reset" : "Project deleted").AnyContext(); return; } if (!wi.Reset) { await context.ReportProgressAsync(20, "Removing tokens").AnyContext(); await _tokenRepository.RemoveAllByProjectIdAsync(project.OrganizationId, project.Id).AnyContext(); await context.ReportProgressAsync(40, "Removing web hooks").AnyContext(); await _webHookRepository.RemoveAllByProjectIdAsync(project.OrganizationId, project.Id).AnyContext(); } await context.ReportProgressAsync(60, "Resetting project data").AnyContext(); await _eventRepository.RemoveAllByProjectIdAsync(project.OrganizationId, project.Id).AnyContext(); await _stackRepository.RemoveAllByProjectIdAsync(project.OrganizationId, project.Id).AnyContext(); if (!wi.Reset) { await context.ReportProgressAsync(80, "Removing project").AnyContext(); await _projectRepository.RemoveAsync(project.Id).AnyContext(); } await context.ReportProgressAsync(100, wi.Reset? "Project data reset" : "Project deleted").AnyContext(); } }
public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData <OrganizationNotificationWorkItem>(); Log.LogInformation("Received organization notification work item for: {organization} IsOverHourlyLimit: {IsOverHourlyLimit} IsOverMonthlyLimit: {IsOverMonthlyLimit}", workItem.OrganizationId, workItem.IsOverHourlyLimit, workItem.IsOverMonthlyLimit); var organization = await _organizationRepository.GetByIdAsync(workItem.OrganizationId, o => o.Cache()).AnyContext(); if (organization == null) { return; } if (workItem.IsOverMonthlyLimit) { await SendOverageNotificationsAsync(organization, workItem.IsOverHourlyLimit, workItem.IsOverMonthlyLimit).AnyContext(); } }
public XElement GetQueryXml(WorkItemContext context, FieldList fields) { var parser = new LexalParser(this.QueryText); var nodes = parser.ProcessWherePart(); nodes.Optimize(); nodes.ExtractOperatorForward(); nodes.FixFields(fields); nodes.FillFieldTypes(fields); var manager = new ParameterManager(context); manager.EvalParameters(nodes); var xmlTransformer = new NodesToXml(nodes); return(XElement.Parse(xmlTransformer.WriteXml())); }
public override async Task HandleItemAsync(WorkItemContext context) { const int LIMIT = 100; var workItem = context.GetData <ProjectMaintenanceWorkItem>(); Log.Info("Received upgrade projects work item. Update Default Bot List: {0} IncrementConfigurationVersion: {1}", workItem.UpdateDefaultBotList, workItem.IncrementConfigurationVersion); var results = await _projectRepository.GetAllAsync(o => o.PageLimit(LIMIT)).AnyContext(); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { foreach (var project in results.Documents) { if (workItem.UpdateDefaultBotList) { project.SetDefaultUserAgentBotPatterns(); } if (workItem.IncrementConfigurationVersion) { project.Configuration.IncrementVersion(); } } if (workItem.UpdateDefaultBotList) { await _projectRepository.SaveAsync(results.Documents).AnyContext(); } // Sleep so we are not hammering the backend. await SystemClock.SleepAsync(TimeSpan.FromSeconds(2.5)).AnyContext(); if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync().AnyContext()) { break; } if (results.Documents.Count > 0) { await context.RenewLockAsync().AnyContext(); } } }
public override void Canceling() { using (var ctx = new WorkItemContext()) { Item = (from row in ctx.WorkItems where row.Oid == Item.Oid select row).FirstOrDefault(); DateTime now = DateTime.Now; if (Item != null) { Item.Progress = Progress; Item.Status = WorkItemStatusEnum.Canceling; } ctx.SaveChanges(); } }
public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData <RemoveProjectWorkItem>(); Log.Info("Received remove project work item for: {0} Reset Data: {1}", workItem.ProjectId, workItem.Reset); await context.ReportProgressAsync(0, "Starting deletion...").AnyContext(); var project = await _projectRepository.GetByIdAsync(workItem.ProjectId).AnyContext(); if (project == null) { await context.ReportProgressAsync(100, workItem.Reset? "Project data reset" : "Project deleted").AnyContext(); return; } if (!workItem.Reset) { await context.ReportProgressAsync(20, "Removing tokens").AnyContext(); await _tokenRepository.RemoveAllByOrganizationIdsAsync(new[] { project.Id }).AnyContext(); await context.ReportProgressAsync(40, "Removing web hooks").AnyContext(); await _webHookRepository.RemoveAllByOrganizationIdsAsync(new[] { project.Id }).AnyContext(); } await context.ReportProgressAsync(60, "Resetting project data").AnyContext(); await _eventRepository.RemoveAllByProjectIdsAsync(new[] { project.Id }).AnyContext(); await _stackRepository.RemoveAllByProjectIdsAsync(new[] { project.Id }).AnyContext(); if (!workItem.Reset) { await context.ReportProgressAsync(80, "Removing project").AnyContext(); await _projectRepository.RemoveAsync(project.Id).AnyContext(); } await context.ReportProgressAsync(100, workItem.Reset? "Project data reset" : "Project deleted").AnyContext(); }
public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData <SetLocationFromGeoWorkItem>(); GeoResult result; if (!GeoResult.TryParse(workItem.Geo, out result)) { return; } var location = await _cacheClient.GetAsync <Location>(workItem.Geo, null).AnyContext(); if (location == null) { try { result = await _geocodeService.ReverseGeocodeAsync(result.Latitude.GetValueOrDefault(), result.Longitude.GetValueOrDefault()).AnyContext(); location = result.ToLocation(); await _metricsClient.CounterAsync(MetricNames.UsageGeocodingApi).AnyContext(); } catch (Exception ex) { Logger.Error().Exception(ex).Message($"Error occurred looking up reverse geocode: {workItem.Geo}").Write(); } } if (location == null) { return; } await _cacheClient.SetAsync(workItem.Geo, location, TimeSpan.FromDays(3)).AnyContext(); var ev = await _eventRepository.GetByIdAsync(workItem.EventId).AnyContext(); if (ev == null) { return; } ev.SetLocation(location); await _eventRepository.SaveAsync(ev).AnyContext(); }
// NOTE: Uncomment to ensure only one work item handler is called at a time. //public override Task<ILock> GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = new CancellationToken()) { // return _lockProvider.AcquireAsync(nameof(DeleteValueWorkItemHandler), cancellationToken: cancellationToken); //} public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData<DeleteValueWorkItem>(); await context.ReportProgressAsync(0, $"Starting to delete item: {workItem.Id}."); await context.ReportProgressAsync(1, "If you are seeing multiple progresses. Please uncomment the lock in the DeleteValueWorkItemHandler."); await Task.Delay(TimeSpan.FromSeconds(2.5)); await context.ReportProgressAsync(50, "Deleting"); await Task.Delay(TimeSpan.FromSeconds(.5)); await context.ReportProgressAsync(70, "Deleting."); await Task.Delay(TimeSpan.FromSeconds(.5)); await context.ReportProgressAsync(90, "Deleting.."); await Task.Delay(TimeSpan.FromSeconds(.5)); await _cacheClient.RemoveAsync(workItem.Id); await _publisher.PublishAsync(new EntityChanged { ChangeType = ChangeType.Removed, Id = workItem.Id }); await context.ReportProgressAsync(100); }
public override Task HandleItemAsync(WorkItemContext context) { var wi = context.GetData <OrganizationNotificationWorkItem>(); string cacheKey = $"{nameof(OrganizationNotificationWorkItemHandler)}:{wi.OrganizationId}"; return(_lockProvider.TryUsingAsync(cacheKey, async() => { Log.LogInformation("Received organization notification work item for: {organization} IsOverHourlyLimit: {IsOverHourlyLimit} IsOverMonthlyLimit: {IsOverMonthlyLimit}", wi.OrganizationId, wi.IsOverHourlyLimit, wi.IsOverMonthlyLimit); var organization = await _organizationRepository.GetByIdAsync(wi.OrganizationId, o => o.Cache()).AnyContext(); if (organization == null) { return; } if (wi.IsOverMonthlyLimit) { await SendOverageNotificationsAsync(organization, wi.IsOverHourlyLimit, wi.IsOverMonthlyLimit).AnyContext(); } }, TimeSpan.FromMinutes(15), new CancellationToken(true))); }
public override void Idle() { using (var ctx = new WorkItemContext()) { Item = (from row in ctx.WorkItems where row.Oid == Item.Oid select row).FirstOrDefault(); DateTime now = DateTime.Now; if (Item != null) { Item.Progress = Progress; Item.ExpirationTime = now; Item.DeleteTime = now.AddMinutes(240); Item.Status = WorkItemStatusEnum.Idle; } ctx.SaveChanges(); } }
public override async Task HandleItemAsync(WorkItemContext context) { const int LIMIT = 100; var workItem = context.GetData <UserMaintenanceWorkItem>(); Log.Info("Received user maintenance work item. Normalize: {0}", workItem.Normalize); var results = await _userRepository.GetAllAsync(paging : new PagingOptions().WithLimit(LIMIT)).AnyContext(); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { foreach (var user in results.Documents) { if (workItem.Normalize) { NormalizeUser(user); } } if (workItem.Normalize) { await _userRepository.SaveAsync(results.Documents).AnyContext(); } // Sleep so we are not hammering the backend. await SystemClock.SleepAsync(TimeSpan.FromSeconds(2.5)).AnyContext(); if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync().AnyContext()) { break; } if (results.Documents.Count > 0) { await context.RenewLockAsync().AnyContext(); } } }
public override async Task HandleItemAsync(WorkItemContext context) { const int LIMIT = 100; var workItem = context.GetData<OrganizationMaintenanceWorkItem>(); _logger.Info("Received upgrade organizations work item. Upgrade Plans: {0}", workItem.UpgradePlans); var results = await _organizationRepository.GetAllAsync(paging: new PagingOptions().WithLimit(LIMIT)).AnyContext(); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { foreach (var organization in results.Documents) { if (workItem.UpgradePlans) UpgradePlan(organization); } if (workItem.UpgradePlans) await _organizationRepository.SaveAsync(results.Documents).AnyContext(); // Sleep so we are not hammering the backend. await Task.Delay(TimeSpan.FromSeconds(2.5)).AnyContext(); await results.NextPageAsync().AnyContext(); if (results.Documents.Count > 0) await context.RenewLockAsync().AnyContext(); } }
public override Task HandleItem(WorkItemContext context) { Assert.NotNull(Dependency); var jobData = context.GetData<MyWorkItem>(); Assert.Equal("Test", jobData.SomeData); for (int i = 0; i < 10; i++) { Thread.Sleep(100); context.ReportProgress(10 * i); } return TaskHelper.Completed(); }
public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData<RemoveOrganizationWorkItem>(); Logger.Info().Message($"Received remove organization work item for: {workItem.OrganizationId}").Write(); await context.ReportProgressAsync(0, "Starting deletion...").AnyContext(); var organization = await _organizationRepository.GetByIdAsync(workItem.OrganizationId).AnyContext(); if (organization == null) { await context.ReportProgressAsync(100, "Organization deleted").AnyContext(); return; } await context.ReportProgressAsync(10, "Removing subscriptions").AnyContext(); if (!String.IsNullOrEmpty(organization.StripeCustomerId)) { Logger.Info().Message($"Canceling stripe subscription for the organization '{organization.Name}' with Id: '{organization.Id}'.").Write(); var subscriptionService = new StripeSubscriptionService(Settings.Current.StripeApiKey); var subscriptions = subscriptionService.List(organization.StripeCustomerId).Where(s => !s.CanceledAt.HasValue); foreach (var subscription in subscriptions) subscriptionService.Cancel(organization.StripeCustomerId, subscription.Id); } await context.ReportProgressAsync(20, "Removing users").AnyContext(); var users = await _userRepository.GetByOrganizationIdAsync(organization.Id).AnyContext(); foreach (User user in users.Documents) { // delete the user if they are not associated to any other organizations and they are not the current user if (user.OrganizationIds.All(oid => String.Equals(oid, organization.Id)) && !String.Equals(user.Id, workItem.CurrentUserId)) { Logger.Info().Message("Removing user '{0}' as they do not belong to any other organizations.", user.Id, organization.Name, organization.Id).Write(); await _userRepository.RemoveAsync(user.Id).AnyContext(); } else { Logger.Info().Message("Removing user '{0}' from organization '{1}' with Id: '{2}'", user.Id, organization.Name, organization.Id).Write(); user.OrganizationIds.Remove(organization.Id); await _userRepository.SaveAsync(user, true).AnyContext(); } } await context.ReportProgressAsync(30, "Removing tokens").AnyContext(); await _tokenRepository.RemoveAllByOrganizationIdsAsync(new[] { organization.Id }).AnyContext(); await context.ReportProgressAsync(40, "Removing web hooks").AnyContext(); await _webHookRepository.RemoveAllByOrganizationIdsAsync(new[] { organization.Id }).AnyContext(); await context.ReportProgressAsync(50, "Removing projects").AnyContext(); var projects = await _projectRepository.GetByOrganizationIdAsync(organization.Id).AnyContext(); if (workItem.IsGlobalAdmin && projects.Total > 0) { var completed = 1; foreach (Project project in projects.Documents) { Logger.Info().Message("Resetting all project data for project '{0}' with Id: '{1}'.", project.Name, project.Id).Write(); await _eventRepository.RemoveAllByProjectIdsAsync(new[] { project.Id }).AnyContext(); await _stackRepository.RemoveAllByProjectIdsAsync(new[] { project.Id }).AnyContext(); await context.ReportProgressAsync(CalculateProgress(projects.Total, completed++, 51, 89), "Removing projects...").AnyContext(); } Logger.Info().Message("Deleting all projects for organization '{0}' with Id: '{1}'.", organization.Name, organization.Id).Write(); await _projectRepository.RemoveAsync(projects.Documents).AnyContext(); } Logger.Info().Message("Deleting organization '{0}' with Id: '{1}'.", organization.Name, organization.Id).Write(); await context.ReportProgressAsync(90, "Removing organization").AnyContext(); await _organizationRepository.RemoveAsync(organization.Id).AnyContext(); await context.ReportProgressAsync(100, "Organization deleted").AnyContext(); }
public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData<ReindexWorkItem>(); await _reindexer.ReindexAsync(workItem, context.ReportProgressAsync).AnyContext(); }
private async Task<ReindexResult> ReindexAsync(ReindexWorkItem workItem, WorkItemContext context, int startProgress = 0, int endProgress = 100, DateTime? startTime = null) { const int pageSize = 100; const string scroll = "5m"; string timestampField = workItem.TimestampField ?? "_timestamp"; long completed = 0; var scanResults = await _client.SearchAsync<JObject>(s => s .Index(workItem.OldIndex) .AllTypes() .Filter(f => startTime.HasValue ? f.Range(r => r.OnField(timestampField).Greater(startTime.Value)) : f.MatchAll()) .From(0).Take(pageSize) .SearchType(SearchType.Scan) .Scroll(scroll)).AnyContext(); if (!scanResults.IsValid || scanResults.ScrollId == null) { Logger.Error().Message("Invalid search result: message={0}", scanResults.GetErrorMessage()).Write(); return new ReindexResult(); } long totalHits = scanResults.Total; var parentMap = workItem.ParentMaps?.ToDictionary(p => p.Type, p => p.ParentPath) ?? new Dictionary<string, string>(); var results = await _client.ScrollAsync<JObject>(scroll, scanResults.ScrollId).AnyContext(); while (results.Documents.Any()) { var bulkDescriptor = new BulkDescriptor(); foreach (var hit in results.Hits) { var h = hit; // TODO: Add support for doing JObject based schema migrations bulkDescriptor.Index<JObject>(idx => { idx .Index(workItem.NewIndex) .Type(h.Type) .Id(h.Id) .Document(h.Source); if (String.IsNullOrEmpty(h.Type)) Logger.Error().Message("Hit type empty. id={0}", h.Id).Write(); if (parentMap.ContainsKey(h.Type)) { if (String.IsNullOrEmpty(parentMap[h.Type])) Logger.Error().Message("Parent map has empty value. id={0} type={1}", h.Id, h.Type).Write(); var parentId = h.Source.SelectToken(parentMap[h.Type]); if (!String.IsNullOrEmpty(parentId?.ToString())) idx.Parent(parentId.ToString()); else Logger.Error().Message("Unable to get parent id. id={0} path={1}", h.Id, parentMap[h.Type]).Write(); } return idx; }); } var bulkResponse = await _client.BulkAsync(bulkDescriptor).AnyContext(); if (!bulkResponse.IsValid) { string message = $"Reindex bulk error: old={workItem.OldIndex} new={workItem.NewIndex} completed={completed} message={bulkResponse.GetErrorMessage()}"; Logger.Warn().Message(message).Write(); // try each doc individually so we can see which doc is breaking us foreach (var hit in results.Hits) { var h = hit; var response = await _client.IndexAsync<JObject>(h.Source, d => { d .Index(workItem.NewIndex) .Type(h.Type) .Id(h.Id); if (parentMap.ContainsKey(h.Type)) { var parentId = h.Source.SelectToken(parentMap[h.Type]); if (!String.IsNullOrEmpty(parentId?.ToString())) d.Parent(parentId.ToString()); else Logger.Error().Message("Unable to get parent id. id={0} path={1}", h.Id, parentMap[h.Type]).Write(); } return d; }).AnyContext(); if (response.IsValid) continue; message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={hit.Id} completed={completed} message={response.GetErrorMessage()}"; Logger.Error().Message(message).Write(); var errorDoc = new JObject(new { h.Type, Content = h.Source.ToString(Formatting.Indented) }); if (parentMap.ContainsKey(h.Type)) { var parentId = h.Source.SelectToken(parentMap[h.Type]); if (!String.IsNullOrEmpty(parentId?.ToString())) errorDoc["ParentId"] = parentId.ToString(); else Logger.Error().Message("Unable to get parent id. id={0} path={1}", h.Id, parentMap[h.Type]).Write(); } // put the document into an error index response = await _client.IndexAsync<JObject>(errorDoc, d => { d .Index(workItem.NewIndex + "-error") .Id(h.Id); return d; }).AnyContext(); if (response.IsValid) continue; throw new ReindexException(response.ConnectionStatus, message); } } completed += bulkResponse.Items.Count(); await context.ReportProgressAsync(CalculateProgress(totalHits, completed, startProgress, endProgress), $"Total: {totalHits} Completed: {completed}").AnyContext(); Logger.Info().Message($"Reindex Progress: {CalculateProgress(totalHits, completed, startProgress, endProgress)} Completed: {completed} Total: {totalHits}").Write(); results = await _client.ScrollAsync<JObject>(scroll, results.ScrollId).AnyContext(); } return new ReindexResult { Total = totalHits, Completed = completed }; }
public override async Task HandleItemAsync(WorkItemContext context) { Assert.NotNull(Dependency); var jobData = context.GetData<MyWorkItem>(); Assert.Equal("Test", jobData.SomeData); for (int i = 1; i < 10; i++) { await Task.Delay(100); await context.ReportProgressAsync(10 * i); } }
public override Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData<ThrottleBotsWorkItem>(); return _eventRepository.HideAllByClientIpAndDateAsync(workItem.OrganizationId, workItem.ClientIpAddress, workItem.UtcStartDate, workItem.UtcEndDate); }
public override async Task HandleItemAsync(WorkItemContext context) { var jobData = context.GetData<MyWorkItem>(); Assert.Equal("Test", jobData.SomeData); for (int i = 1; i < 10; i++) { await SystemClock.SleepAsync(100); await context.ReportProgressAsync(10 * i); } }