Exemplo n.º 1
0
        public List <WorkItem> GetWorkItems(int count, WorkItemPriorityEnum priority)
        {
            _nowRunningWorkItems.Clear();

            using (var context = new WorkItemContext())
            {
                var now = DateTime.Now;

                var workItems = (from row in context.WorkItems
                                 where row.ScheduledTime < now && row.Priority == priority &&
                                 (row.Status == WorkItemStatusEnum.Pending || row.Status == WorkItemStatusEnum.Idle)
                                 orderby row.Priority, row.ScheduledTime
                                 select row).Take(count);


                foreach (var item in workItems)
                {
                    // 标记任务处于正在运行状态,避免任务的多次执行
                    item.Status = WorkItemStatusEnum.InProgress;
                    _nowRunningWorkItems.Add(item);
                }

                return(new List <WorkItem>(_nowRunningWorkItems));
            }
        }
        public override async Task HandleItemAsync(WorkItemContext context) {
            var workItem = context.GetData<RemoveProjectWorkItem>();
            _logger.Info("Received remove project work item for: {0} Reset Data: {1}", workItem.ProjectId, workItem.Reset);

            await context.ReportProgressAsync(0, "Starting deletion...").AnyContext();
            var project = await _projectRepository.GetByIdAsync(workItem.ProjectId).AnyContext();
            if (project == null) {
                await context.ReportProgressAsync(100, workItem.Reset ? "Project data reset" : "Project deleted").AnyContext();
                return;
            }

            if (!workItem.Reset) {
                await context.ReportProgressAsync(20, "Removing tokens").AnyContext();
                await _tokenRepository.RemoveAllByOrganizationIdsAsync(new[] { project.Id }).AnyContext();

                await context.ReportProgressAsync(40, "Removing web hooks").AnyContext();
                await _webHookRepository.RemoveAllByOrganizationIdsAsync(new[] { project.Id }).AnyContext();
            }

            await context.ReportProgressAsync(60, "Resetting project data").AnyContext();
            await _eventRepository.RemoveAllByProjectIdsAsync(new[] { project.Id }).AnyContext();
            await _stackRepository.RemoveAllByProjectIdsAsync(new[] { project.Id }).AnyContext();

            if (!workItem.Reset) {
                await context.ReportProgressAsync(80, "Removing project").AnyContext();
                await _projectRepository.RemoveAsync(project.Id).AnyContext();
            }

            await context.ReportProgressAsync(100, workItem.Reset ? "Project data reset" : "Project deleted").AnyContext();
        }
        public override async Task HandleItemAsync(WorkItemContext context) {
            var workItem = context.GetData<SetLocationFromGeoWorkItem>();

            GeoResult result;
            if (!GeoResult.TryParse(workItem.Geo, out result))
                return;
            
            var location = await _cacheClient.GetAsync<Location>(workItem.Geo, null).AnyContext();
            if (location == null) {
                try {
                    result = await _geocodeService.ReverseGeocodeAsync(result.Latitude.GetValueOrDefault(), result.Longitude.GetValueOrDefault()).AnyContext();
                    location = result.ToLocation();
                    await _metricsClient.CounterAsync(MetricNames.UsageGeocodingApi).AnyContext();
                } catch (Exception ex) {
                    Log.Error(ex, "Error occurred looking up reverse geocode: {0}", workItem.Geo);
                }
            }
            
            if (location == null)
                return;
            
            await _cacheClient.SetAsync(workItem.Geo, location, TimeSpan.FromDays(3)).AnyContext();

            var ev = await _eventRepository.GetByIdAsync(workItem.EventId).AnyContext();
            if (ev == null)
                return;

            ev.SetLocation(location);
            await _eventRepository.SaveAsync(ev).AnyContext();
        }
        public Task HandleItem(WorkItemContext context) {
            var workItem = context.GetData<ReindexWorkItem>();

            Log.Info().Message("Received reindex work item for new index {0}", workItem.NewIndex).Write();
            var startTime = DateTime.UtcNow.AddSeconds(-1);
            context.ReportProgress(0, "Starting reindex...");
            var result = Reindex(workItem, context, 0, 90, workItem.StartUtc);
            context.ReportProgress(90, String.Format("Total: {0} Completed: {1}", result.Total, result.Completed));

            // TODO: Check to make sure the docs have been added to the new index before changing alias

            if (!String.IsNullOrEmpty(workItem.Alias)) {
                _client.Alias(x => x.Remove(a => a.Alias(workItem.Alias).Index(workItem.OldIndex)).Add(a => a.Alias(workItem.Alias).Index(workItem.NewIndex)));
                context.ReportProgress(98, String.Format("Updated alias: {0} Remove: {1} Add: {2}", workItem.Alias, workItem.OldIndex, workItem.NewIndex));
            }

            _client.Refresh();
            var secondPassResult = Reindex(workItem, context, 90, 98, startTime);
            context.ReportProgress(98, String.Format("Total: {0} Completed: {1}", secondPassResult.Total, secondPassResult.Completed));

            if (workItem.DeleteOld) {
                _client.Refresh();
                long newDocCount = _client.Count(d => d.Index(workItem.OldIndex)).Count;
                long oldDocCount = _client.Count(d => d.Index(workItem.OldIndex)).Count;
                context.ReportProgress(98, String.Format("Old Docs: {0} New Docs: {1}", oldDocCount, newDocCount));
                if (newDocCount >= oldDocCount)
                    _client.DeleteIndex(d => d.Index(workItem.OldIndex));
                context.ReportProgress(98, String.Format("Deleted index: {0}", workItem.OldIndex));
            }
            context.ReportProgress(100);

            return TaskHelper.Completed();
        }
        public override async Task HandleItemAsync(WorkItemContext context) {
            var workItem = context.GetData<ReindexWorkItem>();

            Logger.Info().Message("Received reindex work item for new index {0}", workItem.NewIndex).Write();
            var startTime = DateTime.UtcNow.AddSeconds(-1);
            await context.ReportProgressAsync(0, "Starting reindex...").AnyContext();
            var result = await ReindexAsync(workItem, context, 0, 90, workItem.StartUtc).AnyContext();
            await context.ReportProgressAsync(90, $"Total: {result.Total} Completed: {result.Completed}").AnyContext();

            // TODO: Check to make sure the docs have been added to the new index before changing alias

            if (!String.IsNullOrEmpty(workItem.Alias)) {
                await _client.AliasAsync(x => x.Remove(a => a.Alias(workItem.Alias).Index(workItem.OldIndex)).Add(a => a.Alias(workItem.Alias).Index(workItem.NewIndex))).AnyContext();
                await context.ReportProgressAsync(98, $"Updated alias: {workItem.Alias} Remove: {workItem.OldIndex} Add: {workItem.NewIndex}").AnyContext();
            }

            await _client.RefreshAsync().AnyContext();
            var secondPassResult = await ReindexAsync(workItem, context, 90, 98, startTime).AnyContext();
            await context.ReportProgressAsync(98, $"Total: {secondPassResult.Total} Completed: {secondPassResult.Completed}").AnyContext();

            if (workItem.DeleteOld) {
                await _client.RefreshAsync().AnyContext();
                long newDocCount = (await _client.CountAsync(d => d.Index(workItem.OldIndex)).AnyContext()).Count;
                long oldDocCount = (await _client.CountAsync(d => d.Index(workItem.OldIndex)).AnyContext()).Count;
                await context.ReportProgressAsync(98, $"Old Docs: {oldDocCount} New Docs: {newDocCount}").AnyContext();
                if (newDocCount >= oldDocCount)
                    await _client.DeleteIndexAsync(d => d.Index(workItem.OldIndex)).AnyContext();
                await context.ReportProgressAsync(98, $"Deleted index: {workItem.OldIndex}").AnyContext();
            }
            await context.ReportProgressAsync(100).AnyContext();
        }
Exemplo n.º 6
0
        public void ResetInProgressWorkItems()
        {
            using (var context = new WorkItemContext())
            {
                var query = from row in context.WorkItems
                            where row.Status == WorkItemStatusEnum.InProgress ||
                            row.Status == WorkItemStatusEnum.DeleteInProgress ||
                            row.Status == WorkItemStatusEnum.Canceling
                            orderby row.Priority
                            select row;

                foreach (var item in query.ToList())
                {
                    switch (item.Status)
                    {
                    case WorkItemStatusEnum.InProgress:
                        item.Status = WorkItemStatusEnum.Pending;
                        break;

                    case WorkItemStatusEnum.DeleteInProgress:
                        item.Status = WorkItemStatusEnum.Deleted;
                        break;

                    case WorkItemStatusEnum.Canceling:
                        item.Status = WorkItemStatusEnum.Canceled;
                        break;
                    }
                }

                context.SaveChanges();
            }
        }
Exemplo n.º 7
0
        public override async Task HandleItemAsync(WorkItemContext context)
        {
            var wi = context.GetData <StackWorkItem>();

            using (Log.BeginScope(new ExceptionlessState().Organization(wi.OrganizationId).Project(wi.ProjectId))) {
                if (wi.Delete)
                {
                    await _eventRepository.RemoveAllByStackIdAsync(wi.OrganizationId, wi.ProjectId, wi.StackId).AnyContext();

                    await _stackRepository.RemoveAsync(wi.StackId).AnyContext();

                    return;
                }

                if (wi.UpdateIsFixed)
                {
                    await _eventRepository.UpdateFixedByStackAsync(wi.OrganizationId, wi.ProjectId, wi.StackId, wi.IsFixed).AnyContext();
                }

                if (wi.UpdateIsHidden)
                {
                    await _eventRepository.UpdateHiddenByStackAsync(wi.OrganizationId, wi.ProjectId, wi.StackId, wi.IsHidden).AnyContext();
                }
            }
        }
Exemplo n.º 8
0
        // NOTE: Uncomment to ensure only one work item handler is called at a time.
        //public override Task<ILock> GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = new CancellationToken()) {
        //    return _lockProvider.AcquireAsync(nameof(DeleteValueWorkItemHandler), cancellationToken: cancellationToken);
        //}

        public override async Task HandleItemAsync(WorkItemContext context)
        {
            var workItem = context.GetData <DeleteValueWorkItem>();
            await context.ReportProgressAsync(0, $"Starting to delete item: {workItem.Id}.");

            await context.ReportProgressAsync(1, "If you are seeing multiple progresses. Please uncomment the lock in the DeleteValueWorkItemHandler.");

            await Task.Delay(TimeSpan.FromSeconds(2.5));

            await context.ReportProgressAsync(50, "Deleting");

            await Task.Delay(TimeSpan.FromSeconds(.5));

            await context.ReportProgressAsync(70, "Deleting.");

            await Task.Delay(TimeSpan.FromSeconds(.5));

            await context.ReportProgressAsync(90, "Deleting..");

            await Task.Delay(TimeSpan.FromSeconds(.5));

            await _cacheClient.RemoveAsync(workItem.Id);

            await _publisher.PublishAsync(new EntityChanged {
                ChangeType = ChangeType.Removed,
                Id         = workItem.Id
            });

            await context.ReportProgressAsync(100);
        }
        public override async Task HandleItemAsync(WorkItemContext context) {
            const int LIMIT = 100;

            var workItem = context.GetData<ProjectMaintenanceWorkItem>();
            Log.Info("Received upgrade projects work item. Update Default Bot List: {0} IncrementConfigurationVersion: {1}", workItem.UpdateDefaultBotList, workItem.IncrementConfigurationVersion);

            var results = await _projectRepository.GetAllAsync(paging: new PagingOptions().WithLimit(LIMIT)).AnyContext();
            while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) {
                foreach (var project in results.Documents) {
                    if (workItem.UpdateDefaultBotList)
                        project.SetDefaultUserAgentBotPatterns();

                    if (workItem.IncrementConfigurationVersion)
                        project.Configuration.IncrementVersion();
                }

                if (workItem.UpdateDefaultBotList)
                    await _projectRepository.SaveAsync(results.Documents).AnyContext();

                // Sleep so we are not hammering the backend.
                await Task.Delay(TimeSpan.FromSeconds(2.5)).AnyContext();

                await results.NextPageAsync().AnyContext();
                if (results.Documents.Count > 0)
                    await context.RenewLockAsync().AnyContext();
            }
        }
Exemplo n.º 10
0
    public override async Task HandleItemAsync(WorkItemContext context)
    {
        const int LIMIT = 100;

        var workItem = context.GetData <ProjectMaintenanceWorkItem>();

        Log.LogInformation("Received upgrade projects work item. Update Default Bot List: {UpdateDefaultBotList} IncrementConfigurationVersion: {IncrementConfigurationVersion}", workItem.UpdateDefaultBotList, workItem.IncrementConfigurationVersion);

        var results = await _projectRepository.GetAllAsync(o => o.PageLimit(LIMIT)).AnyContext();

        while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested)
        {
            foreach (var project in results.Documents)
            {
                if (workItem.UpdateDefaultBotList)
                {
                    project.SetDefaultUserAgentBotPatterns();
                }

                if (workItem.IncrementConfigurationVersion)
                {
                    project.Configuration.IncrementVersion();
                }

                if (workItem.RemoveOldUsageStats)
                {
                    foreach (var usage in project.OverageHours.Where(u => u.Date < SystemClock.UtcNow.Subtract(TimeSpan.FromDays(3))).ToList())
                    {
                        project.OverageHours.Remove(usage);
                    }

                    foreach (var usage in project.Usage.Where(u => u.Date < SystemClock.UtcNow.Subtract(TimeSpan.FromDays(366))).ToList())
                    {
                        project.Usage.Remove(usage);
                    }
                }
            }

            if (workItem.UpdateDefaultBotList || workItem.IncrementConfigurationVersion || workItem.RemoveOldUsageStats)
            {
                await _projectRepository.SaveAsync(results.Documents).AnyContext();
            }

            // Sleep so we are not hammering the backend.
            await SystemClock.SleepAsync(TimeSpan.FromSeconds(2.5)).AnyContext();

            if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync().AnyContext())
            {
                break;
            }

            if (results.Documents.Count > 0)
            {
                await context.RenewLockAsync().AnyContext();
            }
        }
    }
Exemplo n.º 11
0
 public override void Postpone(TimeSpan delay)
 {
     using (var ctx = new WorkItemContext())
     {
         var result = from row in ctx.WorkItems
                      where row.Oid == Item.Oid
                      select row;
     }
 }
        public List <int> GetWorkItemIds(StoredQuery query, FieldList fields)
        {
            WorkItemContext context = new WorkItemContext {
                ProjectId = query.ProjectId, Me = WorkItemsContext.WhoAmI
            };

            var invoker  = new SoapInvoker(this);
            var envelope = invoker.CreateEnvelope("QueryWorkitems", headerName);

            envelope.Header.Add(GetHeaderElement());
            XNamespace queryNs = XNamespace.Get("");

            envelope.Body.Add(new XElement(MessageNs + "psQuery",
                                           new XElement(queryNs + "Query", new XAttribute("Product", this.Url.ToString()), query.GetQueryXml(context, fields))));

            XElement sorting = query.GetSortingXml();

            foreach (var items in sorting.DescendantsAndSelf())
            {
                items.Name = MessageNs + items.Name.LocalName;
            }
            envelope.Body.Add(sorting);
            envelope.Body.Add(new XElement(MessageNs + "useMaster", "false"));
            var response = invoker.InvokeResponse();

            var queryIds = response.Element(MessageNs + "resultIds").Element("QueryIds");

            if (queryIds == null)
            {
                return(new List <int>());
            }
            var list = new List <int>();

            foreach (var item in queryIds.Elements("id"))
            {
                var startId = item.Attribute("s");
                if (startId == null)
                {
                    continue;
                }
                var s     = Convert.ToInt32(startId.Value);
                var endId = item.Attribute("e");
                if (endId != null)
                {
                    var e     = Convert.ToInt32(endId.Value);
                    var range = Enumerable.Range(s, e - s + 1);
                    list.AddRange(range);
                }
                else
                {
                    list.Add(s);
                }
            }
            return(list);
        }
Exemplo n.º 13
0
        public bool AddWorkItem(WorkItem item)
        {
            using (var ctx = new WorkItemContext())
            {
                ctx.WorkItems.Add(item);

                ctx.SaveChanges();
            }

            return(true);
        }
        public override async Task HandleItemAsync(WorkItemContext context) {
            var workItem = context.GetData<SetProjectIsConfiguredWorkItem>();
            Logger.Info().Message($"Setting Is Configured for project: {workItem.ProjectId}").Write();
            
            var project = await _projectRepository.GetByIdAsync(workItem.ProjectId).AnyContext();
            if (project == null || project.IsConfigured.GetValueOrDefault())
                return;

            project.IsConfigured = workItem.IsConfigured || await _eventRepository.GetCountByProjectIdAsync(project.Id).AnyContext() > 0;
            await _projectRepository.SaveAsync(project, true).AnyContext();
        }
        public override async Task HandleItemAsync(WorkItemContext context) {
            var workItem = context.GetData<OrganizationNotificationWorkItem>();
            _logger.Info("Received organization notification work item for: {0} IsOverHourlyLimit: {1} IsOverMonthlyLimit: {2}", workItem.OrganizationId, workItem.IsOverHourlyLimit, workItem.IsOverMonthlyLimit);

            var organization = await _organizationRepository.GetByIdAsync(workItem.OrganizationId, true).AnyContext();
            if (organization == null)
                return;
            
            if (workItem.IsOverMonthlyLimit)
                await SendOverageNotificationsAsync(organization, workItem.IsOverHourlyLimit, workItem.IsOverMonthlyLimit).AnyContext();
        }
Exemplo n.º 16
0
        public WorkItem GetWorkItem(long oid)
        {
            using (var ctx = new WorkItemContext())
            {
                var obj = (from row in ctx.WorkItems
                           where row.Oid == oid
                           select row).FirstOrDefault();

                return(obj);
            }
        }
Exemplo n.º 17
0
        public override async Task HandleItemAsync(WorkItemContext context) {
            var workItem = context.GetData<OrganizationNotificationWorkItem>();
            Log.Info("Received organization notification work item for: {0} IsOverHourlyLimit: {1} IsOverMonthlyLimit: {2}", workItem.OrganizationId, workItem.IsOverHourlyLimit, workItem.IsOverMonthlyLimit);

            var organization = await _organizationRepository.GetByIdAsync(workItem.OrganizationId, true).AnyContext();
            if (organization == null)
                return;
            
            if (workItem.IsOverMonthlyLimit)
                await SendOverageNotificationsAsync(organization, workItem.IsOverHourlyLimit, workItem.IsOverMonthlyLimit).AnyContext();
        }
        private async Task<ReindexResult> ReindexAsync(ReindexWorkItem workItem, WorkItemContext context, int startProgress = 0, int endProgress = 100, DateTime? startTime = null) {
            const int pageSize = 100;
            const string scroll = "10s";

            var scanResults = await _client.SearchAsync<JObject>(s => s.Index(workItem.OldIndex).AllTypes().Filter(f => startTime.HasValue ? f.Range(r => r.OnField(workItem.TimestampField ?? "_timestamp").Greater(startTime.Value)) : f.MatchAll()).From(0).Take(pageSize).SearchType(SearchType.Scan).Scroll(scroll)).AnyContext();

            if (!scanResults.IsValid || scanResults.ScrollId == null) {
                Logger.Error().Message("Invalid search result: message={0}", scanResults.GetErrorMessage()).Write();
                return new ReindexResult();
            }

            long totalHits = scanResults.Total;
            long completed = 0;
            int page = 0;
            var results = await _client.ScrollAsync<JObject>(scroll, scanResults.ScrollId).AnyContext();
            while (results.Documents.Any()) {
                var bulkDescriptor = new BulkDescriptor();
                foreach (var hit in results.Hits) {
                    var h = hit;
                    // TODO: Add support for doing JObject based schema migrations
                    bulkDescriptor.Index<JObject>(idx => idx.Index(workItem.NewIndex).Type(h.Type).Id(h.Id).Document(h.Source));
                }

                var bulkResponse = await _client.BulkAsync(bulkDescriptor).AnyContext();
                if (!bulkResponse.IsValid) {
                    string message = $"Reindex bulk error: old={workItem.OldIndex} new={workItem.NewIndex} page={page} message={bulkResponse.GetErrorMessage()}";
                    Logger.Warn().Message(message).Write();
                    // try each doc individually so we can see which doc is breaking us
                    foreach (var hit in results.Hits) {
                        var h = hit;
                        var response = await _client.IndexAsync(h.Source, d => d.Index(workItem.NewIndex).Type(h.Type).Id(h.Id)).AnyContext();

                        if (response.IsValid)
                            continue;

                        message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={hit.Id} page={page} message={response.GetErrorMessage()}";
                        Logger.Error().Message(message).Write();
                        throw new ReindexException(response.ConnectionStatus, message);
                    }
                }

                completed += bulkResponse.Items.Count();
                await context.ReportProgressAsync(CalculateProgress(totalHits, completed, startProgress, endProgress), $"Total: {totalHits} Completed: {completed}").AnyContext();
                results = await _client.ScrollAsync<JObject>(scroll, results.ScrollId).AnyContext();
                page++;
            }

            return new ReindexResult {
                Total = totalHits,
                Completed = completed
            };
        }
Exemplo n.º 19
0
        public override async Task HandleItemAsync(WorkItemContext context)
        {
            var jobData = context.GetData <MyWorkItem>();

            Assert.Equal("Test", jobData.SomeData);

            for (int i = 1; i < 10; i++)
            {
                await SystemClock.SleepAsync(100);

                await context.ReportProgressAsync(10 *i);
            }
        }
        public override async Task HandleItemAsync(WorkItemContext context)
        {
            var wi = context.GetData <RemoveStacksWorkItem>();

            using (Log.BeginScope(new ExceptionlessState().Organization(wi.OrganizationId).Project(wi.ProjectId))) {
                Log.LogInformation("Received remove stacks work item for project: {ProjectId}", wi.ProjectId);
                await context.ReportProgressAsync(0, "Starting soft deleting of stacks...").AnyContext();

                long deleted = await _stackRepository.SoftDeleteByProjectIdAsync(wi.OrganizationId, wi.ProjectId).AnyContext();

                await context.ReportProgressAsync(100, $"Stacks soft deleted: {deleted}").AnyContext();
            }
        }
Exemplo n.º 21
0
    public override async Task HandleItemAsync(WorkItemContext context)
    {
        var wi = context.GetData <RemoveBotEventsWorkItem>();

        using var _ = Log.BeginScope(new ExceptionlessState().Organization(wi.OrganizationId).Project(wi.ProjectId));
        Log.LogInformation("Received remove bot events work item OrganizationId={OrganizationId} ProjectId={ProjectId}, ClientIpAddress={ClientIpAddress}, UtcStartDate={UtcStartDate}, UtcEndDate={UtcEndDate}", wi.OrganizationId, wi.ProjectId, wi.ClientIpAddress, wi.UtcStartDate, wi.UtcEndDate);

        await context.ReportProgressAsync(0, $"Starting deleting of bot events... OrganizationId={wi.OrganizationId}").AnyContext();

        long deleted = await _eventRepository.RemoveAllAsync(wi.OrganizationId, wi.ClientIpAddress, wi.UtcStartDate, wi.UtcEndDate).AnyContext();

        await context.ReportProgressAsync(100, $"Bot events deleted: {deleted} OrganizationId={wi.OrganizationId}").AnyContext();
    }
Exemplo n.º 22
0
        public override async Task HandleItemAsync(WorkItemContext context)
        {
            const int LIMIT = 100;
            var       wi    = context.GetData <OrganizationMaintenanceWorkItem>();

            Log.LogInformation("Received upgrade organizations work item. Upgrade Plans: {UpgradePlans}", wi.UpgradePlans);

            var results = await _organizationRepository.GetAllAsync(o => o.PageLimit(LIMIT)).AnyContext();

            while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested)
            {
                foreach (var organization in results.Documents)
                {
                    if (wi.UpgradePlans)
                    {
                        UpgradePlan(organization);
                    }

                    if (wi.RemoveOldUsageStats)
                    {
                        foreach (var usage in organization.OverageHours.Where(u => u.Date < SystemClock.UtcNow.Subtract(TimeSpan.FromDays(3))).ToList())
                        {
                            organization.OverageHours.Remove(usage);
                        }

                        foreach (var usage in organization.Usage.Where(u => u.Date < SystemClock.UtcNow.Subtract(TimeSpan.FromDays(366))).ToList())
                        {
                            organization.Usage.Remove(usage);
                        }
                    }
                }

                if (wi.UpgradePlans || wi.RemoveOldUsageStats)
                {
                    await _organizationRepository.SaveAsync(results.Documents).AnyContext();
                }

                // Sleep so we are not hammering the backend.
                await SystemClock.SleepAsync(TimeSpan.FromSeconds(2.5)).AnyContext();

                if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync().AnyContext())
                {
                    break;
                }

                if (results.Documents.Count > 0)
                {
                    await context.RenewLockAsync().AnyContext();
                }
            }
        }
Exemplo n.º 23
0
        public override async Task HandleItemAsync(WorkItemContext context) {
            var workItem = context.GetData<StackWorkItem>();
            if (workItem.Delete) {
                await _eventRepository.RemoveAllByStackIdsAsync(new[] { workItem.StackId }).AnyContext();
                await _stackRepository.RemoveAsync(workItem.StackId).AnyContext();
                return;
            }

            if (workItem.UpdateIsFixed)
                await _eventRepository.UpdateFixedByStackAsync(workItem.OrganizationId, workItem.StackId, workItem.IsFixed).AnyContext();

            if (workItem.UpdateIsHidden)
                await _eventRepository.UpdateHiddenByStackAsync(workItem.OrganizationId, workItem.StackId, workItem.IsHidden).AnyContext();
        }
Exemplo n.º 24
0
        public override async Task HandleItemAsync(WorkItemContext context)
        {
            Assert.NotNull(Dependency);

            var jobData = context.GetData <MyWorkItem>();

            Assert.Equal("Test", jobData.SomeData);

            for (int i = 1; i < 10; i++)
            {
                await Task.Delay(100);

                await context.ReportProgressAsync(10 *i);
            }
        }
Exemplo n.º 25
0
        public Task HandleItem(WorkItemContext context)
        {
            Assert.NotNull(Dependency);

            var jobData = context.GetData <MyWorkItem>();

            Assert.Equal("Test", jobData.SomeData);

            for (int i = 0; i < 10; i++)
            {
                Thread.Sleep(100);
                context.ReportProgress(10 * i);
            }

            return(TaskHelper.Completed());
        }
        public override async Task HandleItemAsync(WorkItemContext context)
        {
            var workItem = context.GetData <SetProjectIsConfiguredWorkItem>();

            Logger.Info().Message($"Setting Is Configured for project: {workItem.ProjectId}").Write();

            var project = await _projectRepository.GetByIdAsync(workItem.ProjectId).AnyContext();

            if (project == null || project.IsConfigured.GetValueOrDefault())
            {
                return;
            }

            project.IsConfigured = workItem.IsConfigured || await _eventRepository.GetCountByProjectIdAsync(project.Id).AnyContext() > 0;

            await _projectRepository.SaveAsync(project, true).AnyContext();
        }
Exemplo n.º 27
0
        public void SaveWorkItem(WorkItem item)
        {
            using (var ctx = new WorkItemContext())
            {
                var obj = (from row in ctx.WorkItems
                           where row.Oid == item.Oid
                           select row).FirstOrDefault();

                if (obj != null)
                {
                    obj.Status  = item.Status;
                    obj.Request = item.Request;
                }

                ctx.SaveChanges();
            }
        }
Exemplo n.º 28
0
    public override async Task HandleItemAsync(WorkItemContext context)
    {
        var workItem = context.GetData <SetProjectIsConfiguredWorkItem>();

        Log.LogInformation("Setting Is Configured for project: {project}", workItem.ProjectId);

        var project = await _projectRepository.GetByIdAsync(workItem.ProjectId).AnyContext();

        if (project == null || project.IsConfigured.GetValueOrDefault())
        {
            return;
        }

        project.IsConfigured = workItem.IsConfigured || await _eventRepository.CountAsync(q => q.Project(project.Id)).AnyContext() > 0;

        await _projectRepository.SaveAsync(project, o => o.Cache()).AnyContext();
    }
        public override async Task HandleItemAsync(WorkItemContext context)
        {
            var workItem = context.GetData <ReindexWorkItem>();

            long existingDocCount = (await _client.CountAsync(d => d.Index(workItem.NewIndex)).AnyContext()).Count;

            Log.Info().Message("Received reindex work item for new index {0}", workItem.NewIndex).Write();
            var startTime = DateTime.UtcNow.AddSeconds(-1);
            await context.ReportProgressAsync(0, "Starting reindex...").AnyContext();

            var result = await ReindexAsync(workItem, context, 0, 90, workItem.StartUtc).AnyContext();

            await context.ReportProgressAsync(90, $"Total: {result.Total} Completed: {result.Completed}").AnyContext();

            // TODO: Check to make sure the docs have been added to the new index before changing alias
            if (!String.IsNullOrEmpty(workItem.Alias))
            {
                await _client.AliasAsync(x => x
                                         .Remove(a => a.Alias(workItem.Alias).Index(workItem.OldIndex))
                                         .Add(a => a.Alias(workItem.Alias).Index(workItem.NewIndex))).AnyContext();

                await context.ReportProgressAsync(98, $"Updated alias: {workItem.Alias} Remove: {workItem.OldIndex} Add: {workItem.NewIndex}").AnyContext();
            }

            await _client.RefreshAsync().AnyContext();

            var secondPassResult = await ReindexAsync(workItem, context, 90, 98, startTime).AnyContext();

            await context.ReportProgressAsync(98, $"Total: {secondPassResult.Total} Completed: {secondPassResult.Completed}").AnyContext();

            if (workItem.DeleteOld)
            {
                await _client.RefreshAsync().AnyContext();

                long newDocCount = (await _client.CountAsync(d => d.Index(workItem.NewIndex)).AnyContext()).Count - existingDocCount;
                long oldDocCount = (await _client.CountAsync(d => d.Index(workItem.OldIndex)).AnyContext()).Count;
                await context.ReportProgressAsync(98, $"Old Docs: {oldDocCount} New Docs: {newDocCount}").AnyContext();

                if (newDocCount >= oldDocCount)
                {
                    await _client.DeleteIndexAsync(d => d.Index(workItem.OldIndex)).AnyContext();
                }
                await context.ReportProgressAsync(98, $"Deleted index: {workItem.OldIndex}").AnyContext();
            }
            await context.ReportProgressAsync(100).AnyContext();
        }
        public override async Task HandleItemAsync(WorkItemContext context)
        {
            var wi = context.GetData <RemoveProjectWorkItem>();

            using (Log.BeginScope(new ExceptionlessState().Organization(wi.OrganizationId).Project(wi.ProjectId))) {
                Log.LogInformation("Received remove project work item for: {0} Reset Data: {1}", wi.ProjectId, wi.Reset);

                await context.ReportProgressAsync(0, "Starting deletion...").AnyContext();

                var project = await _projectRepository.GetByIdAsync(wi.ProjectId).AnyContext();

                if (project == null)
                {
                    await context.ReportProgressAsync(100, wi.Reset? "Project data reset" : "Project deleted").AnyContext();

                    return;
                }

                if (!wi.Reset)
                {
                    await context.ReportProgressAsync(20, "Removing tokens").AnyContext();

                    await _tokenRepository.RemoveAllByProjectIdAsync(project.OrganizationId, project.Id).AnyContext();

                    await context.ReportProgressAsync(40, "Removing web hooks").AnyContext();

                    await _webHookRepository.RemoveAllByProjectIdAsync(project.OrganizationId, project.Id).AnyContext();
                }

                await context.ReportProgressAsync(60, "Resetting project data").AnyContext();

                await _eventRepository.RemoveAllByProjectIdAsync(project.OrganizationId, project.Id).AnyContext();

                await _stackRepository.RemoveAllByProjectIdAsync(project.OrganizationId, project.Id).AnyContext();

                if (!wi.Reset)
                {
                    await context.ReportProgressAsync(80, "Removing project").AnyContext();

                    await _projectRepository.RemoveAsync(project.Id).AnyContext();
                }

                await context.ReportProgressAsync(100, wi.Reset? "Project data reset" : "Project deleted").AnyContext();
            }
        }
        public override async Task HandleItemAsync(WorkItemContext context)
        {
            var workItem = context.GetData <OrganizationNotificationWorkItem>();

            Log.LogInformation("Received organization notification work item for: {organization} IsOverHourlyLimit: {IsOverHourlyLimit} IsOverMonthlyLimit: {IsOverMonthlyLimit}", workItem.OrganizationId, workItem.IsOverHourlyLimit, workItem.IsOverMonthlyLimit);

            var organization = await _organizationRepository.GetByIdAsync(workItem.OrganizationId, o => o.Cache()).AnyContext();

            if (organization == null)
            {
                return;
            }

            if (workItem.IsOverMonthlyLimit)
            {
                await SendOverageNotificationsAsync(organization, workItem.IsOverHourlyLimit, workItem.IsOverMonthlyLimit).AnyContext();
            }
        }
        public XElement GetQueryXml(WorkItemContext context, FieldList fields)
        {
            var parser = new LexalParser(this.QueryText);
            var nodes  = parser.ProcessWherePart();

            nodes.Optimize();
            nodes.ExtractOperatorForward();
            nodes.FixFields(fields);
            nodes.FillFieldTypes(fields);

            var manager = new ParameterManager(context);

            manager.EvalParameters(nodes);

            var xmlTransformer = new NodesToXml(nodes);

            return(XElement.Parse(xmlTransformer.WriteXml()));
        }
        public override async Task HandleItemAsync(WorkItemContext context)
        {
            const int LIMIT = 100;

            var workItem = context.GetData <ProjectMaintenanceWorkItem>();

            Log.Info("Received upgrade projects work item. Update Default Bot List: {0} IncrementConfigurationVersion: {1}", workItem.UpdateDefaultBotList, workItem.IncrementConfigurationVersion);

            var results = await _projectRepository.GetAllAsync(o => o.PageLimit(LIMIT)).AnyContext();

            while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested)
            {
                foreach (var project in results.Documents)
                {
                    if (workItem.UpdateDefaultBotList)
                    {
                        project.SetDefaultUserAgentBotPatterns();
                    }

                    if (workItem.IncrementConfigurationVersion)
                    {
                        project.Configuration.IncrementVersion();
                    }
                }

                if (workItem.UpdateDefaultBotList)
                {
                    await _projectRepository.SaveAsync(results.Documents).AnyContext();
                }

                // Sleep so we are not hammering the backend.
                await SystemClock.SleepAsync(TimeSpan.FromSeconds(2.5)).AnyContext();

                if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync().AnyContext())
                {
                    break;
                }

                if (results.Documents.Count > 0)
                {
                    await context.RenewLockAsync().AnyContext();
                }
            }
        }
Exemplo n.º 34
0
        public override void Canceling()
        {
            using (var ctx = new WorkItemContext())
            {
                Item = (from row in ctx.WorkItems
                        where row.Oid == Item.Oid
                        select row).FirstOrDefault();

                DateTime now = DateTime.Now;

                if (Item != null)
                {
                    Item.Progress = Progress;
                    Item.Status   = WorkItemStatusEnum.Canceling;
                }

                ctx.SaveChanges();
            }
        }
        public override async Task HandleItemAsync(WorkItemContext context)
        {
            var workItem = context.GetData <RemoveProjectWorkItem>();

            Log.Info("Received remove project work item for: {0} Reset Data: {1}", workItem.ProjectId, workItem.Reset);

            await context.ReportProgressAsync(0, "Starting deletion...").AnyContext();

            var project = await _projectRepository.GetByIdAsync(workItem.ProjectId).AnyContext();

            if (project == null)
            {
                await context.ReportProgressAsync(100, workItem.Reset? "Project data reset" : "Project deleted").AnyContext();

                return;
            }

            if (!workItem.Reset)
            {
                await context.ReportProgressAsync(20, "Removing tokens").AnyContext();

                await _tokenRepository.RemoveAllByOrganizationIdsAsync(new[] { project.Id }).AnyContext();

                await context.ReportProgressAsync(40, "Removing web hooks").AnyContext();

                await _webHookRepository.RemoveAllByOrganizationIdsAsync(new[] { project.Id }).AnyContext();
            }

            await context.ReportProgressAsync(60, "Resetting project data").AnyContext();

            await _eventRepository.RemoveAllByProjectIdsAsync(new[] { project.Id }).AnyContext();

            await _stackRepository.RemoveAllByProjectIdsAsync(new[] { project.Id }).AnyContext();

            if (!workItem.Reset)
            {
                await context.ReportProgressAsync(80, "Removing project").AnyContext();

                await _projectRepository.RemoveAsync(project.Id).AnyContext();
            }

            await context.ReportProgressAsync(100, workItem.Reset? "Project data reset" : "Project deleted").AnyContext();
        }
Exemplo n.º 36
0
        public override async Task HandleItemAsync(WorkItemContext context)
        {
            var workItem = context.GetData <SetLocationFromGeoWorkItem>();

            GeoResult result;

            if (!GeoResult.TryParse(workItem.Geo, out result))
            {
                return;
            }

            var location = await _cacheClient.GetAsync <Location>(workItem.Geo, null).AnyContext();

            if (location == null)
            {
                try {
                    result = await _geocodeService.ReverseGeocodeAsync(result.Latitude.GetValueOrDefault(), result.Longitude.GetValueOrDefault()).AnyContext();

                    location = result.ToLocation();
                    await _metricsClient.CounterAsync(MetricNames.UsageGeocodingApi).AnyContext();
                } catch (Exception ex) {
                    Logger.Error().Exception(ex).Message($"Error occurred looking up reverse geocode: {workItem.Geo}").Write();
                }
            }

            if (location == null)
            {
                return;
            }

            await _cacheClient.SetAsync(workItem.Geo, location, TimeSpan.FromDays(3)).AnyContext();

            var ev = await _eventRepository.GetByIdAsync(workItem.EventId).AnyContext();

            if (ev == null)
            {
                return;
            }

            ev.SetLocation(location);
            await _eventRepository.SaveAsync(ev).AnyContext();
        }
        // NOTE: Uncomment to ensure only one work item handler is called at a time.
        //public override Task<ILock> GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = new CancellationToken()) {
        //    return _lockProvider.AcquireAsync(nameof(DeleteValueWorkItemHandler), cancellationToken: cancellationToken);
        //}

        public override async Task HandleItemAsync(WorkItemContext context) {
            var workItem = context.GetData<DeleteValueWorkItem>();
            await context.ReportProgressAsync(0, $"Starting to delete item: {workItem.Id}.");
            await context.ReportProgressAsync(1, "If you are seeing multiple progresses. Please uncomment the lock in the DeleteValueWorkItemHandler.");
            await Task.Delay(TimeSpan.FromSeconds(2.5));
            await context.ReportProgressAsync(50, "Deleting");
            await Task.Delay(TimeSpan.FromSeconds(.5));
            await context.ReportProgressAsync(70, "Deleting.");
            await Task.Delay(TimeSpan.FromSeconds(.5));
            await context.ReportProgressAsync(90, "Deleting..");
            await Task.Delay(TimeSpan.FromSeconds(.5));

            await _cacheClient.RemoveAsync(workItem.Id);
            await _publisher.PublishAsync(new EntityChanged {
                ChangeType = ChangeType.Removed,
                Id = workItem.Id
            });

            await context.ReportProgressAsync(100);
        }
Exemplo n.º 38
0
    public override Task HandleItemAsync(WorkItemContext context)
    {
        var    wi       = context.GetData <OrganizationNotificationWorkItem>();
        string cacheKey = $"{nameof(OrganizationNotificationWorkItemHandler)}:{wi.OrganizationId}";

        return(_lockProvider.TryUsingAsync(cacheKey, async() => {
            Log.LogInformation("Received organization notification work item for: {organization} IsOverHourlyLimit: {IsOverHourlyLimit} IsOverMonthlyLimit: {IsOverMonthlyLimit}", wi.OrganizationId, wi.IsOverHourlyLimit, wi.IsOverMonthlyLimit);

            var organization = await _organizationRepository.GetByIdAsync(wi.OrganizationId, o => o.Cache()).AnyContext();
            if (organization == null)
            {
                return;
            }

            if (wi.IsOverMonthlyLimit)
            {
                await SendOverageNotificationsAsync(organization, wi.IsOverHourlyLimit, wi.IsOverMonthlyLimit).AnyContext();
            }
        }, TimeSpan.FromMinutes(15), new CancellationToken(true)));
    }
Exemplo n.º 39
0
        public override void Idle()
        {
            using (var ctx = new WorkItemContext())
            {
                Item = (from row in ctx.WorkItems
                        where row.Oid == Item.Oid
                        select row).FirstOrDefault();

                DateTime now = DateTime.Now;

                if (Item != null)
                {
                    Item.Progress       = Progress;
                    Item.ExpirationTime = now;
                    Item.DeleteTime     = now.AddMinutes(240);
                    Item.Status         = WorkItemStatusEnum.Idle;
                }

                ctx.SaveChanges();
            }
        }
        public override async Task HandleItemAsync(WorkItemContext context)
        {
            const int LIMIT = 100;

            var workItem = context.GetData <UserMaintenanceWorkItem>();

            Log.Info("Received user maintenance work item. Normalize: {0}", workItem.Normalize);

            var results = await _userRepository.GetAllAsync(paging : new PagingOptions().WithLimit(LIMIT)).AnyContext();

            while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested)
            {
                foreach (var user in results.Documents)
                {
                    if (workItem.Normalize)
                    {
                        NormalizeUser(user);
                    }
                }

                if (workItem.Normalize)
                {
                    await _userRepository.SaveAsync(results.Documents).AnyContext();
                }

                // Sleep so we are not hammering the backend.
                await SystemClock.SleepAsync(TimeSpan.FromSeconds(2.5)).AnyContext();

                if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync().AnyContext())
                {
                    break;
                }

                if (results.Documents.Count > 0)
                {
                    await context.RenewLockAsync().AnyContext();
                }
            }
        }
        public override async Task HandleItemAsync(WorkItemContext context) {
            const int LIMIT = 100;

            var workItem = context.GetData<OrganizationMaintenanceWorkItem>();
            _logger.Info("Received upgrade organizations work item. Upgrade Plans: {0}", workItem.UpgradePlans);

            var results = await _organizationRepository.GetAllAsync(paging: new PagingOptions().WithLimit(LIMIT)).AnyContext();
            while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) {
                foreach (var organization in results.Documents) {
                    if (workItem.UpgradePlans)
                        UpgradePlan(organization);
                }

                if (workItem.UpgradePlans)
                    await _organizationRepository.SaveAsync(results.Documents).AnyContext();

                // Sleep so we are not hammering the backend.
                await Task.Delay(TimeSpan.FromSeconds(2.5)).AnyContext();

                await results.NextPageAsync().AnyContext();
                if (results.Documents.Count > 0)
                    await context.RenewLockAsync().AnyContext();
            }
        }
Exemplo n.º 42
0
        public override Task HandleItem(WorkItemContext context) {
            Assert.NotNull(Dependency);

            var jobData = context.GetData<MyWorkItem>();
            Assert.Equal("Test", jobData.SomeData);

            for (int i = 0; i < 10; i++) {
                Thread.Sleep(100);
                context.ReportProgress(10 * i);
            }

            return TaskHelper.Completed();
        }
        public override async Task HandleItemAsync(WorkItemContext context) {
            var workItem = context.GetData<RemoveOrganizationWorkItem>();
            Logger.Info().Message($"Received remove organization work item for: {workItem.OrganizationId}").Write();

            await context.ReportProgressAsync(0, "Starting deletion...").AnyContext();
            var organization = await _organizationRepository.GetByIdAsync(workItem.OrganizationId).AnyContext();
            if (organization == null) {
                await context.ReportProgressAsync(100, "Organization deleted").AnyContext();
                return;
            }

            await context.ReportProgressAsync(10, "Removing subscriptions").AnyContext();
            if (!String.IsNullOrEmpty(organization.StripeCustomerId)) {
                Logger.Info().Message($"Canceling stripe subscription for the organization '{organization.Name}' with Id: '{organization.Id}'.").Write();

                var subscriptionService = new StripeSubscriptionService(Settings.Current.StripeApiKey);
                var subscriptions = subscriptionService.List(organization.StripeCustomerId).Where(s => !s.CanceledAt.HasValue);
                foreach (var subscription in subscriptions)
                    subscriptionService.Cancel(organization.StripeCustomerId, subscription.Id);
            }

            await context.ReportProgressAsync(20, "Removing users").AnyContext();
            var users = await _userRepository.GetByOrganizationIdAsync(organization.Id).AnyContext();
            foreach (User user in users.Documents) {
                // delete the user if they are not associated to any other organizations and they are not the current user
                if (user.OrganizationIds.All(oid => String.Equals(oid, organization.Id)) && !String.Equals(user.Id, workItem.CurrentUserId)) {
                    Logger.Info().Message("Removing user '{0}' as they do not belong to any other organizations.", user.Id, organization.Name, organization.Id).Write();
                    await _userRepository.RemoveAsync(user.Id).AnyContext();
                } else {
                    Logger.Info().Message("Removing user '{0}' from organization '{1}' with Id: '{2}'", user.Id, organization.Name, organization.Id).Write();
                    user.OrganizationIds.Remove(organization.Id);
                    await _userRepository.SaveAsync(user, true).AnyContext();
                }
            }

            await context.ReportProgressAsync(30, "Removing tokens").AnyContext();
            await _tokenRepository.RemoveAllByOrganizationIdsAsync(new[] { organization.Id }).AnyContext();

            await context.ReportProgressAsync(40, "Removing web hooks").AnyContext();
            await _webHookRepository.RemoveAllByOrganizationIdsAsync(new[] { organization.Id }).AnyContext();

            await context.ReportProgressAsync(50, "Removing projects").AnyContext();
            var projects = await _projectRepository.GetByOrganizationIdAsync(organization.Id).AnyContext();
            if (workItem.IsGlobalAdmin && projects.Total > 0) {
                var completed = 1;
                foreach (Project project in projects.Documents) {
                    Logger.Info().Message("Resetting all project data for project '{0}' with Id: '{1}'.", project.Name, project.Id).Write();
                    await _eventRepository.RemoveAllByProjectIdsAsync(new[] { project.Id }).AnyContext();
                    await _stackRepository.RemoveAllByProjectIdsAsync(new[] { project.Id }).AnyContext();
                    await context.ReportProgressAsync(CalculateProgress(projects.Total, completed++, 51, 89), "Removing projects...").AnyContext();
                }

                Logger.Info().Message("Deleting all projects for organization '{0}' with Id: '{1}'.", organization.Name, organization.Id).Write();
                await _projectRepository.RemoveAsync(projects.Documents).AnyContext();
            }

            Logger.Info().Message("Deleting organization '{0}' with Id: '{1}'.", organization.Name, organization.Id).Write();
            await context.ReportProgressAsync(90, "Removing organization").AnyContext();
            await _organizationRepository.RemoveAsync(organization.Id).AnyContext();

            await context.ReportProgressAsync(100, "Organization deleted").AnyContext();
        }
 public override async Task HandleItemAsync(WorkItemContext context) {
     var workItem = context.GetData<ReindexWorkItem>();
     await _reindexer.ReindexAsync(workItem, context.ReportProgressAsync).AnyContext();
 }
Exemplo n.º 45
0
        private async Task<ReindexResult> ReindexAsync(ReindexWorkItem workItem, WorkItemContext context, int startProgress = 0, int endProgress = 100, DateTime? startTime = null) {
            const int pageSize = 100;
            const string scroll = "5m";
            string timestampField = workItem.TimestampField ?? "_timestamp";

            long completed = 0;

            var scanResults = await _client.SearchAsync<JObject>(s => s
                .Index(workItem.OldIndex)
                .AllTypes()
                .Filter(f => startTime.HasValue
                    ? f.Range(r => r.OnField(timestampField).Greater(startTime.Value))
                    : f.MatchAll())
                .From(0).Take(pageSize)
                .SearchType(SearchType.Scan)
                .Scroll(scroll)).AnyContext();

            if (!scanResults.IsValid || scanResults.ScrollId == null) {
                Logger.Error().Message("Invalid search result: message={0}", scanResults.GetErrorMessage()).Write();
                return new ReindexResult();
            }

            long totalHits = scanResults.Total;

            var parentMap = workItem.ParentMaps?.ToDictionary(p => p.Type, p => p.ParentPath) ?? new Dictionary<string, string>();

            var results = await _client.ScrollAsync<JObject>(scroll, scanResults.ScrollId).AnyContext();
            while (results.Documents.Any()) {
                var bulkDescriptor = new BulkDescriptor();
                foreach (var hit in results.Hits) {
                    var h = hit;
                    // TODO: Add support for doing JObject based schema migrations
                    bulkDescriptor.Index<JObject>(idx => {
                        idx
                            .Index(workItem.NewIndex)
                            .Type(h.Type)
                            .Id(h.Id)
                            .Document(h.Source);

                        if (String.IsNullOrEmpty(h.Type))
                            Logger.Error().Message("Hit type empty. id={0}", h.Id).Write();

                        if (parentMap.ContainsKey(h.Type)) {
                            if (String.IsNullOrEmpty(parentMap[h.Type]))
                                Logger.Error().Message("Parent map has empty value. id={0} type={1}", h.Id, h.Type).Write();

                            var parentId = h.Source.SelectToken(parentMap[h.Type]);
                            if (!String.IsNullOrEmpty(parentId?.ToString()))
                                idx.Parent(parentId.ToString());
                            else
                                Logger.Error().Message("Unable to get parent id. id={0} path={1}", h.Id, parentMap[h.Type]).Write();
                        }

                        return idx;
                    });
                }

                var bulkResponse = await _client.BulkAsync(bulkDescriptor).AnyContext();
                if (!bulkResponse.IsValid) {
                    string message = $"Reindex bulk error: old={workItem.OldIndex} new={workItem.NewIndex} completed={completed} message={bulkResponse.GetErrorMessage()}";
                    Logger.Warn().Message(message).Write();
                    // try each doc individually so we can see which doc is breaking us
                    foreach (var hit in results.Hits) {
                        var h = hit;
                        var response = await _client.IndexAsync<JObject>(h.Source, d => {
                            d
                                .Index(workItem.NewIndex)
                                .Type(h.Type)
                                .Id(h.Id);

                            if (parentMap.ContainsKey(h.Type)) {
                                var parentId = h.Source.SelectToken(parentMap[h.Type]);
                                if (!String.IsNullOrEmpty(parentId?.ToString()))
                                    d.Parent(parentId.ToString());
                                else
                                    Logger.Error().Message("Unable to get parent id. id={0} path={1}", h.Id, parentMap[h.Type]).Write();
                            }

                            return d;
                        }).AnyContext();

                        if (response.IsValid)
                            continue;

                        message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={hit.Id} completed={completed} message={response.GetErrorMessage()}";
                        Logger.Error().Message(message).Write();

                        var errorDoc = new JObject(new {
                            h.Type,
                            Content = h.Source.ToString(Formatting.Indented)
                        });

                        if (parentMap.ContainsKey(h.Type)) {
                            var parentId = h.Source.SelectToken(parentMap[h.Type]);
                            if (!String.IsNullOrEmpty(parentId?.ToString()))
                                errorDoc["ParentId"] = parentId.ToString();
                            else
                                Logger.Error().Message("Unable to get parent id. id={0} path={1}", h.Id, parentMap[h.Type]).Write();
                        }

                        // put the document into an error index
                        response = await _client.IndexAsync<JObject>(errorDoc, d => {
                            d
                                .Index(workItem.NewIndex + "-error")
                                .Id(h.Id);

                            return d;
                        }).AnyContext();

                        if (response.IsValid)
                            continue;

                        throw new ReindexException(response.ConnectionStatus, message);
                    }
                }

                completed += bulkResponse.Items.Count();
                await context.ReportProgressAsync(CalculateProgress(totalHits, completed, startProgress, endProgress),
                    $"Total: {totalHits} Completed: {completed}").AnyContext();

                Logger.Info().Message($"Reindex Progress: {CalculateProgress(totalHits, completed, startProgress, endProgress)} Completed: {completed} Total: {totalHits}").Write();
                results = await _client.ScrollAsync<JObject>(scroll, results.ScrollId).AnyContext();
            }

            return new ReindexResult { Total = totalHits, Completed = completed };
        }
Exemplo n.º 46
0
        public override async Task HandleItemAsync(WorkItemContext context) {
            Assert.NotNull(Dependency);

            var jobData = context.GetData<MyWorkItem>();
            Assert.Equal("Test", jobData.SomeData);

            for (int i = 1; i < 10; i++) {
                await Task.Delay(100);
                await context.ReportProgressAsync(10 * i);
            }
        }
 public override Task HandleItemAsync(WorkItemContext context) {
     var workItem = context.GetData<ThrottleBotsWorkItem>();
     return _eventRepository.HideAllByClientIpAndDateAsync(workItem.OrganizationId, workItem.ClientIpAddress, workItem.UtcStartDate, workItem.UtcEndDate);
 }
Exemplo n.º 48
0
        public override async Task HandleItemAsync(WorkItemContext context) {
            var jobData = context.GetData<MyWorkItem>();
            Assert.Equal("Test", jobData.SomeData);

            for (int i = 1; i < 10; i++) {
                await SystemClock.SleepAsync(100);
                await context.ReportProgressAsync(10 * i);
            }
        }