public async Task Invoke(HttpContext context) { if (!IsEventPost(context)) { await _next(context); return; } if (_appOptions.EventSubmissionDisabled) { context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable; return; } string organizationId = context.Request.GetDefaultOrganizationId(); var organizationTask = _organizationRepository.GetByIdAsync(organizationId, o => o.Cache()); string projectId = context.Request.GetDefaultProjectId(); var projectTask = _projectRepository.GetByIdAsync(projectId, o => o.Cache()); bool tooBig = false; if (String.Equals(context.Request.Method, "POST", StringComparison.OrdinalIgnoreCase) && context.Request.Headers != null) { if (context.Request.Headers.ContentLength.HasValue && context.Request.Headers.ContentLength.Value <= 0) { //_metricsClient.Counter(MetricNames.PostsBlocked); context.Response.StatusCode = StatusCodes.Status411LengthRequired; await Task.WhenAll(organizationTask, projectTask); return; } long size = context.Request.Headers.ContentLength.GetValueOrDefault(); if (size > 0) { _metricsClient.Gauge(MetricNames.PostsSize, size); } if (size > _appOptions.MaximumEventPostSize) { if (_logger.IsEnabled(LogLevel.Warning)) { using (_logger.BeginScope(new ExceptionlessState().Value(size).Tag(context.Request.Headers.TryGetAndReturn(Headers.ContentEncoding)))) _logger.SubmissionTooLarge(size); } _metricsClient.Counter(MetricNames.PostsDiscarded); tooBig = true; } } var organization = await organizationTask; var project = await projectTask; // block large submissions, client should break them up or remove some of the data. if (tooBig) { await _usageService.IncrementTooBigAsync(organization, project).AnyContext(); context.Response.StatusCode = StatusCodes.Status413RequestEntityTooLarge; return; } bool overLimit = await _usageService.IsOverLimitAsync(organization).AnyContext(); if (overLimit) { _metricsClient.Counter(MetricNames.PostsBlocked); context.Response.StatusCode = StatusCodes.Status402PaymentRequired; return; } context.Request.SetOrganization(organization); context.Request.SetProject(project); await _next(context); }
protected override async Task <JobResult> ProcessQueueEntryAsync(QueueEntryContext <EventPost> context) { var entry = context.QueueEntry; var ep = entry.Value; string payloadPath = Path.ChangeExtension(entry.Value.FilePath, ".payload"); var payloadTask = _metrics.TimeAsync(() => _eventPostService.GetEventPostPayloadAsync(payloadPath), MetricNames.PostsMarkFileActiveTime); var projectTask = _projectRepository.GetByIdAsync(ep.ProjectId, o => o.Cache()); var organizationTask = _organizationRepository.GetByIdAsync(ep.OrganizationId, o => o.Cache()); byte[] payload = await payloadTask.AnyContext(); if (payload == null) { await Task.WhenAll(AbandonEntryAsync(entry), projectTask, organizationTask).AnyContext(); return(JobResult.FailedWithMessage($"Unable to retrieve payload '{payloadPath}'.")); } _metrics.Gauge(MetricNames.PostsMessageSize, payload.LongLength); if (payload.LongLength > _maximumEventPostFileSize) { await Task.WhenAll(_metrics.TimeAsync(() => entry.CompleteAsync(), MetricNames.PostsCompleteTime), projectTask, organizationTask).AnyContext(); return(JobResult.FailedWithMessage($"Unable to process payload '{payloadPath}' ({payload.LongLength} bytes): Maximum event post size limit ({_appOptions.MaximumEventPostSize} bytes) reached.")); } using (_logger.BeginScope(new ExceptionlessState().Organization(ep.OrganizationId).Project(ep.ProjectId))) { _metrics.Gauge(MetricNames.PostsCompressedSize, payload.Length); bool isDebugLogLevelEnabled = _logger.IsEnabled(LogLevel.Debug); bool isInternalProject = ep.ProjectId == _appOptions.InternalProjectId; if (!isInternalProject && _logger.IsEnabled(LogLevel.Information)) { using (_logger.BeginScope(new ExceptionlessState().Tag("processing").Tag("compressed").Tag(ep.ContentEncoding).Value(payload.Length))) _logger.LogInformation("Processing post: id={QueueEntryId} path={FilePath} project={project} ip={IpAddress} v={ApiVersion} agent={UserAgent}", entry.Id, payloadPath, ep.ProjectId, ep.IpAddress, ep.ApiVersion, ep.UserAgent); } var project = await projectTask.AnyContext(); if (project == null) { if (!isInternalProject) { _logger.LogError("Unable to process EventPost {FilePath}: Unable to load project: {Project}", payloadPath, ep.ProjectId); } await Task.WhenAll(CompleteEntryAsync(entry, ep, SystemClock.UtcNow), organizationTask).AnyContext(); return(JobResult.Success); } long maxEventPostSize = _appOptions.MaximumEventPostSize; byte[] uncompressedData = payload; if (!String.IsNullOrEmpty(ep.ContentEncoding)) { if (!isInternalProject && isDebugLogLevelEnabled) { using (_logger.BeginScope(new ExceptionlessState().Tag("decompressing").Tag(ep.ContentEncoding))) _logger.LogDebug("Decompressing EventPost: {QueueEntryId} ({CompressedBytes} bytes)", entry.Id, payload.Length); } maxEventPostSize = _maximumUncompressedEventPostSize; try { _metrics.Time(() => { uncompressedData = uncompressedData.Decompress(ep.ContentEncoding); }, MetricNames.PostsDecompressionTime); } catch (Exception ex) { _metrics.Counter(MetricNames.PostsDecompressionErrors); await Task.WhenAll(CompleteEntryAsync(entry, ep, SystemClock.UtcNow), organizationTask).AnyContext(); return(JobResult.FailedWithMessage($"Unable to decompress EventPost data '{payloadPath}' ({payload.Length} bytes compressed): {ex.Message}")); } } _metrics.Gauge(MetricNames.PostsUncompressedSize, payload.LongLength); if (uncompressedData.Length > maxEventPostSize) { var org = await organizationTask.AnyContext(); await _usageService.IncrementTooBigAsync(org, project).AnyContext(); await CompleteEntryAsync(entry, ep, SystemClock.UtcNow).AnyContext(); return(JobResult.FailedWithMessage($"Unable to process decompressed EventPost data '{payloadPath}' ({payload.Length} bytes compressed, {uncompressedData.Length} bytes): Maximum uncompressed event post size limit ({maxEventPostSize} bytes) reached.")); } if (!isInternalProject && isDebugLogLevelEnabled) { using (_logger.BeginScope(new ExceptionlessState().Tag("uncompressed").Value(uncompressedData.Length))) _logger.LogDebug("Processing uncompressed EventPost: {QueueEntryId} ({UncompressedBytes} bytes)", entry.Id, uncompressedData.Length); } var createdUtc = SystemClock.UtcNow; var events = ParseEventPost(ep, createdUtc, uncompressedData, entry.Id, isInternalProject); if (events == null || events.Count == 0) { await Task.WhenAll(CompleteEntryAsync(entry, ep, createdUtc), organizationTask).AnyContext(); return(JobResult.Success); } if (context.CancellationToken.IsCancellationRequested) { await Task.WhenAll(AbandonEntryAsync(entry), organizationTask).AnyContext(); return(JobResult.Cancelled); } var organization = await organizationTask.AnyContext(); if (organization == null) { if (!isInternalProject) { _logger.LogError("Unable to process EventPost {FilePath}: Unable to load organization: {OrganizationId}", payloadPath, project.OrganizationId); } await CompleteEntryAsync(entry, ep, SystemClock.UtcNow).AnyContext(); return(JobResult.Success); } bool isSingleEvent = events.Count == 1; if (!isSingleEvent) { // Don't process all the events if it will put the account over its limits. int eventsToProcess = await _usageService.GetRemainingEventLimitAsync(organization).AnyContext(); // Add 1 because we already counted 1 against their limit when we received the event post. if (eventsToProcess < Int32.MaxValue) { eventsToProcess += 1; } // Discard any events over the plan limit. if (eventsToProcess < events.Count) { int discarded = events.Count - eventsToProcess; events = events.Take(eventsToProcess).ToList(); _metrics.Counter(MetricNames.EventsDiscarded, discarded); } } int errorCount = 0; var eventsToRetry = new List <PersistentEvent>(); try { var contexts = await _eventPipeline.RunAsync(events, organization, project, ep).AnyContext(); if (!isInternalProject && isDebugLogLevelEnabled) { using (_logger.BeginScope(new ExceptionlessState().Value(contexts.Count))) _logger.LogDebug("Ran {@value} events through the pipeline: id={QueueEntryId} success={SuccessCount} error={ErrorCount}", contexts.Count, entry.Id, contexts.Count(r => r.IsProcessed), contexts.Count(r => r.HasError)); } // increment the plan usage counters (note: OverageHandler already incremented usage by 1) int processedEvents = contexts.Count(c => c.IsProcessed); await _usageService.IncrementUsageAsync(organization, project, processedEvents, applyHourlyLimit : false).AnyContext(); int discardedEvents = contexts.Count(c => c.IsDiscarded); _metrics.Counter(MetricNames.EventsDiscarded, discardedEvents); foreach (var ctx in contexts) { if (ctx.IsCancelled) { continue; } if (!ctx.HasError) { continue; } if (!isInternalProject) { _logger.LogError(ctx.Exception, "Error processing EventPost {QueueEntryId} {FilePath}: {Message}", entry.Id, payloadPath, ctx.ErrorMessage); } if (ctx.Exception is ValidationException) { continue; } errorCount++; if (!isSingleEvent) { // Put this single event back into the queue so we can retry it separately. eventsToRetry.Add(ctx.Event); } } } catch (Exception ex) { if (!isInternalProject) { _logger.LogError(ex, "Error processing EventPost {QueueEntryId} {FilePath}: {Message}", entry.Id, payloadPath, ex.Message); } if (ex is ArgumentException || ex is DocumentNotFoundException) { await CompleteEntryAsync(entry, ep, createdUtc).AnyContext(); return(JobResult.Success); } errorCount++; if (!isSingleEvent) { eventsToRetry.AddRange(events); } } if (eventsToRetry.Count > 0) { await _metrics.TimeAsync(() => RetryEventsAsync(eventsToRetry, ep, entry, project, isInternalProject), MetricNames.PostsRetryTime).AnyContext(); } if (isSingleEvent && errorCount > 0) { await AbandonEntryAsync(entry).AnyContext(); } else { await CompleteEntryAsync(entry, ep, createdUtc).AnyContext(); } return(JobResult.Success); } }