public void GetDocumentIsSuccesfull() { var identity = new TestIdentity(); var state0 = new TestState() { Val = "Initial" }; var savedState = stateProvider.SaveStateAsync(CancellationToken.None, identity, state0).Result; //blocking on a task for test purposes, should have a timeout var readState = stateProvider.GetStateAsync(CancellationToken.None, identity).Result; Assert.True(savedState.ETag == readState.ETag && savedState.Val == readState.Val && savedState.Val != null); }
private async Task _processEntry(int idx, int total, IResourceMetadata info, ResourceState lastState, CancellationToken ctk = default) { try { _logger.Info("({4}/{5}) Detected change on Resource.Id=\"{0}\", Resource.Modified={1}, OldState.Modified={2}, OldState.Retry={3}. Processing..." , info.ResourceId , info.Modified , lastState?.Modified , lastState?.RetryCount , idx , total ); var sw = Stopwatch.StartNew(); AsyncLazy <T> payload = new AsyncLazy <T>(() => _retrievePayload(info, lastState, ctk)); var state = new ResourceState() { Tenant = _config.Tenant, ResourceId = info.ResourceId, Modified = lastState?.Modified ?? info.Modified, // we want to update modified only on success or Ban or first run LastEvent = SystemClock.Instance.GetCurrentInstant(), RetryCount = lastState?.RetryCount ?? 0, CheckSum = lastState?.CheckSum, RetrievedAt = lastState?.RetrievedAt, Extensions = info.Extensions }; try { await _processResource(new ChangedStateContext <T>(info, lastState, payload), ctk).ConfigureAwait(false); // if handlers retrived data, fetch the result to check the checksum if (payload.IsStarted) { // if the retrievePayload task gone in exception but the _processResource did not ... // here we care only if we have a payload to use if (payload.Task.Status == TaskStatus.RanToCompletion) { var newState = await payload; if (newState != null) { if (!string.IsNullOrWhiteSpace(newState.CheckSum) && state.CheckSum != newState.CheckSum) { _logger.Info("Checksum changed on Resource.Id=\"{0}\" from \"{1}\" to \"{2}\"", state.ResourceId, state.CheckSum, newState.CheckSum); } state.CheckSum = newState.CheckSum; state.RetrievedAt = newState.RetrievedAt; } else // no payload retrived, so no new state. Generally due to a same-checksum { } state.Modified = info.Modified; state.RetryCount = 0; // success } } else // for some reason, no action has been and payload has not been retrieved. We do not change the state { } } catch (Exception ex) { state.LastException = ex; LogLevel lvl = ++state.RetryCount == _config.MaxRetries ? LogLevel.Fatal : LogLevel.Warn; _logger.Log(lvl, ex, "Error while processing ResourceId=\"{0}\"", info.ResourceId); // if we're in BAN and we tried to exit BAN and we failed, update the Modified anw if (state.RetryCount > _config.MaxRetries) { state.Modified = info.Modified; } } await _stateProvider.SaveStateAsync(new[] { state }, ctk).ConfigureAwait(false); _logger.Info("({3}/{4}) ResourceId=\"{0}\" handled {2}successfully in {1}", state.ResourceId, sw.Elapsed, state.RetryCount == 0 ? "" : "not ", idx , total); if (sw.Elapsed > _config.ResourceDurationNotificationLimit) { _logger.Fatal("Processing of ResourceId=\"{0}\" took too much: {1}", state.ResourceId, sw.Elapsed); } } catch (Exception ex) { // chomp it, we'll retry this file next time, forever, fuckit _logger.Error(ex, "({4}/{5}) Error in processing the ResourceId=\"{0}\". The execution will be automatically retried.", info.ResourceId); } }
private async Task _processEntry(ProcessContext pc, CancellationToken ctk = default) { var info = pc.CurrentInfo; var lastState = pc.LastState; var dataType = pc.ProcessType; try { var processActivity = _diagnosticSource.ProcessResourceStart(pc); AsyncLazy <T> payload = new AsyncLazy <T>(() => _fetchResource(pc, ctk)); var state = pc.NewState = new ResourceState() { Tenant = _config.Tenant, ResourceId = info.ResourceId, Modified = lastState?.Modified ?? default, // we want to update modified only on success or Ban or first run ModifiedSources = lastState?.ModifiedSources, // we want to update modified multiple only on success or Ban or first run LastEvent = SystemClock.Instance.GetCurrentInstant(), RetryCount = lastState?.RetryCount ?? 0, CheckSum = lastState?.CheckSum, RetrievedAt = lastState?.RetrievedAt, Extensions = info.Extensions }; try { pc.ResultType = ResultType.Normal; IResourceState newState = default; await _processResource(new ChangedStateContext <T>(info, lastState, payload), ctk).ConfigureAwait(false); // if handlers retrived data, fetch the result to check the checksum if (payload.IsStarted) { // if the retrievePayload task gone in exception but the _processResource did not ... // here we care only if we have a payload to use if (payload.Task.Status == TaskStatus.RanToCompletion) { newState = await payload; if (newState != null) { if (!string.IsNullOrWhiteSpace(newState.CheckSum) && state.CheckSum != newState.CheckSum) { _logger.Info("Checksum changed on ResourceId=\"{0}\" from \"{1}\" to \"{2}\"", state.ResourceId, state.CheckSum, newState.CheckSum); } state.CheckSum = newState.CheckSum; state.RetrievedAt = newState.RetrievedAt; pc.ResultType = ResultType.Normal; } else // no payload retrived, so no new state. Generally due to a same-checksum { pc.ResultType = ResultType.NoNewData; } state.Extensions = info.Extensions; state.Modified = info.Modified; state.ModifiedSources = info.ModifiedSources; state.RetryCount = 0; // success } else { throw new NotSupportedException($"({pc.Index}/{pc.Total}) ResourceId=\"{state.ResourceId}\" we cannot reach this point!"); } } else // for some reason, no action has been and payload has not been retrieved. We do not change the state { pc.ResultType = ResultType.NoAction; } _diagnosticSource.ProcessResourceSuccessful(processActivity, pc); if (processActivity.Duration > _config.ResourceDurationNotificationLimit) { _diagnosticSource.ProcessResourceTookTooLong(info.ResourceId, processActivity); } } catch (Exception ex) { state.LastException = ex; pc.ResultType = ResultType.Error; var isBanned = ++state.RetryCount == _config.MaxRetries; state.Extensions = info.Extensions; _diagnosticSource.ProcessResourceFailed(processActivity, pc, isBanned, ex); } await _stateProvider.SaveStateAsync(new[] { state }, ctk).ConfigureAwait(false); } catch (Exception ex) { // chomp it, we'll retry this file next time, forever, fuckit pc.ResultType = ResultType.Error; _diagnosticSource.ProcessResourceSaveFailed(info.ResourceId, ex); } }