/// <summary>Parses an error message into a <see cref="CosmosError"/>.</summary> /// <param name="message">The error message.</param> /// <returns>An instance of <see cref="CosmosError"/>.</returns> public static CosmosError ParseCosmosErrorMessage(string message) { var result = new CosmosError(); foreach (var el in message.Split(',')) { var attr = el.Trim().Split('='); if (attr.Count() != 2) { continue; } if (attr[0] == "Error") { result.ErrorCode = int.Parse(attr[1]); } if (attr[0] == "RetryAfterMs") { result.RetryAfterMs = int.Parse(attr[1]); } } return(result); }
/// <summary>Bulk upsert records.</summary> /// <param name="observations">The set of observations to be loaded.</param> /// <param name="maxRetries">Max number of retries that can be used if rate limited.</param> /// <param name="carriedOverErrors">Accumulates errors over retries.</param> /// <param name="performCheck"> /// If True, check if the record already exists and delete it. If False, the method attempts /// a basic Insert operation. /// </param> /// <returns>True if success, False otherwise.</returns> public bool UpsertMany( IEnumerable <WeatherStationObservation> observations, int maxRetries = 10, IList <CosmosError> carriedOverErrors = null, bool performCheck = true) { var collection = database.GetCollection <WeatherStationObservation>(config.CollectionName); WeatherStationObservation[] observationsArr = observations.ToArray(); if (observationsArr.Count() == 0) { logger.LogInformation("No records to insert - nothing to do."); return(true); } if (performCheck) { var checkId = observationsArr.First().Id; var filter = Builders <WeatherStationObservation> .Filter.Eq("_id", checkId); try { if (collection.CountDocuments(filter) > 0) { logger.LogInformation("Datastore not updated as data exists"); return(false); } } catch (MongoCommandException e) { logger.LogInformation($"Couldn't perform check: {e.Message}"); if (maxRetries == 0) { logger.LogError($"Datastore update exception (no retries available): {e.Message}"); return(false); } var cosmosError = CosmosError.ParseCosmosErrorMessage(e.Message); Task.Delay(cosmosError.RetryAfterMs); UpsertMany(observations, maxRetries: maxRetries - 1, carriedOverErrors: carriedOverErrors, performCheck: true); } } try { logger.LogInformation($"Records to be inserted: {observationsArr.Count()}"); collection.InsertMany(observationsArr); logger.LogInformation("Datastore updated"); return(true); } catch (MongoCommandException e) { logger.LogError($"Unhandled exception: {e.Source} - {e.Message}"); return(false); } catch (MongoBulkWriteException <WeatherStationObservation> e) { if (maxRetries == 0) { logger.LogError($"Datastore update exception (no retries available): {e.Source} - {e.Message}"); return(false); } var errorList = CosmosError.ParseBulkWriteException(e); var errorRecords = errorList.Select(we => observationsArr[we.Index]); var unprocessedRecords = e.UnprocessedRequests .Where(ur => ur.ModelType == WriteModelType.InsertOne) .OfType <InsertOneModel <WeatherStationObservation> >() .Select(ur => ur.Document); var remainingObservations = unprocessedRecords.Union(errorRecords); if (remainingObservations.Count() > 0) { int retryInterval = errorList.Max(rec => rec.RetryAfterMs) + rand.Next(1000, 5000); logger.LogInformation($"Retrying {remainingObservations.Count()} records in {retryInterval}ms"); Task.Delay(retryInterval); return(UpsertMany( remainingObservations, maxRetries: maxRetries - 1, carriedOverErrors: carriedOverErrors, performCheck: false)); } return(false); } }