private void HandleIndexReplaceError(IndexReplaceInformation indexReplaceInformation) { indexReplaceInformation.ErrorCount++; if (indexReplaceInformation.ReplaceTimer != null) { if (indexReplaceInformation.ErrorCount <= 10) { indexReplaceInformation.ReplaceTimer.Change(TimeSpan.FromMinutes(1), TimeSpan.FromDays(7)); // try again in one minute } else { Database.TimerManager.ReleaseTimer(indexReplaceInformation.ReplaceTimer); indexReplaceInformation.ReplaceTimer = null; } } var message = string.Format("Index replace failed. Could not replace index '{0}' with '{1}'. Number of tries: '{2}'.", indexReplaceInformation.IndexToReplace, indexReplaceInformation.ReplaceIndex, indexReplaceInformation.ErrorCount); Database.AddAlert(new Alert { AlertLevel = indexReplaceInformation.ErrorCount <= 10 ? AlertLevel.Warning : AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Message = message, Title = "Index replace failed", UniqueKey = string.Format("Index '{0}' errored, dbid: {1}", indexReplaceInformation.ReplaceIndex, Database.TransactionalStorage.Id), }); log.Error(message); }
private void ReplaceSingleIndex(IndexReplaceInformation indexReplaceInformation) { var key = Constants.IndexReplacePrefix + indexReplaceInformation.ReplaceIndex; var originalReplaceDocument = Database.Documents.Get(key, null); var etag = originalReplaceDocument != null ? originalReplaceDocument.Etag : null; bool wasReplaced = false; try { wasReplaced = Database.IndexStorage.TryReplaceIndex(indexReplaceInformation.ReplaceIndex, indexReplaceInformation.IndexToReplace); } //this is thrown when a locked index (with LockError mode) is been replaced by a side by side index //we need to stop the timer and delete the document so it will not continue throwing errors catch (InvalidOperationException ie) { HandleSideBySideFailureBecauseLockError(indexReplaceInformation, key, etag); throw; } if (wasReplaced) { DeleteIndexReplaceDocument(key, etag); } else { HandleIndexReplaceError(indexReplaceInformation); } }
private void ReplaceSingleIndex(IndexReplaceInformation indexReplaceInformation) { var key = Constants.IndexReplacePrefix + indexReplaceInformation.ReplaceIndex; var originalReplaceDocument = Database.Documents.Get(key); var etag = originalReplaceDocument != null ? originalReplaceDocument.Etag : null; var wasReplaced = Database.IndexStorage.TryReplaceIndex(indexReplaceInformation.ReplaceIndex, indexReplaceInformation.IndexToReplace); if (wasReplaced) { try { Database.Documents.Delete(key, etag, null); } catch (ConcurrencyException) { if (log.IsDebugEnabled) // side by side document changed, probably means that we created a new side by side index and updated the index { log.Debug("Failed to delete the side by side document after index replace."); } } } else { HandleIndexReplaceError(indexReplaceInformation); } }
private bool ShouldReplace(IndexReplaceInformation indexReplaceInformation, int indexId) { bool shouldReplace = false; Database.TransactionalStorage.Batch(accessor => { if (indexReplaceInformation.Forced || Database.IndexStorage.IsIndexStale(indexId, Database.LastCollectionEtags) == false) { shouldReplace = true; // always replace non-stale or forced indexes } else { var replaceIndex = Database.IndexStorage.GetIndexInstance(indexId); var statistics = accessor.Indexing.GetIndexStats(indexId); if (replaceIndex.IsMapReduce == false) { if (statistics.LastIndexedEtag != null && EtagUtil.IsGreaterThanOrEqual(statistics.LastIndexedEtag, indexReplaceInformation.MinimumEtagBeforeReplace)) { shouldReplace = true; } } if (shouldReplace == false && indexReplaceInformation.ReplaceTimeUtc.HasValue && (indexReplaceInformation.ReplaceTimeUtc.Value - SystemTime.UtcNow).TotalSeconds < 0) { shouldReplace = true; } } }); return(shouldReplace); }
private void ReplaceSingleIndex(IndexReplaceInformation indexReplaceInformation) { var wasReplaced = Database.IndexStorage.TryReplaceIndex(indexReplaceInformation.ReplaceIndex, indexReplaceInformation.IndexToReplace); if (wasReplaced) { Database.Documents.Delete(Constants.IndexReplacePrefix + indexReplaceInformation.ReplaceIndex, null, null); } else { HandleIndexReplaceError(indexReplaceInformation); } }
private void HandleSideBySideFailureBecauseLockError(IndexReplaceInformation indexReplaceInformation, string key, Etag etag) { if (indexReplaceInformation.ReplaceTimer != null) { Database.TimerManager.ReleaseTimer(indexReplaceInformation.ReplaceTimer); } Database.IndexStorage.DeleteIndex(indexReplaceInformation.ReplaceIndex); DeleteIndexReplaceDocument(key, etag); var message = string.Format("Index {0} is locked with LockError mode but was attempted to be replaced by a side by side index {1}", indexReplaceInformation.IndexToReplace, indexReplaceInformation.ReplaceIndex); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Message = message, Title = "Index replace failed", UniqueKey = string.Format("Index '{0}' errored, dbid: {1}", indexReplaceInformation.ReplaceIndex, Database.TransactionalStorage.Id), }); log.Error(message); }
private void HandleIndexReplaceError(IndexReplaceInformation indexReplaceInformation) { indexReplaceInformation.ErrorCount++; if (indexReplaceInformation.ReplaceTimer != null) { if (indexReplaceInformation.ErrorCount <= 10) indexReplaceInformation.ReplaceTimer.Change(TimeSpan.FromMinutes(1), TimeSpan.FromDays(7)); // try again in one minute else { Database.TimerManager.ReleaseTimer(indexReplaceInformation.ReplaceTimer); indexReplaceInformation.ReplaceTimer = null; } } var message = string.Format("Index replace failed. Could not replace index '{0}' with '{1}'.", indexReplaceInformation.IndexToReplace, indexReplaceInformation.ReplaceIndex); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Message = message, Title = "Index replace failed", UniqueKey = string.Format("Index '{0}' errored, dbid: {1}", indexReplaceInformation.ReplaceIndex, Database.TransactionalStorage.Id), }); log.Error(message); }
private void ReplaceSingleIndex(IndexReplaceInformation indexReplaceInformation) { var key = Constants.IndexReplacePrefix + indexReplaceInformation.ReplaceIndex; var originalReplaceDocument = Database.Documents.Get(key, null); var etag = originalReplaceDocument != null ? originalReplaceDocument.Etag : null; var wasReplaced = Database.IndexStorage.TryReplaceIndex(indexReplaceInformation.ReplaceIndex, indexReplaceInformation.IndexToReplace); if (wasReplaced) { try { Database.Documents.Delete(key, etag, null); } catch (ConcurrencyException e) { // side by side document changed, probably means that we created a new side by side index and updated the index log.Debug("Failed to delete the side by side document after index replace."); } } else HandleIndexReplaceError(indexReplaceInformation); }
private bool ShouldReplace(IndexReplaceInformation indexReplaceInformation, int indexId) { bool shouldReplace = false; Database.TransactionalStorage.Batch(accessor => { if (indexReplaceInformation.Forced || Database.IndexStorage.IsIndexStale(indexId, Database.LastCollectionEtags) == false) shouldReplace = true; // always replace non-stale or forced indexes else { var replaceIndex = Database.IndexStorage.GetIndexInstance(indexId); var statistics = accessor.Indexing.GetIndexStats(indexId); if (replaceIndex.IsMapReduce) { if (statistics.LastReducedEtag != null && EtagUtil.IsGreaterThanOrEqual(statistics.LastReducedEtag, indexReplaceInformation.MinimumEtagBeforeReplace)) shouldReplace = true; } else { if (statistics.LastIndexedEtag != null && EtagUtil.IsGreaterThanOrEqual(statistics.LastIndexedEtag, indexReplaceInformation.MinimumEtagBeforeReplace)) shouldReplace = true; } if (shouldReplace == false && indexReplaceInformation.ReplaceTimeUtc.HasValue && (indexReplaceInformation.ReplaceTimeUtc.Value - SystemTime.UtcNow).TotalSeconds < 0) shouldReplace = true; } }); return shouldReplace; }
private void HandleSideBySideFailureBecauseLockError(IndexReplaceInformation indexReplaceInformation, string key, Etag etag) { if (indexReplaceInformation.ReplaceTimer != null) Database.TimerManager.ReleaseTimer(indexReplaceInformation.ReplaceTimer); Database.IndexStorage.DeleteIndex(indexReplaceInformation.ReplaceIndex); DeleteIndexReplaceDocument(key, etag); var message = string.Format("Index {0} is locked with LockError mode but was attempted to be replaced by a side by side index {1}", indexReplaceInformation.IndexToReplace, indexReplaceInformation.ReplaceIndex); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Message = message, Title = "Index replace failed", UniqueKey = string.Format("Index '{0}' errored, dbid: {1}", indexReplaceInformation.ReplaceIndex, Database.TransactionalStorage.Id), }); log.Error(message); }
private void ReplaceSingleIndex(IndexReplaceInformation indexReplaceInformation) { var key = Constants.IndexReplacePrefix + indexReplaceInformation.ReplaceIndex; var originalReplaceDocument = Database.Documents.Get(key, null); var etag = originalReplaceDocument != null ? originalReplaceDocument.Etag : null; bool wasReplaced = false; try { wasReplaced = Database.IndexStorage.TryReplaceIndex(indexReplaceInformation.ReplaceIndex, indexReplaceInformation.IndexToReplace); } //this is thrown when a locked inedx (with LockError mode) is been replaced by a side by side index //we need to stop the timer and delete the document so it will not continue throwing errors catch (InvalidOperationException ie) { HandleSideBySideFailureBecauseLockError(indexReplaceInformation, key, etag); throw; } if (wasReplaced) { DeleteIndexReplaceDocument(key, etag); } else HandleIndexReplaceError(indexReplaceInformation); }
private void ReplaceSingleIndex(IndexReplaceInformation indexReplaceInformation) { var wasReplaced = Database.IndexStorage.TryReplaceIndex(indexReplaceInformation.ReplaceIndex, indexReplaceInformation.IndexToReplace); if (wasReplaced) Database.Documents.Delete(Constants.IndexReplacePrefix + indexReplaceInformation.ReplaceIndex, null, null); else HandleIndexReplaceError(indexReplaceInformation); }