private void HandleIndexReplaceError(IndexReplaceInformation indexReplaceInformation)
{
indexReplaceInformation.ErrorCount++;
if (indexReplaceInformation.ReplaceTimer != null)
{
if (indexReplaceInformation.ErrorCount <= 10)
{
indexReplaceInformation.ReplaceTimer.Change(TimeSpan.FromMinutes(1), TimeSpan.FromDays(7)); // try again in one minute
}
else
{
Database.TimerManager.ReleaseTimer(indexReplaceInformation.ReplaceTimer);
indexReplaceInformation.ReplaceTimer = null;
}
}
var message = string.Format("Index replace failed. Could not replace index '{0}' with '{1}'. Number of tries: '{2}'.", indexReplaceInformation.IndexToReplace, indexReplaceInformation.ReplaceIndex, indexReplaceInformation.ErrorCount);
Database.AddAlert(new Alert
{
AlertLevel = indexReplaceInformation.ErrorCount <= 10 ? AlertLevel.Warning : AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Message = message,
Title = "Index replace failed",
UniqueKey = string.Format("Index '{0}' errored, dbid: {1}", indexReplaceInformation.ReplaceIndex, Database.TransactionalStorage.Id),
});
log.Error(message);
}
private void ReplaceSingleIndex(IndexReplaceInformation indexReplaceInformation)
{
var key = Constants.IndexReplacePrefix + indexReplaceInformation.ReplaceIndex;
var originalReplaceDocument = Database.Documents.Get(key, null);
var etag = originalReplaceDocument != null ? originalReplaceDocument.Etag : null;
bool wasReplaced = false;
try
{
wasReplaced = Database.IndexStorage.TryReplaceIndex(indexReplaceInformation.ReplaceIndex, indexReplaceInformation.IndexToReplace);
}
//this is thrown when a locked index (with LockError mode) is been replaced by a side by side index
//we need to stop the timer and delete the document so it will not continue throwing errors
catch (InvalidOperationException ie)
{
HandleSideBySideFailureBecauseLockError(indexReplaceInformation, key, etag);
throw;
}
if (wasReplaced)
{
DeleteIndexReplaceDocument(key, etag);
}
else
{
HandleIndexReplaceError(indexReplaceInformation);
}
}
private void ReplaceSingleIndex(IndexReplaceInformation indexReplaceInformation)
{
var key = Constants.IndexReplacePrefix + indexReplaceInformation.ReplaceIndex;
var originalReplaceDocument = Database.Documents.Get(key);
var etag = originalReplaceDocument != null ? originalReplaceDocument.Etag : null;
var wasReplaced = Database.IndexStorage.TryReplaceIndex(indexReplaceInformation.ReplaceIndex, indexReplaceInformation.IndexToReplace);
if (wasReplaced)
{
try
{
Database.Documents.Delete(key, etag, null);
}
catch (ConcurrencyException)
{
if (log.IsDebugEnabled) // side by side document changed, probably means that we created a new side by side index and updated the index
{
log.Debug("Failed to delete the side by side document after index replace.");
}
}
}
else
{
HandleIndexReplaceError(indexReplaceInformation);
}
}
private bool ShouldReplace(IndexReplaceInformation indexReplaceInformation, int indexId)
{
bool shouldReplace = false;
Database.TransactionalStorage.Batch(accessor =>
{
if (indexReplaceInformation.Forced ||
Database.IndexStorage.IsIndexStale(indexId, Database.LastCollectionEtags) == false)
{
shouldReplace = true; // always replace non-stale or forced indexes
}
else
{
var replaceIndex = Database.IndexStorage.GetIndexInstance(indexId);
var statistics = accessor.Indexing.GetIndexStats(indexId);
if (replaceIndex.IsMapReduce == false)
{
if (statistics.LastIndexedEtag != null && EtagUtil.IsGreaterThanOrEqual(statistics.LastIndexedEtag, indexReplaceInformation.MinimumEtagBeforeReplace))
{
shouldReplace = true;
}
}
if (shouldReplace == false && indexReplaceInformation.ReplaceTimeUtc.HasValue && (indexReplaceInformation.ReplaceTimeUtc.Value - SystemTime.UtcNow).TotalSeconds < 0)
{
shouldReplace = true;
}
}
});
return(shouldReplace);
}
private void ReplaceSingleIndex(IndexReplaceInformation indexReplaceInformation)
{
var wasReplaced = Database.IndexStorage.TryReplaceIndex(indexReplaceInformation.ReplaceIndex, indexReplaceInformation.IndexToReplace);
if (wasReplaced)
{
Database.Documents.Delete(Constants.IndexReplacePrefix + indexReplaceInformation.ReplaceIndex, null, null);
}
else
{
HandleIndexReplaceError(indexReplaceInformation);
}
}
private void HandleSideBySideFailureBecauseLockError(IndexReplaceInformation indexReplaceInformation, string key, Etag etag)
{
if (indexReplaceInformation.ReplaceTimer != null)
{
Database.TimerManager.ReleaseTimer(indexReplaceInformation.ReplaceTimer);
}
Database.IndexStorage.DeleteIndex(indexReplaceInformation.ReplaceIndex);
DeleteIndexReplaceDocument(key, etag);
var message = string.Format("Index {0} is locked with LockError mode but was attempted to be replaced by a side by side index {1}",
indexReplaceInformation.IndexToReplace, indexReplaceInformation.ReplaceIndex);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Message = message,
Title = "Index replace failed",
UniqueKey = string.Format("Index '{0}' errored, dbid: {1}", indexReplaceInformation.ReplaceIndex, Database.TransactionalStorage.Id),
});
log.Error(message);
}
private void HandleIndexReplaceError(IndexReplaceInformation indexReplaceInformation)
{
indexReplaceInformation.ErrorCount++;
if (indexReplaceInformation.ReplaceTimer != null)
{
if (indexReplaceInformation.ErrorCount <= 10)
indexReplaceInformation.ReplaceTimer.Change(TimeSpan.FromMinutes(1), TimeSpan.FromDays(7)); // try again in one minute
else
{
Database.TimerManager.ReleaseTimer(indexReplaceInformation.ReplaceTimer);
indexReplaceInformation.ReplaceTimer = null;
}
}
var message = string.Format("Index replace failed. Could not replace index '{0}' with '{1}'.", indexReplaceInformation.IndexToReplace, indexReplaceInformation.ReplaceIndex);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Message = message,
Title = "Index replace failed",
UniqueKey = string.Format("Index '{0}' errored, dbid: {1}", indexReplaceInformation.ReplaceIndex, Database.TransactionalStorage.Id),
});
log.Error(message);
}
private void ReplaceSingleIndex(IndexReplaceInformation indexReplaceInformation)
{
var key = Constants.IndexReplacePrefix + indexReplaceInformation.ReplaceIndex;
var originalReplaceDocument = Database.Documents.Get(key, null);
var etag = originalReplaceDocument != null ? originalReplaceDocument.Etag : null;
var wasReplaced = Database.IndexStorage.TryReplaceIndex(indexReplaceInformation.ReplaceIndex, indexReplaceInformation.IndexToReplace);
if (wasReplaced)
{
try
{
Database.Documents.Delete(key, etag, null);
}
catch (ConcurrencyException e)
{
// side by side document changed, probably means that we created a new side by side index and updated the index
log.Debug("Failed to delete the side by side document after index replace.");
}
}
else
HandleIndexReplaceError(indexReplaceInformation);
}
private bool ShouldReplace(IndexReplaceInformation indexReplaceInformation, int indexId)
{
bool shouldReplace = false;
Database.TransactionalStorage.Batch(accessor =>
{
if (indexReplaceInformation.Forced
|| Database.IndexStorage.IsIndexStale(indexId, Database.LastCollectionEtags) == false)
shouldReplace = true; // always replace non-stale or forced indexes
else
{
var replaceIndex = Database.IndexStorage.GetIndexInstance(indexId);
var statistics = accessor.Indexing.GetIndexStats(indexId);
if (replaceIndex.IsMapReduce)
{
if (statistics.LastReducedEtag != null && EtagUtil.IsGreaterThanOrEqual(statistics.LastReducedEtag, indexReplaceInformation.MinimumEtagBeforeReplace))
shouldReplace = true;
}
else
{
if (statistics.LastIndexedEtag != null && EtagUtil.IsGreaterThanOrEqual(statistics.LastIndexedEtag, indexReplaceInformation.MinimumEtagBeforeReplace))
shouldReplace = true;
}
if (shouldReplace == false && indexReplaceInformation.ReplaceTimeUtc.HasValue && (indexReplaceInformation.ReplaceTimeUtc.Value - SystemTime.UtcNow).TotalSeconds < 0)
shouldReplace = true;
}
});
return shouldReplace;
}
private void HandleSideBySideFailureBecauseLockError(IndexReplaceInformation indexReplaceInformation, string key, Etag etag)
{
if (indexReplaceInformation.ReplaceTimer != null)
Database.TimerManager.ReleaseTimer(indexReplaceInformation.ReplaceTimer);
Database.IndexStorage.DeleteIndex(indexReplaceInformation.ReplaceIndex);
DeleteIndexReplaceDocument(key, etag);
var message = string.Format("Index {0} is locked with LockError mode but was attempted to be replaced by a side by side index {1}",
indexReplaceInformation.IndexToReplace, indexReplaceInformation.ReplaceIndex);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Message = message,
Title = "Index replace failed",
UniqueKey = string.Format("Index '{0}' errored, dbid: {1}", indexReplaceInformation.ReplaceIndex, Database.TransactionalStorage.Id),
});
log.Error(message);
}
private void ReplaceSingleIndex(IndexReplaceInformation indexReplaceInformation)
{
var key = Constants.IndexReplacePrefix + indexReplaceInformation.ReplaceIndex;
var originalReplaceDocument = Database.Documents.Get(key, null);
var etag = originalReplaceDocument != null ? originalReplaceDocument.Etag : null;
bool wasReplaced = false;
try
{
wasReplaced = Database.IndexStorage.TryReplaceIndex(indexReplaceInformation.ReplaceIndex, indexReplaceInformation.IndexToReplace);
}
//this is thrown when a locked inedx (with LockError mode) is been replaced by a side by side index
//we need to stop the timer and delete the document so it will not continue throwing errors
catch (InvalidOperationException ie)
{
HandleSideBySideFailureBecauseLockError(indexReplaceInformation, key, etag);
throw;
}
if (wasReplaced)
{
DeleteIndexReplaceDocument(key, etag);
}
else
HandleIndexReplaceError(indexReplaceInformation);
}
private void ReplaceSingleIndex(IndexReplaceInformation indexReplaceInformation)
{
var wasReplaced = Database.IndexStorage.TryReplaceIndex(indexReplaceInformation.ReplaceIndex, indexReplaceInformation.IndexToReplace);
if (wasReplaced)
Database.Documents.Delete(Constants.IndexReplacePrefix + indexReplaceInformation.ReplaceIndex, null, null);
else
HandleIndexReplaceError(indexReplaceInformation);
}
