private void HandleNestedValuesReduction(TransactionOperationContext indexContext, IndexingStatsScope stats, CancellationToken token, MapReduceResultsStore modifiedStore, IndexWriteOperation writer, LazyStringValue reduceKeyHash) { EnsureValidNestedValuesReductionStats(stats); var numberOfEntriesToReduce = 0; try { var section = modifiedStore.GetNestedResultsSection(); if (section.IsModified == false) { return; } using (_nestedValuesReductionStats.NestedValuesRead.Start()) { numberOfEntriesToReduce += section.GetResults(indexContext, _aggregationBatch.Items); } stats.RecordReduceAttempts(numberOfEntriesToReduce); AggregationResult result; using (_nestedValuesReductionStats.NestedValuesAggregation.Start()) { result = AggregateOn(_aggregationBatch.Items, indexContext, token); } if (section.IsNew == false) { writer.DeleteReduceResult(reduceKeyHash, stats); } foreach (var output in result.GetOutputs()) { writer.IndexDocument(reduceKeyHash, output, stats, indexContext); } _index.ReducesPerSec.Mark(numberOfEntriesToReduce); _metrics.MapReduceIndexes.ReducedPerSec.Mark(numberOfEntriesToReduce); stats.RecordReduceSuccesses(numberOfEntriesToReduce); } catch (Exception e) { _index.HandleError(e); var message = $"Failed to execute reduce function for reduce key '{reduceKeyHash}' on nested values of '{_indexDefinition.Name}' index."; if (_logger.IsInfoEnabled) { _logger.Info(message, e); } stats.RecordReduceErrors(numberOfEntriesToReduce); stats.AddReduceError(message + $" Exception: {e}"); } }
private void LogReductionError(Exception error, LazyStringValue reduceKeyHash, IndexingStatsScope stats, bool updateStats, TreePage page, int numberOfNestedValues = -1) { var builder = new StringBuilder("Failed to execute reduce function on "); if (page != null) { builder.Append($"page {page} "); } else { builder.Append("nested values "); } builder.Append($"of '{_indexDefinition.Name}' index (reduce key hash: {reduceKeyHash}"); var sampleItem = _aggregationBatch?.Items?.FirstOrDefault(); if (sampleItem != null) { builder.Append($", sample item to reduce: {sampleItem}"); } builder.Append(")"); var message = builder.ToString(); if (_logger.IsInfoEnabled) { _logger.Info(message, error); } if (updateStats) { var errorCount = page?.NumberOfEntries ?? numberOfNestedValues; Debug.Assert(errorCount != -1); stats.RecordReduceErrors(errorCount); stats.AddReduceError(message + $" Exception: {error}"); } }
private void HandleReductionError(Exception error, LazyStringValue reduceKeyHash, IndexWriteOperation writer, IndexingStatsScope stats, bool updateStats, TreePage page, int numberOfNestedValues = -1) { var builder = new StringBuilder("Failed to execute reduce function on "); if (page != null) { builder.Append($"page {page} "); } else { builder.Append("nested values "); } builder.Append($"of '{_indexDefinition.Name}' index. The relevant reduce result is going to be removed from the index "); builder.Append($"as it would be incorrect due to encountered errors (reduce key hash: {reduceKeyHash}"); var sampleItem = _aggregationBatch?.Items?.FirstOrDefault(); if (sampleItem != null) { builder.Append($", sample item to reduce: {sampleItem}"); } builder.Append(")"); var message = builder.ToString(); if (_logger.IsInfoEnabled) { _logger.Info(message, error); } try { writer.DeleteReduceResult(reduceKeyHash, stats); } catch (Exception e) { if (_logger.IsInfoEnabled) { _logger.Info($"Failed to delete an index result from '${_indexDefinition.Name}' index on reduce error (reduce key hash: ${reduceKeyHash})", e); } } if (updateStats) { var numberOfEntries = page?.NumberOfEntries ?? numberOfNestedValues; Debug.Assert(numberOfEntries != -1); // we'll only want to record exceptions on some of these, to give the // user information about what is going on, otherwise we'll have to wait a lot if we // are processing a big batch, and this can be a perf killer. See: RavenDB-11038 stats.RecordReduceErrors(numberOfEntries); if (stats.NumberOfKeptReduceErrors < IndexStorage.MaxNumberOfKeptErrors) { stats.AddReduceError(message + $" Exception: {error}"); } var failureInfo = new IndexFailureInformation { Name = _index.Name, MapErrors = stats.MapErrors, MapAttempts = stats.MapAttempts, ReduceErrors = stats.ReduceErrors, ReduceAttempts = stats.ReduceAttempts }; if (failureInfo.IsInvalidIndex(_isStaleBecauseOfRunningReduction)) { throw new ExcessiveNumberOfReduceErrorsException("Excessive number of errors during the reduce phase for the current batch. Failure info: " + failureInfo.GetErrorMessage()); } } }
private void HandleTreeReduction(TransactionOperationContext indexContext, IndexingStatsScope stats, CancellationToken token, MapReduceResultsStore modifiedStore, LowLevelTransaction lowLevelTransaction, IndexWriteOperation writer, LazyStringValue reduceKeyHash, Table table) { EnsureValidTreeReductionStats(stats); var tree = modifiedStore.Tree; var branchesToAggregate = new HashSet <long>(); var parentPagesToAggregate = new HashSet <long>(); var page = new TreePage(null, Constants.Storage.PageSize); foreach (var modifiedPage in modifiedStore.ModifiedPages) { token.ThrowIfCancellationRequested(); page.Base = lowLevelTransaction.GetPage(modifiedPage).Pointer; stats.RecordReduceTreePageModified(page.IsLeaf); if (page.IsLeaf == false) { Debug.Assert(page.IsBranch); branchesToAggregate.Add(modifiedPage); continue; } var leafPage = page; var compressed = leafPage.IsCompressed; if (compressed) { stats.RecordCompressedLeafPage(); } using (compressed ? (DecompressedLeafPage)(leafPage = tree.DecompressPage(leafPage, skipCache: true)) : null) { if (leafPage.NumberOfEntries == 0) { if (leafPage.PageNumber != tree.State.RootPageNumber) { throw new InvalidOperationException( $"Encountered empty page which isn't a root. Page #{leafPage.PageNumber} in '{tree.Name}' tree."); } writer.DeleteReduceResult(reduceKeyHash, stats); var emptyPageNumber = Bits.SwapBytes(leafPage.PageNumber); using (Slice.External(indexContext.Allocator, (byte *)&emptyPageNumber, sizeof(long), out Slice pageNumSlice)) table.DeleteByKey(pageNumSlice); continue; } var parentPage = tree.GetParentPageOf(leafPage); stats.RecordReduceAttempts(leafPage.NumberOfEntries); try { using (var result = AggregateLeafPage(leafPage, lowLevelTransaction, indexContext, token)) { if (parentPage == -1) { writer.DeleteReduceResult(reduceKeyHash, stats); foreach (var output in result.GetOutputs()) { writer.IndexDocument(reduceKeyHash, output, stats, indexContext); } } else { StoreAggregationResult(leafPage.PageNumber, leafPage.NumberOfEntries, table, result); parentPagesToAggregate.Add(parentPage); } _metrics.MapReduceReducedPerSecond.Mark(leafPage.NumberOfEntries); stats.RecordReduceSuccesses(leafPage.NumberOfEntries); } } catch (Exception e) { _index.HandleError(e); var message = $"Failed to execute reduce function for reduce key '{tree.Name}' on a leaf page #{leafPage} of '{_indexDefinition.Name}' index."; if (_logger.IsInfoEnabled) { _logger.Info(message, e); } if (parentPage == -1) { stats.RecordReduceErrors(leafPage.NumberOfEntries); stats.AddReduceError(message + $" Exception: {e}"); } } } } long tmp = 0; using (Slice.External(indexContext.Allocator, (byte *)&tmp, sizeof(long), out Slice pageNumberSlice)) { foreach (var freedPage in modifiedStore.FreedPages) { tmp = Bits.SwapBytes(freedPage); table.DeleteByKey(pageNumberSlice); } } while (parentPagesToAggregate.Count > 0 || branchesToAggregate.Count > 0) { token.ThrowIfCancellationRequested(); var branchPages = parentPagesToAggregate; parentPagesToAggregate = new HashSet <long>(); foreach (var pageNumber in branchPages) { page.Base = lowLevelTransaction.GetPage(pageNumber).Pointer; try { if (page.IsBranch == false) { throw new InvalidOperationException("Parent page was found that wasn't a branch, error at " + page.PageNumber); } stats.RecordReduceAttempts(page.NumberOfEntries); var parentPage = tree.GetParentPageOf(page); using (var result = AggregateBranchPage(page, table, indexContext, branchesToAggregate, token)) { if (parentPage == -1) { writer.DeleteReduceResult(reduceKeyHash, stats); foreach (var output in result.GetOutputs()) { writer.IndexDocument(reduceKeyHash, output, stats, indexContext); } } else { parentPagesToAggregate.Add(parentPage); StoreAggregationResult(page.PageNumber, page.NumberOfEntries, table, result); } _metrics.MapReduceReducedPerSecond.Mark(page.NumberOfEntries); stats.RecordReduceSuccesses(page.NumberOfEntries); } } catch (Exception e) { _index.HandleError(e); var message = $"Failed to execute reduce function for reduce key '{tree.Name}' on a branch page #{page} of '{_indexDefinition.Name}' index."; if (_logger.IsInfoEnabled) { _logger.Info(message, e); } stats.RecordReduceErrors(page.NumberOfEntries); stats.AddReduceError(message + $" Exception: {e}"); } finally { branchesToAggregate.Remove(pageNumber); } } if (parentPagesToAggregate.Count == 0 && branchesToAggregate.Count > 0) { // we still have unaggregated branches which were modified but their children were not modified (branch page splitting) so we missed them parentPagesToAggregate.Add(branchesToAggregate.First()); } } }
private void HandleReductionError(Exception error, LazyStringValue reduceKeyHash, Lazy <IndexWriteOperation> writer, IndexingStatsScope stats, bool updateStats, TreePage page, int numberOfNestedValues = -1) { var builder = new StringBuilder("Failed to execute reduce function on "); if (page != null) { builder.Append($"page {page} "); } else { builder.Append("nested values "); } builder.Append($"of '{_indexDefinition.Name}' index. The relevant reduce result is going to be removed from the index "); builder.Append($"as it would be incorrect due to encountered errors (reduce key hash: {reduceKeyHash}"); var erroringResult = OnErrorResult; if (erroringResult != null) { builder.Append($", current item to reduce: {erroringResult}"); } else { erroringResult = _aggregationBatch?.Items?.FirstOrDefault(); if (erroringResult != null) { builder.Append($", sample item to reduce: {erroringResult}"); } } builder.Append(")"); var message = builder.ToString(); if (_logger.IsInfoEnabled) { _logger.Info(message, error); } try { writer.Value.DeleteReduceResult(reduceKeyHash, stats); } catch (Exception e) { if (_logger.IsInfoEnabled) { _logger.Info($"Failed to delete an index result from '${_indexDefinition.Name}' index on reduce error (reduce key hash: ${reduceKeyHash})", e); } } if (updateStats) { var numberOfEntries = page?.NumberOfEntries ?? numberOfNestedValues; Debug.Assert(numberOfEntries != -1); // we'll only want to record exceptions on some of these, to give the // user information about what is going on, otherwise we'll have to wait a lot if we // are processing a big batch, and this can be a perf killer. See: RavenDB-11038 stats.RecordReduceErrors(numberOfEntries); if (stats.NumberOfKeptReduceErrors < IndexStorage.MaxNumberOfKeptErrors) { var reduceKey = GetReduceKey(); stats.AddReduceError(message + $" Exception: {error}", reduceKey); } var failureInfo = new IndexFailureInformation { Name = _index.Name, MapErrors = stats.MapErrors, MapAttempts = stats.MapAttempts, ReduceErrors = stats.ReduceErrors, ReduceAttempts = stats.ReduceAttempts }; if (failureInfo.IsInvalidIndex(true)) { throw new ExcessiveNumberOfReduceErrorsException("Excessive number of errors during the reduce phase for the current batch. Failure info: " + failureInfo.GetErrorMessage()); } string GetReduceKey() { if (erroringResult == null) { return(null); } try { var mapReduceDef = _index.Definition as MapReduceIndexDefinition; var autoMapReduceDef = _index.Definition as AutoMapReduceIndexDefinition; var groupByKeys = (mapReduceDef?.GroupByFields.Select(x => x.Key) ?? autoMapReduceDef?.GroupByFields.Select(x => x.Key))?.ToList(); StringBuilder reduceKeyValue = null; if (groupByKeys != null) { foreach (var key in groupByKeys) { if (erroringResult.TryGetMember(key, out var result)) { if (reduceKeyValue == null) { reduceKeyValue = new StringBuilder("Reduce key: { "); } reduceKeyValue.Append($"'{key}' : {result?.ToString() ?? "null"}"); } } reduceKeyValue?.Append(" }"); } return(reduceKeyValue?.ToString()); } catch { // ignore - make sure we don't error on error reporting return(null); } } } }