public IndexFailureInformation GetFailureRate(int id)
        {
            var key = new Slice(CreateKey(id));

            ushort version;
            var    indexStats  = LoadStruct(tableStorage.IndexingStats, key, out version);
            var    reduceStats = LoadStruct(tableStorage.ReduceStats, key, out version);

            var reduceAttempts  = reduceStats.ReadInt(ReducingWorkStatsFields.ReduceAttempts);
            var reduceErrors    = reduceStats.ReadInt(ReducingWorkStatsFields.ReduceErrors);
            var reduceSuccesses = reduceStats.ReadInt(ReducingWorkStatsFields.ReduceSuccesses);

            var indexFailureInformation = new IndexFailureInformation
            {
                Attempts        = indexStats.ReadInt(IndexingWorkStatsFields.IndexingAttempts),
                Errors          = indexStats.ReadInt(IndexingWorkStatsFields.IndexingErrors),
                Successes       = indexStats.ReadInt(IndexingWorkStatsFields.IndexingSuccesses),
                ReduceAttempts  = reduceAttempts == -1 ? (int?)null : reduceAttempts,
                ReduceErrors    = reduceErrors == -1 ? (int?)null : reduceErrors,
                ReduceSuccesses = reduceSuccesses == -1 ? (int?)null : reduceSuccesses,
                Id = indexStats.ReadInt(IndexingWorkStatsFields.IndexId)
            };

            return(indexFailureInformation);
        }
Example #2
0
        public bool IsIndexInvalid(RavenTransaction tx)
        {
            var statsTree = tx.InnerTransaction.ReadTree(IndexSchema.StatsTree);

            var mapAttempts = statsTree.Read(IndexSchema.MapAttemptsSlice)?.Reader.ReadLittleEndianInt32() ?? 0;
            var mapErrors   = statsTree.Read(IndexSchema.MapErrorsSlice)?.Reader.ReadLittleEndianInt32() ?? 0;

            int?reduceAttempts = null, reduceErrors = null;

            if (_index.Type.IsMapReduce())
            {
                reduceAttempts = statsTree.Read(IndexSchema.ReduceAttemptsSlice)?.Reader.ReadLittleEndianInt32() ?? 0;
                reduceErrors   = statsTree.Read(IndexSchema.ReduceErrorsSlice)?.Reader.ReadLittleEndianInt32() ?? 0;
            }

            int mapReferenceAttempts = 0, mapReferenceErrors = 0;

            if (_index.GetReferencedCollections()?.Count > 0)
            {
                mapReferenceAttempts = statsTree.Read(IndexSchema.MapReferencedAttemptsSlice)?.Reader.ReadLittleEndianInt32() ?? 0;
                mapReferenceErrors   = statsTree.Read(IndexSchema.MapReferenceErrorsSlice)?.Reader.ReadLittleEndianInt32() ?? 0;
            }

            return(IndexFailureInformation.CheckIndexInvalid(mapAttempts, mapErrors,
                                                             mapReferenceAttempts, mapReferenceErrors, reduceAttempts, reduceErrors, false));
        }
Example #3
0
        public IndexFailureInformation GetFailureRate(string index)
        {
            var readResult = storage.IndexingStats.Read(index);

            if (readResult == null)
            {
                throw new IndexDoesNotExistsException("There is no index named: " + index);
            }
            var indexFailureInformation = new IndexFailureInformation
            {
                Attempts        = readResult.Key.Value <int>("attempts"),
                Errors          = readResult.Key.Value <int>("failures"),
                Successes       = readResult.Key.Value <int>("successes"),
                ReduceAttempts  = readResult.Key.Value <int?>("reduce_attempts"),
                ReduceErrors    = readResult.Key.Value <int?>("reduce_failures"),
                ReduceSuccesses = readResult.Key.Value <int?>("reduce_successes"),
                Name            = readResult.Key.Value <string>("index"),
            };

            return(indexFailureInformation);
        }
        public IndexFailureInformation GetFailureRate(int id)
        {
            var key = CreateKey(id);

            ushort version;
            var    indexStats  = Load(tableStorage.IndexingStats, key, out version);
            var    reduceStats = Load(tableStorage.ReduceStats, key, out version);

            var indexFailureInformation = new IndexFailureInformation
            {
                Attempts        = indexStats.Value <int>("attempts"),
                Errors          = indexStats.Value <int>("failures"),
                Successes       = indexStats.Value <int>("successes"),
                ReduceAttempts  = reduceStats.Value <int?>("reduce_attempts"),
                ReduceErrors    = reduceStats.Value <int?>("reduce_failures"),
                ReduceSuccesses = reduceStats.Value <int?>("reduce_successes"),
                Id = indexStats.Value <int>("index")
            };

            return(indexFailureInformation);
        }
        private void HandleReductionError(Exception error, LazyStringValue reduceKeyHash, IndexWriteOperation writer, IndexingStatsScope stats, bool updateStats, TreePage page,
                                          int numberOfNestedValues = -1)
        {
            var builder = new StringBuilder("Failed to execute reduce function on ");

            if (page != null)
            {
                builder.Append($"page {page} ");
            }
            else
            {
                builder.Append("nested values ");
            }

            builder.Append($"of '{_indexDefinition.Name}' index. The relevant reduce result is going to be removed from the index ");
            builder.Append($"as it would be incorrect due to encountered errors (reduce key hash: {reduceKeyHash}");

            var sampleItem = _aggregationBatch?.Items?.FirstOrDefault();

            if (sampleItem != null)
            {
                builder.Append($", sample item to reduce: {sampleItem}");
            }

            builder.Append(")");

            var message = builder.ToString();

            if (_logger.IsInfoEnabled)
            {
                _logger.Info(message, error);
            }

            try
            {
                writer.DeleteReduceResult(reduceKeyHash, stats);
            }
            catch (Exception e)
            {
                if (_logger.IsInfoEnabled)
                {
                    _logger.Info($"Failed to delete an index result from '${_indexDefinition.Name}' index on reduce error (reduce key hash: ${reduceKeyHash})", e);
                }
            }

            if (updateStats)
            {
                var numberOfEntries = page?.NumberOfEntries ?? numberOfNestedValues;

                Debug.Assert(numberOfEntries != -1);

                // we'll only want to record exceptions on some of these, to give the
                // user information about what is going on, otherwise we'll have to wait a lot if we
                // are processing a big batch, and this can be a perf killer. See: RavenDB-11038

                stats.RecordReduceErrors(numberOfEntries);

                if (stats.NumberOfKeptReduceErrors < IndexStorage.MaxNumberOfKeptErrors)
                {
                    stats.AddReduceError(message + $" Exception: {error}");
                }

                var failureInfo = new IndexFailureInformation
                {
                    Name           = _index.Name,
                    MapErrors      = stats.MapErrors,
                    MapAttempts    = stats.MapAttempts,
                    ReduceErrors   = stats.ReduceErrors,
                    ReduceAttempts = stats.ReduceAttempts
                };

                if (failureInfo.IsInvalidIndex(_isStaleBecauseOfRunningReduction))
                {
                    throw new ExcessiveNumberOfReduceErrorsException("Excessive number of errors during the reduce phase for the current batch. Failure info: " +
                                                                     failureInfo.GetErrorMessage());
                }
            }
        }
 /// <summary>
 /// Initializes a new instance of the <see cref="IndexDisabledException"/> class.
 /// </summary>
 /// <param name="information">The information.</param>
 public IndexDisabledException(IndexFailureInformation information)
 {
     Information = information;
 }
Example #7
0
        public unsafe IndexFailureInformation UpdateStats(DateTime indexingTime, IndexingRunStats stats)
        {
            if (_logger.IsInfoEnabled)
            {
                _logger.Info($"Updating statistics for '{_index.Name}'. Stats: {stats}.");
            }

            using (_contextPool.AllocateOperationContext(out TransactionOperationContext context))
                using (var tx = context.OpenWriteTransaction())
                {
                    var result = new IndexFailureInformation
                    {
                        Name = _index.Name
                    };

                    var table = tx.InnerTransaction.OpenTable(_errorsSchema, "Errors");

                    var statsTree = tx.InnerTransaction.ReadTree(IndexSchema.StatsTree);

                    result.MapAttempts  = statsTree.Increment(IndexSchema.MapAttemptsSlice, stats.MapAttempts);
                    result.MapSuccesses = statsTree.Increment(IndexSchema.MapSuccessesSlice, stats.MapSuccesses);
                    result.MapErrors    = statsTree.Increment(IndexSchema.MapErrorsSlice, stats.MapErrors);

                    var currentMaxNumberOfOutputs = statsTree.Read(IndexSchema.MaxNumberOfOutputsPerDocument)?.Reader.ReadLittleEndianInt32();

                    using (statsTree.DirectAdd(IndexSchema.MaxNumberOfOutputsPerDocument, sizeof(int), out byte *ptr))
                    {
                        *(int *)ptr = currentMaxNumberOfOutputs > stats.MaxNumberOfOutputsPerDocument
                        ? currentMaxNumberOfOutputs.Value
                        : stats.MaxNumberOfOutputsPerDocument;
                    }

                    if (_index.Type.IsMapReduce())
                    {
                        result.ReduceAttempts  = statsTree.Increment(IndexSchema.ReduceAttemptsSlice, stats.ReduceAttempts);
                        result.ReduceSuccesses = statsTree.Increment(IndexSchema.ReduceSuccessesSlice, stats.ReduceSuccesses);
                        result.ReduceErrors    = statsTree.Increment(IndexSchema.ReduceErrorsSlice, stats.ReduceErrors);
                    }

                    var binaryDate = indexingTime.ToBinary();
                    using (Slice.External(context.Allocator, (byte *)&binaryDate, sizeof(long), out Slice binaryDateslice))
                        statsTree.Add(IndexSchema.LastIndexingTimeSlice, binaryDateslice);

                    if (stats.Errors != null)
                    {
                        for (var i = Math.Max(stats.Errors.Count - MaxNumberOfKeptErrors, 0); i < stats.Errors.Count; i++)
                        {
                            var error          = stats.Errors[i];
                            var ticksBigEndian = Bits.SwapBytes(error.Timestamp.Ticks);
                            using (var document = context.GetLazyString(error.Document))
                                using (var action = context.GetLazyString(error.Action))
                                    using (var e = context.GetLazyString(error.Error))
                                    {
                                        var tvb = new TableValueBuilder
                                        {
                                            { (byte *)&ticksBigEndian, sizeof(long) },
                                            { document.Buffer, document.Size },
                                            { action.Buffer, action.Size },
                                            { e.Buffer, e.Size }
                                        };
                                        table.Insert(tvb);
                                    }
                        }

                        CleanupErrors(table);
                    }

                    tx.Commit();

                    return(result);
                }
        }
Example #8
0
		public IndexFailureInformation GetFailureRate(string index)
		{
			var readResult = storage.IndexingStats.Read(index);
			if (readResult == null)
				throw new IndexDoesNotExistsException("There is no index named: " + index);
			var indexFailureInformation = new IndexFailureInformation
			{
				Attempts = readResult.Key.Value<int>("attempts"),
				Errors = readResult.Key.Value<int>("failures"),
				Successes = readResult.Key.Value<int>("successes"),
				ReduceAttempts = readResult.Key.Value<int?>("reduce_attempts"),
				ReduceErrors = readResult.Key.Value<int?>("reduce_failures"),
				ReduceSuccesses = readResult.Key.Value<int?>("reduce_successes"),
				Name = readResult.Key.Value<string>("index"),
			};
			return indexFailureInformation;
		}
Example #9
0
        private void HandleReductionError(Exception error, LazyStringValue reduceKeyHash, Lazy <IndexWriteOperation> writer, IndexingStatsScope stats, bool updateStats, TreePage page,
                                          int numberOfNestedValues = -1)
        {
            var builder = new StringBuilder("Failed to execute reduce function on ");

            if (page != null)
            {
                builder.Append($"page {page} ");
            }
            else
            {
                builder.Append("nested values ");
            }

            builder.Append($"of '{_indexDefinition.Name}' index. The relevant reduce result is going to be removed from the index ");
            builder.Append($"as it would be incorrect due to encountered errors (reduce key hash: {reduceKeyHash}");

            var erroringResult = OnErrorResult;

            if (erroringResult != null)
            {
                builder.Append($", current item to reduce: {erroringResult}");
            }
            else
            {
                erroringResult = _aggregationBatch?.Items?.FirstOrDefault();

                if (erroringResult != null)
                {
                    builder.Append($", sample item to reduce: {erroringResult}");
                }
            }


            builder.Append(")");

            var message = builder.ToString();

            if (_logger.IsInfoEnabled)
            {
                _logger.Info(message, error);
            }

            try
            {
                writer.Value.DeleteReduceResult(reduceKeyHash, stats);
            }
            catch (Exception e)
            {
                if (_logger.IsInfoEnabled)
                {
                    _logger.Info($"Failed to delete an index result from '${_indexDefinition.Name}' index on reduce error (reduce key hash: ${reduceKeyHash})", e);
                }
            }

            if (updateStats)
            {
                var numberOfEntries = page?.NumberOfEntries ?? numberOfNestedValues;

                Debug.Assert(numberOfEntries != -1);

                // we'll only want to record exceptions on some of these, to give the
                // user information about what is going on, otherwise we'll have to wait a lot if we
                // are processing a big batch, and this can be a perf killer. See: RavenDB-11038

                stats.RecordReduceErrors(numberOfEntries);


                if (stats.NumberOfKeptReduceErrors < IndexStorage.MaxNumberOfKeptErrors)
                {
                    var reduceKey = GetReduceKey();

                    stats.AddReduceError(message + $" Exception: {error}", reduceKey);
                }

                var failureInfo = new IndexFailureInformation
                {
                    Name           = _index.Name,
                    MapErrors      = stats.MapErrors,
                    MapAttempts    = stats.MapAttempts,
                    ReduceErrors   = stats.ReduceErrors,
                    ReduceAttempts = stats.ReduceAttempts
                };

                if (failureInfo.IsInvalidIndex(true))
                {
                    throw new ExcessiveNumberOfReduceErrorsException("Excessive number of errors during the reduce phase for the current batch. Failure info: " +
                                                                     failureInfo.GetErrorMessage());
                }

                string GetReduceKey()
                {
                    if (erroringResult == null)
                    {
                        return(null);
                    }

                    try
                    {
                        var mapReduceDef     = _index.Definition as MapReduceIndexDefinition;
                        var autoMapReduceDef = _index.Definition as AutoMapReduceIndexDefinition;

                        var groupByKeys = (mapReduceDef?.GroupByFields.Select(x => x.Key) ??
                                           autoMapReduceDef?.GroupByFields.Select(x => x.Key))?.ToList();

                        StringBuilder reduceKeyValue = null;

                        if (groupByKeys != null)
                        {
                            foreach (var key in groupByKeys)
                            {
                                if (erroringResult.TryGetMember(key, out var result))
                                {
                                    if (reduceKeyValue == null)
                                    {
                                        reduceKeyValue = new StringBuilder("Reduce key: { ");
                                    }

                                    reduceKeyValue.Append($"'{key}' : {result?.ToString() ?? "null"}");
                                }
                            }

                            reduceKeyValue?.Append(" }");
                        }

                        return(reduceKeyValue?.ToString());
                    }
                    catch
                    {
                        // ignore - make sure we don't error on error reporting

                        return(null);
                    }
                }
            }
        }