Beispiel #1
0
        public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize,
                                                   CancellationToken token, ref int nextStart, Action <JsonDocument> addDoc,
                                                   string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null,
                                                   string skipAfter   = null)
        {
            if (idPrefix == null)
            {
                throw new ArgumentNullException("idPrefix");
            }
            idPrefix = idPrefix.Trim();

            var canPerformRapidPagination = nextStart > 0 && start == nextStart;
            var actualStart       = canPerformRapidPagination ? start : 0;
            var addedDocs         = 0;
            var docCountOnLastAdd = 0;
            var matchedDocs       = 0;

            TransactionalStorage.Batch(
                actions =>
            {
                var docsToSkip = canPerformRapidPagination ? 0 : start;
                int docCount;

                AbstractTransformer storedTransformer = null;
                var hasTransformer = transformer != null;
                if (hasTransformer)
                {
                    storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
                    if (storedTransformer == null)
                    {
                        throw new InvalidOperationException("No transformer with the name: " + transformer);
                    }
                }

                do
                {
                    Database.WorkContext.UpdateFoundWork();

                    docCount = 0;
                    var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter);
                    var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, transformerParameters, hasTransformer: hasTransformer);

                    foreach (var doc in docs)
                    {
                        token.ThrowIfCancellationRequested();
                        docCount++;
                        if (docCount - docCountOnLastAdd > 1000)
                        {
                            addDoc(null);     // heartbeat
                        }

                        var keyTest = doc.Key.Substring(idPrefix.Length);

                        if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
                        {
                            continue;
                        }

                        JsonDocument.EnsureIdInMetadata(doc);
                        var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);

                        var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc;
                        document     = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
                        if (document == null)
                        {
                            continue;
                        }

                        matchedDocs++;

                        if (matchedDocs <= docsToSkip)
                        {
                            continue;
                        }

                        token.ThrowIfCancellationRequested();

                        document = TransformDocumentIfNeeded(document, storedTransformer, documentRetriever);
                        addDoc(document);

                        addedDocs++;
                        docCountOnLastAdd = docCount;

                        if (addedDocs >= pageSize)
                        {
                            break;
                        }
                    }

                    actualStart += pageSize;
                } while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue);
            });

            if (addedDocs != pageSize)
            {
                nextStart = start; // will mark as last page
            }
            else if (canPerformRapidPagination)
            {
                nextStart = start + matchedDocs;
            }
            else
            {
                nextStart = actualStart;
            }
        }
 private bool IsGcGen3SizeName(IReadOnlyList <WildcardMatcher> disabledMetrics) => !WildcardMatcher.IsAnyMatch(disabledMetrics, GcGen3SizeName);
Beispiel #3
0
        internal static ITransaction StartTransactionAsync(HttpContext context, IApmLogger logger, ITracer tracer, IConfigSnapshot configSnapshot)
        {
            try
            {
                if (WildcardMatcher.IsAnyMatch(configSnapshot?.TransactionIgnoreUrls, context.Request.Path))
                {
                    logger.Debug()?.Log("Request ignored based on TransactionIgnoreUrls, url: {urlPath}", context.Request.Path);
                    return(null);
                }

                ITransaction transaction;
                var          transactionName = $"{context.Request.Method} {context.Request.Path}";

                var containsTraceParentHeader =
                    context.Request.Headers.TryGetValue(TraceContext.TraceParentHeaderName, out var traceParentHeader);

                var containsPrefixedTraceParentHeader = false;
                if (!containsTraceParentHeader)
                {
                    containsPrefixedTraceParentHeader = context.Request.Headers.TryGetValue(TraceContext.TraceParentHeaderNamePrefixed, out traceParentHeader);
                }

                if (containsPrefixedTraceParentHeader || containsTraceParentHeader)
                {
                    var tracingData = context.Request.Headers.TryGetValue(TraceContext.TraceStateHeaderName, out var traceStateHeader)
                                                ? TraceContext.TryExtractTracingData(traceParentHeader, traceStateHeader)
                                                : TraceContext.TryExtractTracingData(traceParentHeader);

                    if (tracingData != null)
                    {
                        logger.Debug()
                        ?.Log(
                            "Incoming request with {TraceParentHeaderName} header. DistributedTracingData: {DistributedTracingData}. Continuing trace.",
                            containsPrefixedTraceParentHeader ? TraceContext.TraceParentHeaderNamePrefixed : TraceContext.TraceParentHeaderName,
                            tracingData);

                        transaction = tracer.StartTransaction(transactionName, ApiConstants.TypeRequest, tracingData);
                    }
                    else
                    {
                        logger.Debug()
                        ?.Log(
                            "Incoming request with invalid {TraceParentHeaderName} header (received value: {TraceParentHeaderValue}). Starting trace with new trace id.",
                            containsPrefixedTraceParentHeader ? TraceContext.TraceParentHeaderNamePrefixed : TraceContext.TraceParentHeaderName,
                            traceParentHeader);

                        transaction = tracer.StartTransaction(transactionName, ApiConstants.TypeRequest);
                    }
                }
                else
                {
                    logger.Debug()?.Log("Incoming request. Starting Trace.");
                    transaction = tracer.StartTransaction(transactionName, ApiConstants.TypeRequest);
                }

                return(transaction);
            }
            catch (Exception ex)
            {
                logger?.Error()?.LogException(ex, "Exception thrown while trying to start transaction");
                return(null);
            }
        }
Beispiel #4
0
        internal static ITransaction CreateConsumerTransaction(
            IApmAgent agent,
            string topic,
            Partition?partition,
            Offset?offset,
            IMessage message)
        {
            ITransaction transaction = null;

            try
            {
                if (agent.Tracer.CurrentTransaction is not null)
                {
                    return(null);
                }

                if (agent is ApmAgent apmAgent)
                {
                    var matcher = WildcardMatcher.AnyMatch(apmAgent.ConfigurationStore.CurrentSnapshot.IgnoreMessageQueues, topic);
                    if (matcher != null)
                    {
                        agent.Logger.Trace()
                        ?.Log(
                            "Not tracing message from {Queue} because it matched IgnoreMessageQueues pattern {Matcher}",
                            topic,
                            matcher.GetMatcher());

                        return(null);
                    }
                }

                DistributedTracingData distributedTracingData = null;
                if (message?.Headers != null)
                {
                    var headers = new KafkaHeadersCollection(message.Headers, agent.Logger);
                    try
                    {
                        var traceParent = string.Join(",", headers.GetValues(TraceContext.TraceParentBinaryHeaderName));
                        var traceState  = headers.GetValues(TraceContext.TraceStateHeaderName).FirstOrDefault();
                        distributedTracingData = TraceContext.TryExtractTracingData(traceParent, traceState);
                    }
                    catch (Exception ex)
                    {
                        agent.Logger.Error()?.LogException(ex, "Error extracting propagated headers from Kafka message");
                    }
                }

                var name = string.IsNullOrEmpty(topic)
                                        ? "Kafka RECEIVE"
                                        : $"Kafka RECEIVE from {topic}";

                transaction = agent.Tracer.StartTransaction(name, ApiConstants.TypeMessaging, distributedTracingData);

                if (partition is not null)
                {
                    transaction.SetLabel("partition", partition.ToString());
                }

                if (offset is not null)
                {
                    transaction.SetLabel("offset", offset.ToString());
                }

                // record only queue topic name and age on context for now. capture body and headers potentially in future
                transaction.Context.Message = new Message {
                    Queue = new Queue {
                        Name = topic
                    }
                };
                if (transaction is Transaction realTransaction && message is not null && message.Timestamp.Type != 0)
                {
                    var consumeTime = TimeUtils.ToDateTime(realTransaction.Timestamp);
                    var produceTime = message.Timestamp.UtcDateTime;

                    var age = Math.Max(0, (consumeTime - produceTime).TotalMilliseconds);
                    if (age > 0 && age < MaxAge)
                    {
                        transaction.Context.Message.Age = new Age {
                            Ms = (long)age
                        }
                    }
                    ;
                }

                if (message is not null && message.Value is null)
                {
                    transaction.SetLabel("tombstone", "true");
                }
            }
            catch (Exception ex)
            {
                agent.Logger.Error()?.LogException(ex, "Error creating or populating transaction.");
            }

            return(transaction);
        }
 private bool IsGcCountNameEnabled(IReadOnlyList <WildcardMatcher> disabledMetrics) => !WildcardMatcher.IsAnyMatch(disabledMetrics, GcCountName);
 public bool IsEnabled(IReadOnlyList <WildcardMatcher> disabledMetrics) => !WildcardMatcher.IsAnyMatch(disabledMetrics, ProcessCpuTotalPct);
 public bool IsEnabled(IReadOnlyList <WildcardMatcher> disabledMetrics) =>
 WildcardMatcher.IsAnyMatch(disabledMetrics, BreakdownMetricsProvider.SpanSelfTime);
        public async Task <IEnumerable <TagRequest> > QueryTaggedRequestsAsync(int limit, string[] filterTags = null)
        {
            var result = await Task.Run(() =>
            {
                // Get tagged requests collection
                var taggedRequests = ServerContext.Database.GetCollection <TagRequest>(DatabaseConstants.TaggedRequestDataKey);
                // Log by descending timestamp
                return(taggedRequests.Find(
                           Query.And(
                               Query.All(nameof(TagRequest.Timestamp), Query.Descending),
                               Query.Where(nameof(TagRequest.Tag), v => filterTags == null || filterTags.Any(f => WildcardMatcher.IsMatch(v.AsString, f)))
                               ), limit: limit
                           ));
            });

            return(result);
        }
        /// <summary>
        /// OnMethodEnd callback
        /// </summary>
        /// <typeparam name="TTarget">Type of the target</typeparam>
        /// <typeparam name="TResult">Type of the BasicGetResult</typeparam>
        /// <param name="instance">Instance value, aka `this` of the instrumented method.</param>
        /// <param name="basicGetResult">BasicGetResult instance</param>
        /// <param name="exception">Exception instance in case the original code threw an exception.</param>
        /// <param name="state">Calltarget state value</param>
        /// <returns>A default CallTargetReturn to satisfy the CallTarget contract</returns>
        public static CallTargetReturn <TResult> OnMethodEnd <TTarget, TResult>(TTarget instance, TResult basicGetResult, Exception exception, CallTargetState state)
            where TResult : IBasicGetResult, IDuckType
        {
            var queue       = (string)state.State;
            var startTime   = state.StartTime;
            var agent       = Agent.Instance;
            var transaction = agent.Tracer.CurrentTransaction;

            if (transaction is null)
            {
                return(new CallTargetReturn <TResult>(basicGetResult));
            }

            var matcher = WildcardMatcher.AnyMatch(transaction.Configuration.IgnoreMessageQueues, queue);

            if (matcher != null)
            {
                agent.Logger.Trace()
                ?.Log(
                    "Not tracing message from {Queue} because it matched IgnoreMessageQueues pattern {Matcher}",
                    queue,
                    matcher.GetMatcher());

                return(new CallTargetReturn <TResult>(basicGetResult));
            }

            // check if there is an actual instance of the duck-typed type. RabbitMQ client can return null when the server
            // answers that there are no messages available
            var instanceNotNull = basicGetResult.Instance != null;

            if (instanceNotNull)
            {
                var normalizedExchange = RabbitMqIntegration.NormalizeExchangeName(basicGetResult.Exchange);
                matcher = WildcardMatcher.AnyMatch(transaction.Configuration.IgnoreMessageQueues, normalizedExchange);
                if (matcher != null)
                {
                    agent.Logger.Trace()
                    ?.Log(
                        "Not tracing message from {Queue} because it matched IgnoreMessageQueues pattern {Matcher}",
                        normalizedExchange,
                        matcher.GetMatcher());

                    return(new CallTargetReturn <TResult>(basicGetResult));
                }
            }

            var normalizedQueue = RabbitMqIntegration.NormalizeQueueName(queue);
            var span            = agent.Tracer.CurrentExecutionSegment().StartSpan(
                $"{RabbitMqIntegration.Name} POLL from {normalizedQueue}",
                ApiConstants.TypeMessaging,
                RabbitMqIntegration.Subtype);

            if (startTime.HasValue && span is Span realSpan)
            {
                realSpan.Timestamp = TimeUtils.ToTimestamp(startTime.Value);
            }

            span.Context.Message = new Message {
                Queue = new Queue {
                    Name = queue
                }
            };

            if (instanceNotNull)
            {
                span.SetLabel("message_size", basicGetResult.Body?.Length ?? 0);

                if (!string.IsNullOrEmpty(basicGetResult.RoutingKey))
                {
                    span.Context.Message.RoutingKey = basicGetResult.RoutingKey;
                }
            }

            span.EndCapturingException(exception);
            return(new CallTargetReturn <TResult>(basicGetResult));
        }
        /// <summary>
        /// Extracts the request body using measure to prevent the 'read once' problem (cannot read after the body ha been already
        /// read).
        /// </summary>
        /// <param name="request"></param>
        /// <param name="logger"></param>
        /// <returns></returns>
        public static async Task <string> ExtractRequestBodyAsync(this HttpRequest request, IApmLogger logger, IConfigSnapshot configSnapshot)
        {
            string body = null;

            try
            {
                if (request.HasFormContentType)
                {
                    var form = await request.ReadFormAsync(Consts.FormContentOptions);

                    var itemProcessed = 0;
                    if (form != null && form.Count > 0)
                    {
                        var sb = new StringBuilder();

                        foreach (var item in form)
                        {
                            sb.Append(item.Key);
                            sb.Append("=");

                            if (WildcardMatcher.IsAnyMatch(configSnapshot.SanitizeFieldNames, item.Key))
                            {
                                sb.Append(Elastic.Apm.Consts.Redacted);
                            }
                            else
                            {
                                sb.Append(item.Value);
                            }

                            itemProcessed++;
                            if (itemProcessed != form.Count)
                            {
                                sb.Append("&");
                            }
                        }

                        body = sb.ToString();
                    }
                }
                else
                {
                    request.EnableBuffering();
                    request.Body.Position = 0;

                    using (var reader = new StreamReader(request.Body,
                                                         Encoding.UTF8,
                                                         false,
                                                         1024 * 2,
                                                         true))
                        body = await reader.ReadToEndAsync();

                    // Truncate the body to the first 2kb if it's longer
                    if (body.Length > Consts.RequestBodyMaxLength)
                    {
                        body = body.Substring(0, Consts.RequestBodyMaxLength);
                    }
                    request.Body.Position = 0;
                }
            }
            catch (IOException ioException)
            {
                logger.Error()?.LogException(ioException, "IO Error reading request body");
            }
            catch (Exception e)
            {
                logger.Error()?.LogException(e, "Error reading request body");
            }
            return(body);
        }
Beispiel #11
0
        /// <summary>
        /// Extracts the request body, up to a specified maximum length.
        /// The request body that is read is buffered.
        /// </summary>
        /// <param name="request">The request</param>
        /// <param name="logger">The logger</param>
        /// <param name="configSnapshot">The configuration snapshot</param>
        /// <returns></returns>
        public static string ExtractRequestBody(this HttpRequest request, IApmLogger logger, IConfigSnapshot configSnapshot)
        {
            string body = null;
            var    longerThanMaxLength = false;

            try
            {
                if (request.HasFormContentType)
                {
                    var form = request.Form;

                    var itemProcessed = 0;
                    if (form != null && form.Count > 0)
                    {
                        var sb = new StringBuilder();
                        foreach (var item in form)
                        {
                            sb.Append(item.Key);
                            sb.Append("=");

                            if (WildcardMatcher.IsAnyMatch(configSnapshot.SanitizeFieldNames, item.Key))
                            {
                                sb.Append(Elastic.Apm.Consts.Redacted);
                            }
                            else
                            {
                                sb.Append(item.Value);
                            }

                            itemProcessed++;
                            if (itemProcessed != form.Count)
                            {
                                sb.Append("&");
                            }

                            // perf: check length once per iteration and truncate at the end, rather than each append
                            if (sb.Length > RequestBodyMaxLength)
                            {
                                longerThanMaxLength = true;
                                break;
                            }
                        }

                        body = sb.ToString(0, Math.Min(sb.Length, RequestBodyMaxLength));
                    }
                }
                else
                {
                    // allow synchronous reading of the request stream, which is false by default from 3.0 onwards.
                    // Reading must be synchronous as it can happen within a synchronous diagnostic listener method
                    var bodyControlFeature = request.HttpContext.Features.Get <IHttpBodyControlFeature>();
                    if (bodyControlFeature != null)
                    {
                        bodyControlFeature.AllowSynchronousIO = true;
                    }

                    request.EnableBuffering();
                    var requestBody = request.Body;
                    requestBody.Position = 0;
                    var arrayPool = ArrayPool <char> .Shared;
                    var capacity  = 512;
                    var buffer    = arrayPool.Rent(capacity);
                    var totalRead = 0;
                    int read;

                    // requestBody.Length is 0 on initial buffering - length relates to how much has been read and buffered.
                    // Read to just beyond request body max length so that we can determine if truncation will occur
                    try
                    {
                        // TODO: can we assume utf-8 encoding?
                        using var reader = new StreamReader(requestBody, Encoding.UTF8, false, buffer.Length, true);
                        while ((read = reader.Read(buffer, 0, capacity)) != 0)
                        {
                            totalRead += read;
                            if (totalRead > RequestBodyMaxLength)
                            {
                                longerThanMaxLength = true;
                                break;
                            }
                        }
                    }
                    finally
                    {
                        arrayPool.Return(buffer);
                    }

                    requestBody.Position = 0;
                    capacity             = Math.Min(totalRead, RequestBodyMaxLength);
                    buffer = arrayPool.Rent(capacity);

                    try
                    {
                        using var reader = new StreamReader(requestBody, Encoding.UTF8, false, RequestBodyMaxLength, true);
                        read             = reader.ReadBlock(buffer, 0, capacity);
                        body             = new string(buffer, 0, read);
                    }
                    finally
                    {
                        arrayPool.Return(buffer);
                    }

                    requestBody.Position = 0;
                }
            }
            catch (IOException ioException)
            {
                logger.Error()?.LogException(ioException, "IO Error reading request body");
            }
            catch (Exception e)
            {
                logger.Error()?.LogException(e, "Error reading request body");
            }

            if (longerThanMaxLength)
            {
                logger.Debug()?.Log("truncated body to max length {MaxLength}", RequestBodyMaxLength);
            }

            return(body);
        }
Beispiel #12
0
 public void CanMatch(string pattern, string input, bool expected)
 {
     Assert.Equal(expected, WildcardMatcher.Matches(pattern, input));
 }
 private bool IsProcessWorkingSetMemoryEnabled(IReadOnlyList <WildcardMatcher> disabledMetrics) =>
 !WildcardMatcher.IsAnyMatch(disabledMetrics, ProcessWorkingSetMemory);
 private bool IsTotalMemoryEnabled(IReadOnlyList <WildcardMatcher> disabledMetrics) => !WildcardMatcher.IsAnyMatch(disabledMetrics, TotalMemory);
        public MetricsCollector(IApmLogger logger, IPayloadSender payloadSender, IConfigSnapshotProvider configSnapshotProvider)
        {
            _logger                 = logger.Scoped(nameof(MetricsCollector));
            _payloadSender          = payloadSender;
            _configSnapshotProvider = configSnapshotProvider;

            var currentConfigSnapshot = configSnapshotProvider.CurrentSnapshot;

            var interval = currentConfigSnapshot.MetricsIntervalInMilliseconds;

            // ReSharper disable once CompareOfFloatsByEqualityOperator
            if (interval == 0)
            {
                _logger.Info()?.Log("Collecting metrics is disabled - the agent won't collect metrics");
                return;
            }

            MetricsProviders = new List <IMetricsProvider>();

            if (!WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics, ProcessTotalCpuTimeProvider.ProcessCpuTotalPct))
            {
                MetricsProviders.Add(new ProcessTotalCpuTimeProvider(_logger));
            }
            if (!WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics, SystemTotalCpuProvider.SystemCpuTotalPct))
            {
                MetricsProviders.Add(new SystemTotalCpuProvider(_logger));
            }

            var collectProcessWorkingSet = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
                                                                       ProcessWorkingSetAndVirtualMemoryProvider.ProcessWorkingSetMemory);
            var collectProcessVirtualMemory = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
                                                                          ProcessWorkingSetAndVirtualMemoryProvider.ProcessVirtualMemory);

            if (collectProcessVirtualMemory || collectProcessWorkingSet)
            {
                MetricsProviders.Add(new ProcessWorkingSetAndVirtualMemoryProvider(collectProcessVirtualMemory, collectProcessWorkingSet));
            }

            var collectTotalMemory = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
                                                                 FreeAndTotalMemoryProvider.TotalMemory);
            var collectFreeMemory = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
                                                                FreeAndTotalMemoryProvider.FreeMemory);

            if (collectFreeMemory || collectTotalMemory)
            {
                MetricsProviders.Add(new FreeAndTotalMemoryProvider(collectFreeMemory, collectTotalMemory));
            }

            var collectGcCount = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
                                                             GcMetricsProvider.GcCountName);
            var collectGen0Size = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
                                                              GcMetricsProvider.GcGen0SizeName);
            var collectGen1Size = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
                                                              GcMetricsProvider.GcGen1SizeName);
            var collectGen2Size = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
                                                              GcMetricsProvider.GcGen2SizeName);
            var collectGen3Size = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
                                                              GcMetricsProvider.GcGen3SizeName);

            if (collectGcCount || collectGen0Size || collectGen1Size || collectGen2Size || collectGen3Size)
            {
                MetricsProviders.Add(new GcMetricsProvider(_logger, collectGcCount, collectGen0Size, collectGen1Size, collectGen2Size,
                                                           collectGen3Size));
            }

            var collectCgroupMemLimitBytes = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
                                                                         CgroupMetricsProvider.SystemProcessCgroupMemoryMemLimitBytes);
            var collectCgroupMemUsageBytes = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
                                                                         CgroupMetricsProvider.SystemProcessCgroupMemoryMemUsageBytes);
            var collectCgroupStatsInactiveFileBytes = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
                                                                                  CgroupMetricsProvider.SystemProcessCgroupMemoryStatsInactiveFileBytes);

            if (collectCgroupMemLimitBytes || collectCgroupMemUsageBytes || collectCgroupStatsInactiveFileBytes)
            {
                MetricsProviders.Add(
                    new CgroupMetricsProvider(_logger, collectCgroupMemLimitBytes, collectCgroupMemUsageBytes, collectCgroupStatsInactiveFileBytes));
            }

            _logger.Info()?.Log("Collecting metrics in {interval} milliseconds interval", interval);
            _timer          = new Timer(interval);
            _timer.Elapsed += (sender, args) => { CollectAllMetrics(); };
        }
Beispiel #16
0
 public void TestComplexExpressions()
 {
     WildcardMatcher.ValueOf("/foo/*/baz*").Matches("/foo/a/bar/b/baz").Should().BeTrue();
     WildcardMatcher.ValueOf("/foo/*/bar/*/baz").Matches("/foo/a/bar/b/baz").Should().BeTrue();
 }
Beispiel #17
0
        public HttpResponseMessage Get([FromUri] string[] fileNames)
        {
            var list = new List <FileHeader>();

            var startsWith = GetQueryStringValue("startsWith");

            if (string.IsNullOrEmpty(startsWith) == false)
            {
                var matches = GetQueryStringValue("matches");

                var endsWithSlash = startsWith.EndsWith("/") || startsWith.EndsWith("\\");
                startsWith = FileHeader.Canonize(startsWith);
                if (endsWithSlash)
                {
                    startsWith += "/";
                }

                Storage.Batch(accessor =>
                {
                    var actualStart             = 0;
                    var filesToSkip             = Paging.Start;
                    int fileCount, matchedFiles = 0, addedFiles = 0;

                    do
                    {
                        fileCount = 0;

                        foreach (var file in accessor.GetFilesStartingWith(startsWith, actualStart, Paging.PageSize))
                        {
                            fileCount++;

                            var keyTest = file.FullPath.Substring(startsWith.Length);

                            if (WildcardMatcher.Matches(matches, keyTest) == false)
                            {
                                continue;
                            }

                            if (FileSystem.ReadTriggers.CanReadFile(file.FullPath, file.Metadata, ReadOperation.Load) == false)
                            {
                                continue;
                            }

                            matchedFiles++;

                            if (matchedFiles <= filesToSkip)
                            {
                                continue;
                            }

                            list.Add(file);
                            addedFiles++;
                        }

                        actualStart += Paging.PageSize;
                    }while (fileCount > 0 && addedFiles < Paging.PageSize && actualStart > 0 && actualStart < int.MaxValue);
                });
            }
            else
            {
                if (fileNames != null && fileNames.Length > 0)
                {
                    Storage.Batch(accessor =>
                    {
                        foreach (var path in fileNames.Where(x => x != null).Select(FileHeader.Canonize))
                        {
                            var file = accessor.ReadFile(path);

                            if (file == null || file.Metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker))
                            {
                                list.Add(null);
                                continue;
                            }

                            list.Add(file);
                        }
                    });
                }
                else
                {
                    int results;
                    var keys = Search.Query(null, null, Paging.Start, Paging.PageSize, out results);

                    Storage.Batch(accessor => list.AddRange(keys.Select(accessor.ReadFile).Where(x => x != null)));
                }
            }

            return(GetMessageWithObject(list)
                   .WithNoCache());
        }
Beispiel #18
0
        internal static ISpan CreateProducerSpan(IApmAgent agent, ITopicPartition topicPartition, bool isTombstone, bool finishOnClose)
        {
            ISpan span = null;

            try
            {
                // no current transaction, don't create a span
                var currentTransaction = agent.Tracer.CurrentTransaction;
                if (currentTransaction is null)
                {
                    return(span);
                }

                var topic   = topicPartition?.Topic;
                var matcher = WildcardMatcher.AnyMatch(currentTransaction.Configuration.IgnoreMessageQueues, topic);
                if (matcher != null)
                {
                    agent.Logger.Trace()
                    ?.Log(
                        "Not tracing message from {Queue} because it matched IgnoreMessageQueues pattern {Matcher}",
                        topic,
                        matcher.GetMatcher());

                    return(span);
                }

                var spanName = string.IsNullOrEmpty(topic)
                                        ? "Kafka SEND"
                                        : $"Kafka SEND to {topic}";

                span = agent.GetCurrentExecutionSegment().StartSpan(
                    spanName,
                    ApiConstants.TypeMessaging,
                    Subtype,
                    isExitSpan: true);

                if (!string.IsNullOrEmpty(topic))
                {
                    span.Context.Message = new Message {
                        Queue = new Queue {
                            Name = topic
                        }
                    }
                }
                ;

                if (topicPartition?.Partition is not null && !topicPartition.Partition.IsSpecial)
                {
                    span.SetLabel("partition", topicPartition.Partition.ToString());
                }

                if (isTombstone)
                {
                    span.SetLabel("tombstone", "true");
                }
            }
            catch (Exception ex)
            {
                agent.Logger.Error()?.LogException(ex, "Error creating or populating kafka span.");
            }

            return(span);
        }
Beispiel #19
0
        public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize,
                                                   CancellationToken token, ref int nextStart, Action <RavenJObject> addDoc,
                                                   string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null,
                                                   string skipAfter   = null)
        {
            if (idPrefix == null)
            {
                throw new ArgumentNullException("idPrefix");
            }
            idPrefix = idPrefix.Trim();

            var canPerformRapidPagination = nextStart > 0 && start == nextStart;
            var actualStart = canPerformRapidPagination ? start : 0;
            var addedDocs   = 0;
            var matchedDocs = 0;

            TransactionalStorage.Batch(
                actions =>
            {
                var docsToSkip = canPerformRapidPagination ? 0 : start;
                int docCount;

                AbstractTransformer storedTransformer = null;
                if (transformer != null)
                {
                    storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
                    if (storedTransformer == null)
                    {
                        throw new InvalidOperationException("No transformer with the name: " + transformer);
                    }
                }

                do
                {
                    docCount = 0;
                    var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter);
                    var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState, transformerParameters);

                    foreach (var doc in docs)
                    {
                        token.ThrowIfCancellationRequested();
                        docCount++;
                        var keyTest = doc.Key.Substring(idPrefix.Length);

                        if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
                        {
                            continue;
                        }

                        DocumentRetriever.EnsureIdInMetadata(doc);
                        var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);

                        var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc;
                        document     = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
                        if (document == null)
                        {
                            continue;
                        }

                        matchedDocs++;

                        if (matchedDocs <= docsToSkip)
                        {
                            continue;
                        }

                        token.ThrowIfCancellationRequested();

                        if (storedTransformer != null)
                        {
                            using (new CurrentTransformationScope(Database, documentRetriever))
                            {
                                var transformed =
                                    storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) })
                                    .Select(x => JsonExtensions.ToJObject(x))
                                    .ToArray();

                                if (transformed.Length == 0)
                                {
                                    throw new InvalidOperationException("The transform results function failed on a document: " + document.Key);
                                }

                                var transformedJsonDocument = new JsonDocument
                                {
                                    Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(documentRetriever.Etag),
                                    NonAuthoritativeInformation = document.NonAuthoritativeInformation,
                                    LastModified = document.LastModified,
                                    DataAsJson   = new RavenJObject {
                                        { "$values", new RavenJArray(transformed) }
                                    },
                                };

                                addDoc(transformedJsonDocument.ToJson());
                            }
                        }
                        else
                        {
                            addDoc(document.ToJson());
                        }

                        addedDocs++;

                        if (addedDocs >= pageSize)
                        {
                            break;
                        }
                    }

                    actualStart += pageSize;
                }while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue);
            });

            if (addedDocs != pageSize)
            {
                nextStart = start; // will mark as last page
            }
            else if (canPerformRapidPagination)
            {
                nextStart = start + matchedDocs;
            }
            else
            {
                nextStart = actualStart;
            }
        }
        public FieldTermStack(IndexReader reader, int docId, String fieldName, FieldQuery fieldQuery, IState state)
        {
            this.fieldName = fieldName;

            var tfv = reader.GetTermFreqVector(docId, fieldName, state);

            if (tfv == null)
            {
                return; // just return to make null snippets
            }
            TermPositionVector tpv = null;

            try
            {
                tpv = (TermPositionVector)tfv;
            }
            catch (InvalidCastException)
            {
                return; // just return to make null snippets
            }

            List <String> termSet = fieldQuery.getTermSet(fieldName);

            // just return to make null snippet if un-matched fieldName specified when fieldMatch == true
            if (termSet == null)
            {
                return;
            }
            var needwildcard = termSet.Any(x => x.IndexOfAny(new char[] { '*', '?' }) != -1);

            foreach (String term in tpv.GetTerms())
            {
                if (needwildcard)
                {
                    if (termSet.Any(ts => WildcardMatcher.Matches(ts, term)) == false)
                    {
                        continue;
                    }
                }
                else if (!termSet.Contains(term))
                {
                    continue;
                }

                int index = tpv.IndexOf(term);
                TermVectorOffsetInfo[] tvois = tpv.GetOffsets(index);
                if (tvois == null)
                {
                    return; // just return to make null snippets
                }
                int[] poss = tpv.GetTermPositions(index);
                if (poss == null)
                {
                    return; // just return to make null snippets
                }
                for (int i = 0; i < tvois.Length; i++)
                {
                    termList.AddLast(new TermInfo(term, tvois[i].StartOffset, tvois[i].EndOffset, poss[i]));
                }
            }

            // sort by position
            //Collections.sort(termList);
            Sort(termList);
        }