Ejemplo n.º 1
0
        public HttpLogShipper(
            string serverUrl,
            string bufferBaseFilename,
            string apiKey,
            int batchPostingLimit,
            TimeSpan period,
            long?eventBodyLimitBytes,
            LoggingLevelSwitch levelControlSwitch,
            HttpMessageHandler messageHandler,
            long?retainedInvalidPayloadsLimitBytes)
        {
            _apiKey              = apiKey;
            _batchPostingLimit   = batchPostingLimit;
            _eventBodyLimitBytes = eventBodyLimitBytes;
            _controlledSwitch    = new ControlledLevelSwitch(levelControlSwitch);
            _connectionSchedule  = new ExponentialBackoffConnectionSchedule(period);
            _retainedInvalidPayloadsLimitBytes = retainedInvalidPayloadsLimitBytes;
            _httpClient = messageHandler != null ?
                          new HttpClient(messageHandler) :
                          new HttpClient();
            _httpClient.BaseAddress = new Uri(SeqApi.NormalizeServerBaseAddress(serverUrl));

            _bookmarkFilename    = Path.GetFullPath(bufferBaseFilename + ".bookmark");
            _logFolder           = Path.GetDirectoryName(_bookmarkFilename);
            _candidateSearchPath = Path.GetFileName(bufferBaseFilename) + "*.json";

            _timer = new PortableTimer(c => OnTick());

            SetTimer();
        }
Ejemplo n.º 2
0
        protected override async Task EmitBatchAsync(IEnumerable <LogEvent> events)
        {
            _nextRequiredLevelCheckUtc = DateTime.UtcNow.Add(RequiredLevelCheckInterval);

            string payload, payloadContentType;

            if (_useCompactFormat)
            {
                payloadContentType = SeqApi.CompactLogEventFormatMimeType;
                payload            = FormatCompactPayload(events, _eventBodyLimitBytes);
            }
            else
            {
                payloadContentType = SeqApi.RawEventFormatMimeType;
                payload            = FormatRawPayload(events, _eventBodyLimitBytes);
            }

            var content = new StringContent(payload, Encoding.UTF8, payloadContentType);

            if (!string.IsNullOrWhiteSpace(_apiKey))
            {
                content.Headers.Add(SeqApi.ApiKeyHeaderName, _apiKey);
            }

            var result = await _httpClient.PostAsync(SeqApi.BulkUploadResource, content).ConfigureAwait(false);

            if (!result.IsSuccessStatusCode)
            {
                throw new LoggingFailedException($"Received failed result {result.StatusCode} when posting events to Seq");
            }

            var returned = await result.Content.ReadAsStringAsync();

            _controlledSwitch.Update(SeqApi.ReadEventInputResult(returned));
        }
Ejemplo n.º 3
0
 public SeqAuditSink(
     string serverUrl,
     string apiKey,
     HttpMessageHandler messageHandler,
     bool useCompactFormat)
 {
     if (serverUrl == null)
     {
         throw new ArgumentNullException(nameof(serverUrl));
     }
     _apiKey                 = apiKey;
     _useCompactFormat       = useCompactFormat;
     _httpClient             = messageHandler != null ? new HttpClient(messageHandler) : new HttpClient();
     _httpClient.BaseAddress = new Uri(SeqApi.NormalizeServerBaseAddress(serverUrl));
 }
Ejemplo n.º 4
0
 public SeqSink(
     string serverUrl,
     string apiKey,
     int batchPostingLimit,
     TimeSpan period,
     long?eventBodyLimitBytes,
     LoggingLevelSwitch levelControlSwitch,
     HttpMessageHandler messageHandler)
     : base(batchPostingLimit, period)
 {
     if (serverUrl == null)
     {
         throw new ArgumentNullException(nameof(serverUrl));
     }
     _apiKey = apiKey;
     _eventBodyLimitBytes    = eventBodyLimitBytes;
     _controlledSwitch       = new ControlledLevelSwitch(levelControlSwitch);
     _httpClient             = messageHandler != null ? new HttpClient(messageHandler) : new HttpClient();
     _httpClient.BaseAddress = new Uri(SeqApi.NormalizeServerBaseAddress(serverUrl));
 }
Ejemplo n.º 5
0
        protected override void EmitBatch(IEnumerable <LogEvent> events)
        {
            var payload = new StringWriter();

            payload.Write("{\"events\":[");

            var formatter  = new JsonFormatter();
            var delimStart = "";

            foreach (var logEvent in events)
            {
                payload.Write(delimStart);
                formatter.Format(logEvent, payload);
                delimStart = ",";
            }

            payload.Write("]}");

            var content = new StringContent(payload.ToString(), Encoding.UTF8, "application/json");

            if (!string.IsNullOrWhiteSpace(_apiKey))
            {
                content.Headers.Add(ApiKeyHeaderName, _apiKey);
            }

            var result = _httpClient.PostAsync(BulkUploadResource, content).Result;

            if (!result.IsSuccessStatusCode)
            {
                SelfLog.WriteLine("Received failed result {0}: {1}", result.StatusCode, result.Content.ReadAsStringAsync().Result);
            }

            var returned = result.Content.ReadAsStringAsync().Result;

            _minimumAcceptedLevel = SeqApi.ReadEventInputResult(returned);
        }
Ejemplo n.º 6
0
        protected override async Task EmitBatchAsync(IEnumerable <LogEvent> events)
        {
            var payload = new StringWriter();

            payload.Write("{\"events\":[");

            var formatter  = new JsonFormatter(closingDelimiter: "");
            var delimStart = "";

            foreach (var logEvent in events)
            {
                payload.Write(delimStart);
                formatter.Format(logEvent, payload);
                delimStart = ",";
            }

            payload.Write("]}");

            var content = new StringContent(payload.ToString(), Encoding.UTF8, "application/json");

            if (!string.IsNullOrWhiteSpace(_apiKey))
            {
                content.Headers.Add(ApiKeyHeaderName, _apiKey);
            }

            var result = await _httpClient.PostAsync(BulkUploadResource, content);

            if (!result.IsSuccessStatusCode)
            {
                throw new LoggingFailedException(string.Format("Received failed result {0} when posting events to Seq", result.StatusCode));
            }

            var returned = await result.Content.ReadAsStringAsync();

            _minimumAcceptedLevel = SeqApi.ReadEventInputResult(returned);
        }
Ejemplo n.º 7
0
        void OnTick()
        {
            LogEventLevel?minimumAcceptedLevel = null;

            try
            {
                int count;
                do
                {
                    count = 0;

                    // Locking the bookmark ensures that though there may be multiple instances of this
                    // class running, only one will ship logs at a time.

                    using (var bookmark = IOFile.Open(_bookmarkFilename, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read))
                    {
                        long   nextLineBeginsAtOffset;
                        string currentFile;

                        TryReadBookmark(bookmark, out nextLineBeginsAtOffset, out currentFile);

                        var fileSet = GetFileSet();

                        if (currentFile == null || !IOFile.Exists(currentFile))
                        {
                            nextLineBeginsAtOffset = 0;
                            currentFile            = fileSet.FirstOrDefault();
                        }

                        if (currentFile == null)
                        {
                            continue;
                        }

                        var payload = ReadPayload(currentFile, ref nextLineBeginsAtOffset, ref count);

                        if (count > 0 || _levelControlSwitch != null && _nextRequiredLevelCheckUtc < DateTime.UtcNow)
                        {
                            lock (_stateLock)
                            {
                                _nextRequiredLevelCheckUtc = DateTime.UtcNow.Add(RequiredLevelCheckInterval);
                            }

                            var content = new StringContent(payload, Encoding.UTF8, "application/json");
                            if (!string.IsNullOrWhiteSpace(_apiKey))
                            {
                                content.Headers.Add(ApiKeyHeaderName, _apiKey);
                            }

                            var result = _httpClient.PostAsync(BulkUploadResource, content).Result;
                            if (result.IsSuccessStatusCode)
                            {
                                _connectionSchedule.MarkSuccess();
                                WriteBookmark(bookmark, nextLineBeginsAtOffset, currentFile);
                                var returned = result.Content.ReadAsStringAsync().Result;
                                minimumAcceptedLevel = SeqApi.ReadEventInputResult(returned);
                            }
                            else if (result.StatusCode == HttpStatusCode.BadRequest ||
                                     result.StatusCode == HttpStatusCode.RequestEntityTooLarge)
                            {
                                // The connection attempt was successful - the payload we sent was the problem.
                                _connectionSchedule.MarkSuccess();

                                DumpInvalidPayload(result, payload).Wait();

                                WriteBookmark(bookmark, nextLineBeginsAtOffset, currentFile);
                            }
                            else
                            {
                                _connectionSchedule.MarkFailure();
                                SelfLog.WriteLine("Received failed HTTP shipping result {0}: {1}", result.StatusCode, result.Content.ReadAsStringAsync().Result);
                                break;
                            }
                        }
                        else
                        {
                            // For whatever reason, there's nothing waiting to send. This means we should try connecting again at the
                            // regular interval, so mark the attempt as successful.
                            _connectionSchedule.MarkSuccess();

                            // Only advance the bookmark if no other process has the
                            // current file locked, and its length is as we found it.

                            if (fileSet.Length == 2 && fileSet.First() == currentFile && IsUnlockedAtLength(currentFile, nextLineBeginsAtOffset))
                            {
                                WriteBookmark(bookmark, 0, fileSet[1]);
                            }

                            if (fileSet.Length > 2)
                            {
                                // Once there's a third file waiting to ship, we do our
                                // best to move on, though a lock on the current file
                                // will delay this.

                                IOFile.Delete(fileSet[0]);
                            }
                        }
                    }
                }while (count == _batchPostingLimit);
            }
            catch (Exception ex)
            {
                SelfLog.WriteLine("Exception while emitting periodic batch from {0}: {1}", this, ex);
                _connectionSchedule.MarkFailure();
            }
            finally
            {
                lock (_stateLock)
                {
                    UpdateLevelControlSwitch(minimumAcceptedLevel);

                    if (!_unloading)
                    {
                        SetTimer();
                    }
                }
            }
        }
Ejemplo n.º 8
0
        void OnTick()
        {
            LogEventLevel?minimumAcceptedLevel = null;

            try
            {
                var count = 0;

                do
                {
                    // Locking the bookmark ensures that though there may be multiple instances of this
                    // class running, only one will ship logs at a time.

                    using (var bookmark = File.Open(_bookmarkFilename, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read))
                    {
                        long   nextLineBeginsAtOffset;
                        string currentFile;

                        TryReadBookmark(bookmark, out nextLineBeginsAtOffset, out currentFile);

                        var fileSet = GetFileSet();

                        if (currentFile == null || !File.Exists(currentFile))
                        {
                            nextLineBeginsAtOffset = 0;
                            currentFile            = fileSet.FirstOrDefault();
                        }

                        if (currentFile != null)
                        {
                            var payload = new StringWriter();
                            payload.Write("{\"events\":[");
                            count = 0;
                            var delimStart = "";

                            using (var current = File.Open(currentFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                            {
                                current.Position = nextLineBeginsAtOffset;

                                string nextLine;
                                while (count < _batchPostingLimit &&
                                       TryReadLine(current, ref nextLineBeginsAtOffset, out nextLine))
                                {
                                    ++count;
                                    payload.Write(delimStart);
                                    payload.Write(nextLine);
                                    delimStart = ",";
                                }

                                payload.Write("]}");
                            }

                            if (count > 0)
                            {
                                var content = new StringContent(payload.ToString(), Encoding.UTF8, "application/json");
                                if (!string.IsNullOrWhiteSpace(_apiKey))
                                {
                                    content.Headers.Add(ApiKeyHeaderName, _apiKey);
                                }

                                var result = _httpClient.PostAsync(BulkUploadResource, content).Result;
                                if (result.IsSuccessStatusCode)
                                {
                                    WriteBookmark(bookmark, nextLineBeginsAtOffset, currentFile);
                                    var returned = result.Content.ReadAsStringAsync().Result;
                                    minimumAcceptedLevel = SeqApi.ReadEventInputResult(returned);
                                }
                                else
                                {
                                    SelfLog.WriteLine("Received failed HTTP shipping result {0}: {1}", result.StatusCode, result.Content.ReadAsStringAsync().Result);
                                }
                            }
                            else
                            {
                                // Only advance the bookmark if no other process has the
                                // current file locked, and its length is as we found it.

                                if (fileSet.Length == 2 && fileSet.First() == currentFile && IsUnlockedAtLength(currentFile, nextLineBeginsAtOffset))
                                {
                                    WriteBookmark(bookmark, 0, fileSet[1]);
                                }

                                if (fileSet.Length > 2)
                                {
                                    // Once there's a third file waiting to ship, we do our
                                    // best to move on, though a lock on the current file
                                    // will delay this.

                                    File.Delete(fileSet[0]);
                                }
                            }
                        }
                    }
                }while (count == _batchPostingLimit);
            }
            catch (Exception ex)
            {
                SelfLog.WriteLine("Exception while emitting periodic batch from {0}: {1}", this, ex);
            }
            finally
            {
                lock (_stateLock)
                {
                    _minimumAcceptedLevel = minimumAcceptedLevel;
                    if (!_unloading)
                    {
                        SetTimer();
                    }
                }
            }
        }
Ejemplo n.º 9
0
        protected override async Task EmitBatchAsync(IEnumerable <LogEvent> events)
        {
            _nextRequiredLevelCheckUtc = DateTime.UtcNow.Add(RequiredLevelCheckInterval);

            var payload = new StringWriter();

            payload.Write("{\"Events\":[");

            var delimStart = "";

            foreach (var logEvent in events)
            {
                if (_eventBodyLimitBytes.HasValue)
                {
                    var scratch = new StringWriter();
                    RawJsonFormatter.FormatContent(logEvent, scratch);
                    var buffered = scratch.ToString();

                    if (Encoding.UTF8.GetByteCount(buffered) > _eventBodyLimitBytes.Value)
                    {
                        SelfLog.WriteLine("Event JSON representation exceeds the byte size limit of {0} set for this sink and will be dropped; data: {1}", _eventBodyLimitBytes, buffered);
                    }
                    else
                    {
                        payload.Write(delimStart);
                        payload.Write(buffered);
                        delimStart = ",";
                    }
                }
                else
                {
                    payload.Write(delimStart);
                    RawJsonFormatter.FormatContent(logEvent, payload);
                    delimStart = ",";
                }
            }

            payload.Write("]}");

            var content = new StringContent(payload.ToString(), Encoding.UTF8, "application/json");

            if (!string.IsNullOrWhiteSpace(_apiKey))
            {
                content.Headers.Add(ApiKeyHeaderName, _apiKey);
            }

            var result = await _httpClient.PostAsync(BulkUploadResource, content).ConfigureAwait(false);

            if (!result.IsSuccessStatusCode)
            {
                throw new LoggingFailedException($"Received failed result {result.StatusCode} when posting events to Seq");
            }

            var returned = await result.Content.ReadAsStringAsync();

            var minimumAcceptedLevel = SeqApi.ReadEventInputResult(returned);

            if (minimumAcceptedLevel == null)
            {
                if (_levelControlSwitch != null)
                {
                    _levelControlSwitch.MinimumLevel = LevelAlias.Minimum;
                }
            }
            else
            {
                if (_levelControlSwitch == null)
                {
                    _levelControlSwitch = new LoggingLevelSwitch(minimumAcceptedLevel.Value);
                }
                else
                {
                    _levelControlSwitch.MinimumLevel = minimumAcceptedLevel.Value;
                }
            }
        }