private static JobRunnerContext BuildJobRunnerContext(SubscriptionConfiguration config) { var result = new JobRunnerContext { Id = config.Id, EventFilterPattern = config.EventFilterPattern, TargetUrl = config.TargetUrl, AwsRegion = config.AwsRegion, LogGroupName = config.LogGroupName, LogStreamNames = config.LogStreamNames, LogStreamNamePrefix = config.LogStreamNamePrefix, ReadMaxBatchSize = config.ReadMaxBatchSize.GetValueOrDefault(), MinIntervalSeconds = config.MinIntervalSeconds.GetValueOrDefault(), MaxIntervalSeconds = config.MaxIntervalSeconds.GetValueOrDefault(), ClockSkewProtectionSeconds = config.ClockSkewProtectionSeconds.GetValueOrDefault(), TargetMaxBatchSize = config.TargetMaxBatchSize.GetValueOrDefault(), TargetSubscriptionData = config.TargetSubscriptionData, StartInstant = ParseAbsoluteOrRelativeTime(config.StartTimeIso, config.StartTimeSecondsAgo) ?? InstantUtils.Now, EndInstant = ParseAbsoluteOrRelativeTime(config.EndTimeIso, config.EndTimeSecondsAgo), Logger = Log.Logger.ForContext("SubscriptionId", config.Id), AwsClient = new AmazonCloudWatchLogsClient(RegionEndpoint.GetBySystemName(config.AwsRegion)), HttpClient = new HttpClient(), Random = new Random(), NextInputBatchSequence = 1 }; result.HttpClient.Timeout = TimeSpan.FromSeconds(config.TargetTimeoutSeconds.GetValueOrDefault()); return(result); }
private static void ValidateSubscription(SubscriptionConfiguration config) { if (config.Id.IsNullOrWhitespace()) { throw new ArgumentException("Subscription ID is required for all subscriptions."); } if (config.Id.Length > 100) { throw new ArgumentException("Subscription ID cannot be longer than 100 characters."); } if (!ConfigurationIdRegex.IsMatch(config.Id)) { throw new ArgumentException($"Subscription '{config.Id}' - Invalid characters in Subscription ID. Allowed characters are alpha numeric, dash, underscore and dot."); } if (config.TargetUrl.IsNullOrWhitespace()) { throw new ArgumentException($"Subscription '{config.Id}' - targetUrl is required to be specified"); } if (config.AwsRegion.IsNullOrWhitespace()) { throw new ArgumentException($"Subscription '{config.Id}' - awsRegion is required to be specified"); } if (config.LogGroupName.IsNullOrWhitespace()) { throw new ArgumentException($"Subscription '{config.Id}' - logGroupName is required to be specified"); } if (config.LogGroupPattern.HasValue()) { throw new ArgumentException($"Subscription '{config.Id}' - logGroupPattern should have been expanded at this point and set to null"); } if (config.LogStreamNames != null && config.LogStreamNames.SafeAny(lsn => lsn.HasValue()) && config.LogStreamNamePrefix.HasValue()) { throw new ArgumentException($"Subscription '{config.Id}' - logStreamNames and logStreamNamePrefix fields cannot be specified at the same time. AWS API allows only one of them to be specified in each call."); } if (config.StartTimeIso.HasValue() && config.StartTimeSecondsAgo.HasValue) { throw new ArgumentException($"Subscription '{config.Id}' - startTimeIso and startTimeSecondsAgo cannot be specified at the same time. Only one of them allowed to be present in the configuration."); } if (config.EndTimeIso.HasValue() && config.EndTimeSecondsAgo.HasValue) { throw new ArgumentException($"Subscription '{config.Id}' - endTimeIso and endTimeSecondsAgo cannot be specified at the same time. Only one of them allowed to be present in the configuration."); } if (config.Children != null) { throw new ArgumentException($"Subscription '{config.Id}' - children need to be flattened in this point and Children property must have been set to null."); } }
private static void CoerceSubscription(SubscriptionConfiguration config) { if (config.EventFilterPattern.IsNullOrWhitespace()) { config.EventFilterPattern = null; } if (config.LogStreamNamePrefix.IsNullOrWhitespace()) { config.LogStreamNamePrefix = null; } if (config.LogStreamNames.SafeAny()) { config.LogStreamNames = config.LogStreamNames.Where(sn => sn.HasValue()).ToList(); if (!config.LogStreamNames.Any()) { config.LogStreamNames = null; } } config.Enabled ??= true; config.ReadMaxBatchSize ??= 10000; config.MinIntervalSeconds ??= 15; config.MaxIntervalSeconds ??= 300; config.ClockSkewProtectionSeconds ??= 15; config.TargetTimeoutSeconds ??= 60; config.TargetMaxBatchSize ??= 1; config.ReadMaxBatchSize = Math.Max(config.ReadMaxBatchSize.Value, 1); config.ReadMaxBatchSize = Math.Min(config.ReadMaxBatchSize.Value, 10000); config.MinIntervalSeconds = Math.Max(config.MinIntervalSeconds.Value, 5); config.MinIntervalSeconds = Math.Min(config.MinIntervalSeconds.Value, 10 * 60); config.MaxIntervalSeconds = Math.Max(config.MaxIntervalSeconds.Value, config.MinIntervalSeconds.Value); config.MaxIntervalSeconds = Math.Min(config.MaxIntervalSeconds.Value, 8 * 60 * 60); config.ClockSkewProtectionSeconds = Math.Max(config.ClockSkewProtectionSeconds.Value, 5); config.ClockSkewProtectionSeconds = Math.Min(config.ClockSkewProtectionSeconds.Value, 120); config.TargetTimeoutSeconds = Math.Max(config.TargetTimeoutSeconds.Value, 1); config.TargetTimeoutSeconds = Math.Min(config.TargetTimeoutSeconds.Value, 15 * 60); config.TargetMaxBatchSize = Math.Max(config.TargetMaxBatchSize.Value, 1); config.TargetMaxBatchSize = Math.Min(config.TargetMaxBatchSize.Value, 20000); if (config.Enabled.GetValueOrDefault()) { DependencyContext.RunnerContexts.Add(config.Id, BuildJobRunnerContext(config)); } }
private static void FlattenSubscriptionItem(List <SubscriptionConfiguration> flattenedList, SubscriptionConfiguration parent, SubscriptionConfiguration item) { var extendedItem = item.Clone(false); extendedItem.ExtendFrom(parent); extendedItem.Children = null; if (item.Children.SafeAny()) { foreach (var child in item.Children) { FlattenSubscriptionItem(flattenedList, extendedItem, child); } } else { flattenedList.Add(extendedItem); } }
public void ExtendFrom(SubscriptionConfiguration parent) { if (parent == null) { return; } if (parent.Id.HasValue()) { Id = Id.HasValue() ? parent.Id + "." + Id : parent.Id; } Enabled ??= parent.Enabled; Comments = Comments.Or(parent.Comments); AwsRegion = AwsRegion.Or(parent.AwsRegion); LogGroupName = LogGroupName.Or(parent.LogGroupName); LogGroupPattern = LogGroupPattern.Or(parent.LogGroupPattern); StartTimeIso = StartTimeIso.Or(parent.StartTimeIso); EndTimeIso = EndTimeIso.Or(parent.EndTimeIso); StartTimeSecondsAgo ??= parent.StartTimeSecondsAgo; EndTimeSecondsAgo ??= parent.EndTimeSecondsAgo; EventFilterPattern = EventFilterPattern.Or(parent.EventFilterPattern); LogStreamNamePrefix = LogStreamNamePrefix.Or(parent.LogStreamNamePrefix); if (parent.LogStreamNames.SafeAny()) { LogStreamNames = LogStreamNames.SafeUnion(parent.LogStreamNames).ToList(); } ReadMaxBatchSize ??= parent.ReadMaxBatchSize; MinIntervalSeconds ??= parent.MinIntervalSeconds; MaxIntervalSeconds ??= parent.MaxIntervalSeconds; ClockSkewProtectionSeconds ??= parent.ClockSkewProtectionSeconds; TargetUrl = TargetUrl.Or(parent.TargetUrl); TargetTimeoutSeconds ??= parent.TargetTimeoutSeconds; TargetMaxBatchSize ??= parent.TargetMaxBatchSize; TargetSubscriptionData = TargetSubscriptionData.Or(parent.TargetSubscriptionData); }
private static async Task <List <SubscriptionConfiguration> > ExpandSubscriptionPattern(SubscriptionConfiguration originalSubscription) { var cw = new AmazonCloudWatchLogsClient(RegionEndpoint.GetBySystemName(originalSubscription.AwsRegion)); var regex = new Regex(originalSubscription.LogGroupPattern, RegexOptions.IgnoreCase); var expandedList = new List <SubscriptionConfiguration>(); var nextToken = (string)null; do { var response = await cw.DescribeLogGroupsAsync(new DescribeLogGroupsRequest { NextToken = nextToken }); if (response.HttpStatusCode != HttpStatusCode.OK) { throw new InvalidOperationException($"Subscription '{originalSubscription.Id}' - " + $"Could not query AWS CloudWatch Logs for list of log groups"); } nextToken = response.NextToken; foreach (var logGroup in response.LogGroups.OrEmpty()) { if (!regex.IsMatch(logGroup.LogGroupName)) { continue; } var clonedSubscription = originalSubscription.Clone(false); clonedSubscription.LogGroupPattern = null; clonedSubscription.LogGroupName = logGroup.LogGroupName; clonedSubscription.Id += "." + ConfigurationIdCleanupRegex.Replace(logGroup.LogGroupName, "_"); expandedList.Add(clonedSubscription); } }while (nextToken.HasValue()); Log.Logger.Information( "Expanded {OriginalId} into {ExpandedCount} subscriptions for log groups: {ExpandedList}", originalSubscription.Id, expandedList.Count, string.Join(", ", expandedList.Select(s => s.LogGroupName))); return(expandedList); }