private static JobRunnerContext BuildJobRunnerContext(SubscriptionConfiguration config)
{
var result = new JobRunnerContext
{
Id = config.Id,
EventFilterPattern = config.EventFilterPattern,
TargetUrl = config.TargetUrl,
AwsRegion = config.AwsRegion,
LogGroupName = config.LogGroupName,
LogStreamNames = config.LogStreamNames,
LogStreamNamePrefix = config.LogStreamNamePrefix,
ReadMaxBatchSize = config.ReadMaxBatchSize.GetValueOrDefault(),
MinIntervalSeconds = config.MinIntervalSeconds.GetValueOrDefault(),
MaxIntervalSeconds = config.MaxIntervalSeconds.GetValueOrDefault(),
ClockSkewProtectionSeconds = config.ClockSkewProtectionSeconds.GetValueOrDefault(),
TargetMaxBatchSize = config.TargetMaxBatchSize.GetValueOrDefault(),
TargetSubscriptionData = config.TargetSubscriptionData,
StartInstant = ParseAbsoluteOrRelativeTime(config.StartTimeIso, config.StartTimeSecondsAgo) ?? InstantUtils.Now,
EndInstant = ParseAbsoluteOrRelativeTime(config.EndTimeIso, config.EndTimeSecondsAgo),
Logger = Log.Logger.ForContext("SubscriptionId", config.Id),
AwsClient = new AmazonCloudWatchLogsClient(RegionEndpoint.GetBySystemName(config.AwsRegion)),
HttpClient = new HttpClient(),
Random = new Random(),
NextInputBatchSequence = 1
};
result.HttpClient.Timeout = TimeSpan.FromSeconds(config.TargetTimeoutSeconds.GetValueOrDefault());
return(result);
}
private static void ValidateSubscription(SubscriptionConfiguration config)
{
if (config.Id.IsNullOrWhitespace())
{
throw new ArgumentException("Subscription ID is required for all subscriptions.");
}
if (config.Id.Length > 100)
{
throw new ArgumentException("Subscription ID cannot be longer than 100 characters.");
}
if (!ConfigurationIdRegex.IsMatch(config.Id))
{
throw new ArgumentException($"Subscription '{config.Id}' - Invalid characters in Subscription ID. Allowed characters are alpha numeric, dash, underscore and dot.");
}
if (config.TargetUrl.IsNullOrWhitespace())
{
throw new ArgumentException($"Subscription '{config.Id}' - targetUrl is required to be specified");
}
if (config.AwsRegion.IsNullOrWhitespace())
{
throw new ArgumentException($"Subscription '{config.Id}' - awsRegion is required to be specified");
}
if (config.LogGroupName.IsNullOrWhitespace())
{
throw new ArgumentException($"Subscription '{config.Id}' - logGroupName is required to be specified");
}
if (config.LogGroupPattern.HasValue())
{
throw new ArgumentException($"Subscription '{config.Id}' - logGroupPattern should have been expanded at this point and set to null");
}
if (config.LogStreamNames != null &&
config.LogStreamNames.SafeAny(lsn => lsn.HasValue()) &&
config.LogStreamNamePrefix.HasValue())
{
throw new ArgumentException($"Subscription '{config.Id}' - logStreamNames and logStreamNamePrefix fields cannot be specified at the same time. AWS API allows only one of them to be specified in each call.");
}
if (config.StartTimeIso.HasValue() && config.StartTimeSecondsAgo.HasValue)
{
throw new ArgumentException($"Subscription '{config.Id}' - startTimeIso and startTimeSecondsAgo cannot be specified at the same time. Only one of them allowed to be present in the configuration.");
}
if (config.EndTimeIso.HasValue() && config.EndTimeSecondsAgo.HasValue)
{
throw new ArgumentException($"Subscription '{config.Id}' - endTimeIso and endTimeSecondsAgo cannot be specified at the same time. Only one of them allowed to be present in the configuration.");
}
if (config.Children != null)
{
throw new ArgumentException($"Subscription '{config.Id}' - children need to be flattened in this point and Children property must have been set to null.");
}
}
private static void CoerceSubscription(SubscriptionConfiguration config)
{
if (config.EventFilterPattern.IsNullOrWhitespace())
{
config.EventFilterPattern = null;
}
if (config.LogStreamNamePrefix.IsNullOrWhitespace())
{
config.LogStreamNamePrefix = null;
}
if (config.LogStreamNames.SafeAny())
{
config.LogStreamNames = config.LogStreamNames.Where(sn => sn.HasValue()).ToList();
if (!config.LogStreamNames.Any())
{
config.LogStreamNames = null;
}
}
config.Enabled ??= true;
config.ReadMaxBatchSize ??= 10000;
config.MinIntervalSeconds ??= 15;
config.MaxIntervalSeconds ??= 300;
config.ClockSkewProtectionSeconds ??= 15;
config.TargetTimeoutSeconds ??= 60;
config.TargetMaxBatchSize ??= 1;
config.ReadMaxBatchSize = Math.Max(config.ReadMaxBatchSize.Value, 1);
config.ReadMaxBatchSize = Math.Min(config.ReadMaxBatchSize.Value, 10000);
config.MinIntervalSeconds = Math.Max(config.MinIntervalSeconds.Value, 5);
config.MinIntervalSeconds = Math.Min(config.MinIntervalSeconds.Value, 10 * 60);
config.MaxIntervalSeconds = Math.Max(config.MaxIntervalSeconds.Value, config.MinIntervalSeconds.Value);
config.MaxIntervalSeconds = Math.Min(config.MaxIntervalSeconds.Value, 8 * 60 * 60);
config.ClockSkewProtectionSeconds = Math.Max(config.ClockSkewProtectionSeconds.Value, 5);
config.ClockSkewProtectionSeconds = Math.Min(config.ClockSkewProtectionSeconds.Value, 120);
config.TargetTimeoutSeconds = Math.Max(config.TargetTimeoutSeconds.Value, 1);
config.TargetTimeoutSeconds = Math.Min(config.TargetTimeoutSeconds.Value, 15 * 60);
config.TargetMaxBatchSize = Math.Max(config.TargetMaxBatchSize.Value, 1);
config.TargetMaxBatchSize = Math.Min(config.TargetMaxBatchSize.Value, 20000);
if (config.Enabled.GetValueOrDefault())
{
DependencyContext.RunnerContexts.Add(config.Id, BuildJobRunnerContext(config));
}
}
private static void FlattenSubscriptionItem(List <SubscriptionConfiguration> flattenedList,
SubscriptionConfiguration parent, SubscriptionConfiguration item)
{
var extendedItem = item.Clone(false);
extendedItem.ExtendFrom(parent);
extendedItem.Children = null;
if (item.Children.SafeAny())
{
foreach (var child in item.Children)
{
FlattenSubscriptionItem(flattenedList, extendedItem, child);
}
}
else
{
flattenedList.Add(extendedItem);
}
}
public void ExtendFrom(SubscriptionConfiguration parent)
{
if (parent == null)
{
return;
}
if (parent.Id.HasValue())
{
Id = Id.HasValue() ? parent.Id + "." + Id : parent.Id;
}
Enabled ??= parent.Enabled;
Comments = Comments.Or(parent.Comments);
AwsRegion = AwsRegion.Or(parent.AwsRegion);
LogGroupName = LogGroupName.Or(parent.LogGroupName);
LogGroupPattern = LogGroupPattern.Or(parent.LogGroupPattern);
StartTimeIso = StartTimeIso.Or(parent.StartTimeIso);
EndTimeIso = EndTimeIso.Or(parent.EndTimeIso);
StartTimeSecondsAgo ??= parent.StartTimeSecondsAgo;
EndTimeSecondsAgo ??= parent.EndTimeSecondsAgo;
EventFilterPattern = EventFilterPattern.Or(parent.EventFilterPattern);
LogStreamNamePrefix = LogStreamNamePrefix.Or(parent.LogStreamNamePrefix);
if (parent.LogStreamNames.SafeAny())
{
LogStreamNames = LogStreamNames.SafeUnion(parent.LogStreamNames).ToList();
}
ReadMaxBatchSize ??= parent.ReadMaxBatchSize;
MinIntervalSeconds ??= parent.MinIntervalSeconds;
MaxIntervalSeconds ??= parent.MaxIntervalSeconds;
ClockSkewProtectionSeconds ??= parent.ClockSkewProtectionSeconds;
TargetUrl = TargetUrl.Or(parent.TargetUrl);
TargetTimeoutSeconds ??= parent.TargetTimeoutSeconds;
TargetMaxBatchSize ??= parent.TargetMaxBatchSize;
TargetSubscriptionData = TargetSubscriptionData.Or(parent.TargetSubscriptionData);
}
private static async Task <List <SubscriptionConfiguration> > ExpandSubscriptionPattern(SubscriptionConfiguration originalSubscription)
{
var cw = new AmazonCloudWatchLogsClient(RegionEndpoint.GetBySystemName(originalSubscription.AwsRegion));
var regex = new Regex(originalSubscription.LogGroupPattern, RegexOptions.IgnoreCase);
var expandedList = new List <SubscriptionConfiguration>();
var nextToken = (string)null;
do
{
var response = await cw.DescribeLogGroupsAsync(new DescribeLogGroupsRequest { NextToken = nextToken });
if (response.HttpStatusCode != HttpStatusCode.OK)
{
throw new InvalidOperationException($"Subscription '{originalSubscription.Id}' - " +
$"Could not query AWS CloudWatch Logs for list of log groups");
}
nextToken = response.NextToken;
foreach (var logGroup in response.LogGroups.OrEmpty())
{
if (!regex.IsMatch(logGroup.LogGroupName))
{
continue;
}
var clonedSubscription = originalSubscription.Clone(false);
clonedSubscription.LogGroupPattern = null;
clonedSubscription.LogGroupName = logGroup.LogGroupName;
clonedSubscription.Id += "." + ConfigurationIdCleanupRegex.Replace(logGroup.LogGroupName, "_");
expandedList.Add(clonedSubscription);
}
}while (nextToken.HasValue());
Log.Logger.Information(
"Expanded {OriginalId} into {ExpandedCount} subscriptions for log groups: {ExpandedList}",
originalSubscription.Id,
expandedList.Count,
string.Join(", ", expandedList.Select(s => s.LogGroupName)));
return(expandedList);
}
