private void ExecuteRestrictedToMinimumLevelTest(string logGroupName)
{
// Log 4 Debug messages
for (int i = 0; i < 3; i++)
{
Log.Debug(string.Format("Test logging message {0} SeriLog, Thread Id:{1}", i, Thread.CurrentThread.ManagedThreadId));
}
// Log 5 Error messages
for (int i = 0; i < 5; i++)
{
Log.Error(string.Format("Test logging message {0} SeriLog, Thread Id:{1}", i, Thread.CurrentThread.ManagedThreadId));
}
Log.Error(LASTMESSAGE);
GetLogEventsResponse getLogEventsResponse = new GetLogEventsResponse();
if (NotifyLoggingCompleted("AWSSeriLogGroupRestrictedtoMinimumLevel", "LASTMESSAGE"))
{
DescribeLogStreamsResponse describeLogstreamsResponse = Client.DescribeLogStreamsAsync(new DescribeLogStreamsRequest
{
Descending = true,
LogGroupName = logGroupName,
OrderBy = "LastEventTime"
}).Result;
getLogEventsResponse = Client.GetLogEventsAsync(new GetLogEventsRequest
{
LogGroupName = logGroupName,
LogStreamName = describeLogstreamsResponse.LogStreams[0].LogStreamName
}).Result;
}
Assert.Equal(6, getLogEventsResponse.Events.Count);
}
public async Task DescribeLogStreamsTest_TwoResponses()
{
var request = new DescribeLogStreamsRequest();
var firstResponse = new DescribeLogStreamsResponse()
{
NextToken = "foo"
};
var secondResponse = new DescribeLogStreamsResponse()
{
NextToken = null
};
var token = new CancellationToken();
_mockClient.SetupSequence(x => x.DescribeLogStreamsAsync(It.IsAny <DescribeLogStreamsRequest>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(firstResponse)
.ReturnsAsync(secondResponse);
var numResponses = 0;
await foreach (var desination in _mockClient.Object.Paginators.DescribeLogStreams(request).Responses.WithCancellation(token))
{
numResponses += 1;
}
Assert.Equal(2, numResponses);
}
public void MessageHasToBeBrokenUp()
{
string logGroupName = "AWSNLogGroupEventSizeExceededTest";
CreateLoggerFromConfiguration("AWSNLogGroupEventSizeExceededTest.config");
Logger = LogManager.GetLogger("loggerRegularEventSizeExceeded");
// This will get broken up into 3 CloudWatch Log messages
Logger.Debug(new string('a', 600000));
Logger.Debug(LASTMESSAGE);
GetLogEventsResponse getLogEventsResponse = new GetLogEventsResponse();
if (NotifyLoggingCompleted(logGroupName, "LASTMESSAGE"))
{
DescribeLogStreamsResponse describeLogstreamsResponse =
Client.DescribeLogStreamsAsync(new DescribeLogStreamsRequest
{
Descending = true,
LogGroupName = logGroupName,
OrderBy = "LastEventTime"
}).Result;
// Wait for the large messages to propagate
Thread.Sleep(5000);
getLogEventsResponse = Client.GetLogEventsAsync(new GetLogEventsRequest
{
LogGroupName = logGroupName,
LogStreamName = describeLogstreamsResponse.LogStreams[0].LogStreamName
}).Result;
}
_testFixture.LogGroupNameList.Add(logGroupName);
Assert.Equal(4, getLogEventsResponse.Events.Count());
}
public void SimpleLoggingTest(string logGroupName)
{
LogMessages(SIMPLELOGTEST_COUNT);
GetLogEventsResponse getLogEventsResponse = new GetLogEventsResponse();
if (NotifyLoggingCompleted(logGroupName, "LASTMESSAGE"))
{
DescribeLogStreamsResponse describeLogstreamsResponse =
Client.DescribeLogStreamsAsync(new DescribeLogStreamsRequest
{
Descending = true,
LogGroupName = logGroupName,
OrderBy = "LastEventTime"
}).Result;
getLogEventsResponse = Client.GetLogEventsAsync(new GetLogEventsRequest
{
LogGroupName = logGroupName,
LogStreamName = describeLogstreamsResponse.LogStreams[0].LogStreamName
}).Result;
var customStreamSuffix = describeLogstreamsResponse.LogStreams[0].LogStreamName.Split('-').Last().Trim();
Assert.Equal(CUSTOMSTREAMSUFFIX, customStreamSuffix);
}
Assert.Equal(SIMPLELOGTEST_COUNT, getLogEventsResponse.Events.Count());
_testFixture.LogGroupNameList.Add(logGroupName);
}
public void Log4Net()
{
XmlConfigurator.Configure(new System.IO.FileInfo("log4net.config"));
logger = LogManager.GetLogger("Log4Net");
for (int i = 0; i < 10; i++)
{
logger.Debug(string.Format("Test logging message {0} Log4Net", i));
}
//Added Sleep to give sufficient time for the log stream to get posted on CloudWatch
Thread.Sleep(10000);
string region = "us-west-2";
string logGroupName = "AWSLog4NetGroupLog4Net";
AmazonCloudWatchLogsClient client = new AmazonCloudWatchLogsClient(
Amazon.RegionEndpoint.GetBySystemName(region));
DescribeLogStreamsResponse describeLogstreamsResponse = client.DescribeLogStreamsAsync(new DescribeLogStreamsRequest
{
Descending = true,
LogGroupName = logGroupName,
OrderBy = "LastEventTime"
}).Result;
GetLogEventsResponse getLogEventsResponse = client.GetLogEventsAsync(new GetLogEventsRequest
{
LogGroupName = logGroupName,
LogStreamName = describeLogstreamsResponse.LogStreams[0].LogStreamName
}).Result;
Assert.Equal(10, getLogEventsResponse.Events.Count());
}
public void ILogger()
{
LoggerConfigSectionSetup("appsettings.json", null);
var logger = LoggerSetup();
for (int i = 0; i < 10; i++)
{
logger.LogDebug(string.Format("Test logging message {0} Ilogger", i));
}
//Sleep is introduced to give suffiecient time for the logstream to get posted on CloudWatchLogs
Thread.Sleep(5000);
string region = _configSection.Config.Region;
string logGroupName = _configSection.Config.LogGroup;
AmazonCloudWatchLogsClient client = new AmazonCloudWatchLogsClient(
Amazon.RegionEndpoint.GetBySystemName(region));
DescribeLogStreamsResponse describeLogstreamsResponse = client.DescribeLogStreamsAsync(new DescribeLogStreamsRequest
{
Descending = true,
LogGroupName = logGroupName,
OrderBy = "LastEventTime"
}).Result;
GetLogEventsResponse getLogEventsResponse = client.GetLogEventsAsync(new GetLogEventsRequest
{
LogGroupName = logGroupName,
LogStreamName = describeLogstreamsResponse.LogStreams[0].LogStreamName
}).Result;
Assert.Equal(10, getLogEventsResponse.Events.Count());
}
/// <summary>
/// Unmarshaller the response from the service to the response class.
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
DescribeLogStreamsResponse response = new DescribeLogStreamsResponse();
context.Read();
int targetDepth = context.CurrentDepth;
while (context.ReadAtDepth(targetDepth))
{
if (context.TestExpression("logStreams", targetDepth))
{
var unmarshaller = new ListUnmarshaller <LogStream, LogStreamUnmarshaller>(LogStreamUnmarshaller.Instance);
response.LogStreams = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("nextToken", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.NextToken = unmarshaller.Unmarshall(context);
continue;
}
}
return(response);
}
protected virtual async Task GetSequenceTokenAsync(string logStreamName, bool createStreamIfNotExists)
{
// Failover
if (_failoverSinkEnabled)
{
// Failover to Secondary Region
_ = FailOverToSecondaryRegion(_throttle);
}
try
{
var request = new DescribeLogStreamsRequest
{
LogGroupName = _logGroupName,
LogStreamNamePrefix = logStreamName
};
DescribeLogStreamsResponse describeLogsStreamsResponse = null;
try
{
describeLogsStreamsResponse = await CloudWatchLogsClient.DescribeLogStreamsAsync(request);
}
catch (ResourceNotFoundException rex)
{
// Create the log group if it doesn't exist.
if (rex.Message.IndexOf("log group does not exist") > -1)
{
_logger?.LogInformation("Log group {0} does not exist. Creating it.", _logGroupName);
await CreateLogGroupAsync();
if (createStreamIfNotExists)
{
await CreateLogStreamAsync(logStreamName);
}
return;
}
}
var logStream = describeLogsStreamsResponse.LogStreams
.FirstOrDefault(s => s.LogStreamName.Equals(logStreamName, StringComparison.CurrentCultureIgnoreCase));
if (logStream == null)
{
if (createStreamIfNotExists)
{
await CreateLogStreamAsync(logStreamName);
}
}
else
{
_sequenceToken = logStream.UploadSequenceToken;
}
}
catch (Exception)
{
// Set error count
_throttle.SetError();
throw;
}
}
public LogStream GetLogStream(LogGroup logGroup, string logStream = null)
{
List <LogStream> allStreams = new List <LogStream>();
DescribeLogStreamsResponse lsResponse = null;
do
{
DebugLogger.WriteLine("Getting logStreams...");
lsResponse = client.DescribeLogStreams(new DescribeLogStreamsRequest
{
NextToken = (lsResponse != null ? lsResponse.NextToken : null),
LogGroupName = logGroup.LogGroupName
});
allStreams.AddRange(lsResponse.LogStreams);
DebugLogger.WriteLine("Got logStreams, have {0}, {1} more pages", allStreams.Count, (!string.IsNullOrWhiteSpace(lsResponse.NextToken) ? "still" : "no"));
} while (!string.IsNullOrWhiteSpace(lsResponse.NextToken));
if (string.IsNullOrWhiteSpace(logStream) || logStream[logStream.Length - 1] == '*')
{
if (!string.IsNullOrWhiteSpace(logStream))
{
logStream = logStream.Substring(0, logStream.Length - 1);
allStreams = allStreams.Where(x => x.LogStreamName.StartsWith(logStream)).ToList();
}
allStreams = allStreams.OrderByDescending(x => x.CreationTime).ToList();
for (int i = 0, len = allStreams.Count; i < len; ++i)
{
Console.WriteLine(i + ") " + allStreams[i].LogStreamName);
}
int num = ReadIntBetween("Choose log stream: ", -1, allStreams.Count - 1);
if (num == -1)
{
if (logStream[logStream.Length - 1] == '*')
{
return(CreateLogStream(logGroup));
}
}
Console.Clear();
Console.WriteLine("You choose LogGroup: " + logGroup.LogGroupName + Environment.NewLine + "You choose LogStream: " + allStreams[num].LogStreamName);
return(allStreams[num]);
}
var ls = allStreams.FirstOrDefault(x => x.LogStreamName == logStream);
if (ls == null)
{
Console.WriteLine("The log stream '" + logGroup + "' does not exist.");
ls = CreateLogStream(logGroup);
}
Console.WriteLine("You choose LogStream: " + ls.LogStreamName);
return(ls);
}
/// <summary>
/// The collecting process in a thread. when done is will set the Done bool to true
/// </summary>
/// <param name="LogGroup">AWS LogGroup </param>
/// <param name="LogStreamRespon">DescribeLogStreamsResponse logsteam from loggroup</param>
/// <param name="From">TimeSpan date from</param>
/// <param name="To">TimeSpan date to</param>
/// <param name="CSV_filepath">Path to where the log file shall be saved</param>
private void CollectLogDataThread(string LogGroup, DescribeLogStreamsResponse LogStreamRespon, TimeSpan From, TimeSpan To, string CSV_filepath)
{
Logfile l = new Logfile(CSV_filepath);
List <GetLogEventsResponse> Logs = awslog.GetLogsFrom(LogGroup, LogStreamRespon, From.TotalDays, To.TotalDays);
l.write(Logs);
Done = true;
}
/// <summary>
/// When user selecting a Loggroup is will get all logstreams from that loggroup
/// </summary>
private void cbox_LogGroups_SelectionChanged(object sender, SelectionChangedEventArgs e)
{
btn_SetTimeSpan.IsEnabled = true;
dp_datefrom.IsEnabled = true;
dp_dateto.IsEnabled = true;
dp_datefrom.SelectedDate = DateTime.Today;
dp_dateto.SelectedDate = DateTime.Today;
LogStreamRespon = vbn.GetLogStreams(cbox_LogGroups.SelectedValue.ToString());
}
public void MultiThreadTest(string logGroupName)
{
var tasks = new List <Task>();
var streamNames = new List <string>();
var count = MULTITHREADTEST_COUNT;
var totalCount = 0;
for (int i = 0; i < THREAD_COUNT; i++)
{
tasks.Add(Task.Factory.StartNew(() => LogMessages(count)));
totalCount = totalCount + count;
}
Task.WaitAll(tasks.ToArray(), TimeSpan.FromSeconds(THREAD_WAITTIME));
int testCount = -1;
if (NotifyLoggingCompleted(logGroupName, "LASTMESSAGE"))
{
DescribeLogStreamsResponse describeLogstreamsResponse =
Client.DescribeLogStreamsAsync(new DescribeLogStreamsRequest
{
Descending = true,
LogGroupName = logGroupName,
OrderBy = "LastEventTime"
}).Result;
if (describeLogstreamsResponse.LogStreams.Count > 0)
{
testCount = 0;
foreach (var logStream in describeLogstreamsResponse.LogStreams)
{
GetLogEventsResponse getLogEventsResponse =
Client.GetLogEventsAsync(new GetLogEventsRequest
{
LogGroupName = logGroupName,
LogStreamName = logStream.LogStreamName
}).Result;
if (getLogEventsResponse != null)
{
testCount += getLogEventsResponse.Events.Count();
}
}
}
}
Assert.Equal(totalCount, testCount);
_testFixture.LogGroupNameList.Add(logGroupName);
}
public void MultiThreadBufferFullTest()
{
global::NLog.Logger logger;
try
{
LogManager.Configuration = new XmlLoggingConfiguration("Regular.config");
}
catch (FileNotFoundException)
{
LogManager.Configuration = new XmlLoggingConfiguration("./test/AWS.Logger.NLog.Tests/Regular.config");
}
logger = LogManager.GetLogger("loggerMultiThreadBufferFull");
var tasks = new List <Task>();
var streamNames = new List <string>();
var count = 200;
var totcount = 0;
for (int i = 0; i < 2; i++)
{
tasks.Add(Task.Factory.StartNew(() => NLogThread(count, logger)));
totcount = totcount + count;
}
Task.WaitAll(tasks.ToArray());
string region = "us-west-2";
string logGroupName = "AWSNLogGroupMultiThreadBufferFullTest";
Thread.Sleep(10000);
AmazonCloudWatchLogsClient client = new AmazonCloudWatchLogsClient(
Amazon.RegionEndpoint.GetBySystemName(region));
DescribeLogStreamsResponse describeLogstreamsResponse = client.DescribeLogStreamsAsync(new DescribeLogStreamsRequest
{
Descending = true,
LogGroupName = logGroupName,
OrderBy = "LastEventTime"
}).Result;
List <string> logStreamNames = new List <string>();
logStreamNames.Add(describeLogstreamsResponse.LogStreams[0].LogStreamName);
FilterLogEventsResponse filterLogEventsResponse = client.FilterLogEventsAsync(new FilterLogEventsRequest
{
FilterPattern = "maximum",
LogGroupName = logGroupName,
LogStreamNames = logStreamNames
}).Result;
Assert.NotEmpty(filterLogEventsResponse.Events);
}
public async Task DescribeLogStreamsTest__OnlyUsedOnce()
{
var request = new DescribeLogStreamsRequest();
var response = new DescribeLogStreamsResponse();
var token = new CancellationToken();
var paginator = _mockClient.Object.Paginators.DescribeLogStreams(request);
_mockClient.Setup(x => x.DescribeLogStreamsAsync(It.IsAny <DescribeLogStreamsRequest>(), It.IsAny <CancellationToken>())).ReturnsAsync(response);
// Should work the first time
await LoopOverLogStreams(paginator, token);
// Second time should throw an exception
await Assert.ThrowsAsync <System.InvalidOperationException>(async() => await LoopOverLogStreams(paginator, token));
}
static async Task Main(string[] args)
{
var accessKey = ConfigurationManager.AppSettings["accessKey"];
var secretAccessKey = ConfigurationManager.AppSettings["secretAccessKey"];
var region = Amazon.RegionEndpoint.GetBySystemName(ConfigurationManager.AppSettings["region"]);
var groupName = ConfigurationManager.AppSettings["groupName"];
var cloudWatchLogClient = new AmazonCloudWatchLogsClient(accessKey, secretAccessKey, region);
var reqestCounter = 0;
BigInteger firstLogStreamName = 0;
DescribeLogStreamsResponse lastResponse = new DescribeLogStreamsResponse();
var isEndOfStream = false;
while (isEndOfStream == false)
{
var describeLogStreamsRequest = (reqestCounter == 0)
? new DescribeLogStreamsRequest(groupName)
{
Limit = 50
}
: new DescribeLogStreamsRequest(groupName)
{
Limit = 50, NextToken = lastResponse.NextToken
};
var describeLogStreamsResponse = await cloudWatchLogClient.DescribeLogStreamsAsync(describeLogStreamsRequest);
var logStreams = describeLogStreamsResponse.LogStreams;
for (int j = 0; j < logStreams.Count; j++)
{
//0は正の符号
var i = BigInteger.Parse("0" + logStreams[j].LogStreamName, NumberStyles.HexNumber);
if (firstLogStreamName < i)
{
Console.WriteLine(logStreams[j].LogStreamName);
}
else
{
isEndOfStream = true;
break;
}
}
firstLogStreamName = (reqestCounter == 0)
? BigInteger.Parse("0" + describeLogStreamsResponse.LogStreams[0].LogStreamName, NumberStyles.HexNumber)
: firstLogStreamName;
lastResponse = describeLogStreamsResponse;
reqestCounter++;
}
Console.ReadKey();
}
/// <summary>
/// Get and write logs to chosen file, This is not at right way to do it, but every task, thread, async/await either lock the GUI or is not working.
/// </summary>
/// <param name="awsLogGroupName"></param>
/// <param name="awsLogStreamName"></param>
/// <param name="fromdays"></param>
/// <param name="todays"></param>
public List <GetLogEventsResponse> GetLogsFrom(string awsLogGroupName, DescribeLogStreamsResponse awsLogStreamName, double fromdays, double todays)
{
List <GetLogEventsResponse> Logs = new List <GetLogEventsResponse>();
foreach (var item in awsLogStreamName.LogStreams)
{
awsGetLogsEvent = new GetLogEventsRequest();
awsGetLogsEvent.LogGroupName = awsLogGroupName;
awsGetLogsEvent.LogStreamName = item.LogStreamName;
awsGetLogsEvent.StartTime = DateTime.Today.AddDays(fromdays);
awsGetLogsEvent.EndTime = DateTime.Today.AddDays(todays + 1);
Logs.Add(awsCloudwatchClient.GetLogEvents(awsGetLogsEvent));
}
return(Logs);
}
private async Task GetSequenceTokenAsync(string logStreamName, bool createStreamIfNotExists)
{
var request = new DescribeLogStreamsRequest
{
LogGroupName = _logGroupName,
LogStreamNamePrefix = logStreamName
};
DescribeLogStreamsResponse describeLogsStreamsResponse = null;
try
{
describeLogsStreamsResponse = await _client.DescribeLogStreamsAsync(request);
}
catch (ResourceNotFoundException rex)
{
// Create the log group if it doesn't exist.
if (rex.Message.IndexOf("log group does not exist") > -1)
{
_logger?.LogInformation("Log group {0} does not exist. Creating it.", _logGroupName);
await this.CreateLogGroupAsync();
if (createStreamIfNotExists)
{
await this.CreateLogStreamAsync(logStreamName);
}
return;
}
}
var logStream = describeLogsStreamsResponse.LogStreams
.FirstOrDefault(s => s.LogStreamName.Equals(logStreamName, StringComparison.CurrentCultureIgnoreCase));
if (logStream == null)
{
if (createStreamIfNotExists)
{
await this.CreateLogStreamAsync(logStreamName);
}
}
else
{
_sequenceToken = logStream.UploadSequenceToken;
}
}
public void MultiThreadBufferFullTest()
{
var tasks = new List <Task>();
var streamNames = new List <string>();
ILoggerMultiThreadTestSetup("multiThreadBufferFullTest.json");
logger = ILoggerTest.LoggerSetup();
var count = 200;
var totcount = 0;
for (int i = 0; i < 2; i++)
{
tasks.Add(Task.Factory.StartNew(() => ILoggerThread(count)));
totcount = totcount + count;
}
Task.WaitAll(tasks.ToArray(), 10000);
//Sleep is introduced to give suffiecient time for the logstream to get posted on CloudWatchLogs
Thread.Sleep(5000);
string region = ILoggerTest._configSection.Config.Region;
string logGroupName = ILoggerTest._configSection.Config.LogGroup;
AmazonCloudWatchLogsClient client = new AmazonCloudWatchLogsClient(
Amazon.RegionEndpoint.GetBySystemName(region));
DescribeLogStreamsResponse describeLogstreamsResponse = client.DescribeLogStreamsAsync(new DescribeLogStreamsRequest
{
Descending = true,
LogGroupName = logGroupName,
OrderBy = "LastEventTime"
}).Result;
List <string> logStreamNames = new List <string>();
logStreamNames.Add(describeLogstreamsResponse.LogStreams[0].LogStreamName);
FilterLogEventsResponse filterLogEventsResponse = client.FilterLogEventsAsync(new FilterLogEventsRequest
{
FilterPattern = "maximum",
LogGroupName = logGroupName,
LogStreamNames = logStreamNames
}).Result;
Assert.NotEmpty(filterLogEventsResponse.Events);
}
public void MultiThreadBufferFullTest()
{
ILog logger;
XmlConfigurator.Configure(new System.IO.FileInfo("log4net.config"));
logger = LogManager.GetLogger("MultiThreadBufferFullTest");
var tasks = new List <Task>();
var count = 200;
var totcount = 0;
for (int i = 0; i < 2; i++)
{
tasks.Add(Task.Factory.StartNew(() => Log4NetThread(count, logger)));
totcount = totcount + count;
}
Task.WaitAll(tasks.ToArray(), 10000);
Thread.Sleep(10000);
string region = "us-west-2";
string logGroupName = "AWSLog4NetGroupMultiThreadBufferFullTest";
AmazonCloudWatchLogsClient client = new AmazonCloudWatchLogsClient(
Amazon.RegionEndpoint.GetBySystemName(region));
DescribeLogStreamsResponse describeLogstreamsResponse = client.DescribeLogStreamsAsync(new DescribeLogStreamsRequest
{
Descending = true,
LogGroupName = logGroupName,
OrderBy = "LastEventTime"
}).Result;
List <string> logStreamNames = new List <string>();
logStreamNames.Add(describeLogstreamsResponse.LogStreams[0].LogStreamName);
FilterLogEventsResponse filterLogEventsResponse = client.FilterLogEventsAsync(new FilterLogEventsRequest
{
FilterPattern = "maximum",
LogGroupName = logGroupName,
LogStreamNames = logStreamNames
}).Result;
Assert.NotEmpty(filterLogEventsResponse.Events);
}
public void MultiThreadTest()
{
ILog logger;
XmlConfigurator.Configure(new System.IO.FileInfo("log4net.config"));
logger = LogManager.GetLogger("MultiThreadTest");
var log = LogManager.GetCurrentLoggers();
var tasks = new List <Task>();
var count = 200;
var totcount = 0;
for (int i = 0; i < 2; i++)
{
tasks.Add(Task.Factory.StartNew(() => Log4NetThread(count, logger)));
totcount = totcount + count;
}
Task.WaitAll(tasks.ToArray(), 10000);
//Added Sleep to give sufficient time for the log stream to get posted on CloudWatch
Thread.Sleep(5000);
string region = "us-west-2";
string logGroupName = "AWSLog4NetGroupLog4NetMultiThreadTest";
AmazonCloudWatchLogsClient client = new AmazonCloudWatchLogsClient(
Amazon.RegionEndpoint.GetBySystemName(region));
DescribeLogStreamsResponse describeLogstreamsResponse = client.DescribeLogStreamsAsync(new DescribeLogStreamsRequest
{
Descending = true,
LogGroupName = logGroupName,
OrderBy = "LastEventTime"
}).Result;
GetLogEventsResponse getLogEventsResponse = client.GetLogEventsAsync(new GetLogEventsRequest
{
LogGroupName = logGroupName,
LogStreamName = describeLogstreamsResponse.LogStreams[0].LogStreamName
}).Result;
Assert.Equal(totcount, getLogEventsResponse.Events.Count());
}
public void awsGetLogStream() //Pull all LogStream from a LogGroup on a aws account
{
string awsLogGroup = "";
DescribeLogGroupsResponse LogGroupRespon = awsC.GetLogGroups();
foreach (var item in LogGroupRespon.LogGroups)
{
if (item.LogGroupName.Contains("ecs")) // /ecs/blade-*** is from the ML logs
{
awsLogGroup = item.LogGroupName;
break;
}
}
DescribeLogStreamsResponse LogStreamRespon = awsC.GetLogStreams(awsLogGroup);
Assert.IsNotNull(LogStreamRespon);
}
public void Nlog()
{
try
{
LogManager.Configuration = new XmlLoggingConfiguration("Regular.config");
}
catch (FileNotFoundException)
{
LogManager.Configuration = new XmlLoggingConfiguration("./test/AWS.Logger.NLog.Tests/Regular.config");
}
global::NLog.Logger logger = LogManager.GetLogger("loggerRegular");
for (int i = 0; i < 10; i++)
{
logger.Debug(string.Format("Test logging message {0} NLog, Thread Id:{1}", i, Thread.CurrentThread.ManagedThreadId));
}
string region = "us-west-2";
string logGroupName = "AWSNLogGroup";
Thread.Sleep(10000);
AmazonCloudWatchLogsClient client = new AmazonCloudWatchLogsClient(
Amazon.RegionEndpoint.GetBySystemName(region));
DescribeLogStreamsResponse describeLogstreamsResponse = client.DescribeLogStreamsAsync(new DescribeLogStreamsRequest
{
Descending = true,
LogGroupName = logGroupName,
OrderBy = "LastEventTime"
}).Result;
GetLogEventsResponse getLogEventsResponse = client.GetLogEventsAsync(new GetLogEventsRequest
{
LogGroupName = logGroupName,
LogStreamName = describeLogstreamsResponse.LogStreams[0].LogStreamName
}).Result;
Assert.Equal(10, getLogEventsResponse.Events.Count());
}
[TestMethod] // May fail if there are no logs within the timespan that are set and may take some time to be done, if there is
public void awsGetLogsFrom() //Pull all Logs from a Logstream from a timespan on a aws account
{
string awsLogGroup = "";
int awsLogFrom = -20;
int awsLogTo = -15;
DescribeLogGroupsResponse LogGroupRespon = awsC.GetLogGroups();
foreach (var item in LogGroupRespon.LogGroups)
{
if (item.LogGroupName.Contains("/ecs/blade-seg")) // /ecs/blade-seg is from the ML logs
{
awsLogGroup = item.LogGroupName;
break;
}
}
DescribeLogStreamsResponse LogStreamRespon = awsC.GetLogStreams(awsLogGroup);
List <GetLogEventsResponse> Logs = awsC.GetLogsFrom(awsLogGroup, LogStreamRespon, awsLogFrom, awsLogTo);
bool containdata = false;
foreach (var log in Logs)
{
foreach (var item in log.Events)
{
if (!string.IsNullOrEmpty(item.Message))
{
containdata = true;
break;
}
}
}
Assert.IsTrue(containdata);
}
public async Task DescribeLogStreamsTest__CancellationToken()
{
var request = new DescribeLogStreamsRequest();
var firstResponse = new DescribeLogStreamsResponse()
{
NextToken = "foo"
};
var secondResponse = new DescribeLogStreamsResponse()
{
NextToken = null
};
var paginator = _mockClient.Object.Paginators.DescribeLogStreams(request);
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
tokenSource.Cancel();
_mockClient.SetupSequence(x => x.DescribeLogStreamsAsync(It.IsAny <DescribeLogStreamsRequest>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(firstResponse)
.ReturnsAsync(secondResponse);
await Assert.ThrowsAsync <OperationCanceledException>(async() => await LoopOverLogStreams(paginator, token));
}
/// <summary>
/// Starting the log collection process
/// </summary>
/// <param name="LogGroup">AWS LogGroup </param>
/// <param name="LogStreamRespon">DescribeLogStreamsResponse logsteam from loggroup</param>
/// <param name="From">TimeSpan date from</param>
/// <param name="To">TimeSpan date to</param>
/// <param name="CSV_filepath">Path to where the log file shall be saved</param>
public void CollectLogData(string LogGroup, DescribeLogStreamsResponse LogStreamRespon, TimeSpan From, TimeSpan To, string CSV_filepath)
{
new Thread(() => CollectLogDataThread(LogGroup, LogStreamRespon, From, To, CSV_filepath)).Start();
}
public void CRUDLogGroup()
{
var logGroupName = "sdk-dotnet-" + DateTime.Now.Ticks;
Client.CreateLogGroupAsync(new CreateLogGroupRequest
{
LogGroupName = logGroupName
}).Wait();
try
{
{
DescribeLogGroupsResponse response = Client.DescribeLogGroupsAsync(new DescribeLogGroupsRequest
{
LogGroupNamePrefix = logGroupName
}).Result;
Assert.AreEqual(1, response.LogGroups.Count);
Assert.IsNotNull(response.LogGroups[0].Arn);
Assert.IsNull(response.LogGroups[0].RetentionInDays);
Client.PutRetentionPolicyAsync(new PutRetentionPolicyRequest
{
LogGroupName = logGroupName,
RetentionInDays = 1
}).Wait();
response = Client.DescribeLogGroupsAsync(new DescribeLogGroupsRequest
{
LogGroupNamePrefix = logGroupName
}).Result;
Assert.AreEqual(1, response.LogGroups.Count);
Assert.AreEqual(1, response.LogGroups[0].RetentionInDays.GetValueOrDefault());
}
{
Client.CreateLogStreamAsync(new CreateLogStreamRequest
{
LogGroupName = logGroupName,
LogStreamName = "sample"
}).Wait();
DescribeLogStreamsResponse describeResponse = Client.DescribeLogStreamsAsync(new DescribeLogStreamsRequest
{
LogGroupName = logGroupName,
LogStreamNamePrefix = "sample"
}).Result;
Assert.AreEqual(1, describeResponse.LogStreams.Count);
Assert.IsNotNull(describeResponse.LogStreams[0].Arn);
PutLogEventsResponse putResponse1 = Client.PutLogEventsAsync(new PutLogEventsRequest
{
LogGroupName = logGroupName,
LogStreamName = "sample",
LogEvents = new List <InputLogEvent>
{
new InputLogEvent
{
Message = "First Data",
Timestamp = DateTime.Now
}
}
}).Result;
// Pad the time so the 2 events are not at the same time.
UtilityMethods.Sleep(TimeSpan.FromSeconds(.1));
Client.PutLogEventsAsync(new PutLogEventsRequest
{
SequenceToken = putResponse1.NextSequenceToken,
LogGroupName = logGroupName,
LogStreamName = "sample",
LogEvents = new List <InputLogEvent>
{
new InputLogEvent
{
Message = "Second Data",
Timestamp = DateTime.Now
}
}
}).Wait();
GetLogEventsResponse getResponse = null;
// Doing this in a loop to wait for the eventual consistency of the events
// being written to cloudwatch logs.
for (int i = 0; i < 20; i++)
{
getResponse = Client.GetLogEventsAsync(new GetLogEventsRequest
{
LogGroupName = logGroupName,
LogStreamName = "sample",
StartTime = DateTime.Now.AddDays(-2),
EndTime = DateTime.Now
}).Result;
if (getResponse.Events.Count == 2)
{
break;
}
UtilityMethods.Sleep(TimeSpan.FromSeconds(2));
}
Assert.AreEqual(2, getResponse.Events.Count);
Assert.AreEqual("First Data", getResponse.Events[0].Message);
Assert.AreEqual(DateTime.UtcNow.Date, getResponse.Events[0].Timestamp.Date);
Assert.AreEqual("Second Data", getResponse.Events[1].Message);
Assert.AreEqual(DateTime.UtcNow.Date, getResponse.Events[1].Timestamp.Date);
Assert.IsTrue(getResponse.Events[0].Timestamp < getResponse.Events[1].Timestamp);
Client.DeleteLogStreamAsync(new DeleteLogStreamRequest
{
LogGroupName = logGroupName,
LogStreamName = "sample"
}).Wait();
}
}
finally
{
Client.DeleteLogGroupAsync(new DeleteLogGroupRequest
{
LogGroupName = logGroupName
}).Wait();
}
}
public async Task DoWorkAsync()
{
_logger.LogInformation("Update Background Service is running");
try
{
_logger.LogInformation("Update Background Service is checking VPCs");
if (context.VPCs.Any())
{
DescribeVpcsResponse responseDescribeVPC = await ec2Client.DescribeVpcsAsync();
List <VPC> vpcs = await context.VPCs.ToListAsync();
foreach (VPC vpc in vpcs)
{
Boolean Flag = false;
foreach (Vpc AWSVPC in responseDescribeVPC.Vpcs)
{
if (vpc.AWSVPCReference.Equals(AWSVPC.VpcId))
{
Flag = true;
break;
}
}
if (Flag == false)
{
context.VPCs.Remove(vpc);
}
}
context.SaveChanges();
_logger.LogInformation("Update Background Service completed checking of VPCs");
}
if (context.VPCs.Any())
{
VPC vpc = await context.VPCs.FindAsync(1);
if (context.Servers.Any())
{
_logger.LogInformation("Update Background Service is checking servers");
List <Server> servers = await context.Servers.ToListAsync();
DescribeInstancesResponse response = await ec2Client.DescribeInstancesAsync(new DescribeInstancesRequest
{
Filters = new List <Filter>
{
new Filter {
Name = "vpc-id", Values = new List <string> {
vpc.AWSVPCReference
}
}
}
});
foreach (var server in servers)
{
foreach (var reservation in response.Reservations)
{
foreach (var instance in reservation.Instances)
{
bool Flag = false;
if (server.AWSEC2Reference.Equals(instance.InstanceId))
{
if (instance.State.Code == 0 && server.State != State.Starting)
{
server.State = State.Starting;
Flag = true;
}
else if (instance.State.Code == 16 && server.State != State.Running)
{
server.State = State.Running;
Flag = true;
}
else if (instance.State.Code == 64 && server.State != State.Stopping)
{
server.State = State.Stopping;
Flag = true;
}
else if (instance.State.Code == 80 && server.State != State.Stopped)
{
server.State = State.Stopped;
Flag = true;
}
if (server.Visibility == Visibility.Internet && (server.IPAddress != instance.PublicIpAddress || server.DNSHostname != instance.PublicDnsName) && server.State != State.Stopped)
{
server.IPAddress = instance.PublicIpAddress;
server.DNSHostname = instance.PublicDnsName;
Flag = true;
}
else if (server.Visibility == Visibility.Internet && (server.IPAddress != instance.PublicIpAddress || server.DNSHostname != instance.PublicDnsName))
{
server.IPAddress = "Public IP Address is not available when server is stopped";
server.DNSHostname = "Public DNS Hostname is not available when server is stopped";
Flag = true;
}
else if ((server.Visibility == Visibility.Extranet || server.Visibility == Visibility.Intranet) && (server.IPAddress != instance.PrivateIpAddress || server.DNSHostname != instance.PrivateDnsName))
{
server.IPAddress = instance.PrivateIpAddress;
server.DNSHostname = instance.PrivateDnsName;
Flag = true;
}
if (Flag == true)
{
context.Servers.Update(server);
}
break;
}
}
}
}
context.SaveChanges();
}
if (context.CloudWatchLogGroups.Any() && context.CloudWatchLogStreams.Any())
{
_logger.LogInformation("Update Background Service is checking cloudwatch groups");
List <CloudWatchLogGroup> allStreams = await context.CloudWatchLogGroups.ToListAsync();
foreach (CloudWatchLogGroup g in allStreams)
{
DescribeLogStreamsResponse response = await cwlClient.DescribeLogStreamsAsync(new DescribeLogStreamsRequest
{
LogGroupName = g.Name.Replace("@", "/")
});
foreach (LogStream ls in response.LogStreams)
{
Boolean Flag = false;
foreach (CloudWatchLogStream CWLS in g.LogStreams)
{
if (ls.Arn.Equals(CWLS.ARN))
{
Flag = true;
break;
}
}
if (Flag == false)
{
CloudWatchLogStream newS = new CloudWatchLogStream
{
ARN = ls.Arn,
CreationTime = ls.CreationTime,
FirstEventTime = ls.FirstEventTimestamp,
LastEventTime = ls.LastEventTimestamp,
Name = ls.LogStreamName,
LinkedGroupID = g.ID
};
if (g.Name.Equals("VMVPCLogs"))
{
newS.DisplayName = "Network Flow Log For Challenge Network Interface (" + newS.Name.Substring(0, newS.Name.Length - 4) + ")";
}
else if (g.Name.Equals("PlatformVPCLogs"))
{
newS.DisplayName = "Network Flow Log For Platform Network Interface (" + newS.Name.Substring(0, newS.Name.Length - 4) + ")";
}
else if (g.Name.Equals("@aws@elasticbeanstalk@User-Side@IIS-Log"))
{
newS.DisplayName = "IIS Logs for User Side Web Server";
}
else if (g.Name.Equals("@aws@elasticbeanstalk@Admin-Side@IIS-Log"))
{
newS.DisplayName = "IIS Logs for Admin Side Web Server";
}
else if (g.Name.Equals("@aws@elasticbeanstalk@User-Side@EBDeploy-Log"))
{
newS.DisplayName = "Elastic Beanstalk Deployment Tool Logs for User Side";
}
else if (g.Name.Equals("@aws@elasticbeanstalk@Admin-Side@EBDeploy-Log"))
{
newS.DisplayName = "Elastic Beanstalk Deployment Tool Logs for Admin Side";
}
else if (g.Name.Equals("@aws@elasticbeanstalk@User-Side@EBHooks-Log"))
{
newS.DisplayName = "Elastic Beanstalk Deployment Hook Logs for User Side";
}
else if (g.Name.Equals("@aws@elasticbeanstalk@Admin-Side@EBHooks-Log"))
{
newS.DisplayName = "Elastic Beanstalk Deployment Hook Logs for Admin Side";
}
else
{
newS.DisplayName = newS.Name;
}
if (!g.Name.Equals("RDSOSMetrics") || !g.Name.Contains("sns"))
{
context.CloudWatchLogStreams.Add(newS);
}
}
}
foreach (CloudWatchLogStream CWLS in g.LogStreams)
{
Boolean Flag = false;
foreach (LogStream ls in response.LogStreams)
{
if (CWLS.ARN.Equals(ls.Arn))
{
Flag = true;
break;
}
}
if (Flag == false)
{
context.CloudWatchLogStreams.Remove(CWLS);
}
}
}
await context.SaveChangesAsync();
}
if (context.Templates.Any())
{
DescribeSnapshotsResponse response = await ec2Client.DescribeSnapshotsAsync(new DescribeSnapshotsRequest());
List <Template> templates = await context.Templates.FromSql("SELECT * FROM dbo.Templates WHERE AWSSnapshotReference = NULL").ToListAsync();
foreach (Template t in templates)
{
foreach (Snapshot s in response.Snapshots)
{
if (s.Description.Contains(t.AWSAMIReference))
{
t.AWSSnapshotReference = s.SnapshotId;
break;
}
}
}
await context.SaveChangesAsync();
}
}
_logger.LogInformation("Update Background Service has completed!");
}
catch (SqlException e)
{
_logger.LogInformation("Update Background Service faced an SQL exception! " + e.Message + " | " + e.Source);
return;
}
catch (Exception e)
{
_logger.LogInformation("Update Background Service faced an exception! " + e.Message + " | " + e.Source);
return;
}
}
