public static async Task ListInstanceEventsAsync(
this EntriesResource entriesResource,
IEnumerable <string> projectIds,
DateTime startTime,
IEventProcessor processor)
{
using (TraceSources.LogAnalysis.TraceMethod().WithParameters(
string.Join(", ", projectIds),
startTime))
{
var request = new ListLogEntriesRequest()
{
ResourceNames = projectIds.Select(p => "projects/" + p).ToList(),
Filter = $"protoPayload.methodName=(\"{string.Join("\" OR \"", processor.SupportedMethods)}\") " +
$"AND severity=(\"{string.Join("\" OR \"", processor.SupportedSeverities)}\") " +
$"AND resource.type=\"gce_instance\" " +
$"AND timestamp > {startTime:yyyy-MM-dd} ",
PageSize = MaxPageSize,
OrderBy = "timestamp desc"
};
await ListEventsAsync(
entriesResource,
request,
processor.Process,
new ExponentialBackOff(initialBackOff, MaxRetries));
}
}
protected override void ProcessRecord()
{
ListLogEntriesRequest logEntriesRequest = new ListLogEntriesRequest();
// Set resource to "projects/{Project}" so we will only find log entries in project Project.
logEntriesRequest.ResourceNames = new List <string> {
$"projects/{Project}"
};
string logName = PrefixProjectToLogName(LogName, Project);
logEntriesRequest.Filter = ConstructLogFilterString(
logName: logName,
logSeverity: Severity,
selectedType: SelectedResourceType,
before: Before,
after: After,
otherFilter: Filter);
do
{
EntriesResource.ListRequest listLogRequest = Service.Entries.List(logEntriesRequest);
ListLogEntriesResponse response = listLogRequest.Execute();
if (response.Entries != null)
{
foreach (LogEntry logEntry in response.Entries)
{
WriteObject(logEntry);
}
}
logEntriesRequest.PageToken = response.NextPageToken;
}while (!Stopping && logEntriesRequest.PageToken != null);
}
public async Task WhenUsingInvalidProjectId_ThenListEventsAsyncThrowsException(
[Credential(Role = PredefinedRole.LogsViewer)] CredentialRequest credential)
{
var startDate = DateTime.UtcNow.AddDays(-30);
var request = new ListLogEntriesRequest()
{
ResourceNames = new[]
{
"projects/invalid"
},
Filter = $"resource.type=\"gce_instance\" " +
$"AND protoPayload.methodName:{InsertInstanceEvent.Method} " +
$"AND timestamp > {startDate:yyyy-MM-dd}",
PageSize = 1000,
OrderBy = "timestamp desc"
};
var adapter = new AuditLogAdapter(await credential.GetCredentialAsync());
AssertEx.ThrowsAggregateException <GoogleApiException>(
() => adapter.ListEventsAsync(
request,
_ => { },
new Apis.Util.ExponentialBackOff(),
CancellationToken.None).Wait());
}
/// <summary>
/// Returns the first page of log entries of the project id.
/// </summary>
/// <param name="filter">
/// Optional,
/// Refert to https://cloud.google.com/logging/docs/view/advanced_filters.
/// </param>
/// <param name="orderBy">
/// Optional, "timestamp desc" or "timestamp asc"
/// </param>
/// <param name="pageSize">
/// Optional,
/// If page size is not specified, a server side default value is used.
/// </param>
/// <param name="nextPageToken">
/// Optional,
/// The page token from last list request response.
/// If the value is null, fetch the first page results.
/// If the value is not null, it is hard requirement that the filter, orderBy and pageSize parameters
/// must stay same as the prior call.
/// </param>
/// <param name="cancelToken">Optional. A cancellation token.</param>
/// <returns>
/// <seealso ref="LogEntryRequestResult" /> object that contains log entries and next page token.
/// </returns>
public async Task <LogEntryRequestResult> ListLogEntriesAsync(
string filter = null, string orderBy = null, int?pageSize = null, string nextPageToken = null,
CancellationToken cancelToken = default(CancellationToken))
{
try
{
var requestData = new ListLogEntriesRequest
{
ResourceNames = _resourceNames,
Filter = filter,
OrderBy = orderBy,
PageSize = pageSize
};
requestData.PageToken = nextPageToken;
var response = await Service.Entries.List(requestData).ExecuteAsync(cancelToken);
return(new LogEntryRequestResult(response.Entries, response.NextPageToken));
}
catch (GoogleApiException ex)
{
Debug.WriteLine($"Failed to get log entries: {ex.Message}");
throw new DataSourceException(ex.Message, ex);
}
}
/// <summary>
/// Gets log entries that contain the passed in testId in the log message. Will poll
/// and wait for the entries to appear.
/// </summary>
/// <param name="startTime">The earliest log entry time that will be looked at.</param>
/// <param name="testId">The test id to filter log entries on.</param>
/// <param name="minEntries">The minimum number of logs entries that should be waited for.
/// If minEntries is zero this method will wait the full timeout before checking for the
/// entries.</param>
private IEnumerable <LogEntry> GetEntries(DateTime startTime, string testId, int minEntries)
{
TimeSpan totalSleepTime = TimeSpan.Zero;
while (totalSleepTime < _timeout)
{
TimeSpan sleepTime = minEntries > 0 ? _sleepInterval : _timeout;
totalSleepTime += sleepTime;
Thread.Sleep(sleepTime);
// Convert the time to RFC3339 UTC format.
string time = XmlConvert.ToString(startTime, XmlDateTimeSerializationMode.Utc);
var request = new ListLogEntriesRequest
{
ResourceNames = { $"projects/{_projectId}" },
Filter = $"timestamp >= \"{time}\""
};
var results = _client.ListLogEntries(request);
var entries = results.Where(p => p.TextPayload.Contains(testId)).ToList();
if (minEntries == 0 || entries.Count() >= minEntries)
{
return(entries);
}
}
return(new List <LogEntry>());
}
internal async Task ListEventsAsync(
ListLogEntriesRequest request,
Action <EventBase> callback,
ExponentialBackOff backOff,
CancellationToken cancellationToken)
{
using (ApplicationTraceSources.Default.TraceMethod().WithParameters(request.Filter))
{
try
{
string nextPageToken = null;
do
{
request.PageToken = nextPageToken;
using (var stream = await this.service.Entries
.List(request)
.ExecuteAsStreamWithRetryAsync(backOff, cancellationToken)
.ConfigureAwait(false))
using (var reader = new JsonTextReader(new StreamReader(stream)))
{
nextPageToken = ListLogEntriesParser.Read(reader, callback);
}
}while (nextPageToken != null);
}
catch (GoogleApiException e) when(e.Error != null && e.Error.Code == 403)
{
throw new ResourceAccessDeniedException(
"You do not have sufficient permissions to view logs. " +
"You need the 'Logs Viewer' role (or an equivalent custom role) " +
"to perform this action.",
e);
}
}
}
public async Task ProcessInstanceEventsAsync(
IEnumerable <string> projectIds,
IEnumerable <string> zones,
IEnumerable <ulong> instanceIds,
DateTime startTime,
IEventProcessor processor,
CancellationToken cancellationToken)
{
Utilities.ThrowIfNull(projectIds, nameof(projectIds));
using (ApplicationTraceSources.Default.TraceMethod().WithParameters(
string.Join(", ", projectIds),
startTime))
{
var request = new ListLogEntriesRequest()
{
ResourceNames = projectIds.Select(p => "projects/" + p).ToList(),
Filter = CreateFilterString(
zones,
instanceIds,
processor.SupportedMethods,
processor.SupportedSeverities,
startTime),
PageSize = MaxPageSize,
OrderBy = "timestamp desc"
};
await ListEventsAsync(
request,
processor.Process,
new ExponentialBackOff(initialBackOff, MaxRetries),
cancellationToken).ConfigureAwait(false);
}
}
private static async Task <IList <LogEntry> > GetIapAccessLogsForPortAsync(ushort port)
{
var loggingService = TestProject.CreateService <LoggingService>();
var request = new ListLogEntriesRequest()
{
ResourceNames = new[] { "projects/" + TestProject.ProjectId },
Filter =
"protoPayload.methodName=\"AuthorizeUser\"\n" +
$"logName=\"projects/{TestProject.ProjectId}/logs/cloudaudit.googleapis.com%2Fdata_access\"\n" +
$"protoPayload.requestMetadata.destinationAttributes.port=\"{port}\"",
OrderBy = "timestamp desc"
};
// Logs take some time to show up, so retry a few times.
for (int retry = 0; retry < 20; retry++)
{
var entries = await loggingService.Entries.List(request).ExecuteAsync();
if (entries.Entries.EnsureNotNull().Any())
{
return(entries.Entries);
}
await Task.Delay(1000);
}
return(new List <LogEntry>());
}
public async Task WhenInstanceCreated_ThenListLogEntriesReturnsInsertEvent(
[LinuxInstance] InstanceRequest testInstance)
{
await testInstance.AwaitReady();
var instanceRef = await testInstance.GetInstanceAsync();
var loggingService = new LoggingService(new BaseClientService.Initializer
{
HttpClientInitializer = Defaults.GetCredential()
});
var startDate = DateTime.UtcNow.AddDays(-30);
var endDate = DateTime.UtcNow;
var request = new ListLogEntriesRequest()
{
ResourceNames = new[]
{
"projects/" + Defaults.ProjectId
},
Filter = $"resource.type=\"gce_instance\" " +
$"AND protoPayload.methodName:{InsertInstanceEvent.Method} " +
$"AND timestamp > {startDate:yyyy-MM-dd}",
PageSize = 1000,
OrderBy = "timestamp desc"
};
var events = new List <EventBase>();
var instanceBuilder = new InstanceSetHistoryBuilder(startDate, endDate);
// Creating the VM might be quicker than the logs become available.
for (int retry = 0; retry < 4 && !events.Any(); retry++)
{
await loggingService.Entries.ListEventsAsync(
request,
events.Add,
new Apis.Util.ExponentialBackOff());
if (!events.Any())
{
await Task.Delay(20 * 1000);
}
}
var insertEvent = events.OfType <InsertInstanceEvent>()
.First(e => e.InstanceReference == instanceRef);
Assert.IsNotNull(insertEvent);
}
/// <summary>
/// Gets log entries that contain the passed in testId in the log message. Will poll
/// and wait for the entries to appear.
/// </summary>
/// <param name="startTime">The earliest log entry time that will be looked at.</param>
/// <param name="testId">The test id to filter log entries on.</param>
/// <param name="minEntries">The minimum number of logs entries that should be waited for.
/// If minEntries is zero this method will wait the full timeout before checking for the
/// entries.</param>
public IEnumerable <LogEntry> GetEntries(DateTime startTime, string testId, int minEntries)
{
return(GetEntries(minEntries, () =>
{
// Convert the time to RFC3339 UTC format.
string time = XmlConvert.ToString(startTime, XmlDateTimeSerializationMode.Utc);
var request = new ListLogEntriesRequest
{
ResourceNames = { $"projects/{_projectId}" },
Filter = $"timestamp >= \"{time}\" AND textPayload:{testId}",
PageSize = 1000,
};
return _client.ListLogEntries(request);
}));
}
/// <summary>
/// Gets log entries that contain the passed in testId in the log message. Will poll
/// and wait for the entries to appear.
/// </summary>
/// <param name="startTime">The earliest log entry time that will be looked at.</param>
/// <param name="testId">The test id to filter log entries on.</param>
/// <param name="minEntries">The minimum number of logs entries that should be waited for.
/// If minEntries is zero this method will wait the full timeout before checking for the
/// entries.</param>
/// <param name="minSeverity">The minimum severity a log can be.</param>
public IEnumerable <LogEntry> GetEntries(DateTime startTime, string testId, int minEntries, LogSeverity minSeverity)
{
return(GetEntries(minEntries, () =>
{
// Convert the time to RFC3339 UTC format.
string time = XmlConvert.ToString(startTime, XmlDateTimeSerializationMode.Utc);
var request = new ListLogEntriesRequest
{
ResourceNames = { $"projects/{_projectId}" },
Filter = $"timestamp >= \"{time}\" AND jsonPayload.message:\"{testId}\" AND severity >= {minSeverity} AND logName=\"projects/{_projectId}/logs/aspnetcore\"",
PageSize = 1000,
};
return _client.ListLogEntries(request);
}));
}
public async Task WhenInstanceCreated_ThenListLogEntriesReturnsInsertEvent(
[LinuxInstance] InstanceRequest testInstance,
[Credential(Role = PredefinedRole.LogsViewer)] CredentialRequest credential)
{
await testInstance.AwaitReady();
var instanceRef = await testInstance.GetInstanceAsync();
var startDate = DateTime.UtcNow.AddDays(-30);
var endDate = DateTime.UtcNow;
var adapter = new AuditLogAdapter(await credential.GetCredentialAsync());
var request = new ListLogEntriesRequest()
{
ResourceNames = new[]
{
"projects/" + TestProject.ProjectId
},
Filter = $"resource.type=\"gce_instance\" " +
$"AND protoPayload.methodName:{InsertInstanceEvent.Method} " +
$"AND timestamp > {startDate:yyyy-MM-dd}",
PageSize = 1000,
OrderBy = "timestamp desc"
};
var events = new List <EventBase>();
var instanceBuilder = new InstanceSetHistoryBuilder(startDate, endDate);
// Creating the VM might be quicker than the logs become available.
for (int retry = 0; retry < 4 && !events.Any(); retry++)
{
await adapter.ListEventsAsync(
request,
events.Add,
new Apis.Util.ExponentialBackOff(),
CancellationToken.None);
if (!events.Any())
{
await Task.Delay(20 * 1000);
}
}
var insertEvent = events.OfType <InsertInstanceEvent>()
.First(e => e.InstanceReference == instanceRef);
Assert.IsNotNull(insertEvent);
}
/// <summary>
/// Gets log entries that contain the passed in testId in the log message. Will poll
/// and wait for the entries to appear.
/// </summary>
/// <param name="startTime">The earliest log entry time that will be looked at.</param>
/// <param name="testId">The test id to filter log entries on.</param>
/// <param name="minEntries">The minimum number of logs entries that should be waited for.
/// If minEntries is zero this method will wait the full timeout before checking for the
/// entries.</param>
public IEnumerable <LogEntry> GetEntries(DateTime startTime, string testId, int minEntries)
{
return(GetEntries(minEntries, () =>
{
// Convert the time to RFC3339 UTC format.
string time = XmlConvert.ToString(startTime, XmlDateTimeSerializationMode.Utc);
var request = new ListLogEntriesRequest
{
ResourceNames = { $"projects/{_projectId}" },
Filter = $"timestamp >= \"{time}\""
};
var results = _client.ListLogEntries(request);
return results.Where(p => p.TextPayload.Contains(testId)).ToList();
}));
}
/// <summary>Snippet for ListLogEntriesAsync</summary>
public async Task ListLogEntriesAsync_RequestObject()
{
// Snippet: ListLogEntriesAsync(ListLogEntriesRequest,CallSettings)
// Create client
LoggingServiceV2Client loggingServiceV2Client = await LoggingServiceV2Client.CreateAsync();
// Initialize request argument(s)
ListLogEntriesRequest request = new ListLogEntriesRequest
{
ResourceNames = { },
};
// Make the request
PagedAsyncEnumerable <ListLogEntriesResponse, LogEntry> response =
loggingServiceV2Client.ListLogEntriesAsync(request);
// Iterate over all response items, lazily performing RPCs as required
await response.ForEachAsync((LogEntry item) =>
{
// Do something with each item
Console.WriteLine(item);
});
// Or iterate over pages (of server-defined size), performing one RPC per page
await response.AsRawResponses().ForEachAsync((ListLogEntriesResponse page) =>
{
// Do something with each page of items
Console.WriteLine("A page of results:");
foreach (LogEntry item in page)
{
Console.WriteLine(item);
}
});
// Or retrieve a single page of known size (unless it's the final page), performing as many RPCs as required
int pageSize = 10;
Page <LogEntry> singlePage = await response.ReadPageAsync(pageSize);
// Do something with the page of items
Console.WriteLine($"A page of {pageSize} results (unless it's the final page):");
foreach (LogEntry item in singlePage)
{
Console.WriteLine(item);
}
// Store the pageToken, for when the next page is required.
string nextPageToken = singlePage.NextPageToken;
// End snippet
}
/// <summary>
/// Gets log entries that contain the passed in testId in the log message. Will poll
/// and wait for the entries to appear.
/// </summary>
/// <param name="startTime">The earliest log entry time that will be looked at.</param>
/// <param name="testId">The test id to filter log entries on.</param>
/// <param name="minEntries">The minimum number of logs entries that should be waited for.
/// If minEntries is zero this method will wait the full timeout before checking for the
/// entries.</param>
/// <param name="minSeverity">The minimum severity a log can be.</param>
public IEnumerable <LogEntry> GetEntries(DateTimeOffset startTime, string testId, int minEntries, LogSeverity minSeverity)
{
return(GetEntries(minEntries, () =>
{
// Convert the time to RFC3339 UTC format.
// We substract 5 minutes because on occasion we were polling too fast after startTime
// and the backend clock was a little behind, we were getting InvalidArgument.
string time = XmlConvert.ToString(startTime.DateTime - TimeSpan.FromMinutes(5), XmlDateTimeSerializationMode.Utc);
var request = new ListLogEntriesRequest
{
ResourceNames = { $"projects/{_projectId}" },
Filter = $"timestamp >= \"{time}\" AND jsonPayload.message:\"{testId}\" AND severity >= {minSeverity} AND logName=\"projects/{_projectId}/logs/aspnetcore\"",
PageSize = 1000,
};
return _client.ListLogEntries(request);
}));
}
/// <summary>
/// Lists log entries. Use this method to retrieve log entries from Stackdriver Logging. For ways to export log entries, see Exporting Logs.
/// Documentation https://developers.google.com/logging/v2beta1/reference/entries/list
/// Generation Note: This does not always build corectly. Google needs to standardise things I need to figuer out which ones are wrong.
/// </summary>
/// <param name="service">Authenticated Logging service.</param>
/// <param name="body">A valid Logging v2beta1 body.</param>
/// <returns>ListLogEntriesResponseResponse</returns>
public static ListLogEntriesResponse List(LoggingService service, ListLogEntriesRequest body)
{
try
{
// Initial validation.
if (service == null)
{
throw new ArgumentNullException("service");
}
if (body == null)
{
throw new ArgumentNullException("body");
}
// Make the request.
return(service.Entries.List(body).Execute());
}
catch (Exception ex)
{
throw new Exception("Request Entries.List failed.", ex);
}
}
public static async Task ListEventsAsync(
this EntriesResource entriesResource,
ListLogEntriesRequest request,
Action <EventBase> callback,
ExponentialBackOff backOff)
{
using (TraceSources.LogAnalysis.TraceMethod().WithParameters(request.Filter))
{
string nextPageToken = null;
do
{
request.PageToken = nextPageToken;
using (var stream = await entriesResource
.List(request)
.ExecuteAsStreamWithRetryAsync(backOff))
using (var reader = new JsonTextReader(new StreamReader(stream)))
{
nextPageToken = ListLogEntriesParser.Read(reader, callback);
}
}while (nextPageToken != null);
}
}
protected override void ProcessRecord()
{
ListLogEntriesRequest logEntriesRequest = new ListLogEntriesRequest();
// Set resource to "projects/{Project}" so we will only find log entries in project Project.
logEntriesRequest.ResourceNames = new List <string> {
$"projects/{Project}"
};
if (ParameterSetName == ParameterSetNames.Filter)
{
logEntriesRequest.Filter = Filter;
}
else
{
string andOp = " AND ";
string filterString = "";
if (!string.IsNullOrWhiteSpace(LogName))
{
LogName = PrefixProject(LogName, Project);
// By setting logName = LogName in the filter, the list request
// will only return log entry that belongs to LogName.
// Example: logName = "Name of log".
filterString = $"logName = '{LogName}'{andOp}".Replace('\'', '"');
}
if (Severity.HasValue)
{
// Example: severity >= ERROR.
string severityString = Enum.GetName(typeof(LogSeverity), Severity.Value).ToUpper();
filterString += $"severity = {severityString}{andOp}";
}
string selectedType = _dynamicParameters["ResourceType"].Value?.ToString().ToLower();
if (selectedType != null)
{
// Example: resource.type = "gce_instance".
filterString += $"resource.type = '{selectedType}'{andOp}".Replace('\'', '"');
}
if (Before.HasValue)
{
// Example: timestamp <= "2016-06-27T14:40:00-04:00".
string beforeTimestamp = XmlConvert.ToString(Before.Value, XmlDateTimeSerializationMode.Local);
filterString += $"timestamp <= '{beforeTimestamp}'{andOp}".Replace('\'', '"');
}
if (After.HasValue)
{
// Example: timestamp >= "2016-06-27T14:40:00-04:00".
string afterTimestamp = XmlConvert.ToString(After.Value, XmlDateTimeSerializationMode.Local);
filterString += $"timestamp >= '{afterTimestamp}'{andOp}".Replace('\'', '"');
}
// Strip the "AND " at the end if we have it.
if (filterString.EndsWith(andOp))
{
logEntriesRequest.Filter = filterString.Substring(0, filterString.Length - andOp.Length);
}
}
do
{
EntriesResource.ListRequest listLogRequest = Service.Entries.List(logEntriesRequest);
ListLogEntriesResponse response = listLogRequest.Execute();
if (response.Entries != null)
{
foreach (LogEntry logEntry in response.Entries)
{
WriteObject(logEntry);
}
}
logEntriesRequest.PageToken = response.NextPageToken;
}while (!Stopping && logEntriesRequest.PageToken != null);
}
protected override void ProcessRecord()
{
ListLogEntriesRequest logEntriesRequest = new ListLogEntriesRequest();
// Set resource to "projects/{Project}" so we will only find log entries in project Project.
logEntriesRequest.ResourceNames = new List<string> { $"projects/{Project}" };
if (ParameterSetName == ParameterSetNames.Filter)
{
logEntriesRequest.Filter = Filter;
}
else
{
string andOp = " AND ";
string filterString = "";
if (!string.IsNullOrWhiteSpace(LogName))
{
LogName = PrefixProject(LogName, Project);
// By setting logName = LogName in the filter, the list request
// will only return log entry that belongs to LogName.
// Example: logName = "Name of log".
filterString = $"logName = '{LogName}'{andOp}".Replace('\'', '"');
}
if (Severity.HasValue)
{
// Example: severity >= ERROR.
string severityString = Enum.GetName(typeof(LogSeverity), Severity.Value).ToUpper();
filterString += $"severity = {severityString}{andOp}";
}
string selectedType = _dynamicParameters["ResourceType"].Value?.ToString().ToLower();
if (selectedType != null)
{
// Example: resource.type = "gce_instance".
filterString += $"resource.type = '{selectedType}'{andOp}".Replace('\'', '"');
}
if (Before.HasValue)
{
// Example: timestamp <= "2016-06-27T14:40:00-04:00".
string beforeTimestamp = XmlConvert.ToString(Before.Value, XmlDateTimeSerializationMode.Local);
filterString += $"timestamp <= '{beforeTimestamp}'{andOp}".Replace('\'', '"');
}
if (After.HasValue)
{
// Example: timestamp >= "2016-06-27T14:40:00-04:00".
string afterTimestamp = XmlConvert.ToString(After.Value, XmlDateTimeSerializationMode.Local);
filterString += $"timestamp >= '{afterTimestamp}'{andOp}".Replace('\'', '"');
}
// Strip the "AND " at the end if we have it.
if (filterString.EndsWith(andOp))
{
logEntriesRequest.Filter = filterString.Substring(0, filterString.Length - andOp.Length);
}
}
do
{
EntriesResource.ListRequest listLogRequest = Service.Entries.List(logEntriesRequest);
ListLogEntriesResponse response = listLogRequest.Execute();
if (response.Entries != null)
{
foreach (LogEntry logEntry in response.Entries)
{
WriteObject(logEntry);
}
}
logEntriesRequest.PageToken = response.NextPageToken;
}
while (!Stopping && logEntriesRequest.PageToken != null);
}
