public static async Task ListInstanceEventsAsync( this EntriesResource entriesResource, IEnumerable <string> projectIds, DateTime startTime, IEventProcessor processor) { using (TraceSources.LogAnalysis.TraceMethod().WithParameters( string.Join(", ", projectIds), startTime)) { var request = new ListLogEntriesRequest() { ResourceNames = projectIds.Select(p => "projects/" + p).ToList(), Filter = $"protoPayload.methodName=(\"{string.Join("\" OR \"", processor.SupportedMethods)}\") " + $"AND severity=(\"{string.Join("\" OR \"", processor.SupportedSeverities)}\") " + $"AND resource.type=\"gce_instance\" " + $"AND timestamp > {startTime:yyyy-MM-dd} ", PageSize = MaxPageSize, OrderBy = "timestamp desc" }; await ListEventsAsync( entriesResource, request, processor.Process, new ExponentialBackOff(initialBackOff, MaxRetries)); } }
protected override void ProcessRecord() { ListLogEntriesRequest logEntriesRequest = new ListLogEntriesRequest(); // Set resource to "projects/{Project}" so we will only find log entries in project Project. logEntriesRequest.ResourceNames = new List <string> { $"projects/{Project}" }; string logName = PrefixProjectToLogName(LogName, Project); logEntriesRequest.Filter = ConstructLogFilterString( logName: logName, logSeverity: Severity, selectedType: SelectedResourceType, before: Before, after: After, otherFilter: Filter); do { EntriesResource.ListRequest listLogRequest = Service.Entries.List(logEntriesRequest); ListLogEntriesResponse response = listLogRequest.Execute(); if (response.Entries != null) { foreach (LogEntry logEntry in response.Entries) { WriteObject(logEntry); } } logEntriesRequest.PageToken = response.NextPageToken; }while (!Stopping && logEntriesRequest.PageToken != null); }
public async Task WhenUsingInvalidProjectId_ThenListEventsAsyncThrowsException( [Credential(Role = PredefinedRole.LogsViewer)] CredentialRequest credential) { var startDate = DateTime.UtcNow.AddDays(-30); var request = new ListLogEntriesRequest() { ResourceNames = new[] { "projects/invalid" }, Filter = $"resource.type=\"gce_instance\" " + $"AND protoPayload.methodName:{InsertInstanceEvent.Method} " + $"AND timestamp > {startDate:yyyy-MM-dd}", PageSize = 1000, OrderBy = "timestamp desc" }; var adapter = new AuditLogAdapter(await credential.GetCredentialAsync()); AssertEx.ThrowsAggregateException <GoogleApiException>( () => adapter.ListEventsAsync( request, _ => { }, new Apis.Util.ExponentialBackOff(), CancellationToken.None).Wait()); }
/// <summary> /// Returns the first page of log entries of the project id. /// </summary> /// <param name="filter"> /// Optional, /// Refert to https://cloud.google.com/logging/docs/view/advanced_filters. /// </param> /// <param name="orderBy"> /// Optional, "timestamp desc" or "timestamp asc" /// </param> /// <param name="pageSize"> /// Optional, /// If page size is not specified, a server side default value is used. /// </param> /// <param name="nextPageToken"> /// Optional, /// The page token from last list request response. /// If the value is null, fetch the first page results. /// If the value is not null, it is hard requirement that the filter, orderBy and pageSize parameters /// must stay same as the prior call. /// </param> /// <param name="cancelToken">Optional. A cancellation token.</param> /// <returns> /// <seealso ref="LogEntryRequestResult" /> object that contains log entries and next page token. /// </returns> public async Task <LogEntryRequestResult> ListLogEntriesAsync( string filter = null, string orderBy = null, int?pageSize = null, string nextPageToken = null, CancellationToken cancelToken = default(CancellationToken)) { try { var requestData = new ListLogEntriesRequest { ResourceNames = _resourceNames, Filter = filter, OrderBy = orderBy, PageSize = pageSize }; requestData.PageToken = nextPageToken; var response = await Service.Entries.List(requestData).ExecuteAsync(cancelToken); return(new LogEntryRequestResult(response.Entries, response.NextPageToken)); } catch (GoogleApiException ex) { Debug.WriteLine($"Failed to get log entries: {ex.Message}"); throw new DataSourceException(ex.Message, ex); } }
/// <summary> /// Gets log entries that contain the passed in testId in the log message. Will poll /// and wait for the entries to appear. /// </summary> /// <param name="startTime">The earliest log entry time that will be looked at.</param> /// <param name="testId">The test id to filter log entries on.</param> /// <param name="minEntries">The minimum number of logs entries that should be waited for. /// If minEntries is zero this method will wait the full timeout before checking for the /// entries.</param> private IEnumerable <LogEntry> GetEntries(DateTime startTime, string testId, int minEntries) { TimeSpan totalSleepTime = TimeSpan.Zero; while (totalSleepTime < _timeout) { TimeSpan sleepTime = minEntries > 0 ? _sleepInterval : _timeout; totalSleepTime += sleepTime; Thread.Sleep(sleepTime); // Convert the time to RFC3339 UTC format. string time = XmlConvert.ToString(startTime, XmlDateTimeSerializationMode.Utc); var request = new ListLogEntriesRequest { ResourceNames = { $"projects/{_projectId}" }, Filter = $"timestamp >= \"{time}\"" }; var results = _client.ListLogEntries(request); var entries = results.Where(p => p.TextPayload.Contains(testId)).ToList(); if (minEntries == 0 || entries.Count() >= minEntries) { return(entries); } } return(new List <LogEntry>()); }
internal async Task ListEventsAsync( ListLogEntriesRequest request, Action <EventBase> callback, ExponentialBackOff backOff, CancellationToken cancellationToken) { using (ApplicationTraceSources.Default.TraceMethod().WithParameters(request.Filter)) { try { string nextPageToken = null; do { request.PageToken = nextPageToken; using (var stream = await this.service.Entries .List(request) .ExecuteAsStreamWithRetryAsync(backOff, cancellationToken) .ConfigureAwait(false)) using (var reader = new JsonTextReader(new StreamReader(stream))) { nextPageToken = ListLogEntriesParser.Read(reader, callback); } }while (nextPageToken != null); } catch (GoogleApiException e) when(e.Error != null && e.Error.Code == 403) { throw new ResourceAccessDeniedException( "You do not have sufficient permissions to view logs. " + "You need the 'Logs Viewer' role (or an equivalent custom role) " + "to perform this action.", e); } } }
public async Task ProcessInstanceEventsAsync( IEnumerable <string> projectIds, IEnumerable <string> zones, IEnumerable <ulong> instanceIds, DateTime startTime, IEventProcessor processor, CancellationToken cancellationToken) { Utilities.ThrowIfNull(projectIds, nameof(projectIds)); using (ApplicationTraceSources.Default.TraceMethod().WithParameters( string.Join(", ", projectIds), startTime)) { var request = new ListLogEntriesRequest() { ResourceNames = projectIds.Select(p => "projects/" + p).ToList(), Filter = CreateFilterString( zones, instanceIds, processor.SupportedMethods, processor.SupportedSeverities, startTime), PageSize = MaxPageSize, OrderBy = "timestamp desc" }; await ListEventsAsync( request, processor.Process, new ExponentialBackOff(initialBackOff, MaxRetries), cancellationToken).ConfigureAwait(false); } }
private static async Task <IList <LogEntry> > GetIapAccessLogsForPortAsync(ushort port) { var loggingService = TestProject.CreateService <LoggingService>(); var request = new ListLogEntriesRequest() { ResourceNames = new[] { "projects/" + TestProject.ProjectId }, Filter = "protoPayload.methodName=\"AuthorizeUser\"\n" + $"logName=\"projects/{TestProject.ProjectId}/logs/cloudaudit.googleapis.com%2Fdata_access\"\n" + $"protoPayload.requestMetadata.destinationAttributes.port=\"{port}\"", OrderBy = "timestamp desc" }; // Logs take some time to show up, so retry a few times. for (int retry = 0; retry < 20; retry++) { var entries = await loggingService.Entries.List(request).ExecuteAsync(); if (entries.Entries.EnsureNotNull().Any()) { return(entries.Entries); } await Task.Delay(1000); } return(new List <LogEntry>()); }
public async Task WhenInstanceCreated_ThenListLogEntriesReturnsInsertEvent( [LinuxInstance] InstanceRequest testInstance) { await testInstance.AwaitReady(); var instanceRef = await testInstance.GetInstanceAsync(); var loggingService = new LoggingService(new BaseClientService.Initializer { HttpClientInitializer = Defaults.GetCredential() }); var startDate = DateTime.UtcNow.AddDays(-30); var endDate = DateTime.UtcNow; var request = new ListLogEntriesRequest() { ResourceNames = new[] { "projects/" + Defaults.ProjectId }, Filter = $"resource.type=\"gce_instance\" " + $"AND protoPayload.methodName:{InsertInstanceEvent.Method} " + $"AND timestamp > {startDate:yyyy-MM-dd}", PageSize = 1000, OrderBy = "timestamp desc" }; var events = new List <EventBase>(); var instanceBuilder = new InstanceSetHistoryBuilder(startDate, endDate); // Creating the VM might be quicker than the logs become available. for (int retry = 0; retry < 4 && !events.Any(); retry++) { await loggingService.Entries.ListEventsAsync( request, events.Add, new Apis.Util.ExponentialBackOff()); if (!events.Any()) { await Task.Delay(20 * 1000); } } var insertEvent = events.OfType <InsertInstanceEvent>() .First(e => e.InstanceReference == instanceRef); Assert.IsNotNull(insertEvent); }
/// <summary> /// Gets log entries that contain the passed in testId in the log message. Will poll /// and wait for the entries to appear. /// </summary> /// <param name="startTime">The earliest log entry time that will be looked at.</param> /// <param name="testId">The test id to filter log entries on.</param> /// <param name="minEntries">The minimum number of logs entries that should be waited for. /// If minEntries is zero this method will wait the full timeout before checking for the /// entries.</param> public IEnumerable <LogEntry> GetEntries(DateTime startTime, string testId, int minEntries) { return(GetEntries(minEntries, () => { // Convert the time to RFC3339 UTC format. string time = XmlConvert.ToString(startTime, XmlDateTimeSerializationMode.Utc); var request = new ListLogEntriesRequest { ResourceNames = { $"projects/{_projectId}" }, Filter = $"timestamp >= \"{time}\" AND textPayload:{testId}", PageSize = 1000, }; return _client.ListLogEntries(request); })); }
/// <summary> /// Gets log entries that contain the passed in testId in the log message. Will poll /// and wait for the entries to appear. /// </summary> /// <param name="startTime">The earliest log entry time that will be looked at.</param> /// <param name="testId">The test id to filter log entries on.</param> /// <param name="minEntries">The minimum number of logs entries that should be waited for. /// If minEntries is zero this method will wait the full timeout before checking for the /// entries.</param> /// <param name="minSeverity">The minimum severity a log can be.</param> public IEnumerable <LogEntry> GetEntries(DateTime startTime, string testId, int minEntries, LogSeverity minSeverity) { return(GetEntries(minEntries, () => { // Convert the time to RFC3339 UTC format. string time = XmlConvert.ToString(startTime, XmlDateTimeSerializationMode.Utc); var request = new ListLogEntriesRequest { ResourceNames = { $"projects/{_projectId}" }, Filter = $"timestamp >= \"{time}\" AND jsonPayload.message:\"{testId}\" AND severity >= {minSeverity} AND logName=\"projects/{_projectId}/logs/aspnetcore\"", PageSize = 1000, }; return _client.ListLogEntries(request); })); }
public async Task WhenInstanceCreated_ThenListLogEntriesReturnsInsertEvent( [LinuxInstance] InstanceRequest testInstance, [Credential(Role = PredefinedRole.LogsViewer)] CredentialRequest credential) { await testInstance.AwaitReady(); var instanceRef = await testInstance.GetInstanceAsync(); var startDate = DateTime.UtcNow.AddDays(-30); var endDate = DateTime.UtcNow; var adapter = new AuditLogAdapter(await credential.GetCredentialAsync()); var request = new ListLogEntriesRequest() { ResourceNames = new[] { "projects/" + TestProject.ProjectId }, Filter = $"resource.type=\"gce_instance\" " + $"AND protoPayload.methodName:{InsertInstanceEvent.Method} " + $"AND timestamp > {startDate:yyyy-MM-dd}", PageSize = 1000, OrderBy = "timestamp desc" }; var events = new List <EventBase>(); var instanceBuilder = new InstanceSetHistoryBuilder(startDate, endDate); // Creating the VM might be quicker than the logs become available. for (int retry = 0; retry < 4 && !events.Any(); retry++) { await adapter.ListEventsAsync( request, events.Add, new Apis.Util.ExponentialBackOff(), CancellationToken.None); if (!events.Any()) { await Task.Delay(20 * 1000); } } var insertEvent = events.OfType <InsertInstanceEvent>() .First(e => e.InstanceReference == instanceRef); Assert.IsNotNull(insertEvent); }
/// <summary> /// Gets log entries that contain the passed in testId in the log message. Will poll /// and wait for the entries to appear. /// </summary> /// <param name="startTime">The earliest log entry time that will be looked at.</param> /// <param name="testId">The test id to filter log entries on.</param> /// <param name="minEntries">The minimum number of logs entries that should be waited for. /// If minEntries is zero this method will wait the full timeout before checking for the /// entries.</param> public IEnumerable <LogEntry> GetEntries(DateTime startTime, string testId, int minEntries) { return(GetEntries(minEntries, () => { // Convert the time to RFC3339 UTC format. string time = XmlConvert.ToString(startTime, XmlDateTimeSerializationMode.Utc); var request = new ListLogEntriesRequest { ResourceNames = { $"projects/{_projectId}" }, Filter = $"timestamp >= \"{time}\"" }; var results = _client.ListLogEntries(request); return results.Where(p => p.TextPayload.Contains(testId)).ToList(); })); }
/// <summary>Snippet for ListLogEntriesAsync</summary> public async Task ListLogEntriesAsync_RequestObject() { // Snippet: ListLogEntriesAsync(ListLogEntriesRequest,CallSettings) // Create client LoggingServiceV2Client loggingServiceV2Client = await LoggingServiceV2Client.CreateAsync(); // Initialize request argument(s) ListLogEntriesRequest request = new ListLogEntriesRequest { ResourceNames = { }, }; // Make the request PagedAsyncEnumerable <ListLogEntriesResponse, LogEntry> response = loggingServiceV2Client.ListLogEntriesAsync(request); // Iterate over all response items, lazily performing RPCs as required await response.ForEachAsync((LogEntry item) => { // Do something with each item Console.WriteLine(item); }); // Or iterate over pages (of server-defined size), performing one RPC per page await response.AsRawResponses().ForEachAsync((ListLogEntriesResponse page) => { // Do something with each page of items Console.WriteLine("A page of results:"); foreach (LogEntry item in page) { Console.WriteLine(item); } }); // Or retrieve a single page of known size (unless it's the final page), performing as many RPCs as required int pageSize = 10; Page <LogEntry> singlePage = await response.ReadPageAsync(pageSize); // Do something with the page of items Console.WriteLine($"A page of {pageSize} results (unless it's the final page):"); foreach (LogEntry item in singlePage) { Console.WriteLine(item); } // Store the pageToken, for when the next page is required. string nextPageToken = singlePage.NextPageToken; // End snippet }
/// <summary> /// Gets log entries that contain the passed in testId in the log message. Will poll /// and wait for the entries to appear. /// </summary> /// <param name="startTime">The earliest log entry time that will be looked at.</param> /// <param name="testId">The test id to filter log entries on.</param> /// <param name="minEntries">The minimum number of logs entries that should be waited for. /// If minEntries is zero this method will wait the full timeout before checking for the /// entries.</param> /// <param name="minSeverity">The minimum severity a log can be.</param> public IEnumerable <LogEntry> GetEntries(DateTimeOffset startTime, string testId, int minEntries, LogSeverity minSeverity) { return(GetEntries(minEntries, () => { // Convert the time to RFC3339 UTC format. // We substract 5 minutes because on occasion we were polling too fast after startTime // and the backend clock was a little behind, we were getting InvalidArgument. string time = XmlConvert.ToString(startTime.DateTime - TimeSpan.FromMinutes(5), XmlDateTimeSerializationMode.Utc); var request = new ListLogEntriesRequest { ResourceNames = { $"projects/{_projectId}" }, Filter = $"timestamp >= \"{time}\" AND jsonPayload.message:\"{testId}\" AND severity >= {minSeverity} AND logName=\"projects/{_projectId}/logs/aspnetcore\"", PageSize = 1000, }; return _client.ListLogEntries(request); })); }
/// <summary> /// Lists log entries. Use this method to retrieve log entries from Stackdriver Logging. For ways to export log entries, see Exporting Logs. /// Documentation https://developers.google.com/logging/v2beta1/reference/entries/list /// Generation Note: This does not always build corectly. Google needs to standardise things I need to figuer out which ones are wrong. /// </summary> /// <param name="service">Authenticated Logging service.</param> /// <param name="body">A valid Logging v2beta1 body.</param> /// <returns>ListLogEntriesResponseResponse</returns> public static ListLogEntriesResponse List(LoggingService service, ListLogEntriesRequest body) { try { // Initial validation. if (service == null) { throw new ArgumentNullException("service"); } if (body == null) { throw new ArgumentNullException("body"); } // Make the request. return(service.Entries.List(body).Execute()); } catch (Exception ex) { throw new Exception("Request Entries.List failed.", ex); } }
public static async Task ListEventsAsync( this EntriesResource entriesResource, ListLogEntriesRequest request, Action <EventBase> callback, ExponentialBackOff backOff) { using (TraceSources.LogAnalysis.TraceMethod().WithParameters(request.Filter)) { string nextPageToken = null; do { request.PageToken = nextPageToken; using (var stream = await entriesResource .List(request) .ExecuteAsStreamWithRetryAsync(backOff)) using (var reader = new JsonTextReader(new StreamReader(stream))) { nextPageToken = ListLogEntriesParser.Read(reader, callback); } }while (nextPageToken != null); } }
protected override void ProcessRecord() { ListLogEntriesRequest logEntriesRequest = new ListLogEntriesRequest(); // Set resource to "projects/{Project}" so we will only find log entries in project Project. logEntriesRequest.ResourceNames = new List <string> { $"projects/{Project}" }; if (ParameterSetName == ParameterSetNames.Filter) { logEntriesRequest.Filter = Filter; } else { string andOp = " AND "; string filterString = ""; if (!string.IsNullOrWhiteSpace(LogName)) { LogName = PrefixProject(LogName, Project); // By setting logName = LogName in the filter, the list request // will only return log entry that belongs to LogName. // Example: logName = "Name of log". filterString = $"logName = '{LogName}'{andOp}".Replace('\'', '"'); } if (Severity.HasValue) { // Example: severity >= ERROR. string severityString = Enum.GetName(typeof(LogSeverity), Severity.Value).ToUpper(); filterString += $"severity = {severityString}{andOp}"; } string selectedType = _dynamicParameters["ResourceType"].Value?.ToString().ToLower(); if (selectedType != null) { // Example: resource.type = "gce_instance". filterString += $"resource.type = '{selectedType}'{andOp}".Replace('\'', '"'); } if (Before.HasValue) { // Example: timestamp <= "2016-06-27T14:40:00-04:00". string beforeTimestamp = XmlConvert.ToString(Before.Value, XmlDateTimeSerializationMode.Local); filterString += $"timestamp <= '{beforeTimestamp}'{andOp}".Replace('\'', '"'); } if (After.HasValue) { // Example: timestamp >= "2016-06-27T14:40:00-04:00". string afterTimestamp = XmlConvert.ToString(After.Value, XmlDateTimeSerializationMode.Local); filterString += $"timestamp >= '{afterTimestamp}'{andOp}".Replace('\'', '"'); } // Strip the "AND " at the end if we have it. if (filterString.EndsWith(andOp)) { logEntriesRequest.Filter = filterString.Substring(0, filterString.Length - andOp.Length); } } do { EntriesResource.ListRequest listLogRequest = Service.Entries.List(logEntriesRequest); ListLogEntriesResponse response = listLogRequest.Execute(); if (response.Entries != null) { foreach (LogEntry logEntry in response.Entries) { WriteObject(logEntry); } } logEntriesRequest.PageToken = response.NextPageToken; }while (!Stopping && logEntriesRequest.PageToken != null); }
protected override void ProcessRecord() { ListLogEntriesRequest logEntriesRequest = new ListLogEntriesRequest(); // Set resource to "projects/{Project}" so we will only find log entries in project Project. logEntriesRequest.ResourceNames = new List<string> { $"projects/{Project}" }; if (ParameterSetName == ParameterSetNames.Filter) { logEntriesRequest.Filter = Filter; } else { string andOp = " AND "; string filterString = ""; if (!string.IsNullOrWhiteSpace(LogName)) { LogName = PrefixProject(LogName, Project); // By setting logName = LogName in the filter, the list request // will only return log entry that belongs to LogName. // Example: logName = "Name of log". filterString = $"logName = '{LogName}'{andOp}".Replace('\'', '"'); } if (Severity.HasValue) { // Example: severity >= ERROR. string severityString = Enum.GetName(typeof(LogSeverity), Severity.Value).ToUpper(); filterString += $"severity = {severityString}{andOp}"; } string selectedType = _dynamicParameters["ResourceType"].Value?.ToString().ToLower(); if (selectedType != null) { // Example: resource.type = "gce_instance". filterString += $"resource.type = '{selectedType}'{andOp}".Replace('\'', '"'); } if (Before.HasValue) { // Example: timestamp <= "2016-06-27T14:40:00-04:00". string beforeTimestamp = XmlConvert.ToString(Before.Value, XmlDateTimeSerializationMode.Local); filterString += $"timestamp <= '{beforeTimestamp}'{andOp}".Replace('\'', '"'); } if (After.HasValue) { // Example: timestamp >= "2016-06-27T14:40:00-04:00". string afterTimestamp = XmlConvert.ToString(After.Value, XmlDateTimeSerializationMode.Local); filterString += $"timestamp >= '{afterTimestamp}'{andOp}".Replace('\'', '"'); } // Strip the "AND " at the end if we have it. if (filterString.EndsWith(andOp)) { logEntriesRequest.Filter = filterString.Substring(0, filterString.Length - andOp.Length); } } do { EntriesResource.ListRequest listLogRequest = Service.Entries.List(logEntriesRequest); ListLogEntriesResponse response = listLogRequest.Execute(); if (response.Entries != null) { foreach (LogEntry logEntry in response.Entries) { WriteObject(logEntry); } } logEntriesRequest.PageToken = response.NextPageToken; } while (!Stopping && logEntriesRequest.PageToken != null); }