internal static bool IsCorrectLogType(ICloudBlob logBlob, LoggingOperations operations)
{
IDictionary <string, string> metadata = logBlob.Metadata;
string logTypeValue;
bool hasLogtype = metadata.TryGetValue("LogType", out logTypeValue);
if (!hasLogtype)
{
return(true);
}
if (operations.HasFlag(LoggingOperations.Read) && logTypeValue.Contains("read"))
{
return(true);
}
if (operations.HasFlag(LoggingOperations.Write) && logTypeValue.Contains("write"))
{
return(true);
}
if (operations.HasFlag(LoggingOperations.Delete) && logTypeValue.Contains("delete"))
{
return(true);
}
return(false);
}
/// <summary>
/// Returns an enumerable collection of logs stored as blobs, retrieved lazily.
/// </summary>
/// <param name="service">A <see cref="StorageService"/> enumeration value.</param>
/// <param name="operations">A <see cref="LoggingOperations"/> enumeration value that indicates which log types to return.</param>
/// <param name="details">A <see cref="BlobListingDetails"/> enumeration value that indicates whether or not blob metadata should be returned. Only <c>None</c> and <c>Metadata</c> are valid values. </param>
/// <param name="options">A <see cref="BlobRequestOptions"/> object that specifies additional options for the request.</param>
/// <param name="operationContext">An <see cref="OperationContext"/> object that represents the context for the current operation.</param>
/// <returns>An enumerable collection of objects that implement <see cref="ICloudBlob"/> and are retrieved lazily.</returns>
/// <remarks>Note that specifying a log type for the <paramref name="operations"/> parameter will return any log that contains the specified log type,
/// even if that log also includes other log types. Also note that the only currently supported values for the <paramref name="details"/>
/// parameter are <c>None</c> and <c>Metadata</c>.</remarks>
public IEnumerable <ICloudBlob> ListLogs(StorageService service, LoggingOperations operations, BlobListingDetails details, BlobRequestOptions options, OperationContext operationContext)
{
BlobListingDetails metadataDetails = BlobListingDetails.None;
// Currently only support the ability to retrieve metadata on logs.
if (details.HasFlag(BlobListingDetails.Copy) || details.HasFlag(BlobListingDetails.Snapshots) || details.HasFlag(BlobListingDetails.UncommittedBlobs))
{
throw new ArgumentException(SR.InvalidListingDetails);
}
// At least one LogType must be specified.
if (operations == LoggingOperations.None)
{
throw new ArgumentException(SR.InvalidLoggingLevel);
}
if (details.HasFlag(BlobListingDetails.Metadata) || !operations.HasFlag(LoggingOperations.All))
{
metadataDetails = BlobListingDetails.Metadata;
}
IEnumerable <IListBlobItem> logs = this.GetLogDirectory(service).ListBlobs(true, metadataDetails, options, operationContext);
return(logs.Select(log => (ICloudBlob)log).Where(log => IsCorrectLogType(log, operations)));
}
public static LoggingProperties ConvertLoggingProperties(TableAnalyticsLoggingSettings loggingSettings)
{
LoggingOperations loggingOperations = LoggingOperations.None;
if (loggingSettings.Delete)
{
loggingOperations |= LoggingOperations.Delete;
}
if (loggingSettings.Read)
{
loggingOperations |= LoggingOperations.Read;
}
if (loggingSettings.Write)
{
loggingOperations |= LoggingOperations.Write;
}
return(new LoggingProperties()
{
Version = loggingSettings.Version,
RetentionDays = loggingSettings.RetentionPolicy?.Days,
LoggingOperations = loggingOperations,
});
}
/// <summary>
/// Constructs a <c>LoggingProperties</c> object from an XML element.
/// </summary>
/// <param name="element">The XML element.</param>
/// <returns>A <c>LoggingProperties</c> object containing the properties in the element.</returns>
private static LoggingProperties ReadLoggingPropertiesFromXml(XElement element)
{
LoggingOperations state = LoggingOperations.None;
if (bool.Parse(element.Element(DeleteName).Value))
{
state |= LoggingOperations.Delete;
}
if (bool.Parse(element.Element(ReadName).Value))
{
state |= LoggingOperations.Read;
}
if (bool.Parse(element.Element(WriteName).Value))
{
state |= LoggingOperations.Write;
}
return(new LoggingProperties()
{
Version = element.Element(VersionName).Value,
LoggingOperations = state,
RetentionDays = ReadRetentionPolicyFromXml(element.Element(RetentionPolicyName))
});
}
private static LoggingProperties ReadLoggingPropertiesFromXml(XElement element)
{
if (element == null)
{
return(null);
}
LoggingOperations loggingOperations = LoggingOperations.None;
if (bool.Parse(element.Element("Delete").Value))
{
loggingOperations |= LoggingOperations.Delete;
}
if (bool.Parse(element.Element("Read").Value))
{
loggingOperations |= LoggingOperations.Read;
}
if (bool.Parse(element.Element("Write").Value))
{
loggingOperations |= LoggingOperations.Write;
}
return(new LoggingProperties
{
Version = element.Element("Version").Value,
LoggingOperations = loggingOperations,
RetentionDays = ReadRetentionPolicyFromXml(element.Element("RetentionPolicy"))
});
}
/// <summary>
/// Update the specified service properties according to the input
/// </summary>
/// <param name="logging">Service properties</param>
internal void UpdateServiceProperties(LoggingProperties logging)
{
if (Version != null)
{
logging.Version = Version.ToString();
}
if (RetentionDays != null)
{
if (RetentionDays == -1)
{
//Disable logging retention policy
logging.RetentionDays = null;
}
else if (RetentionDays < 1 || RetentionDays > 365)
{
throw new ArgumentException(string.Format(Resources.InvalidRetentionDay, RetentionDays));
}
else
{
logging.RetentionDays = RetentionDays;
}
}
if (LoggingOperations != null && LoggingOperations.Length > 0)
{
LoggingOperations logOperations = default(LoggingOperations);
for (int i = 0; i < LoggingOperations.Length; i++)
{
if (LoggingOperations[i] == StorageClient.LoggingOperations.None ||
LoggingOperations[i] == StorageClient.LoggingOperations.All)
{
if (LoggingOperations.Length > 1)
{
throw new ArgumentException(Resources.NoneAndAllOperationShouldBeAlone);
}
}
logOperations |= LoggingOperations[i];
}
logging.LoggingOperations = logOperations;
// Set default logging version
if (string.IsNullOrEmpty(logging.Version))
{
string defaultLoggingVersion = StorageNouns.DefaultLoggingVersion;
logging.Version = defaultLoggingVersion;
}
}
}
internal void ExpectValidLoggingOperation(ServiceType serviceType, string operations, GetLoggingProperty <ServiceProperties> getServiceProperties)
{
int retentionDays = Utility.GetRandomTestCount(1, 365 + 1);
Test.Assert(CommandAgent.SetAzureStorageServiceLogging(serviceType, operations, retentionDays.ToString(), string.Empty), "Set logging operation should succeed");
ServiceProperties retrievedProperties = getServiceProperties(retentionDays, operations);
LoggingOperations expectOperation = (LoggingOperations)Enum.Parse(typeof(LoggingOperations), operations, true);
bool?read = (expectOperation & LoggingOperations.Read) != 0;
bool?write = (expectOperation & LoggingOperations.Write) != 0;
bool?delete = (expectOperation & LoggingOperations.Delete) != 0;
Utility.ValidateLoggingOperationProperty(operations, read, write, delete);
}
internal void GenericSetInvalidLoggingOperation(ServiceType serviceType, Func <ServiceProperties> getServiceProperties)
{
Test.Info("Set invalid service logging operation for {0}", serviceType);
if (lang == Language.PowerShell)
{
LoggingOperations[] loggingOperations = { LoggingOperations.All, LoggingOperations.Read };
ExpectInvalidLoggingOperation(serviceType, loggingOperations, getServiceProperties);
loggingOperations = new LoggingOperations[] { LoggingOperations.Delete, LoggingOperations.None };
ExpectInvalidLoggingOperation(serviceType, loggingOperations, getServiceProperties);
string operation = "delete,xxx";
ExpectInvalidLoggingOperation(serviceType, operation, getServiceProperties, "Cannot bind parameter 'LoggingOperations'. Cannot convert value");
}
else
{
string operation = " --read --read-off ";
string expectedError = "--read and --read-off cannot be both defined";
ExpectInvalidLoggingOperation(serviceType, operation, getServiceProperties, expectedError);
}
}
/// <summary>
/// Returns an enumerable collection of logs stored as blobs, retrieved lazily.
/// </summary>
/// <param name="service">A <see cref="StorageService"/> enumeration value.</param>
/// <param name="startTime">A <see cref="DateTimeOffset"/> object representing the start of the time range for which logs should be retrieved.</param>
/// <param name="endTime">A <see cref="DateTimeOffset"/> object representing the end of the time range for which logs should be retrieved.</param>
/// <param name="operations">A <see cref="LoggingOperations"/> enumeration value that indicates which log types to return</param>
/// <param name="details">A <see cref="BlobListingDetails"/> enumeration value that indicates whether or not blob metadata should be returned. Only <c>None</c> and <c>Metadata</c> are valid values. </param>
/// <param name="options">A <see cref="BlobRequestOptions"/> object that specifies additional options for the request.</param>
/// <param name="operationContext">An <see cref="OperationContext"/> object that represents the context for the current operation.</param>
/// <returns>An enumerable collection of objects that implement <see cref="ICloudBlob"/> and are retrieved lazily.</returns>
/// <remarks>Note that specifying a log type for the <paramref name="operations"/> parameter will return any log that contains the specified log type,
/// even if that log also includes other log types. Also note that the only currently supported values for the <paramref name="details"/>
/// parameter are <c>None</c> and <c>Metadata</c>.</remarks>
public IEnumerable <ICloudBlob> ListLogs(StorageService service, DateTimeOffset startTime, DateTimeOffset?endTime, LoggingOperations operations, BlobListingDetails details, BlobRequestOptions options, OperationContext operationContext)
{
CloudBlobDirectory logDirectory = this.GetLogDirectory(service);
BlobListingDetails metadataDetails = details;
DateTimeOffset utcStartTime = startTime.ToUniversalTime();
DateTimeOffset dateCounter = new DateTimeOffset(utcStartTime.Ticks - (utcStartTime.Ticks % TimeSpan.TicksPerHour), utcStartTime.Offset);
DateTimeOffset? utcEndTime = null;
string endPrefix = null;
// Ensure that the date range is correct.
if (endTime.HasValue)
{
utcEndTime = endTime.Value.ToUniversalTime();
endPrefix = logDirectory.Prefix + utcEndTime.Value.ToString("yyyy/MM/dd/HH", CultureInfo.InvariantCulture);
if (utcStartTime > utcEndTime.Value)
{
string errorString = string.Format(CultureInfo.InvariantCulture, SR.StartTimeExceedsEndTime, startTime, endTime.Value);
throw new ArgumentException(errorString);
}
}
// Currently only support the ability to retrieve metadata on logs.
if (details.HasFlag(BlobListingDetails.Copy) || details.HasFlag(BlobListingDetails.Snapshots) || details.HasFlag(BlobListingDetails.UncommittedBlobs))
{
throw new ArgumentException(SR.InvalidListingDetails);
}
// At least one LogType must be specified.
if (operations == LoggingOperations.None)
{
throw new ArgumentException(SR.InvalidLoggingLevel);
}
// If metadata or a specific LogType is specified, metadata should be retrieved.
if (details.HasFlag(BlobListingDetails.Metadata) || !operations.HasFlag(LoggingOperations.All))
{
metadataDetails = BlobListingDetails.Metadata;
}
// Check logs using an hour-based prefix until we reach a day boundary.
while (dateCounter.Hour > 0)
{
string currentPrefix = logDirectory.Prefix + dateCounter.ToString("yyyy/MM/dd/HH", CultureInfo.InvariantCulture);
IEnumerable <IListBlobItem> currentLogs = logDirectory.Container.ListBlobs(currentPrefix, true, metadataDetails, options, operationContext);
foreach (ICloudBlob log in currentLogs)
{
if (!utcEndTime.HasValue || string.Compare(log.Parent.Prefix, endPrefix) <= 0)
{
if (IsCorrectLogType(log, operations))
{
yield return(log);
}
}
else
{
yield break;
}
}
dateCounter = dateCounter.AddHours(1);
if (dateCounter > DateTimeOffset.UtcNow.AddHours(1))
{
yield break;
}
}
// Check logs using a day-based prefix until we reach a month boundary.
while (dateCounter.Day > 1)
{
string currentPrefix = logDirectory.Prefix + dateCounter.ToString("yyyy/MM/dd", CultureInfo.InvariantCulture);
IEnumerable <IListBlobItem> currentLogs = logDirectory.Container.ListBlobs(currentPrefix, true, metadataDetails, options, operationContext);
foreach (ICloudBlob log in currentLogs)
{
if (!utcEndTime.HasValue || string.Compare(log.Parent.Prefix, endPrefix) <= 0)
{
if (IsCorrectLogType(log, operations))
{
yield return(log);
}
}
else
{
yield break;
}
}
dateCounter = dateCounter.AddDays(1);
if (dateCounter > DateTimeOffset.UtcNow.AddHours(1))
{
yield break;
}
}
// Check logs using a month-based prefix until we reach a year boundary.
while (dateCounter.Month > 1)
{
string currentPrefix = logDirectory.Prefix + dateCounter.ToString("yyyy/MM", CultureInfo.InvariantCulture);
IEnumerable <IListBlobItem> currentLogs = logDirectory.Container.ListBlobs(currentPrefix, true, metadataDetails, options, operationContext);
foreach (ICloudBlob log in currentLogs)
{
if (!utcEndTime.HasValue || string.Compare(log.Parent.Prefix, endPrefix) <= 0)
{
if (IsCorrectLogType(log, operations))
{
yield return(log);
}
}
else
{
yield break;
}
}
dateCounter = dateCounter.AddMonths(1);
if (dateCounter > DateTimeOffset.UtcNow.AddHours(1))
{
yield break;
}
}
// Continue using a year-based prefix.
while (true)
{
string currentPrefix = logDirectory.Prefix + dateCounter.ToString("yyyy", CultureInfo.InvariantCulture);
IEnumerable <IListBlobItem> currentLogs = logDirectory.Container.ListBlobs(currentPrefix, true, metadataDetails, options, operationContext);
foreach (ICloudBlob log in currentLogs)
{
if (!utcEndTime.HasValue || string.Compare(log.Parent.Prefix, endPrefix) <= 0)
{
if (IsCorrectLogType(log, operations))
{
yield return(log);
}
}
else
{
yield break;
}
}
dateCounter = dateCounter.AddYears(1);
if (dateCounter > DateTimeOffset.UtcNow.AddHours(1))
{
yield break;
}
}
}
internal static bool IsCorrectLogType(ICloudBlob logBlob, LoggingOperations operations)
{
IDictionary<string, string> metadata = logBlob.Metadata;
string logTypeValue;
bool hasLogtype = metadata.TryGetValue("LogType", out logTypeValue);
if (!hasLogtype)
{
return true;
}
if (operations.HasFlag(LoggingOperations.Read) && logTypeValue.Contains("read"))
{
return true;
}
if (operations.HasFlag(LoggingOperations.Write) && logTypeValue.Contains("write"))
{
return true;
}
if (operations.HasFlag(LoggingOperations.Delete) && logTypeValue.Contains("delete"))
{
return true;
}
return false;
}
/// <summary>
/// Returns an enumerable collection of log blobs containing Analytics log records. The blobs are retrieved lazily.
/// </summary>
/// <param name="service">A <see cref="StorageService"/> enumeration value.</param>
/// <param name="startTime">A <see cref="DateTimeOffset"/> object representing the start of the time range for which logs should be retrieved.</param>
/// <param name="endTime">A <see cref="DateTimeOffset"/> object representing the end of the time range for which logs should be retrieved.</param>
/// <param name="operations">A <see cref="LoggingOperations"/> enumeration value that indicates the types of logging operations on which to filter the log blobs.</param>
/// <param name="details">A <see cref="BlobListingDetails"/> enumeration value that indicates whether or not blob metadata should be returned. Only <c>None</c> and <c>Metadata</c> are valid values. </param>
/// <param name="options">A <see cref="BlobRequestOptions"/> object that specifies additional options for the request.</param>
/// <param name="operationContext">An <see cref="OperationContext"/> object that represents the context for the current operation.</param>
/// <returns>An enumerable collection of objects that implement <see cref="ICloudBlob"/> and are retrieved lazily.</returns>
/// <remarks>Note that specifying a logging operation type for the <paramref name="operations"/> parameter will return any Analytics log blob that contains the specified logging operation,
/// even if that log blob also includes other types of logging operations. Also note that the only currently supported values for the <paramref name="details"/>
/// parameter are <c>None</c> and <c>Metadata</c>.</remarks>
public IEnumerable<ICloudBlob> ListLogs(StorageService service, DateTimeOffset startTime, DateTimeOffset? endTime, LoggingOperations operations, BlobListingDetails details, BlobRequestOptions options, OperationContext operationContext)
{
CloudBlobDirectory logDirectory = this.GetLogDirectory(service);
BlobListingDetails metadataDetails = details;
DateTimeOffset utcStartTime = startTime.ToUniversalTime();
DateTimeOffset dateCounter = new DateTimeOffset(utcStartTime.Ticks - (utcStartTime.Ticks % TimeSpan.TicksPerHour), utcStartTime.Offset);
DateTimeOffset? utcEndTime = null;
string endPrefix = null;
// Ensure that the date range is correct.
if (endTime.HasValue)
{
utcEndTime = endTime.Value.ToUniversalTime();
endPrefix = logDirectory.Prefix + utcEndTime.Value.ToString("yyyy/MM/dd/HH", CultureInfo.InvariantCulture);
if (utcStartTime > utcEndTime.Value)
{
string errorString = string.Format(CultureInfo.InvariantCulture, SR.StartTimeExceedsEndTime, startTime, endTime.Value);
throw new ArgumentException(errorString);
}
}
// Currently only support the ability to retrieve metadata on logs.
if (details.HasFlag(BlobListingDetails.Copy) || details.HasFlag(BlobListingDetails.Snapshots) || details.HasFlag(BlobListingDetails.UncommittedBlobs))
{
throw new ArgumentException(SR.InvalidListingDetails);
}
// At least one LogType must be specified.
if (operations == LoggingOperations.None)
{
throw new ArgumentException(SR.InvalidLoggingLevel);
}
// If metadata or a specific LogType is specified, metadata should be retrieved.
if (details.HasFlag(BlobListingDetails.Metadata) || !operations.HasFlag(LoggingOperations.All))
{
metadataDetails = BlobListingDetails.Metadata;
}
// Check logs using an hour-based prefix until we reach a day boundary.
while (dateCounter.Hour > 0)
{
string currentPrefix = logDirectory.Prefix + dateCounter.ToString("yyyy/MM/dd/HH", CultureInfo.InvariantCulture);
IEnumerable<IListBlobItem> currentLogs = logDirectory.Container.ListBlobs(currentPrefix, true, metadataDetails, options, operationContext);
foreach (ICloudBlob log in currentLogs)
{
if (!utcEndTime.HasValue || string.Compare(log.Parent.Prefix, endPrefix) <= 0)
{
if (IsCorrectLogType(log, operations))
{
yield return log;
}
}
else
{
yield break;
}
}
dateCounter = dateCounter.AddHours(1);
if (dateCounter > DateTimeOffset.UtcNow.AddHours(1))
{
yield break;
}
}
// Check logs using a day-based prefix until we reach a month boundary.
while (dateCounter.Day > 1)
{
string currentPrefix = logDirectory.Prefix + dateCounter.ToString("yyyy/MM/dd", CultureInfo.InvariantCulture);
IEnumerable<IListBlobItem> currentLogs = logDirectory.Container.ListBlobs(currentPrefix, true, metadataDetails, options, operationContext);
foreach (ICloudBlob log in currentLogs)
{
if (!utcEndTime.HasValue || string.Compare(log.Parent.Prefix, endPrefix) <= 0)
{
if (IsCorrectLogType(log, operations))
{
yield return log;
}
}
else
{
yield break;
}
}
dateCounter = dateCounter.AddDays(1);
if (dateCounter > DateTimeOffset.UtcNow.AddHours(1))
{
yield break;
}
}
// Check logs using a month-based prefix until we reach a year boundary.
while (dateCounter.Month > 1)
{
string currentPrefix = logDirectory.Prefix + dateCounter.ToString("yyyy/MM", CultureInfo.InvariantCulture);
IEnumerable<IListBlobItem> currentLogs = logDirectory.Container.ListBlobs(currentPrefix, true, metadataDetails, options, operationContext);
foreach (ICloudBlob log in currentLogs)
{
if (!utcEndTime.HasValue || string.Compare(log.Parent.Prefix, endPrefix) <= 0)
{
if (IsCorrectLogType(log, operations))
{
yield return log;
}
}
else
{
yield break;
}
}
dateCounter = dateCounter.AddMonths(1);
if (dateCounter > DateTimeOffset.UtcNow.AddHours(1))
{
yield break;
}
}
// Continue using a year-based prefix.
while (true)
{
string currentPrefix = logDirectory.Prefix + dateCounter.ToString("yyyy", CultureInfo.InvariantCulture);
IEnumerable<IListBlobItem> currentLogs = logDirectory.Container.ListBlobs(currentPrefix, true, metadataDetails, options, operationContext);
foreach (ICloudBlob log in currentLogs)
{
if (!utcEndTime.HasValue || string.Compare(log.Parent.Prefix, endPrefix) <= 0)
{
if (IsCorrectLogType(log, operations))
{
yield return log;
}
}
else
{
yield break;
}
}
dateCounter = dateCounter.AddYears(1);
if (dateCounter > DateTimeOffset.UtcNow.AddHours(1))
{
yield break;
}
}
}
/// <summary>
/// Returns an enumerable collection of log blobs containing Analytics log records. The blobs are retrieved lazily.
/// </summary>
/// <param name="service">A <see cref="StorageService"/> enumeration value.</param>
/// <param name="operations">A <see cref="LoggingOperations"/> enumeration value that indicates the types of logging operations on which to filter the log blobs.</param>
/// <param name="details">A <see cref="BlobListingDetails"/> enumeration value that indicates whether or not blob metadata should be returned. Only <c>None</c> and <c>Metadata</c> are valid values. </param>
/// <param name="options">A <see cref="BlobRequestOptions"/> object that specifies additional options for the request.</param>
/// <param name="operationContext">An <see cref="OperationContext"/> object that represents the context for the current operation.</param>
/// <returns>An enumerable collection of objects that implement <see cref="ICloudBlob"/> and are retrieved lazily.</returns>
/// <remarks>Note that specifying a logging operation type for the <paramref name="operations"/> parameter will return any Analytics log blob that contains the specified logging operation,
/// even if that log blob also includes other types of logging operations. Also note that the only currently supported values for the <paramref name="details"/>
/// parameter are <c>None</c> and <c>Metadata</c>.</remarks>
public IEnumerable<ICloudBlob> ListLogs(StorageService service, LoggingOperations operations, BlobListingDetails details, BlobRequestOptions options, OperationContext operationContext)
{
BlobListingDetails metadataDetails = BlobListingDetails.None;
// Currently only support the ability to retrieve metadata on logs.
if (details.HasFlag(BlobListingDetails.Copy) || details.HasFlag(BlobListingDetails.Snapshots) || details.HasFlag(BlobListingDetails.UncommittedBlobs))
{
throw new ArgumentException(SR.InvalidListingDetails);
}
// At least one LogType must be specified.
if (operations == LoggingOperations.None)
{
throw new ArgumentException(SR.InvalidLoggingLevel);
}
if (details.HasFlag(BlobListingDetails.Metadata) || !operations.HasFlag(LoggingOperations.All))
{
metadataDetails = BlobListingDetails.Metadata;
}
IEnumerable<IListBlobItem> logs = this.GetLogDirectory(service).ListBlobs(true, metadataDetails, options, operationContext);
return logs.Select(log => (ICloudBlob)log).Where(log => IsCorrectLogType(log, operations));
}
