/// <summary>
/// Wrapper method to handle any filtering (Azure table queries can't do wild card matching)
/// </summary>
/// <param name="appenderName"></param>
/// <param name="partitionKey"></param>
/// <param name="rowKey"></param>
/// <param name="hostName"></param>
/// <param name="loggerName"></param>
/// <param name="messageIntro"></param>
/// <returns></returns>
internal IEnumerable <LogTableEntity> ReadLogTableEntities(string appenderName, string partitionKey, string rowKey, string hostName, string loggerName, Level minLevel, string message, string sessionId, int take)
{
// flags to indiciate if filtering is requied here in c#
bool hostNameWildcardFiltering = !string.IsNullOrWhiteSpace(hostName) && !IndexService.Instance.GetMachineNames(appenderName).Any(x => x == hostName);
bool loggerNameWildcardFiltering = !string.IsNullOrWhiteSpace(loggerName) && !IndexService.Instance.GetLoggerNames(appenderName).Any(x => x == loggerName);
// check to see if any filtering needs to be done here (always filter message as it has no index)
if (!string.IsNullOrWhiteSpace(message) || hostNameWildcardFiltering || loggerNameWildcardFiltering)
{
// additional filtering here- may require additional azure table queries
List <LogTableEntity> logTableEntities = new List <LogTableEntity>(); // collection to return
// calculate excess amount to request from a cloud query, which will filter down to the required take
// machine name: low
// logger name: low
// level-debug: low
// level-info: medium
// level-warn: high
// level-error: high
// level-fatal: high
// message: medium
int lastCount;
string lastPartitionKey = partitionKey;
string lastRowKey = rowKey;
int attempts = 0;
bool finished = false;
do
{
attempts++;
if (attempts >= 3)
{
throw new TableQueryTimeoutException(lastPartitionKey, lastRowKey, logTableEntities.ToArray());
}
lastCount = logTableEntities.Count;
// take a large chunk to filter here - take size should be relative to the filter granularity
IEnumerable <LogTableEntity> returnedLogTableEntities = this.ReadLogTableEntities(
appenderName,
lastPartitionKey,
lastRowKey,
minLevel,
loggerNameWildcardFiltering ? null : loggerName, // only set if
hostNameWildcardFiltering ? null : hostName,
sessionId) // not using wildcards
.Take(100);
if (returnedLogTableEntities.Any())
{
logTableEntities.AddRange(returnedLogTableEntities);
// set last known, before filtering out
lastPartitionKey = logTableEntities.Last().PartitionKey;
lastRowKey = logTableEntities.Last().RowKey;
// performing filtering on local list, otherwise it seems to affect table query performance (as every row in table returned and cast)
logTableEntities = logTableEntities
.Where(x => string.IsNullOrWhiteSpace(hostName) || (x.log4net_HostName != null && x.log4net_HostName.IndexOf(hostName, StringComparison.InvariantCultureIgnoreCase) > -1))
.Where(x => string.IsNullOrWhiteSpace(loggerName) || (x.LoggerName != null && x.LoggerName.IndexOf(loggerName, StringComparison.InvariantCultureIgnoreCase) > -1))
.Where(x => string.IsNullOrWhiteSpace(message) || (x.Message != null && x.Message.IndexOf(message, StringComparison.InvariantCultureIgnoreCase) > -1))
.ToList();
}
else
{
// no data returned from Azure query
finished = true;
}
}while (logTableEntities.Count < take && !finished);
return(logTableEntities.Take(take)); // trim any excess
}
else
{
// filter at azure table query level only
return(this.ReadLogTableEntities(appenderName, partitionKey, rowKey, minLevel, loggerName, hostName, sessionId).Take(take));
}
}
/// <summary>
/// Attempts to perform an Azure table query
/// https://azure.microsoft.com/en-gb/documentation/articles/storage-dotnet-how-to-use-tables/
/// Gets a collection of LogTableEntity objs suitable for casting to LogItemInto
/// </summary>
/// <param name="appenderName"></param>
/// <param name="partitionKey">null or the last known partition key</param>
/// <param name="rowKey">null or the last known row key</param>
/// <param name="minLevel"></param>
/// <param name="loggerName">if set, looks for an exact match</param>
/// <param name="hostName">if set, looks for an exact match</param>
/// <returns>a collection of log items matching the supplied filter criteria</returns>
private IEnumerable <LogTableEntity> ReadLogTableEntities(string appenderName, string partitionKey, string rowKey, Level minLevel, string loggerName, string hostName, string sessionId)
{
CloudTable cloudTable = this.GetCloudTable(appenderName);
if (cloudTable != null)
{
TableQuery <LogTableEntity> tableQuery = new TableQuery <LogTableEntity>()
.Select(new string[] { // reduce data fields returned from Azure
"Level",
"LoggerName",
"Message",
"EventTimeStamp",
"log4net_HostName"
});
if (!string.IsNullOrWhiteSpace(partitionKey))
{
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThanOrEqual, partitionKey));
}
if (!string.IsNullOrWhiteSpace(rowKey))
{
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, rowKey));
}
if (minLevel != Level.DEBUG)
{
// a number comparrison would be better, but log4net level and enum level don't match
switch (minLevel)
{
case Level.INFO: // show all except debug
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("Level", QueryComparisons.NotEqual, Level.DEBUG.ToString()));
break;
case Level.WARN: // show all except debug and info
tableQuery.AndWhere(TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("Level", QueryComparisons.NotEqual, Level.DEBUG.ToString()),
TableOperators.And,
TableQuery.GenerateFilterCondition("Level", QueryComparisons.NotEqual, Level.INFO.ToString())));
break;
case Level.ERROR: // show if error or fatal
tableQuery.AndWhere(TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("Level", QueryComparisons.Equal, Level.ERROR.ToString()),
TableOperators.Or,
TableQuery.GenerateFilterCondition("Level", QueryComparisons.Equal, Level.FATAL.ToString())));
break;
case Level.FATAL: // show fatal only
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("Level", QueryComparisons.Equal, Level.FATAL.ToString()));
break;
}
}
if (!string.IsNullOrWhiteSpace(loggerName))
{
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("LoggerName", QueryComparisons.Equal, loggerName));
}
else
{
// HACK: ensure index entities are not returned
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("LoggerName", QueryComparisons.NotEqual, string.Empty));
}
if (!string.IsNullOrWhiteSpace(hostName))
{
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("log4net_HostName", QueryComparisons.Equal, hostName));
}
if (!string.IsNullOrWhiteSpace(sessionId))
{
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("sessionId", QueryComparisons.Equal, sessionId));
}
return(cloudTable.ExecuteQuery(
tableQuery,
new TableRequestOptions()
{
ServerTimeout = new TimeSpan(0, 0, 2)
}));
}
return(Enumerable.Empty <LogTableEntity>()); // fallback
}
/// <summary>
/// Queries Azure table storage until results are returned or a timeout is thown
/// </summary>
/// <param name="appenderName">name of the log4net appender</param>
/// <param name="partitionKey">a partional key to begin search from (can be null)</param>
/// <param name="rowKey">a row key to begin search from (can be null)</param>
/// <param name="hostName">host name to filter on</param>
/// <param name="loggerName">logger name to filter on</param>
/// <param name="minLevel">logger level to filter</param>
/// <param name="message">message text to filter</param>
/// <param name="sessionId">session id to filter</param>
/// <returns></returns>
internal LogTableEntity[] ReadLogTableEntities(string appenderName, string partitionKey, string rowKey, string hostName, string loggerName, Level minLevel, string message, string sessionId)
{
LogTableEntity[] logTableEntities = new LogTableEntity[] { }; // default return value
CloudTable cloudTable = this.GetCloudTable(appenderName);
if (cloudTable == null)
{
return(logTableEntities);
}
int take = 50; // default take for Azure query
bool hostNameWildcardFiltering = !string.IsNullOrWhiteSpace(hostName) && !IndexService.Instance.GetMachineNames(appenderName).Any(x => x == hostName);
bool loggerNameWildcardFiltering = !string.IsNullOrWhiteSpace(loggerName) && !IndexService.Instance.GetLoggerNames(appenderName).Any(x => x == loggerName);
// local filtering function applied to returned Azure table results
Func <LogTableEntity, bool> customFiltering = (x) => { return(true); }; // default empty method (no custom filtering performed)
// check to see if custom filtering (in c#) is required in addition to the Azure query
if (hostNameWildcardFiltering || loggerNameWildcardFiltering || !string.IsNullOrWhiteSpace(message)) // message filtering always done in c#
{
customFiltering = (x) =>
{
return((string.IsNullOrWhiteSpace(hostName) || x.log4net_HostName != null && x.log4net_HostName.IndexOf(hostName, StringComparison.InvariantCultureIgnoreCase) > -1) &&
(string.IsNullOrWhiteSpace(loggerName) || x.LoggerName != null && x.LoggerName.IndexOf(loggerName, StringComparison.InvariantCultureIgnoreCase) > -1) &&
(string.IsNullOrWhiteSpace(message) || x.Message != null && x.Message.IndexOf(message, StringComparison.InvariantCultureIgnoreCase) > -1));
};
// increase take, to account for customFiltering further reducing dataset
take = 1000;
}
// build the Azure table query
TableQuery <LogTableEntity> tableQuery = new TableQuery <LogTableEntity>()
.Select(new string[] { // reduce data fields returned from Azure
"Level",
"LoggerName",
"Message",
"EventTimeStamp",
"log4net_HostName"
});
if (!string.IsNullOrWhiteSpace(partitionKey))
{
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThanOrEqual, partitionKey));
}
if (!string.IsNullOrWhiteSpace(rowKey))
{
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, rowKey));
}
if (minLevel != Level.DEBUG)
{
// a number comparrison would be better, but log4net level and enum level don't match
switch (minLevel)
{
case Level.INFO: // show all except debug
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("Level", QueryComparisons.NotEqual, Level.DEBUG.ToString()));
break;
case Level.WARN: // show all except debug and info
tableQuery.AndWhere(TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("Level", QueryComparisons.NotEqual, Level.DEBUG.ToString()),
TableOperators.And,
TableQuery.GenerateFilterCondition("Level", QueryComparisons.NotEqual, Level.INFO.ToString())));
break;
case Level.ERROR: // show if error or fatal
tableQuery.AndWhere(TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("Level", QueryComparisons.Equal, Level.ERROR.ToString()),
TableOperators.Or,
TableQuery.GenerateFilterCondition("Level", QueryComparisons.Equal, Level.FATAL.ToString())));
break;
case Level.FATAL: // show fatal only
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("Level", QueryComparisons.Equal, Level.FATAL.ToString()));
break;
}
}
if (!loggerNameWildcardFiltering && !string.IsNullOrWhiteSpace(loggerName))
{
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("LoggerName", QueryComparisons.Equal, loggerName));
}
else
{
// HACK: ensure index entities are not returned
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("LoggerName", QueryComparisons.NotEqual, string.Empty));
}
if (!hostNameWildcardFiltering && !string.IsNullOrWhiteSpace(hostName))
{
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("log4net_HostName", QueryComparisons.Equal, hostName));
}
if (!string.IsNullOrWhiteSpace(sessionId))
{
tableQuery.AndWhere(TableQuery.GenerateFilterCondition("sessionId", QueryComparisons.Equal, sessionId));
}
tableQuery.Take(take);
TableContinuationToken tableContinuationToken = null;
TableQuerySegment <LogTableEntity> response;
do
{
// single Azure table storage requsest
response = cloudTable.ExecuteQuerySegmented(tableQuery, tableContinuationToken); // blocking
logTableEntities = response.Results.Where(x => customFiltering(x)).ToArray();
tableContinuationToken = response.ContinuationToken;
} while (!logTableEntities.Any() && tableContinuationToken != null);
return(logTableEntities);
}
