/// <summary> /// Wrapper method to handle any filtering (Azure table queries can't do wild card matching) /// </summary> /// <param name="appenderName"></param> /// <param name="partitionKey"></param> /// <param name="rowKey"></param> /// <param name="hostName"></param> /// <param name="loggerName"></param> /// <param name="messageIntro"></param> /// <returns></returns> internal IEnumerable <LogTableEntity> ReadLogTableEntities(string appenderName, string partitionKey, string rowKey, string hostName, string loggerName, Level minLevel, string message, string sessionId, int take) { // flags to indiciate if filtering is requied here in c# bool hostNameWildcardFiltering = !string.IsNullOrWhiteSpace(hostName) && !IndexService.Instance.GetMachineNames(appenderName).Any(x => x == hostName); bool loggerNameWildcardFiltering = !string.IsNullOrWhiteSpace(loggerName) && !IndexService.Instance.GetLoggerNames(appenderName).Any(x => x == loggerName); // check to see if any filtering needs to be done here (always filter message as it has no index) if (!string.IsNullOrWhiteSpace(message) || hostNameWildcardFiltering || loggerNameWildcardFiltering) { // additional filtering here- may require additional azure table queries List <LogTableEntity> logTableEntities = new List <LogTableEntity>(); // collection to return // calculate excess amount to request from a cloud query, which will filter down to the required take // machine name: low // logger name: low // level-debug: low // level-info: medium // level-warn: high // level-error: high // level-fatal: high // message: medium int lastCount; string lastPartitionKey = partitionKey; string lastRowKey = rowKey; int attempts = 0; bool finished = false; do { attempts++; if (attempts >= 3) { throw new TableQueryTimeoutException(lastPartitionKey, lastRowKey, logTableEntities.ToArray()); } lastCount = logTableEntities.Count; // take a large chunk to filter here - take size should be relative to the filter granularity IEnumerable <LogTableEntity> returnedLogTableEntities = this.ReadLogTableEntities( appenderName, lastPartitionKey, lastRowKey, minLevel, loggerNameWildcardFiltering ? null : loggerName, // only set if hostNameWildcardFiltering ? null : hostName, sessionId) // not using wildcards .Take(100); if (returnedLogTableEntities.Any()) { logTableEntities.AddRange(returnedLogTableEntities); // set last known, before filtering out lastPartitionKey = logTableEntities.Last().PartitionKey; lastRowKey = logTableEntities.Last().RowKey; // performing filtering on local list, otherwise it seems to affect table query performance (as every row in table returned and cast) logTableEntities = logTableEntities .Where(x => string.IsNullOrWhiteSpace(hostName) || (x.log4net_HostName != null && x.log4net_HostName.IndexOf(hostName, StringComparison.InvariantCultureIgnoreCase) > -1)) .Where(x => string.IsNullOrWhiteSpace(loggerName) || (x.LoggerName != null && x.LoggerName.IndexOf(loggerName, StringComparison.InvariantCultureIgnoreCase) > -1)) .Where(x => string.IsNullOrWhiteSpace(message) || (x.Message != null && x.Message.IndexOf(message, StringComparison.InvariantCultureIgnoreCase) > -1)) .ToList(); } else { // no data returned from Azure query finished = true; } }while (logTableEntities.Count < take && !finished); return(logTableEntities.Take(take)); // trim any excess } else { // filter at azure table query level only return(this.ReadLogTableEntities(appenderName, partitionKey, rowKey, minLevel, loggerName, hostName, sessionId).Take(take)); } }
/// <summary> /// Attempts to perform an Azure table query /// https://azure.microsoft.com/en-gb/documentation/articles/storage-dotnet-how-to-use-tables/ /// Gets a collection of LogTableEntity objs suitable for casting to LogItemInto /// </summary> /// <param name="appenderName"></param> /// <param name="partitionKey">null or the last known partition key</param> /// <param name="rowKey">null or the last known row key</param> /// <param name="minLevel"></param> /// <param name="loggerName">if set, looks for an exact match</param> /// <param name="hostName">if set, looks for an exact match</param> /// <returns>a collection of log items matching the supplied filter criteria</returns> private IEnumerable <LogTableEntity> ReadLogTableEntities(string appenderName, string partitionKey, string rowKey, Level minLevel, string loggerName, string hostName, string sessionId) { CloudTable cloudTable = this.GetCloudTable(appenderName); if (cloudTable != null) { TableQuery <LogTableEntity> tableQuery = new TableQuery <LogTableEntity>() .Select(new string[] { // reduce data fields returned from Azure "Level", "LoggerName", "Message", "EventTimeStamp", "log4net_HostName" }); if (!string.IsNullOrWhiteSpace(partitionKey)) { tableQuery.AndWhere(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThanOrEqual, partitionKey)); } if (!string.IsNullOrWhiteSpace(rowKey)) { tableQuery.AndWhere(TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, rowKey)); } if (minLevel != Level.DEBUG) { // a number comparrison would be better, but log4net level and enum level don't match switch (minLevel) { case Level.INFO: // show all except debug tableQuery.AndWhere(TableQuery.GenerateFilterCondition("Level", QueryComparisons.NotEqual, Level.DEBUG.ToString())); break; case Level.WARN: // show all except debug and info tableQuery.AndWhere(TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("Level", QueryComparisons.NotEqual, Level.DEBUG.ToString()), TableOperators.And, TableQuery.GenerateFilterCondition("Level", QueryComparisons.NotEqual, Level.INFO.ToString()))); break; case Level.ERROR: // show if error or fatal tableQuery.AndWhere(TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("Level", QueryComparisons.Equal, Level.ERROR.ToString()), TableOperators.Or, TableQuery.GenerateFilterCondition("Level", QueryComparisons.Equal, Level.FATAL.ToString()))); break; case Level.FATAL: // show fatal only tableQuery.AndWhere(TableQuery.GenerateFilterCondition("Level", QueryComparisons.Equal, Level.FATAL.ToString())); break; } } if (!string.IsNullOrWhiteSpace(loggerName)) { tableQuery.AndWhere(TableQuery.GenerateFilterCondition("LoggerName", QueryComparisons.Equal, loggerName)); } else { // HACK: ensure index entities are not returned tableQuery.AndWhere(TableQuery.GenerateFilterCondition("LoggerName", QueryComparisons.NotEqual, string.Empty)); } if (!string.IsNullOrWhiteSpace(hostName)) { tableQuery.AndWhere(TableQuery.GenerateFilterCondition("log4net_HostName", QueryComparisons.Equal, hostName)); } if (!string.IsNullOrWhiteSpace(sessionId)) { tableQuery.AndWhere(TableQuery.GenerateFilterCondition("sessionId", QueryComparisons.Equal, sessionId)); } return(cloudTable.ExecuteQuery( tableQuery, new TableRequestOptions() { ServerTimeout = new TimeSpan(0, 0, 2) })); } return(Enumerable.Empty <LogTableEntity>()); // fallback }
/// <summary> /// Queries Azure table storage until results are returned or a timeout is thown /// </summary> /// <param name="appenderName">name of the log4net appender</param> /// <param name="partitionKey">a partional key to begin search from (can be null)</param> /// <param name="rowKey">a row key to begin search from (can be null)</param> /// <param name="hostName">host name to filter on</param> /// <param name="loggerName">logger name to filter on</param> /// <param name="minLevel">logger level to filter</param> /// <param name="message">message text to filter</param> /// <param name="sessionId">session id to filter</param> /// <returns></returns> internal LogTableEntity[] ReadLogTableEntities(string appenderName, string partitionKey, string rowKey, string hostName, string loggerName, Level minLevel, string message, string sessionId) { LogTableEntity[] logTableEntities = new LogTableEntity[] { }; // default return value CloudTable cloudTable = this.GetCloudTable(appenderName); if (cloudTable == null) { return(logTableEntities); } int take = 50; // default take for Azure query bool hostNameWildcardFiltering = !string.IsNullOrWhiteSpace(hostName) && !IndexService.Instance.GetMachineNames(appenderName).Any(x => x == hostName); bool loggerNameWildcardFiltering = !string.IsNullOrWhiteSpace(loggerName) && !IndexService.Instance.GetLoggerNames(appenderName).Any(x => x == loggerName); // local filtering function applied to returned Azure table results Func <LogTableEntity, bool> customFiltering = (x) => { return(true); }; // default empty method (no custom filtering performed) // check to see if custom filtering (in c#) is required in addition to the Azure query if (hostNameWildcardFiltering || loggerNameWildcardFiltering || !string.IsNullOrWhiteSpace(message)) // message filtering always done in c# { customFiltering = (x) => { return((string.IsNullOrWhiteSpace(hostName) || x.log4net_HostName != null && x.log4net_HostName.IndexOf(hostName, StringComparison.InvariantCultureIgnoreCase) > -1) && (string.IsNullOrWhiteSpace(loggerName) || x.LoggerName != null && x.LoggerName.IndexOf(loggerName, StringComparison.InvariantCultureIgnoreCase) > -1) && (string.IsNullOrWhiteSpace(message) || x.Message != null && x.Message.IndexOf(message, StringComparison.InvariantCultureIgnoreCase) > -1)); }; // increase take, to account for customFiltering further reducing dataset take = 1000; } // build the Azure table query TableQuery <LogTableEntity> tableQuery = new TableQuery <LogTableEntity>() .Select(new string[] { // reduce data fields returned from Azure "Level", "LoggerName", "Message", "EventTimeStamp", "log4net_HostName" }); if (!string.IsNullOrWhiteSpace(partitionKey)) { tableQuery.AndWhere(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThanOrEqual, partitionKey)); } if (!string.IsNullOrWhiteSpace(rowKey)) { tableQuery.AndWhere(TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, rowKey)); } if (minLevel != Level.DEBUG) { // a number comparrison would be better, but log4net level and enum level don't match switch (minLevel) { case Level.INFO: // show all except debug tableQuery.AndWhere(TableQuery.GenerateFilterCondition("Level", QueryComparisons.NotEqual, Level.DEBUG.ToString())); break; case Level.WARN: // show all except debug and info tableQuery.AndWhere(TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("Level", QueryComparisons.NotEqual, Level.DEBUG.ToString()), TableOperators.And, TableQuery.GenerateFilterCondition("Level", QueryComparisons.NotEqual, Level.INFO.ToString()))); break; case Level.ERROR: // show if error or fatal tableQuery.AndWhere(TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("Level", QueryComparisons.Equal, Level.ERROR.ToString()), TableOperators.Or, TableQuery.GenerateFilterCondition("Level", QueryComparisons.Equal, Level.FATAL.ToString()))); break; case Level.FATAL: // show fatal only tableQuery.AndWhere(TableQuery.GenerateFilterCondition("Level", QueryComparisons.Equal, Level.FATAL.ToString())); break; } } if (!loggerNameWildcardFiltering && !string.IsNullOrWhiteSpace(loggerName)) { tableQuery.AndWhere(TableQuery.GenerateFilterCondition("LoggerName", QueryComparisons.Equal, loggerName)); } else { // HACK: ensure index entities are not returned tableQuery.AndWhere(TableQuery.GenerateFilterCondition("LoggerName", QueryComparisons.NotEqual, string.Empty)); } if (!hostNameWildcardFiltering && !string.IsNullOrWhiteSpace(hostName)) { tableQuery.AndWhere(TableQuery.GenerateFilterCondition("log4net_HostName", QueryComparisons.Equal, hostName)); } if (!string.IsNullOrWhiteSpace(sessionId)) { tableQuery.AndWhere(TableQuery.GenerateFilterCondition("sessionId", QueryComparisons.Equal, sessionId)); } tableQuery.Take(take); TableContinuationToken tableContinuationToken = null; TableQuerySegment <LogTableEntity> response; do { // single Azure table storage requsest response = cloudTable.ExecuteQuerySegmented(tableQuery, tableContinuationToken); // blocking logTableEntities = response.Results.Where(x => customFiltering(x)).ToArray(); tableContinuationToken = response.ContinuationToken; } while (!logTableEntities.Any() && tableContinuationToken != null); return(logTableEntities); }