protected ISet <string> GetAllSupportedCollections(IArtifactProcessor artifactProcessor, string rootLogLocation)
{
return(artifactProcessor.GetParserFactory(rootLogLocation)
.GetAllParsers()
.Select(parser => parser.CollectionSchema.CollectionName)
.ToHashSet());
}
/// <summary>
/// Loads all requested plugins for a given artifact processor.
/// </summary>
/// <returns>Set of loaded plugins.</returns>
public ISet <Type> LoadPlugins(ISet <string> requestedPlugins, IArtifactProcessor artifactProcessor)
{
var pluginLoadingOption = GetPluginLoadingOption(requestedPlugins);
if (pluginLoadingOption == PluginLoadingOption.None)
{
Log.Info("No plugins loaded due to user request.");
return(new HashSet <Type>());
}
// Load everything for the given artifact type.
ISet <Type> plugins = LoadSupportedPlugins(artifactProcessor);
// Filter down, if user requested it.
if (pluginLoadingOption == PluginLoadingOption.Default)
{
plugins = FilterPluginsByConfiguredDefaults(artifactProcessor, plugins);
}
else if (pluginLoadingOption == PluginLoadingOption.Named)
{
plugins = FilterPluginsByName(plugins, requestedPlugins);
}
DisplayPluginLoadingMessage(plugins);
return(plugins);
}
/// <summary>
/// Processes a full logset from end-to-end.
/// </summary>
public virtual void ProcessRequest(LogsharkRequest request)
{
var runTimer = request.RunContext.CreateTimer("Logshark Run", request.Target);
// Update log4net to contain the CustomId and RunId properties for any consumers which wish to log them.
LogicalThreadContext.Properties["CustomId"] = request.CustomId;
LogicalThreadContext.Properties["RunId"] = request.RunId;
LocalMongoProcessManager localMongoProcessManager = StartLocalMongoIfRequested(request);
request.RunContext.CurrentPhase = ProcessingPhase.Pending;
try
{
ExtractLogset(request);
IArtifactProcessor artifactProcessor = InitializeArtifactProcessor(request);
ProcessLogset(request, artifactProcessor);
ExecutePlugins(request);
SetRunSuccess(request);
}
catch (Exception ex)
{
SetRunFailed(request, ex);
throw;
}
finally
{
LogsharkController.TearDown(request);
StopLocalMongoIfRequested(request, localMongoProcessManager);
runTimer.Stop();
Log.InfoFormat("Logshark run complete! [{0}]", runTimer.Elapsed.Print());
LogsharkController.DisplayRunSummary(request);
}
}
public RunInitializationResult Initialize(RunInitializationRequest request)
{
if (request.Target.Type != LogsetTarget.File && request.Target.Type != LogsetTarget.Directory)
{
throw new ArgumentException("Request target must be a file or directory!", "request");
}
ExtractionResult extractionResult = ExtractLogset(request.Target, request.RunId);
var artifactProcessorLoader = new ArchiveArtifactProcessorLoader();
IArtifactProcessor artifactProcessor = artifactProcessorLoader.LoadArtifactProcessor(extractionResult.RootLogDirectory);
string logsetHash = ComputeLogsetHash(extractionResult.RootLogDirectory, artifactProcessor);
var pluginLoader = new PluginLoader(request.ArtifactProcessorOptions);
ISet <Type> pluginsToExecute = pluginLoader.LoadPlugins(request.RequestedPlugins, artifactProcessor);
var extractedTarget = new LogsharkRequestTarget(extractionResult.RootLogDirectory);
ISet <string> collectionDependencies;
if (request.ParseFullLogset)
{
collectionDependencies = GetAllSupportedCollections(artifactProcessor, extractionResult.RootLogDirectory);
}
else
{
collectionDependencies = pluginLoader.GetCollectionDependencies(pluginsToExecute);
}
return(new RunInitializationResult(extractedTarget, artifactProcessor, collectionDependencies, logsetHash, pluginsToExecute));
}
protected IArtifactProcessor InitializeArtifactProcessor(LogsharkRequest request)
{
StartPhase(request, ProcessingPhase.Initializing);
IArtifactProcessor artifactProcessor = LogsharkController.InitializeArtifactProcessor(request);
metadataWriter.WriteCustomMetadata(request);
return(artifactProcessor);
}
public RunInitializationResult(LogsharkRequestTarget target, IArtifactProcessor artifactProcessor, ISet <string> collectionsRequested, string logsetHash, ISet <Type> pluginsToExecute)
{
Target = target;
ArtifactProcessor = artifactProcessor;
ArtifactProcessorVersion = artifactProcessor.GetType().Assembly.GetName().Version;
CollectionsRequested = collectionsRequested;
CollectionsRequested.UnionWith(artifactProcessor.RequiredCollections);
LogsetHash = logsetHash;
PluginTypesToExecute = pluginsToExecute;
}
/// <summary>
/// Filters a collection of plugins by the defaults specified for that Artifact Processor in the LogsharkConfiguration.
/// </summary>
/// <param name="artifactProcessor">The artifact processor to load the defaults for.</param>
/// <param name="plugins">The set of plugins to filter.</param>
/// <returns>Set of plugins filtered down to only those that are specified as defaults for the given artifact processor.</returns>
protected ISet <Type> FilterPluginsByConfiguredDefaults(IArtifactProcessor artifactProcessor, ISet <Type> plugins)
{
try
{
LogsharkArtifactProcessorConfiguration artifactProcessorConfiguration = artifactProcessorOptions.LoadConfiguration(artifactProcessor.GetType());
return(FilterPluginsByName(plugins, artifactProcessorConfiguration.DefaultPlugins));
}
catch (KeyNotFoundException)
{
// If the user has not specified a default configuration for the given artifact processor, just default to running everything.
return(plugins);
}
}
protected string ComputeLogsetHash(string rootLogDirectory, IArtifactProcessor artifactProcessor)
{
Log.Info("Computing logset hash..");
try
{
string logsetHash = artifactProcessor.ComputeArtifactHash(rootLogDirectory);
Log.InfoFormat("Logset hash is '{0}'.", logsetHash);
return(logsetHash);
}
catch (Exception ex)
{
Log.FatalFormat("Unable to determine logset hash: {0}", ex.Message);
throw;
}
}
private static string ComputeLogsetHash(string rootLogDirectory, IArtifactProcessor artifactProcessor)
{
Log.Info("Computing logset hash..");
try
{
var logsetHash = artifactProcessor.ComputeArtifactHash(rootLogDirectory);
Log.InfoFormat($"Logset hash is '{logsetHash}'.");
return(logsetHash);
}
catch (Exception ex)
{
Log.FatalFormat($"Unable to determine logset hash: {ex.Message}");
throw;
}
}
/// <summary>
/// Encapsulates extracting and parsing logset.
/// </summary>
protected void ParseLogset(LogsharkRequest request, IArtifactProcessor artifactProcessor)
{
StartPhase(request, ProcessingPhase.Parsing);
try
{
LogsharkController.ParseLogset(request, artifactProcessor.GetParserFactory(request.RunContext.RootLogDirectory));
}
catch (Exception ex)
{
Log.FatalFormat("Encountered a fatal error while processing logset: {0}", ex.Message);
if (ex.InnerException != null)
{
Log.DebugFormat(ex.InnerException.StackTrace);
}
throw;
}
}
/// <summary>
/// Preprocesses the logset associated with the given artifact processor.
/// Partitions any partitionable files that exceed the configured size threshold.
/// </summary>
/// <returns>Queue of preprocessed log files.</returns>
public Queue <LogFileContext> Preprocess(string rootLogDirectory, IArtifactProcessor artifactProcessor, ISet <string> collectionsRequested)
{
// Sanity check.
if (!Directory.Exists(rootLogDirectory))
{
throw new ArgumentException(String.Format("{0} is not a valid directory.", rootLogDirectory));
}
IParserFactory parserFactory = artifactProcessor.GetParserFactory(rootLogDirectory);
// Gather a list of log files required to run the requested plugins for that type.
IEnumerable <LogFileContext> requiredLogs = LoadRequiredLogs(rootLogDirectory, artifactProcessor, parserFactory, collectionsRequested);
// Split large files into chunks for faster parallel processing.
var partitioner = new ConcurrentFilePartitioner(parserFactory, tuningOptions);
var filesToProcess = partitioner.PartitionLargeFiles(requiredLogs);
return(new Queue <LogFileContext>(filesToProcess.OrderByDescending(file => file.FileSize)));
}
/// <summary>
/// Loads all plugins in the Plugins directory which are supported by a given artifact processor.
/// </summary>
/// <returns>All plugins present in assemblies within the plugins directory supported by the given artifact processor.</returns>
protected static ISet <Type> LoadSupportedPlugins(IArtifactProcessor artifactProcessor)
{
ISet <Type> plugins = new HashSet <Type>();
Log.DebugFormat("Loading all available Logshark plugins for artifact processor {0}..", artifactProcessor.ArtifactType);
string pluginsDirectory = GetPluginsDirectory();
foreach (string pluginDll in Directory.GetFiles(pluginsDirectory, "*.dll"))
{
IList <Type> pluginTypes = LoadPluginsFromAssembly(pluginDll).ToList();
foreach (var pluginType in pluginTypes)
{
if (IsImplementationOfSupportedPluginInterface(pluginType, artifactProcessor.SupportedPluginInterfaces))
{
plugins.Add(pluginType);
}
}
}
return(plugins);
}
public RunInitializationResult Initialize(RunInitializationRequest request)
{
if (request.Target.Type != LogsetTarget.Hash)
{
throw new ArgumentException("Request target must be a logset hash!", "request");
}
var metadataReader = new MongoLogProcessingMetadataWriter(mongoConnectionInfo);
LogProcessingMetadata logsetMetadata = metadataReader.Read(request.Target);
if (logsetMetadata == null)
{
throw new InvalidTargetHashException(String.Format("No logset exists that matches logset hash '{0}'. Aborting..", request.Target));
}
var artifactProcessorLoader = new HashArtifactProcessorLoader();
IArtifactProcessor artifactProcessor = artifactProcessorLoader.LoadArtifactProcessor(logsetMetadata.ArtifactProcessorType);
var pluginLoader = new PluginLoader(request.ArtifactProcessorOptions);
ISet <Type> pluginsToExecute = pluginLoader.LoadPlugins(request.RequestedPlugins, artifactProcessor);
return(new RunInitializationResult(request.Target, artifactProcessor, logsetMetadata.CollectionsParsed, request.Target, pluginsToExecute));
}
/// <summary>
/// Takes action to process a logset based on the current status of the Logset.
/// </summary>
protected void ProcessLogset(LogsharkRequest request, IArtifactProcessor artifactProcessor)
{
LogsetStatus existingProcessedLogsetStatus = LogsharkController.GetExistingLogsetStatus(request);
if (request.ForceParse && !request.Target.IsHashId)
{
// If we are forcing a reparsing of this logset, first drop any existing logset in our MongoCluster which matches this hash-id.
if (existingProcessedLogsetStatus != LogsetStatus.NonExistent)
{
Log.InfoFormat("'Force Parse' request issued, dropping existing Mongo database '{0}'..", request.RunContext.MongoDatabaseName);
MongoAdminUtil.DropDatabase(request.Configuration.MongoConnectionInfo.GetClient(), request.RunContext.MongoDatabaseName);
}
ParseLogset(request, artifactProcessor);
return;
}
switch (existingProcessedLogsetStatus)
{
case LogsetStatus.NonExistent:
if (request.Target.IsHashId)
{
request.RunContext.IsValidLogset = false;
throw new InvalidTargetHashException(String.Format("No logset exists that matches logset hash '{0}'. Aborting..", request.RunContext.LogsetHash));
}
ParseLogset(request, artifactProcessor);
break;
case LogsetStatus.Corrupt:
if (request.Target.IsHashId)
{
request.RunContext.IsValidLogset = false;
throw new InvalidTargetHashException(String.Format("Mongo database matching logset hash '{0}' exists but is corrupted. Aborting..", request.RunContext.LogsetHash));
}
Log.InfoFormat("Logset matching hash '{0}' exists but is corrupted. Dropping it and reprocessing..", request.RunContext.MongoDatabaseName);
MongoAdminUtil.DropDatabase(request.Configuration.MongoConnectionInfo.GetClient(), request.RunContext.MongoDatabaseName);
ParseLogset(request, artifactProcessor);
break;
case LogsetStatus.InFlight:
string collisionErrorMessage = String.Format("Logset matching hash '{0}' exists but is currently being processed by another user. Aborting..", request.RunContext.MongoDatabaseName);
Log.InfoFormat(collisionErrorMessage);
throw new ProcessingUserCollisionException(collisionErrorMessage);
case LogsetStatus.Incomplete:
if (request.Target.IsHashId)
{
throw new InvalidTargetHashException("Found existing logset matching hash, but it is a partial logset that does not contain all of the data required to run specified plugins. Aborting..");
}
MongoAdminUtil.DropDatabase(request.Configuration.MongoConnectionInfo.GetClient(), request.RunContext.MongoDatabaseName);
Log.Info("Found existing logset matching hash, but it is a partial logset that does not contain all of the data required to run specified plugins. Dropping it and reprocessing..");
ParseLogset(request, artifactProcessor);
break;
case LogsetStatus.Indeterminable:
throw new IndeterminableLogsetStatusException("Unable to determine status of logset. Aborting..");
case LogsetStatus.Valid:
request.RunContext.UtilizedExistingProcessedLogset = true;
request.RunContext.IsValidLogset = true;
Log.Info("Found existing logset matching hash, skipping extraction and parsing.");
LogsetMetadataReader.SetLogsetType(request);
LogsetMetadataReader.SetLogsetSize(request);
break;
default:
throw new ArgumentOutOfRangeException(String.Format("'{0}' is not a valid LogsetStatus!", existingProcessedLogsetStatus));
}
LogsharkController.ValidateMongoDatabaseContainsData(request);
}
public MarkdownCodeArtifactCollection(IArtifactProcessor artifactProcessor) : base(artifactProcessor) { }
public MarkdownCodeArtifactCollection(IArtifactProcessor artifactProcessor) : base(artifactProcessor)
{
}
/// <summary>
/// Loads all of the logs found in the root log directory which are supported by the given artifact processor.
/// </summary>
/// <returns>Log contexts for all logs required for request.</returns>
protected IEnumerable <LogFileContext> LoadRequiredLogs(string rootLogDirectory, IArtifactProcessor artifactProcessor, IParserFactory parserFactory, ISet <string> collectionsRequested)
{
var logsToProcess = new List <LogFileContext>();
// Filter down to only supported files.
IEnumerable <FileInfo> supportedFiles = GetSupportedFiles(rootLogDirectory, parserFactory);
// Filter supported files to keep only what we need to populate the required collections.
foreach (var supportedFile in supportedFiles)
{
var parser = parserFactory.GetParser(supportedFile.FullName);
string collectionName = parser.CollectionSchema.CollectionName;
if (collectionsRequested.Contains(collectionName, StringComparer.InvariantCultureIgnoreCase))
{
logsToProcess.Add(new LogFileContext(supportedFile.FullName, rootLogDirectory, artifactProcessor.GetAdditionalFileMetadata));
}
}
return(logsToProcess);
}
