/// <summary> /// Processes a full logset from end-to-end. /// </summary> public virtual void ProcessRequest(LogsharkRequest request) { var runTimer = request.RunContext.CreateTimer("Logshark Run", request.Target); // Update log4net to contain the CustomId and RunId properties for any consumers which wish to log them. LogicalThreadContext.Properties["CustomId"] = request.CustomId; LogicalThreadContext.Properties["RunId"] = request.RunId; LocalMongoProcessManager localMongoProcessManager = StartLocalMongoIfRequested(request); request.RunContext.CurrentPhase = ProcessingPhase.Pending; try { ExtractLogset(request); IArtifactProcessor artifactProcessor = InitializeArtifactProcessor(request); ProcessLogset(request, artifactProcessor); ExecutePlugins(request); SetRunSuccess(request); } catch (Exception ex) { SetRunFailed(request, ex); throw; } finally { LogsharkController.TearDown(request); StopLocalMongoIfRequested(request, localMongoProcessManager); runTimer.Stop(); Log.InfoFormat("Logshark run complete! [{0}]", runTimer.Elapsed.Print()); LogsharkController.DisplayRunSummary(request); } }
protected void StopLocalMongoIfRequested(LogsharkRequest request, LocalMongoProcessManager localMongoProcessManager) { if (request.StartLocalMongo) { LogsharkController.ShutDownLocalMongoDbInstance(localMongoProcessManager); } }
/// <summary> /// Processes a full logset from end-to-end. /// </summary> /// <param name="request">The user's processing request.</param> /// <returns>Run context containing the run outcome and details of what happened during the run.</returns> public LogsharkRunContext ProcessRequest(LogsharkRequest request) { // Clear any cached event timing data. GlobalEventTimingData.Clear(); // Update log4net to contain the CustomId and RunId properties for any consumers which wish to log them. LogicalThreadContext.Properties["CustomId"] = request.CustomId; LogicalThreadContext.Properties["RunId"] = request.RunId; try { using (new LocalMongoDatabaseManager(request)) { // Verify all external dependencies are up and available. var serviceDependencyValidator = new ServiceDependencyValidator(request.Configuration); serviceDependencyValidator.ValidateAllDependencies(); var metadataWriter = new LogsharkRunMetadataPostgresWriter(request.Configuration.PostgresConnectionInfo); return(ExecuteLogsharkRun(request, metadataWriter)); } } catch (Exception ex) { Log.FatalFormat("Logshark run failed: {0}", ex.Message); if (!String.IsNullOrWhiteSpace(ex.StackTrace)) { Log.Debug(ex.StackTrace); } throw; } }
public PluginExecutor(LogsharkRequest request) { logsharkRequest = request; mongoDatabase = request.Configuration.MongoConnectionInfo.GetDatabase(logsharkRequest.RunContext.MongoDatabaseName); outputDatabaseConnectionFactory = request.Configuration.PostgresConnectionInfo.GetConnectionFactory(logsharkRequest.PostgresDatabaseName); workbookPublisher = new WorkbookPublisher(logsharkRequest); }
protected void ExecutePlugins(LogsharkRequest request) { // Execute plugins. StartPhase(request, ProcessingPhase.ExecutingPlugins); LogsharkController.ExecutePlugins(request); metadataWriter.WritePluginExecutionMetadata(request); }
public LogsetMetadataReader(LogsharkRequest logsharkRequest) { IMongoDatabase mongoDatabase = logsharkRequest.Configuration.MongoConnectionInfo.GetDatabase(logsharkRequest.RunContext.MongoDatabaseName); MetadataCollection = mongoDatabase.GetCollection <LogsetMetadata>(LogsharkConstants.MONGO_METADATA_COLLECTION_NAME); MetadataDocumentId = logsharkRequest.RunContext.LogsetHash; }
public MongoWriter(LogsharkRequest request, IParserFactory parserFactory) { logsharkRequest = request; this.parserFactory = parserFactory; logsetPreprocessor = new LogsetPreprocessor(request, parserFactory); database = request.Configuration.MongoConnectionInfo.GetDatabase(request.RunContext.MongoDatabaseName); }
/// <summary> /// Set state on a request as to what kind of logset we are working with. /// </summary> /// <param name="request">The request to set the LogsetType parameter on.</param> public static void SetLogsetType(LogsharkRequest request) { var metadataReader = new LogsetMetadataReader(request); LogsetType logsetType = metadataReader.GetLogsetType(); request.RunContext.LogsetType = logsetType; }
/// <summary> /// Validates that a parsed logset contains at least one record. /// </summary> public static void ValidateMongoDatabaseContainsData(LogsharkRequest request) { if (!LogsetValidator.MongoDatabaseContainsRecords(request)) { throw new ProcessingException(String.Format("Mongo database {0} contains no valid log data!", request.RunContext.MongoDatabaseName)); } }
/// <summary> /// Display a summary of the run to the user, including locations of any assets. /// </summary> public static void DisplayRunSummary(LogsharkRequest request) { // Display logset hash, if relevant. if (!String.IsNullOrWhiteSpace(request.RunContext.LogsetHash)) { Log.InfoFormat("Logset hash for this run was '{0}'.", request.RunContext.LogsetHash); } // Display Postgres output location, if relevant. int pluginSuccesses = request.RunContext.PluginResponses.Count(pluginResponse => pluginResponse.SuccessfulExecution); if (pluginSuccesses > 0) { Log.InfoFormat("Plugin backing data was written to Postgres database '{0}\\{1}'.", request.Configuration.PostgresConnectionInfo, request.PostgresDatabaseName); } // A plugin may run successfully, yet not output a workbook. We only want to display the workbook output location if at least one workbook was output. int workbooksOutput = request.RunContext.PluginResponses.Sum(pluginResponse => pluginResponse.WorkbooksOutput.Count); if (workbooksOutput > 0) { Log.InfoFormat("Plugin workbook output was saved to '{0}'.", PluginExecutor.GetOutputLocation(request.RunId)); } // Display information about any published workbooks, if relevant. if (request.PublishWorkbooks && pluginSuccesses > 0) { Log.Info(WorkbookPublisher.BuildPublishingSummary(request.RunContext.PublishedWorkbooks)); } }
public MongoProcessingHeartbeatTimer(LogsharkRequest logsharkRequest) { metadataWriter = new LogsetMetadataWriter(logsharkRequest); long heartbeatDelayMs = 1000 * CoreConstants.MONGO_PROCESSING_HEARTBEAT_INTERVAL; timer = new Timer(WriteHeartbeat, null, 0, heartbeatDelayMs); }
/// <summary> /// Creates the appropriate collections & indexes, then parses the target logset into Mongo. /// </summary> public static void ParseLogset(LogsharkRequest request, IParserFactory parserFactory) { var mongoWriter = new MongoWriter(request, parserFactory); mongoWriter.ProcessLogset(); request.RunContext.IsValidLogset = true; }
/// <summary> /// Sets up and issues the LogsharkRequest to the LogsharkController. /// </summary> public void Execute() { if (commandLineOptions.ListPlugins) { try { LogsharkController.PrintAvailablePlugins(); return; } catch (Exception ex) { Log.FatalFormat("Unable to retrieve list of available plugins: {0}", ex.Message); throw; } } try { LogsharkRequest request = BuildLogsharkRequest(commandLineOptions); LogsharkRequestProcessor requestProcessor = InitializeRequestProcessor(); requestProcessor.ProcessRequest(request); } catch (Exception ex) { // Certain known exception types have already had their errors logged out by the core; we want to avoid duplicating error logging on these. if (!IsKnownExceptionType(ex)) { Log.Fatal(ex.GetFlattenedMessage()); } Log.Debug(ex); throw; } }
/// <summary> /// Handles any initialization tasks that need to happen with the Logshark request prior to processing a logset. /// </summary> public static void InitializeRequest(LogsharkRequest request) { Log.Info("Initializing Logshark.."); // Purge temp directory of any data left over from aborted runs. PurgeTempDirectory(); // Load all plugins required for this request. PluginLoader pluginLoader = new PluginLoader(request); request.RunContext.PluginTypesToExecute = pluginLoader.LoadPlugins(); // Compute the hash of the logset we are working with. try { request.RunContext.LogsetHash = ComputeLogsetHash(request.Target); } catch (Exception ex) { Log.FatalFormat("Unable to determine logset hash: {0}", ex.Message); throw; } if (request.Target.IsHashId) { // Disable force parse as we aren't actually working with a logset payload. request.ForceParse = false; } }
/// <summary> /// Sets up and issues the <see cref="LogsharkRequest"/> to the <see cref="LogsharkRequestProcessor"/>. /// </summary> /// <returns>Exit code</returns> public ExitCode Execute(LogsharkCommandLineOptions commandLineOptions) { if (commandLineOptions.ListPlugins) { try { LogsharkRequestProcessor.PrintAvailablePlugins(); return(ExitCode.Success); } catch (Exception ex) { Log.FatalFormat("Unable to retrieve list of available plugins: {0}", ex.Message); return(ExitCode.ExecutionError); } } try { LogsharkRequest request = BuildLogsharkRequest(commandLineOptions); var requestProcessor = new LogsharkRequestProcessor(); LogsharkRunContext outcome = requestProcessor.ProcessRequest(request); return(outcome.IsRunSuccessful.Equals(true) ? ExitCode.Success : ExitCode.ExecutionError); } catch (Exception ex) { Log.Debug(ex.GetFlattenedMessage()); Log.Debug(ex.StackTrace); return(ExitCode.ExecutionError); } }
public LogsetMetadataWriter(LogsharkRequest logsharkRequest) { this.logsharkRequest = logsharkRequest; logsetDatabase = logsharkRequest.Configuration.MongoConnectionInfo.GetDatabase(logsharkRequest.RunContext.MongoDatabaseName); logsetMetadataCollection = GetOrCreateMetadataCollection(); masterMetadataCollection = logsharkRequest.Configuration.MongoConnectionInfo.GetDatabase(CoreConstants.MONGO_METADATA_DATABASE_NAME).GetCollection <BsonDocument>(CoreConstants.MONGO_METADATA_COLLECTION_NAME); }
public static bool MongoDatabaseContainsRecords(LogsharkRequest request) { IMongoDatabase database = request.Configuration.MongoConnectionInfo.GetDatabase(request.RunContext.MongoDatabaseName); try { foreach (var collectionDocument in database.ListCollections().ToList()) { string collectionName = collectionDocument.GetValue("name").AsString; IMongoCollection <BsonDocument> collection = database.GetCollection <BsonDocument>(collectionName); if (MongoCollectionContainsRecords(collection)) { return(true); } } } catch (Exception ex) { Log.ErrorFormat("Encountered exception while validating contents of Mongo database {0}: {1}", request.RunContext.MongoDatabaseName, ex.Message); return(false); } return(false); }
protected IArtifactProcessor InitializeArtifactProcessor(LogsharkRequest request) { StartPhase(request, ProcessingPhase.Initializing); IArtifactProcessor artifactProcessor = LogsharkController.InitializeArtifactProcessor(request); metadataWriter.WriteCustomMetadata(request); return(artifactProcessor); }
/// <summary> /// Set state on a request as to the size of logset we are working with. /// </summary> /// <param name="request">The request to set the logset size on.</param> public static void SetLogsetSize(LogsharkRequest request) { var metadataReader = new LogsetMetadataReader(request); request.Target.UncompressedSize = metadataReader.GetLogsetUncompressedSize(); request.Target.CompressedSize = metadataReader.GetLogsetCompressedSize(); request.Target.ProcessedSize = metadataReader.GetLogsetProcessedSize(); }
/// <summary> /// Retrieves the list of files in the target directory which are required to process this request. /// </summary> protected static IEnumerable <string> GetRequiredFilesInDirectory(LogsharkRequest request) { IEnumerable <string> allFiles = Directory.GetFiles(request.Target, "*", SearchOption.AllDirectories); var whiteListPattern = LogsharkConstants.EXTRACTION_FILE_WHITELIST; var whiteListedFiles = allFiles.Where(file => whiteListPattern.IsMatch(Path.GetFileName(file))); return(whiteListedFiles.Where(file => LogsetDependencyHelper.IsLogfileRequiredForRequest(file, request.Target, request))); }
/// <summary> /// Diffs the collections we need to process the current logset against an existing set of collections for a given product type. /// </summary> /// <param name="request">The Logshark request object.</param> /// <param name="existingLogsetType">The product type of the existing logset.</param> /// <param name="existingCollections">The collections present in the existing logset.</param> /// <returns>Set of collections which are required to process the current request, but which don't exist already.</returns> public static ISet <string> GetMissingRequiredCollections(LogsharkRequest request, LogsetType existingLogsetType, IEnumerable <string> existingCollections) { var requiredCollections = GetCollectionDependencies(request, existingLogsetType); requiredCollections.ExceptWith(existingCollections); return(requiredCollections); }
/// <summary> /// Execute plugins requested by the user against an initialized logset. /// </summary> private PluginExecutionResult ExecutePlugins(LogsharkRequest request, RunInitializationResult initializationResult) { PublishingOptions publishingOptions = BuildPublishingOptions(request, initializationResult); var pluginExecutionRequest = new PluginExecutionRequest(initializationResult, publishingOptions, request.PluginCustomArguments, request.RunId, request.PostgresDatabaseName); var pluginExecutor = new PluginExecutor(request.Configuration); return(pluginExecutor.ExecutePlugins(pluginExecutionRequest)); }
public MongoInsertionFileProcessor(LogFileContext logFile, LogsharkRequest request, IParserFactory parserFactory) { this.logFile = logFile; parser = parserFactory.GetParser(logFile); mongoDatabase = request.Configuration.MongoConnectionInfo.GetDatabase(request.RunContext.MongoDatabaseName); ignoreDebugLogs = request.IgnoreDebugLogs; inFlightInsertions = new List <Thread>(); insertionQueue = new List <BsonDocument>(); }
/// <summary> /// Handles any initialization tasks associated with the run, such as extracting any archives, loading the relevant artifact processor and plugins. /// </summary> private RunInitializationResult InitializeRun(LogsharkRequest request) { // Blow out the application temp directory so that we start with as much disk space as possible. PurgeTempDirectory(request.Configuration); IRunInitializer runInitializer = RunInitializerFactory.GetRunInitializer(request.Target, request.Configuration); var initializationRequest = new RunInitializationRequest(request.Target, request.RunId, request.PluginsToExecute, request.ProcessFullLogset, request.Configuration.ArtifactProcessorOptions); return(runInitializer.Initialize(initializationRequest)); }
/// <summary> /// Spin up local MongoDB instance if the user requested it. /// </summary> protected LocalMongoProcessManager StartLocalMongoIfRequested(LogsharkRequest request) { LocalMongoProcessManager localMongoProcessManager = null; if (request.StartLocalMongo) { localMongoProcessManager = LogsharkController.StartLocalMongoDbInstance(request); } return(localMongoProcessManager); }
/// <summary> /// Unpacks the target logset and sets the root log directory. Contains logic to copy files locally if target is on a remote server. /// </summary> public static void ExtractLogFiles(LogsharkRequest request) { // Purge temp directory of any data left over from aborted runs. PurgeTempDirectory(); var availableArtifactProcessors = ArtifactProcessorLoader.LoadAllArtifactProcessors(); var extractionWhitelist = BuildExtractionWhitelist(availableArtifactProcessors); var extractor = new LogsetExtractor(request, extractionWhitelist); extractor.Process(); }
public LogsharkRunMetadata(LogsharkRequest request) { // Request & target data. CustomId = request.CustomId; DatabaseName = request.PostgresDatabaseName; RunId = request.RunId; RunByUser = Environment.UserName; RunByMachine = Environment.MachineName; Source = request.Source; Target = request.Target.OriginalTarget; TargetSize = request.Target.UncompressedSize; TargetCompressedSize = request.Target.CompressedSize; TargetProcessedSize = request.Target.ProcessedSize; VersionLogshark = typeof(LogsharkController).Assembly.GetName().Version.ToString(); // Timing data. FullRunStartTime = request.RequestCreationDate; FullRunElapsedSeconds = (DateTime.UtcNow - request.RequestCreationDate).TotalSeconds; LastMetadataUpdateTime = DateTime.UtcNow; LogsetExtractionStartTime = request.RunContext.GetStartTime("Unpack Logset"); LogsetExtractionElapsedSeconds = request.RunContext.GetElapsedTime("Unpack Logset"); LogParsingStartTime = request.RunContext.GetStartTime("Parsed Files"); LogParsingElapsedSeconds = request.RunContext.GetElapsedTime("Parsed Files"); PluginExecutionStartTime = request.RunContext.GetStartTime("Executed Plugins"); PluginExecutionElapsedSeconds = request.RunContext.GetElapsedTime("Executed Plugins"); // Context data. if (request.RunContext.MetadataRecordId.HasValue) { Id = request.RunContext.MetadataRecordId.Value; } ContainsSuccessfulPluginExecution = request.RunContext.PluginResponses.Any(pluginResponse => pluginResponse.SuccessfulExecution); CurrentProcessingPhase = request.RunContext.CurrentPhase.ToString(); CustomMetadataRecords = GetCustomMetadataRecords(request); IsRunSuccessful = request.RunContext.IsRunSuccessful; if (request.RunContext.CurrentPhase == ProcessingPhase.Complete) { IsRunComplete = true; } IsValidLogset = request.RunContext.IsValidLogset; LogsetHash = request.RunContext.LogsetHash; LogsetType = request.RunContext.LogsetType; PluginExecutionMetadataRecords = GetPluginExecutionMetadataRecords(request); PluginsExecuted = GetExecutedPluginsString(request); PluginsFailed = String.Join(",", request.RunContext.PluginResponses.Where(pluginResponse => !pluginResponse.SuccessfulExecution)); PublishedWorkbookMetadataRecords = request.RunContext.PublishedWorkbooks.Select(publishedWorkbook => new LogsharkPublishedWorkbookMetadata(request, this, publishedWorkbook)); RunFailureExceptionType = request.RunContext.RunFailureExceptionType; if (request.RunContext.RunFailurePhase.HasValue) { RunFailurePhase = request.RunContext.RunFailurePhase.ToString(); } RunFailureReason = request.RunContext.RunFailureReason; UtilizedExistingLogsetHash = request.RunContext.UtilizedExistingProcessedLogset; }
private string GetExecutedPluginsString(LogsharkRequest request) { ICollection <string> executedPlugins = GetExecutedPlugins(request); if (executedPlugins == null || executedPlugins.Count == 0) { return(null); } return(String.Join(",", executedPlugins)); }
public void WriteCustomMetadata(LogsharkRequest request) { try { var metadata = new LogsharkRunMetadata(request); InsertCustomMetadata(metadata); } catch (Exception ex) { throw new MetadataWriterException(String.Format("Failed to insert Logshark custom metadata records for run '{0}' in database: {1}", request.RunId, ex.Message)); } }
private string GetPluginVersion(LogsharkRequest request, string pluginName) { foreach (Type plugin in request.RunContext.PluginTypesToExecute) { if (plugin.Name.Equals(pluginName, StringComparison.InvariantCultureIgnoreCase)) { return(plugin.Assembly.GetName().Version.ToString()); } } return(null); }