/// <summary>
/// Processes a full logset from end-to-end.
/// </summary>
public virtual void ProcessRequest(LogsharkRequest request)
{
var runTimer = request.RunContext.CreateTimer("Logshark Run", request.Target);
// Update log4net to contain the CustomId and RunId properties for any consumers which wish to log them.
LogicalThreadContext.Properties["CustomId"] = request.CustomId;
LogicalThreadContext.Properties["RunId"] = request.RunId;
LocalMongoProcessManager localMongoProcessManager = StartLocalMongoIfRequested(request);
request.RunContext.CurrentPhase = ProcessingPhase.Pending;
try
{
ExtractLogset(request);
IArtifactProcessor artifactProcessor = InitializeArtifactProcessor(request);
ProcessLogset(request, artifactProcessor);
ExecutePlugins(request);
SetRunSuccess(request);
}
catch (Exception ex)
{
SetRunFailed(request, ex);
throw;
}
finally
{
LogsharkController.TearDown(request);
StopLocalMongoIfRequested(request, localMongoProcessManager);
runTimer.Stop();
Log.InfoFormat("Logshark run complete! [{0}]", runTimer.Elapsed.Print());
LogsharkController.DisplayRunSummary(request);
}
}
protected void StopLocalMongoIfRequested(LogsharkRequest request, LocalMongoProcessManager localMongoProcessManager)
{
if (request.StartLocalMongo)
{
LogsharkController.ShutDownLocalMongoDbInstance(localMongoProcessManager);
}
}
/// <summary>
/// Processes a full logset from end-to-end.
/// </summary>
/// <param name="request">The user's processing request.</param>
/// <returns>Run context containing the run outcome and details of what happened during the run.</returns>
public LogsharkRunContext ProcessRequest(LogsharkRequest request)
{
// Clear any cached event timing data.
GlobalEventTimingData.Clear();
// Update log4net to contain the CustomId and RunId properties for any consumers which wish to log them.
LogicalThreadContext.Properties["CustomId"] = request.CustomId;
LogicalThreadContext.Properties["RunId"] = request.RunId;
try
{
using (new LocalMongoDatabaseManager(request))
{
// Verify all external dependencies are up and available.
var serviceDependencyValidator = new ServiceDependencyValidator(request.Configuration);
serviceDependencyValidator.ValidateAllDependencies();
var metadataWriter = new LogsharkRunMetadataPostgresWriter(request.Configuration.PostgresConnectionInfo);
return(ExecuteLogsharkRun(request, metadataWriter));
}
}
catch (Exception ex)
{
Log.FatalFormat("Logshark run failed: {0}", ex.Message);
if (!String.IsNullOrWhiteSpace(ex.StackTrace))
{
Log.Debug(ex.StackTrace);
}
throw;
}
}
public PluginExecutor(LogsharkRequest request)
{
logsharkRequest = request;
mongoDatabase = request.Configuration.MongoConnectionInfo.GetDatabase(logsharkRequest.RunContext.MongoDatabaseName);
outputDatabaseConnectionFactory = request.Configuration.PostgresConnectionInfo.GetConnectionFactory(logsharkRequest.PostgresDatabaseName);
workbookPublisher = new WorkbookPublisher(logsharkRequest);
}
protected void ExecutePlugins(LogsharkRequest request)
{
// Execute plugins.
StartPhase(request, ProcessingPhase.ExecutingPlugins);
LogsharkController.ExecutePlugins(request);
metadataWriter.WritePluginExecutionMetadata(request);
}
public LogsetMetadataReader(LogsharkRequest logsharkRequest)
{
IMongoDatabase mongoDatabase = logsharkRequest.Configuration.MongoConnectionInfo.GetDatabase(logsharkRequest.RunContext.MongoDatabaseName);
MetadataCollection = mongoDatabase.GetCollection <LogsetMetadata>(LogsharkConstants.MONGO_METADATA_COLLECTION_NAME);
MetadataDocumentId = logsharkRequest.RunContext.LogsetHash;
}
public MongoWriter(LogsharkRequest request, IParserFactory parserFactory)
{
logsharkRequest = request;
this.parserFactory = parserFactory;
logsetPreprocessor = new LogsetPreprocessor(request, parserFactory);
database = request.Configuration.MongoConnectionInfo.GetDatabase(request.RunContext.MongoDatabaseName);
}
/// <summary>
/// Set state on a request as to what kind of logset we are working with.
/// </summary>
/// <param name="request">The request to set the LogsetType parameter on.</param>
public static void SetLogsetType(LogsharkRequest request)
{
var metadataReader = new LogsetMetadataReader(request);
LogsetType logsetType = metadataReader.GetLogsetType();
request.RunContext.LogsetType = logsetType;
}
/// <summary>
/// Validates that a parsed logset contains at least one record.
/// </summary>
public static void ValidateMongoDatabaseContainsData(LogsharkRequest request)
{
if (!LogsetValidator.MongoDatabaseContainsRecords(request))
{
throw new ProcessingException(String.Format("Mongo database {0} contains no valid log data!", request.RunContext.MongoDatabaseName));
}
}
/// <summary>
/// Display a summary of the run to the user, including locations of any assets.
/// </summary>
public static void DisplayRunSummary(LogsharkRequest request)
{
// Display logset hash, if relevant.
if (!String.IsNullOrWhiteSpace(request.RunContext.LogsetHash))
{
Log.InfoFormat("Logset hash for this run was '{0}'.", request.RunContext.LogsetHash);
}
// Display Postgres output location, if relevant.
int pluginSuccesses = request.RunContext.PluginResponses.Count(pluginResponse => pluginResponse.SuccessfulExecution);
if (pluginSuccesses > 0)
{
Log.InfoFormat("Plugin backing data was written to Postgres database '{0}\\{1}'.", request.Configuration.PostgresConnectionInfo, request.PostgresDatabaseName);
}
// A plugin may run successfully, yet not output a workbook. We only want to display the workbook output location if at least one workbook was output.
int workbooksOutput = request.RunContext.PluginResponses.Sum(pluginResponse => pluginResponse.WorkbooksOutput.Count);
if (workbooksOutput > 0)
{
Log.InfoFormat("Plugin workbook output was saved to '{0}'.", PluginExecutor.GetOutputLocation(request.RunId));
}
// Display information about any published workbooks, if relevant.
if (request.PublishWorkbooks && pluginSuccesses > 0)
{
Log.Info(WorkbookPublisher.BuildPublishingSummary(request.RunContext.PublishedWorkbooks));
}
}
public MongoProcessingHeartbeatTimer(LogsharkRequest logsharkRequest)
{
metadataWriter = new LogsetMetadataWriter(logsharkRequest);
long heartbeatDelayMs = 1000 * CoreConstants.MONGO_PROCESSING_HEARTBEAT_INTERVAL;
timer = new Timer(WriteHeartbeat, null, 0, heartbeatDelayMs);
}
/// <summary>
/// Creates the appropriate collections & indexes, then parses the target logset into Mongo.
/// </summary>
public static void ParseLogset(LogsharkRequest request, IParserFactory parserFactory)
{
var mongoWriter = new MongoWriter(request, parserFactory);
mongoWriter.ProcessLogset();
request.RunContext.IsValidLogset = true;
}
/// <summary>
/// Sets up and issues the LogsharkRequest to the LogsharkController.
/// </summary>
public void Execute()
{
if (commandLineOptions.ListPlugins)
{
try
{
LogsharkController.PrintAvailablePlugins();
return;
}
catch (Exception ex)
{
Log.FatalFormat("Unable to retrieve list of available plugins: {0}", ex.Message);
throw;
}
}
try
{
LogsharkRequest request = BuildLogsharkRequest(commandLineOptions);
LogsharkRequestProcessor requestProcessor = InitializeRequestProcessor();
requestProcessor.ProcessRequest(request);
}
catch (Exception ex)
{
// Certain known exception types have already had their errors logged out by the core; we want to avoid duplicating error logging on these.
if (!IsKnownExceptionType(ex))
{
Log.Fatal(ex.GetFlattenedMessage());
}
Log.Debug(ex);
throw;
}
}
/// <summary>
/// Handles any initialization tasks that need to happen with the Logshark request prior to processing a logset.
/// </summary>
public static void InitializeRequest(LogsharkRequest request)
{
Log.Info("Initializing Logshark..");
// Purge temp directory of any data left over from aborted runs.
PurgeTempDirectory();
// Load all plugins required for this request.
PluginLoader pluginLoader = new PluginLoader(request);
request.RunContext.PluginTypesToExecute = pluginLoader.LoadPlugins();
// Compute the hash of the logset we are working with.
try
{
request.RunContext.LogsetHash = ComputeLogsetHash(request.Target);
}
catch (Exception ex)
{
Log.FatalFormat("Unable to determine logset hash: {0}", ex.Message);
throw;
}
if (request.Target.IsHashId)
{
// Disable force parse as we aren't actually working with a logset payload.
request.ForceParse = false;
}
}
/// <summary>
/// Sets up and issues the <see cref="LogsharkRequest"/> to the <see cref="LogsharkRequestProcessor"/>.
/// </summary>
/// <returns>Exit code</returns>
public ExitCode Execute(LogsharkCommandLineOptions commandLineOptions)
{
if (commandLineOptions.ListPlugins)
{
try
{
LogsharkRequestProcessor.PrintAvailablePlugins();
return(ExitCode.Success);
}
catch (Exception ex)
{
Log.FatalFormat("Unable to retrieve list of available plugins: {0}", ex.Message);
return(ExitCode.ExecutionError);
}
}
try
{
LogsharkRequest request = BuildLogsharkRequest(commandLineOptions);
var requestProcessor = new LogsharkRequestProcessor();
LogsharkRunContext outcome = requestProcessor.ProcessRequest(request);
return(outcome.IsRunSuccessful.Equals(true) ? ExitCode.Success : ExitCode.ExecutionError);
}
catch (Exception ex)
{
Log.Debug(ex.GetFlattenedMessage());
Log.Debug(ex.StackTrace);
return(ExitCode.ExecutionError);
}
}
public LogsetMetadataWriter(LogsharkRequest logsharkRequest)
{
this.logsharkRequest = logsharkRequest;
logsetDatabase = logsharkRequest.Configuration.MongoConnectionInfo.GetDatabase(logsharkRequest.RunContext.MongoDatabaseName);
logsetMetadataCollection = GetOrCreateMetadataCollection();
masterMetadataCollection = logsharkRequest.Configuration.MongoConnectionInfo.GetDatabase(CoreConstants.MONGO_METADATA_DATABASE_NAME).GetCollection <BsonDocument>(CoreConstants.MONGO_METADATA_COLLECTION_NAME);
}
public static bool MongoDatabaseContainsRecords(LogsharkRequest request)
{
IMongoDatabase database = request.Configuration.MongoConnectionInfo.GetDatabase(request.RunContext.MongoDatabaseName);
try
{
foreach (var collectionDocument in database.ListCollections().ToList())
{
string collectionName = collectionDocument.GetValue("name").AsString;
IMongoCollection <BsonDocument> collection = database.GetCollection <BsonDocument>(collectionName);
if (MongoCollectionContainsRecords(collection))
{
return(true);
}
}
}
catch (Exception ex)
{
Log.ErrorFormat("Encountered exception while validating contents of Mongo database {0}: {1}", request.RunContext.MongoDatabaseName, ex.Message);
return(false);
}
return(false);
}
protected IArtifactProcessor InitializeArtifactProcessor(LogsharkRequest request)
{
StartPhase(request, ProcessingPhase.Initializing);
IArtifactProcessor artifactProcessor = LogsharkController.InitializeArtifactProcessor(request);
metadataWriter.WriteCustomMetadata(request);
return(artifactProcessor);
}
/// <summary>
/// Set state on a request as to the size of logset we are working with.
/// </summary>
/// <param name="request">The request to set the logset size on.</param>
public static void SetLogsetSize(LogsharkRequest request)
{
var metadataReader = new LogsetMetadataReader(request);
request.Target.UncompressedSize = metadataReader.GetLogsetUncompressedSize();
request.Target.CompressedSize = metadataReader.GetLogsetCompressedSize();
request.Target.ProcessedSize = metadataReader.GetLogsetProcessedSize();
}
/// <summary>
/// Retrieves the list of files in the target directory which are required to process this request.
/// </summary>
protected static IEnumerable <string> GetRequiredFilesInDirectory(LogsharkRequest request)
{
IEnumerable <string> allFiles = Directory.GetFiles(request.Target, "*", SearchOption.AllDirectories);
var whiteListPattern = LogsharkConstants.EXTRACTION_FILE_WHITELIST;
var whiteListedFiles = allFiles.Where(file => whiteListPattern.IsMatch(Path.GetFileName(file)));
return(whiteListedFiles.Where(file => LogsetDependencyHelper.IsLogfileRequiredForRequest(file, request.Target, request)));
}
/// <summary>
/// Diffs the collections we need to process the current logset against an existing set of collections for a given product type.
/// </summary>
/// <param name="request">The Logshark request object.</param>
/// <param name="existingLogsetType">The product type of the existing logset.</param>
/// <param name="existingCollections">The collections present in the existing logset.</param>
/// <returns>Set of collections which are required to process the current request, but which don't exist already.</returns>
public static ISet <string> GetMissingRequiredCollections(LogsharkRequest request, LogsetType existingLogsetType, IEnumerable <string> existingCollections)
{
var requiredCollections = GetCollectionDependencies(request, existingLogsetType);
requiredCollections.ExceptWith(existingCollections);
return(requiredCollections);
}
/// <summary>
/// Execute plugins requested by the user against an initialized logset.
/// </summary>
private PluginExecutionResult ExecutePlugins(LogsharkRequest request, RunInitializationResult initializationResult)
{
PublishingOptions publishingOptions = BuildPublishingOptions(request, initializationResult);
var pluginExecutionRequest = new PluginExecutionRequest(initializationResult, publishingOptions, request.PluginCustomArguments, request.RunId, request.PostgresDatabaseName);
var pluginExecutor = new PluginExecutor(request.Configuration);
return(pluginExecutor.ExecutePlugins(pluginExecutionRequest));
}
public MongoInsertionFileProcessor(LogFileContext logFile, LogsharkRequest request, IParserFactory parserFactory)
{
this.logFile = logFile;
parser = parserFactory.GetParser(logFile);
mongoDatabase = request.Configuration.MongoConnectionInfo.GetDatabase(request.RunContext.MongoDatabaseName);
ignoreDebugLogs = request.IgnoreDebugLogs;
inFlightInsertions = new List <Thread>();
insertionQueue = new List <BsonDocument>();
}
/// <summary>
/// Handles any initialization tasks associated with the run, such as extracting any archives, loading the relevant artifact processor and plugins.
/// </summary>
private RunInitializationResult InitializeRun(LogsharkRequest request)
{
// Blow out the application temp directory so that we start with as much disk space as possible.
PurgeTempDirectory(request.Configuration);
IRunInitializer runInitializer = RunInitializerFactory.GetRunInitializer(request.Target, request.Configuration);
var initializationRequest = new RunInitializationRequest(request.Target, request.RunId, request.PluginsToExecute, request.ProcessFullLogset, request.Configuration.ArtifactProcessorOptions);
return(runInitializer.Initialize(initializationRequest));
}
/// <summary>
/// Spin up local MongoDB instance if the user requested it.
/// </summary>
protected LocalMongoProcessManager StartLocalMongoIfRequested(LogsharkRequest request)
{
LocalMongoProcessManager localMongoProcessManager = null;
if (request.StartLocalMongo)
{
localMongoProcessManager = LogsharkController.StartLocalMongoDbInstance(request);
}
return(localMongoProcessManager);
}
/// <summary>
/// Unpacks the target logset and sets the root log directory. Contains logic to copy files locally if target is on a remote server.
/// </summary>
public static void ExtractLogFiles(LogsharkRequest request)
{
// Purge temp directory of any data left over from aborted runs.
PurgeTempDirectory();
var availableArtifactProcessors = ArtifactProcessorLoader.LoadAllArtifactProcessors();
var extractionWhitelist = BuildExtractionWhitelist(availableArtifactProcessors);
var extractor = new LogsetExtractor(request, extractionWhitelist);
extractor.Process();
}
public LogsharkRunMetadata(LogsharkRequest request)
{
// Request & target data.
CustomId = request.CustomId;
DatabaseName = request.PostgresDatabaseName;
RunId = request.RunId;
RunByUser = Environment.UserName;
RunByMachine = Environment.MachineName;
Source = request.Source;
Target = request.Target.OriginalTarget;
TargetSize = request.Target.UncompressedSize;
TargetCompressedSize = request.Target.CompressedSize;
TargetProcessedSize = request.Target.ProcessedSize;
VersionLogshark = typeof(LogsharkController).Assembly.GetName().Version.ToString();
// Timing data.
FullRunStartTime = request.RequestCreationDate;
FullRunElapsedSeconds = (DateTime.UtcNow - request.RequestCreationDate).TotalSeconds;
LastMetadataUpdateTime = DateTime.UtcNow;
LogsetExtractionStartTime = request.RunContext.GetStartTime("Unpack Logset");
LogsetExtractionElapsedSeconds = request.RunContext.GetElapsedTime("Unpack Logset");
LogParsingStartTime = request.RunContext.GetStartTime("Parsed Files");
LogParsingElapsedSeconds = request.RunContext.GetElapsedTime("Parsed Files");
PluginExecutionStartTime = request.RunContext.GetStartTime("Executed Plugins");
PluginExecutionElapsedSeconds = request.RunContext.GetElapsedTime("Executed Plugins");
// Context data.
if (request.RunContext.MetadataRecordId.HasValue)
{
Id = request.RunContext.MetadataRecordId.Value;
}
ContainsSuccessfulPluginExecution = request.RunContext.PluginResponses.Any(pluginResponse => pluginResponse.SuccessfulExecution);
CurrentProcessingPhase = request.RunContext.CurrentPhase.ToString();
CustomMetadataRecords = GetCustomMetadataRecords(request);
IsRunSuccessful = request.RunContext.IsRunSuccessful;
if (request.RunContext.CurrentPhase == ProcessingPhase.Complete)
{
IsRunComplete = true;
}
IsValidLogset = request.RunContext.IsValidLogset;
LogsetHash = request.RunContext.LogsetHash;
LogsetType = request.RunContext.LogsetType;
PluginExecutionMetadataRecords = GetPluginExecutionMetadataRecords(request);
PluginsExecuted = GetExecutedPluginsString(request);
PluginsFailed = String.Join(",", request.RunContext.PluginResponses.Where(pluginResponse => !pluginResponse.SuccessfulExecution));
PublishedWorkbookMetadataRecords = request.RunContext.PublishedWorkbooks.Select(publishedWorkbook => new LogsharkPublishedWorkbookMetadata(request, this, publishedWorkbook));
RunFailureExceptionType = request.RunContext.RunFailureExceptionType;
if (request.RunContext.RunFailurePhase.HasValue)
{
RunFailurePhase = request.RunContext.RunFailurePhase.ToString();
}
RunFailureReason = request.RunContext.RunFailureReason;
UtilizedExistingLogsetHash = request.RunContext.UtilizedExistingProcessedLogset;
}
private string GetExecutedPluginsString(LogsharkRequest request)
{
ICollection <string> executedPlugins = GetExecutedPlugins(request);
if (executedPlugins == null || executedPlugins.Count == 0)
{
return(null);
}
return(String.Join(",", executedPlugins));
}
public void WriteCustomMetadata(LogsharkRequest request)
{
try
{
var metadata = new LogsharkRunMetadata(request);
InsertCustomMetadata(metadata);
}
catch (Exception ex)
{
throw new MetadataWriterException(String.Format("Failed to insert Logshark custom metadata records for run '{0}' in database: {1}", request.RunId, ex.Message));
}
}
private string GetPluginVersion(LogsharkRequest request, string pluginName)
{
foreach (Type plugin in request.RunContext.PluginTypesToExecute)
{
if (plugin.Name.Equals(pluginName, StringComparison.InvariantCultureIgnoreCase))
{
return(plugin.Assembly.GetName().Version.ToString());
}
}
return(null);
}
