public LogsetParsingResult ProcessLogset(LogsetParsingRequest request, LogsetProcessingStatus existingProcessedLogsetStatus)
{
switch (existingProcessedLogsetStatus.State)
{
case ProcessedLogsetState.NonExistent:
throw new InvalidTargetHashException(String.Format("No logset exists that matches logset hash '{0}'. Aborting..", request.LogsetHash));
case ProcessedLogsetState.Corrupt:
throw new InvalidTargetHashException(String.Format("Mongo database matching logset hash '{0}' exists but is corrupted. Aborting..", request.LogsetHash));
case ProcessedLogsetState.InFlight:
throw new ProcessingUserCollisionException(String.Format("Logset matching hash '{0}' exists but is currently being processed by another user. Aborting..", request.LogsetHash));
case ProcessedLogsetState.Incomplete:
throw new InvalidTargetHashException("Found existing logset matching hash, but it is a partial logset that does not contain all of the data required to run specified plugins. Aborting..");
case ProcessedLogsetState.Indeterminable:
throw new IndeterminableLogsetStatusException("Unable to determine status of logset. Aborting..");
case ProcessedLogsetState.Valid:
Log.Info("Found existing logset matching hash! Skipping extraction and parsing.");
return(new LogsetParsingResult(new List <string>(), existingProcessedLogsetStatus.ProcessedDataVolumeBytes, utilizedExistingProcessedLogset: true));
default:
throw new ArgumentOutOfRangeException(String.Format("'{0}' is not a valid LogsetProcessingState!", existingProcessedLogsetStatus.State));
}
}
protected override void Initialize(LogsetParsingRequest request)
{
IMongoDatabase database = mongoConnectionInfo.GetDatabase(request.LogsetHash);
IParserFactory parserFactory = request.ArtifactProcessor.GetParserFactory(request.Target);
var metadata = new LogProcessingMetadata(request);
metadataWriter.Write(metadata, request.LogsetHash);
CreateMongoDbCollections(request.CollectionsToParse, database, parserFactory);
}
protected override void Finalize(LogsetParsingRequest request, LogsetParsingResult result)
{
var metadata = new LogProcessingMetadata(request)
{
ProcessedSuccessfully = true,
ProcessedSize = result.ParsedDataVolumeBytes,
FailedFileParses = result.FailedFileParses
};
metadataWriter.Write(metadata, request.LogsetHash);
metadataWriter.WriteMasterMetadataRecord(metadata);
}
/// <summary>
/// Takes action to process a logset based on the current status of the Logset.
/// </summary>
private LogsetParsingResult ProcessLogset(LogsharkRequest request, RunInitializationResult runInitializationResult)
{
var statusChecker = new LogsetProcessingStatusChecker(request.Configuration.MongoConnectionInfo);
LogsetProcessingStatus existingProcessedLogsetStatus = statusChecker.GetStatus(runInitializationResult.LogsetHash, runInitializationResult.CollectionsRequested);
Func <LogsetParsingRequest, LogsetParsingResult> parseLogset = logsetParsingRequest => ParseLogset(logsetParsingRequest, request.Configuration);
Action <string> dropLogset = logsetHash => MongoAdminHelper.DropDatabase(request.Configuration.MongoConnectionInfo.GetClient(), logsetHash);
var parsingRequest = new LogsetParsingRequest(runInitializationResult, request.ForceParse);
ILogsetProcessingStrategy processingStrategy = LogsetProcessingStrategyFactory.GetLogsetProcessingStrategy(request.Target, parseLogset, dropLogset, request.Configuration);
return(processingStrategy.ProcessLogset(parsingRequest, existingProcessedLogsetStatus));
}
public LogProcessingMetadata(LogsetParsingRequest request)
{
Id = request.LogsetHash;
Target = request.Target;
TargetSize = request.Target.Size;
LogsetType = request.ArtifactProcessor.ArtifactType;
ProcessedSuccessfully = false;
ProcessingTimestamp = request.CreationTimestamp;
ProcessingTime = GlobalEventTimingData.GetElapsedTime("Parsed Files", request.LogsetHash);
User = Environment.UserName;
Machine = Environment.MachineName;
LogsharkVersion = typeof(LogsharkRequestProcessor).Assembly.GetName().Version.ToString();
ArtifactProcessorType = request.ArtifactProcessor.GetType().Name;
ArtifactProcessorVersion = request.ArtifactProcessor.GetType().Assembly.GetName().Version;
CollectionsParsed = new SortedSet <string>(request.CollectionsToParse);
}
/// <summary>
/// Encapsulates extracting and parsing logset.
/// </summary>
private LogsetParsingResult ParseLogset(LogsetParsingRequest parsingRequest, LogsharkConfiguration config)
{
try
{
var parser = new MongoLogsetParser(config.MongoConnectionInfo, config.TuningOptions);
return(parser.ParseLogset(parsingRequest));
}
catch (Exception ex)
{
Log.FatalFormat("Encountered a fatal error while processing logset: {0}", ex.Message);
if (ex.InnerException != null)
{
Log.DebugFormat(ex.InnerException.StackTrace);
}
throw;
}
}
public LogsetParsingResult ProcessLogset(LogsetParsingRequest request, LogsetProcessingStatus existingProcessedLogsetStatus)
{
// If the user requested a forced reparsing of this logset, first drop the existing logset.
if (request.ForceParse && existingProcessedLogsetStatus.State != ProcessedLogsetState.NonExistent)
{
Log.InfoFormat("'Force Parse' request issued, dropping existing logset '{0}'..", request.LogsetHash);
dropExistingLogset(request.LogsetHash);
return(parseLogset(request));
}
switch (existingProcessedLogsetStatus.State)
{
case ProcessedLogsetState.NonExistent:
return(parseLogset(request));
case ProcessedLogsetState.Corrupt:
Log.InfoFormat("Logset matching hash '{0}' exists but is corrupted. Dropping it and reprocessing..", request.LogsetHash);
dropExistingLogset(request.LogsetHash);
return(parseLogset(request));
case ProcessedLogsetState.InFlight:
throw new ProcessingUserCollisionException(String.Format("Logset matching hash '{0}' exists but is currently being processed by another user. Aborting..", request.LogsetHash));
case ProcessedLogsetState.Incomplete:
dropExistingLogset(request.LogsetHash);
Log.Info("Found existing logset matching hash, but it is a partial logset that does not contain all of the data required to run specified plugins. Dropping it and reprocessing..");
return(parseLogset(request));
case ProcessedLogsetState.Indeterminable:
throw new IndeterminableLogsetStatusException("Unable to determine status of logset. Aborting..");
case ProcessedLogsetState.Valid:
Log.Info("Found existing logset matching hash! Skipping extraction and parsing.");
return(new LogsetParsingResult(new List <string>(), existingProcessedLogsetStatus.ProcessedDataVolumeBytes, utilizedExistingProcessedLogset: true));
default:
throw new ArgumentOutOfRangeException(String.Format("'{0}' is not a valid LogsetProcessingState!", existingProcessedLogsetStatus.State));
}
}
protected override IDisposable GetProcessingWrapper(LogsetParsingRequest request)
{
return(new MongoProcessingHeartbeatTimer(metadataWriter, request.LogsetHash));
}
