/// <summary>
/// Creates all of the required MongoDB collections that this logset requires.
/// </summary>
private void CreateMongoDbCollections()
{
var collections = new Dictionary <string, HashSet <string> >();
ISet <IParser> parsers = parserFactory.GetAllParsers();
// Stuff collection names & indexes into the dictionary, deduping in the process.
foreach (var parser in parsers)
{
var collectionName = parser.CollectionSchema.CollectionName.ToLowerInvariant();
IList <string> indexes = parser.CollectionSchema.Indexes;
if (!collections.ContainsKey(collectionName))
{
if (LogsetDependencyHelper.IsCollectionRequiredForRequest(collectionName, logsharkRequest))
{
collections.Add(collectionName, new HashSet <string>());
}
}
// Add indexes.
if (collections.ContainsKey(collectionName))
{
foreach (var index in indexes)
{
if (collections.ContainsKey(collectionName))
{
collections[collectionName].Add(index);
}
}
}
}
// New up collections & indexes using the dictionary.
foreach (var collection in collections)
{
var collectionName = collection.Key;
ISet <string> indexes = collection.Value;
var dbCollection = database.GetCollection <BsonDocument>(collectionName);
logsharkRequest.RunContext.CollectionsGenerated.Add(collectionName);
foreach (var index in indexes)
{
var indexKeysBuilder = new IndexKeysDefinitionBuilder <BsonDocument>();
CreateIndexOptions indexOptions = new CreateIndexOptions {
Sparse = false
};
dbCollection.Indexes.CreateOne(indexKeysBuilder.Ascending(index), indexOptions);
}
// If we are working against a sharded Mongo cluster, we need to explicitly shard each collection.
MongoConnectionInfo mongoConnectionInfo = logsharkRequest.Configuration.MongoConnectionInfo;
if (mongoConnectionInfo.ConnectionType == MongoConnectionType.ShardedCluster)
{
MongoAdminUtil.EnableShardingOnCollectionIfNotEnabled(mongoConnectionInfo.GetClient(), logsharkRequest.RunContext.MongoDatabaseName, collectionName);
}
}
}
public IMongoDatabase GetDatabase(string name)
{
var mongoClient = GetClient();
IMongoDatabase database = mongoClient.GetDatabase(name);
// If we are using a connection to a sharded database, we need to explicitly enable sharding on the db.
if (ConnectionType == MongoConnectionType.ShardedCluster)
{
MongoAdminUtil.EnableShardingOnDatabaseIfNotEnabled(mongoClient, name);
}
return(database);
}
protected MongoConnectionType GetConnectionType()
{
bool isCluster;
try
{
isCluster = MongoAdminUtil.IsMongoCluster(client);
}
catch (MongoException ex)
{
Log.DebugFormat("Unable to determine if current MongoDB connection is a sharded cluster: {0}", ex.Message);
return(MongoConnectionType.Undetermined);
}
if (isCluster)
{
return(MongoConnectionType.ShardedCluster);
}
return(MongoConnectionType.SingleNode);
}
/// <summary>
/// Performs any teardown tasks.
/// </summary>
public static void TearDown(LogsharkRequest request)
{
// Drop logset if user didn't want to retain it, assuming they didn't piggyback on an existing processed logset.
if (request.DropMongoDBPostRun && !request.RunContext.UtilizedExistingProcessedLogset)
{
try
{
Log.InfoFormat("Dropping Mongo database {0}..", request.RunContext.MongoDatabaseName);
MongoAdminUtil.DropDatabase(request.Configuration.MongoConnectionInfo.GetClient(), request.RunContext.MongoDatabaseName);
// Remove metadata record for this run from master metadata DB.
LogsetMetadataWriter metadataWriter = new LogsetMetadataWriter(request);
metadataWriter.DeleteMasterMetadataRecord();
}
catch (Exception ex)
{
Log.ErrorFormat("Failed to clean up DB {0}: {1}", request.RunContext.MongoDatabaseName, ex.Message);
}
}
LogsetExtractor.CleanUpRun(request.RunId);
}
private static bool RemoteLogsetHasData(LogsharkRequest request)
{
return(MongoAdminUtil.DatabaseExists(request.Configuration.MongoConnectionInfo.GetClient(), request.RunContext.MongoDatabaseName));
}
/// <summary>
/// Takes action to process a logset based on the current status of the Logset.
/// </summary>
protected void ProcessLogset(LogsharkRequest request, IArtifactProcessor artifactProcessor)
{
LogsetStatus existingProcessedLogsetStatus = LogsharkController.GetExistingLogsetStatus(request);
if (request.ForceParse && !request.Target.IsHashId)
{
// If we are forcing a reparsing of this logset, first drop any existing logset in our MongoCluster which matches this hash-id.
if (existingProcessedLogsetStatus != LogsetStatus.NonExistent)
{
Log.InfoFormat("'Force Parse' request issued, dropping existing Mongo database '{0}'..", request.RunContext.MongoDatabaseName);
MongoAdminUtil.DropDatabase(request.Configuration.MongoConnectionInfo.GetClient(), request.RunContext.MongoDatabaseName);
}
ParseLogset(request, artifactProcessor);
return;
}
switch (existingProcessedLogsetStatus)
{
case LogsetStatus.NonExistent:
if (request.Target.IsHashId)
{
request.RunContext.IsValidLogset = false;
throw new InvalidTargetHashException(String.Format("No logset exists that matches logset hash '{0}'. Aborting..", request.RunContext.LogsetHash));
}
ParseLogset(request, artifactProcessor);
break;
case LogsetStatus.Corrupt:
if (request.Target.IsHashId)
{
request.RunContext.IsValidLogset = false;
throw new InvalidTargetHashException(String.Format("Mongo database matching logset hash '{0}' exists but is corrupted. Aborting..", request.RunContext.LogsetHash));
}
Log.InfoFormat("Logset matching hash '{0}' exists but is corrupted. Dropping it and reprocessing..", request.RunContext.MongoDatabaseName);
MongoAdminUtil.DropDatabase(request.Configuration.MongoConnectionInfo.GetClient(), request.RunContext.MongoDatabaseName);
ParseLogset(request, artifactProcessor);
break;
case LogsetStatus.InFlight:
string collisionErrorMessage = String.Format("Logset matching hash '{0}' exists but is currently being processed by another user. Aborting..", request.RunContext.MongoDatabaseName);
Log.InfoFormat(collisionErrorMessage);
throw new ProcessingUserCollisionException(collisionErrorMessage);
case LogsetStatus.Incomplete:
if (request.Target.IsHashId)
{
throw new InvalidTargetHashException("Found existing logset matching hash, but it is a partial logset that does not contain all of the data required to run specified plugins. Aborting..");
}
MongoAdminUtil.DropDatabase(request.Configuration.MongoConnectionInfo.GetClient(), request.RunContext.MongoDatabaseName);
Log.Info("Found existing logset matching hash, but it is a partial logset that does not contain all of the data required to run specified plugins. Dropping it and reprocessing..");
ParseLogset(request, artifactProcessor);
break;
case LogsetStatus.Indeterminable:
throw new IndeterminableLogsetStatusException("Unable to determine status of logset. Aborting..");
case LogsetStatus.Valid:
request.RunContext.UtilizedExistingProcessedLogset = true;
request.RunContext.IsValidLogset = true;
Log.Info("Found existing logset matching hash, skipping extraction and parsing.");
LogsetMetadataReader.SetLogsetType(request);
LogsetMetadataReader.SetLogsetSize(request);
break;
default:
throw new ArgumentOutOfRangeException(String.Format("'{0}' is not a valid LogsetStatus!", existingProcessedLogsetStatus));
}
LogsharkController.ValidateMongoDatabaseContainsData(request);
}
