public IEnumerable <IndexQueryResult> Query()
{
using (IndexStorage.EnsureInvariantCulture())
{
AssertQueryDoesNotContainFieldsThatAreNotIndexes();
IndexSearcher indexSearcher;
using (parent.GetSearcher(out indexSearcher))
{
var luceneQuery = GetLuceneQuery();
foreach (var indexQueryTrigger in indexQueryTriggers)
{
luceneQuery = indexQueryTrigger.Value.ProcessQuery(parent.name, luceneQuery, indexQuery);
}
int start = indexQuery.Start;
int pageSize = indexQuery.PageSize;
int returnedResults = 0;
int skippedResultsInCurrentLoop = 0;
do
{
if (skippedResultsInCurrentLoop > 0)
{
start = start + pageSize;
// trying to guesstimate how many results we will need to read from the index
// to get enough unique documents to match the page size
pageSize = skippedResultsInCurrentLoop * indexQuery.PageSize;
skippedResultsInCurrentLoop = 0;
}
TopDocs search = ExecuteQuery(indexSearcher, luceneQuery, start, pageSize, indexQuery);
indexQuery.TotalSize.Value = search.TotalHits;
RecordResultsAlreadySeenForDistinctQuery(indexSearcher, search, start);
for (int i = start; i < search.TotalHits && (i - start) < pageSize; i++)
{
Document document = indexSearcher.Doc(search.ScoreDocs[i].doc);
IndexQueryResult indexQueryResult = parent.RetrieveDocument(document, fieldsToFetch, search.ScoreDocs[i].score);
if (ShouldIncludeInResults(indexQueryResult) == false)
{
indexQuery.SkippedResults.Value++;
skippedResultsInCurrentLoop++;
continue;
}
returnedResults++;
yield return(indexQueryResult);
if (returnedResults == indexQuery.PageSize)
{
yield break;
}
}
} while (skippedResultsInCurrentLoop > 0 && returnedResults < indexQuery.PageSize);
}
}
}
public DocumentDatabase(RavenConfiguration configuration)
{
Configuration = configuration;
configuration.Container.SatisfyImportsOnce(this);
workContext = new WorkContext
{
IndexUpdateTriggers = IndexUpdateTriggers,
ReadTriggers = ReadTriggers
};
TransactionalStorage = configuration.CreateTransactionalStorage(workContext.NotifyAboutWork);
configuration.Container.SatisfyImportsOnce(TransactionalStorage);
bool newDb;
try
{
newDb = TransactionalStorage.Initialize();
}
catch (Exception)
{
TransactionalStorage.Dispose();
throw;
}
IndexDefinitionStorage = new IndexDefinitionStorage(
TransactionalStorage,
configuration.DataDirectory,
configuration.Container.GetExportedValues<AbstractViewGenerator>(),
Extensions);
IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration);
workContext.PerformanceCounters = new PerformanceCounters("Instance @ " + configuration.Port);
workContext.IndexStorage = IndexStorage;
workContext.TransactionaStorage = TransactionalStorage;
workContext.IndexDefinitionStorage = IndexDefinitionStorage;
InitializeTriggers();
ExecuteStartupTasks();
if (!newDb)
return;
OnNewlyCreatedDatabase();
}
protected override void HandleCommitPoints(IndexedItemsInfo itemsInfo, IndexSegmentsInfo segmentsInfo)
{
if (ShouldStoreCommitPoint(itemsInfo) && itemsInfo.HighestETag != null)
{
context.IndexStorage.StoreCommitPoint(indexId.ToString(), new IndexCommitPoint
{
HighestCommitedETag = itemsInfo.HighestETag,
TimeStamp = LastIndexTime,
SegmentsInfo = segmentsInfo ?? IndexStorage.GetCurrentSegmentsInfo(indexDefinition.Name, directory)
});
LastCommitPointStoreTime = SystemTime.UtcNow;
}
else if (itemsInfo.DeletedKeys != null && directory is RAMDirectory == false)
{
context.IndexStorage.AddDeletedKeysToCommitPoints(indexDefinition, itemsInfo.DeletedKeys);
}
}
public DocumentDatabase(RavenConfiguration configuration)
{
this.configuration = configuration;
workContext = new WorkContext();
TransactionalStorage = new TransactionalStorage(configuration.DataDirectory, workContext.NotifyAboutWork);
bool newDb;
try
{
newDb = TransactionalStorage.Initialize();
}
catch (Exception)
{
TransactionalStorage.Dispose();
throw;
}
IndexDefinitionStorage = new IndexDefinitionStorage(configuration.DataDirectory);
IndexStorage = new IndexStorage(configuration.DataDirectory, IndexDefinitionStorage);
workContext.IndexStorage = IndexStorage;
workContext.TransactionaStorage = TransactionalStorage;
workContext.IndexDefinitionStorage = IndexDefinitionStorage;
if (!newDb)
return;
if(configuration.ShouldCreateDefaultsWhenBuildingNewDatabaseFromScratch)
{
PutIndex("Raven/DocumentsByEntityName",
new IndexDefinition
{
Map =
@"
from doc in docs
where doc[""@metadata""][""Raven-Entity-Name""] != null
select new { Tag = doc[""@metadata""][""Raven-Entity-Name""] };
"
});
}
configuration.RaiseDatabaseCreatedFromScratch(this);
}
public DocumentDatabase(InMemoryRavenConfiguration configuration)
{
this.configuration = configuration;
using (LogManager.OpenMappedContext("database", configuration.DatabaseName ?? Constants.SystemDatabase))
{
if (configuration.IsTenantDatabase == false)
{
validateLicense = new ValidateLicense();
validateLicense.Execute(configuration);
}
AppDomain.CurrentDomain.DomainUnload += DomainUnloadOrProcessExit;
AppDomain.CurrentDomain.ProcessExit += DomainUnloadOrProcessExit;
Name = configuration.DatabaseName;
backgroundTaskScheduler = configuration.CustomTaskScheduler ?? TaskScheduler.Current;
ExtensionsState = new AtomicDictionary<object>();
Configuration = configuration;
ExecuteAlterConfiguration();
configuration.Container.SatisfyImportsOnce(this);
workContext = new WorkContext
{
Database = this,
DatabaseName = Name,
IndexUpdateTriggers = IndexUpdateTriggers,
ReadTriggers = ReadTriggers,
RaiseIndexChangeNotification = RaiseNotifications,
TaskScheduler = backgroundTaskScheduler,
Configuration = configuration
};
TransactionalStorage = configuration.CreateTransactionalStorage(workContext.HandleWorkNotifications);
try
{
sequentialUuidGenerator = new SequentialUuidGenerator();
TransactionalStorage.Initialize(sequentialUuidGenerator, DocumentCodecs);
}
catch (Exception)
{
TransactionalStorage.Dispose();
throw;
}
try
{
TransactionalStorage.Batch(actions =>
sequentialUuidGenerator.EtagBase = actions.General.GetNextIdentityValue("Raven/Etag"));
TransportState = new TransportState();
// Index codecs must be initialized before we try to read an index
InitializeIndexCodecTriggers();
IndexDefinitionStorage = new IndexDefinitionStorage(
configuration,
TransactionalStorage,
configuration.DataDirectory,
configuration.Container.GetExportedValues<AbstractViewGenerator>(),
Extensions);
IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration, this);
CompleteWorkContextSetup();
indexingExecuter = new IndexingExecuter(workContext);
InitializeTriggersExceptIndexCodecs();
SecondStageInitialization();
ExecuteStartupTasks();
}
catch (Exception)
{
Dispose();
throw;
}
}
}
public DocumentDatabase(InMemoryRavenConfiguration configuration)
{
ExternalState = new ConcurrentDictionary<string, object>();
Name = configuration.DatabaseName;
if (configuration.BackgroundTasksPriority != ThreadPriority.Normal)
{
backgroundTaskScheduler = new TaskSchedulerWithCustomPriority(
// we need a minimum of four task threads - one for indexing dispatch, one for reducing dispatch, one for tasks, one for indexing/reducing ops
Math.Max(4, configuration.MaxNumberOfParallelIndexTasks + 2),
configuration.BackgroundTasksPriority);
}
else
{
backgroundTaskScheduler = TaskScheduler.Current;
}
ExtensionsState = new ConcurrentDictionary<object, object>();
Configuration = configuration;
ExecuteAlterConfiguration();
configuration.Container.SatisfyImportsOnce(this);
workContext = new WorkContext
{
IndexUpdateTriggers = IndexUpdateTriggers,
ReadTriggers = ReadTriggers
};
TransactionalStorage = configuration.CreateTransactionalStorage(workContext.HandleWorkNotifications);
configuration.Container.SatisfyImportsOnce(TransactionalStorage);
try
{
TransactionalStorage.Initialize(this);
}
catch (Exception)
{
TransactionalStorage.Dispose();
throw;
}
TransactionalStorage.Batch(actions => currentEtagBase = actions.General.GetNextIdentityValue("Raven/Etag"));
IndexDefinitionStorage = new IndexDefinitionStorage(
configuration,
TransactionalStorage,
configuration.DataDirectory,
configuration.Container.GetExportedValues<AbstractViewGenerator>(),
Extensions);
IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration);
workContext.Configuration = configuration;
workContext.IndexStorage = IndexStorage;
workContext.TransactionaStorage = TransactionalStorage;
workContext.IndexDefinitionStorage = IndexDefinitionStorage;
try
{
InitializeTriggers();
ExecuteStartupTasks();
}
catch (Exception)
{
Dispose();
throw;
}
}
public IEnumerable <IndexQueryResult> IntersectionQuery()
{
using (IndexStorage.EnsureInvariantCulture())
{
AssertQueryDoesNotContainFieldsThatAreNotIndexes();
IndexSearcher indexSearcher;
using (parent.GetSearcher(out indexSearcher))
{
var subQueries = indexQuery.Query.Split(new[] { Constants.IntersectSeperator }, StringSplitOptions.RemoveEmptyEntries);
if (subQueries.Length <= 1)
{
throw new InvalidOperationException("Invalid INTRESECT query, must have multiple intersect clauses.");
}
//Not sure how to select the page size here??? The problem is that only docs in this search can be part
//of the final result because we're doing an intersection query (but we might exclude some of them)
int pageSizeBestGuess = (indexQuery.Start + indexQuery.PageSize) * 2;
int intersectMatches = 0, skippedResultsInCurrentLoop = 0;
int previousBaseQueryMatches = 0, currentBaseQueryMatches = 0;
var firstSubLuceneQuery = ApplyIndexTriggers(GetLuceneQuery(subQueries[0], indexQuery.DefaultField));
//Do the first sub-query in the normal way, so that sorting, filtering etc is accounted for
var search = ExecuteQuery(indexSearcher, firstSubLuceneQuery, 0, pageSizeBestGuess, indexQuery);
currentBaseQueryMatches = search.ScoreDocs.Length;
var intersectionCollector = new IntersectionCollector(indexSearcher, search.ScoreDocs);
do
{
if (skippedResultsInCurrentLoop > 0)
{
// We get here because out first attempt didn't get enough docs (after INTERSECTION was calculated)
pageSizeBestGuess = pageSizeBestGuess * 2;
search = ExecuteQuery(indexSearcher, firstSubLuceneQuery, 0, pageSizeBestGuess, indexQuery);
previousBaseQueryMatches = currentBaseQueryMatches;
currentBaseQueryMatches = search.ScoreDocs.Length;
intersectionCollector = new IntersectionCollector(indexSearcher, search.ScoreDocs);
}
for (int i = 1; i < subQueries.Length; i++)
{
var luceneSubQuery = ApplyIndexTriggers(GetLuceneQuery(subQueries[i], indexQuery.DefaultField));
indexSearcher.Search(luceneSubQuery, null, intersectionCollector);
}
var currentIntersectResults = intersectionCollector.DocumentsIdsForCount(subQueries.Length).ToList();
intersectMatches = currentIntersectResults.Count;
skippedResultsInCurrentLoop = pageSizeBestGuess - intersectMatches;
} while (intersectMatches < indexQuery.PageSize && //stop if we've got enough results to satisfy the pageSize
currentBaseQueryMatches < search.TotalHits && //stop if increasing the page size wouldn't make any difference
previousBaseQueryMatches < currentBaseQueryMatches); //stop if increasing the page size didn't result in any more "base query" results
var intersectResults = intersectionCollector.DocumentsIdsForCount(subQueries.Length).ToList();
//It's hard to know what to do here, the TotalHits from the base search isn't really the TotalSize,
//because it's before the INTERSECTION has been applied, so only some of those results make it out.
//Trying to give an accurate answer is going to be too costly, so we aren't going to try.
indexQuery.TotalSize.Value = search.TotalHits;
indexQuery.SkippedResults.Value = skippedResultsInCurrentLoop;
//Using the final set of results in the intersectionCollector
int returnedResults = 0;
for (int i = indexQuery.Start; i < intersectResults.Count && (i - indexQuery.Start) < pageSizeBestGuess; i++)
{
Document document = indexSearcher.Doc(intersectResults[i].LuceneId);
IndexQueryResult indexQueryResult = parent.RetrieveDocument(document, fieldsToFetch, search.ScoreDocs[i].score);
if (ShouldIncludeInResults(indexQueryResult) == false)
{
indexQuery.SkippedResults.Value++;
skippedResultsInCurrentLoop++;
continue;
}
returnedResults++;
yield return(indexQueryResult);
if (returnedResults == indexQuery.PageSize)
{
yield break;
}
}
}
}
}
public DocumentDatabase(InMemoryRavenConfiguration configuration, TransportState transportState = null)
{
DocumentLock = new PutSerialLock();
this.configuration = configuration;
this.transportState = transportState ?? new TransportState();
using (LogManager.OpenMappedContext("database", configuration.DatabaseName ?? Constants.SystemDatabase))
{
log.Debug("Start loading the following database: {0}", configuration.DatabaseName ?? Constants.SystemDatabase);
if (configuration.IsTenantDatabase == false)
{
validateLicense = new ValidateLicense();
validateLicense.Execute(configuration);
}
AppDomain.CurrentDomain.DomainUnload += DomainUnloadOrProcessExit;
AppDomain.CurrentDomain.ProcessExit += DomainUnloadOrProcessExit;
Name = configuration.DatabaseName;
backgroundTaskScheduler = configuration.CustomTaskScheduler ?? TaskScheduler.Default;
ExtensionsState = new AtomicDictionary<object>();
Configuration = configuration;
ExecuteAlterConfiguration();
recentTouches = new SizeLimitedConcurrentDictionary<string, TouchedDocumentInfo>(configuration.MaxRecentTouchesToRemember, StringComparer.OrdinalIgnoreCase);
configuration.Container.SatisfyImportsOnce(this);
workContext = new WorkContext
{
Database = this,
DatabaseName = Name,
IndexUpdateTriggers = IndexUpdateTriggers,
ReadTriggers = ReadTriggers,
RaiseIndexChangeNotification = RaiseNotifications,
TaskScheduler = backgroundTaskScheduler,
Configuration = configuration,
IndexReaderWarmers = IndexReaderWarmers
};
TransactionalStorage = configuration.CreateTransactionalStorage(workContext.HandleWorkNotifications);
try
{
sequentialUuidGenerator = new SequentialUuidGenerator();
TransactionalStorage.Initialize(sequentialUuidGenerator, DocumentCodecs);
}
catch (Exception)
{
TransactionalStorage.Dispose();
throw;
}
try
{
inFlightTransactionalState = TransactionalStorage.GetInFlightTransactionalState(Put, Delete);
TransactionalStorage.Batch(actions =>
sequentialUuidGenerator.EtagBase = actions.General.GetNextIdentityValue("Raven/Etag"));
// Index codecs must be initialized before we try to read an index
InitializeIndexCodecTriggers();
IndexDefinitionStorage = new IndexDefinitionStorage(
configuration,
TransactionalStorage,
configuration.DataDirectory,
configuration.Container.GetExportedValues<AbstractViewGenerator>(),
Extensions);
IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration, this);
CompleteWorkContextSetup();
prefetcher = new Prefetcher(workContext);
indexingExecuter = new IndexingExecuter(workContext, prefetcher);
InitializeTriggersExceptIndexCodecs();
SecondStageInitialization();
ExecuteStartupTasks();
log.Debug("Finish loading the following database: {0}", configuration.DatabaseName ?? Constants.SystemDatabase);
}
catch (Exception)
{
Dispose();
throw;
}
}
}
public DocumentDatabase(InMemoryRavenConfiguration configuration)
{
if (configuration.IsTenantDatabase == false)
{
validateLicense = new ValidateLicense();
validateLicense.Execute(configuration);
}
AppDomain.CurrentDomain.DomainUnload += DomainUnloadOrProcessExit;
AppDomain.CurrentDomain.ProcessExit += DomainUnloadOrProcessExit;
Name = configuration.DatabaseName;
if (configuration.CustomTaskScheduler != null)
{
backgroundTaskScheduler = configuration.CustomTaskScheduler;
}
else if (configuration.BackgroundTasksPriority != ThreadPriority.Normal)
{
backgroundTaskScheduler = new TaskSchedulerWithCustomPriority(
// we need a minimum of four task threads - one for indexing dispatch, one for reducing dispatch, one for tasks, one for indexing/reducing ops
Math.Max(4, configuration.MaxNumberOfParallelIndexTasks + 2),
configuration.BackgroundTasksPriority);
}
else
{
backgroundTaskScheduler = TaskScheduler.Current;
}
ExtensionsState = new AtomicDictionary<object>();
Configuration = configuration;
ExecuteAlterConfiguration();
configuration.Container.SatisfyImportsOnce(this);
workContext = new WorkContext
{
DatabaseName = Name,
IndexUpdateTriggers = IndexUpdateTriggers,
ReadTriggers = ReadTriggers,
RaiseIndexChangeNotification = RaiseNotifications,
TaskScheduler = backgroundTaskScheduler,
Configuration = configuration
};
TransactionalStorage = configuration.CreateTransactionalStorage(workContext.HandleWorkNotifications);
try
{
TransactionalStorage.Initialize(this, DocumentCodecs);
}
catch (Exception)
{
TransactionalStorage.Dispose();
throw;
}
try
{
TransactionalStorage.Batch(actions => currentEtagBase = actions.General.GetNextIdentityValue("Raven/Etag"));
TransportState = new TransportState();
// Index codecs must be initialized before we try to read an index
InitializeIndexCodecTriggers();
IndexDefinitionStorage = new IndexDefinitionStorage(
configuration,
TransactionalStorage,
configuration.DataDirectory,
configuration.Container.GetExportedValues<AbstractViewGenerator>(),
Extensions);
IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration, this);
CompleteWorkContextSetup();
indexingExecuter = new IndexingExecuter(workContext);
InitializeTriggersExceptIndexCodecs();
ExecuteStartupTasks();
}
catch (Exception)
{
Dispose();
throw;
}
}
public DocumentDatabase(InMemoryRavenConfiguration configuration)
{
Configuration = configuration;
configuration.Container.SatisfyImportsOnce(this);
workContext = new WorkContext
{
IndexUpdateTriggers = IndexUpdateTriggers,
ReadTriggers = ReadTriggers
};
dynamicQueryRunner = new DynamicQueryRunner(this);
suggestionQueryRunner = new SuggestionQueryRunner(this);
TransactionalStorage = configuration.CreateTransactionalStorage(workContext.HandleWorkNotifications);
configuration.Container.SatisfyImportsOnce(TransactionalStorage);
bool newDb;
try
{
newDb = TransactionalStorage.Initialize(this);
}
catch (Exception)
{
TransactionalStorage.Dispose();
throw;
}
TransactionalStorage.Batch(actions => currentEtagBase = actions.General.GetNextIdentityValue("Raven/Etag"));
IndexDefinitionStorage = new IndexDefinitionStorage(
configuration,
TransactionalStorage,
configuration.DataDirectory,
configuration.Container.GetExportedValues<AbstractViewGenerator>(),
Extensions);
IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration);
workContext.IndexStorage = IndexStorage;
workContext.TransactionaStorage = TransactionalStorage;
workContext.IndexDefinitionStorage = IndexDefinitionStorage;
try
{
InitializeTriggers();
ExecuteStartupTasks();
}
catch (Exception)
{
Dispose();
throw;
}
if (!newDb)
return;
OnNewlyCreatedDatabase();
}