public DocumentDatabase(RavenConfiguration configuration)
{
Configuration = configuration;
configuration.Container.SatisfyImportsOnce(this);
workContext = new WorkContext {
IndexUpdateTriggers = IndexUpdateTriggers
};
TransactionalStorage = new TransactionalStorage(configuration, workContext.NotifyAboutWork);
configuration.Container.SatisfyImportsOnce(TransactionalStorage);
bool newDb;
try
{
newDb = TransactionalStorage.Initialize();
}
catch (Exception)
{
TransactionalStorage.Dispose();
throw;
}
IndexDefinitionStorage = new IndexDefinitionStorage(
TransactionalStorage,
configuration.DataDirectory,
configuration.Container.GetExportedValues <AbstractViewGenerator>());
IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration);
workContext.IndexStorage = IndexStorage;
workContext.TransactionaStorage = TransactionalStorage;
workContext.IndexDefinitionStorage = IndexDefinitionStorage;
InitializeTriggers();
ExecuteStartupTasks();
if (!newDb)
{
return;
}
OnNewlyCreatedDatabase();
}
protected Lucene.Net.Store.Directory OpenOrCreateLuceneDirectory(
IndexDefinition indexDefinition,
string indexName = null,
bool createIfMissing = true)
{
Lucene.Net.Store.Directory directory;
if (indexDefinition.IsTemp || configuration.RunInMemory)
{
directory = new RAMDirectory();
new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Close(); // creating index structure
}
else
{
var indexDirectory = indexName ?? IndexDefinitionStorage.FixupIndexName(indexDefinition.Name, path);
var indexFullPath = Path.Combine(path, MonoHttpUtility.UrlEncode(indexDirectory));
directory = FSDirectory.Open(new DirectoryInfo(indexFullPath));
if (!IndexReader.IndexExists(directory))
{
if (createIfMissing == false)
{
throw new InvalidOperationException("Index does not exists: " + indexDirectory);
}
//creating index structure if we need to
new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Close();
}
else
{
if (directory.FileExists("write.lock")) // we had an unclean shutdown
{
if (configuration.ResetIndexOnUncleanShutdown)
{
throw new InvalidOperationException("Rude shutdown detected on: " + indexDirectory);
}
CheckIndexAndRecover(directory, indexDirectory);
IndexWriter.Unlock(directory);
directory.DeleteFile("write.lock");
}
}
}
return(directory);
}
private IEnumerable <RavenJObject> GetQueryResults(IndexQuery query, AbstractViewGenerator viewGenerator, DocumentRetriever docRetriever, IEnumerable <JsonDocument> results, List <string> transformerErrors, Action <double> loadingDocumentsFinish, Action <double> transformerFinish, bool showTimings, CancellationTokenSource tokenSource)
{
if (query.PageSize <= 0) // maybe they just want the stats?
{
return(Enumerable.Empty <RavenJObject>());
}
IndexingFunc transformFunc = null;
// Check an explicitly declared one first
if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
{
var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer);
if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null)
{
transformFunc = transformGenerator.TransformResultsDefinition;
}
else
{
throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found");
}
}
if (transformFunc == null)
{
var resultsWithoutTransformer = results.Select(x => x.ToJson());
return(showTimings ? new TimedEnumerable <RavenJObject>(resultsWithoutTransformer, loadingDocumentsFinish) : resultsWithoutTransformer);
}
var dynamicJsonObjects = results.Select(x => new DynamicLuceneOrParentDocumntObject(docRetriever, x.ToJson()));
var robustEnumerator = new RobustEnumerator(tokenSource.Token, 100)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
var resultsWithTransformer = robustEnumerator
.RobustEnumeration(dynamicJsonObjects.Cast <object>().GetEnumerator(), transformFunc)
.Select(JsonExtensions.ToJObject);
return(showTimings ? new TimedEnumerable <RavenJObject>(resultsWithTransformer, transformerFinish) : resultsWithTransformer);
}
public string PutIndex(string name, IndexDefinition definition)
{
switch (IndexDefinitionStorage.FindIndexCreationOptionsOptions(name, definition))
{
case IndexCreationOptions.Noop:
return(name);
case IndexCreationOptions.Update:
// ensure that the code can compile
new DynamicViewCompiler(name, definition, Extensions).GenerateInstance();
DeleteIndex(name);
break;
}
IndexDefinitionStorage.AddIndex(name, definition);
IndexStorage.CreateIndexImplementation(name, definition);
TransactionalStorage.Batch(actions => AddIndexAndEnqueueIndexingTasks(actions, name));
return(name);
}
protected Lucene.Net.Store.Directory OpenOrCreateLuceneDirectory(IndexDefinition indexDefinition, string indexName = null)
{
Lucene.Net.Store.Directory directory;
if (indexDefinition.IsTemp || configuration.RunInMemory)
{
directory = new RAMDirectory();
}
else
{
var indexDirectory = indexName ?? IndexDefinitionStorage.FixupIndexName(indexDefinition.Name, path);
directory = FSDirectory.Open(new DirectoryInfo(Path.Combine(path, MonoHttpUtility.UrlEncode(indexDirectory))));
}
//creating index structure if we need to
new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Close();
return(directory);
}
public void ResetIndex(string index)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var indexDefinition = IndexDefinitionStorage.GetIndexDefinition(index);
if (indexDefinition == null)
{
throw new InvalidOperationException("There is no index named: " + index);
}
IndexStorage.DeleteIndex(index);
IndexStorage.CreateIndexImplementation(indexDefinition);
TransactionalStorage.Batch(actions =>
{
actions.Indexing.DeleteIndex(index);
actions.Indexing.AddIndex(index, indexDefinition.IsMapReduce);
workContext.ShouldNotifyAboutWork();
});
}
public IndexStorage(IndexDefinitionStorage indexDefinitionStorage, InMemoryRavenConfiguration configuration, DocumentDatabase documentDatabase)
{
try
{
this.indexDefinitionStorage = indexDefinitionStorage;
this.configuration = configuration;
this.documentDatabase = documentDatabase;
path = configuration.IndexStoragePath;
if (Directory.Exists(path) == false && configuration.RunInMemory == false)
{
Directory.CreateDirectory(path);
}
if (configuration.RunInMemory == false)
{
var crashMarkerPath = Path.Combine(path, "indexing.crash-marker");
if (File.Exists(crashMarkerPath))
{
// the only way this can happen is if we crashed because of a power outage
// in this case, we consider all open indexes to be corrupt and force them
// to be reset. This is because to get better perf, we don't flush the files to disk,
// so in the case of a power outage, we can't be sure that there wasn't still stuff in
// the OS buffer that wasn't written yet.
configuration.ResetIndexOnUncleanShutdown = true;
}
// The delete on close ensures that the only way this file will exists is if there was
// a power outage while the server was running.
crashMarker = File.Create(crashMarkerPath, 16, FileOptions.DeleteOnClose);
}
BackgroundTaskExecuter.Instance.ExecuteAll(documentDatabase.WorkContext,
indexDefinitionStorage.IndexNames, (indexName, _) => OpenIndexOnStartup(indexName));
}
catch
{
Dispose();
throw;
}
}
internal void DeleteIndex(IndexDefinition instance, bool removeByNameMapping = true, bool clearErrors = true, bool removeIndexReplaceDocument = true)
{
using (IndexDefinitionStorage.TryRemoveIndexContext())
{
if (instance == null)
{
return;
}
// Set up a flag to signal that this is something we're doing
TransactionalStorage.Batch(actions => actions.Lists.Set("Raven/Indexes/PendingDeletion", instance.IndexId.ToString(CultureInfo.InvariantCulture), (RavenJObject.FromObject(new
{
TimeOfOriginalDeletion = SystemTime.UtcNow,
instance.IndexId,
IndexName = instance.Name
})), UuidType.Tasks));
// Delete the main record synchronously
IndexDefinitionStorage.RemoveIndex(instance.IndexId, removeByNameMapping);
Database.IndexStorage.DeleteIndex(instance.IndexId);
if (clearErrors)
{
WorkContext.ClearErrorsFor(instance.Name);
}
if (removeIndexReplaceDocument && instance.IsSideBySideIndex)
{
Database.Documents.Delete(Constants.IndexReplacePrefix + instance.Name, null, null);
}
// And delete the data in the background
StartDeletingIndexDataAsync(instance.IndexId, instance.Name);
// We raise the notification now because as far as we're concerned it is done *now*
TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() => Database.Notifications.RaiseNotifications(new IndexChangeNotification
{
Name = instance.Name,
Type = IndexChangeTypes.IndexRemoved,
}));
}
}
public string PutTransform(string name, TransformerDefinition definition)
{
if (name == null)
{
throw new ArgumentNullException("name");
}
if (definition == null)
{
throw new ArgumentNullException("definition");
}
name = name.Trim();
var existingDefinition = IndexDefinitionStorage.GetTransformerDefinition(name);
if (existingDefinition != null && existingDefinition.Equals(definition))
{
return(name); // no op for the same transformer
}
var generator = IndexDefinitionStorage.CompileTransform(definition);
if (existingDefinition != null)
{
IndexDefinitionStorage.RemoveTransformer(existingDefinition.TransfomerId);
}
TransactionalStorage.Batch(accessor =>
{
definition.TransfomerId = (int)Database.Documents.GetNextIdentityValueWithoutOverwritingOnExistingDocuments("TransformerId", accessor);
});
IndexDefinitionStorage.CreateAndPersistTransform(definition, generator);
IndexDefinitionStorage.AddTransform(definition.TransfomerId, definition);
TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() => Database.Notifications.RaiseNotifications(new TransformerChangeNotification()
{
Name = name,
Type = TransformerChangeTypes.TransformerAdded,
}));
return(name);
}
public IEnumerable <string> QueryDocumentIds(string index, IndexQuery query, CancellationToken token, out bool stale)
{
var queryStat = AddToCurrentlyRunningQueryList(index, query);
try
{
bool isStale = false;
HashSet <string> loadedIds = null;
TransactionalStorage.Batch(
actions =>
{
var definition = IndexDefinitionStorage.GetIndexDefinition(index);
if (definition == null)
{
throw new ArgumentException("specified index definition was not found", "index");
}
isStale = actions.Staleness.IsIndexStale(definition.IndexId, query.Cutoff, null);
if (isStale == false && query.Cutoff == null)
{
var indexInstance = Database.IndexStorage.GetIndexInstance(index);
isStale = isStale || (indexInstance != null && indexInstance.IsMapIndexingInProgress);
}
var indexFailureInformation = actions.Indexing.GetFailureRate(definition.IndexId);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
loadedIds = new HashSet <string>(from queryResult in Database.IndexStorage.Query(index, query, result => true, new FieldsToFetch(null, false, Constants.DocumentIdFieldName), Database.IndexQueryTriggers, token)
select queryResult.Key);
});
stale = isStale;
return(loadedIds);
}
finally
{
RemoveFromCurrentlyRunningQueryList(index, queryStat);
}
}
private void AddIndexingTask(IStorageActionsAccessor actions, JToken metadata, Func <Task> taskGenerator)
{
foreach (var indexName in IndexDefinitionStorage.IndexNames)
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(indexName);
if (viewGenerator == null)
{
continue;
}
var entityName = metadata.Value <string>("Raven-Entity-Name");
if (viewGenerator.ForEntityName != null &&
viewGenerator.ForEntityName != entityName)
{
continue;
}
var task = taskGenerator();
task.Index = indexName;
actions.Tasks.AddTask(task, DateTime.UtcNow);
}
}
public bool IndexHasChanged(string name, IndexDefinition definition)
{
if (name == null)
{
throw new ArgumentNullException("name");
}
name = name.Trim();
IsIndexNameValid(name);
var existingIndex = IndexDefinitionStorage.GetIndexDefinition(name);
if (existingIndex == null)
{
return(true);
}
var creationOption = FindIndexCreationOptions(definition, ref name);
return(creationOption != IndexCreationOptions.Noop);
}
internal void PutNewIndexIntoStorage(string name, IndexDefinition definition)
{
Debug.Assert(Database.IndexStorage != null);
Debug.Assert(TransactionalStorage != null);
Debug.Assert(WorkContext != null);
TransactionalStorage.Batch(actions =>
{
definition.IndexId = (int)Database.Documents.GetNextIdentityValueWithoutOverwritingOnExistingDocuments("IndexId", actions, null);
IndexDefinitionStorage.RegisterNewIndexInThisSession(name, definition);
// this has to happen in this fashion so we will expose the in memory status after the commit, but
// before the rest of the world is notified about this.
IndexDefinitionStorage.CreateAndPersistIndex(definition);
Database.IndexStorage.CreateIndexImplementation(definition);
InvokeSuggestionIndexing(name, definition);
actions.Indexing.AddIndex(definition.IndexId, definition.IsMapReduce);
});
if (name.Equals(Constants.DocumentsByEntityNameIndex, StringComparison.InvariantCultureIgnoreCase) == false &&
Database.IndexStorage.HasIndex(Constants.DocumentsByEntityNameIndex))
{
// optimization of handling new index creation when the number of document in a database is significantly greater than
// number of documents that this index applies to - let us use built-in RavenDocumentsByEntityName to get just appropriate documents
var index = Database.IndexStorage.GetIndexInstance(definition.IndexId);
TryApplyPrecomputedBatchForNewIndex(index, definition);
}
WorkContext.ShouldNotifyAboutWork(() => "PUT INDEX " + name);
WorkContext.NotifyAboutWork();
// The act of adding it here make it visible to other threads
// we have to do it in this way so first we prepare all the elements of the
// index, then we add it to the storage in a way that make it public
IndexDefinitionStorage.AddIndex(definition.IndexId, definition);
}
public IEnumerable <string> QueryDocumentIds(string index, IndexQuery query, out bool stale)
{
index = IndexDefinitionStorage.FixupIndexName(index);
bool isStale = false;
HashSet <string> loadedIds = null;
TransactionalStorage.Batch(
actions =>
{
isStale = actions.Staleness.IsIndexStale(index, query.Cutoff, null);
var indexFailureInformation = actions.Indexing.GetFailureRate(index)
;
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
loadedIds = new HashSet <string>(from queryResult in IndexStorage.Query(index, query, result => true, new FieldsToFetch(null, AggregationOperation.None, Raven.Abstractions.Data.Constants.DocumentIdFieldName))
select queryResult.Key);
});
stale = isStale;
return(loadedIds);
}
public string PutIndex(string name, IndexDefinition definition)
{
definition.Name = name = IndexDefinitionStorage.FixupIndexName(name);
switch (IndexDefinitionStorage.FindIndexCreationOptions(definition))
{
case IndexCreationOptions.Noop:
return(name);
case IndexCreationOptions.Update:
// ensure that the code can compile
new DynamicViewCompiler(name, definition, Extensions, IndexDefinitionStorage.IndexDefinitionsPath, Configuration).GenerateInstance();
DeleteIndex(name);
break;
}
IndexDefinitionStorage.AddIndex(definition);
IndexStorage.CreateIndexImplementation(definition);
TransactionalStorage.Batch(actions =>
{
actions.Indexing.AddIndex(name, definition.IsMapReduce);
workContext.ShouldNotifyAboutWork();
});
return(name);
}
public IndexStorage(IndexDefinitionStorage indexDefinitionStorage, RavenConfiguration configuration)
{
this.configuration = configuration;
path = Path.Combine(configuration.DataDirectory, "Indexes");
if (Directory.Exists(path) == false)
{
Directory.CreateDirectory(path);
}
foreach (var indexDirectory in indexDefinitionStorage.IndexNames)
{
log.DebugFormat("Loading saved index {0}", indexDirectory);
var indexDefinition = indexDefinitionStorage.GetIndexDefinition(indexDirectory);
if (indexDefinition == null)
{
continue;
}
indexes.TryAdd(indexDirectory,
CreateIndexImplementation(indexDirectory, indexDefinition,
OpenOrCreateLuceneDirectory(indexDirectory)));
}
}
public void DeleteIndex(string name)
{
using (IndexDefinitionStorage.TryRemoveIndexContext())
{
var instance = IndexDefinitionStorage.GetIndexDefinition(name);
if (instance == null)
{
return;
}
// Set up a flag to signal that this is something we're doing
TransactionalStorage.Batch(actions => actions.Lists.Set("Raven/Indexes/PendingDeletion", instance.IndexId.ToString(CultureInfo.InvariantCulture), (RavenJObject.FromObject(new
{
TimeOfOriginalDeletion = SystemTime.UtcNow,
instance.IndexId
})), UuidType.Tasks));
// Delete the main record synchronously
IndexDefinitionStorage.RemoveIndex(name);
Database.IndexStorage.DeleteIndex(instance.IndexId);
ConcurrentSet <string> _;
WorkContext.DoNotTouchAgainIfMissingReferences.TryRemove(instance.IndexId, out _);
WorkContext.ClearErrorsFor(name);
// And delete the data in the background
StartDeletingIndexDataAsync(instance.IndexId);
// We raise the notification now because as far as we're concerned it is done *now*
TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() => Database.Notifications.RaiseNotifications(new IndexChangeNotification
{
Name = name,
Type = IndexChangeTypes.IndexRemoved,
}));
}
}
public QueryResult Query(string index, IndexQuery query)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var list = new List <JObject>();
var stale = false;
Tuple <DateTime, Guid> indexTimestamp = null;
TransactionalStorage.Batch(
actions =>
{
string entityName = null;
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
{
throw new InvalidOperationException("Could not find index named: " + index);
}
entityName = viewGenerator.ForEntityName;
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, entityName);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers);
var indexDefinition = GetIndexDefinition(index);
var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
viewGenerator.ReduceDefinition == null
? Abstractions.Data.Constants.DocumentIdFieldName
: Abstractions.Data.Constants.ReduceKeyFieldName);
var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch), fieldsToFetch)
select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch)
into doc
where doc != null
select doc;
var transformerErrors = new List <string>();
IEnumerable <JObject> results;
if (viewGenerator != null &&
query.SkipTransformResults == false &&
viewGenerator.TransformResultsDefinition != null)
{
var robustEnumerator = new RobustEnumerator
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
results =
robustEnumerator.RobustEnumeration(
dynamicJsonObjects,
source => viewGenerator.TransformResultsDefinition(docRetriever, source))
.Select(JsonExtensions.ToJObject);
}
else
{
results = collection.Select(x => x.ToJson());
}
list.AddRange(results);
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
});
return(new QueryResult
{
IndexName = index,
Results = list,
IsStale = stale,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2
});
}
public IndexDefinition GetIndexDefinition(string index)
{
index = IndexDefinitionStorage.FixupIndexName(index);
return(IndexDefinitionStorage.GetIndexDefinition(index));
}
public IndexDefinition GetIndexDefinition(string index)
{
return(IndexDefinitionStorage.GetIndexDefinition(index));
}
public string[] PutIndexes(string[] names, IndexDefinition[] definitions, IndexingPriority[] priorities)
{
var createdIndexes = new List <string>();
var prioritiesList = new List <IndexingPriority>();
try
{
for (int i = 0; i < names.Length; i++)
{
var name = names[i];
var definition = definitions[i];
var priority = priorities[i];
if (name == null)
{
throw new ArgumentNullException("names", "Names cannot contain null values");
}
IsIndexNameValid(name);
var existingIndex = IndexDefinitionStorage.GetIndexDefinition(name);
if (existingIndex != null)
{
switch (existingIndex.LockMode)
{
case IndexLockMode.SideBySide:
Log.Info("Index {0} not saved because it might be only updated by side-by-side index");
throw new InvalidOperationException("Can not overwrite locked index: " + name + ". This index can be only updated by side-by-side index.");
case IndexLockMode.LockedIgnore:
Log.Info("Index {0} not saved because it was lock (with ignore)", name);
continue;
case IndexLockMode.LockedError:
throw new InvalidOperationException("Can not overwrite locked index: " + name);
}
}
name = name.Trim();
if (name.Equals("dynamic", StringComparison.OrdinalIgnoreCase) ||
name.StartsWith("dynamic/", StringComparison.OrdinalIgnoreCase))
{
throw new ArgumentException("Cannot use index name " + name + " because it clashes with reserved dynamic index names", "name");
}
if (name.Contains("//"))
{
throw new ArgumentException("Cannot use an index with // in the name, but got: " + name, "name");
}
AssertAnalyzersValid(definition);
switch (FindIndexCreationOptions(definition, ref name))
{
case IndexCreationOptions.Noop:
continue;
case IndexCreationOptions.UpdateWithoutUpdatingCompiledIndex:
// ensure that the code can compile
new DynamicViewCompiler(definition.Name, definition, Database.Extensions, IndexDefinitionStorage.IndexDefinitionsPath, Database.Configuration).GenerateInstance();
IndexDefinitionStorage.UpdateIndexDefinitionWithoutUpdatingCompiledIndex(definition);
break;
case IndexCreationOptions.Update:
// ensure that the code can compile
new DynamicViewCompiler(definition.Name, definition, Database.Extensions, IndexDefinitionStorage.IndexDefinitionsPath, Database.Configuration).GenerateInstance();
DeleteIndex(name);
break;
}
PutNewIndexIntoStorage(name, definition, true);
WorkContext.ClearErrorsFor(name);
TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() => Database.Notifications.RaiseNotifications(new IndexChangeNotification
{
Name = name,
Type = IndexChangeTypes.IndexAdded,
}));
createdIndexes.Add(name);
prioritiesList.Add(priority);
}
var indexesIds = createdIndexes.Select(x => Database.IndexStorage.GetIndexInstance(x).indexId).ToArray();
Database.TransactionalStorage.Batch(accessor => accessor.Indexing.SetIndexesPriority(indexesIds, prioritiesList.ToArray()));
return(createdIndexes.ToArray());
}
catch (Exception e)
{
Log.WarnException("Could not create index batch", e);
foreach (var index in createdIndexes)
{
DeleteIndex(index);
}
throw;
}
}
internal Lucene.Net.Store.Directory MakeRAMDirectoryPhysical(RAMDirectory ramDir, string indexName)
{
var newDir = new LuceneCodecDirectory(Path.Combine(path, MonoHttpUtility.UrlEncode(IndexDefinitionStorage.FixupIndexName(indexName, path))), documentDatabase.IndexCodecs.OfType <AbstractIndexCodec>());
Lucene.Net.Store.Directory.Copy(ramDir, newDir, true);
return(newDir);
}
public Etag GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Func <JsonDocument, bool> addDocument,
string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null, long?maxSize = null, TimeSpan?timeout = null)
{
Etag lastDocumentReadEtag = null;
using (DocumentCacher.SkipSetDocumentsInDocumentCache())
TransactionalStorage.Batch(actions =>
{
AbstractTransformer storedTransformer = null;
if (transformer != null)
{
storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
{
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
}
var returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize, token, maxSize: maxSize, timeout: timeout);
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, transformerParameters);
var docCount = 0;
var docCountOnLastAdd = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
if (docCount - docCountOnLastAdd > 1000)
{
addDocument(null); // heartbeat
}
if (etag != null)
{
etag = doc.Etag;
}
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
returnedDocs = true;
Database.WorkContext.UpdateFoundWork();
document = TransformDocumentIfNeeded(document, storedTransformer, documentRetriever);
var canContinue = addDocument(document);
if (!canContinue)
{
break;
}
lastDocumentReadEtag = etag;
docCountOnLastAdd = docCount;
}
if (returnedDocs || docCount == 0)
{
break;
}
// No document was found that matches the requested criteria
// If we had a failure happen, we update the etag as we don't need to process those documents again (no matches there anyways).
if (lastDocumentReadEtag != null)
{
etag = lastDocumentReadEtag;
}
start += docCount;
}
});
return(lastDocumentReadEtag);
}
public DocumentDatabase(InMemoryRavenConfiguration configuration)
{
ExternalState = new ConcurrentDictionary <string, object>();
if (configuration.BackgroundTasksPriority != ThreadPriority.Normal)
{
backgroundTaskScheduler = new TaskSchedulerWithCustomPriority(
// we need a minimum of three task threads - one for indexing dispatch, one for tasks, one for indexing ops
Math.Max(3, configuration.MaxNumberOfParallelIndexTasks + 2),
configuration.BackgroundTasksPriority);
}
else
{
backgroundTaskScheduler = TaskScheduler.Current;
}
ExtensionsState = new ConcurrentDictionary <object, object>();
Configuration = configuration;
configuration.Container.SatisfyImportsOnce(this);
workContext = new WorkContext
{
IndexUpdateTriggers = IndexUpdateTriggers,
ReadTriggers = ReadTriggers
};
TransactionalStorage = configuration.CreateTransactionalStorage(workContext.HandleWorkNotifications);
configuration.Container.SatisfyImportsOnce(TransactionalStorage);
try
{
TransactionalStorage.Initialize(this);
}
catch (Exception)
{
TransactionalStorage.Dispose();
throw;
}
TransactionalStorage.Batch(actions => currentEtagBase = actions.General.GetNextIdentityValue("Raven/Etag"));
IndexDefinitionStorage = new IndexDefinitionStorage(
configuration,
TransactionalStorage,
configuration.DataDirectory,
configuration.Container.GetExportedValues <AbstractViewGenerator>(),
Extensions);
IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration);
workContext.Configuration = configuration;
workContext.IndexStorage = IndexStorage;
workContext.TransactionaStorage = TransactionalStorage;
workContext.IndexDefinitionStorage = IndexDefinitionStorage;
try
{
InitializeTriggers();
ExecuteStartupTasks();
}
catch (Exception)
{
Dispose();
throw;
}
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize,
CancellationToken token, ref int nextStart, Action <JsonDocument> addDoc,
string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null,
string skipAfter = null)
{
if (idPrefix == null)
{
throw new ArgumentNullException("idPrefix");
}
idPrefix = idPrefix.Trim();
var canPerformRapidPagination = nextStart > 0 && start == nextStart;
var actualStart = canPerformRapidPagination ? start : 0;
var addedDocs = 0;
var docCountOnLastAdd = 0;
var matchedDocs = 0;
TransactionalStorage.Batch(
actions =>
{
var docsToSkip = canPerformRapidPagination ? 0 : start;
int docCount;
AbstractTransformer storedTransformer = null;
if (transformer != null)
{
storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
{
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
}
do
{
Database.WorkContext.UpdateFoundWork();
docCount = 0;
var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter);
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, transformerParameters);
foreach (var doc in docs)
{
token.ThrowIfCancellationRequested();
docCount++;
if (docCount - docCountOnLastAdd > 1000)
{
addDoc(null); // heartbeat
}
var keyTest = doc.Key.Substring(idPrefix.Length);
if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
{
continue;
}
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc;
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
matchedDocs++;
if (matchedDocs <= docsToSkip)
{
continue;
}
token.ThrowIfCancellationRequested();
document = TransformDocumentIfNeeded(document, storedTransformer, documentRetriever);
addDoc(document);
addedDocs++;
docCountOnLastAdd = docCount;
if (addedDocs >= pageSize)
{
break;
}
}
actualStart += pageSize;
} while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue);
});
if (addedDocs != pageSize)
{
nextStart = start; // will mark as last page
}
else if (canPerformRapidPagination)
{
nextStart = start + matchedDocs;
}
else
{
nextStart = actualStart;
}
}
public string[] GetIndexFields(string index)
{
var abstractViewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
return(abstractViewGenerator == null ? new string[0] : abstractViewGenerator.Fields);
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize,
CancellationToken token, ref int nextStart, Action <RavenJObject> addDoc,
string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null,
string skipAfter = null)
{
if (idPrefix == null)
{
throw new ArgumentNullException("idPrefix");
}
idPrefix = idPrefix.Trim();
var canPerformRapidPagination = nextStart > 0 && start == nextStart;
var actualStart = canPerformRapidPagination ? start : 0;
var addedDocs = 0;
var matchedDocs = 0;
TransactionalStorage.Batch(
actions =>
{
var docsToSkip = canPerformRapidPagination ? 0 : start;
int docCount;
AbstractTransformer storedTransformer = null;
if (transformer != null)
{
storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
{
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
}
do
{
docCount = 0;
var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter);
var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState, transformerParameters);
foreach (var doc in docs)
{
token.ThrowIfCancellationRequested();
docCount++;
var keyTest = doc.Key.Substring(idPrefix.Length);
if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
{
continue;
}
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc;
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
matchedDocs++;
if (matchedDocs <= docsToSkip)
{
continue;
}
token.ThrowIfCancellationRequested();
if (storedTransformer != null)
{
using (new CurrentTransformationScope(Database, documentRetriever))
{
var transformed =
storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) })
.Select(x => JsonExtensions.ToJObject(x))
.ToArray();
if (transformed.Length == 0)
{
throw new InvalidOperationException("The transform results function failed on a document: " + document.Key);
}
var transformedJsonDocument = new JsonDocument
{
Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(documentRetriever.Etag),
NonAuthoritativeInformation = document.NonAuthoritativeInformation,
LastModified = document.LastModified,
DataAsJson = new RavenJObject {
{ "$values", new RavenJArray(transformed) }
},
};
addDoc(transformedJsonDocument.ToJson());
}
}
else
{
addDoc(document.ToJson());
}
addedDocs++;
if (addedDocs >= pageSize)
{
break;
}
}
actualStart += pageSize;
}while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue);
});
if (addedDocs != pageSize)
{
nextStart = start; // will mark as last page
}
else if (canPerformRapidPagination)
{
nextStart = start + matchedDocs;
}
else
{
nextStart = actualStart;
}
}
private Action TryCreateTaskForApplyingPrecomputedBatchForNewIndex(Index index, IndexDefinition definition)
{
var generator = IndexDefinitionStorage.GetViewGenerator(definition.IndexId);
if (generator.ForEntityNames.Count == 0 && index.IsTestIndex == false)
{
// we don't optimize if we don't have what to optimize _on_, we know this is going to return all docs.
// no need to try to optimize that, then
index.IsMapIndexingInProgress = false;
return(null);
}
try
{
var cts = new CancellationTokenSource();
var task = new Task(() =>
{
try
{
ApplyPrecomputedBatchForNewIndex(index, generator, index.IsTestIndex == false ? Database.Configuration.MaxNumberOfItemsToProcessInSingleBatch : Database.Configuration.Indexing.MaxNumberOfItemsToProcessInTestIndexes, cts);
}
catch (Exception e)
{
Log.Warn("Could not apply precomputed batch for index " + index, e);
}
finally
{
index.IsMapIndexingInProgress = false;
WorkContext.ShouldNotifyAboutWork(() => "Precomputed indexing batch for " + index.PublicName + " is completed");
WorkContext.NotifyAboutWork();
}
}, TaskCreationOptions.LongRunning);
return(() =>
{
try
{
task.Start();
long id;
Database
.Tasks
.AddTask(
task,
new TaskBasedOperationState(task),
new TaskActions.PendingTaskDescription
{
StartTime = DateTime.UtcNow,
Payload = index.PublicName,
TaskType = TaskActions.PendingTaskType.NewIndexPrecomputedBatch
},
out id,
cts);
}
catch (Exception)
{
index.IsMapIndexingInProgress = false;
throw;
}
});
}
catch (Exception)
{
index.IsMapIndexingInProgress = false;
throw;
}
}
internal void PutNewIndexIntoStorage(string name, IndexDefinition definition, bool disableIndex = false)
{
Debug.Assert(Database.IndexStorage != null);
Debug.Assert(TransactionalStorage != null);
Debug.Assert(WorkContext != null);
Index index = null;
TransactionalStorage.Batch(actions =>
{
var maxId = 0;
if (Database.IndexStorage.Indexes.Length > 0)
{
maxId = Database.IndexStorage.Indexes.Max();
}
definition.IndexId = (int)Database.Documents.GetNextIdentityValueWithoutOverwritingOnExistingDocuments("IndexId", actions);
if (definition.IndexId <= maxId)
{
actions.General.SetIdentityValue("IndexId", maxId + 1);
definition.IndexId = (int)Database.Documents.GetNextIdentityValueWithoutOverwritingOnExistingDocuments("IndexId", actions);
}
IndexDefinitionStorage.RegisterNewIndexInThisSession(name, definition);
// this has to happen in this fashion so we will expose the in memory status after the commit, but
// before the rest of the world is notified about this.
IndexDefinitionStorage.CreateAndPersistIndex(definition);
Database.IndexStorage.CreateIndexImplementation(definition);
index = Database.IndexStorage.GetIndexInstance(definition.IndexId);
// If we execute multiple indexes at once and want to activate them all at once we will disable the index from the endpoint
if (disableIndex)
{
index.Priority = IndexingPriority.Disabled;
}
//ensure that we don't start indexing it right away, let the precomputation run first, if applicable
index.IsMapIndexingInProgress = true;
if (definition.IsTestIndex)
{
index.MarkQueried(); // test indexes should be mark queried, so the cleanup task would not delete them immediately
}
InvokeSuggestionIndexing(name, definition, index);
actions.Indexing.AddIndex(definition.IndexId, definition.IsMapReduce);
});
Debug.Assert(index != null);
Action precomputeTask = null;
if (WorkContext.RunIndexing &&
name.Equals(Constants.DocumentsByEntityNameIndex, StringComparison.InvariantCultureIgnoreCase) == false &&
Database.IndexStorage.HasIndex(Constants.DocumentsByEntityNameIndex))
{
// optimization of handling new index creation when the number of document in a database is significantly greater than
// number of documents that this index applies to - let us use built-in RavenDocumentsByEntityName to get just appropriate documents
precomputeTask = TryCreateTaskForApplyingPrecomputedBatchForNewIndex(index, definition);
}
else
{
index.IsMapIndexingInProgress = false; // we can't apply optimization, so we'll make it eligible for running normally
}
// The act of adding it here make it visible to other threads
// we have to do it in this way so first we prepare all the elements of the
// index, then we add it to the storage in a way that make it public
IndexDefinitionStorage.AddIndex(definition.IndexId, definition);
// we start the precomuteTask _after_ we finished adding the index
if (precomputeTask != null)
{
precomputeTask();
}
WorkContext.ShouldNotifyAboutWork(() => "PUT INDEX " + name);
WorkContext.NotifyAboutWork();
}
public Etag GetIndexEtag(string indexName, Etag previousEtag, string resultTransformer = null)
{
Etag lastDocEtag = Etag.Empty;
Etag lastIndexedEtag = null;
Etag lastReducedEtag = null;
bool isStale = false;
int touchCount = 0;
TransactionalStorage.Batch(accessor =>
{
var indexInstance = Database.IndexStorage.GetIndexInstance(indexName);
if (indexInstance == null)
{
return;
}
isStale = (indexInstance.IsMapIndexingInProgress) ||
accessor.Staleness.IsIndexStale(indexInstance.indexId, null, null);
lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag();
var indexStats = accessor.Indexing.GetIndexStats(indexInstance.indexId);
if (indexStats != null)
{
lastReducedEtag = indexStats.LastReducedEtag;
lastIndexedEtag = indexStats.LastIndexedEtag;
}
touchCount = accessor.Staleness.GetIndexTouchCount(indexInstance.indexId);
});
var indexDefinition = GetIndexDefinition(indexName);
if (indexDefinition == null)
{
return(Etag.Empty); // this ensures that we will get the normal reaction of IndexNotFound later on.
}
var list = new List <byte>();
list.AddRange(indexDefinition.GetIndexHash());
list.AddRange(Encoding.Unicode.GetBytes(indexName));
if (string.IsNullOrWhiteSpace(resultTransformer) == false)
{
var abstractTransformer = IndexDefinitionStorage.GetTransformer(resultTransformer);
if (abstractTransformer == null)
{
throw new InvalidOperationException("The result transformer: " + resultTransformer + " was not found");
}
list.AddRange(abstractTransformer.GetHashCodeBytes());
}
list.AddRange(lastDocEtag.ToByteArray());
list.AddRange(BitConverter.GetBytes(touchCount));
list.AddRange(BitConverter.GetBytes(isStale));
if (lastReducedEtag != null)
{
list.AddRange(lastReducedEtag.ToByteArray());
}
if (lastIndexedEtag != null)
{
list.AddRange(lastIndexedEtag.ToByteArray());
}
list.AddRange(BitConverter.GetBytes(UuidGenerator.LastDocumentTransactionEtag));
var indexEtag = Etag.Parse(Encryptor.Current.Hash.Compute16(list.ToArray()));
if (previousEtag != null && previousEtag != indexEtag)
{
// the index changed between the time when we got it and the time
// we actually call this, we need to return something random so that
// the next time we won't get 304
return(Etag.InvalidEtag);
}
return(indexEtag);
}
