public IEnumerable <string> QueryDocumentIds(string index, IndexQuery query, CancellationTokenSource tokenSource, out bool stale)
{
var queryStat = AddToCurrentlyRunningQueryList(index, query, tokenSource);
try
{
bool isStale = false;
HashSet <string> loadedIds = null;
TransactionalStorage.Batch(
actions =>
{
var definition = IndexDefinitionStorage.GetIndexDefinition(index);
if (definition == null)
{
throw new ArgumentException("specified index definition was not found", "index");
}
isStale = actions.Staleness.IsIndexStale(definition.IndexId, query.Cutoff, null);
if (isStale == false && query.Cutoff == null)
{
var indexInstance = Database.IndexStorage.GetIndexInstance(index);
isStale = isStale || (indexInstance != null && indexInstance.IsMapIndexingInProgress);
}
if (isStale && actions.Staleness.IsIndexStaleByTask(definition.IndexId, query.Cutoff) == false &&
actions.Staleness.IsReduceStale(definition.IndexId) == false)
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
{
throw new ArgumentException("specified index definition was not found", "index");
}
var forEntityNames = viewGenerator.ForEntityNames.ToList();
var lastIndexedEtag = actions.Indexing.GetIndexStats(definition.IndexId).LastIndexedEtag;
if (Database.LastCollectionEtags.HasEtagGreaterThan(forEntityNames, lastIndexedEtag) == false)
{
isStale = false;
}
}
var indexFailureInformation = actions.Indexing.GetFailureRate(definition.IndexId);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
loadedIds = new HashSet <string>(from queryResult in Database.IndexStorage.Query(index, query, result => true, new FieldsToFetch(null, false, Constants.DocumentIdFieldName), Database.IndexQueryTriggers, tokenSource.Token)
select queryResult.Key);
});
stale = isStale;
return(loadedIds);
}
finally
{
RemoveFromCurrentlyRunningQueryList(index, queryStat);
}
}
internal void DeleteDocumentFromIndexesForCollection(string key, string collection, IStorageActionsAccessor actions)
{
foreach (var indexName in IndexDefinitionStorage.IndexNames)
{
AbstractViewGenerator abstractViewGenerator =
IndexDefinitionStorage.GetViewGenerator(indexName);
if (abstractViewGenerator == null)
{
continue;
}
if (collection != null && // the document has a entity name
abstractViewGenerator.ForEntityNames.Count > 0)
// the index operations on specific entities
{
if (abstractViewGenerator.ForEntityNames.Contains(collection) == false)
{
continue;
}
}
var instance = IndexDefinitionStorage.GetIndexDefinition(indexName);
var task = actions.GetTask(x => x.Index == instance.IndexId, new RemoveFromIndexTask
{
Index = instance.IndexId
});
task.Keys.Add(key);
}
}
private void TryApplyPrecomputedBatchForNewIndex(Index index, IndexDefinition definition)
{
var generator = IndexDefinitionStorage.GetViewGenerator(definition.IndexId);
if (generator.ForEntityNames.Count == 0)
{
// we don't optimize if we don't have what to optimize _on, we know this is going to return all docs.
// no need to try to optimize that, then
index.IsMapIndexingInProgress = false;
return;
}
try
{
Task.Factory.StartNew(() => ApplyPrecomputedBatchForNewIndex(index, generator),
TaskCreationOptions.LongRunning)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
Log.Warn("Could not apply precomputed batch for index " + index, t.Exception);
}
index.IsMapIndexingInProgress = false;
WorkContext.ShouldNotifyAboutWork(() => "Precomputed indexing batch for " + index.PublicName + " is completed");
WorkContext.NotifyAboutWork();
});
}
catch (Exception)
{
index.IsMapIndexingInProgress = false;
throw;
}
}
public string[] GetIndexFields(string index)
{
var abstractViewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (abstractViewGenerator == null)
{
return(new string[0]);
}
return(abstractViewGenerator.Fields);
}
private Index CreateIndexImplementation(string directoryPath, IndexDefinition indexDefinition, Lucene.Net.Store.Directory directory)
{
var viewGenerator = indexDefinitionStorage.GetViewGenerator(indexDefinition.Name);
var indexImplementation = indexDefinition.IsMapReduce
? (Index) new MapReduceIndex(directory, directoryPath, indexDefinition, viewGenerator, documentDatabase.WorkContext)
: new SimpleIndex(directory, directoryPath, indexDefinition, viewGenerator, documentDatabase.WorkContext);
configuration.Container.SatisfyImportsOnce(indexImplementation);
return(indexImplementation);
}
private void TryApplyPrecomputedBatchForNewIndex(Index index, IndexDefinition definition)
{
var generator = IndexDefinitionStorage.GetViewGenerator(definition.IndexId);
if (generator.ForEntityNames.Count == 0 && index.IsTestIndex == false)
{
// we don't optimize if we don't have what to optimize _on_, we know this is going to return all docs.
// no need to try to optimize that, then
index.IsMapIndexingInProgress = false;
return;
}
try
{
var cts = new CancellationTokenSource();
var task = Task
.Factory
.StartNew(() => ApplyPrecomputedBatchForNewIndex(index, generator, index.IsTestIndex == false ? Database.Configuration.MaxNumberOfItemsToProcessInSingleBatch : Database.Configuration.Indexing.MaxNumberOfItemsToProcessInTestIndexes, cts), TaskCreationOptions.LongRunning)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
Log.Warn("Could not apply precomputed batch for index " + index, t.Exception);
}
index.IsMapIndexingInProgress = false;
WorkContext.ShouldNotifyAboutWork(() => "Precomputed indexing batch for " + index.PublicName + " is completed");
WorkContext.NotifyAboutWork();
});
long id;
Database
.Tasks
.AddTask(
task,
new TaskBasedOperationState(task),
new TaskActions.PendingTaskDescription
{
StartTime = DateTime.UtcNow,
Payload = index.PublicName,
TaskType = TaskActions.PendingTaskType.NewIndexPrecomputedBatch
},
out id,
cts);
}
catch (Exception)
{
index.IsMapIndexingInProgress = false;
throw;
}
}
private void AddIndexingTask(IStorageActionsAccessor actions, JToken metadata, Func <Task> taskGenerator)
{
foreach (var indexName in IndexDefinitionStorage.IndexNames)
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(indexName);
if (viewGenerator == null)
{
continue;
}
var entityName = metadata.Value <string>("Raven-Entity-Name");
if (viewGenerator.ForEntityName != null &&
viewGenerator.ForEntityName != entityName)
{
continue;
}
var task = taskGenerator();
task.Index = indexName;
actions.Tasks.AddTask(task, DateTime.UtcNow);
}
}
public QueryResult Query(string index, IndexQuery query)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var list = new List <JObject>();
var stale = false;
Tuple <DateTime, Guid> indexTimestamp = null;
TransactionalStorage.Batch(
actions =>
{
string entityName = null;
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
{
throw new InvalidOperationException("Could not find index named: " + index);
}
entityName = viewGenerator.ForEntityName;
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, entityName);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers);
var indexDefinition = GetIndexDefinition(index);
var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
viewGenerator.ReduceDefinition == null
? Abstractions.Data.Constants.DocumentIdFieldName
: Abstractions.Data.Constants.ReduceKeyFieldName);
var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch), fieldsToFetch)
select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch)
into doc
where doc != null
select doc;
var transformerErrors = new List <string>();
IEnumerable <JObject> results;
if (viewGenerator != null &&
query.SkipTransformResults == false &&
viewGenerator.TransformResultsDefinition != null)
{
var robustEnumerator = new RobustEnumerator
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
results =
robustEnumerator.RobustEnumeration(
dynamicJsonObjects,
source => viewGenerator.TransformResultsDefinition(docRetriever, source))
.Select(JsonExtensions.ToJObject);
}
else
{
results = collection.Select(x => x.ToJson());
}
list.AddRange(results);
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
});
return(new QueryResult
{
IndexName = index,
Results = list,
IsStale = stale,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2
});
}
private Action TryCreateTaskForApplyingPrecomputedBatchForNewIndex(Index index, IndexDefinition definition)
{
if (Database.Configuration.MaxPrecomputedBatchSizeForNewIndex <= 0) //precaution -> should never be lower than 0
{
index.IsMapIndexingInProgress = false;
return(null);
}
var generator = IndexDefinitionStorage.GetViewGenerator(definition.IndexId);
if (generator.ForEntityNames.Count == 0 && index.IsTestIndex == false)
{
// we don't optimize if we don't have what to optimize _on_, we know this is going to return all docs.
// no need to try to optimize that, then
index.IsMapIndexingInProgress = false;
return(null);
}
//only one precomputed batch can run at a time except for test indexes
if (index.IsTestIndex == false)
{
lock (precomputedLock)
{
if (isPrecomputedBatchForNewIndexIsRunning)
{
index.IsMapIndexingInProgress = false;
return(null);
}
isPrecomputedBatchForNewIndexIsRunning = true;
}
}
try
{
var cts = new CancellationTokenSource();
var task = new Task(() =>
{
try
{
ApplyPrecomputedBatchForNewIndex(index, generator,
index.IsTestIndex == false ?
Database.Configuration.MaxPrecomputedBatchSizeForNewIndex :
Database.Configuration.Indexing.MaxNumberOfItemsToProcessInTestIndexes, cts);
}
catch (TotalDataSizeExceededException e)
{
//expected error
Log.Info(e.Message);
}
catch (Exception e)
{
Log.Warn("Could not apply precomputed batch for index " + index, e);
}
finally
{
if (index.IsTestIndex == false)
{
isPrecomputedBatchForNewIndexIsRunning = false;
}
index.IsMapIndexingInProgress = false;
WorkContext.ShouldNotifyAboutWork(() => "Precomputed indexing batch for " + index.PublicName + " is completed");
WorkContext.NotifyAboutWork();
}
}, TaskCreationOptions.LongRunning);
return(() =>
{
try
{
task.Start();
long id;
Database
.Tasks
.AddTask(
task,
new TaskBasedOperationState(task),
new TaskActions.PendingTaskDescription
{
StartTime = DateTime.UtcNow,
Description = index.PublicName,
TaskType = TaskActions.PendingTaskType.NewIndexPrecomputedBatch
},
out id,
cts);
}
catch (Exception)
{
index.IsMapIndexingInProgress = false;
if (index.IsTestIndex == false)
{
isPrecomputedBatchForNewIndexIsRunning = false;
}
throw;
}
});
}
catch (Exception)
{
index.IsMapIndexingInProgress = false;
if (index.IsTestIndex == false)
{
isPrecomputedBatchForNewIndexIsRunning = false;
}
throw;
}
}
public bool Delete(string key, Etag etag, TransactionInformation transactionInformation, out RavenJObject metadata)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
key = key.Trim();
var deleted = false;
Log.Debug("Delete a document with key: {0} and etag {1}", key, etag);
RavenJObject metadataVar = null;
using (Database.DocumentLock.Lock())
{
TransactionalStorage.Batch(actions =>
{
AssertDeleteOperationNotVetoed(key, transactionInformation);
if (transactionInformation == null)
{
Database.DeleteTriggers.Apply(trigger => trigger.OnDelete(key, null));
Etag deletedETag;
if (actions.Documents.DeleteDocument(key, etag, out metadataVar, out deletedETag))
{
deleted = true;
actions.Indexing.RemoveAllDocumentReferencesFrom(key);
WorkContext.MarkDeleted(key);
Database.Indexes.CheckReferenceBecauseOfDocumentUpdate(key, actions);
foreach (var indexName in IndexDefinitionStorage.IndexNames)
{
AbstractViewGenerator abstractViewGenerator =
IndexDefinitionStorage.GetViewGenerator(indexName);
if (abstractViewGenerator == null)
{
continue;
}
var token = metadataVar.Value <string>(Constants.RavenEntityName);
if (token != null && // the document has a entity name
abstractViewGenerator.ForEntityNames.Count > 0)
// the index operations on specific entities
{
if (abstractViewGenerator.ForEntityNames.Contains(token) == false)
{
continue;
}
}
var instance = IndexDefinitionStorage.GetIndexDefinition(indexName);
var task = actions.GetTask(x => x.Index == instance.IndexId, new RemoveFromIndexTask
{
Index = instance.IndexId
});
task.Keys.Add(key);
}
if (deletedETag != null)
{
Database.Prefetcher.AfterDelete(key, deletedETag);
}
Database.DeleteTriggers.Apply(trigger => trigger.AfterDelete(key, null));
}
TransactionalStorage
.ExecuteImmediatelyOrRegisterForSynchronization(() =>
{
Database.DeleteTriggers.Apply(trigger => trigger.AfterCommit(key));
Database.Notifications.RaiseNotifications(new DocumentChangeNotification
{
Id = key,
Type = DocumentChangeTypes.Delete,
TypeName = (metadataVar != null) ? metadataVar.Value <string>(Constants.RavenClrType) : null,
CollectionName = (metadataVar != null) ? metadataVar.Value <string>(Constants.RavenEntityName) : null
}, metadataVar);
});
}
else
{
var doc = actions.Documents.DocumentMetadataByKey(key, null);
Database.InFlightTransactionalState.DeleteDocumentInTransaction(transactionInformation, key,
etag,
doc == null ? Etag.Empty : doc.Etag,
UuidGenerator);
deleted = doc != null;
}
WorkContext.ShouldNotifyAboutWork(() => "DEL " + key);
});
metadata = metadataVar;
return(deleted);
}
}