public JArray GetDocuments(int start, int pageSize, Guid?etag)
{
var list = new JArray();
TransactionalStorage.Batch(actions =>
{
IEnumerable <JsonDocument> documents;
if (etag == null)
{
documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start);
}
else
{
documents = actions.Documents.GetDocumentsAfter(etag.Value);
}
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents.Take(pageSize))
{
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
{
continue;
}
list.Add(document.ToJson());
}
});
return(list);
}
public Etag GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Func <JsonDocument, bool> addDocument)
{
Etag lastDocumentReadEtag = null;
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize, token);
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, Database.InFlightTransactionalState);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
if (etag != null)
{
etag = doc.Etag;
}
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
returnedDocs = true;
Database.WorkContext.UpdateFoundWork();
bool canContinue = addDocument(document);
if (!canContinue)
{
break;
}
lastDocumentReadEtag = etag;
}
if (returnedDocs || docCount == 0)
{
break;
}
start += docCount;
}
});
return(lastDocumentReadEtag);
}
private void IndexDocuments(IStorageActionsAccessor actions, string index, JsonDocument[] jsonDocs)
{
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
{
return; // index was deleted, probably
}
var dateTime = jsonDocs.Min(x => x.LastModified);
var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
try
{
log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index);
context.IndexStorage.Index(index, viewGenerator,
jsonDocs
.Select(doc => documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Index))
.Where(doc => doc != null)
.Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime);
}
catch (Exception e)
{
if (actions.IsWriteConflict(e))
{
return;
}
log.WarnFormat(e, "Failed to index documents for index: {0}", index);
}
}
public Etag GetDocumentsWithIdStartingWith(string idPrefix, int pageSize, Etag etag, CancellationToken token, Action <JsonDocument> addDocument)
{
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = actions.Documents.GetDocumentsAfterWithIdStartingWith(etag, idPrefix, pageSize, token, timeout: TimeSpan.FromSeconds(2));
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, Database.InFlightTransactionalState);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
etag = doc.Etag;
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
addDocument(document);
returnedDocs = true;
Database.WorkContext.UpdateFoundWork();
}
if (returnedDocs || docCount == 0)
{
break;
}
}
});
return(etag);
}
public static int UpdateDocumentsAfter(this DocumentDatabase db, Guid eTag, Func <JsonDocument, bool> updateDoc,
CancellationToken cancellationToken, TransactionInformation transactionInformation)
{
var initialTime = DateTime.UtcNow;
log.Trace("UpdateDocumentsAfter started whith Etag {0}", eTag);
int updatedDocCount = 0;
db.TransactionalStorage.Batch(action =>
{
var documentRetriever = new DocumentRetriever(action, db.ReadTriggers);
foreach (var doc in action.Documents.GetDocumentsAfter(eTag, int.MaxValue))
{
DocumentRetriever.EnsureIdInMetadata(doc);
if (cancellationToken.IsCancellationRequested)
{
log.Trace("UpdateDocumentsAfter has been cancelled");
}
cancellationToken.ThrowIfCancellationRequested();
var document = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document != null && updateDoc(document))
{
try
{
db.Put(document.Key, document.Etag, document.DataAsJson, document.Metadata, transactionInformation);
updatedDocCount++;
log.Debug("{0} document has been updated in UpdateDocumentsAfter. {1} documents updated so far", document.Key, updatedDocCount);
}
catch (ConcurrencyException)
{
log.Trace("ConcurrencyException caught in UpdateDocumentsAfter");
if (db.UpdateDocumentWithRetries(document.Key, updateDoc, transactionInformation))
{
updatedDocCount++;
log.Debug("{0} document has been updated in UpdateDocumentsAfter. {1} documents updated so far", document.Key, updatedDocCount);
}
}
}
}
});
log.Trace("UpdateDocumentsAfter ETag {0} completed successfully. {1} documents updated in {2}", eTag, updatedDocCount, DateTime.UtcNow.Subtract(initialTime));
return(updatedDocCount);
}
public void GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Action <RavenJObject> addDocument)
{
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize, WorkContext.CancellationToken);
var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
if (etag != null)
{
etag = doc.Etag;
}
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever
.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
addDocument(document.ToJson());
returnedDocs = true;
}
if (returnedDocs || docCount == 0)
{
break;
}
start += docCount;
}
});
}
public static IEnumerable <JsonDocument> GetDocsWithIdStartingWith(this DocumentDatabase db, string idPrefix, int start, int pageSize)
{
if (idPrefix == null)
{
throw new ArgumentNullException("idPrefix");
}
idPrefix = idPrefix.Trim();
var docs = new List <JsonDocument>(pageSize);
db.TransactionalStorage.Batch(actions =>
{
var retriever = new DocumentRetriever(actions, db.ReadTriggers);
foreach (var doc in actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize))
{
var document = retriever.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
{
continue;
}
docs.Add(document);
}
});
return(docs);
}
public JArray GetDocumentsWithIdStartingWith(string idPrefix, int start, int pageSize)
{
var list = new JArray();
TransactionalStorage.Batch(actions =>
{
var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start)
.Take(pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents)
{
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
{
continue;
}
list.Add(document.ToJson());
}
});
return(list);
}
public RavenJArray GetDocuments(int start, int pageSize, Guid? etag)
{
var list = new RavenJArray();
TransactionalStorage.Batch(actions =>
{
IEnumerable<JsonDocument> documents;
if (etag == null)
documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start);
else
documents = actions.Documents.GetDocumentsAfter(etag.Value);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents.Take(pageSize))
{
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
list.Add(document.ToJson());
}
});
return list;
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize,
CancellationToken token, ref int nextStart, Action <RavenJObject> addDoc,
string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null,
string skipAfter = null)
{
if (idPrefix == null)
{
throw new ArgumentNullException("idPrefix");
}
idPrefix = idPrefix.Trim();
var canPerformRapidPagination = nextStart > 0 && start == nextStart;
var actualStart = canPerformRapidPagination ? start : 0;
var addedDocs = 0;
var matchedDocs = 0;
TransactionalStorage.Batch(
actions =>
{
var docsToSkip = canPerformRapidPagination ? 0 : start;
int docCount;
AbstractTransformer storedTransformer = null;
if (transformer != null)
{
storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
{
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
}
do
{
docCount = 0;
var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter);
var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState, transformerParameters);
foreach (var doc in docs)
{
token.ThrowIfCancellationRequested();
docCount++;
var keyTest = doc.Key.Substring(idPrefix.Length);
if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
{
continue;
}
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc;
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
matchedDocs++;
if (matchedDocs <= docsToSkip)
{
continue;
}
token.ThrowIfCancellationRequested();
if (storedTransformer != null)
{
using (new CurrentTransformationScope(Database, documentRetriever))
{
var transformed =
storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) })
.Select(x => JsonExtensions.ToJObject(x))
.ToArray();
if (transformed.Length == 0)
{
throw new InvalidOperationException("The transform results function failed on a document: " + document.Key);
}
var transformedJsonDocument = new JsonDocument
{
Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(documentRetriever.Etag),
NonAuthoritativeInformation = document.NonAuthoritativeInformation,
LastModified = document.LastModified,
DataAsJson = new RavenJObject {
{ "$values", new RavenJArray(transformed) }
},
};
addDoc(transformedJsonDocument.ToJson());
}
}
else
{
addDoc(document.ToJson());
}
addedDocs++;
if (addedDocs >= pageSize)
{
break;
}
}
actualStart += pageSize;
}while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue);
});
if (addedDocs != pageSize)
{
nextStart = start; // will mark as last page
}
else if (canPerformRapidPagination)
{
nextStart = start + matchedDocs;
}
else
{
nextStart = actualStart;
}
}
public Etag GetDocumentsWithIdStartingWith(string idPrefix, int pageSize, Etag etag, CancellationToken token, Func <JsonDocument, bool> addDocument)
{
Etag lastDocumentReadEtag = null;
TransactionalStorage.Batch(actions =>
{
var returnedDocs = false;
while (true)
{
var documents = actions.Documents.GetDocumentsAfterWithIdStartingWith(etag, idPrefix, pageSize, token, timeout: TimeSpan.FromSeconds(2), lastProcessedDocument: x => lastDocumentReadEtag = x);
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers);
var docCount = 0;
var docCountOnLastAdd = 0;
foreach (var doc in documents)
{
docCount++;
if (docCount - docCountOnLastAdd > 1000)
{
addDocument(null); // heartbeat
}
token.ThrowIfCancellationRequested();
etag = doc.Etag;
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
returnedDocs = true;
Database.WorkContext.UpdateFoundWork();
var canContinue = addDocument(document);
docCountOnLastAdd = docCount;
if (!canContinue)
{
break;
}
}
if (returnedDocs)
{
break;
}
// No document was found that matches the requested criteria
if (docCount == 0)
{
// If we had a failure happen, we update the etag as we don't need to process those documents again (no matches there anyways).
if (lastDocumentReadEtag != null)
{
etag = lastDocumentReadEtag;
}
break;
}
}
});
return(etag);
}
public Etag GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Func <JsonDocument, bool> addDocument,
string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null, long?maxSize = null, TimeSpan?timeout = null)
{
Etag lastDocumentReadEtag = null;
using (DocumentCacher.SkipSetDocumentsInDocumentCache())
TransactionalStorage.Batch(actions =>
{
AbstractTransformer storedTransformer = null;
if (transformer != null)
{
storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
{
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
}
var returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize, token, maxSize: maxSize, timeout: timeout);
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, transformerParameters);
var docCount = 0;
var docCountOnLastAdd = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
if (docCount - docCountOnLastAdd > 1000)
{
addDocument(null); // heartbeat
}
if (etag != null)
{
etag = doc.Etag;
}
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
returnedDocs = true;
Database.WorkContext.UpdateFoundWork();
document = TransformDocumentIfNeeded(document, storedTransformer, documentRetriever);
var canContinue = addDocument(document);
if (!canContinue)
{
break;
}
lastDocumentReadEtag = etag;
docCountOnLastAdd = docCount;
}
if (returnedDocs || docCount == 0)
{
break;
}
// No document was found that matches the requested criteria
// If we had a failure happen, we update the etag as we don't need to process those documents again (no matches there anyways).
if (lastDocumentReadEtag != null)
{
etag = lastDocumentReadEtag;
}
start += docCount;
}
});
return(lastDocumentReadEtag);
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize,
CancellationToken token, ref int nextStart, Action <JsonDocument> addDoc,
string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null,
string skipAfter = null)
{
if (idPrefix == null)
{
throw new ArgumentNullException("idPrefix");
}
idPrefix = idPrefix.Trim();
var canPerformRapidPagination = nextStart > 0 && start == nextStart;
var actualStart = canPerformRapidPagination ? start : 0;
var addedDocs = 0;
var docCountOnLastAdd = 0;
var matchedDocs = 0;
TransactionalStorage.Batch(
actions =>
{
var docsToSkip = canPerformRapidPagination ? 0 : start;
int docCount;
AbstractTransformer storedTransformer = null;
if (transformer != null)
{
storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
{
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
}
do
{
Database.WorkContext.UpdateFoundWork();
docCount = 0;
var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter);
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, transformerParameters);
foreach (var doc in docs)
{
token.ThrowIfCancellationRequested();
docCount++;
if (docCount - docCountOnLastAdd > 1000)
{
addDoc(null); // heartbeat
}
var keyTest = doc.Key.Substring(idPrefix.Length);
if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
{
continue;
}
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc;
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
matchedDocs++;
if (matchedDocs <= docsToSkip)
{
continue;
}
token.ThrowIfCancellationRequested();
document = TransformDocumentIfNeeded(document, storedTransformer, documentRetriever);
addDoc(document);
addedDocs++;
docCountOnLastAdd = docCount;
if (addedDocs >= pageSize)
{
break;
}
}
actualStart += pageSize;
} while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue);
});
if (addedDocs != pageSize)
{
nextStart = start; // will mark as last page
}
else if (canPerformRapidPagination)
{
nextStart = start + matchedDocs;
}
else
{
nextStart = actualStart;
}
}
private IEnumerable <Tuple <IndexToWorkOn, IndexingBatch> > FilterIndexes(IList <IndexToWorkOn> indexesToWorkOn, JsonDocument[] jsonDocs)
{
var last = jsonDocs.Last();
Debug.Assert(last.Etag != null);
Debug.Assert(last.LastModified != null);
var lastEtag = last.Etag.Value;
var lastModified = last.LastModified.Value;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
var filteredDocs =
BackgroundTaskExecuter.Instance.Apply(jsonDocs, doc =>
{
doc = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Index);
return(doc == null ? null : new { Doc = doc, Json = JsonToExpando.Convert(doc.ToJson()) });
});
log.Debug("After read triggers executed, {0} documents remained", filteredDocs.Count);
var results = new Tuple <IndexToWorkOn, IndexingBatch> [indexesToWorkOn.Count];
var actions = new Action <IStorageActionsAccessor> [indexesToWorkOn.Count];
BackgroundTaskExecuter.Instance.ExecuteAll(context.Configuration, scheduler, indexesToWorkOn, (indexToWorkOn, i) =>
{
var indexLastInedexEtag = new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray());
if (indexLastInedexEtag.CompareTo(lastIndexedEtag) >= 0)
{
return;
}
var indexName = indexToWorkOn.IndexName;
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName);
if (viewGenerator == null)
{
return; // probably deleted
}
var batch = new IndexingBatch();
foreach (var item in filteredDocs)
{
// did we already indexed this document in this index?
if (indexLastInedexEtag.CompareTo(new ComparableByteArray(item.Doc.Etag.Value.ToByteArray())) >= 0)
{
continue;
}
// is the Raven-Entity-Name a match for the things the index executes on?
if (viewGenerator.ForEntityNames.Count != 0 &&
viewGenerator.ForEntityNames.Contains(item.Doc.Metadata.Value <string>(Constants.RavenEntityName)) == false)
{
continue;
}
batch.Add(item.Doc, item.Json);
if (batch.DateTime == null)
{
batch.DateTime = item.Doc.LastModified;
}
else
{
batch.DateTime = batch.DateTime > item.Doc.LastModified
? item.Doc.LastModified
: batch.DateTime;
}
}
if (batch.Docs.Count == 0)
{
log.Debug("All documents have been filtered for {0}, no indexing will be performed, updating to {1}, {2}", indexName,
lastEtag, lastModified);
// we use it this way to batch all the updates together
actions[i] = accessor => accessor.Indexing.UpdateLastIndexed(indexName, lastEtag, lastModified);
return;
}
log.Debug("Going to index {0} documents in {1}", batch.Ids.Count, indexToWorkOn);
results[i] = Tuple.Create(indexToWorkOn, batch);
});
transactionalStorage.Batch(actionsAccessor =>
{
foreach (var action in actions)
{
if (action != null)
{
action(actionsAccessor);
}
}
});
return(results.Where(x => x != null));
}
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, string matches, int start, int pageSize)
{
if (idPrefix == null)
throw new ArgumentNullException("idPrefix");
idPrefix = idPrefix.Trim();
var list = new RavenJArray();
TransactionalStorage.Batch(actions =>
{
var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents)
{
if (WildcardMatcher.Matches(matches, doc.Key.Substring(idPrefix.Length)) == false)
continue;
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
list.Add(document.ToJson());
}
});
return list;
}
public void GetDocuments(int start, int pageSize, Etag etag, Action<RavenJObject> addDocument)
{
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
if (etag != null)
etag = doc.Etag;
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
addDocument(document.ToJson());
returnedDocs = true;
}
if (returnedDocs || docCount == 0)
break;
start += docCount;
}
});
}
public JArray GetDocuments(int start, int pageSize, Guid? etag)
{
var list = new JArray();
TransactionalStorage.Batch(actions =>
{
IEnumerable<JsonDocument> documents;
if (etag == null)
documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start);
else
documents = actions.Documents.GetDocumentsAfter(etag.Value);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents.Take(pageSize))
{
var document = documentRetriever.ExecuteReadTriggers(doc, null,
// here we want to have the Load semantic, not Query, because we need this to be
// as close as possible to the full database contents
ReadOperation.Load);
if (document == null)
continue;
if (document.Metadata.Property("@id") == null)
document.Metadata.Add("@id", new JValue(doc.Key));
list.Add(document.ToJson());
}
});
return list;
}
private IEnumerable <IndexingBatchForIndex> FilterIndexes(IList <IndexToWorkOn> indexesToWorkOn, List <JsonDocument> jsonDocs, Etag highestETagInBatch)
{
var last = jsonDocs.Last();
Debug.Assert(last.Etag != null);
Debug.Assert(last.LastModified != null);
var lastEtag = last.Etag;
var lastModified = last.LastModified.Value;
var documentRetriever = new DocumentRetriever(null, null, context.ReadTriggers, context.Database.InFlightTransactionalState);
var filteredDocs =
BackgroundTaskExecuter.Instance.Apply(context, jsonDocs, doc =>
{
var filteredDoc = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Index);
return(filteredDoc == null ? new
{
Doc = doc,
Json = (object)new FilteredDocument(doc)
} : new
{
Doc = filteredDoc,
Json = JsonToExpando.Convert(doc.ToJson())
});
});
Log.Debug("After read triggers executed, {0} documents remained", filteredDocs.Count);
var results = new IndexingBatchForIndex[indexesToWorkOn.Count];
var actions = new Action <IStorageActionsAccessor> [indexesToWorkOn.Count];
BackgroundTaskExecuter.Instance.ExecuteAll(context, indexesToWorkOn, (indexToWorkOn, i) =>
{
var indexName = indexToWorkOn.Index.PublicName;
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName);
if (viewGenerator == null)
{
return; // probably deleted
}
var batch = new IndexingBatch(highestETagInBatch);
foreach (var item in filteredDocs)
{
if (defaultPrefetchingBehavior.FilterDocuments(item.Doc) == false)
{
continue;
}
// did we already indexed this document in this index?
var etag = item.Doc.Etag;
if (etag == null)
{
continue;
}
// is the Raven-Entity-Name a match for the things the index executes on?
if (viewGenerator.ForEntityNames.Count != 0 &&
viewGenerator.ForEntityNames.Contains(item.Doc.Metadata.Value <string>(Constants.RavenEntityName)) == false)
{
continue;
}
batch.Add(item.Doc, item.Json, defaultPrefetchingBehavior.ShouldSkipDeleteFromIndex(item.Doc));
if (batch.DateTime == null)
{
batch.DateTime = item.Doc.LastModified;
}
else
{
batch.DateTime = batch.DateTime > item.Doc.LastModified
? item.Doc.LastModified
: batch.DateTime;
}
}
if (batch.Docs.Count == 0)
{
Log.Debug("All documents have been filtered for {0}, no indexing will be performed, updating to {1}, {2}", indexName,
lastEtag, lastModified);
// we use it this way to batch all the updates together
actions[i] = accessor =>
{
accessor.Indexing.UpdateLastIndexed(indexToWorkOn.Index.indexId, lastEtag, lastModified);
accessor.AfterStorageCommit += () =>
{
indexToWorkOn.Index.EnsureIndexWriter();
indexToWorkOn.Index.Flush(lastEtag);
};
};
return;
}
if (Log.IsDebugEnabled)
{
Log.Debug("Going to index {0} documents in {1}: ({2})", batch.Ids.Count, indexToWorkOn, string.Join(", ", batch.Ids));
}
results[i] = new IndexingBatchForIndex
{
Batch = batch,
IndexId = indexToWorkOn.IndexId,
Index = indexToWorkOn.Index,
LastIndexedEtag = indexToWorkOn.LastIndexedEtag
};
});
transactionalStorage.Batch(actionsAccessor =>
{
foreach (var action in actions)
{
if (action != null)
{
action(actionsAccessor);
}
}
});
return(results.Where(x => x != null));
}
private IEnumerable<IndexingBatchForIndex> FilterIndexes(IList<IndexToWorkOn> indexesToWorkOn, List<JsonDocument> jsonDocs, Etag highestETagInBatch, out List<IndexToWorkOn> filteredOutIndexes)
{
var innerFilteredOutIndexes = new ConcurrentStack<IndexToWorkOn>();
var last = jsonDocs.Last();
Debug.Assert(last.Etag != null);
Debug.Assert(last.LastModified != null);
var lastEtag = last.Etag;
var lastModified = last.LastModified.Value;
var documentRetriever = new DocumentRetriever(null, null, context.ReadTriggers);
var filteredDocs =
BackgroundTaskExecuter.Instance.Apply(context, jsonDocs, doc =>
{
var filteredDoc = documentRetriever.ExecuteReadTriggers(doc, ReadOperation.Index);
return filteredDoc == null ? new
{
Doc = doc,
Json = (object)new FilteredDocument(doc)
} : new
{
Doc = filteredDoc,
Json = JsonToExpando.Convert(doc.ToJson())
};
});
if ( Log.IsDebugEnabled )
Log.Debug("After read triggers executed, {0} documents remained", filteredDocs.Count);
var results = new ConcurrentQueue<IndexingBatchForIndex>();
var actions = new ConcurrentQueue<Action<IStorageActionsAccessor>>();
context.Database.MappingThreadPool.ExecuteBatch(indexesToWorkOn, indexToWorkOn =>
{
var indexName = indexToWorkOn.Index.PublicName;
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName);
if (viewGenerator == null)
return; // probably deleted
var batch = new IndexingBatch(highestETagInBatch);
foreach (var filteredDoc in filteredDocs)
{
var doc = filteredDoc.Doc;
var json = filteredDoc.Json;
if (defaultPrefetchingBehavior.FilterDocuments(doc) == false
|| doc.Etag.CompareTo(indexToWorkOn.LastIndexedEtag) <= 0)
continue;
// did we already indexed this document in this index?
var etag = doc.Etag;
if (etag == null)
continue;
// is the Raven-Entity-Name a match for the things the index executes on?
if (viewGenerator.ForEntityNames.Count != 0 &&
viewGenerator.ForEntityNames.Contains(doc.Metadata.Value<string>(Constants.RavenEntityName)) == false)
{
continue;
}
batch.Add(doc, json, defaultPrefetchingBehavior.ShouldSkipDeleteFromIndex(doc));
if (batch.DateTime == null)
batch.DateTime = doc.LastModified;
else
batch.DateTime = batch.DateTime > doc.LastModified
? doc.LastModified
: batch.DateTime;
}
if (batch.Docs.Count == 0)
{
if ( Log.IsDebugEnabled )
Log.Debug("All documents have been filtered for {0}, no indexing will be performed, updating to {1}, {2}", indexName, lastEtag, lastModified);
// we use it this way to batch all the updates together
if (indexToWorkOn.LastIndexedEtag.CompareTo(lastEtag) < 0)
actions.Enqueue(accessor =>
{
accessor.Indexing.UpdateLastIndexed(indexToWorkOn.Index.indexId, lastEtag, lastModified);
accessor.AfterStorageCommit += () =>
{
indexToWorkOn.Index.EnsureIndexWriter();
indexToWorkOn.Index.Flush(lastEtag);
};
});
innerFilteredOutIndexes.Push(indexToWorkOn);
context.MarkIndexFilteredOut(indexName);
return;
}
if (Log.IsDebugEnabled)
Log.Debug("Going to index {0} documents in {1}: ({2})", batch.Ids.Count, indexToWorkOn, string.Join(", ", batch.Ids));
results.Enqueue(new IndexingBatchForIndex
{
Batch = batch,
IndexId = indexToWorkOn.IndexId,
Index = indexToWorkOn.Index,
LastIndexedEtag = indexToWorkOn.LastIndexedEtag
});
}, description: string.Format("Filtering documents for {0} indexes", indexesToWorkOn.Count));
filteredOutIndexes = innerFilteredOutIndexes.ToList();
foreach (var action in actions)
{
bool keepTrying = true;
for (int i = 0; i < 10 && keepTrying; i++)
{
keepTrying = false;
transactionalStorage.Batch(actionsAccessor =>
{
if (action != null)
{
try
{
action(actionsAccessor);
}
catch (Exception e)
{
if (actionsAccessor.IsWriteConflict(e))
{
keepTrying = true;
return;
}
throw;
}
}
});
if (keepTrying)
Thread.Sleep(11);
}
}
return results.Where(x => x != null);
}
public RavenJArray GetDocuments(int start, int pageSize, Guid? etag)
{
var list = new RavenJArray();
TransactionalStorage.Batch(actions =>
{
while (true)
{
var documents = etag == null ?
actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) :
actions.Documents.GetDocumentsAfter(etag.Value, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
if(etag != null)
etag = doc.Etag;
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
list.Add(document.ToJson());
}
if (list.Length != 0 || docCount == 0)
break;
start += docCount;
}
});
return list;
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize, Action<RavenJObject> addDoc)
{
if (idPrefix == null)
throw new ArgumentNullException("idPrefix");
idPrefix = idPrefix.Trim();
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
int docCount = 0;
var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers, inFlightTransactionalState);
foreach (var doc in documents)
{
docCount++;
string keyTest = doc.Key.Substring(idPrefix.Length);
if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
continue;
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = inFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key);
JsonDocument document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc;
document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
addDoc(document.ToJson());
returnedDocs = true;
}
if (returnedDocs || docCount == 0)
break;
start += docCount;
}
});
}
public void GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Action<RavenJObject> addDocument)
{
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers, inFlightTransactionalState);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
if (etag != null)
etag = doc.Etag;
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = inFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever
.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
continue;
addDocument(document.ToJson());
returnedDocs = true;
}
if (returnedDocs || docCount == 0)
break;
start += docCount;
}
});
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize, CancellationToken token, Action<RavenJObject> addDoc)
{
if (idPrefix == null)
throw new ArgumentNullException("idPrefix");
idPrefix = idPrefix.Trim();
TransactionalStorage.Batch(
actions =>
{
var docsToSkip = start;
var addedDocs = 0;
var matchedDocs = 0;
int docCount;
start = 0;
do
{
docCount = 0;
var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers, inFlightTransactionalState);
foreach (var doc in docs)
{
token.ThrowIfCancellationRequested();
docCount++;
var keyTest = doc.Key.Substring(idPrefix.Length);
if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
continue;
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = inFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc;
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
continue;
matchedDocs++;
if (matchedDocs <= docsToSkip)
continue;
token.ThrowIfCancellationRequested();
addDoc(document.ToJson());
addedDocs++;
if (addedDocs >= pageSize)
break;
}
start += pageSize;
}
while (docCount > 0 && addedDocs < pageSize && start >= 0 && start < int.MaxValue);
});
}
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, int start, int pageSize)
{
var list = new RavenJArray();
TransactionalStorage.Batch(actions =>
{
var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start)
.Take(pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents)
{
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
list.Add(document.ToJson());
}
});
return list;
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, int start, int pageSize, Action<RavenJObject> addDoc)
{
if (idPrefix == null)
throw new ArgumentNullException("idPrefix");
idPrefix = idPrefix.Trim();
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
int docCount = 0;
var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents)
{
docCount++;
if (WildcardMatcher.Matches(matches, doc.Key.Substring(idPrefix.Length)) == false)
continue;
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
addDoc(document.ToJson());
returnedDocs = true;
}
if (returnedDocs || docCount == 0)
break;
start += docCount;
}
});
}