public CurrentTransformationScope(DocumentDatabase database, DocumentRetriever documentRetriever)
{
this.database = database;
retriever = documentRetriever;
old = current;
current = this;
}
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray jsonDocuments = null;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag, 100)
.Where(destination.FilterDocuments)
.Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return(jsonDocuments);
}
private void IndexDocuments(IStorageActionsAccessor actions, string index, JsonDocument[] jsonDocs)
{
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
{
return; // index was deleted, probably
}
var dateTime = jsonDocs.Min(x => x.LastModified);
var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
try
{
log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index);
context.IndexStorage.Index(index, viewGenerator,
jsonDocs
.Select(doc => documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Index))
.Where(doc => doc != null)
.Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime);
}
catch (Exception e)
{
if (actions.IsWriteConflict(e))
{
return;
}
log.WarnFormat(e, "Failed to index documents for index: {0}", index);
}
}
private Dictionary <string, List <ItemToReplicate> > ApplyConversionScript(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs)
{
var dictionary = new Dictionary <string, List <ItemToReplicate> >();
foreach (var jsonDocument in docs)
{
if (string.IsNullOrEmpty(cfg.RavenEntityName) == false)
{
var entityName = jsonDocument.Metadata.Value <string>(Constants.RavenEntityName);
if (string.Equals(cfg.RavenEntityName, entityName, StringComparison.InvariantCultureIgnoreCase) == false)
{
continue;
}
}
var patcher = new SqlReplicationScriptedJsonPatcher(Database, dictionary, jsonDocument.Key);
try
{
DocumentRetriever.EnsureIdInMetadata(jsonDocument);
jsonDocument.Metadata[Constants.DocumentIdFieldName] = jsonDocument.Key;
var document = jsonDocument.ToJson();
patcher.Apply(document, new ScriptedPatchRequest
{
Script = cfg.Script
});
}
catch (Exception e)
{
log.WarnException("Could not process SQL Replication script for " + cfg.Name + ", skipping this document", e);
}
}
return(dictionary);
}
private JArray GetJsonDocuments(Guid etag)
{
JArray jsonDocuments = null;
try
{
var instanceId = docDb.TransactionalStorage.Id.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonDocuments = new JArray(actions.Documents.GetDocumentsAfter(etag)
.Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs
.Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) == instanceId) // only replicate documents created on this instance
.Where(x => x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Take(100)
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.Warn("Could not get documents to replicate after: " + etag, e);
}
return(jsonDocuments);
}
public void AfterCommit(JsonDocument[] docs)
{
if (context.Configuration.DisableDocumentPreFetchingForIndexing)
{
return;
}
if (futureIndexBatches.Count > 512 || // this is optimization, and we need to make sure we don't overuse memory
docs.Length == 0)
{
return;
}
foreach (var doc in docs)
{
DocumentRetriever.EnsureIdInMetadata(doc);
}
futureIndexBatches.Add(new FutureIndexBatch
{
StartingEtag = GetLowestEtag(docs),
Task = new CompletedTask <JsonResults>(new JsonResults
{
Results = docs,
LoadedFromDisk = false
}),
Age = Interlocked.Increment(ref currentIndexingAge)
});
}
public Etag GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Func <JsonDocument, bool> addDocument)
{
Etag lastDocumentReadEtag = null;
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize, token);
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, Database.InFlightTransactionalState);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
if (etag != null)
{
etag = doc.Etag;
}
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
returnedDocs = true;
Database.WorkContext.UpdateFoundWork();
bool canContinue = addDocument(document);
if (!canContinue)
{
break;
}
lastDocumentReadEtag = etag;
}
if (returnedDocs || docCount == 0)
{
break;
}
start += docCount;
}
});
return(lastDocumentReadEtag);
}
private List <JsonDocument> GetJsonDocsFromDisk(Etag etag, Etag untilEtag)
{
List <JsonDocument> jsonDocs = null;
context.TransactionalStorage.Batch(actions =>
{
jsonDocs = actions.Documents
.GetDocumentsAfter(
etag,
autoTuner.NumberOfItemsToIndexInSingleBatch,
context.CancellationToken,
maxSize: autoTuner.MaximumSizeAllowedToFetchFromStorage,
untilEtag: untilEtag,
timeout: autoTuner.FetchingDocumentsFromDiskTimeout)
.Where(x => x != null)
.Select(doc =>
{
DocumentRetriever.EnsureIdInMetadata(doc);
return(doc);
})
.ToList();
});
if (untilEtag == null)
{
MaybeAddFutureBatch(jsonDocs);
}
return(jsonDocs);
}
public JArray GetDocuments(int start, int pageSize, Guid?etag)
{
var list = new JArray();
TransactionalStorage.Batch(actions =>
{
IEnumerable <JsonDocument> documents;
if (etag == null)
{
documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start);
}
else
{
documents = actions.Documents.GetDocumentsAfter(etag.Value);
}
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents.Take(pageSize))
{
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
{
continue;
}
list.Add(document.ToJson());
}
});
return(list);
}
private List <JsonDocument> GetJsonDocsFromDisk(Guid etag)
{
List <JsonDocument> jsonDocs = null;
var untilEtag = GetNextEtagInMemory();
context.TransactionalStorage.Batch(actions =>
{
jsonDocs = actions.Documents
.GetDocumentsAfter(
etag,
autoTuner.NumberOfItemsToIndexInSingleBatch,
autoTuner.MaximumSizeAllowedToFetchFromStorage,
untilEtag: untilEtag)
.Where(x => x != null)
.Select(doc =>
{
DocumentRetriever.EnsureIdInMetadata(doc);
return(doc);
})
.ToList();
});
if (untilEtag == null)
{
MaybeAddFutureBatch(jsonDocs);
}
return(jsonDocs);
}
private void StartGetDocumentOnBackground()
{
while (run)
{
DocumentDatabase.TransactionalStorage.Batch(accessor =>
{
var documents = accessor.Documents.GetDocumentsAfter(lastEtagSeen, 128)
.Where(x => x != null)
.Select(doc =>
{
DocumentRetriever.EnsureIdInMetadata(doc);
return(doc);
})
.ToArray();
if (documents.Length == 0)
{
return;
}
lastEtagSeen = documents.Last().Etag;
log.Debug("Docs: {0}", string.Join(", ", documents.Select(x => x.Key)));
getDocumentsState.Enqueue(new GetDocumentState(lastEtagSeen, documents.Length));
});
}
}
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource)
{
RavenJArray jsonDocuments = null;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag)
.Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs
.Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source
.Where(x => x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Take(100)
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.Warn("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return(jsonDocuments);
}
public JsonDocument Get(string key, TransactionInformation transactionInformation)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
key = key.Trim();
JsonDocument document = null;
if (transactionInformation == null ||
Database.InFlightTransactionalState.TryGet(key, transactionInformation, out document) == false)
{
// first we check the dtc state, then the storage, to avoid race conditions
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(transactionInformation, key);
TransactionalStorage.Batch(actions => { document = actions.Documents.DocumentByKey(key); });
if (nonAuthoritativeInformationBehavior != null)
{
document = nonAuthoritativeInformationBehavior(document);
}
}
DocumentRetriever.EnsureIdInMetadata(document);
return(new DocumentRetriever(null, Database.ReadTriggers, Database.InFlightTransactionalState)
.ExecuteReadTriggers(document, transactionInformation, ReadOperation.Load));
}
public void AfterCommit(JsonDocument[] docs)
{
if (futureIndexBatches.Count > 512 || // this is optimization, and we need to make sure we don't overuse memory
docs.Length == 0)
{
return;
}
foreach (var doc in docs)
{
DocumentRetriever.EnsureIdInMetadata(doc);
}
futureIndexBatches.Add(new FutureIndexBatch
{
StartingEtag = DecrementEtag(GetLowestEtag(docs)),
Task = new CompletedTask <JsonResults>(new JsonResults
{
Results = docs,
LoadedFromDisk = false
}),
Age = currentIndexingAge
});
}
public JsonDocumentMetadata GetDocumentMetadata(string key, TransactionInformation transactionInformation)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
key = key.Trim();
JsonDocumentMetadata document = null;
if (transactionInformation == null ||
Database.InFlightTransactionalState.TryGet(key, transactionInformation, out document) == false)
{
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocumentMetadata>(transactionInformation, key);
TransactionalStorage.Batch(actions =>
{
document = actions.Documents.DocumentMetadataByKey(key);
});
if (nonAuthoritativeInformationBehavior != null)
{
document = nonAuthoritativeInformationBehavior(document);
}
}
DocumentRetriever.EnsureIdInMetadata(document);
return(new DocumentRetriever(null, Database.ReadTriggers, Database.InFlightTransactionalState)
.ProcessReadVetoes(document, transactionInformation, ReadOperation.Load));
}
private ConversionScriptResult ApplyConversionScript(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs)
{
var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name));
var result = new ConversionScriptResult();
foreach (var jsonDocument in docs)
{
Database.WorkContext.CancellationToken.ThrowIfCancellationRequested();
if (string.IsNullOrEmpty(cfg.RavenEntityName) == false)
{
var entityName = jsonDocument.Metadata.Value <string>(Constants.RavenEntityName);
if (string.Equals(cfg.RavenEntityName, entityName, StringComparison.InvariantCultureIgnoreCase) == false)
{
continue;
}
}
var patcher = new SqlReplicationScriptedJsonPatcher(Database, result, cfg, jsonDocument.Key);
using (var scope = new SqlReplicationScriptedJsonPatcherOperationScope(Database))
{
try
{
DocumentRetriever.EnsureIdInMetadata(jsonDocument);
var document = jsonDocument.ToJson();
document[Constants.DocumentIdFieldName] = jsonDocument.Key;
patcher.Apply(scope, document, new ScriptedPatchRequest {
Script = cfg.Script
}, jsonDocument.SerializedSizeOnDisk);
if (log.IsDebugEnabled && patcher.Debug.Count > 0)
{
log.Debug("Debug output for doc: {0} for script {1}:\r\n.{2}", jsonDocument.Key, cfg.Name, string.Join("\r\n", patcher.Debug));
patcher.Debug.Clear();
}
replicationStats.ScriptSuccess();
}
catch (ParseException e)
{
replicationStats.MarkScriptAsInvalid(Database, cfg.Script);
log.WarnException("Could not parse SQL Replication script for " + cfg.Name, e);
return(result);
}
catch (Exception e)
{
replicationStats.RecordScriptError(Database);
log.WarnException("Could not process SQL Replication script for " + cfg.Name + ", skipping document: " + jsonDocument.Key, e);
}
}
}
return(result);
}
private void ExecuteIndexingInternal(IEnumerable <IndexToWorkOn> indexesToWorkOn, Action <JsonDocument[]> indexingOp)
{
var lastIndexedGuidForAllIndexes = indexesToWorkOn.Min(x => new ComparableByteArray(x.LastIndexedEtag.ToByteArray())).ToGuid();
JsonDocument[] jsonDocs = null;
try
{
transactionalStorage.Batch(actions =>
{
jsonDocs = actions.Documents.GetDocumentsAfter(lastIndexedGuidForAllIndexes)
.Where(x => x != null)
.Select(doc =>
{
DocumentRetriever.EnsureIdInMetadata(doc);
return(doc);
})
.Take(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) // ensure that we won't go overboard with reading and blow up with OOM
.ToArray();
});
if (jsonDocs.Length > 0)
{
indexingOp(jsonDocs);
}
}
finally
{
if (jsonDocs != null && jsonDocs.Length > 0)
{
var last = jsonDocs.Last();
Debug.Assert(last.Etag != null);
Debug.Assert(last.LastModified != null);
var lastEtag = last.Etag.Value;
var lastModified = last.LastModified.Value;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed documents
transactionalStorage.Batch(actions =>
{
foreach (var indexToWorkOn in indexesToWorkOn)
{
if (new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()).CompareTo(lastIndexedEtag) > 0)
{
continue;
}
actions.Indexing.UpdateLastIndexed(indexToWorkOn.IndexName, lastEtag, lastModified);
}
});
}
}
}
private IEnumerable <RavenJObject> GetQueryResults(IndexQuery query,
AbstractViewGenerator viewGenerator,
DocumentRetriever docRetriever,
IEnumerable <JsonDocument> results,
List <string> transformerErrors,
CancellationToken token)
{
if (query.PageSize <= 0) // maybe they just want the stats?
{
return(Enumerable.Empty <RavenJObject>());
}
IndexingFunc transformFunc = null;
// Check an explicitly declared one first
if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
{
var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer);
if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null)
{
transformFunc = transformGenerator.TransformResultsDefinition;
}
else
{
throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found");
}
}
else if (query.SkipTransformResults == false && viewGenerator.TransformResultsDefinition != null)
{
transformFunc = source => viewGenerator.TransformResultsDefinition(docRetriever, source);
}
if (transformFunc == null)
{
return(results.Select(x => x.ToJson()));
}
var dynamicJsonObjects = results.Select(x => new DynamicLuceneOrParentDocumntObject(docRetriever, x.ToJson()));
var robustEnumerator = new RobustEnumerator(token, 100)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
return(robustEnumerator.RobustEnumeration(
dynamicJsonObjects.Cast <object>().GetEnumerator(),
transformFunc)
.Select(JsonExtensions.ToJObject));
}
public JsonDocument GetWithTransformer(string key, string transformer, TransactionInformation transactionInformation, Dictionary <string, RavenJToken> transformerParameters, out HashSet <string> itemsToInclude)
{
JsonDocument result = null;
DocumentRetriever docRetriever = null;
TransactionalStorage.Batch(
actions =>
{
docRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState, transformerParameters);
using (new CurrentTransformationScope(Database, docRetriever))
{
var document = Get(key, transactionInformation);
if (document == null)
{
return;
}
if (document.Metadata.ContainsKey("Raven-Read-Veto"))
{
result = document;
return;
}
var storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
{
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
var transformed = storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) })
.Select(x => JsonExtensions.ToJObject(x))
.ToArray();
if (transformed.Length == 0)
{
return;
}
result = new JsonDocument
{
Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(docRetriever.Etag),
NonAuthoritativeInformation = document.NonAuthoritativeInformation,
LastModified = document.LastModified,
DataAsJson = new RavenJObject {
{ "$values", new RavenJArray(transformed) }
},
};
}
});
itemsToInclude = docRetriever.ItemsToInclude;
return(result);
}
protected override bool TryGetOrCreateResourceStore(string tenantId, out IResourceStore database)
{
if (ResourcesStoresCache.TryGetValue(tenantId, out database))
{
return(true);
}
JsonDocument jsonDocument;
using (DocumentRetriever.DisableReadTriggers())
jsonDocument = DefaultDatabase.Get("Raven/Databases/" + tenantId, null);
if (jsonDocument == null)
{
return(false);
}
var document = jsonDocument.DataAsJson.JsonDeserialization <DatabaseDocument>();
database = ResourcesStoresCache.GetOrAdd(tenantId, s =>
{
var config = new InMemoryRavenConfiguration
{
Settings = DefaultConfiguration.Settings,
};
foreach (var setting in document.Settings)
{
config.Settings[setting.Key] = setting.Value;
}
var dataDir = config.Settings["Raven/DataDir"];
if (dataDir == null)
{
throw new InvalidOperationException("Could not find Raven/DataDir");
}
if (dataDir.StartsWith("~/") || dataDir.StartsWith(@"~\"))
{
var baseDataPath = Path.GetDirectoryName(DefaultDatabase.Configuration.DataDirectory);
if (baseDataPath == null)
{
throw new InvalidOperationException("Could not find root data path");
}
config.Settings["Raven/DataDir"] = Path.Combine(baseDataPath, dataDir.Substring(2));
}
config.Settings["Raven/VirtualDir"] = config.Settings["Raven/VirtualDir"] + "/" + tenantId;
config.Initialize();
var documentDatabase = new DocumentDatabase(config);
documentDatabase.SpinBackgroundWorkers();
return(documentDatabase);
});
return(true);
}
public JsonDocumentMetadata GetDocumentMetadata(string key, TransactionInformation transactionInformation)
{
JsonDocumentMetadata document = null;
TransactionalStorage.Batch(actions =>
{
document = actions.Documents.DocumentMetadataByKey(key, transactionInformation);
});
DocumentRetriever.EnsureIdInMetadata(document);
return(new DocumentRetriever(null, ReadTriggers)
.ProcessReadVetoes(document, transactionInformation, ReadOperation.Load));
}
private ConversionScriptResult ApplyConversionScript(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs)
{
var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name));
var result = new ConversionScriptResult();
foreach (var jsonDocument in docs)
{
if (string.IsNullOrEmpty(cfg.RavenEntityName) == false)
{
var entityName = jsonDocument.Metadata.Value <string>(Constants.RavenEntityName);
if (string.Equals(cfg.RavenEntityName, entityName, StringComparison.InvariantCultureIgnoreCase) == false)
{
continue;
}
}
var patcher = new SqlReplicationScriptedJsonPatcher(Database, result, cfg, jsonDocument.Key);
try
{
DocumentRetriever.EnsureIdInMetadata(jsonDocument);
jsonDocument.Metadata[Constants.DocumentIdFieldName] = jsonDocument.Key;
var document = jsonDocument.ToJson();
patcher.Apply(document, new ScriptedPatchRequest
{
Script = cfg.Script
});
replicationStats.ScriptSuccess();
}
catch (ParseException e)
{
replicationStats.MarkScriptAsInvalid(Database, cfg.Script);
log.WarnException("Could parse SQL Replication script for " + cfg.Name, e);
return(result);
}
catch (Exception e)
{
replicationStats.RecordScriptError(Database);
log.WarnException("Could not process SQL Replication script for " + cfg.Name + ", skipping this document", e);
}
}
return(result);
}
public Etag GetDocumentsWithIdStartingWith(string idPrefix, int pageSize, Etag etag, CancellationToken token, Action <JsonDocument> addDocument)
{
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = actions.Documents.GetDocumentsAfterWithIdStartingWith(etag, idPrefix, pageSize, token, timeout: TimeSpan.FromSeconds(2));
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, Database.InFlightTransactionalState);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
etag = doc.Etag;
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
addDocument(document);
returnedDocs = true;
Database.WorkContext.UpdateFoundWork();
}
if (returnedDocs || docCount == 0)
{
break;
}
}
});
return(etag);
}
public void AfterStorageCommitBeforeWorkNotifications(JsonDocument[] docs)
{
if (context.Configuration.DisableDocumentPreFetchingForIndexing || docs.Length == 0)
{
return;
}
if (inMemoryDocs.Count > // don't use too much, this is an optimization and we need to be careful about using too much mem
context.Configuration.MaxNumberOfItemsToIndexInSingleBatch)
{
return;
}
foreach (var jsonDocument in docs)
{
DocumentRetriever.EnsureIdInMetadata(jsonDocument);
inMemoryDocs.Enqueue(jsonDocument);
}
}
public static int UpdateDocumentsAfter(this DocumentDatabase db, Guid eTag, Func <JsonDocument, bool> updateDoc,
CancellationToken cancellationToken, TransactionInformation transactionInformation)
{
var initialTime = DateTime.UtcNow;
log.Trace("UpdateDocumentsAfter started whith Etag {0}", eTag);
int updatedDocCount = 0;
db.TransactionalStorage.Batch(action =>
{
var documentRetriever = new DocumentRetriever(action, db.ReadTriggers);
foreach (var doc in action.Documents.GetDocumentsAfter(eTag, int.MaxValue))
{
DocumentRetriever.EnsureIdInMetadata(doc);
if (cancellationToken.IsCancellationRequested)
{
log.Trace("UpdateDocumentsAfter has been cancelled");
}
cancellationToken.ThrowIfCancellationRequested();
var document = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document != null && updateDoc(document))
{
try
{
db.Put(document.Key, document.Etag, document.DataAsJson, document.Metadata, transactionInformation);
updatedDocCount++;
log.Debug("{0} document has been updated in UpdateDocumentsAfter. {1} documents updated so far", document.Key, updatedDocCount);
}
catch (ConcurrencyException)
{
log.Trace("ConcurrencyException caught in UpdateDocumentsAfter");
if (db.UpdateDocumentWithRetries(document.Key, updateDoc, transactionInformation))
{
updatedDocCount++;
log.Debug("{0} document has been updated in UpdateDocumentsAfter. {1} documents updated so far", document.Key, updatedDocCount);
}
}
}
}
});
log.Trace("UpdateDocumentsAfter ETag {0} completed successfully. {1} documents updated in {2}", eTag, updatedDocCount, DateTime.UtcNow.Subtract(initialTime));
return(updatedDocCount);
}
public void AfterStorageCommitBeforeWorkNotifications(JsonDocument[] docs)
{
if (context.Configuration.DisableDocumentPreFetchingForIndexing || docs.Length == 0)
{
return;
}
if (prefetchingQueue.Count >= // don't use too much, this is an optimization and we need to be careful about using too much mem
context.Configuration.MaxNumberOfItemsToPreFetchForIndexing ||
prefetchingQueue.Aggregate(0, (x, c) => x + SelectSerializedSizeOnDiskIfNotNull(c)) > context.Configuration.AvailableMemoryForRaisingIndexBatchSizeLimit)
{
return;
}
foreach (var jsonDocument in docs)
{
DocumentRetriever.EnsureIdInMetadata(jsonDocument);
prefetchingQueue.Add(jsonDocument);
}
}
public void GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Action <RavenJObject> addDocument)
{
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize, WorkContext.CancellationToken);
var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
if (etag != null)
{
etag = doc.Etag;
}
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever
.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
addDocument(document.ToJson());
returnedDocs = true;
}
if (returnedDocs || docCount == 0)
{
break;
}
start += docCount;
}
});
}
private List <JsonDocument> GetJsonDocsFromDisk(Etag etag, Etag untilEtag)
{
List <JsonDocument> jsonDocs = null;
context.TransactionalStorage.Batch(actions =>
{
//limit how much data we load from disk --> better adhere to memory limits
var totalSizeAllowedToLoadInBytes =
(context.Configuration.MemoryLimitForIndexingInMb * 1024 * 1024) -
(prefetchingQueue.Aggregate(0, (acc, doc) => acc + doc.SerializedSizeOnDisk) + autoTuner.CurrentlyUsedBatchSizesInBytes.Values.Sum());
// at any rate, we will load a min of 512Kb docs
var maxSize = Math.Max(
Math.Min(totalSizeAllowedToLoadInBytes, autoTuner.MaximumSizeAllowedToFetchFromStorageInBytes),
1024 * 512);
jsonDocs = actions.Documents
.GetDocumentsAfter(
etag,
autoTuner.NumberOfItemsToIndexInSingleBatch,
context.CancellationToken,
maxSize,
untilEtag,
autoTuner.FetchingDocumentsFromDiskTimeout
)
.Where(x => x != null)
.Select(doc =>
{
DocumentRetriever.EnsureIdInMetadata(doc);
return(doc);
})
.ToList();
});
if (untilEtag == null)
{
MaybeAddFutureBatch(jsonDocs);
}
return(jsonDocs);
}
public void AfterStorageCommitBeforeWorkNotifications(JsonDocument[] docs)
{
if (context.Configuration.DisableDocumentPreFetching || docs.Length == 0 || DisableCollectingDocumentsAfterCommit)
{
return;
}
if (prefetchingQueue.Count >= // don't use too much, this is an optimization and we need to be careful about using too much mem
context.Configuration.MaxNumberOfItemsToPreFetch ||
prefetchingQueue.Aggregate(0, (x, c) => x + SelectSerializedSizeOnDiskIfNotNull(c)) > context.Configuration.AvailableMemoryForRaisingBatchSizeLimit)
{
return;
}
Etag lowestEtag = null;
foreach (var jsonDocument in docs)
{
DocumentRetriever.EnsureIdInMetadata(jsonDocument);
prefetchingQueue.Add(jsonDocument);
if (ShouldHandleUnusedDocumentsAddedAfterCommit && (lowestEtag == null || jsonDocument.Etag.CompareTo(lowestEtag) < 0))
{
lowestEtag = jsonDocument.Etag;
}
}
if (ShouldHandleUnusedDocumentsAddedAfterCommit && lowestEtag != null)
{
if (lowestInMemoryDocumentAddedAfterCommit == null || lowestEtag.CompareTo(lowestInMemoryDocumentAddedAfterCommit.Etag) < 0)
{
lowestInMemoryDocumentAddedAfterCommit = new DocAddedAfterCommit
{
Etag = lowestEtag,
AddedAt = SystemTime.UtcNow
};
}
}
}
public void AfterCommit(JsonDocument[] docs)
{
if (docs.Length == 0)
{
return;
}
foreach (var doc in docs)
{
DocumentRetriever.EnsureIdInMetadata(doc);
}
futureIndexBatches.Add(new FutureIndexBatch
{
StartingEtag = DecrementEtag(GetLowestEtag(docs)),
Task = new CompletedTask <JsonResults>(new JsonResults
{
Results = docs,
LoadedFromDisk = false
}),
Age = currentIndexingAge
});
}
private IEnumerable<RavenJObject> GetQueryResults(IndexQuery query,
AbstractViewGenerator viewGenerator,
DocumentRetriever docRetriever,
IEnumerable<JsonDocument> results,
List<string> transformerErrors,
CancellationToken token)
{
if (query.PageSize <= 0) // maybe they just want the stats?
{
return Enumerable.Empty<RavenJObject>();
}
IndexingFunc transformFunc = null;
// Check an explicitly declared one first
if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
{
var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer);
if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null)
transformFunc = transformGenerator.TransformResultsDefinition;
else
throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found");
}
else if (query.SkipTransformResults == false && viewGenerator.TransformResultsDefinition != null)
{
transformFunc = source => viewGenerator.TransformResultsDefinition(docRetriever, source);
}
if (transformFunc == null)
return results.Select(x => x.ToJson());
var dynamicJsonObjects = results.Select(x => new DynamicLuceneOrParentDocumntObject(docRetriever, x.ToJson()));
var robustEnumerator = new RobustEnumerator(token, 100)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
return robustEnumerator.RobustEnumeration(
dynamicJsonObjects.Cast<object>().GetEnumerator(),
transformFunc)
.Select(JsonExtensions.ToJObject);
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize, CancellationToken token, Action<RavenJObject> addDoc)
{
if (idPrefix == null)
throw new ArgumentNullException("idPrefix");
idPrefix = idPrefix.Trim();
TransactionalStorage.Batch(
actions =>
{
var docsToSkip = start;
var addedDocs = 0;
var matchedDocs = 0;
int docCount;
start = 0;
do
{
docCount = 0;
var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers, inFlightTransactionalState);
foreach (var doc in docs)
{
token.ThrowIfCancellationRequested();
docCount++;
var keyTest = doc.Key.Substring(idPrefix.Length);
if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
continue;
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = inFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc;
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
continue;
matchedDocs++;
if (matchedDocs <= docsToSkip)
continue;
token.ThrowIfCancellationRequested();
addDoc(document.ToJson());
addedDocs++;
if (addedDocs >= pageSize)
break;
}
start += pageSize;
}
while (docCount > 0 && addedDocs < pageSize && start >= 0 && start < int.MaxValue);
});
}
public void GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Action<RavenJObject> addDocument)
{
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers, inFlightTransactionalState);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
if (etag != null)
etag = doc.Etag;
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = inFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever
.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
continue;
addDocument(document.ToJson());
returnedDocs = true;
}
if (returnedDocs || docCount == 0)
break;
start += docCount;
}
});
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize, Action<RavenJObject> addDoc)
{
if (idPrefix == null)
throw new ArgumentNullException("idPrefix");
idPrefix = idPrefix.Trim();
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
int docCount = 0;
var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers, inFlightTransactionalState);
foreach (var doc in documents)
{
docCount++;
string keyTest = doc.Key.Substring(idPrefix.Length);
if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
continue;
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = inFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key);
JsonDocument document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc;
document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
addDoc(document.ToJson());
returnedDocs = true;
}
if (returnedDocs || docCount == 0)
break;
start += docCount;
}
});
}
public RavenJArray GetDocuments(int start, int pageSize, Guid? etag)
{
var list = new RavenJArray();
TransactionalStorage.Batch(actions =>
{
while (true)
{
var documents = etag == null ?
actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) :
actions.Documents.GetDocumentsAfter(etag.Value, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
if(etag != null)
etag = doc.Etag;
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
list.Add(document.ToJson());
}
if (list.Length != 0 || docCount == 0)
break;
start += docCount;
}
});
return list;
}
private IEnumerable<RavenJObject> GetQueryResults(IndexQuery query,
AbstractViewGenerator viewGenerator,
DocumentRetriever docRetriever,
IEnumerable<JsonDocument> results,
List<string> transformerErrors)
{
if (query.PageSize <= 0) // maybe they just want the stats?
{
return Enumerable.Empty<RavenJObject>();
}
IndexingFunc transformFunc = null;
// Check an explicitly declared one first
if (query.ResultsTransformer != null)
{
var transformGenerator = IndexDefinitionStorage.GetTransfomer(query.ResultsTransformer);
if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null)
transformFunc = transformGenerator.TransformResultsDefinition;
}
else if (query.SkipTransformResults == false && viewGenerator.TransformResultsDefinition != null)
{
transformFunc = source => viewGenerator.TransformResultsDefinition(docRetriever, source);
}
if (transformFunc == null)
return results.Select(x => x.ToJson());
var dynamicJsonObjects = results.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
var robustEnumerator = new RobustEnumerator(workContext, dynamicJsonObjects.Length)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
return robustEnumerator.RobustEnumeration(
dynamicJsonObjects.Cast<object>().GetEnumerator(),
transformFunc)
.Select(JsonExtensions.ToJObject);
}
public QueryResult Query(string index, IndexQuery query)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var list = new List<JObject>();
var stale = false;
Tuple<DateTime, Guid> indexTimestamp = null;
TransactionalStorage.Batch(
actions =>
{
string entityName = null;
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator != null)
entityName = viewGenerator.ForEntityName;
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, entityName);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers);
var indexDefinition = GetIndexDefinition(index);
var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, query.FieldsToFetch,query.AggregationOperation))
select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, query.FieldsToFetch, query.AggregationOperation)
into doc
where doc != null
select doc;
var transformerErrors = new List<string>();
IEnumerable<JObject> results;
if (viewGenerator != null &&
query.SkipTransformResults == false &&
viewGenerator.TransformResultsDefinition != null)
{
var robustEnumerator = new RobustEnumerator
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
results =
robustEnumerator.RobustEnumeration(
dynamicJsonObjects,
source => viewGenerator.TransformResultsDefinition(docRetriever, source))
.Select(JsonExtensions.ToJObject);
}
else
{
results = collection.Select(x => x.ToJson());
}
list.AddRange(results);
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
});
return new QueryResult
{
IndexName = index,
Results = list,
IsStale = stale,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2
};
}
public QueryResultWithIncludes Query(string index, IndexQuery query)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var list = new List<RavenJObject>();
var highlightings = new Dictionary<string, Dictionary<string, string[]>>();
var stale = false;
Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty);
Guid resultEtag = Guid.Empty;
var nonAuthoritativeInformation = false;
var idsToLoad = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);
TransactionalStorage.Batch(
actions =>
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
throw new IndexDoesNotExistsException("Could not find index named: " + index);
resultEtag = GetIndexEtag(index, null);
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers, idsToLoad);
var indexDefinition = GetIndexDefinition(index);
var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
viewGenerator.ReduceDefinition == null
? Constants.DocumentIdFieldName
: Constants.ReduceKeyFieldName);
Func<IndexQueryResult, bool> shouldIncludeInResults =
result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch);
var collection =
from queryResult in
IndexStorage.Query(index, query, shouldIncludeInResults, fieldsToFetch, IndexQueryTriggers)
select new
{
Document = docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch),
Fragments = queryResult.Highligtings
}
into docWithFragments
where docWithFragments.Document != null
let _ = nonAuthoritativeInformation |= (docWithFragments.Document.NonAuthoritativeInformation ?? false)
select docWithFragments;
var transformerErrors = new List<string>();
IEnumerable<RavenJObject> results;
if (query.SkipTransformResults == false &&
query.PageSize > 0 && // maybe they just want the stats?
viewGenerator.TransformResultsDefinition != null)
{
var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.Document.ToJson())).ToArray();
var robustEnumerator = new RobustEnumerator(workContext, dynamicJsonObjects.Length)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
results =
robustEnumerator.RobustEnumeration(
dynamicJsonObjects.Cast<object>().GetEnumerator(),
source => viewGenerator.TransformResultsDefinition(docRetriever, source))
.Select(JsonExtensions.ToJObject);
}
else
{
var resultList = new List<RavenJObject>();
foreach (var docWithFragments in collection)
{
resultList.Add(docWithFragments.Document.ToJson());
if (docWithFragments.Fragments != null && docWithFragments.Document.Key != null)
highlightings.Add(docWithFragments.Document.Key, docWithFragments.Fragments);
}
results = resultList;
}
list.AddRange(results);
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
});
return new QueryResultWithIncludes
{
IndexName = index,
Results = list,
IsStale = stale,
NonAuthoritativeInformation = nonAuthoritativeInformation,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2,
ResultEtag = resultEtag,
IdsToInclude = idsToLoad,
LastQueryTime = SystemTime.UtcNow,
Highlightings = highlightings
};
}
public QueryResultWithIncludes Query(string index, IndexQuery query, Action<QueryHeaderInformation> headerInfo, Action<RavenJObject> onResult)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var highlightings = new Dictionary<string, Dictionary<string, string[]>>();
Func<IndexQueryResult, object> tryRecordHighlighting = queryResult =>
{
if (queryResult.Highligtings != null && queryResult.Key != null)
highlightings.Add(queryResult.Key, queryResult.Highligtings);
return null;
};
var stale = false;
Tuple<DateTime, Etag> indexTimestamp = Tuple.Create(DateTime.MinValue, Etag.Empty);
Etag resultEtag = Etag.Empty;
var nonAuthoritativeInformation = false;
if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
{
query.FieldsToFetch = new[] { Constants.AllFields };
}
var idsToLoad = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
TransactionalStorage.Batch(
actions =>
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
throw new IndexDoesNotExistsException("Could not find index named: " + index);
resultEtag = GetIndexEtag(index, null);
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers, query.QueryInputs, idsToLoad);
var indexDefinition = GetIndexDefinition(index);
var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
viewGenerator.ReduceDefinition == null
? Constants.DocumentIdFieldName
: Constants.ReduceKeyFieldName);
Func<IndexQueryResult, bool> shouldIncludeInResults =
result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch);
var indexQueryResults = IndexStorage.Query(index, query, shouldIncludeInResults, fieldsToFetch, IndexQueryTriggers);
indexQueryResults = new ActiveEnumerable<IndexQueryResult>(indexQueryResults);
var transformerErrors = new List<string>();
var results = GetQueryResults(query, viewGenerator, docRetriever,
from queryResult in indexQueryResults
let doc = docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch)
where doc != null
let _ = nonAuthoritativeInformation |= (doc.NonAuthoritativeInformation ?? false)
let __ = tryRecordHighlighting(queryResult)
select doc, transformerErrors);
if (headerInfo != null)
{
headerInfo(new QueryHeaderInformation
{
Index = index,
IsStable = stale,
ResultEtag = resultEtag,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2,
TotalResults = query.TotalSize.Value
});
}
using (new CurrentTransformationScope(docRetriever))
{
foreach (var result in results)
{
onResult(result);
}
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
}
});
return new QueryResultWithIncludes
{
IndexName = index,
IsStale = stale,
NonAuthoritativeInformation = nonAuthoritativeInformation,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2,
ResultEtag = resultEtag,
IdsToInclude = idsToLoad,
LastQueryTime = SystemTime.UtcNow,
Highlightings = highlightings
};
}
public RavenJArray GetDocuments(int start, int pageSize, Guid? etag)
{
var list = new RavenJArray();
TransactionalStorage.Batch(actions =>
{
IEnumerable<JsonDocument> documents;
if (etag == null)
documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start);
else
documents = actions.Documents.GetDocumentsAfter(etag.Value);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents.Take(pageSize))
{
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
list.Add(document.ToJson());
}
});
return list;
}
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, int start, int pageSize)
{
var list = new RavenJArray();
TransactionalStorage.Batch(actions =>
{
var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start)
.Take(pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents)
{
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
list.Add(document.ToJson());
}
});
return list;
}
public QueryResult Query(string index, IndexQuery query)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var list = new List<RavenJObject>();
var stale = false;
Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty);
TransactionalStorage.Batch(
actions =>
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
throw new InvalidOperationException("Could not find index named: " + index);
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers);
var indexDefinition = GetIndexDefinition(index);
var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
viewGenerator.ReduceDefinition == null
? Constants.DocumentIdFieldName
: Constants.ReduceKeyFieldName);
var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch), fieldsToFetch, IndexQueryTriggers)
select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch)
into doc
where doc != null
select doc;
var transformerErrors = new List<string>();
IEnumerable<RavenJObject> results;
if (query.SkipTransformResults == false &&
query.PageSize > 0 && // maybe they just want the stats?
viewGenerator.TransformResultsDefinition != null)
{
var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
var robustEnumerator = new RobustEnumerator(dynamicJsonObjects.Length)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
results =
robustEnumerator.RobustEnumeration(
dynamicJsonObjects,
source => viewGenerator.TransformResultsDefinition(docRetriever, source))
.Select(JsonExtensions.ToJObject);
}
else
{
results = collection.Select(x => x.ToJson());
}
if (query.PageSize > 0) // maybe they just want the query stats?
list.AddRange(results);
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
});
return new QueryResult
{
IndexName = index,
Results = list,
IsStale = stale,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2
};
}
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, string matches, int start, int pageSize)
{
if (idPrefix == null)
throw new ArgumentNullException("idPrefix");
idPrefix = idPrefix.Trim();
var list = new RavenJArray();
TransactionalStorage.Batch(actions =>
{
var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents)
{
if (WildcardMatcher.Matches(matches, doc.Key.Substring(idPrefix.Length)) == false)
continue;
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
list.Add(document.ToJson());
}
});
return list;
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, int start, int pageSize, Action<RavenJObject> addDoc)
{
if (idPrefix == null)
throw new ArgumentNullException("idPrefix");
idPrefix = idPrefix.Trim();
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
int docCount = 0;
var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents)
{
docCount++;
if (WildcardMatcher.Matches(matches, doc.Key.Substring(idPrefix.Length)) == false)
continue;
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
addDoc(document.ToJson());
returnedDocs = true;
}
if (returnedDocs || docCount == 0)
break;
start += docCount;
}
});
}
private JArray GetJsonDocuments(Guid etag)
{
JArray jsonDocuments = null;
try
{
var instanceId = docDb.TransactionalStorage.Id.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
var docRetr = new DocumentRetriever(actions, Enumerable.Empty<AbstractReadTrigger>());
jsonDocuments = new JArray(actions.Documents.GetDocumentsAfter(etag)
.Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs
.Where(x => x.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) == instanceId) // only replicate documents created on this instance
.Where(x=> x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict
.Select(x=>
{
docRetr.EnsureIdInMetadata(x);
return x;
})
.Take(100)
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.Warn("Could not get documents to replicate after: " + etag, e);
}
return jsonDocuments;
}
public void GetDocuments(int start, int pageSize, Etag etag, Action<RavenJObject> addDocument)
{
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
if (etag != null)
etag = doc.Etag;
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
addDocument(document.ToJson());
returnedDocs = true;
}
if (returnedDocs || docCount == 0)
break;
start += docCount;
}
});
}
public JArray GetDocuments(int start, int pageSize, Guid? etag)
{
var list = new JArray();
TransactionalStorage.Batch(actions =>
{
IEnumerable<JsonDocument> documents;
if (etag == null)
documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start);
else
documents = actions.Documents.GetDocumentsAfter(etag.Value);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents.Take(pageSize))
{
var document = documentRetriever.ExecuteReadTriggers(doc, null,
// here we want to have the Load semantic, not Query, because we need this to be
// as close as possible to the full database contents
ReadOperation.Load);
if (document == null)
continue;
if (document.Metadata.Property("@id") == null)
document.Metadata.Add("@id", new JValue(doc.Key));
list.Add(document.ToJson());
}
});
return list;
}
public JsonDocument GetWithTransformer(string key, string transformer, TransactionInformation transactionInformation, Dictionary<string, RavenJToken> queryInputs)
{
JsonDocument result = null;
TransactionalStorage.Batch(
actions =>
{
var docRetriever = new DocumentRetriever(actions, ReadTriggers, queryInputs);
using (new CurrentTransformationScope(docRetriever))
{
var document = Get(key, transactionInformation);
if (document == null)
return;
var storedTransformer = IndexDefinitionStorage.GetTransfomer(transformer);
if (storedTransformer == null)
throw new InvalidOperationException("No transfomer with the name: " + transformer);
var transformed = storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) })
.Select(x => JsonExtensions.ToJObject(x))
.ToArray();
if (transformed.Length == 0)
return;
result = new JsonDocument
{
Etag = document.Etag,
NonAuthoritativeInformation = document.NonAuthoritativeInformation,
LastModified = document.LastModified,
};
result.DataAsJson = new RavenJObject { { "$values", new RavenJArray(transformed) } };
}
});
return result;
}
public bool IndexDocuments(IStorageActionsAccessor actions, string index, Guid etagToIndexFrom)
{
log.DebugFormat("Indexing documents for {0}, etag to index from: {1}", index, etagToIndexFrom);
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
return false; // index was deleted, probably
var jsonDocs = actions.Documents.GetDocumentsAfter(etagToIndexFrom)
.Where(x => x != null)
.Take(10000) // ensure that we won't go overboard with reading and blow up with OOM
.ToArray();
if(jsonDocs.Length == 0)
return false;
var dateTime = jsonDocs.Select(x=>x.LastModified).Min();
var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
try
{
log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index);
context.IndexStorage.Index(index, viewGenerator,
jsonDocs
.Select(doc => documentRetriever.ProcessReadVetoes(doc, null, ReadOperation.Index))
.Where(doc => doc != null)
.Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime);
return true;
}
catch (Exception e)
{
log.WarnFormat(e, "Failed to index documents for index: {0}", index);
return false;
}
finally
{
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed documents
var last = jsonDocs.Last();
actions.Indexing.UpdateLastIndexed(index, last.Etag, last.LastModified);
}
}
public JsonDocument GetWithTransformer(string key, string transformer, TransactionInformation transactionInformation, Dictionary<string, RavenJToken> queryInputs)
{
JsonDocument result = null;
TransactionalStorage.Batch(
actions =>
{
var docRetriever = new DocumentRetriever(actions, ReadTriggers, inFlightTransactionalState, queryInputs);
using (new CurrentTransformationScope(this, docRetriever))
{
var document = Get(key, transactionInformation);
if (document == null)
return;
if (document.Metadata.ContainsKey("Raven-Read-Veto") || document.Metadata.ContainsKey(Constants.RavenReplicationConflict))
{
result = document;
return;
}
var storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
throw new InvalidOperationException("No transformer with the name: " + transformer);
var transformed = storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) })
.Select(x => JsonExtensions.ToJObject(x))
.ToArray();
if (transformed.Length == 0)
return;
result = new JsonDocument
{
Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(docRetriever.Etag),
NonAuthoritativeInformation = document.NonAuthoritativeInformation,
LastModified = document.LastModified,
DataAsJson = new RavenJObject { { "$values", new RavenJArray(transformed) } },
};
}
});
return result;
}
