public JArray GetDocuments(int start, int pageSize, Guid?etag)
{
var list = new JArray();
TransactionalStorage.Batch(actions =>
{
IEnumerable <JsonDocument> documents;
if (etag == null)
{
documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start);
}
else
{
documents = actions.Documents.GetDocumentsAfter(etag.Value);
}
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents.Take(pageSize))
{
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
{
continue;
}
list.Add(document.ToJson());
}
});
return(list);
}
private List <JsonDocument> GetJsonDocsFromDisk(Guid etag)
{
List <JsonDocument> jsonDocs = null;
var untilEtag = GetNextEtagInMemory();
context.TransactionalStorage.Batch(actions =>
{
jsonDocs = actions.Documents
.GetDocumentsAfter(
etag,
autoTuner.NumberOfItemsToIndexInSingleBatch,
autoTuner.MaximumSizeAllowedToFetchFromStorage,
untilEtag: untilEtag)
.Where(x => x != null)
.Select(doc =>
{
DocumentRetriever.EnsureIdInMetadata(doc);
return(doc);
})
.ToList();
});
if (untilEtag == null)
{
MaybeAddFutureBatch(jsonDocs);
}
return(jsonDocs);
}
private List <JsonDocument> GetJsonDocsFromDisk(Etag etag, Etag untilEtag)
{
List <JsonDocument> jsonDocs = null;
context.TransactionalStorage.Batch(actions =>
{
jsonDocs = actions.Documents
.GetDocumentsAfter(
etag,
autoTuner.NumberOfItemsToIndexInSingleBatch,
context.CancellationToken,
maxSize: autoTuner.MaximumSizeAllowedToFetchFromStorage,
untilEtag: untilEtag,
timeout: autoTuner.FetchingDocumentsFromDiskTimeout)
.Where(x => x != null)
.Select(doc =>
{
DocumentRetriever.EnsureIdInMetadata(doc);
return(doc);
})
.ToList();
});
if (untilEtag == null)
{
MaybeAddFutureBatch(jsonDocs);
}
return(jsonDocs);
}
public void AfterCommit(JsonDocument[] docs)
{
if (context.Configuration.DisableDocumentPreFetchingForIndexing)
{
return;
}
if (futureIndexBatches.Count > 512 || // this is optimization, and we need to make sure we don't overuse memory
docs.Length == 0)
{
return;
}
foreach (var doc in docs)
{
DocumentRetriever.EnsureIdInMetadata(doc);
}
futureIndexBatches.Add(new FutureIndexBatch
{
StartingEtag = GetLowestEtag(docs),
Task = new CompletedTask <JsonResults>(new JsonResults
{
Results = docs,
LoadedFromDisk = false
}),
Age = Interlocked.Increment(ref currentIndexingAge)
});
}
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource)
{
RavenJArray jsonDocuments = null;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag)
.Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs
.Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source
.Where(x => x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Take(100)
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.Warn("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return(jsonDocuments);
}
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray jsonDocuments = null;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag, 100)
.Where(destination.FilterDocuments)
.Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return(jsonDocuments);
}
public JsonDocument Get(string key, TransactionInformation transactionInformation)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
key = key.Trim();
JsonDocument document = null;
if (transactionInformation == null ||
Database.InFlightTransactionalState.TryGet(key, transactionInformation, out document) == false)
{
// first we check the dtc state, then the storage, to avoid race conditions
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(transactionInformation, key);
TransactionalStorage.Batch(actions => { document = actions.Documents.DocumentByKey(key); });
if (nonAuthoritativeInformationBehavior != null)
{
document = nonAuthoritativeInformationBehavior(document);
}
}
DocumentRetriever.EnsureIdInMetadata(document);
return(new DocumentRetriever(null, Database.ReadTriggers, Database.InFlightTransactionalState)
.ExecuteReadTriggers(document, transactionInformation, ReadOperation.Load));
}
private Dictionary <string, List <ItemToReplicate> > ApplyConversionScript(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs)
{
var dictionary = new Dictionary <string, List <ItemToReplicate> >();
foreach (var jsonDocument in docs)
{
if (string.IsNullOrEmpty(cfg.RavenEntityName) == false)
{
var entityName = jsonDocument.Metadata.Value <string>(Constants.RavenEntityName);
if (string.Equals(cfg.RavenEntityName, entityName, StringComparison.InvariantCultureIgnoreCase) == false)
{
continue;
}
}
var patcher = new SqlReplicationScriptedJsonPatcher(Database, dictionary, jsonDocument.Key);
try
{
DocumentRetriever.EnsureIdInMetadata(jsonDocument);
jsonDocument.Metadata[Constants.DocumentIdFieldName] = jsonDocument.Key;
var document = jsonDocument.ToJson();
patcher.Apply(document, new ScriptedPatchRequest
{
Script = cfg.Script
});
}
catch (Exception e)
{
log.WarnException("Could not process SQL Replication script for " + cfg.Name + ", skipping this document", e);
}
}
return(dictionary);
}
private JArray GetJsonDocuments(Guid etag)
{
JArray jsonDocuments = null;
try
{
var instanceId = docDb.TransactionalStorage.Id.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonDocuments = new JArray(actions.Documents.GetDocumentsAfter(etag)
.Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs
.Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) == instanceId) // only replicate documents created on this instance
.Where(x => x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Take(100)
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.Warn("Could not get documents to replicate after: " + etag, e);
}
return(jsonDocuments);
}
public void AfterCommit(JsonDocument[] docs)
{
if (futureIndexBatches.Count > 512 || // this is optimization, and we need to make sure we don't overuse memory
docs.Length == 0)
{
return;
}
foreach (var doc in docs)
{
DocumentRetriever.EnsureIdInMetadata(doc);
}
futureIndexBatches.Add(new FutureIndexBatch
{
StartingEtag = DecrementEtag(GetLowestEtag(docs)),
Task = new CompletedTask <JsonResults>(new JsonResults
{
Results = docs,
LoadedFromDisk = false
}),
Age = currentIndexingAge
});
}
private void StartGetDocumentOnBackground()
{
while (run)
{
DocumentDatabase.TransactionalStorage.Batch(accessor =>
{
var documents = accessor.Documents.GetDocumentsAfter(lastEtagSeen, 128)
.Where(x => x != null)
.Select(doc =>
{
DocumentRetriever.EnsureIdInMetadata(doc);
return(doc);
})
.ToArray();
if (documents.Length == 0)
{
return;
}
lastEtagSeen = documents.Last().Etag;
log.Debug("Docs: {0}", string.Join(", ", documents.Select(x => x.Key)));
getDocumentsState.Enqueue(new GetDocumentState(lastEtagSeen, documents.Length));
});
}
}
public JsonDocumentMetadata GetDocumentMetadata(string key, TransactionInformation transactionInformation)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
key = key.Trim();
JsonDocumentMetadata document = null;
if (transactionInformation == null ||
Database.InFlightTransactionalState.TryGet(key, transactionInformation, out document) == false)
{
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocumentMetadata>(transactionInformation, key);
TransactionalStorage.Batch(actions =>
{
document = actions.Documents.DocumentMetadataByKey(key);
});
if (nonAuthoritativeInformationBehavior != null)
{
document = nonAuthoritativeInformationBehavior(document);
}
}
DocumentRetriever.EnsureIdInMetadata(document);
return(new DocumentRetriever(null, Database.ReadTriggers, Database.InFlightTransactionalState)
.ProcessReadVetoes(document, transactionInformation, ReadOperation.Load));
}
private ConversionScriptResult ApplyConversionScript(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs)
{
var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name));
var result = new ConversionScriptResult();
foreach (var jsonDocument in docs)
{
Database.WorkContext.CancellationToken.ThrowIfCancellationRequested();
if (string.IsNullOrEmpty(cfg.RavenEntityName) == false)
{
var entityName = jsonDocument.Metadata.Value <string>(Constants.RavenEntityName);
if (string.Equals(cfg.RavenEntityName, entityName, StringComparison.InvariantCultureIgnoreCase) == false)
{
continue;
}
}
var patcher = new SqlReplicationScriptedJsonPatcher(Database, result, cfg, jsonDocument.Key);
using (var scope = new SqlReplicationScriptedJsonPatcherOperationScope(Database))
{
try
{
DocumentRetriever.EnsureIdInMetadata(jsonDocument);
var document = jsonDocument.ToJson();
document[Constants.DocumentIdFieldName] = jsonDocument.Key;
patcher.Apply(scope, document, new ScriptedPatchRequest {
Script = cfg.Script
}, jsonDocument.SerializedSizeOnDisk);
if (log.IsDebugEnabled && patcher.Debug.Count > 0)
{
log.Debug("Debug output for doc: {0} for script {1}:\r\n.{2}", jsonDocument.Key, cfg.Name, string.Join("\r\n", patcher.Debug));
patcher.Debug.Clear();
}
replicationStats.ScriptSuccess();
}
catch (ParseException e)
{
replicationStats.MarkScriptAsInvalid(Database, cfg.Script);
log.WarnException("Could not parse SQL Replication script for " + cfg.Name, e);
return(result);
}
catch (Exception e)
{
replicationStats.RecordScriptError(Database);
log.WarnException("Could not process SQL Replication script for " + cfg.Name + ", skipping document: " + jsonDocument.Key, e);
}
}
}
return(result);
}
private void ExecuteIndexingInternal(IEnumerable <IndexToWorkOn> indexesToWorkOn, Action <JsonDocument[]> indexingOp)
{
var lastIndexedGuidForAllIndexes = indexesToWorkOn.Min(x => new ComparableByteArray(x.LastIndexedEtag.ToByteArray())).ToGuid();
JsonDocument[] jsonDocs = null;
try
{
transactionalStorage.Batch(actions =>
{
jsonDocs = actions.Documents.GetDocumentsAfter(lastIndexedGuidForAllIndexes)
.Where(x => x != null)
.Select(doc =>
{
DocumentRetriever.EnsureIdInMetadata(doc);
return(doc);
})
.Take(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) // ensure that we won't go overboard with reading and blow up with OOM
.ToArray();
});
if (jsonDocs.Length > 0)
{
indexingOp(jsonDocs);
}
}
finally
{
if (jsonDocs != null && jsonDocs.Length > 0)
{
var last = jsonDocs.Last();
Debug.Assert(last.Etag != null);
Debug.Assert(last.LastModified != null);
var lastEtag = last.Etag.Value;
var lastModified = last.LastModified.Value;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed documents
transactionalStorage.Batch(actions =>
{
foreach (var indexToWorkOn in indexesToWorkOn)
{
if (new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()).CompareTo(lastIndexedEtag) > 0)
{
continue;
}
actions.Indexing.UpdateLastIndexed(indexToWorkOn.IndexName, lastEtag, lastModified);
}
});
}
}
}
public JsonDocumentMetadata GetDocumentMetadata(string key, TransactionInformation transactionInformation)
{
JsonDocumentMetadata document = null;
TransactionalStorage.Batch(actions =>
{
document = actions.Documents.DocumentMetadataByKey(key, transactionInformation);
});
DocumentRetriever.EnsureIdInMetadata(document);
return(new DocumentRetriever(null, ReadTriggers)
.ProcessReadVetoes(document, transactionInformation, ReadOperation.Load));
}
private ConversionScriptResult ApplyConversionScript(SqlReplicationConfig cfg, IEnumerable <JsonDocument> docs)
{
var replicationStats = statistics.GetOrAdd(cfg.Name, name => new SqlReplicationStatistics(name));
var result = new ConversionScriptResult();
foreach (var jsonDocument in docs)
{
if (string.IsNullOrEmpty(cfg.RavenEntityName) == false)
{
var entityName = jsonDocument.Metadata.Value <string>(Constants.RavenEntityName);
if (string.Equals(cfg.RavenEntityName, entityName, StringComparison.InvariantCultureIgnoreCase) == false)
{
continue;
}
}
var patcher = new SqlReplicationScriptedJsonPatcher(Database, result, cfg, jsonDocument.Key);
try
{
DocumentRetriever.EnsureIdInMetadata(jsonDocument);
jsonDocument.Metadata[Constants.DocumentIdFieldName] = jsonDocument.Key;
var document = jsonDocument.ToJson();
patcher.Apply(document, new ScriptedPatchRequest
{
Script = cfg.Script
});
replicationStats.ScriptSuccess();
}
catch (ParseException e)
{
replicationStats.MarkScriptAsInvalid(Database, cfg.Script);
log.WarnException("Could parse SQL Replication script for " + cfg.Name, e);
return(result);
}
catch (Exception e)
{
replicationStats.RecordScriptError(Database);
log.WarnException("Could not process SQL Replication script for " + cfg.Name + ", skipping this document", e);
}
}
return(result);
}
public static int UpdateDocumentsAfter(this DocumentDatabase db, Guid eTag, Func <JsonDocument, bool> updateDoc,
CancellationToken cancellationToken, TransactionInformation transactionInformation)
{
var initialTime = DateTime.UtcNow;
log.Trace("UpdateDocumentsAfter started whith Etag {0}", eTag);
int updatedDocCount = 0;
db.TransactionalStorage.Batch(action =>
{
var documentRetriever = new DocumentRetriever(action, db.ReadTriggers);
foreach (var doc in action.Documents.GetDocumentsAfter(eTag, int.MaxValue))
{
DocumentRetriever.EnsureIdInMetadata(doc);
if (cancellationToken.IsCancellationRequested)
{
log.Trace("UpdateDocumentsAfter has been cancelled");
}
cancellationToken.ThrowIfCancellationRequested();
var document = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document != null && updateDoc(document))
{
try
{
db.Put(document.Key, document.Etag, document.DataAsJson, document.Metadata, transactionInformation);
updatedDocCount++;
log.Debug("{0} document has been updated in UpdateDocumentsAfter. {1} documents updated so far", document.Key, updatedDocCount);
}
catch (ConcurrencyException)
{
log.Trace("ConcurrencyException caught in UpdateDocumentsAfter");
if (db.UpdateDocumentWithRetries(document.Key, updateDoc, transactionInformation))
{
updatedDocCount++;
log.Debug("{0} document has been updated in UpdateDocumentsAfter. {1} documents updated so far", document.Key, updatedDocCount);
}
}
}
}
});
log.Trace("UpdateDocumentsAfter ETag {0} completed successfully. {1} documents updated in {2}", eTag, updatedDocCount, DateTime.UtcNow.Subtract(initialTime));
return(updatedDocCount);
}
public void AfterStorageCommitBeforeWorkNotifications(JsonDocument[] docs)
{
if (context.Configuration.DisableDocumentPreFetchingForIndexing || docs.Length == 0)
{
return;
}
if (inMemoryDocs.Count > // don't use too much, this is an optimization and we need to be careful about using too much mem
context.Configuration.MaxNumberOfItemsToIndexInSingleBatch)
{
return;
}
foreach (var jsonDocument in docs)
{
DocumentRetriever.EnsureIdInMetadata(jsonDocument);
inMemoryDocs.Enqueue(jsonDocument);
}
}
public void AfterStorageCommitBeforeWorkNotifications(JsonDocument[] docs)
{
if (context.Configuration.DisableDocumentPreFetchingForIndexing || docs.Length == 0)
{
return;
}
if (prefetchingQueue.Count >= // don't use too much, this is an optimization and we need to be careful about using too much mem
context.Configuration.MaxNumberOfItemsToPreFetchForIndexing ||
prefetchingQueue.Aggregate(0, (x, c) => x + SelectSerializedSizeOnDiskIfNotNull(c)) > context.Configuration.AvailableMemoryForRaisingIndexBatchSizeLimit)
{
return;
}
foreach (var jsonDocument in docs)
{
DocumentRetriever.EnsureIdInMetadata(jsonDocument);
prefetchingQueue.Add(jsonDocument);
}
}
public void GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Action <RavenJObject> addDocument)
{
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize, WorkContext.CancellationToken);
var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
if (etag != null)
{
etag = doc.Etag;
}
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever
.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
addDocument(document.ToJson());
returnedDocs = true;
}
if (returnedDocs || docCount == 0)
{
break;
}
start += docCount;
}
});
}
private List <JsonDocument> GetJsonDocsFromDisk(Etag etag, Etag untilEtag)
{
List <JsonDocument> jsonDocs = null;
context.TransactionalStorage.Batch(actions =>
{
//limit how much data we load from disk --> better adhere to memory limits
var totalSizeAllowedToLoadInBytes =
(context.Configuration.MemoryLimitForIndexingInMb * 1024 * 1024) -
(prefetchingQueue.Aggregate(0, (acc, doc) => acc + doc.SerializedSizeOnDisk) + autoTuner.CurrentlyUsedBatchSizesInBytes.Values.Sum());
// at any rate, we will load a min of 512Kb docs
var maxSize = Math.Max(
Math.Min(totalSizeAllowedToLoadInBytes, autoTuner.MaximumSizeAllowedToFetchFromStorageInBytes),
1024 * 512);
jsonDocs = actions.Documents
.GetDocumentsAfter(
etag,
autoTuner.NumberOfItemsToIndexInSingleBatch,
context.CancellationToken,
maxSize,
untilEtag,
autoTuner.FetchingDocumentsFromDiskTimeout
)
.Where(x => x != null)
.Select(doc =>
{
DocumentRetriever.EnsureIdInMetadata(doc);
return(doc);
})
.ToList();
});
if (untilEtag == null)
{
MaybeAddFutureBatch(jsonDocs);
}
return(jsonDocs);
}
public void AfterStorageCommitBeforeWorkNotifications(JsonDocument[] docs)
{
if (context.Configuration.DisableDocumentPreFetching || docs.Length == 0 || DisableCollectingDocumentsAfterCommit)
{
return;
}
if (prefetchingQueue.Count >= // don't use too much, this is an optimization and we need to be careful about using too much mem
context.Configuration.MaxNumberOfItemsToPreFetch ||
prefetchingQueue.Aggregate(0, (x, c) => x + SelectSerializedSizeOnDiskIfNotNull(c)) > context.Configuration.AvailableMemoryForRaisingBatchSizeLimit)
{
return;
}
Etag lowestEtag = null;
foreach (var jsonDocument in docs)
{
DocumentRetriever.EnsureIdInMetadata(jsonDocument);
prefetchingQueue.Add(jsonDocument);
if (ShouldHandleUnusedDocumentsAddedAfterCommit && (lowestEtag == null || jsonDocument.Etag.CompareTo(lowestEtag) < 0))
{
lowestEtag = jsonDocument.Etag;
}
}
if (ShouldHandleUnusedDocumentsAddedAfterCommit && lowestEtag != null)
{
if (lowestInMemoryDocumentAddedAfterCommit == null || lowestEtag.CompareTo(lowestInMemoryDocumentAddedAfterCommit.Etag) < 0)
{
lowestInMemoryDocumentAddedAfterCommit = new DocAddedAfterCommit
{
Etag = lowestEtag,
AddedAt = SystemTime.UtcNow
};
}
}
}
public void AfterCommit(JsonDocument[] docs)
{
if (docs.Length == 0)
{
return;
}
foreach (var doc in docs)
{
DocumentRetriever.EnsureIdInMetadata(doc);
}
futureIndexBatches.Add(new FutureIndexBatch
{
StartingEtag = DecrementEtag(GetLowestEtag(docs)),
Task = new CompletedTask <JsonResults>(new JsonResults
{
Results = docs,
LoadedFromDisk = false
}),
Age = currentIndexingAge
});
}
private JsonResults GetJsonDocsFromDisk(Guid lastIndexed)
{
JsonDocument[] jsonDocs = null;
transactionalStorage.Batch(actions =>
{
jsonDocs = actions.Documents
.GetDocumentsAfter(
lastIndexed,
autoTuner.NumberOfItemsToIndexInSingleBatch,
autoTuner.MaximumSizeAllowedToFetchFromStorage)
.Where(x => x != null)
.Select(doc =>
{
DocumentRetriever.EnsureIdInMetadata(doc);
return(doc);
})
.ToArray();
});
return(new JsonResults
{
Results = jsonDocs,
LoadedFromDisk = true
});
}
public JArray GetDocumentsWithIdStartingWith(string idPrefix, int start, int pageSize)
{
var list = new JArray();
TransactionalStorage.Batch(actions =>
{
var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start)
.Take(pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents)
{
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
{
continue;
}
list.Add(document.ToJson());
}
});
return(list);
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize,
CancellationToken token, ref int nextStart, Action <RavenJObject> addDoc,
string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null,
string skipAfter = null)
{
if (idPrefix == null)
{
throw new ArgumentNullException("idPrefix");
}
idPrefix = idPrefix.Trim();
var canPerformRapidPagination = nextStart > 0 && start == nextStart;
var actualStart = canPerformRapidPagination ? start : 0;
var addedDocs = 0;
var matchedDocs = 0;
TransactionalStorage.Batch(
actions =>
{
var docsToSkip = canPerformRapidPagination ? 0 : start;
int docCount;
AbstractTransformer storedTransformer = null;
if (transformer != null)
{
storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
{
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
}
do
{
docCount = 0;
var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter);
var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState, transformerParameters);
foreach (var doc in docs)
{
token.ThrowIfCancellationRequested();
docCount++;
var keyTest = doc.Key.Substring(idPrefix.Length);
if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
{
continue;
}
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc;
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
matchedDocs++;
if (matchedDocs <= docsToSkip)
{
continue;
}
token.ThrowIfCancellationRequested();
if (storedTransformer != null)
{
using (new CurrentTransformationScope(Database, documentRetriever))
{
var transformed =
storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) })
.Select(x => JsonExtensions.ToJObject(x))
.ToArray();
if (transformed.Length == 0)
{
throw new InvalidOperationException("The transform results function failed on a document: " + document.Key);
}
var transformedJsonDocument = new JsonDocument
{
Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(documentRetriever.Etag),
NonAuthoritativeInformation = document.NonAuthoritativeInformation,
LastModified = document.LastModified,
DataAsJson = new RavenJObject {
{ "$values", new RavenJArray(transformed) }
},
};
addDoc(transformedJsonDocument.ToJson());
}
}
else
{
addDoc(document.ToJson());
}
addedDocs++;
if (addedDocs >= pageSize)
{
break;
}
}
actualStart += pageSize;
}while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue);
});
if (addedDocs != pageSize)
{
nextStart = start; // will mark as last page
}
else if (canPerformRapidPagination)
{
nextStart = start + matchedDocs;
}
else
{
nextStart = actualStart;
}
}
private void BackgroundSqlReplication()
{
int workCounter = 0;
while (Database.WorkContext.DoWork)
{
var config = GetConfiguredReplicationDestinations();
if (config.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var localReplicationStatus = GetReplicationStatus();
var relevantConfigs = config.Where(x =>
{
if (x.Disabled)
{
return(false);
}
var sqlReplicationStatistics = statistics.GetOrDefault(x.Name);
if (sqlReplicationStatistics == null)
{
return(true);
}
return(SystemTime.UtcNow >= sqlReplicationStatistics.LastErrorTime);
}) // have error or the timeout expired
.ToList();
if (relevantConfigs.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var leastReplicatedEtag = GetLeastReplicatedEtag(relevantConfigs, localReplicationStatus);
if (leastReplicatedEtag == null)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
List <JsonDocument> documents;
using (prefetchingBehavior.DocumentBatchFrom(leastReplicatedEtag, out documents))
{
Etag latestEtag = null, lastBatchEtag = null;
if (documents.Count != 0)
{
lastBatchEtag = documents[documents.Count - 1].Etag;
}
var replicationDuration = Stopwatch.StartNew();
documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here
if (documents.Count != 0)
{
latestEtag = documents[documents.Count - 1].Etag;
}
documents.RemoveAll(x => prefetchingBehavior.FilterDocuments(x) == false);
var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >();
foreach (var relevantConfig in relevantConfigs)
{
var cfg = relevantConfig;
Database.TransactionalStorage.Batch(accessor =>
{
deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg),
GetLastEtagFor(localReplicationStatus, cfg),
latestEtag,
1024)
.ToList();
});
}
// No documents AND there aren't any deletes to replicate
if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0)
{
if (latestEtag != null)
{
// so we filtered some documents, let us update the etag about that.
foreach (var lastReplicatedEtag in localReplicationStatus.LastReplicatedEtags)
{
if (lastReplicatedEtag.LastDocEtag.CompareTo(latestEtag) <= 0)
{
lastReplicatedEtag.LastDocEtag = latestEtag;
}
}
latestEtag = Etag.Max(latestEtag, lastBatchEtag);
SaveNewReplicationStatus(localReplicationStatus, latestEtag);
}
else // no point in waiting if we just saved a new doc
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
}
continue;
}
var successes = new ConcurrentQueue <Tuple <SqlReplicationConfig, Etag> >();
try
{
var itemsToReplicate = documents.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
var doc = x.ToJson();
doc[Constants.DocumentIdFieldName] = x.Key;
return(new ReplicatedDoc
{
Document = doc,
Etag = x.Etag,
Key = x.Key,
SerializedSizeOnDisk = x.SerializedSizeOnDisk
});
}).ToList();
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, relevantConfigs, replicationConfig =>
{
try
{
var startTime = SystemTime.UtcNow;
Stopwatch spRepTime = new Stopwatch();
spRepTime.Start();
var lastReplicatedEtag = GetLastEtagFor(localReplicationStatus, replicationConfig);
var deletedDocs = deletedDocsByConfig[replicationConfig];
var docsToReplicate = itemsToReplicate
.Where(x => lastReplicatedEtag.CompareTo(x.Etag) <= 0) // haven't replicate the etag yet
.Where(document =>
{
var info = Database.Documents.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (info.TouchedEtag.CompareTo(lastReplicatedEtag) > 0)
{
log.Debug(
"Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches",
document.Key, replicationConfig.Name, info.TouchedEtag);
return(false);
}
}
return(true);
})
.ToList();
var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicate);
if (currentLatestEtag == null && itemsToReplicate.Count > 0 && docsToReplicate.Count == 0)
{
currentLatestEtag = lastBatchEtag;
}
int countOfReplicatedItems = 0;
if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) &&
ReplicateChangesToDestination(replicationConfig, docsToReplicate, out countOfReplicatedItems))
{
if (deletedDocs.Count > 0)
{
Database.TransactionalStorage.Batch(accessor =>
accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag));
}
successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag));
}
spRepTime.Stop();
var elapsedMicroseconds = (long)(spRepTime.ElapsedTicks * SystemTime.MicroSecPerTick);
var sqlReplicationMetricsCounters = GetSqlReplicationMetricsManager(replicationConfig);
sqlReplicationMetricsCounters.SqlReplicationBatchSizeMeter.Mark(countOfReplicatedItems);
sqlReplicationMetricsCounters.SqlReplicationBatchSizeHistogram.Update(countOfReplicatedItems);
sqlReplicationMetricsCounters.SqlReplicationDurationHistogram.Update(elapsedMicroseconds);
UpdateReplicationPerformance(replicationConfig, startTime, spRepTime.Elapsed, docsToReplicate.Count);
}
catch (Exception e)
{
log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Exception = e.ToString(),
Title = "Sql Replication failure to replication",
Message = "Sql Replication could not replicate to " + replicationConfig.Name,
UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name
});
}
});
if (successes.Count == 0)
{
continue;
}
foreach (var t in successes)
{
var cfg = t.Item1;
var currentLatestEtag = t.Item2;
var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase));
if (destEtag == null)
{
localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag
{
Name = cfg.Name,
LastDocEtag = currentLatestEtag ?? Etag.Empty
});
}
else
{
destEtag.LastDocEtag = currentLatestEtag = currentLatestEtag ?? destEtag.LastDocEtag;
}
latestEtag = Etag.Max(latestEtag, currentLatestEtag);
}
latestEtag = Etag.Max(latestEtag, lastBatchEtag);
SaveNewReplicationStatus(localReplicationStatus, latestEtag);
}
finally
{
AfterReplicationCompleted(successes.Count);
var min = localReplicationStatus.LastReplicatedEtags.Min(x => new ComparableByteArray(x.LastDocEtag.ToByteArray()));
if (min != null)
{
var lastMinReplicatedEtag = min.ToEtag();
prefetchingBehavior.CleanupDocuments(lastMinReplicatedEtag);
prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed);
}
}
}
}
}
public RelationalDatabaseWriter.TableQuerySummary[] SimulateSqlReplicationSQLQueries(string strDocumentId, SqlReplicationConfig sqlReplication, bool performRolledbackTransaction, out Alert alert)
{
alert = null;
RelationalDatabaseWriter.TableQuerySummary[] resutls = null;
try
{
var stats = new SqlReplicationStatistics(sqlReplication.Name);
var jsonDocument = Database.Documents.Get(strDocumentId, null);
DocumentRetriever.EnsureIdInMetadata(jsonDocument);
var doc = jsonDocument.ToJson();
doc[Constants.DocumentIdFieldName] = jsonDocument.Key;
var docs = new List <ReplicatedDoc>
{
new ReplicatedDoc
{
Document = doc,
Etag = jsonDocument.Etag,
Key = jsonDocument.Key,
SerializedSizeOnDisk = jsonDocument.SerializedSizeOnDisk
}
};
var scriptResult = ApplyConversionScript(sqlReplication, docs);
var connectionsDoc = Database.Documents.Get(ConnectionsDocumentName, null);
var sqlReplicationConnections = connectionsDoc.DataAsJson.JsonDeserialization <SqlReplicationConnections>();
if (PrepareSqlReplicationConfig(sqlReplication, sqlReplication.Name, stats, sqlReplicationConnections, false, false))
{
if (performRolledbackTransaction)
{
using (var writer = new RelationalDatabaseWriter(Database, sqlReplication, stats))
{
resutls = writer.RolledBackExecute(scriptResult).ToArray();
}
}
else
{
var simulatedwriter = new RelationalDatabaseWriterSimulator(Database, sqlReplication, stats);
resutls = new List <RelationalDatabaseWriter.TableQuerySummary>()
{
new RelationalDatabaseWriter.TableQuerySummary()
{
Commands = simulatedwriter.SimulateExecuteCommandText(scriptResult)
.Select(x => new RelationalDatabaseWriter.TableQuerySummary.CommandData()
{
CommandText = x
}).ToArray()
}
}.ToArray();
}
}
alert = stats.LastAlert;
}
catch (Exception e)
{
alert = new Alert()
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Message = "Last SQL replication operation for " + sqlReplication.Name + " was failed",
Title = "SQL replication error",
Exception = e.ToString(),
UniqueKey = "Sql Replication Error: " + sqlReplication.Name
};
}
return(resutls);
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
var result = new JsonDocumentsToReplicate();
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int docsSinceLastReplEtag = 0;
List <JsonDocument> docsToReplicate;
List <JsonDocument> filteredDocsToReplicate;
result.LastEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
while (true)
{
docsToReplicate = GetDocsToReplicate(actions, result);
filteredDocsToReplicate =
docsToReplicate
.Where(document =>
{
var info = docDb.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (Etag.IsGreaterThan(info.PreTouchEtag, result.LastEtag) == false)
{
return(false);
}
}
return(destination.FilterDocuments(destinationId, document.Key, document.Metadata));
})
.ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
if (docsToReplicate.Count > 0)
{
var lastDoc = docsToReplicate.Last();
Debug.Assert(lastDoc.Etag != null);
result.LastEtag = lastDoc.Etag.Value;
if (lastDoc.LastModified.HasValue)
{
result.LastLastModified = lastDoc.LastModified.Value;
}
}
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag);
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
{
return(string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag));
}
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
{
return(string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination));
}
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return(string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag));
});
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return(result);
}
public RemoteQueryResults Query(LinearQuery query)
{
var viewGenerator = queryCache.GetOrAdd(query.Query,
s =>
new DynamicViewCompiler("query", new IndexDefinition {
Map = query.Query,
},
new AbstractDynamicCompilationExtension[0])
{
RequiresSelectNewAnonymousType = false
}.GenerateInstance());
var results = new List <string>();
var errors = new List <string>();
int lastResult = 0;
int finalResult = 0;
remoteStorage.Batch(actions =>
{
var firstAndLastDocumentIds = actions.Documents.FirstAndLastDocumentIds();
finalResult = firstAndLastDocumentIds.Item2;
var start = Math.Max(firstAndLastDocumentIds.Item1, query.Start);
var matchingDocs = actions.Documents.DocumentsById(start, firstAndLastDocumentIds.Item2);
if (string.IsNullOrEmpty(viewGenerator.ForEntityName) == false) //optimization
{
matchingDocs =
matchingDocs.Where(x => x.Item1.Metadata.Value <string>("Raven-Entity-Name") == viewGenerator.ForEntityName);
}
var docs = matchingDocs
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x.Item1);
return(x);
})
.Select(x =>
{
lastResult = x.Item2;
return(new DynamicJsonObject(x.Item1.ToJson()));
});
var robustEnumerator = new RobustEnumerator
{
OnError =
(exception, o) =>
errors.Add(String.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
results.AddRange(
robustEnumerator.RobustEnumeration(docs, viewGenerator.MapDefinition)
.Take(query.PageSize)
.Select(result => JsonExtensions.ToJObject(result).ToString())
);
});
return(new RemoteQueryResults
{
LastScannedResult = lastResult,
TotalResults = finalResult,
Errors = errors.ToArray(),
QueryCacheSize = queryCache.Count,
Results = results.ToArray()
});
}