public override void AfterDelete(string key, TransactionInformation transactionInformation)
{
var versioningConfig = Database.GetDocumentVersioningConfiguration(versionInformer.Value[key]);
using (Database.DisableAllTriggersForCurrentThread())
{
Database.TransactionalStorage.Batch(accessor =>
{
using (DocumentCacher.SkipSetDocumentsInDocumentCache())
{
foreach (var jsonDocument in accessor.Documents.GetDocumentsWithIdStartingWith(key + "/revisions/", 0, int.MaxValue, null))
{
if (jsonDocument == null)
{
continue;
}
if (versioningConfig != null && versioningConfig.PurgeOnDelete)
{
Database.Documents.Delete(jsonDocument.Key, null, transactionInformation);
}
else
{
jsonDocument.Metadata.Remove(Constants.RavenReadOnly);
accessor.Documents.AddDocument(jsonDocument.Key, jsonDocument.Etag, jsonDocument.DataAsJson, jsonDocument.Metadata);
}
}
}
});
}
}
protected override Task SerializeToStreamAsync(Stream stream, TransportContext context)
{
var bufferSize = queryOp.Header.TotalResults > 1024 ? 1024 * 64 : 1024 * 8;
using (var bufferedStream = new BufferedStream(stream, bufferSize))
using (queryOp)
using (accessor)
using (_timeout)
using (var writer = GetOutputWriter(req, bufferedStream))
// we may be sending a LOT of documents to the user, and most
// of them aren't going to be relevant for other ops, so we are going to skip
// the cache for that, to avoid filling it up very quickly
using (DocumentCacher.SkipSettingDocumentsInDocumentCache())
{
outputContentTypeSetter(writer.ContentType);
writer.WriteHeader();
try
{
queryOp.Execute(o =>
{
_timeout.Delay();
writer.Write(o);
});
}
catch (Exception e)
{
writer.WriteError(e);
}
}
return(Task.FromResult(true));
}
public void Index(string index,
AbstractViewGenerator viewGenerator,
IndexingBatch batch,
WorkContext context,
IStorageActionsAccessor actions,
DateTime minimumTimestamp)
{
Index value;
if (indexes.TryGetValue(index, out value) == false)
{
log.Debug("Tried to index on a non existent index {0}, ignoring", index);
return;
}
using (EnsureInvariantCulture())
using (DocumentCacher.SkipSettingDocumentsInDocumentCache())
{
value.IndexDocuments(viewGenerator, batch, context, actions, minimumTimestamp);
context.RaiseIndexChangeNotification(new IndexChangeNotification
{
Name = index,
Type = IndexChangeTypes.MapCompleted
});
}
}
private HttpResponseMessage OnBulkOperation(Func <string, IndexQuery, BulkOperationOptions, Action <BulkOperationProgress>, RavenJArray> batchOperation, string index, CancellationTimeout timeout)
{
if (string.IsNullOrEmpty(index))
{
return(GetEmptyMessage(HttpStatusCode.BadRequest));
}
var option = new BulkOperationOptions
{
AllowStale = GetAllowStale(),
MaxOpsPerSec = GetMaxOpsPerSec(),
StaleTimeout = GetStaleTimeout(),
RetrieveDetails = GetRetrieveDetails()
};
var indexQuery = GetIndexQuery(maxPageSize: int.MaxValue);
var status = new BulkOperationStatus();
long id;
var task = Task.Factory.StartNew(() =>
{
using (DocumentCacher.SkipSetDocumentsInDocumentCache())
{
status.State["Batch"] = batchOperation(index, indexQuery, option, x =>
{
status.MarkProgress(x);
});
}
}).ContinueWith(t =>
{
if (timeout != null)
{
timeout.Dispose();
}
if (t.IsFaulted == false)
{
status.MarkCompleted($"Processed {status.OperationProgress.ProcessedEntries} items");
return;
}
var exception = t.Exception.ExtractSingleInnerException();
status.MarkFaulted(exception.Message);
});
Database.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription
{
StartTime = SystemTime.UtcNow,
TaskType = TaskActions.PendingTaskType.IndexBulkOperation,
Description = index
}, out id, timeout.CancellationTokenSource);
return(GetMessageWithObject(new { OperationId = id }, HttpStatusCode.Accepted));
}
public override void Respond(IHttpContext context)
{
using (context.Response.Streaming())
{
context.Response.ContentType = "application/json; charset=utf-8";
var match = urlMatcher.Match(context.GetRequestUrl());
var index = match.Groups[1].Value;
var query = context.GetIndexQueryFromHttpContext(int.MaxValue);
if (string.IsNullOrEmpty(context.Request.QueryString["pageSize"]))
{
query.PageSize = int.MaxValue;
}
var isHeadRequest = context.Request.HttpMethod == "HEAD";
if (isHeadRequest)
{
query.PageSize = 0;
}
using (var writer = GetOutputWriter(context))
{
// we may be sending a LOT of documents to the user, and most
// of them aren't going to be relevant for other ops, so we are going to skip
// the cache for that, to avoid filling it up very quickly
using (DocumentCacher.SkipSettingDocumentsInDocumentCache())
using (var cts = new CancellationTokenSource())
using (var timeout = cts.TimeoutAfter(Settings.DatbaseOperationTimeout))
{
Database.Query(index, query, cts.Token, information =>
{
context.Response.AddHeader("Raven-Result-Etag", information.ResultEtag.ToString());
context.Response.AddHeader("Raven-Index-Etag", information.IndexEtag.ToString());
context.Response.AddHeader("Raven-Is-Stale", information.IsStable ? "true" : "false");
context.Response.AddHeader("Raven-Index", information.Index);
context.Response.AddHeader("Raven-Total-Results", information.TotalResults.ToString(CultureInfo.InvariantCulture));
context.Response.AddHeader("Raven-Index-Timestamp",
information.IndexTimestamp.ToString(Default.DateTimeFormatsToWrite,
CultureInfo.InvariantCulture));
if (isHeadRequest)
{
return;
}
writer.WriteHeader();
}, o =>
{
timeout.Delay();
Database.WorkContext.UpdateFoundWork();
writer.Write(o);
});
}
}
}
}
public override void Respond(IHttpContext context)
{
using (context.Response.Streaming())
{
context.Response.ContentType = "application/json; charset=utf-8";
using (var writer = new JsonTextWriter(new StreamWriter(context.Response.OutputStream)))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
Database.TransactionalStorage.Batch(accessor =>
{
var startsWith = context.Request.QueryString["startsWith"];
int pageSize = context.GetPageSize(int.MaxValue);
if (string.IsNullOrEmpty(context.Request.QueryString["pageSize"]))
{
pageSize = int.MaxValue;
}
// we may be sending a LOT of documents to the user, and most
// of them aren't going to be relevant for other ops, so we are going to skip
// the cache for that, to avoid filling it up very quickly
using (DocumentCacher.SkipSettingDocumentsInDocumentCache())
{
if (string.IsNullOrEmpty(startsWith))
{
Database.GetDocuments(context.GetStart(), pageSize, context.GetEtagFromQueryString(),
doc => doc.WriteTo(writer));
}
else
{
Database.GetDocumentsWithIdStartingWith(
startsWith,
context.Request.QueryString["matches"],
context.Request.QueryString["exclude"],
context.GetStart(),
pageSize,
doc => doc.WriteTo(writer));
}
}
});
writer.WriteEndArray();
writer.WriteEndObject();
writer.Flush();
}
}
}
private void StreamToClient(Stream stream, string startsWith, int start, int pageSize, Etag etag, string matches, int nextPageStart, string skipAfter)
{
using (var cts = new CancellationTokenSource())
using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout))
using (var writer = new JsonTextWriter(new StreamWriter(stream)))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
Database.TransactionalStorage.Batch(accessor =>
{
// we may be sending a LOT of documents to the user, and most
// of them aren't going to be relevant for other ops, so we are going to skip
// the cache for that, to avoid filling it up very quickly
using (DocumentCacher.SkipSettingDocumentsInDocumentCache())
{
if (string.IsNullOrEmpty(startsWith))
{
Database.Documents.GetDocuments(start, pageSize, etag, cts.Token, doc =>
{
timeout.Delay();
doc.WriteTo(writer);
writer.WriteRaw(Environment.NewLine);
});
}
else
{
var nextPageStartInternal = nextPageStart;
Database.Documents.GetDocumentsWithIdStartingWith(startsWith, matches, null, start, pageSize, cts.Token, ref nextPageStartInternal, doc =>
{
timeout.Delay();
doc.WriteTo(writer);
writer.WriteRaw(Environment.NewLine);
}, skipAfter: skipAfter);
nextPageStart = nextPageStartInternal;
}
}
});
writer.WriteEndArray();
writer.WritePropertyName("NextPageStart");
writer.WriteValue(nextPageStart);
writer.WriteEndObject();
writer.Flush();
}
}
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start,
int pageSize, CancellationToken token, ref int nextStart,
string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null,
string skipAfter = null)
{
using (DocumentCacher.SkipSetDocumentsInDocumentCache())
{
var list = new RavenJArray();
GetDocumentsWithIdStartingWith(idPrefix, matches, exclude, start, pageSize, token, ref nextStart,
doc => { if (doc != null)
{
list.Add(doc.ToJson());
}
}, transformer, transformerParameters, skipAfter);
return(list);
}
}
private void WriteDocuments(JsonTextWriter jsonWriter)
{
long totalDocsCount = 0;
storage.Batch(accsesor => totalDocsCount = accsesor.Documents.GetDocumentsCount());
using (DocumentCacher.SkipSetDocumentsInDocumentCache())
{
if (DocumentsStartEtag == Etag.Empty)
{
ExtractDocuments(jsonWriter, totalDocsCount);
}
else
{
ExtractDocumentsFromEtag(jsonWriter, totalDocsCount);
}
}
}
public void Index(string index,
AbstractViewGenerator viewGenerator,
IEnumerable <dynamic> docs,
WorkContext context,
IStorageActionsAccessor actions,
DateTime minimumTimestamp)
{
Index value;
if (indexes.TryGetValue(index, out value) == false)
{
log.Debug("Tried to index on a non existant index {0}, ignoring", index);
return;
}
using (EnsureInvariantCulture())
using (DocumentCacher.SkipSettingDocumentsInDocumentCache())
{
value.IndexDocuments(viewGenerator, docs, context, actions, minimumTimestamp);
}
}
public Etag GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Func <JsonDocument, bool> addDocument,
string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null, long?maxSize = null, TimeSpan?timeout = null)
{
Etag lastDocumentReadEtag = null;
using (DocumentCacher.SkipSetDocumentsInDocumentCache())
TransactionalStorage.Batch(actions =>
{
AbstractTransformer storedTransformer = null;
if (transformer != null)
{
storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
{
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
}
var returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize, token, maxSize: maxSize, timeout: timeout);
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, transformerParameters);
var docCount = 0;
var docCountOnLastAdd = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
if (docCount - docCountOnLastAdd > 1000)
{
addDocument(null); // heartbeat
}
if (etag != null)
{
etag = doc.Etag;
}
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
returnedDocs = true;
Database.WorkContext.UpdateFoundWork();
document = TransformDocumentIfNeeded(document, storedTransformer, documentRetriever);
var canContinue = addDocument(document);
if (!canContinue)
{
break;
}
lastDocumentReadEtag = etag;
docCountOnLastAdd = docCount;
}
if (returnedDocs || docCount == 0)
{
break;
}
// No document was found that matches the requested criteria
// If we had a failure happen, we update the etag as we don't need to process those documents again (no matches there anyways).
if (lastDocumentReadEtag != null)
{
etag = lastDocumentReadEtag;
}
start += docCount;
}
});
return(lastDocumentReadEtag);
}
private void ApplyPrecomputedBatchForNewIndex(Index index, AbstractViewGenerator generator, int pageSize, CancellationTokenSource cts)
{
PrecomputedIndexingBatch result = null;
var docsToIndex = new List <JsonDocument>();
TransactionalStorage.Batch(actions =>
{
var query = QueryBuilder.GetQueryForAllMatchingDocumentsForIndex(Database, generator.ForEntityNames);
using (DocumentCacher.SkipSetDocumentsInDocumentCache())
using (var linked = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, WorkContext.CancellationToken))
using (var op = new QueryActions.DatabaseQueryOperation(Database, Constants.DocumentsByEntityNameIndex, new IndexQuery
{
Query = query,
PageSize = pageSize
}, actions, linked)
{
ShouldSkipDuplicateChecking = true
})
{
op.Init();
//if we are working on a test index, apply the optimization anyway, as the index is capped by small number of results
if (op.Header.TotalResults > pageSize && index.IsTestIndex == false)
{
// we don't apply this optimization if the total number of results
// to index is more than the max numbers to index in a single batch.
// The idea here is that we need to keep the amount
// of memory we use to a manageable level even when introducing a new index to a BIG
// database
try
{
cts.Cancel();
// we have to run just a little bit of the query to properly setup the disposal
op.Execute(o => { });
}
catch (OperationCanceledException)
{
}
return;
}
if (Log.IsDebugEnabled)
{
Log.Debug("For new index {0}, using precomputed indexing batch optimization for {1} docs",
index, op.Header.TotalResults);
}
var totalLoadedDocumentSize = 0;
const int totalSizeToCheck = 16 * 1024 * 1024; //16MB
var localLoadedDocumentSize = 0;
op.Execute(document =>
{
var metadata = document.Value <RavenJObject>(Constants.Metadata);
var key = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
var lastModified = DateTime.Parse(metadata.Value <string>(Constants.LastModified));
document.Remove(Constants.Metadata);
var serializedSizeOnDisk = metadata.Value <int>(Constants.SerializedSizeOnDisk);
metadata.Remove(Constants.SerializedSizeOnDisk);
var doc = new JsonDocument
{
DataAsJson = document,
Etag = etag,
Key = key,
SerializedSizeOnDisk = serializedSizeOnDisk,
LastModified = lastModified,
SkipDeleteFromIndex = true,
Metadata = metadata
};
docsToIndex.Add(doc);
totalLoadedDocumentSize += serializedSizeOnDisk;
localLoadedDocumentSize += serializedSizeOnDisk;
if (totalLoadedDocumentSize > Database.Configuration.MaxPrecomputedBatchTotalDocumentSizeInBytes)
{
var error = $"Aborting applying precomputed batch for index id: {index.indexId}, " +
$"name: {index.PublicName} because we have {totalLoadedDocumentSize}MB of documents that were fetched" +
$"and the configured max data to fetch is " +
$"{Database.Configuration.MaxPrecomputedBatchTotalDocumentSizeInBytes/1024/1024}MB";
//we are aborting operation, so don't keep the references
docsToIndex.Clear();
throw new TotalDataSizeExceededException(error);
}
if (localLoadedDocumentSize <= totalSizeToCheck)
{
return;
}
localLoadedDocumentSize = 0;
if (Database.Configuration.MemoryLimitForProcessingInMb > MemoryStatistics.AvailableMemoryInMb)
{
var error = $"Aborting applying precomputed batch for index id: {index.indexId}, " +
$"name: {index.PublicName} because we have {MemoryStatistics.AvailableMemoryInMb}MB " +
$"of available memory and the available memory for processing is: " +
$"{Database.Configuration.MemoryLimitForProcessingInMb}MB";
//we are aborting operation, so don't keep the references
docsToIndex.Clear();
throw new TotalDataSizeExceededException(error);
}
});
result = new PrecomputedIndexingBatch
{
LastIndexed = op.Header.IndexEtag,
LastModified = op.Header.IndexTimestamp,
Documents = docsToIndex,
Index = index
};
}
});
if (result != null && result.Documents != null && result.Documents.Count >= 0)
{
using (var linked = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, WorkContext.CancellationToken))
{
Database.IndexingExecuter.IndexPrecomputedBatch(result, linked.Token);
if (index.IsTestIndex)
{
TransactionalStorage.Batch(accessor => accessor.Indexing.TouchIndexEtag(index.IndexId));
}
}
}
}
private void StreamToClient(long id, SubscriptionActions subscriptions, Stream stream)
{
var sentDocuments = false;
var bufferStream = new BufferedStream(stream, 1024 * 64);
using (var writer = new JsonTextWriter(new StreamWriter(bufferStream)))
{
var options = subscriptions.GetBatchOptions(id);
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
using (var cts = new CancellationTokenSource())
using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout))
{
Etag lastProcessedDocEtag = null;
var batchSize = 0;
var batchDocCount = 0;
var processedDocuments = 0;
var hasMoreDocs = false;
var config = subscriptions.GetSubscriptionConfig(id);
var startEtag = config.AckEtag;
var criteria = config.Criteria;
bool isPrefixCriteria = !string.IsNullOrWhiteSpace(criteria.KeyStartsWith);
Func <JsonDocument, bool> addDocument = doc =>
{
timeout.Delay();
if (doc == null)
{
// we only have this heartbeat when the streaming has gone on for a long time
// and we haven't send anything to the user in a while (because of filtering, skipping, etc).
writer.WriteRaw(Environment.NewLine);
writer.Flush();
return(true);
}
processedDocuments++;
// We cant continue because we have already maxed out the batch bytes size.
if (options.MaxSize.HasValue && batchSize >= options.MaxSize)
{
return(false);
}
// We cant continue because we have already maxed out the amount of documents to send.
if (batchDocCount >= options.MaxDocCount)
{
return(false);
}
// We can continue because we are ignoring system documents.
if (doc.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase))
{
return(true);
}
// We can continue because we are ignoring the document as it doesn't fit the criteria.
if (MatchCriteria(criteria, doc) == false)
{
return(true);
}
doc.ToJson().WriteTo(writer);
writer.WriteRaw(Environment.NewLine);
batchSize += doc.SerializedSizeOnDisk;
batchDocCount++;
return(true); // We get the next document
};
int retries = 0;
do
{
int lastIndex = processedDocuments;
Database.TransactionalStorage.Batch(accessor =>
{
// we may be sending a LOT of documents to the user, and most
// of them aren't going to be relevant for other ops, so we are going to skip
// the cache for that, to avoid filling it up very quickly
using (DocumentCacher.SkipSetAndGetDocumentsInDocumentCache())
{
if (isPrefixCriteria)
{
// If we don't get any document from GetDocumentsWithIdStartingWith it could be that we are in presence of a lagoon of uninteresting documents, so we are hitting a timeout.
lastProcessedDocEtag = Database.Documents.GetDocumentsWithIdStartingWith(criteria.KeyStartsWith, options.MaxDocCount - batchDocCount, startEtag, cts.Token, addDocument);
hasMoreDocs = false;
}
else
{
// It doesn't matter if we match the criteria or not, the document has been already processed.
lastProcessedDocEtag = Database.Documents.GetDocuments(-1, options.MaxDocCount - batchDocCount, startEtag, cts.Token, addDocument);
// If we don't get any document from GetDocuments it may be a signal that something is wrong.
if (lastProcessedDocEtag == null)
{
hasMoreDocs = false;
}
else
{
var lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag();
hasMoreDocs = EtagUtil.IsGreaterThan(lastDocEtag, lastProcessedDocEtag);
startEtag = lastProcessedDocEtag;
}
retries = lastIndex == batchDocCount ? retries : 0;
}
}
});
if (lastIndex == processedDocuments)
{
if (retries == 3)
{
log.Warn("Subscription processing did not end up replicating any documents for 3 times in a row, stopping operation", retries);
}
else
{
log.Warn("Subscription processing did not end up replicating any documents, due to possible storage error, retry number: {0}", retries);
}
retries++;
}
}while (retries < 3 && hasMoreDocs && batchDocCount < options.MaxDocCount && (options.MaxSize.HasValue == false || batchSize < options.MaxSize));
writer.WriteEndArray();
if (batchDocCount > 0 || isPrefixCriteria)
{
writer.WritePropertyName("LastProcessedEtag");
writer.WriteValue(lastProcessedDocEtag.ToString());
sentDocuments = true;
}
writer.WriteEndObject();
writer.Flush();
bufferStream.Flush();
}
}
if (sentDocuments)
{
subscriptions.UpdateBatchSentTime(id);
}
}
private void StreamToClient(Stream stream, string startsWith, int start, int pageSize, Etag etag, string matches, int nextPageStart, string skipAfter, string transformer, Dictionary <string, RavenJToken> transformerParameters,
Lazy <NameValueCollection> headers, IPrincipal user)
{
var old = CurrentOperationContext.Headers.Value;
var oldUser = CurrentOperationContext.User.Value;
try
{
CurrentOperationContext.Headers.Value = headers;
CurrentOperationContext.User.Value = user;
var bufferStream = new BufferedStream(stream, 1024 * 64);
using (var cts = new CancellationTokenSource())
using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout))
using (var writer = new JsonTextWriter(new StreamWriter(bufferStream)))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
Action <JsonDocument> addDocument = doc =>
{
timeout.Delay();
if (doc == null)
{
// we only have this heartbit when the streaming has gone on for a long time
// and we haven't send anything to the user in a while (because of filtering, skipping, etc).
writer.WriteRaw(Environment.NewLine);
writer.Flush();
return;
}
doc.ToJson().WriteTo(writer);
writer.WriteRaw(Environment.NewLine);
};
Database.TransactionalStorage.Batch(accessor =>
{
// we may be sending a LOT of documents to the user, and most
// of them aren't going to be relevant for other ops, so we are going to skip
// the cache for that, to avoid filling it up very quickly
using (DocumentCacher.SkipSetAndGetDocumentsInDocumentCache())
{
if (string.IsNullOrEmpty(startsWith))
{
Database.Documents.GetDocuments(start, pageSize, etag, cts.Token, doc => { addDocument(doc); return(true); }, transformer, transformerParameters);
}
else
{
var nextPageStartInternal = nextPageStart;
Database.Documents.GetDocumentsWithIdStartingWith(startsWith, matches, null, start, pageSize, cts.Token, ref nextPageStartInternal, addDocument, transformer, transformerParameters, skipAfter);
nextPageStart = nextPageStartInternal;
}
}
});
writer.WriteEndArray();
writer.WritePropertyName("NextPageStart");
writer.WriteValue(nextPageStart);
writer.WriteEndObject();
writer.Flush();
bufferStream.Flush();
}
}
finally
{
CurrentOperationContext.Headers.Value = old;
CurrentOperationContext.User.Value = oldUser;
}
}
private void StreamToClient(long id, SubscriptionActions subscriptions, Stream stream)
{
var sentDocuments = false;
using (var streamWriter = new StreamWriter(stream))
using (var writer = new JsonTextWriter(streamWriter))
{
var options = subscriptions.GetBatchOptions(id);
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
using (var cts = new CancellationTokenSource())
using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout))
{
Etag lastProcessedDocEtag = null;
var batchSize = 0;
var batchDocCount = 0;
var hasMoreDocs = false;
var config = subscriptions.GetSubscriptionConfig(id);
var startEtag = config.AckEtag;
var criteria = config.Criteria;
do
{
Database.TransactionalStorage.Batch(accessor =>
{
// we may be sending a LOT of documents to the user, and most
// of them aren't going to be relevant for other ops, so we are going to skip
// the cache for that, to avoid filling it up very quickly
using (DocumentCacher.SkipSettingDocumentsInDocumentCache())
{
Database.Documents.GetDocuments(-1, options.MaxDocCount - batchDocCount, startEtag, cts.Token, doc =>
{
timeout.Delay();
if (options.MaxSize.HasValue && batchSize >= options.MaxSize)
{
return;
}
if (batchDocCount >= options.MaxDocCount)
{
return;
}
lastProcessedDocEtag = doc.Etag;
if (doc.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase))
{
return;
}
if (MatchCriteria(criteria, doc) == false)
{
return;
}
doc.ToJson().WriteTo(writer);
writer.WriteRaw(Environment.NewLine);
batchSize += doc.SerializedSizeOnDisk;
batchDocCount++;
});
}
if (lastProcessedDocEtag == null)
{
hasMoreDocs = false;
}
else
{
var lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag();
hasMoreDocs = EtagUtil.IsGreaterThan(lastDocEtag, lastProcessedDocEtag);
startEtag = lastProcessedDocEtag;
}
});
} while (hasMoreDocs && batchDocCount < options.MaxDocCount && (options.MaxSize.HasValue == false || batchSize < options.MaxSize));
writer.WriteEndArray();
if (batchDocCount > 0)
{
writer.WritePropertyName("LastProcessedEtag");
writer.WriteValue(lastProcessedDocEtag.ToString());
sentDocuments = true;
}
writer.WriteEndObject();
writer.Flush();
}
}
if (sentDocuments)
{
subscriptions.UpdateBatchSentTime(id);
}
}
void Delete()
{
var currentTime = SystemTime.UtcNow;
var currentExpiryThresholdTime = currentTime.AddHours(-Settings.HoursToKeepMessagesBeforeExpiring);
logger.Debug("Trying to find expired documents to delete (with threshold {0})", currentExpiryThresholdTime.ToString(Default.DateTimeFormatsToWrite, CultureInfo.InvariantCulture));
const string queryString = "Status:3 OR Status:4";
var query = new IndexQuery
{
Start = 0,
PageSize = deletionBatchSize,
Cutoff = currentTime,
Query = queryString,
FieldsToFetch = new[]
{
"__document_id",
"ProcessedAt"
},
SortedFields = new[]
{
new SortedField("ProcessedAt")
{
Field = "ProcessedAt",
Descending = false
}
},
};
try
{
var docsToExpire = 0;
// we may be receiving a LOT of documents to delete, so we are going to skip
// the cache for that, to avoid filling it up very quickly
var stopwatch = Stopwatch.StartNew();
int deletionCount;
using (DocumentCacher.SkipSettingDocumentsInDocumentCache())
using (Database.DisableAllTriggersForCurrentThread())
using (var cts = new CancellationTokenSource())
{
var documentWithCurrentThresholdTimeReached = false;
var items = new List <ICommandData>(deletionBatchSize);
try
{
Database.Query(indexName, query, CancellationTokenSource.CreateLinkedTokenSource(Database.WorkContext.CancellationToken, cts.Token).Token,
null,
doc =>
{
if (documentWithCurrentThresholdTimeReached)
{
return;
}
if (doc.Value <DateTime>("ProcessedAt") >= currentExpiryThresholdTime)
{
documentWithCurrentThresholdTimeReached = true;
cts.Cancel();
return;
}
var id = doc.Value <string>("__document_id");
if (!string.IsNullOrEmpty(id))
{
items.Add(new DeleteCommandData
{
Key = id
});
}
});
}
catch (OperationCanceledException)
{
//Ignore
}
logger.Debug("Batching deletion of {0} documents.", items.Count);
docsToExpire += items.Count;
var results = Database.Batch(items.ToArray());
deletionCount = results.Count(x => x.Deleted == true);
items.Clear();
}
if (docsToExpire == 0)
{
logger.Debug("No expired documents found");
}
else
{
logger.Debug("Deleted {0} out of {1} expired documents batch - Execution time:{2}ms", deletionCount, docsToExpire, stopwatch.ElapsedMilliseconds);
}
}
catch (Exception e)
{
logger.ErrorException("Error when trying to find expired documents", e);
}
}
private List <JsonDocument> GetJsonDocsFromDisk(Etag etag, Etag untilEtag, Reference <bool> earlyExit = null)
{
List <JsonDocument> jsonDocs = null;
// We take an snapshot because the implementation of accessing Values from a ConcurrentDictionary involves a lock.
// Taking the snapshot should be safe enough.
var currentlyUsedBatchSizesInBytes = Size.Sum(autoTuner.CurrentlyUsedBatchSizesInBytes.Values);
using (DocumentCacher.SkipSetDocumentsInDocumentCache())
context.TransactionalStorage.Batch(actions =>
{
//limit how much data we load from disk --> better adhere to memory limits
var totalSizeAllowedToLoadInBytes =
(context.Configuration.Memory.DynamicLimitForProcessing) -
(prefetchingQueue.LoadedSize + currentlyUsedBatchSizesInBytes);
// at any rate, we will load a min of 512Kb docs
long maxSize = Size.Max(Size.Min(totalSizeAllowedToLoadInBytes, autoTuner.MaximumSizeAllowedToFetchFromStorage), minSizeToLoadDocs).GetValue(SizeUnit.Bytes);
var sp = Stopwatch.StartNew();
var totalSize = 0L;
var largestDocSize = 0L;
string largestDocKey = null;
jsonDocs = actions.Documents
.GetDocumentsAfter(
etag,
autoTuner.NumberOfItemsToProcessInSingleBatch,
context.CancellationToken,
maxSize,
untilEtag,
autoTuner.FetchingDocumentsFromDiskTimeout,
earlyExit: earlyExit
)
.Where(x => x != null)
.Select(doc =>
{
if (largestDocSize < doc.SerializedSizeOnDisk)
{
largestDocSize = doc.SerializedSizeOnDisk;
largestDocKey = doc.Key;
}
totalSize += doc.SerializedSizeOnDisk;
JsonDocument.EnsureIdInMetadata(doc);
return(doc);
})
.ToList();
loadTimes.Enqueue(new DiskFetchPerformanceStats
{
LoadingTimeInMillseconds = sp.ElapsedMilliseconds,
NumberOfDocuments = jsonDocs.Count,
TotalSize = totalSize,
LargestDocSize = largestDocSize,
LargestDocKey = largestDocKey
});
while (loadTimes.Count > 8)
{
DiskFetchPerformanceStats _;
loadTimes.TryDequeue(out _);
}
});
return(jsonDocs);
}
