private void StreamToClient(Stream stream, int pageSize, Etag etag, OrderedPartCollection<AbstractFileReadTrigger> readTriggers) { using (var cts = new CancellationTokenSource()) using (var timeout = cts.TimeoutAfter(FileSystemsLandlord.SystemConfiguration.DatabaseOperationTimeout)) using (var writer = new JsonTextWriter(new StreamWriter(stream))) { writer.WriteStartObject(); writer.WritePropertyName("Results"); writer.WriteStartArray(); Storage.Batch(accessor => { var files = accessor.GetFilesAfter(etag, pageSize); foreach (var file in files) { if (readTriggers.CanReadFile(file.FullPath, file.Metadata, ReadOperation.Load) == false) continue; timeout.Delay(); var doc = RavenJObject.FromObject(file); doc.WriteTo(writer); writer.WriteRaw(Environment.NewLine); } }); writer.WriteEndArray(); writer.WriteEndObject(); writer.Flush(); } }
private void StreamToClient(Stream stream, int pageSize, Etag etag) { using (var cts = new CancellationTokenSource()) using (var timeout = cts.TimeoutAfter(FileSystemsLandlord.SystemConfiguration.DatabaseOperationTimeout)) using (var writer = new JsonTextWriter(new StreamWriter(stream))) { writer.WriteStartObject(); writer.WritePropertyName("Results"); writer.WriteStartArray(); Storage.Batch(accessor => { var files = accessor.GetFilesAfter(etag, pageSize); foreach (var file in files) { timeout.Delay(); var doc = RavenJObject.FromObject(file); doc.WriteTo(writer); writer.WriteRaw(Environment.NewLine); } }); writer.WriteEndArray(); writer.WriteEndObject(); writer.Flush(); } }
public override void Respond(IHttpContext context) { using (var cts = new CancellationTokenSource()) { var timeout = cts.TimeoutAfter(Settings.DatbaseOperationTimeout); var databaseBulkOperations = new DatabaseBulkOperations(Database, GetRequestTransaction(context), cts.Token, timeout); switch (context.Request.HttpMethod) { case "POST": Batch(context); break; case "DELETE": OnBulkOperation(context, databaseBulkOperations.DeleteByIndex); break; case "PATCH": var patchRequestJson = context.ReadJsonArray(); var patchRequests = patchRequestJson.Cast<RavenJObject>().Select(PatchRequest.FromJson).ToArray(); OnBulkOperation(context, (index, query, allowStale) => databaseBulkOperations.UpdateByIndex(index, query, patchRequests, allowStale)); break; case "EVAL": var advPatchRequestJson = context.ReadJsonObject<RavenJObject>(); var advPatch = ScriptedPatchRequest.FromJson(advPatchRequestJson); OnBulkOperation(context, (index, query, allowStale) => databaseBulkOperations.UpdateByIndex(index, query, advPatch, allowStale)); break; } } }
public async Task<HttpResponseMessage> BulkPost() { using (var cts = new CancellationTokenSource()) using (cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout)) { var jsonCommandArray = await ReadJsonArrayAsync(); cts.Token.ThrowIfCancellationRequested(); var transactionInformation = GetRequestTransaction(); var commands = (from RavenJObject jsonCommand in jsonCommandArray select CommandDataFactory.CreateCommand(jsonCommand, transactionInformation)).ToArray(); Log.Debug( () => { if (commands.Length > 15) // this is probably an import method, we will input minimal information, to avoid filling up the log { return "\tExecuted " + string.Join( ", ", commands.GroupBy(x => x.Method).Select(x => string.Format("{0:#,#;;0} {1} operations", x.Count(), x.Key))); } var sb = new StringBuilder(); foreach (var commandData in commands) { sb.AppendFormat("\t{0} {1}{2}", commandData.Method, commandData.Key, Environment.NewLine); } return sb.ToString(); }); var batchResult = Database.Batch(commands, cts.Token); return GetMessageWithObject(batchResult); } }
public HttpResponseMessage BulkDelete(string id) { using (var cts = new CancellationTokenSource()) using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatbaseOperationTimeout)) { var databaseBulkOperations = new DatabaseBulkOperations(Database, GetRequestTransaction(), cts.Token, timeout); return OnBulkOperation(databaseBulkOperations.DeleteByIndex, id); } }
public async Task<HttpResponseMessage> BulkPost() { using (var cts = new CancellationTokenSource()) using (cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout)) { RavenJArray jsonCommandArray; try { jsonCommandArray = await ReadJsonArrayAsync(); } catch (InvalidOperationException e) { Log.DebugException("Failed to deserialize document batch request." , e); return GetMessageWithObject(new { Message = "Could not understand json, please check its validity." }, (HttpStatusCode)422); //http code 422 - Unprocessable entity } catch (InvalidDataException e) { Log.DebugException("Failed to deserialize document batch request." , e); return GetMessageWithObject(new { e.Message }, (HttpStatusCode)422); //http code 422 - Unprocessable entity } cts.Token.ThrowIfCancellationRequested(); var transactionInformation = GetRequestTransaction(); var commands = (from RavenJObject jsonCommand in jsonCommandArray select CommandDataFactory.CreateCommand(jsonCommand, transactionInformation)).ToArray(); Log.Debug( () => { if (commands.Length > 15) // this is probably an import method, we will input minimal information, to avoid filling up the log { return "\tExecuted " + string.Join( ", ", commands.GroupBy(x => x.Method).Select(x => string.Format("{0:#,#;;0} {1} operations", x.Count(), x.Key))); } var sb = new StringBuilder(); foreach (var commandData in commands) { sb.AppendFormat("\t{0} {1}{2}", commandData.Method, commandData.Key, Environment.NewLine); } return sb.ToString(); }); var batchResult = Database.Batch(commands, cts.Token); return GetMessageWithObject(batchResult); } }
public async Task<HttpResponseMessage> BulkPatch(string id) { using (var cts = new CancellationTokenSource()) using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatbaseOperationTimeout)) { var databaseBulkOperations = new DatabaseBulkOperations(Database, GetRequestTransaction(), cts.Token, timeout); var patchRequestJson = await ReadJsonArrayAsync(); var patchRequests = patchRequestJson.Cast<RavenJObject>().Select(PatchRequest.FromJson).ToArray(); return OnBulkOperation((index, query, allowStale) => databaseBulkOperations.UpdateByIndex(index, query, patchRequests, allowStale), id); } }
public override void Respond(IHttpContext context) { switch (context.Request.HttpMethod) { case "GET": long documentsCount = 0; Etag lastDocEtag = Etag.Empty; Database.TransactionalStorage.Batch(accessor => { lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag(); documentsCount = accessor.Documents.GetDocumentsCount(); }); lastDocEtag = lastDocEtag.HashWith(BitConverter.GetBytes(documentsCount)); if (context.MatchEtag(lastDocEtag)) { context.SetStatusToNotModified(); } else { context.WriteHeaders(new RavenJObject(), lastDocEtag); using (var cts = new CancellationTokenSource()) { cts.TimeoutAfter(Settings.DatbaseOperationTimeout); var startsWith = context.Request.QueryString["startsWith"]; if (string.IsNullOrEmpty(startsWith)) context.WriteJson(Database.GetDocuments(context.GetStart(), context.GetPageSize(Database.Configuration.MaxPageSize), context.GetEtagFromQueryString(), cts.Token)); else context.WriteJson(Database.GetDocumentsWithIdStartingWith( startsWith, context.Request.QueryString["matches"], context.Request.QueryString["exclude"], context.GetStart(), context.GetPageSize(Database.Configuration.MaxPageSize), cts.Token)); } } break; case "POST": var json = context.ReadJson(); var id = Database.Put(null, Etag.Empty, json, context.Request.Headers.FilterHeaders(), GetRequestTransaction(context)); context.SetStatusToCreated("/docs/" + Uri.EscapeUriString(id.Key)); context.WriteJson(id); break; } }
public HttpResponseMessage DocsGet() { using (var cts = new CancellationTokenSource()) using (cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout)) { long documentsCount = 0; var lastDocEtag = Etag.Empty; Database.TransactionalStorage.Batch( accessor => { lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag(); documentsCount = accessor.Documents.GetDocumentsCount(); }); lastDocEtag = lastDocEtag.HashWith(BitConverter.GetBytes(documentsCount)); if (MatchEtag(lastDocEtag)) return GetEmptyMessage(HttpStatusCode.NotModified); var startsWith = GetQueryStringValue("startsWith"); HttpResponseMessage msg; int nextPageStart = GetNextPageStart(); if (string.IsNullOrEmpty(startsWith)) { var results = Database.Documents.GetDocuments(GetStart(), GetPageSize(Database.Configuration.MaxPageSize), GetEtagFromQueryString(), cts.Token); msg = GetMessageWithObject(results); } else { var transformer = GetQueryStringValue("transformer"); var transformerParameters = this.ExtractTransformerParameters(); msg = GetMessageWithObject( Database.Documents.GetDocumentsWithIdStartingWith( startsWith, GetQueryStringValue("matches"), GetQueryStringValue("exclude"), GetStart(), GetPageSize(Database.Configuration.MaxPageSize), cts.Token, ref nextPageStart, transformer, transformerParameters, skipAfter: GetQueryStringValue("skipAfter"))); } WriteHeaders(new RavenJObject { { Constants.NextPageStart, nextPageStart } }, lastDocEtag, msg); return msg; } }
public override void Respond(IHttpContext context) { using (context.Response.Streaming()) { context.Response.ContentType = "application/json; charset=utf-8"; var match = urlMatcher.Match(context.GetRequestUrl()); var index = match.Groups[1].Value; var query = context.GetIndexQueryFromHttpContext(int.MaxValue); if (string.IsNullOrEmpty(context.Request.QueryString["pageSize"])) query.PageSize = int.MaxValue; var isHeadRequest = context.Request.HttpMethod == "HEAD"; if (isHeadRequest) query.PageSize = 0; using (var writer = GetOutputWriter(context)) { // we may be sending a LOT of documents to the user, and most // of them aren't going to be relevant for other ops, so we are going to skip // the cache for that, to avoid filling it up very quickly using (DocumentCacher.SkipSettingDocumentsInDocumentCache()) using (var cts = new CancellationTokenSource()) using(var timeout = cts.TimeoutAfter(Settings.DatbaseOperationTimeout)) { Database.Query(index, query, cts.Token, information => { context.Response.AddHeader("Raven-Result-Etag", information.ResultEtag.ToString()); context.Response.AddHeader("Raven-Index-Etag", information.IndexEtag.ToString()); context.Response.AddHeader("Raven-Is-Stale", information.IsStable ? "true" : "false"); context.Response.AddHeader("Raven-Index", information.Index); context.Response.AddHeader("Raven-Total-Results", information.TotalResults.ToString(CultureInfo.InvariantCulture)); context.Response.AddHeader("Raven-Index-Timestamp", information.IndexTimestamp.ToString(Default.DateTimeFormatsToWrite, CultureInfo.InvariantCulture)); if (isHeadRequest) return; writer.WriteHeader(); }, o => { timeout.Delay(); Database.WorkContext.UpdateFoundWork(); writer.Write(o); }); } } } }
private void StreamToClient(Stream stream, string startsWith, int start, int pageSize, Etag etag, string matches, int nextPageStart, string skipAfter) { var bufferStream = new BufferedStream(stream, 1024 * 64); using (var cts = new CancellationTokenSource()) using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout)) using (var writer = new JsonTextWriter(new StreamWriter(bufferStream))) { writer.WriteStartObject(); writer.WritePropertyName("Results"); writer.WriteStartArray(); Action<JsonDocument> addDocument = doc => { timeout.Delay(); doc.ToJson().WriteTo(writer); writer.WriteRaw(Environment.NewLine); }; Database.TransactionalStorage.Batch(accessor => { // we may be sending a LOT of documents to the user, and most // of them aren't going to be relevant for other ops, so we are going to skip // the cache for that, to avoid filling it up very quickly using (DocumentCacher.SkipSettingDocumentsInDocumentCache()) { if (string.IsNullOrEmpty(startsWith)) { Database.Documents.GetDocuments(start, pageSize, etag, cts.Token, addDocument); } else { var nextPageStartInternal = nextPageStart; Database.Documents.GetDocumentsWithIdStartingWith(startsWith, matches, null, start, pageSize, cts.Token, ref nextPageStartInternal, addDocument, skipAfter: skipAfter); nextPageStart = nextPageStartInternal; } } }); writer.WriteEndArray(); writer.WritePropertyName("NextPageStart"); writer.WriteValue(nextPageStart); writer.WriteEndObject(); writer.Flush(); bufferStream.Flush(); } }
public HttpResponseMessage BulkDelete(string id) { // we don't use using because execution is async var cts = new CancellationTokenSource(); var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout); var indexDefinition = Database.IndexDefinitionStorage.GetIndexDefinition(id); if (indexDefinition == null) throw new IndexDoesNotExistsException(string.Format("Index '{0}' does not exist.", id)); if (indexDefinition.IsMapReduce) throw new InvalidOperationException("Cannot execute DeleteByIndex operation on Map-Reduce indexes."); var databaseBulkOperations = new DatabaseBulkOperations(Database, GetRequestTransaction(), cts, timeout); return OnBulkOperation(databaseBulkOperations.DeleteByIndex, id, cts, timeout); }
public HttpResponseMessage IndexGet(string id) { using (var cts = new CancellationTokenSource()) using (cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout)) { var index = id; if (string.IsNullOrEmpty(GetQueryStringValue("definition")) == false) return GetIndexDefinition(index); if (string.IsNullOrEmpty(GetQueryStringValue("source")) == false) return GetIndexSource(index); if (string.IsNullOrEmpty(GetQueryStringValue("debug")) == false) return DebugIndex(index); if (string.IsNullOrEmpty(GetQueryStringValue("explain")) == false) return GetExplanation(index); return GetIndexQueryResult(index, cts.Token); } }
private void StreamToClient(Stream stream, ExportOptions options, Lazy<NameValueCollection> headers, IPrincipal user) { var old = CurrentOperationContext.Headers.Value; var oldUser = CurrentOperationContext.User.Value; try { CurrentOperationContext.Headers.Value = headers; CurrentOperationContext.User.Value = user; Database.TransactionalStorage.Batch(accessor => { var bufferStream = new BufferedStream(stream, 1024 * 64); using (var cts = new CancellationTokenSource()) using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout)) using (var streamWriter = new StreamWriter(bufferStream)) using (var writer = new JsonTextWriter(streamWriter)) { writer.WriteStartObject(); writer.WritePropertyName("Results"); writer.WriteStartArray(); var exporter = new SmugglerExporter(Database, options); exporter.Export(item => WriteToStream(writer, item, timeout), cts.Token); writer.WriteEndArray(); writer.WriteEndObject(); writer.Flush(); bufferStream.Flush(); } }); } finally { CurrentOperationContext.Headers.Value = old; CurrentOperationContext.User.Value = oldUser; } }
public HttpResponseMessage IndexGet(string id) { using (var cts = new CancellationTokenSource()) using (cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout)) { var index = id; if (string.IsNullOrEmpty(GetQueryStringValue("definition")) == false) return GetIndexDefinition(index); if (string.IsNullOrEmpty(GetQueryStringValue("source")) == false) return GetIndexSource(index); if (string.IsNullOrEmpty(GetQueryStringValue("debug")) == false) return DebugIndex(index); if (string.IsNullOrEmpty(GetQueryStringValue("explain")) == false) return GetExplanation(index); try { return GetIndexQueryResult(index, cts.Token); } catch (OperationCanceledException e) { throw new TimeoutException(string.Format("The query did not produce results in {0}", DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout), e); } } }
public HttpResponseMessage SteamQueryGet(string id) { var cts = new CancellationTokenSource(); var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout); var msg = GetEmptyMessage(); var index = id; var query = GetIndexQuery(int.MaxValue); if (string.IsNullOrEmpty(GetQueryStringValue("pageSize"))) query.PageSize = int.MaxValue; var isHeadRequest = InnerRequest.Method == HttpMethod.Head; if (isHeadRequest) query.PageSize = 0; var accessor = Database.TransactionalStorage.CreateAccessor(); //accessor will be disposed in the StreamQueryContent.SerializeToStreamAsync! try { var queryOp = new QueryActions.DatabaseQueryOperation(Database, index, query, accessor, cts); queryOp.Init(); msg.Content = new StreamQueryContent(InnerRequest, queryOp, accessor, timeout, mediaType => msg.Content.Headers.ContentType = new MediaTypeHeaderValue(mediaType) {CharSet = "utf-8"}); msg.Headers.Add("Raven-Result-Etag", queryOp.Header.ResultEtag.ToString()); msg.Headers.Add("Raven-Index-Etag", queryOp.Header.IndexEtag.ToString()); msg.Headers.Add("Raven-Is-Stale", queryOp.Header.IsStale ? "true" : "false"); msg.Headers.Add("Raven-Index", queryOp.Header.Index); msg.Headers.Add("Raven-Total-Results", queryOp.Header.TotalResults.ToString(CultureInfo.InvariantCulture)); msg.Headers.Add( "Raven-Index-Timestamp", queryOp.Header.IndexTimestamp.ToString(Default.DateTimeFormatsToWrite, CultureInfo.InvariantCulture)); if (IsCsvDownloadRequest(InnerRequest)) { msg.Content.Headers.Add("Content-Disposition", "attachment; filename=export.csv"); } } catch (Exception) { accessor.Dispose(); throw; } return msg; }
private void StreamToClient(long id, SubscriptionActions subscriptions, Stream stream) { var sentDocuments = false; using (var streamWriter = new StreamWriter(stream)) using (var writer = new JsonTextWriter(streamWriter)) { var options = subscriptions.GetBatchOptions(id); writer.WriteStartObject(); writer.WritePropertyName("Results"); writer.WriteStartArray(); using (var cts = new CancellationTokenSource()) using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout)) { Etag lastProcessedDocEtag = null; var batchSize = 0; var batchDocCount = 0; var hasMoreDocs = false; var config = subscriptions.GetSubscriptionConfig(id); var startEtag = config.AckEtag; var criteria = config.Criteria; Action<JsonDocument> addDocument = doc => { timeout.Delay(); if (options.MaxSize.HasValue && batchSize >= options.MaxSize) return; if (batchDocCount >= options.MaxDocCount) return; lastProcessedDocEtag = doc.Etag; if (doc.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)) return; if (MatchCriteria(criteria, doc) == false) return; doc.ToJson().WriteTo(writer); writer.WriteRaw(Environment.NewLine); batchSize += doc.SerializedSizeOnDisk; batchDocCount++; }; int nextStart = 0; do { Database.TransactionalStorage.Batch(accessor => { // we may be sending a LOT of documents to the user, and most // of them aren't going to be relevant for other ops, so we are going to skip // the cache for that, to avoid filling it up very quickly using (DocumentCacher.SkipSettingDocumentsInDocumentCache()) { if (!string.IsNullOrWhiteSpace(criteria.KeyStartsWith)) { Database.Documents.GetDocumentsWithIdStartingWith(criteria.KeyStartsWith, options.MaxDocCount - batchDocCount, startEtag, cts.Token, addDocument); } else { Database.Documents.GetDocuments(-1, options.MaxDocCount - batchDocCount, startEtag, cts.Token, addDocument); } } if (lastProcessedDocEtag == null) hasMoreDocs = false; else { var lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag(); hasMoreDocs = EtagUtil.IsGreaterThan(lastDocEtag, lastProcessedDocEtag); startEtag = lastProcessedDocEtag; } }); } while (hasMoreDocs && batchDocCount < options.MaxDocCount && (options.MaxSize.HasValue == false || batchSize < options.MaxSize)); writer.WriteEndArray(); if (batchDocCount > 0) { writer.WritePropertyName("LastProcessedEtag"); writer.WriteValue(lastProcessedDocEtag.ToString()); sentDocuments = true; } writer.WriteEndObject(); writer.Flush(); } } if (sentDocuments) subscriptions.UpdateBatchSentTime(id); }
private void StreamToClient(long id, SubscriptionActions subscriptions, Stream stream) { var sentDocuments = false; var bufferStream = new BufferedStream(stream, 1024 * 64); var lastBatchSentTime = Stopwatch.StartNew(); using (var writer = new JsonTextWriter(new StreamWriter(bufferStream))) { var options = subscriptions.GetBatchOptions(id); writer.WriteStartObject(); writer.WritePropertyName("Results"); writer.WriteStartArray(); using (var cts = new CancellationTokenSource()) using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout)) { Etag lastProcessedDocEtag = null; var batchSize = 0; var batchDocCount = 0; var processedDocumentsCount = 0; var hasMoreDocs = false; var config = subscriptions.GetSubscriptionConfig(id); var startEtag = config.AckEtag; var criteria = config.Criteria; bool isPrefixCriteria = !string.IsNullOrWhiteSpace(criteria.KeyStartsWith); Func<JsonDocument, bool> addDocument = doc => { timeout.Delay(); if (doc == null) { // we only have this heartbeat when the streaming has gone on for a long time // and we haven't sent anything to the user in a while (because of filtering, skipping, etc). writer.WriteRaw(Environment.NewLine); writer.Flush(); if (lastBatchSentTime.ElapsedMilliseconds > 30000) return false; return true; } processedDocumentsCount++; // We cant continue because we have already maxed out the batch bytes size. if (options.MaxSize.HasValue && batchSize >= options.MaxSize) return false; // We cant continue because we have already maxed out the amount of documents to send. if (batchDocCount >= options.MaxDocCount) return false; // We can continue because we are ignoring system documents. if (doc.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)) return true; // We can continue because we are ignoring the document as it doesn't fit the criteria. if (MatchCriteria(criteria, doc) == false) return true; doc.ToJson().WriteTo(writer); writer.WriteRaw(Environment.NewLine); batchSize += doc.SerializedSizeOnDisk; batchDocCount++; return true; // We get the next document }; var retries = 0; do { var lastProcessedDocumentsCount = processedDocumentsCount; Database.TransactionalStorage.Batch(accessor => { // we may be sending a LOT of documents to the user, and most // of them aren't going to be relevant for other ops, so we are going to skip // the cache for that, to avoid filling it up very quickly using (DocumentCacher.SkipSetAndGetDocumentsInDocumentCache()) { if (isPrefixCriteria) { // If we don't get any document from GetDocumentsWithIdStartingWith it could be that we are in presence of a lagoon of uninteresting documents, so we are hitting a timeout. lastProcessedDocEtag = Database.Documents.GetDocumentsWithIdStartingWith(criteria.KeyStartsWith, options.MaxDocCount - batchDocCount, startEtag, cts.Token, addDocument); hasMoreDocs = false; } else { // It doesn't matter if we match the criteria or not, the document has been already processed. lastProcessedDocEtag = Database.Documents.GetDocuments(-1, options.MaxDocCount - batchDocCount, startEtag, cts.Token, addDocument); // If we don't get any document from GetDocuments it may be a signal that something is wrong. if (lastProcessedDocEtag == null) { hasMoreDocs = false; } else { var lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag(); hasMoreDocs = EtagUtil.IsGreaterThan(lastDocEtag, lastProcessedDocEtag); startEtag = lastProcessedDocEtag; } retries = lastProcessedDocumentsCount == batchDocCount ? retries : 0; } } }); if (lastBatchSentTime.ElapsedMilliseconds >= 30000) { if (batchDocCount == 0) log.Warn("Subscription filtered out all possible documents for {0:#,#;;0} seconds in a row, stopping operation", lastBatchSentTime.Elapsed.TotalSeconds); break; } if (lastProcessedDocumentsCount == processedDocumentsCount) { if (retries == 3) { log.Warn("Subscription processing did not end up replicating any documents for 3 times in a row, stopping operation", retries); } else { log.Warn("Subscription processing did not end up replicating any documents, due to possible storage error, retry number: {0}", retries); } retries++; } } while (retries < 3 && hasMoreDocs && batchDocCount < options.MaxDocCount && (options.MaxSize.HasValue == false || batchSize < options.MaxSize)); writer.WriteEndArray(); if (batchDocCount > 0 || processedDocumentsCount > 0 || isPrefixCriteria) { writer.WritePropertyName("LastProcessedEtag"); writer.WriteValue(lastProcessedDocEtag.ToString()); sentDocuments = true; } writer.WriteEndObject(); writer.Flush(); bufferStream.Flush(); } } if (sentDocuments) subscriptions.UpdateBatchSentTime(id); }
public async Task<HttpResponseMessage> BulkEval(string id) { RavenJObject advPatchRequestJson; try { advPatchRequestJson = await ReadJsonObjectAsync<RavenJObject>(); } catch (InvalidOperationException e) { Log.DebugException("Failed to deserialize document batch request." , e); return GetMessageWithObject(new { Message = "Could not understand json, please check its validity." }, (HttpStatusCode)422); //http code 422 - Unprocessable entity } catch (InvalidDataException e) { Log.DebugException("Failed to deserialize document batch request." , e); return GetMessageWithObject(new { e.Message }, (HttpStatusCode)422); //http code 422 - Unprocessable entity } // we don't use using because execution is async var cts = new CancellationTokenSource(); var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout); var databaseBulkOperations = new DatabaseBulkOperations(Database, GetRequestTransaction(), cts, timeout); var advPatch = ScriptedPatchRequest.FromJson(advPatchRequestJson); return OnBulkOperation((index, query, options) => databaseBulkOperations.UpdateByIndex(index, query, advPatch, options), id, timeout); }
public override void Respond(IHttpContext context) { using (context.Response.Streaming()) { context.Response.ContentType = "application/json; charset=utf-8"; using (var cts = new CancellationTokenSource()) using (var writer = new JsonTextWriter(new StreamWriter(context.Response.OutputStream))) using (var timeout = cts.TimeoutAfter(Settings.DatbaseOperationTimeout)) { writer.WriteStartObject(); writer.WritePropertyName("Results"); writer.WriteStartArray(); Database.TransactionalStorage.Batch(accessor => { var startsWith = context.Request.QueryString["startsWith"]; int pageSize = context.GetPageSize(int.MaxValue); if (string.IsNullOrEmpty(context.Request.QueryString["pageSize"])) pageSize = int.MaxValue; // we may be sending a LOT of documents to the user, and most // of them aren't going to be relevant for other ops, so we are going to skip // the cache for that, to avoid filling it up very quickly using (DocumentCacher.SkipSettingDocumentsInDocumentCache()) { if (string.IsNullOrEmpty(startsWith)) { Database.GetDocuments(context.GetStart(), pageSize, context.GetEtagFromQueryString(), cts.Token, doc => { timeout.Delay(); doc.WriteTo(writer); }); } else { Database.GetDocumentsWithIdStartingWith( startsWith, context.Request.QueryString["matches"], context.Request.QueryString["exclude"], context.GetStart(), pageSize, cts.Token, doc => { timeout.Delay(); Database.WorkContext.UpdateFoundWork(); doc.WriteTo(writer); }); } } }); writer.WriteEndArray(); writer.WriteEndObject(); writer.Flush(); } } }
public async Task<HttpResponseMessage> BulkEval(string id) { // we don't use using because execution is async var cts = new CancellationTokenSource(); var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout); var databaseBulkOperations = new DatabaseBulkOperations(Database, GetRequestTransaction(), cts, timeout); var advPatchRequestJson = await ReadJsonObjectAsync<RavenJObject>(); var advPatch = ScriptedPatchRequest.FromJson(advPatchRequestJson); return OnBulkOperation((index, query, allowStale) => databaseBulkOperations.UpdateByIndex(index, query, advPatch, allowStale), id, cts, timeout); }
public bool TimerCallback() { if (Database.Disposed) { return false; } if (Monitor.TryEnter(locker) == false) return false; try { DateTime currentTime = SystemTime.UtcNow; string nowAsStr = currentTime.GetDefaultRavenFormat(); logger.Debug("Trying to find expired documents to delete"); var query = "Expiry:[* TO " + nowAsStr + "]"; var list = new List<string>(); int start = 0; while (true) { const int pageSize = 1024; QueryResultWithIncludes queryResult; using (var cts = new CancellationTokenSource()) using (Database.DisableAllTriggersForCurrentThread()) using (cts.TimeoutAfter(TimeSpan.FromMinutes(5))) { queryResult = Database.Queries.Query(RavenDocumentsByExpirationDate, new IndexQuery { Start = start, PageSize = pageSize, Cutoff = currentTime, Query = query, FieldsToFetch = new[] { "__document_id" } }, cts.Token); } if (queryResult.Results.Count == 0) break; list.AddRange(queryResult.Results.Select(result => result.Value<string>("__document_id")).Where(x => string.IsNullOrEmpty(x) == false)); if (queryResult.Results.Count < pageSize) break; start += pageSize; if (Database.Disposed) return false; } if (list.Count == 0) return true; logger.Debug( () => string.Format("Deleting {0} expired documents: [{1}]", list.Count, string.Join(", ", list))); foreach (var id in list) { Database.Documents.Delete(id, null, null); if (Database.Disposed) return false; } } catch (Exception e) { logger.ErrorException("Error when trying to find expired documents", e); } finally { Monitor.Exit(locker); } return true; }
public async Task<HttpResponseMessage> BulkInsertPost() { if (string.IsNullOrEmpty(GetQueryStringValue("no-op")) == false) { // this is a no-op request which is there just to force the client HTTP layer to handle the authentication // only used for legacy clients return GetEmptyMessage(); } if ("generate-single-use-auth-token".Equals(GetQueryStringValue("op"), StringComparison.InvariantCultureIgnoreCase)) { // using windows auth with anonymous access = none sometimes generate a 401 even though we made two requests // instead of relying on windows auth, which require request buffering, we generate a one time token and return it. // we KNOW that the user have access to this db for writing, since they got here, so there is no issue in generating // a single use token for them. var authorizer = (MixedModeRequestAuthorizer)Configuration.Properties[typeof(MixedModeRequestAuthorizer)]; var token = authorizer.GenerateSingleUseAuthToken(DatabaseName, User); return GetMessageWithObject(new { Token = token }); } if (HttpContext.Current != null) HttpContext.Current.Server.ScriptTimeout = 60 * 60 * 6; // six hours should do it, I think. var options = new BulkInsertOptions { OverwriteExisting = GetOverwriteExisting(), CheckReferencesInIndexes = GetCheckReferencesInIndexes(), SkipOverwriteIfUnchanged = GetSkipOverwriteIfUnchanged() }; var operationId = ExtractOperationId(); var sp = Stopwatch.StartNew(); var status = new BulkInsertStatus(); status.IsTimedOut = false; var documents = 0; var mre = new ManualResetEventSlim(false); var tre = new CancellationTokenSource(); var inputStream = await InnerRequest.Content.ReadAsStreamAsync().ConfigureAwait(false); var currentDatabase = Database; var timeout = tre.TimeoutAfter(currentDatabase.Configuration.BulkImportBatchTimeout); var user = CurrentOperationContext.User.Value; var headers = CurrentOperationContext.Headers.Value; Exception error = null; var task = Task.Factory.StartNew(() => { try { CurrentOperationContext.User.Value = user; CurrentOperationContext.Headers.Value = headers; currentDatabase.Documents.BulkInsert(options, YieldBatches(timeout, inputStream, mre, batchSize => documents += batchSize), operationId, tre.Token, timeout); } catch (InvalidDataException e) { status.Faulted = true; status.State = RavenJObject.FromObject(new { Error = "Could not understand json.", InnerError = e.SimplifyException().Message }); status.IsSerializationError = true; error = e; } catch (OperationCanceledException) { // happens on timeout currentDatabase.Notifications.RaiseNotifications(new BulkInsertChangeNotification { OperationId = operationId, Message = "Operation cancelled, likely because of a batch timeout", Type = DocumentChangeTypes.BulkInsertError }); status.IsTimedOut = true; status.Faulted = true; } catch (Exception e) { status.Faulted = true; status.State = RavenJObject.FromObject(new { Error = e.SimplifyException().Message }); error = e; } finally { status.Completed = true; status.Documents = documents; CurrentOperationContext.User.Value = null; CurrentOperationContext.Headers.Value = null; timeout.Dispose(); } }, tre.Token); long id; Database.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.BulkInsert, Payload = operationId.ToString() }, out id, tre); await task; if (error != null) { var httpStatusCode = status.IsSerializationError ? (HttpStatusCode)422 : HttpStatusCode.InternalServerError; return GetMessageWithObject(new { error.Message, Error = error.ToString() }, httpStatusCode); } if (status.IsTimedOut) throw new TimeoutException("Bulk insert operation did not receive new data longer than configured threshold"); sp.Stop(); AddRequestTraceInfo(log => log.AppendFormat("\tBulk inserted received {0:#,#;;0} documents in {1}, task #: {2}", documents, sp.Elapsed, id)); return GetMessageWithObject(new { OperationId = id }); }
private void GetIndexQueryResult(IHttpContext context, string index) { Etag indexEtag; QueryResultWithIncludes queryResult; using (var cts = new CancellationTokenSource()) { cts.TimeoutAfter(Settings.DatbaseOperationTimeout); queryResult = ExecuteQuery(context, index, cts.Token, out indexEtag); } if (queryResult == null) return; var includes = context.Request.QueryString.GetValues("include") ?? new string[0]; var loadedIds = new HashSet<string>( queryResult.Results .Where(x => x["@metadata"] != null) .Select(x => x["@metadata"].Value<string>("@id")) .Where(x => x != null) ); var command = new AddIncludesCommand(Database, GetRequestTransaction(context), (etag, doc) => queryResult.Includes.Add(doc), includes, loadedIds); foreach (var result in queryResult.Results) { command.Execute(result); } command.AlsoInclude(queryResult.IdsToInclude); context.WriteETag(indexEtag); if (queryResult.NonAuthoritativeInformation) context.SetStatusToNonAuthoritativeInformation(); context.WriteJson(queryResult); }
private void TimerCallback(object state) { if (executing) return; executing = true; try { var currentTime = SystemTime.UtcNow; var currentExpiryThresholdTime = currentTime.AddHours(-Settings.HoursToKeepMessagesBeforeExpiring); var expiryThresholdAsStr = currentExpiryThresholdTime.ToString(Default.DateTimeFormatsToWrite, CultureInfo.InvariantCulture); logger.Debug("Trying to find expired documents to delete..."); var query = "(Status:3 OR Status:4) AND ProcessedAt:[* TO " + expiryThresholdAsStr + "]"; var list = new List<string>(); var start = 0; while (true) { const int pageSize = 1024; QueryResultWithIncludes queryResult; using (var cts = new CancellationTokenSource()) using (Database.DisableAllTriggersForCurrentThread()) { cts.TimeoutAfter(TimeSpan.FromMinutes(5)); queryResult = Database.Query(indexName, new IndexQuery { Start = start, PageSize = pageSize, Cutoff = currentTime, Query = query, FieldsToFetch = new[] { "__document_id" } } , cts.Token); } if (queryResult.Results.Count == 0) break; list.AddRange(queryResult.Results.Select(result => result.Value<string>("__document_id")).Where(x => string.IsNullOrEmpty(x) == false)); if (queryResult.Results.Count < pageSize) break; start += pageSize; } if (list.Count == 0) { logger.Debug("No expired documents found"); return; } logger.Debug(() => string.Format("Deleting {0} expired documents: [{1}]", list.Count, string.Join(", ", list))); foreach (var id in list) { Database.Delete(id, null, null); } } catch (Exception e) { logger.ErrorException("Error when trying to find expired documents", e); } finally { executing = false; } }