private void GetDocuments(DocumentsOperationContext context, Transformer transformer, bool metadataOnly) { // everything here operates on all docs var actualEtag = ComputeAllDocumentsEtag(context); if (GetLongFromHeaders("If-None-Match") == actualEtag) { HttpContext.Response.StatusCode = 304; return; } HttpContext.Response.Headers["ETag"] = "\"" + actualEtag + "\""; var etag = GetLongQueryString("etag", false); IEnumerable <Document> documents; bool isLoadStartingWith = false; if (etag != null) { documents = Database.DocumentsStorage.GetDocumentsFrom(context, etag.Value, GetStart(), GetPageSize()); } else if (HttpContext.Request.Query.ContainsKey("startsWith")) { isLoadStartingWith = true; documents = Database.DocumentsStorage.GetDocumentsStartingWith(context, HttpContext.Request.Query["startsWith"], HttpContext.Request.Query["matches"], HttpContext.Request.Query["excludes"], GetStart(), GetPageSize() ); } else // recent docs { documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, GetStart(), GetPageSize()); } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { if (isLoadStartingWith) { writer.WriteStartObject(); writer.WritePropertyName("Results"); } if (transformer != null) { var transformerParameters = GetTransformerParameters(context); using (var scope = transformer.OpenTransformationScope(transformerParameters, null, Database.DocumentsStorage, Database.TransformerStore, context)) { writer.WriteDocuments(context, scope.Transform(documents).ToList(), metadataOnly); } } else { writer.WriteDocuments(context, documents, metadataOnly); } if (isLoadStartingWith) { writer.WriteEndObject(); } } }
public async Task Put() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var id = GetQueryStringValueAndAssertIfSingleAndNotEmpty("id"); var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name"); var contentType = GetStringQueryString("contentType", false) ?? ""; AttachmentDetails result; using (var streamsTempFile = Database.DocumentsStorage.AttachmentsStorage.GetTempFile("put")) using (var stream = streamsTempFile.StartNewStream()) { Stream requestBodyStream = RequestBodyStream(); string hash; try { hash = await AttachmentsStorageHelper.CopyStreamToFileAndCalculateHash(context, requestBodyStream, stream, Database.DatabaseShutdown); } catch (Exception) { try { // if we failed to read the entire request body stream, we might leave // data in the pipe still, this will cause us to read and discard the // rest of the attachment stream and return the actual error to the caller requestBodyStream.CopyTo(Stream.Null); } catch (Exception) { // we tried, but we can't clean the request, so let's just kill // the connection HttpContext.Abort(); } throw; } var changeVector = context.GetLazyString(GetStringFromHeaders("If-Match")); var cmd = new MergedPutAttachmentCommand { Database = Database, ExpectedChangeVector = changeVector, DocumentId = id, Name = name, Stream = stream, Hash = hash, ContentType = contentType }; stream.Flush(); await Database.TxMerger.Enqueue(cmd); cmd.ExceptionDispatchInfo?.Throw(); result = cmd.Result; } HttpContext.Response.StatusCode = (int)HttpStatusCode.Created; using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName(nameof(AttachmentDetails.ChangeVector)); writer.WriteString(result.ChangeVector); writer.WriteComma(); writer.WritePropertyName(nameof(AttachmentDetails.Name)); writer.WriteString(result.Name); writer.WriteComma(); writer.WritePropertyName(nameof(AttachmentDetails.DocumentId)); writer.WriteString(result.DocumentId); writer.WriteComma(); writer.WritePropertyName(nameof(AttachmentDetails.ContentType)); writer.WriteString(result.ContentType); writer.WriteComma(); writer.WritePropertyName(nameof(AttachmentDetails.Hash)); writer.WriteString(result.Hash); writer.WriteComma(); writer.WritePropertyName(nameof(AttachmentDetails.Size)); writer.WriteInteger(result.Size); writer.WriteEndObject(); } } }
public async Task Patch() { var id = GetQueryStringValueAndAssertIfSingleAndNotEmpty("id"); var isTest = GetBoolValueQueryString("test", required: false) ?? false; var debugMode = GetBoolValueQueryString("debug", required: false) ?? isTest; var skipPatchIfChangeVectorMismatch = GetBoolValueQueryString("skipPatchIfChangeVectorMismatch", required: false) ?? false; using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var request = context.Read(RequestBodyStream(), "ScriptedPatchRequest"); if (request.TryGet("Patch", out BlittableJsonReaderObject patchCmd) == false || patchCmd == null) { throw new ArgumentException("The 'Patch' field in the body request is mandatory"); } var patch = PatchRequest.Parse(patchCmd, out var patchArgs); PatchRequest patchIfMissing = null; BlittableJsonReaderObject patchIfMissingArgs = null; if (request.TryGet("PatchIfMissing", out BlittableJsonReaderObject patchIfMissingCmd) && patchIfMissingCmd != null) { patchIfMissing = PatchRequest.Parse(patchIfMissingCmd, out patchIfMissingArgs); } var changeVector = context.GetLazyString(GetStringFromHeaders("If-Match")); using (var command = new PatchDocumentCommand(context, id, changeVector, skipPatchIfChangeVectorMismatch, (patch, patchArgs), (patchIfMissing, patchIfMissingArgs), Database, isTest, debugMode, true )) { if (isTest == false) { await Database.TxMerger.Enqueue(command); } else { // PutDocument requires the write access to the docs storage // testing patching is rare enough not to optimize it using (context.OpenWriteTransaction()) { command.Execute(context); } } switch (command.PatchResult.Status) { case PatchStatus.DocumentDoesNotExist: HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound; return; case PatchStatus.Created: HttpContext.Response.StatusCode = (int)HttpStatusCode.Created; break; case PatchStatus.Skipped: HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified; return; case PatchStatus.Patched: case PatchStatus.NotModified: HttpContext.Response.StatusCode = (int)HttpStatusCode.OK; break; default: throw new ArgumentOutOfRangeException(); } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName(nameof(command.PatchResult.Status)); writer.WriteString(command.PatchResult.Status.ToString()); writer.WriteComma(); writer.WritePropertyName(nameof(command.PatchResult.ModifiedDocument)); writer.WriteObject(command.PatchResult.ModifiedDocument); if (debugMode) { writer.WriteComma(); writer.WritePropertyName(nameof(command.PatchResult.OriginalDocument)); if (isTest) { writer.WriteObject(command.PatchResult.OriginalDocument); } else { writer.WriteNull(); } writer.WriteComma(); writer.WritePropertyName(nameof(command.PatchResult.Debug)); context.Write(writer, new DynamicJsonValue { ["Info"] = new DynamicJsonArray(command.DebugOutput), ["Actions"] = command.DebugActions?.GetDebugActions() }); } writer.WriteEndObject(); } } } }
public Task ExtractInfoFromZip() { AssertOnlyInSetupMode(); using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) using (var continueSetupInfoJson = context.ReadForMemory(RequestBodyStream(), "continue-setup-info")) { var continueSetupInfo = JsonDeserializationServer.ContinueSetupInfo(continueSetupInfoJson); byte[] zipBytes; try { zipBytes = Convert.FromBase64String(continueSetupInfo.Zip); } catch (Exception e) { throw new ArgumentException($"Unable to parse the {nameof(continueSetupInfo.Zip)} property, expected a Base64 value", e); } try { var urlByTag = new Dictionary <string, string>(); using (var ms = new MemoryStream(zipBytes)) using (var archive = new ZipArchive(ms, ZipArchiveMode.Read, false)) { foreach (var entry in archive.Entries) { if (entry.Name.Equals("settings.json") == false) { continue; } var tag = entry.FullName[0].ToString(); using (var settingsJson = context.ReadForMemory(entry.Open(), "settings-json")) if (settingsJson.TryGet(nameof(ConfigurationNodeInfo.PublicServerUrl), out string publicServerUrl)) { urlByTag[tag] = publicServerUrl; } } } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartArray(); var first = true; foreach (var node in urlByTag) { if (first == false) { writer.WriteComma(); } writer.WriteStartObject(); writer.WritePropertyName(nameof(ConfigurationNodeInfo.Tag)); writer.WriteString(node.Key); writer.WriteComma(); writer.WritePropertyName(nameof(ConfigurationNodeInfo.PublicServerUrl)); writer.WriteString(node.Value); writer.WriteEndObject(); first = false; } writer.WriteEndArray(); } } catch (Exception e) { throw new InvalidOperationException("Unable to extract setup information from the zip file.", e); } } return(Task.CompletedTask); }
public Task GetCollectionFields() { var collection = GetStringQueryString("collection", required: false); var prefix = GetStringQueryString("prefix", required: false); using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.OpenReadTransaction()) { long totalResults; string changeVector; string etag = null; if (string.IsNullOrEmpty(collection)) { changeVector = DocumentsStorage.GetDatabaseChangeVector(context); totalResults = Database.DocumentsStorage.GetNumberOfDocuments(context); etag = $"{changeVector}/{totalResults}"; } else { changeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context, collection); totalResults = Database.DocumentsStorage.GetCollection(collection, context).Count; if (changeVector != null) { etag = $"{changeVector}/{totalResults}"; } } if (etag != null && GetStringFromHeaders("If-None-Match") == etag) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified; return(Task.CompletedTask); } HttpContext.Response.Headers["ETag"] = "\"" + etag + "\""; if (_buffers == null) { _buffers = new BlittableJsonReaderObject.PropertiesInsertionBuffer(); } var fields = new Dictionary <LazyStringValue, FieldType>(LazyStringValueComparer.Instance); if (string.IsNullOrEmpty(collection)) { foreach (var collectionStats in Database.DocumentsStorage.GetCollections(context)) { FetchFieldsForCollection(context, collectionStats.Name, prefix, fields, _buffers); } } else { FetchFieldsForCollection(context, collection, prefix, fields, _buffers); } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); var first = true; foreach (var field in fields) { if (first == false) { writer.WriteComma(); } first = false; writer.WritePropertyName(field.Key); writer.WriteString(field.Value.ToString()); } writer.WriteEndObject(); } return(Task.CompletedTask); } }
public Task GetAll() { var start = GetStart(); var pageSize = GetPageSize(); var history = GetBoolValueQueryString("history", required: false) ?? false; var running = GetBoolValueQueryString("running", required: false) ?? false; var id = GetLongQueryString("id", required: false); var name = GetStringQueryString("name", required: false); using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (context.OpenReadTransaction()) { IEnumerable <SubscriptionStorage.SubscriptionGeneralDataAndStats> subscriptions; if (string.IsNullOrEmpty(name) && id == null) { subscriptions = running ? Database.SubscriptionStorage.GetAllRunningSubscriptions(context, history, start, pageSize) : Database.SubscriptionStorage.GetAllSubscriptions(context, history, start, pageSize); } else { var subscription = running ? Database .SubscriptionStorage .GetRunningSubscription(context, id, name, history) : Database .SubscriptionStorage .GetSubscription(context, id, name, history); if (subscription == null) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound; return(Task.CompletedTask); } subscriptions = new[] { subscription }; } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); var subscriptionsAsBlittable = subscriptions.Select(x => new DynamicJsonValue() { [nameof(SubscriptionState.SubscriptionId)] = x.SubscriptionId, [nameof(SubscriptionState.SubscriptionName)] = x.SubscriptionName, [nameof(SubscriptionState.ChangeVectorForNextBatchStartingPoint)] = x.ChangeVectorForNextBatchStartingPoint, [nameof(SubscriptionState.Query)] = x.Query, [nameof(SubscriptionState.Disabled)] = x.Disabled, [nameof(SubscriptionState.LastClientConnectionTime)] = x.LastClientConnectionTime, [nameof(SubscriptionState.LastBatchAckTime)] = x.LastBatchAckTime, ["Connection"] = GetSubscriptionConnectionDJV(x.Connection), ["RecentConnections"] = x.RecentConnections?.Select(r => new DynamicJsonValue() { ["State"] = new DynamicJsonValue() { ["LatestChangeVectorClientACKnowledged"] = r.SubscriptionState.ChangeVectorForNextBatchStartingPoint, ["Query"] = r.SubscriptionState.Query }, ["Connection"] = GetSubscriptionConnectionDJV(r) }), ["FailedConnections"] = x.RecentRejectedConnections?.Select(r => new DynamicJsonValue() { ["State"] = new DynamicJsonValue() { ["LatestChangeVectorClientACKnowledged"] = r.SubscriptionState.ChangeVectorForNextBatchStartingPoint, ["Query"] = r.SubscriptionState.Query }, ["Connection"] = GetSubscriptionConnectionDJV(r) }).ToList() }); writer.WriteArray(context, "Results", subscriptionsAsBlittable, (w, c, subscription) => { c.Write(w, subscription); }); writer.WriteEndObject(); } } return(Task.CompletedTask); }
public Task GetIps() { AssertOnlyInSetupMode(); NetworkInterface[] netInterfaces = null; try { netInterfaces = NetworkInterface.GetAllNetworkInterfaces(); } catch (Exception) { // https://github.com/dotnet/corefx/issues/26476 // If GetAllNetworkInterfaces is not supported, we'll just return the default: 127.0.0.1 } using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName("MachineName"); writer.WriteString(Environment.MachineName); writer.WriteComma(); writer.WritePropertyName("NetworkInterfaces"); writer.WriteStartArray(); var first = true; List <string> ips; if (netInterfaces != null) { foreach (var netInterface in netInterfaces) { ips = netInterface.GetIPProperties().UnicastAddresses .Where(x => { // filter 169.254.xxx.xxx out, they are not meaningful for binding if (x.Address.AddressFamily != AddressFamily.InterNetwork) { return(false); } var addressBytes = x.Address.GetAddressBytes(); // filter 127.xxx.xxx.xxx out, in docker only if (SetupParameters.Get(ServerStore).IsDocker&& addressBytes[0] == 127) { return(false); } return(addressBytes[0] != 169 || addressBytes[1] != 254); }) .Select(addr => addr.Address.ToString()) .ToList(); // If there's a hostname in the server url, add it to the list if (SetupParameters.Get(ServerStore).DockerHostname != null && ips.Contains(SetupParameters.Get(ServerStore).DockerHostname) == false) { ips.Add(SetupParameters.Get(ServerStore).DockerHostname); } if (first == false) { writer.WriteComma(); } first = false; writer.WriteStartObject(); writer.WritePropertyName("Name"); writer.WriteString(netInterface.Name); writer.WriteComma(); writer.WritePropertyName("Description"); writer.WriteString(netInterface.Description); writer.WriteComma(); writer.WriteArray("Addresses", ips); writer.WriteEndObject(); } } else { // https://github.com/dotnet/corefx/issues/26476 // If GetAllNetworkInterfaces is not supported, we'll just return the default: 127.0.0.1 ips = new List <string> { "127.0.0.1" }; writer.WriteStartObject(); writer.WritePropertyName("Name"); writer.WriteString("Loopback Interface"); writer.WriteComma(); writer.WritePropertyName("Description"); writer.WriteString("Loopback Interface"); writer.WriteComma(); writer.WriteArray("Addresses", ips); writer.WriteEndObject(); } writer.WriteEndArray(); writer.WriteEndObject(); } return(Task.CompletedTask); }
private static void WriteMemoryStats(BlittableJsonTextWriter writer, JsonOperationContext context, bool includeThreads, bool includeMappings) { writer.WriteStartObject(); var memInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader(); long managedMemoryInBytes = AbstractLowMemoryMonitor.GetManagedMemoryInBytes(); long totalUnmanagedAllocations = NativeMemory.TotalAllocatedMemory; var encryptionBuffers = EncryptionBuffersPool.Instance.GetStats(); var dirtyMemoryState = MemoryInformation.GetDirtyMemoryState(); var memoryUsageRecords = MemoryInformation.GetMemoryUsageRecords(); long totalMapping = 0; var fileMappingByDir = new Dictionary <string, Dictionary <string, ConcurrentDictionary <IntPtr, long> > >(); var fileMappingSizesByDir = new Dictionary <string, long>(); foreach (var mapping in NativeMemory.FileMapping) { var dir = Path.GetDirectoryName(mapping.Key); if (fileMappingByDir.TryGetValue(dir, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value) == false) { value = new Dictionary <string, ConcurrentDictionary <IntPtr, long> >(); fileMappingByDir[dir] = value; } value[mapping.Key] = mapping.Value.Value.Info; foreach (var singleMapping in mapping.Value.Value.Info) { fileMappingSizesByDir.TryGetValue(dir, out long prevSize); fileMappingSizesByDir[dir] = prevSize + singleMapping.Value; totalMapping += singleMapping.Value; } } var djv = new DynamicJsonValue { [nameof(MemoryInfo.PhysicalMemory)] = memInfo.TotalPhysicalMemory.ToString(), [nameof(MemoryInfo.WorkingSet)] = memInfo.WorkingSet.ToString(), [nameof(MemoryInfo.ManagedAllocations)] = Size.Humane(managedMemoryInBytes), [nameof(MemoryInfo.UnmanagedAllocations)] = Size.Humane(totalUnmanagedAllocations), [nameof(MemoryInfo.EncryptionBuffersInUse)] = Size.Humane(encryptionBuffers.CurrentlyInUseSize), [nameof(MemoryInfo.EncryptionBuffersPool)] = Size.Humane(encryptionBuffers.TotalPoolSize), [nameof(MemoryInfo.EncryptionLockedMemory)] = Size.Humane(Sodium.LockedBytes), [nameof(MemoryInfo.MemoryMapped)] = Size.Humane(totalMapping), [nameof(MemoryInfo.ScratchDirtyMemory)] = memInfo.TotalScratchDirtyMemory.ToString(), [nameof(MemoryInfo.IsHighDirty)] = dirtyMemoryState.IsHighDirty, [nameof(MemoryInfo.DirtyMemory)] = Size.Humane(dirtyMemoryState.TotalDirtyInBytes), [nameof(MemoryInfo.AvailableMemory)] = Size.Humane(memInfo.AvailableMemory.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.AvailableMemoryForProcessing)] = memInfo.AvailableMemoryForProcessing.ToString(), [nameof(MemoryInfo.HighMemLastOneMinute)] = memoryUsageRecords.High.LastOneMinute.ToString(), [nameof(MemoryInfo.LowMemLastOneMinute)] = memoryUsageRecords.Low.LastOneMinute.ToString(), [nameof(MemoryInfo.HighMemLastFiveMinute)] = memoryUsageRecords.High.LastFiveMinutes.ToString(), [nameof(MemoryInfo.LowMemLastFiveMinute)] = memoryUsageRecords.Low.LastFiveMinutes.ToString(), [nameof(MemoryInfo.HighMemSinceStartup)] = memoryUsageRecords.High.SinceStartup.ToString(), [nameof(MemoryInfo.LowMemSinceStartup)] = memoryUsageRecords.Low.SinceStartup.ToString(), }; writer.WritePropertyName(nameof(MemoryInformation)); context.Write(writer, djv); writer.WriteComma(); writer.WritePropertyName("Threads"); writer.WriteStartArray(); WriteThreads(includeThreads, writer, context); writer.WriteEndArray(); writer.WriteComma(); writer.WritePropertyName(nameof(MemoryInfo.Mappings)); writer.WriteStartArray(); WriteMappings(includeMappings, writer, context, fileMappingSizesByDir, fileMappingByDir); writer.WriteEndArray(); writer.WriteEndObject(); }
public Task QueriesCacheList() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { var queryCache = Database.QueryMetadataCache.GetQueryCache(); var queriesList = new List <DynamicJsonValue>(); writer.WriteStartObject(); writer.WritePropertyName("TotalCachedQueries"); writer.WriteInteger(queryCache.Length); writer.WriteComma(); foreach (var item in queryCache) { if (item != null) { var curDjvItem = new DynamicJsonValue(); queriesList.Add(curDjvItem); curDjvItem[nameof(Queries.QueryMetadata.CreatedAt)] = item.CreatedAt; curDjvItem[nameof(Queries.QueryMetadata.LastQueriedAt)] = item.LastQueriedAt; if (item.IsGroupBy) { curDjvItem[nameof(Queries.QueryMetadata.IsGroupBy)] = true; } if (item.IsDistinct) { curDjvItem[nameof(Queries.QueryMetadata.IsDistinct)] = true; } if (item.HasFacet) { curDjvItem[nameof(Queries.QueryMetadata.HasFacet)] = true; } if (item.HasMoreLikeThis) { curDjvItem[nameof(Queries.QueryMetadata.HasMoreLikeThis)] = true; } if (item.HasSuggest) { curDjvItem[nameof(Queries.QueryMetadata.HasSuggest)] = true; } if (item.OrderBy != null) { curDjvItem["Sorted"] = true; } if (item.IsOptimizedSortOnly) { curDjvItem[nameof(Queries.QueryMetadata.IsOptimizedSortOnly)] = true; } if (item.HasCmpXchg) { curDjvItem[nameof(Queries.QueryMetadata.HasCmpXchg)] = true; } if (item.HasExplanations) { curDjvItem[nameof(Queries.QueryMetadata.HasExplanations)] = true; } if (item.HasIntersect) { curDjvItem[nameof(Queries.QueryMetadata.HasIntersect)] = true; } if (item.IsDynamic) { curDjvItem[nameof(Queries.QueryMetadata.IsDynamic)] = true; } if (item.SelectFields.Any(x => x.Function != null)) { curDjvItem["IsJSProjection"] = true; } if (string.IsNullOrEmpty(item.CollectionName) == false) { curDjvItem[nameof(Queries.QueryMetadata.CollectionName)] = item.CollectionName; } if (string.IsNullOrEmpty(item.AutoIndexName) == false) { curDjvItem[nameof(Queries.QueryMetadata.AutoIndexName)] = item.AutoIndexName; } if (string.IsNullOrEmpty(item.IndexName) == false) { curDjvItem[nameof(Queries.QueryMetadata.IndexName)] = item.IndexName; } curDjvItem[nameof(Queries.QueryMetadata.QueryText)] = item.QueryText; } } writer.WriteArray("Results", queriesList, context); writer.WriteEndObject(); } return(Task.CompletedTask); }
public Task StreamDocsGet() { var transformerName = GetStringQueryString("transformer", required: false); Transformer transformer = null; if (string.IsNullOrEmpty(transformerName) == false) { transformer = Database.TransformerStore.GetTransformer(transformerName); if (transformer == null) { throw new TransformerDoesNotExistsException("No transformer with the name: " + transformerName); } } DocumentsOperationContext context; using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out context)) using (context.OpenReadTransaction()) { var etag = GetLongQueryString("etag", false); IEnumerable <Document> documents; if (etag != null) { documents = Database.DocumentsStorage.GetDocumentsFrom(context, etag.Value, GetStart(), GetPageSize()); } else if (HttpContext.Request.Query.ContainsKey("startsWith")) { documents = Database.DocumentsStorage.GetDocumentsStartingWith(context, HttpContext.Request.Query["startsWith"], HttpContext.Request.Query["matches"], HttpContext.Request.Query["excludes"], GetStart(), GetPageSize() ); } else // recent docs { documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, GetStart(), GetPageSize()); } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName("Results"); if (transformer != null) { var transformerParameters = GetTransformerParameters(context); using (var scope = transformer.OpenTransformationScope(transformerParameters, null, Database.DocumentsStorage, Database.TransformerStore, context)) { writer.WriteDocuments(context, scope.Transform(documents).ToList(), metadataOnly: false); } } else { writer.WriteDocuments(context, documents, metadataOnly: false); } writer.WriteEndObject(); } } return(Task.CompletedTask); }
public void FullBackupTo(string backupPath) { using (var file = new FileStream(backupPath, FileMode.Create)) using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true)) using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (context.OpenReadTransaction()) { var databaseRecord = _serverStore.Cluster.ReadDatabase(context, Name); Debug.Assert(databaseRecord != null); var zipArchiveEntry = package.CreateEntry(RestoreSettings.SmugglerValuesFileName, CompressionLevel.Optimal); using (var zipStream = zipArchiveEntry.Open()) { var smugglerSource = new DatabaseSource(this, 0); using (DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) using (ctx.OpenReadTransaction()) { var smugglerDestination = new StreamDestination(zipStream, ctx, smugglerSource); var databaseSmugglerOptionsServerSide = new DatabaseSmugglerOptionsServerSide { OperateOnTypes = DatabaseItemType.CmpXchg | DatabaseItemType.Identities }; var smuggler = new DatabaseSmuggler(this, smugglerSource, smugglerDestination, this.Time, options: databaseSmugglerOptionsServerSide); smuggler.Execute(); } } zipArchiveEntry = package.CreateEntry(RestoreSettings.SettingsFileName, CompressionLevel.Optimal); using (var zipStream = zipArchiveEntry.Open()) using (var writer = new BlittableJsonTextWriter(context, zipStream)) { //TODO: encrypt this file using the MasterKey //http://issues.hibernatingrhinos.com/issue/RavenDB-7546 writer.WriteStartObject(); // save the database record writer.WritePropertyName(nameof(RestoreSettings.DatabaseRecord)); var databaseRecordBlittable = EntityToBlittable.ConvertEntityToBlittable(databaseRecord, DocumentConventions.Default, context); context.Write(writer, databaseRecordBlittable); // save the database values (subscriptions, periodic backups statuses, etl states...) writer.WriteComma(); writer.WritePropertyName(nameof(RestoreSettings.DatabaseValues)); writer.WriteStartObject(); var first = true; var prefix = Helpers.ClusterStateMachineValuesPrefix(Name); foreach (var keyValue in ClusterStateMachine.ReadValuesStartingWith(context, prefix)) { if (first == false) { writer.WriteComma(); } first = false; var key = keyValue.Key.ToString().Substring(prefix.Length); writer.WritePropertyName(key); context.Write(writer, keyValue.Value); } writer.WriteEndObject(); // end of values writer.WriteEndObject(); } BackupMethods.Full.ToFile(GetAllStoragesEnvironmentInformation(), package); file.Flush(true); // make sure that we fully flushed to disk } }
public Task GetIps() { AssertOnlyInSetupMode(); using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName("MachineName"); writer.WriteString(Environment.MachineName); writer.WriteComma(); writer.WritePropertyName("NetworkInterfaces"); writer.WriteStartArray(); var first = true; foreach (var netInterface in NetworkInterface.GetAllNetworkInterfaces()) { var ips = netInterface.GetIPProperties().UnicastAddresses .Where(x => { // filter 169.254.xxx.xxx out, they are not meaningful for binding if (x.Address.AddressFamily != AddressFamily.InterNetwork) { return(false); } var addressBytes = x.Address.GetAddressBytes(); // filter 127.xxx.xxx.xxx out, in docker only if (SetupParameters.Get(ServerStore).IsDocker&& addressBytes[0] == 127) { return(false); } return(addressBytes[0] != 169 || addressBytes[1] != 254); }) .Select(addr => addr.Address.ToString()) .ToList(); // If there's a hostname in the server url, add it to the list if (SetupParameters.Get(ServerStore).DockerHostname != null && ips.Contains(SetupParameters.Get(ServerStore).DockerHostname) == false) { ips.Add(SetupParameters.Get(ServerStore).DockerHostname); } if (first == false) { writer.WriteComma(); } first = false; writer.WriteStartObject(); writer.WritePropertyName("Name"); writer.WriteString(netInterface.Name); writer.WriteComma(); writer.WritePropertyName("Description"); writer.WriteString(netInterface.Description); writer.WriteComma(); writer.WriteArray("Addresses", ips); writer.WriteEndObject(); } writer.WriteEndArray(); writer.WriteEndObject(); } return(Task.CompletedTask); }
private async Task HandleRequestAsync( BlittableJsonReaderObject request, JsonOperationContext context, MultiGetHttpResponseStream responseStream, BlittableJsonTextWriter writer, HttpContext httpContext, HostString host, string scheme, LazyStringValue resultProperty, LazyStringValue statusProperty, LazyStringValue headersProperty, StringBuilder trafficWatchStringBuilder) { writer.WriteStartObject(); if (request.TryGet(nameof(GetRequest.Url), out string url) == false || request.TryGet(nameof(GetRequest.Query), out string query) == false) { writer.WriteEndObject(); return; } if (request.TryGet(nameof(GetRequest.Method), out string method) == false || string.IsNullOrEmpty(method)) { method = HttpMethod.Get.Method; } httpContext.Request.Method = method; var routeInformation = Server.Router.GetRoute(method, url, out RouteMatch localMatch); if (routeInformation == null) { HandleNoRoute(context, writer, method, url, query, statusProperty, resultProperty); return; } var requestHandler = routeInformation.GetRequestHandler(); writer.WritePropertyName(resultProperty); writer.Flush(); PrepareHttpContext(request, context, httpContext, method, query, host, scheme, trafficWatchStringBuilder, out var content); var bytesWrittenBeforeRequest = responseStream.BytesWritten; int statusCode; try { if (Server.Configuration.Security.AuthenticationEnabled == false || Server.Router.TryAuthorize(routeInformation, httpContext, Database, out var status)) { await requestHandler(new RequestHandlerContext { Database = Database, RavenServer = Server, RouteMatch = localMatch, HttpContext = httpContext }); } if (bytesWrittenBeforeRequest == responseStream.BytesWritten) { writer.WriteNull(); } statusCode = httpContext.Response.StatusCode == 0 ? (int)HttpStatusCode.OK : httpContext.Response.StatusCode; } catch (Exception e) { if (bytesWrittenBeforeRequest != responseStream.BytesWritten) { throw; } statusCode = (int)HttpStatusCode.InternalServerError; HandleException(context, writer, e, url, query); } writer.WriteComma(); writer.WritePropertyName(statusProperty); writer.WriteInteger(statusCode); writer.WriteComma(); WriteHeaders(writer, httpContext, headersProperty); writer.WriteEndObject(); trafficWatchStringBuilder?.Append(content).AppendLine(); }
public Task Patch() { var id = GetQueryStringValueAndAssertIfSingleAndNotEmpty("id"); var etag = GetLongFromHeaders("If-Match"); var isTestOnly = GetBoolValueQueryString("test", required: false) ?? false; DocumentsOperationContext context; using (ContextPool.AllocateOperationContext(out context)) { var request = context.Read(RequestBodyStream(), "ScriptedPatchRequest"); BlittableJsonReaderObject patchCmd, patchIsMissingCmd; if (request.TryGet("Patch", out patchCmd) == false) { throw new ArgumentException("The 'Patch' field in the body request is mandatory"); } var patch = PatchRequest.Parse(patchCmd); PatchRequest patchIfMissing = null; if (request.TryGet("PatchIfMissing", out patchIsMissingCmd)) { patchIfMissing = PatchRequest.Parse(patchCmd); } // TODO: In order to properly move this to the transaction merger, we need // TODO: move a lot of the costs (such as script parsing) out, so we create // TODO: an object that we'll apply, otherwise we'll slow down a lot the transactions // TODO: just by doing the javascript parsing and preparing the engine PatchResultData patchResult; using (context.OpenWriteTransaction()) { patchResult = Database.Patch.Apply(context, id, etag, patch, patchIfMissing, isTestOnly); context.Transaction.Commit(); } Debug.Assert(patchResult.PatchResult == PatchResult.Patched == isTestOnly == false); using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName(("Patched")); writer.WriteBool(isTestOnly == false); writer.WriteComma(); writer.WritePropertyName(("Debug")); writer.WriteObject(patchResult.ModifiedDocument); if (isTestOnly) { writer.WriteComma(); writer.WritePropertyName(("Document")); writer.WriteObject(patchResult.OriginalDocument); } writer.WriteEndObject(); } } return(Task.CompletedTask); }
public async Task TestJavaScriptIndex() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var input = await context.ReadForMemoryAsync(RequestBodyStream(), "TestJavaScriptIndex"); if (input.TryGet("Definition", out BlittableJsonReaderObject index) == false) { ThrowRequiredPropertyNameInRequest("Definition"); } input.TryGet("Ids", out BlittableJsonReaderArray ids); var indexDefinition = JsonDeserializationServer.IndexDefinition(index); if (indexDefinition.Maps == null || indexDefinition.Maps.Count == 0) { throw new ArgumentException("Index must have a 'Maps' fields"); } indexDefinition.Type = indexDefinition.DetectStaticIndexType(); if (indexDefinition.Type.IsJavaScript() == false) { throw new UnauthorizedAccessException("Testing indexes is only allowed for JavaScript indexes."); } var compiledIndex = new JavaScriptIndex(indexDefinition, Database.Configuration); var inputSize = GetIntValueQueryString("inputSize", false) ?? defaultInputSizeForTestingJavaScriptIndex; var collections = new HashSet <string>(compiledIndex.Maps.Keys); var docsPerCollection = new Dictionary <string, List <DynamicBlittableJson> >(); using (context.OpenReadTransaction()) { if (ids == null) { foreach (var collection in collections) { docsPerCollection.Add(collection, Database.DocumentsStorage.GetDocumentsFrom(context, collection, 0, 0, inputSize).Select(d => new DynamicBlittableJson(d)).ToList()); } } else { var listOfIds = ids.Select(x => x.ToString()); var _ = new Reference <int> { Value = 0 }; var docs = Database.DocumentsStorage.GetDocuments(context, listOfIds, 0, int.MaxValue, _); foreach (var doc in docs) { if (doc.TryGetMetadata(out var metadata) && metadata.TryGet(Constants.Documents.Metadata.Collection, out string collectionStr)) { if (docsPerCollection.TryGetValue(collectionStr, out var listOfDocs) == false) { listOfDocs = docsPerCollection[collectionStr] = new List <DynamicBlittableJson>(); } listOfDocs.Add(new DynamicBlittableJson(doc)); } } } var mapRes = new List <ObjectInstance>(); //all maps foreach (var ListOfFunctions in compiledIndex.Maps) { //multi maps per collection foreach (var mapFunc in ListOfFunctions.Value) { if (docsPerCollection.TryGetValue(ListOfFunctions.Key, out var docs)) { foreach (var res in mapFunc(docs)) { mapRes.Add((ObjectInstance)res); } } } } var first = true; using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName("MapResults"); writer.WriteStartArray(); foreach (var mapResult in mapRes) { if (JavaScriptIndexUtils.StringifyObject(mapResult) is JsString jsStr) { if (first == false) { writer.WriteComma(); } writer.WriteString(jsStr.ToString()); first = false; } } writer.WriteEndArray(); if (indexDefinition.Reduce != null) { using (var bufferPool = new UnmanagedBuffersPoolWithLowMemoryHandling("JavaScriptIndexTest", Database.Name)) { compiledIndex.SetBufferPoolForTestingPurposes(bufferPool); compiledIndex.SetAllocatorForTestingPurposes(context.Allocator); first = true; writer.WritePropertyName("ReduceResults"); writer.WriteStartArray(); var reduceResults = compiledIndex.Reduce(mapRes.Select(mr => new DynamicBlittableJson(JsBlittableBridge.Translate(context, mr.Engine, mr)))); foreach (JsValue reduceResult in reduceResults) { if (JavaScriptIndexUtils.StringifyObject(reduceResult) is JsString jsStr) { if (first == false) { writer.WriteComma(); } writer.WriteString(jsStr.ToString()); first = false; } } } writer.WriteEndArray(); } writer.WriteEndObject(); } } } }
public Task RunningQueries() { var indexes = Database .IndexStore .GetIndexes() .ToList(); using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); var isFirst = true; foreach (var index in indexes) { if (isFirst == false) { writer.WriteComma(); } isFirst = false; writer.WritePropertyName(index.Name); writer.WriteStartArray(); var isFirstInternal = true; foreach (var query in index.CurrentlyRunningQueries) { if (isFirstInternal == false) { writer.WriteComma(); } isFirstInternal = false; writer.WriteStartObject(); writer.WritePropertyName((nameof(query.Duration))); writer.WriteString(query.Duration.ToString()); writer.WriteComma(); writer.WritePropertyName((nameof(query.QueryId))); writer.WriteInteger(query.QueryId); writer.WriteComma(); writer.WritePropertyName((nameof(query.StartTime))); writer.WriteDateTime(query.StartTime, isUtc: true); writer.WriteComma(); writer.WritePropertyName((nameof(query.QueryInfo))); writer.WriteIndexQuery(context, query.QueryInfo); writer.WriteEndObject(); } writer.WriteEndArray(); } writer.WriteEndObject(); } return(Task.CompletedTask); }
public async Task Try() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { BlittableJsonReaderObject json = await context.ReadForMemoryAsync(RequestBodyStream(), null); SubscriptionTryout tryout = JsonDeserializationServer.SubscriptionTryout(json); SubscriptionConnection.ParsedSubscription sub = SubscriptionConnection.ParseSubscriptionQuery(tryout.Query); SubscriptionPatchDocument patch = null; if (string.IsNullOrEmpty(sub.Script) == false) { patch = new SubscriptionPatchDocument(sub.Script, sub.Functions); } if (sub.Collection == null) { throw new ArgumentException("Collection must be specified"); } const int maxPageSize = 1024; var pageSize = GetIntValueQueryString("pageSize") ?? 1; if (pageSize > maxPageSize) { throw new ArgumentException($"Cannot gather more than {maxPageSize} results during tryouts, but requested number was {pageSize}."); } var state = new SubscriptionState { ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector, Query = tryout.Query }; var fetcher = new SubscriptionDocumentsFetcher(Database, int.MaxValue, -0x42, new IPEndPoint(HttpContext.Connection.RemoteIpAddress, HttpContext.Connection.RemotePort), sub.Collection, sub.Revisions, state, patch); var includeCmd = new IncludeDocumentsCommand(Database.DocumentsStorage, context, sub.Includes, isProjection: patch != null); if (Enum.TryParse( tryout.ChangeVector, out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue)) { switch (changeVectorSpecialValue) { case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime: case Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange: state.ChangeVectorForNextBatchStartingPoint = null; break; case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument: using (context.OpenReadTransaction()) { state.ChangeVectorForNextBatchStartingPoint = Database.DocumentsStorage.GetLastDocumentChangeVector(context, sub.Collection); } break; } } else { state.ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector; } var changeVector = state.ChangeVectorForNextBatchStartingPoint.ToChangeVector(); var cv = changeVector.FirstOrDefault(x => x.DbId == Database.DbBase64Id); var sp = Stopwatch.StartNew(); var timeLimit = TimeSpan.FromSeconds(GetIntValueQueryString("timeLimit", false) ?? 15); var startEtag = cv.Etag; using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) using (context.OpenReadTransaction()) { writer.WriteStartObject(); writer.WritePropertyName("Results"); writer.WriteStartArray(); var numberOfDocs = 0; while (numberOfDocs == 0 && sp.Elapsed < timeLimit) { var first = true; var lastEtag = startEtag; foreach (var itemDetails in fetcher.GetDataToSend(context, includeCmd, startEtag)) { if (itemDetails.Doc.Data != null) { using (itemDetails.Doc.Data) { includeCmd.Gather(itemDetails.Doc); if (first == false) { writer.WriteComma(); } if (itemDetails.Exception == null) { writer.WriteDocument(context, itemDetails.Doc, metadataOnly: false); } else { var documentWithException = new DocumentWithException { Exception = itemDetails.Exception.ToString(), ChangeVector = itemDetails.Doc.ChangeVector, Id = itemDetails.Doc.Id, DocumentData = itemDetails.Doc.Data }; writer.WriteObject(context.ReadObject(documentWithException.ToJson(), "")); } first = false; if (++numberOfDocs >= pageSize) { break; } } } if (sp.Elapsed >= timeLimit) { break; } lastEtag = itemDetails.Doc.Etag; } if (startEtag == lastEtag) { break; } startEtag = lastEtag; } writer.WriteEndArray(); writer.WriteComma(); writer.WritePropertyName("Includes"); var includes = new List <Document>(); includeCmd.Fill(includes); writer.WriteIncludes(context, includes); writer.WriteEndObject(); } } }
private void WriteDocument(BlittableJsonTextWriter writer, JsonOperationContext context, Document document, HashSet <string> propertiesPreviewToSend, HashSet <string> fullPropertiesToSend) { writer.WriteStartObject(); document.Data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata); bool first = true; var arrayStubsJson = new DynamicJsonValue(); var objectStubsJson = new DynamicJsonValue(); var trimmedValue = new HashSet <LazyStringValue>(); var prop = new BlittableJsonReaderObject.PropertyDetails(); using (var buffers = document.Data.GetPropertiesByInsertionOrder()) { for (int i = 0; i < buffers.Properties.Count; i++) { document.Data.GetPropertyByIndex(buffers.Properties.Array[i + buffers.Properties.Offset], ref prop); var sendFull = fullPropertiesToSend.Contains(prop.Name); if (sendFull || propertiesPreviewToSend.Contains(prop.Name)) { var strategy = sendFull ? ValueWriteStrategy.Passthrough : FindWriteStrategy(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value); if (strategy == ValueWriteStrategy.Passthrough || strategy == ValueWriteStrategy.Trim) { if (first == false) { writer.WriteComma(); } first = false; } switch (strategy) { case ValueWriteStrategy.Passthrough: writer.WritePropertyName(prop.Name); writer.WriteValue(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value); break; case ValueWriteStrategy.SubstituteWithArrayStub: arrayStubsJson[prop.Name] = ((BlittableJsonReaderArray)prop.Value).Length; break; case ValueWriteStrategy.SubstituteWithObjectStub: objectStubsJson[prop.Name] = ((BlittableJsonReaderObject)prop.Value).Count; break; case ValueWriteStrategy.Trim: writer.WritePropertyName(prop.Name); WriteTrimmedValue(writer, prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value); trimmedValue.Add(prop.Name); break; } } } } if (first == false) { writer.WriteComma(); } var extraMetadataProperties = new DynamicJsonValue { [ObjectStubsKey] = objectStubsJson, [ArrayStubsKey] = arrayStubsJson, [TrimmedValueKey] = new DynamicJsonArray(trimmedValue) }; if (metadata != null) { metadata.Modifications = extraMetadataProperties; metadata = context.ReadObject(metadata, document.Id); } else { metadata = context.ReadObject(extraMetadataProperties, document.Id); } writer.WriteMetadata(document, metadata); writer.WriteEndObject(); }
public Task CommitNonLazyTx() { var modeStr = GetQueryStringValueAndAssertIfSingleAndNotEmpty("mode"); TransactionsMode mode; if (Enum.TryParse(modeStr, true, out mode) == false) { throw new InvalidOperationException("Query string value 'mode' is not a valid mode: " + modeStr); } var configDuration = TimeSpan.FromMinutes(Database.Configuration.Storage.TransactionsModeDuration); var duration = GetTimeSpanQueryString("duration", required: false) ?? configDuration; JsonOperationContext context; using (ContextPool.AllocateOperationContext(out context)) using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName(("Environments")); writer.WriteStartArray(); bool first = true; foreach (var storageEnvironment in Database.GetAllStoragesEnvironment()) { if (storageEnvironment == null) { continue; } if (first == false) { writer.WriteComma(); } first = false; var result = storageEnvironment.Environment.SetTransactionMode(mode, duration); switch (result) { case TransactionsModeResult.ModeAlreadySet: context.Write(writer, new DynamicJsonValue { ["Type"] = mode.ToString(), ["Path"] = storageEnvironment.Environment.Options.BasePath, ["Result"] = "Mode Already Set" }); break; case TransactionsModeResult.SetModeSuccessfully: context.Write(writer, new DynamicJsonValue { ["Type"] = mode.ToString(), ["Path"] = storageEnvironment.Environment.Options.BasePath, ["Result"] = "Mode Set Successfully" }); break; default: throw new ArgumentOutOfRangeException("Result is unexpected value: " + result); } } writer.WriteEndArray(); writer.WriteEndObject(); } return(Task.CompletedTask); }
public Task PreviewCollection() { var start = GetStart(); var pageSize = GetPageSize(); var collection = GetStringQueryString("collection", required: false); var bindings = GetStringValuesQueryString("binding", required: false); var fullBindings = GetStringValuesQueryString("fullBinding", required: false); using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.OpenReadTransaction()) { Document[] documents; HashSet <LazyStringValue> availableColumns; HashSet <string> propertiesPreviewToSend; HashSet <string> fullPropertiesToSend = new HashSet <string>(fullBindings); long totalResults; string changeVector; string etag = null; if (string.IsNullOrEmpty(collection)) { changeVector = DocumentsStorage.GetDatabaseChangeVector(context); totalResults = Database.DocumentsStorage.GetNumberOfDocuments(context); etag = $"{changeVector}/{totalResults}"; } else { changeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context, collection); totalResults = Database.DocumentsStorage.GetCollection(collection, context).Count; if (changeVector != null) { etag = $"{changeVector}/{totalResults}"; } } if (etag != null && GetStringFromHeaders("If-None-Match") == etag) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified; return(Task.CompletedTask); } HttpContext.Response.Headers["ETag"] = "\"" + etag + "\""; if (string.IsNullOrEmpty(collection)) { documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, start, pageSize).ToArray(); availableColumns = ExtractColumnNames(documents, context); propertiesPreviewToSend = bindings.Count > 0 ? new HashSet <string>(bindings) : new HashSet <string>(); } else { documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, collection, start, pageSize).ToArray(); availableColumns = ExtractColumnNames(documents, context); propertiesPreviewToSend = bindings.Count > 0 ? new HashSet <string>(bindings) : availableColumns.Take(ColumnsSamplingLimit).Select(x => x.ToString()).ToHashSet(); } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName("Results"); writer.WriteStartArray(); var first = true; foreach (var document in documents) { if (first == false) { writer.WriteComma(); } first = false; using (document.Data) { WriteDocument(writer, context, document, propertiesPreviewToSend, fullPropertiesToSend); } } writer.WriteEndArray(); writer.WriteComma(); writer.WritePropertyName("TotalResults"); writer.WriteInteger(totalResults); writer.WriteComma(); writer.WriteArray("AvailableColumns", availableColumns); writer.WriteEndObject(); } return(Task.CompletedTask); } }
public Task GetHosts() { AssertOnlyInSetupMode(); using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) using (var certificateJson = context.ReadForMemory(RequestBodyStream(), "setup-certificate")) { var certDef = JsonDeserializationServer.CertificateDefinition(certificateJson); X509Certificate2 certificate = null; string cn; try { certificate = certDef.Password == null ? new X509Certificate2(Convert.FromBase64String(certDef.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet) : new X509Certificate2(Convert.FromBase64String(certDef.Certificate), certDef.Password, X509KeyStorageFlags.MachineKeySet); cn = certificate.GetNameInfo(X509NameType.SimpleName, false); } catch (Exception e) { throw new BadRequestException($"Failed to extract the CN property from the certificate {certificate?.FriendlyName}. Maybe the password is wrong?", e); } if (cn == null) { throw new BadRequestException($"Failed to extract the CN property from the certificate. CN is null"); } if (cn.LastIndexOf('*') > 0) { throw new NotSupportedException("The wildcard CN name contains a '*' which is not at the first character of the string. It is not supported in the Setup Wizard, you can do a manual setup instead."); } try { SecretProtection.ValidateKeyUsages("Setup Wizard", certificate); } catch (Exception e) { throw new InvalidOperationException($"Failed to load the uploaded certificate. Did you accidently upload a client certificate?", e); } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName("CN"); writer.WriteString(cn); writer.WriteComma(); writer.WritePropertyName("AlternativeNames"); writer.WriteStartArray(); var first = true; foreach (var value in SetupManager.GetCertificateAlternativeNames(certificate)) { if (first == false) { writer.WriteComma(); } first = false; writer.WriteString(value); } writer.WriteEndArray(); writer.WriteEndObject(); } } return(Task.CompletedTask); }
public static void WriteIndexQuery(this BlittableJsonTextWriter writer, DocumentConventions conventions, JsonOperationContext context, IndexQuery query) { writer.WriteStartObject(); writer.WritePropertyName(nameof(query.Query)); writer.WriteString(query.Query); writer.WriteComma(); #pragma warning disable 618 if (query.PageSizeSet && query.PageSize >= 0) { writer.WritePropertyName(nameof(query.PageSize)); writer.WriteInteger(query.PageSize); writer.WriteComma(); } #pragma warning restore 618 if (query.WaitForNonStaleResults) { writer.WritePropertyName(nameof(query.WaitForNonStaleResults)); writer.WriteBool(query.WaitForNonStaleResults); writer.WriteComma(); } #pragma warning disable 618 if (query.Start > 0) { writer.WritePropertyName(nameof(query.Start)); writer.WriteInteger(query.Start); writer.WriteComma(); } #pragma warning restore 618 if (query.WaitForNonStaleResultsTimeout.HasValue) { writer.WritePropertyName(nameof(query.WaitForNonStaleResultsTimeout)); writer.WriteString(query.WaitForNonStaleResultsTimeout.Value.ToInvariantString()); writer.WriteComma(); } if (query.DisableCaching) { writer.WritePropertyName(nameof(query.DisableCaching)); writer.WriteBool(query.DisableCaching); writer.WriteComma(); } if (query.SkipDuplicateChecking) { writer.WritePropertyName(nameof(query.SkipDuplicateChecking)); writer.WriteBool(query.SkipDuplicateChecking); writer.WriteComma(); } writer.WritePropertyName(nameof(query.QueryParameters)); if (query.QueryParameters != null) { writer.WriteObject(conventions.Serialization.DefaultConverter.ToBlittable(query.QueryParameters, context)); } else { writer.WriteNull(); } if (query.ProjectionBehavior.HasValue && query.ProjectionBehavior.Value != ProjectionBehavior.Default) { writer.WriteComma(); writer.WritePropertyName(nameof(query.ProjectionBehavior)); writer.WriteString(query.ProjectionBehavior.ToString()); } writer.WriteEndObject(); }
public async Task PostMultiGet() { using (ContextPool.AllocateOperationContext(out JsonOperationContext context)) { var input = await context.ReadForMemoryAsync(RequestBodyStream(), "multi_get"); if (input.TryGet("Requests", out BlittableJsonReaderArray requests) == false) { ThrowRequiredPropertyNameInRequest("Requests"); } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName("Results"); writer.WriteStartArray(); var resultProperty = context.GetLazyStringForFieldWithCaching(nameof(GetResponse.Result)); var statusProperty = context.GetLazyStringForFieldWithCaching(nameof(GetResponse.StatusCode)); var headersProperty = context.GetLazyStringForFieldWithCaching(nameof(GetResponse.Headers)); var features = new FeatureCollection(HttpContext.Features); var responseStream = new MultiGetHttpResponseStream(ResponseBodyStream()); features.Set <IHttpResponseFeature>(new MultiGetHttpResponseFeature(responseStream)); var httpContext = new DefaultHttpContext(features); for (int i = 0; i < requests.Length; i++) { var request = (BlittableJsonReaderObject)requests[i]; if (i != 0) { writer.WriteComma(); } writer.WriteStartObject(); if (request.TryGet("Url", out string url) == false || request.TryGet("Query", out string query) == false) { writer.WriteEndObject(); continue; } if (request.TryGet("Method", out string method) == false || string.IsNullOrEmpty(method)) { method = HttpMethod.Get.Method; } httpContext.Request.Method = method; var routeInformation = Server.Router.GetRoute(method, url, out RouteMatch localMatch); if (routeInformation == null) { writer.WritePropertyName(statusProperty); writer.WriteInteger((int)HttpStatusCode.BadRequest); writer.WritePropertyName(resultProperty); context.Write(writer, new DynamicJsonValue { ["Error"] = $"There is no handler for path: {method} {url}{query}" }); writer.WriteEndObject(); continue; } var requestHandler = routeInformation.GetRequestHandler(); writer.WritePropertyName(resultProperty); writer.Flush(); httpContext.Response.StatusCode = 0; httpContext.Request.Headers.Clear(); httpContext.Response.Headers.Clear(); httpContext.Request.QueryString = new QueryString(query); if (request.TryGet("Headers", out BlittableJsonReaderObject headers)) { foreach (var header in headers.GetPropertyNames()) { if (headers.TryGet(header, out string value) == false) { continue; } if (string.IsNullOrWhiteSpace(value)) { continue; } httpContext.Request.Headers.Add(header, value); } } if (method == HttpMethod.Post.Method && request.TryGet("Content", out object content)) { if (content is LazyStringValue) { var requestBody = GetRequestBody(content.ToString()); HttpContext.Response.RegisterForDispose(requestBody); httpContext.Request.Body = requestBody; } else { var requestBody = new MemoryStream(); var contentWriter = new BlittableJsonTextWriter(context, requestBody); context.Write(contentWriter, (BlittableJsonReaderObject)content); contentWriter.Flush(); HttpContext.Response.RegisterForDispose(requestBody); httpContext.Request.Body = requestBody; httpContext.Request.Body.Position = 0; } } var bytesWrittenBeforeRequest = responseStream.BytesWritten; int statusCode; try { await requestHandler(new RequestHandlerContext { Database = Database, RavenServer = Server, RouteMatch = localMatch, HttpContext = httpContext }); if (bytesWrittenBeforeRequest == responseStream.BytesWritten) { writer.WriteNull(); } statusCode = httpContext.Response.StatusCode == 0 ? (int)HttpStatusCode.OK : httpContext.Response.StatusCode; } catch (Exception e) { if (bytesWrittenBeforeRequest != responseStream.BytesWritten) { throw; } statusCode = (int)HttpStatusCode.InternalServerError; var djv = new DynamicJsonValue { [nameof(ExceptionDispatcher.ExceptionSchema.Url)] = $"{url}{query}", [nameof(ExceptionDispatcher.ExceptionSchema.Type)] = e.GetType().FullName, [nameof(ExceptionDispatcher.ExceptionSchema.Message)] = e.Message, [nameof(ExceptionDispatcher.ExceptionSchema.Error)] = e.ToString() }; using (var json = context.ReadObject(djv, "exception")) writer.WriteObject(json); } writer.WriteComma(); writer.WritePropertyName(statusProperty); writer.WriteInteger(statusCode); writer.WriteComma(); writer.WritePropertyName(headersProperty); writer.WriteStartObject(); bool headerStart = true; foreach (var header in httpContext.Response.Headers) { foreach (var value in header.Value) { if (headerStart == false) { writer.WriteComma(); } headerStart = false; writer.WritePropertyName(header.Key); writer.WriteString(value); } } writer.WriteEndObject(); writer.WriteEndObject(); } writer.WriteEndArray(); writer.WriteEndObject(); } } }
public Task Stats() { var documentId = GetStringQueryString("docId"); using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.OpenReadTransaction()) { var document = Database.DocumentsStorage.Get(context, documentId, DocumentFields.Data); if (document == null) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound; return(Task.CompletedTask); } var timeSeriesNames = GetTimesSeriesNames(document); using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName(nameof(TimeSeriesStatistics.DocumentId)); writer.WriteString(documentId); writer.WriteComma(); writer.WritePropertyName(nameof(TimeSeriesStatistics.TimeSeries)); writer.WriteStartArray(); var first = true; foreach (var tsName in timeSeriesNames) { if (first == false) { writer.WriteComma(); } first = false; var stats = Database.DocumentsStorage.TimeSeriesStorage.Stats.GetStats(context, documentId, tsName); writer.WriteStartObject(); writer.WritePropertyName(nameof(TimeSeriesItemDetail.Name)); writer.WriteString(tsName); writer.WriteComma(); writer.WritePropertyName(nameof(TimeSeriesItemDetail.NumberOfEntries)); writer.WriteInteger(stats.Count); writer.WriteComma(); writer.WritePropertyName(nameof(TimeSeriesItemDetail.StartDate)); writer.WriteDateTime(stats.Start, isUtc: true); writer.WriteComma(); writer.WritePropertyName(nameof(TimeSeriesItemDetail.EndDate)); writer.WriteDateTime(stats.End, isUtc: true); writer.WriteEndObject(); } writer.WriteEndArray(); writer.WriteEndObject(); } } return(Task.CompletedTask); }
public Task GetCertificates() { var thumbprint = GetStringQueryString("thumbprint", required: false); var showSecondary = GetBoolValueQueryString("secondary", required: false) ?? false; var start = GetStart(); var pageSize = GetPageSize(); using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (context.OpenReadTransaction()) { var certificates = new List <(string ItemName, BlittableJsonReaderObject Value)>(); try { if (string.IsNullOrEmpty(thumbprint)) { if (ServerStore.CurrentRachisState == RachisState.Passive) { foreach (var localCertKey in ServerStore.Cluster.GetCertificateKeysFromLocalState(context)) { var localCertificate = ServerStore.Cluster.GetLocalState(context, localCertKey); if (localCertificate == null) { continue; } var def = JsonDeserializationServer.CertificateDefinition(localCertificate); if (showSecondary || string.IsNullOrEmpty(def.CollectionPrimaryKey)) { certificates.Add((localCertKey, localCertificate)); } else { localCertificate.Dispose(); } } } else { foreach (var item in ServerStore.Cluster.ItemsStartingWith(context, Constants.Certificates.Prefix, start, pageSize)) { var def = JsonDeserializationServer.CertificateDefinition(item.Value); if (showSecondary || string.IsNullOrEmpty(def.CollectionPrimaryKey)) { certificates.Add(item); } else { item.Value.Dispose(); } } } } else { var key = Constants.Certificates.Prefix + thumbprint; var certificate = ServerStore.CurrentRachisState == RachisState.Passive ? ServerStore.Cluster.GetLocalState(context, key) : ServerStore.Cluster.Read(context, key); if (certificate == null) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound; return(Task.CompletedTask); } var definition = JsonDeserializationServer.CertificateDefinition(certificate); if (string.IsNullOrEmpty(definition.CollectionPrimaryKey) == false) { certificate = ServerStore.Cluster.Read(context, definition.CollectionPrimaryKey); if (certificate == null) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound; return(Task.CompletedTask); } } certificates.Add((key, certificate)); } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WriteArray(context, "Results", certificates.ToArray(), (w, c, cert) => { c.Write(w, cert.Value); }); writer.WriteEndObject(); } } finally { foreach (var cert in certificates) { cert.Value?.Dispose(); } } } return(Task.CompletedTask); }
public RecoveryStatus Execute(CancellationToken ct) { var sw = new Stopwatch(); StorageEnvironment se = null; sw.Start(); if (_copyOnWrite) { Console.WriteLine("Recovering journal files, this may take a while..."); try { se = new StorageEnvironment(_option); Console.WriteLine( $"Journal recovery has completed successfully within {sw.Elapsed.TotalSeconds:N1} seconds"); } catch (Exception e) { Console.WriteLine($"Journal recovery failed, reason:{Environment.NewLine}{e}"); } finally { se?.Dispose(); } } _option = StorageEnvironmentOptions.ForPath(Path.GetDirectoryName(_datafile)); var mem = Pager.AcquirePagePointer(null, 0); long startOffset = (long)mem; var fi = new FileInfo(_datafile); var fileSize = fi.Length; //making sure eof is page aligned var eof = mem + (fileSize / _pageSize) * _pageSize; DateTime lastProgressReport = DateTime.MinValue; using (var destinationStream = File.OpenWrite(_output)) using (var logFile = File.CreateText(Path.Combine(Path.GetDirectoryName(_output), LogFileName))) using (var gZipStream = new GZipStream(destinationStream, CompressionMode.Compress, true)) using (var context = new JsonOperationContext(_initialContextSize, _initialContextLongLivedSize)) using (var writer = new BlittableJsonTextWriter(context, gZipStream)) { WriteSmugglerHeader(writer); while (mem < eof) { try { if (ct.IsCancellationRequested) { logFile.WriteLine( $"Cancellation requested while recovery was in position {GetFilePosition(startOffset, mem)}"); _cancellationRequested = true; break; } var now = DateTime.UtcNow; if ((now - lastProgressReport).TotalSeconds >= _progressIntervalInSeconds) { if (lastProgressReport != DateTime.MinValue) { Console.Clear(); Console.WriteLine("Press 'q' to quit the recovery process"); } lastProgressReport = now; var currPos = GetFilePosition(startOffset, mem); var eofPos = GetFilePosition(startOffset, eof); Console.WriteLine( $"{now:hh:MM:ss}: Recovering page at position {currPos:#,#;;0}/{eofPos:#,#;;0} ({(double) currPos/eofPos:p}) - Last recovered doc is {_lastRecoveredDocumentKey}"); } var pageHeader = (PageHeader *)mem; //this page is not raw data section move on if ((pageHeader->Flags).HasFlag(PageFlags.RawData) == false) { mem += _pageSize; continue; } if (pageHeader->Flags.HasFlag(PageFlags.Single) && pageHeader->Flags.HasFlag(PageFlags.Overflow)) { var message = $"page #{pageHeader->PageNumber} (offset={GetFilePosition(startOffset, mem)}) has both Overflow and Single flag turned"; mem = PrintErrorAndAdvanceMem(message, mem, logFile); continue; } //overflow page if (pageHeader->Flags.HasFlag(PageFlags.Overflow)) { var endOfOverflow = pageHeader + Pager.GetNumberOfOverflowPages(pageHeader->OverflowSize) * _pageSize; // the endOfOeverFlow can be equal to eof if the last page is overflow if (endOfOverflow > eof) { var message = $"Overflow page #{pageHeader->PageNumber} (offset={GetFilePosition(startOffset, mem)})" + $" size exceeds the end of the file ([{(long) pageHeader}:{(long) endOfOverflow}])"; mem = PrintErrorAndAdvanceMem(message, mem, logFile); continue; } //Should we increase the check size to page size (0=>_pageSize)? if (pageHeader->OverflowSize <= 0) { var message = $"Overflow page #{pageHeader->PageNumber} (offset={GetFilePosition(startOffset, mem)})" + $" OverflowSize is not a positive number ({pageHeader->OverflowSize})"; mem = PrintErrorAndAdvanceMem(message, mem, logFile); continue; } if (WriteDocument((byte *)pageHeader + sizeof(PageHeader), pageHeader->OverflowSize, writer, logFile, context, startOffset)) { var numberOfPages = Pager.GetNumberOfOverflowPages(pageHeader->OverflowSize); mem += numberOfPages * _pageSize; } else //write document failed { mem += _pageSize; } continue; } // small raw data section var rawHeader = (RawDataSmallPageHeader *)mem; if (rawHeader->RawDataFlags.HasFlag(RawDataPageFlags.Header)) { mem += _pageSize; continue; } if (rawHeader->NextAllocation > _pageSize) { var message = $"RawDataSmallPage #{rawHeader->PageNumber} at {GetFilePosition(startOffset, mem)} next allocation is larger than {_pageSize} bytes"; mem = PrintErrorAndAdvanceMem(message, mem, logFile); continue; } for (var pos = sizeof(PageHeader); pos < rawHeader->NextAllocation;) { var debug = GetFilePosition(startOffset, mem); var currMem = mem + pos; var entry = (RawDataSection.RawDataEntrySizes *)currMem; //this indicates that the current entry is invalid because it is outside the size of a page if (pos > _pageSize) { var message = $"RawDataSmallPage #{rawHeader->PageNumber} has an invalid entry at {GetFilePosition(startOffset, currMem)}"; mem = PrintErrorAndAdvanceMem(message, mem, logFile); //we can't retrive entries past the invalid entry break; } //Allocated size of entry exceed the bound of the page next allocation if (entry->AllocatedSize + pos + sizeof(RawDataSection.RawDataEntrySizes) > rawHeader->NextAllocation) { var message = $"RawDataSmallPage #{rawHeader->PageNumber} has an invalid entry at {GetFilePosition(startOffset, currMem)}" + "the allocated entry exceed the bound of the page next allocation."; mem = PrintErrorAndAdvanceMem(message, mem, logFile); //we can't retrive entries past the invalid entry break; } if (entry->UsedSize > entry->AllocatedSize) { var message = $"RawDataSmallPage #{rawHeader->PageNumber} has an invalid entry at {GetFilePosition(startOffset, currMem)}" + "the size of the entry exceed the allocated size"; mem = PrintErrorAndAdvanceMem(message, mem, logFile); //we can't retrive entries past the invalid entry break; } pos += entry->AllocatedSize + sizeof(RawDataSection.RawDataEntrySizes); if (entry->AllocatedSize == 0 || entry->UsedSize == -1) { continue; } if ( WriteDocument(currMem + sizeof(RawDataSection.RawDataEntrySizes), entry->UsedSize, writer, logFile, context, startOffset) == false) { break; } } mem += _pageSize; } catch (Exception e) { var message = $"Unexpected exception at position {GetFilePosition(startOffset, mem)}:{Environment.NewLine} {e}"; mem = PrintErrorAndAdvanceMem(message, mem, logFile); } } writer.WriteEndArray(); writer.WriteEndObject(); logFile.WriteLine( $"Discovered a total of {_numberOfDocumentsRetrived:#,#;00} documents within {sw.Elapsed.TotalSeconds::#,#.#;;00} seconds."); logFile.WriteLine($"Discovered a total of {_numberOfFaultedPages::#,#;00} faulted pages."); } if (_cancellationRequested) { return(RecoveryStatus.CancellationRequested);
public Task Debug() { var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name"); var index = Database.IndexStore.GetIndex(name); if (index == null) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound; return(Task.CompletedTask); } var operation = GetStringQueryString("op"); using (ContextPool.AllocateOperationContext(out JsonOperationContext context)) using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { if (string.Equals(operation, "map-reduce-tree", StringComparison.OrdinalIgnoreCase)) { if (index.Type.IsMapReduce() == false) { HttpContext.Response.StatusCode = (int)HttpStatusCode.BadRequest; context.Write(writer, new DynamicJsonValue { ["Error"] = $"{index.Name} is not map-reduce index" }); return(Task.CompletedTask); } var docIds = GetStringValuesQueryString("docId", required: false); using (index.GetReduceTree(docIds.ToArray(), out IEnumerable <ReduceTree> trees)) { writer.WriteReduceTrees(trees); } return(Task.CompletedTask); } if (string.Equals(operation, "source-doc-ids", StringComparison.OrdinalIgnoreCase)) { using (index.GetIdentifiersOfMappedDocuments(GetStringQueryString("startsWith", required: false), GetStart(), GetPageSize(), out IEnumerable <string> ids)) { writer.WriteArrayOfResultsAndCount(ids); } return(Task.CompletedTask); } if (string.Equals(operation, "entries-fields", StringComparison.OrdinalIgnoreCase)) { var fields = index.GetEntriesFields(); writer.WriteStartObject(); writer.WriteArray("Results", fields); writer.WriteEndObject(); return(Task.CompletedTask); } throw new NotSupportedException($"{operation} is not supported"); } }
public Task Stats() { var name = GetStringQueryString("name", required: false); using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { IndexStats[] indexStats; using (context.OpenReadTransaction()) { if (string.IsNullOrEmpty(name)) { indexStats = Database.IndexStore .GetIndexes() .OrderBy(x => x.Name) .Select(x => { try { return(x.GetStats(calculateLag: true, calculateStaleness: true, documentsContext: context)); } catch (Exception e) { if (Logger.IsOperationsEnabled) { Logger.Operations($"Failed to get stats of '{x.Name}' index", e); } try { Database.NotificationCenter.Add(AlertRaised.Create(Database.Name, $"Failed to get stats of '{x.Name}' index", $"Exception was thrown on getting stats of '{x.Name}' index", AlertType.Indexing_CouldNotGetStats, NotificationSeverity.Error, key: x.Name, details: new ExceptionDetails(e))); } catch (Exception addAlertException) { if (Logger.IsOperationsEnabled && addAlertException.IsOutOfMemory() == false && addAlertException.IsDiskFullException() == false) { Logger.Operations($"Failed to add alert when getting error on retrieving stats of '{x.Name}' index", addAlertException); } } var state = x.State; if (e.IsOutOfMemory() == false && e.IsDiskFullException() == false) { try { state = IndexState.Error; x.SetState(state, inMemoryOnly: true); } catch (Exception ex) { if (Logger.IsOperationsEnabled) { Logger.Operations($"Failed to change state of '{x.Name}' index to error after encountering exception when getting its stats.", ex); } } } return(new IndexStats { Name = x.Name, Type = x.Type, State = state, Status = x.Status, LockMode = x.Definition.LockMode, Priority = x.Definition.Priority, }); } }) .ToArray(); } else { var index = Database.IndexStore.GetIndex(name); if (index == null) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound; return(Task.CompletedTask); } indexStats = new[] { index.GetStats(calculateLag: true, calculateStaleness: true, documentsContext: context) }; } } writer.WriteStartObject(); writer.WriteArray(context, "Results", indexStats, (w, c, stats) => { w.WriteIndexStats(context, stats); }); writer.WriteEndObject(); } return(Task.CompletedTask); }
/// <summary> /// Iterates on a batch in document collection, process it and send documents if found any match /// </summary> /// <param name="docsContext"></param> /// <param name="sendingCurrentBatchStopwatch"></param> /// <returns>Whether succeeded finding any documents to send</returns> private async Task <bool> TrySendingBatchToClient(DocumentsOperationContext docsContext, Stopwatch sendingCurrentBatchStopwatch) { bool anyDocumentsSentInCurrentIteration = false; int docsToFlush = 0; using (var writer = new BlittableJsonTextWriter(docsContext, _buffer)) { using (docsContext.OpenReadTransaction()) { foreach (var result in _documentsFetcher.GetDataToSend(docsContext, _startEtag)) { _startEtag = result.Doc.Etag; _lastChangeVector = string.IsNullOrEmpty(SubscriptionState.ChangeVectorForNextBatchStartingPoint) ? result.Doc.ChangeVector : ChangeVectorUtils.MergeVectors(result.Doc.ChangeVector, SubscriptionState.ChangeVectorForNextBatchStartingPoint); if (result.Doc.Data == null) { if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000) { await SendHeartBeat(); sendingCurrentBatchStopwatch.Restart(); } continue; } anyDocumentsSentInCurrentIteration = true; writer.WriteStartObject(); writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(TypeSegment)); writer.WriteValue(BlittableJsonToken.String, docsContext.GetLazyStringForFieldWithCaching(DataSegment)); writer.WriteComma(); writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(DataSegment)); result.Doc.EnsureMetadata(); if (result.Exception != null) { var metadata = result.Doc.Data[Client.Constants.Documents.Metadata.Key]; writer.WriteValue(BlittableJsonToken.StartObject, docsContext.ReadObject(new DynamicJsonValue { [Client.Constants.Documents.Metadata.Key] = metadata }, result.Doc.Id) ); writer.WriteComma(); writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(ExceptionSegment)); writer.WriteValue(BlittableJsonToken.String, docsContext.GetLazyStringForFieldWithCaching(result.Exception.ToString())); } else { writer.WriteDocument(docsContext, result.Doc, metadataOnly: false); } writer.WriteEndObject(); docsToFlush++; // perform flush for current batch after 1000ms of running or 1 MB if (_buffer.Length > Constants.Size.Megabyte || sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000) { if (docsToFlush > 0) { await FlushDocsToClient(writer, docsToFlush); docsToFlush = 0; sendingCurrentBatchStopwatch.Restart(); } else { await SendHeartBeat(); } } } } if (anyDocumentsSentInCurrentIteration) { docsContext.Write(writer, new DynamicJsonValue { [nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.EndOfBatch) }); await FlushDocsToClient(writer, docsToFlush, true); if (_logger.IsInfoEnabled) { _logger.Info( $"Finished sending a batch with {docsToFlush} documents for subscription {Options.SubscriptionName}"); } } } return(anyDocumentsSentInCurrentIteration); }
public async Task Put() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var id = GetQueryStringValueAndAssertIfSingleAndNotEmpty("id"); var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name"); var contentType = GetStringQueryString("contentType", false) ?? ""; AttachmentDetails result; using (var streamsTempFile = Database.DocumentsStorage.AttachmentsStorage.GetTempFile("put")) using (var stream = streamsTempFile.StartNewStream()) { var hash = await AttachmentsStorageHelper.CopyStreamToFileAndCalculateHash(context, RequestBodyStream(), stream, Database.DatabaseShutdown); var changeVector = context.GetLazyString(GetStringQueryString("If-Match", false)); var cmd = new MergedPutAttachmentCommand { Database = Database, ExpectedChangeVector = changeVector, DocumentId = id, Name = name, Stream = stream, Hash = hash, ContentType = contentType }; await Database.TxMerger.Enqueue(cmd); cmd.ExceptionDispatchInfo?.Throw(); result = cmd.Result; } HttpContext.Response.StatusCode = (int)HttpStatusCode.Created; using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName(nameof(AttachmentDetails.ChangeVector)); writer.WriteString(result.ChangeVector); writer.WriteComma(); writer.WritePropertyName(nameof(AttachmentDetails.Name)); writer.WriteString(result.Name); writer.WriteComma(); writer.WritePropertyName(nameof(AttachmentDetails.DocumentId)); writer.WriteString(result.DocumentId); writer.WriteComma(); writer.WritePropertyName(nameof(AttachmentDetails.ContentType)); writer.WriteString(result.ContentType); writer.WriteComma(); writer.WritePropertyName(nameof(AttachmentDetails.Hash)); writer.WriteString(result.Hash); writer.WriteComma(); writer.WritePropertyName(nameof(AttachmentDetails.Size)); writer.WriteInteger(result.Size); writer.WriteEndObject(); } } }