public static void WriteFacetQuery(this BlittableJsonTextWriter writer, DocumentConventions conventions, JsonOperationContext context, FacetQuery query)
{
writer.WriteObject(EntityToBlittable.ConvertEntityToBlittable(query, conventions, context));
}
public async Task SubscriptionShouldRespectDocumentsWithCompressedData()
{
using (var documentStore = this.GetDocumentStore())
{
Server.ServerStore.Observer.Suspended = true;
var originalDoc = new Doc
{
Id = "doc/1",
StrVal = new string(Enumerable.Repeat('.', 129).ToArray()),
LongByteArray = Enumerable.Repeat((byte)2, 1024).ToArray()
};
using (var session = documentStore.OpenAsyncSession())
{
await session.StoreAsync(originalDoc);
await session.SaveChangesAsync();
}
var database = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(documentStore.Database);
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var doc = database.DocumentsStorage.Get(context, "doc/1");
MemoryStream ms = new MemoryStream();
using (var newContext = JsonOperationContext.ShortTermSingleUse())
using (var writer = new BlittableJsonTextWriter(newContext, ms))
{
writer.WriteDocument(newContext, doc, metadataOnly: false);
writer.Flush();
var bjro = GetReaderFromMemoryStream(ms, context);
var desereializedDoc = (Doc)EntityToBlittable.ConvertToEntity(typeof(Doc), null, bjro, DocumentConventions.Default);
Assert.Equal(originalDoc.StrVal, desereializedDoc.StrVal);
Assert.Equal(originalDoc.LongByteArray, originalDoc.LongByteArray);
}
}
var subscriptionCreationParams = new SubscriptionCreationOptions
{
Query = "from Docs",
};
var subsId = await documentStore.Subscriptions.CreateAsync(subscriptionCreationParams).ConfigureAwait(false);
var amre = new AsyncManualResetEvent();
using (var subscription = documentStore.Subscriptions.GetSubscriptionWorker <Doc>(new SubscriptionWorkerOptions(subsId)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
}))
{
var t = subscription.Run(batch =>
{
var receivedDoc = batch.Items.First().Result;
Assert.Equal(originalDoc.LongByteArray, receivedDoc.LongByteArray);
Assert.Equal(originalDoc.StrVal, receivedDoc.StrVal);
amre.Set();
});
try
{
Assert.True(await amre.WaitAsync(TimeSpan.FromSeconds(60)));
}
catch
{
if (t.IsFaulted)
{
t.Wait();
}
throw;
}
}
}
}
private void WriteDocument(BlittableJsonTextWriter writer, JsonOperationContext context, Document document, HashSet <string> propertiesPreviewToSend, HashSet <string> fullPropertiesToSend)
{
writer.WriteStartObject();
document.Data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata);
bool first = true;
var arrayStubsJson = new DynamicJsonValue();
var objectStubsJson = new DynamicJsonValue();
var trimmedValue = new HashSet <LazyStringValue>();
var prop = new BlittableJsonReaderObject.PropertyDetails();
using (var buffers = document.Data.GetPropertiesByInsertionOrder())
{
for (int i = 0; i < buffers.Properties.Count; i++)
{
document.Data.GetPropertyByIndex(buffers.Properties.Array[i + buffers.Properties.Offset], ref prop);
var sendFull = fullPropertiesToSend.Contains(prop.Name);
if (sendFull || propertiesPreviewToSend.Contains(prop.Name))
{
var strategy = sendFull ? ValueWriteStrategy.Passthrough : FindWriteStrategy(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
if (strategy == ValueWriteStrategy.Passthrough || strategy == ValueWriteStrategy.Trim)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
}
switch (strategy)
{
case ValueWriteStrategy.Passthrough:
writer.WritePropertyName(prop.Name);
writer.WriteValue(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
break;
case ValueWriteStrategy.SubstituteWithArrayStub:
arrayStubsJson[prop.Name] = ((BlittableJsonReaderArray)prop.Value).Length;
break;
case ValueWriteStrategy.SubstituteWithObjectStub:
objectStubsJson[prop.Name] = ((BlittableJsonReaderObject)prop.Value).Count;
break;
case ValueWriteStrategy.Trim:
writer.WritePropertyName(prop.Name);
WriteTrimmedValue(writer, prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
trimmedValue.Add(prop.Name);
break;
}
}
}
}
if (first == false)
{
writer.WriteComma();
}
var extraMetadataProperties = new DynamicJsonValue
{
[ObjectStubsKey] = objectStubsJson,
[ArrayStubsKey] = arrayStubsJson,
[TrimmedValueKey] = new DynamicJsonArray(trimmedValue)
};
if (metadata != null)
{
metadata.Modifications = extraMetadataProperties;
metadata = context.ReadObject(metadata, document.Id);
}
else
{
metadata = context.ReadObject(extraMetadataProperties, document.Id);
}
writer.WriteMetadata(document, metadata);
writer.WriteEndObject();
}
public async Task ApplyCommand()
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
if (ServerStore.IsLeader() == false)
{
throw new NoLeaderException("Not a leader, cannot accept commands.");
}
HttpContext.Response.Headers["Reached-Leader"] = "true";
var commandJson = await context.ReadForMemoryAsync(RequestBodyStream(), "external/rachis/command");
CommandBase command;
try
{
command = CommandBase.CreateFrom(commandJson);
}
catch (InvalidOperationException e)
{
RequestRouter.AssertClientVersion(HttpContext, e);
throw;
}
switch (command)
{
case AddOrUpdateCompareExchangeBatchCommand batchCmpExchange:
batchCmpExchange.ContextToWriteResult = context;
break;
case CompareExchangeCommandBase cmpExchange:
cmpExchange.ContextToWriteResult = context;
break;
}
var isClusterAdmin = IsClusterAdmin();
command.VerifyCanExecuteCommand(ServerStore, context, isClusterAdmin);
var(etag, result) = await ServerStore.Engine.PutAsync(command);
HttpContext.Response.StatusCode = (int)HttpStatusCode.OK;
var ms = context.CheckoutMemoryStream();
try
{
using (var writer = new BlittableJsonTextWriter(context, ms))
{
context.Write(writer, new DynamicJsonValue
{
[nameof(ServerStore.PutRaftCommandResult.RaftCommandIndex)] = etag,
[nameof(ServerStore.PutRaftCommandResult.Data)] = result
});
writer.Flush();
}
// now that we know that we properly serialized it
ms.Position = 0;
await ms.CopyToAsync(ResponseBodyStream());
}
finally
{
context.ReturnMemoryStream(ms);
}
}
}
public async Task Try()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var json = await context.ReadForMemoryAsync(RequestBodyStream(), null);
var tryout = JsonDeserializationServer.SubscriptionTryout(json);
var(collection, (script, functions), revisions) = SubscriptionConnection.ParseSubscriptionQuery(tryout.Query);
SubscriptionPatchDocument patch = null;
if (string.IsNullOrEmpty(script) == false)
{
patch = new SubscriptionPatchDocument(script, functions);
}
if (collection == null)
{
throw new ArgumentException("Collection must be specified");
}
var pageSize = GetIntValueQueryString("pageSize") ?? 1;
var state = new SubscriptionState
{
ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector,
Query = tryout.Query
};
var fetcher = new SubscriptionDocumentsFetcher(Database, pageSize, -0x42,
new IPEndPoint(HttpContext.Connection.RemoteIpAddress, HttpContext.Connection.RemotePort), collection, revisions, state, patch);
if (Enum.TryParse(
tryout.ChangeVector,
out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
{
switch (changeVectorSpecialValue)
{
case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:
case Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange:
state.ChangeVectorForNextBatchStartingPoint = null;
break;
case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
state.ChangeVectorForNextBatchStartingPoint = Database.DocumentsStorage.GetLastDocumentChangeVector(context, collection);
break;
}
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
using (context.OpenReadTransaction())
{
var first = true;
foreach (var itemDetails in fetcher.GetDataToSend(context, 0))
{
if (itemDetails.Doc.Data == null)
{
continue;
}
if (first == false)
{
writer.WriteComma();
}
if (itemDetails.Exception == null)
{
writer.WriteDocument(context, itemDetails.Doc, metadataOnly: false);
}
else
{
var docWithExcepton = new DocumentWithException
{
Exception = itemDetails.Exception.ToString(),
ChangeVector = itemDetails.Doc.ChangeVector,
Id = itemDetails.Doc.Id,
DocumentData = itemDetails.Doc.Data
};
writer.WriteObject(context.ReadObject(docWithExcepton.ToJson(), ""));
}
first = false;
}
}
writer.WriteEndArray();
writer.WriteEndObject();
}
}
}
public Task Stats()
{
var documentId = GetStringQueryString("docId");
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var document = Database.DocumentsStorage.Get(context, documentId, DocumentFields.Data);
if (document == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return(Task.CompletedTask);
}
var timeSeriesNames = new List <string>();
if (document.TryGetMetadata(out var metadata))
{
if (metadata.TryGet(Constants.Documents.Metadata.TimeSeries, out BlittableJsonReaderArray timeSeries) && timeSeries != null)
{
foreach (object name in timeSeries)
{
if (name == null)
{
continue;
}
if (name is string || name is LazyStringValue || name is LazyCompressedStringValue)
{
timeSeriesNames.Add(name.ToString());
}
}
}
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(TimeSeriesStatistics.DocumentId));
writer.WriteString(documentId);
writer.WriteComma();
writer.WritePropertyName(nameof(TimeSeriesStatistics.TimeSeries));
writer.WriteStartArray();
var first = true;
foreach (var tsName in timeSeriesNames)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
var stats = Database.DocumentsStorage.TimeSeriesStorage.Stats.GetStats(context, documentId, tsName);
writer.WriteStartObject();
writer.WritePropertyName(nameof(TimeSeriesItemDetail.Name));
writer.WriteString(tsName);
writer.WriteComma();
writer.WritePropertyName(nameof(TimeSeriesItemDetail.NumberOfEntries));
writer.WriteInteger(stats.Count);
writer.WriteComma();
writer.WritePropertyName(nameof(TimeSeriesItemDetail.StartDate));
writer.WriteDateTime(stats.Start, isUtc: true);
writer.WriteComma();
writer.WritePropertyName(nameof(TimeSeriesItemDetail.EndDate));
writer.WriteDateTime(stats.End, isUtc: true);
writer.WriteEndObject();
}
writer.WriteEndArray();
writer.WriteEndObject();
}
}
return(Task.CompletedTask);
}
public Task GetCertificates()
{
var thumbprint = GetStringQueryString("thumbprint", required: false);
var showSecondary = GetBoolValueQueryString("secondary", required: false) ?? false;
var start = GetStart();
var pageSize = GetPageSize();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
var certificates = new List <(string ItemName, BlittableJsonReaderObject Value)>();
try
{
if (string.IsNullOrEmpty(thumbprint))
{
if (ServerStore.CurrentRachisState == RachisState.Passive)
{
List <string> localCertKeys;
using (context.OpenReadTransaction())
localCertKeys = ServerStore.Cluster.GetCertificateKeysFromLocalState(context).ToList();
var serverCertKey = Constants.Certificates.Prefix + Server.Certificate.Certificate?.Thumbprint;
if (Server.Certificate.Certificate != null && localCertKeys.Contains(serverCertKey) == false)
{
// Since we didn't go through EnsureNotPassive(), the server certificate is not registered yet so we'll add it to the local state.
var serverCertDef = new CertificateDefinition
{
Name = "Server Certificate",
Certificate = Convert.ToBase64String(Server.Certificate.Certificate.Export(X509ContentType.Cert)),
Permissions = new Dictionary <string, DatabaseAccess>(),
SecurityClearance = SecurityClearance.ClusterNode,
Thumbprint = Server.Certificate.Certificate.Thumbprint,
NotAfter = Server.Certificate.Certificate.NotAfter
};
var serverCert = context.ReadObject(serverCertDef.ToJson(), "Server/Certificate/Definition");
using (var tx = context.OpenWriteTransaction())
{
ServerStore.Cluster.PutLocalState(context, Constants.Certificates.Prefix + Server.Certificate.Certificate.Thumbprint, serverCert);
tx.Commit();
}
certificates.Add((serverCertKey, serverCert));
}
foreach (var localCertKey in localCertKeys)
{
BlittableJsonReaderObject localCertificate;
using (context.OpenReadTransaction())
localCertificate = ServerStore.Cluster.GetLocalState(context, localCertKey);
if (localCertificate == null)
{
continue;
}
var def = JsonDeserializationServer.CertificateDefinition(localCertificate);
if (showSecondary || string.IsNullOrEmpty(def.CollectionPrimaryKey))
{
certificates.Add((localCertKey, localCertificate));
}
else
{
localCertificate.Dispose();
}
}
}
else
{
using (context.OpenReadTransaction())
{
foreach (var item in ServerStore.Cluster.ItemsStartingWith(context, Constants.Certificates.Prefix, start, pageSize))
{
var def = JsonDeserializationServer.CertificateDefinition(item.Value);
if (showSecondary || string.IsNullOrEmpty(def.CollectionPrimaryKey))
{
certificates.Add(item);
}
else
{
item.Value.Dispose();
}
}
}
}
}
else
{
using (context.OpenReadTransaction())
{
var key = Constants.Certificates.Prefix + thumbprint;
var certificate = ServerStore.CurrentRachisState == RachisState.Passive
? ServerStore.Cluster.GetLocalState(context, key)
: ServerStore.Cluster.Read(context, key);
if (certificate == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return(Task.CompletedTask);
}
var definition = JsonDeserializationServer.CertificateDefinition(certificate);
if (string.IsNullOrEmpty(definition.CollectionPrimaryKey) == false)
{
certificate = ServerStore.Cluster.Read(context, definition.CollectionPrimaryKey);
if (certificate == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return(Task.CompletedTask);
}
}
certificates.Add((key, certificate));
}
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WriteArray(context, "Results", certificates.ToArray(), (w, c, cert) =>
{
c.Write(w, cert.Value);
});
writer.WriteComma();
writer.WritePropertyName("LoadedServerCert");
writer.WriteString(Server.Certificate.Certificate?.Thumbprint);
writer.WriteEndObject();
}
}
finally
{
foreach (var cert in certificates)
{
cert.Value?.Dispose();
}
}
}
return(Task.CompletedTask);
}
public static void WriteIndexQuery(this BlittableJsonTextWriter writer, JsonOperationContext context, IndexQueryServerSide query)
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(query.CutoffEtag));
if (query.CutoffEtag.HasValue)
{
writer.WriteInteger(query.CutoffEtag.Value);
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(query.ExplainScores));
writer.WriteBool(query.ExplainScores);
writer.WriteComma();
writer.WritePropertyName(nameof(query.PageSize));
writer.WriteInteger(query.PageSize);
writer.WriteComma();
writer.WritePropertyName(nameof(query.Query));
if (query.Query != null)
{
writer.WriteString(query.Query);
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(query.ShowTimings));
writer.WriteBool(query.ShowTimings);
writer.WriteComma();
writer.WritePropertyName(nameof(query.SkipDuplicateChecking));
writer.WriteBool(query.SkipDuplicateChecking);
writer.WriteComma();
writer.WritePropertyName(nameof(query.Start));
writer.WriteInteger(query.Start);
writer.WriteComma();
writer.WritePropertyName(nameof(query.WaitForNonStaleResults));
writer.WriteBool(query.WaitForNonStaleResults);
writer.WriteComma();
writer.WritePropertyName(nameof(query.WaitForNonStaleResultsTimeout));
if (query.WaitForNonStaleResultsTimeout.HasValue)
{
writer.WriteString(query.WaitForNonStaleResultsTimeout.Value.ToString());
}
else
{
writer.WriteNull();
}
writer.WriteEndObject();
}
public static void WriteDatabaseStatistics(this BlittableJsonTextWriter writer, JsonOperationContext context, DatabaseStatistics statistics)
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(statistics.CountOfIndexes));
writer.WriteInteger(statistics.CountOfIndexes);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.CountOfDocuments));
writer.WriteInteger(statistics.CountOfDocuments);
writer.WriteComma();
if (statistics.CountOfRevisionDocuments > 0)
{
writer.WritePropertyName(nameof(statistics.CountOfRevisionDocuments));
writer.WriteInteger(statistics.CountOfRevisionDocuments);
writer.WriteComma();
}
writer.WritePropertyName(nameof(statistics.CountOfDocumentsConflicts));
writer.WriteInteger(statistics.CountOfDocumentsConflicts);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.CountOfConflicts));
writer.WriteInteger(statistics.CountOfConflicts);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.CountOfAttachments));
writer.WriteInteger(statistics.CountOfAttachments);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.CountOfUniqueAttachments));
writer.WriteInteger(statistics.CountOfUniqueAttachments);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.DatabaseChangeVector));
writer.WriteString(statistics.DatabaseChangeVector);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.DatabaseId));
writer.WriteString(statistics.DatabaseId);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.Is64Bit));
writer.WriteBool(statistics.Is64Bit);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.Pager));
writer.WriteString(statistics.Pager);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.LastDocEtag));
if (statistics.LastDocEtag.HasValue)
{
writer.WriteInteger(statistics.LastDocEtag.Value);
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName((nameof(statistics.DatabaseChangeVector)));
writer.WriteString(statistics.DatabaseChangeVector);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.LastIndexingTime));
if (statistics.LastIndexingTime.HasValue)
{
writer.WriteDateTime(statistics.LastIndexingTime.Value, isUtc: true);
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.SizeOnDisk));
writer.WriteStartObject();
writer.WritePropertyName(nameof(statistics.SizeOnDisk.HumaneSize));
writer.WriteString(statistics.SizeOnDisk.HumaneSize);
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.SizeOnDisk.SizeInBytes));
writer.WriteInteger(statistics.SizeOnDisk.SizeInBytes);
writer.WriteEndObject();
writer.WriteComma();
writer.WritePropertyName(nameof(statistics.Indexes));
writer.WriteStartArray();
var isFirstInternal = true;
foreach (var index in statistics.Indexes)
{
if (isFirstInternal == false)
{
writer.WriteComma();
}
isFirstInternal = false;
writer.WriteStartObject();
writer.WritePropertyName(nameof(index.IsStale));
writer.WriteBool(index.IsStale);
writer.WriteComma();
writer.WritePropertyName(nameof(index.Name));
writer.WriteString(index.Name);
writer.WriteComma();
writer.WritePropertyName(nameof(index.Etag));
writer.WriteInteger(index.Etag);
writer.WriteComma();
writer.WritePropertyName(nameof(index.LockMode));
writer.WriteString(index.LockMode.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(index.Priority));
writer.WriteString(index.Priority.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(index.State));
writer.WriteString(index.State.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(index.Type));
writer.WriteString(index.Type.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(index.LastIndexingTime));
if (index.LastIndexingTime.HasValue)
{
writer.WriteDateTime(index.LastIndexingTime.Value, isUtc: true);
}
else
{
writer.WriteNull();
}
writer.WriteEndObject();
}
writer.WriteEndArray();
writer.WriteEndObject();
}
public static void WriteTreePagesRecursively(this BlittableJsonTextWriter writer, IEnumerable <ReduceTreePage> pages)
{
var first = true;
foreach (var page in pages)
{
if (first == false)
{
writer.WriteComma();
}
writer.WriteStartObject();
writer.WritePropertyName(nameof(TreePage.PageNumber));
writer.WriteInteger(page.PageNumber);
writer.WriteComma();
writer.WritePropertyName(nameof(ReduceTreePage.AggregationResult));
if (page.AggregationResult != null)
{
writer.WriteObject(page.AggregationResult);
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(ReduceTreePage.Children));
if (page.Children != null)
{
writer.WriteStartArray();
WriteTreePagesRecursively(writer, page.Children);
writer.WriteEndArray();
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(ReduceTreePage.Entries));
if (page.Entries != null)
{
writer.WriteStartArray();
var firstEntry = true;
foreach (var entry in page.Entries)
{
if (firstEntry == false)
{
writer.WriteComma();
}
writer.WriteStartObject();
writer.WritePropertyName(nameof(MapResultInLeaf.Data));
writer.WriteObject(entry.Data);
writer.WriteComma();
writer.WritePropertyName(nameof(MapResultInLeaf.Source));
writer.WriteString(entry.Source);
writer.WriteEndObject();
firstEntry = false;
}
writer.WriteEndArray();
}
else
{
writer.WriteNull();
}
writer.WriteEndObject();
first = false;
}
}
public static void WriteQueryResult <TResult, TInclude>(this BlittableJsonTextWriter writer, JsonOperationContext context, QueryResultBase <TResult, TInclude> result, bool metadataOnly, out int numberOfResults, bool partial = false)
{
if (partial == false)
{
writer.WriteStartObject();
}
writer.WritePropertyName(nameof(result.IndexName));
writer.WriteString(result.IndexName);
writer.WriteComma();
var results = (object)result.Results;
if (results is List <Document> documents)
{
writer.WritePropertyName(nameof(result.Results));
writer.WriteDocuments(context, documents, metadataOnly, out numberOfResults);
writer.WriteComma();
}
else if (results is List <BlittableJsonReaderObject> objects)
{
writer.WritePropertyName(nameof(result.Results));
writer.WriteObjects(context, objects, out numberOfResults);
writer.WriteComma();
}
else
{
throw new NotSupportedException($"Cannot write query result of '{typeof(TResult)}' type in '{result.GetType()}'.");
}
var includes = (object)result.Includes;
if (includes is List <Document> includeDocuments)
{
writer.WritePropertyName(nameof(result.Includes));
writer.WriteIncludes(context, includeDocuments);
writer.WriteComma();
}
else if (includes is List <BlittableJsonReaderObject> includeObjects)
{
if (includeObjects.Count != 0)
{
throw new NotSupportedException("Cannot write query includes of List<BlittableJsonReaderObject>, but got non zero response");
}
writer.WritePropertyName(nameof(result.Includes));
writer.WriteStartObject();
writer.WriteEndObject();
writer.WriteComma();
}
else
{
throw new NotSupportedException($"Cannot write query includes of '{typeof(TInclude)}' type in '{result.GetType()}'.");
}
writer.WritePropertyName(nameof(result.IndexTimestamp));
writer.WriteString(result.IndexTimestamp.ToString(DefaultFormat.DateTimeFormatsToWrite));
writer.WriteComma();
writer.WritePropertyName(nameof(result.LastQueryTime));
writer.WriteString(result.LastQueryTime.ToString(DefaultFormat.DateTimeFormatsToWrite));
writer.WriteComma();
writer.WritePropertyName(nameof(result.IsStale));
writer.WriteBool(result.IsStale);
writer.WriteComma();
writer.WritePropertyName(nameof(result.ResultEtag));
writer.WriteInteger(result.ResultEtag);
if (partial == false)
{
writer.WriteEndObject();
}
}
private static void WriteDocumentInternal(this BlittableJsonTextWriter writer, JsonOperationContext context, Document document)
{
writer.WriteStartObject();
WriteDocumentProperties(writer, context, document);
writer.WriteEndObject();
}
public static void WriteFacetResult(this BlittableJsonTextWriter writer, JsonOperationContext context, FacetResult result)
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(result.RemainingHits));
writer.WriteInteger(result.RemainingHits);
writer.WriteComma();
writer.WritePropertyName(nameof(result.RemainingTermsCount));
writer.WriteInteger(result.RemainingTermsCount);
writer.WriteComma();
writer.WritePropertyName(nameof(result.RemainingTerms));
writer.WriteStartArray();
var isFirstInternal = true;
foreach (var term in result.RemainingTerms)
{
if (isFirstInternal == false)
{
writer.WriteComma();
}
isFirstInternal = false;
writer.WriteString(term);
}
writer.WriteEndArray();
writer.WriteComma();
writer.WritePropertyName(nameof(result.Values));
writer.WriteStartArray();
isFirstInternal = true;
foreach (var value in result.Values)
{
if (isFirstInternal == false)
{
writer.WriteComma();
}
isFirstInternal = false;
writer.WriteStartObject();
writer.WritePropertyName(nameof(value.Average));
if (value.Average.HasValue)
{
using (var lazyStringValue = context.GetLazyString(value.Average.ToInvariantString()))
writer.WriteDouble(new LazyNumberValue(lazyStringValue));
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(value.Count));
if (value.Count.HasValue)
{
writer.WriteInteger(value.Count.Value);
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(value.Hits));
writer.WriteInteger(value.Hits);
writer.WriteComma();
writer.WritePropertyName(nameof(value.Max));
if (value.Max.HasValue)
{
using (var lazyStringValue = context.GetLazyString(value.Max.ToInvariantString()))
writer.WriteDouble(new LazyNumberValue(lazyStringValue));
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(value.Min));
if (value.Min.HasValue)
{
using (var lazyStringValue = context.GetLazyString(value.Min.ToInvariantString()))
writer.WriteDouble(new LazyNumberValue(lazyStringValue));
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(value.Range));
writer.WriteString(value.Range);
writer.WriteComma();
writer.WritePropertyName(nameof(value.Sum));
if (value.Sum.HasValue)
{
using (var lazyStringValue = context.GetLazyString(value.Sum.ToInvariantString()))
writer.WriteDouble(new LazyNumberValue(lazyStringValue));
}
else
{
writer.WriteNull();
}
writer.WriteEndObject();
}
writer.WriteEndArray();
writer.WriteEndObject();
}
public static void WriteIndexQuery(this BlittableJsonTextWriter writer, DocumentConventions conventions, JsonOperationContext context, IndexQuery query)
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(query.Query));
writer.WriteString(query.Query);
writer.WriteComma();
if (query.PageSizeSet && query.PageSize >= 0)
{
writer.WritePropertyName(nameof(query.PageSize));
writer.WriteInteger(query.PageSize);
writer.WriteComma();
}
if (query.WaitForNonStaleResults)
{
writer.WritePropertyName(nameof(query.WaitForNonStaleResults));
writer.WriteBool(query.WaitForNonStaleResults);
writer.WriteComma();
}
if (query.CutoffEtag.HasValue)
{
writer.WritePropertyName(nameof(query.CutoffEtag));
writer.WriteInteger(query.CutoffEtag.Value);
writer.WriteComma();
}
if (query.Start > 0)
{
writer.WritePropertyName(nameof(query.Start));
writer.WriteInteger(query.Start);
writer.WriteComma();
}
if (query.WaitForNonStaleResultsTimeout.HasValue)
{
writer.WritePropertyName(nameof(query.WaitForNonStaleResultsTimeout));
writer.WriteString(query.WaitForNonStaleResultsTimeout.Value.ToInvariantString());
writer.WriteComma();
}
if (query.DisableCaching)
{
writer.WritePropertyName(nameof(query.DisableCaching));
writer.WriteBool(query.DisableCaching);
writer.WriteComma();
}
if (query.ExplainScores)
{
writer.WritePropertyName(nameof(query.DisableCaching));
writer.WriteBool(query.DisableCaching);
writer.WriteComma();
}
if (query.ShowTimings)
{
writer.WritePropertyName(nameof(query.ShowTimings));
writer.WriteBool(query.ShowTimings);
writer.WriteComma();
}
if (query.SkipDuplicateChecking)
{
writer.WritePropertyName(nameof(query.SkipDuplicateChecking));
writer.WriteBool(query.SkipDuplicateChecking);
writer.WriteComma();
}
writer.WritePropertyName(nameof(query.QueryParameters));
if (query.QueryParameters != null)
{
writer.WriteObject(EntityToBlittable.ConvertEntityToBlittable(query.QueryParameters, conventions, context));
}
else
{
writer.WriteNull();
}
writer.WriteEndObject();
}
private async Task Graph(DocumentsOperationContext context, RequestTimeTracker tracker, HttpMethod method)
{
var indexQuery = await GetIndexQuery(context, method, tracker);
var queryRunner = Database.QueryRunner.GetRunner(indexQuery);
if (!(queryRunner is GraphQueryRunner gqr))
{
throw new InvalidOperationException("The specified query is not a graph query.");
}
using (var token = CreateTimeLimitedQueryToken())
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
{
var results = await gqr.GetAnalyzedQueryResults(indexQuery, ctx, null, token);
var nodes = new DynamicJsonArray();
var edges = new DynamicJsonArray();
var output = new DynamicJsonValue
{
["Nodes"] = nodes,
["Edges"] = edges
};
foreach (var item in results.Nodes)
{
var val = item.Value;
if (val is Document d)
{
d.EnsureMetadata();
val = d.Data;
}
nodes.Add(new DynamicJsonValue
{
["Id"] = item.Key,
["Value"] = val
});
}
foreach (var edge in results.Edges)
{
var array = new DynamicJsonArray();
var djv = new DynamicJsonValue
{
["Name"] = edge.Key,
["Results"] = array
};
foreach (var item in edge.Value)
{
var edgeVal = item.Edge;
if (edgeVal is Document d)
{
edgeVal = d.Id?.ToString() ?? "anonymous/" + Guid.NewGuid();
}
array.Add(new DynamicJsonValue
{
["From"] = item.Source,
["To"] = item.Destination,
["Edge"] = edgeVal
});
}
edges.Add(djv);
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, output);
}
}
}
public static void WriteIndexDefinition(this BlittableJsonTextWriter writer, JsonOperationContext context, IndexDefinition indexDefinition, bool removeAnalyzers = false)
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(indexDefinition.Name));
writer.WriteString(indexDefinition.Name);
writer.WriteComma();
writer.WritePropertyName(nameof(indexDefinition.Etag));
writer.WriteInteger(indexDefinition.Etag);
writer.WriteComma();
writer.WritePropertyName(nameof(indexDefinition.Type));
writer.WriteString(indexDefinition.Type.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(indexDefinition.LockMode));
if (indexDefinition.LockMode.HasValue)
{
writer.WriteString(indexDefinition.LockMode.ToString());
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(indexDefinition.Priority));
if (indexDefinition.Priority.HasValue)
{
writer.WriteString(indexDefinition.Priority.ToString());
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(indexDefinition.OutputReduceToCollection));
writer.WriteString(indexDefinition.OutputReduceToCollection);
writer.WriteComma();
writer.WritePropertyName(nameof(indexDefinition.Configuration));
writer.WriteStartObject();
var isFirstInternal = true;
foreach (var kvp in indexDefinition.Configuration)
{
if (isFirstInternal == false)
{
writer.WriteComma();
}
isFirstInternal = false;
writer.WritePropertyName(kvp.Key);
writer.WriteString(kvp.Value);
}
writer.WriteEndObject();
writer.WriteComma();
writer.WritePropertyName(nameof(indexDefinition.AdditionalSources));
writer.WriteStartObject();
isFirstInternal = true;
foreach (var kvp in indexDefinition.AdditionalSources)
{
if (isFirstInternal == false)
{
writer.WriteComma();
}
isFirstInternal = false;
writer.WritePropertyName(kvp.Key);
writer.WriteString(kvp.Value);
}
writer.WriteEndObject();
writer.WriteComma();
writer.WritePropertyName(nameof(indexDefinition.IsTestIndex));
writer.WriteBool(indexDefinition.IsTestIndex);
writer.WriteComma();
writer.WritePropertyName(nameof(indexDefinition.Reduce));
if (string.IsNullOrWhiteSpace(indexDefinition.Reduce) == false)
{
writer.WriteString(indexDefinition.Reduce);
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(indexDefinition.Maps));
writer.WriteStartArray();
isFirstInternal = true;
foreach (var map in indexDefinition.Maps)
{
if (isFirstInternal == false)
{
writer.WriteComma();
}
isFirstInternal = false;
writer.WriteString(map);
}
writer.WriteEndArray();
writer.WriteComma();
writer.WritePropertyName(nameof(indexDefinition.Fields));
writer.WriteStartObject();
isFirstInternal = true;
foreach (var kvp in indexDefinition.Fields)
{
if (isFirstInternal == false)
{
writer.WriteComma();
}
isFirstInternal = false;
writer.WritePropertyName(kvp.Key);
if (kvp.Value != null)
{
writer.WriteIndexFieldOptions(context, kvp.Value, removeAnalyzers);
}
else
{
writer.WriteNull();
}
}
writer.WriteEndObject();
writer.WriteEndObject();
}
private void WriteTablesArray(string name, List <SqlMigrationTable> tables, BlittableJsonTextWriter writer)
{
writer.WritePropertyName(name);
writer.WriteStartArray();
var first = true;
if (tables != null)
{
foreach (var table in tables)
{
if (first)
{
first = false;
}
else
{
writer.WriteComma();
}
writer.WriteStartObject();
writer.WritePropertyName(nameof(table.Name));
writer.WriteString(table.Name);
writer.WriteComma();
writer.WritePropertyName(nameof(table.Query));
writer.WriteString(table.Query);
writer.WriteComma();
writer.WritePropertyName(nameof(table.Patch));
writer.WriteString(table.Patch);
writer.WriteComma();
writer.WritePropertyName(nameof(table.NewName));
writer.WriteString(table.NewName);
writer.WriteComma();
WriteTablesArray(nameof(table.EmbeddedTables), table.EmbeddedTables, writer);
writer.WriteEndObject();
}
}
writer.WriteEndArray();
}
public static void WriteIndexProgress(this BlittableJsonTextWriter writer, JsonOperationContext context, IndexProgress progress)
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(progress.IsStale));
writer.WriteBool(progress.IsStale);
writer.WriteComma();
writer.WritePropertyName(nameof(progress.Collections));
if (progress.Collections != null)
{
writer.WriteStartObject();
var isFirst = true;
foreach (var kvp in progress.Collections)
{
if (isFirst == false)
{
writer.WriteComma();
}
isFirst = false;
writer.WritePropertyName(kvp.Key);
writer.WriteStartObject();
writer.WritePropertyName(nameof(kvp.Value.LastProcessedDocumentEtag));
writer.WriteInteger(kvp.Value.LastProcessedDocumentEtag);
writer.WriteComma();
writer.WritePropertyName(nameof(kvp.Value.LastProcessedTombstoneEtag));
writer.WriteInteger(kvp.Value.LastProcessedTombstoneEtag);
writer.WriteComma();
writer.WritePropertyName(nameof(kvp.Value.NumberOfDocumentsToProcess));
writer.WriteInteger(kvp.Value.NumberOfDocumentsToProcess);
writer.WriteComma();
writer.WritePropertyName(nameof(kvp.Value.NumberOfTombstonesToProcess));
writer.WriteInteger(kvp.Value.NumberOfTombstonesToProcess);
writer.WriteComma();
writer.WritePropertyName(nameof(kvp.Value.TotalNumberOfDocuments));
writer.WriteInteger(kvp.Value.TotalNumberOfDocuments);
writer.WriteComma();
writer.WritePropertyName(nameof(kvp.Value.TotalNumberOfTombstones));
writer.WriteInteger(kvp.Value.TotalNumberOfTombstones);
writer.WriteEndObject();
}
writer.WriteEndObject();
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(progress.Name));
writer.WriteString(progress.Name);
writer.WriteComma();
writer.WritePropertyName(nameof(progress.Type));
writer.WriteString(progress.Type.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(progress.Etag));
writer.WriteInteger(progress.Etag);
writer.WriteEndObject();
}
public async Task PostImportAsync()
{
DocumentsOperationContext context;
using (ContextPool.AllocateOperationContext(out context))
{
if (HttpContext.Request.HasFormContentType == false)
{
HttpContext.Response.StatusCode = 400; // Bad request
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, new DynamicJsonValue
{
["Type"] = "Error",
["Error"] = "This endpoint requires form content type"
});
return;
}
}
var operationId = GetLongQueryString("operationId", required: true) ?? -1;
var token = CreateOperationToken();
var sp = Stopwatch.StartNew();
var result = new ImportResult();
await Database.Operations.AddOperation("Import to: " + Database.Name,
DatabaseOperations.PendingOperationType.DatabaseImport,
onProgress =>
{
return(Task.Run(async() =>
{
try
{
var boundary = MultipartRequestHelper.GetBoundary(
MediaTypeHeaderValue.Parse(HttpContext.Request.ContentType),
MultipartRequestHelper.MultipartBoundaryLengthLimit);
var reader = new MultipartReader(boundary, HttpContext.Request.Body);
DatabaseSmugglerOptions smugglerOptions = null;
while (true)
{
var section = await reader.ReadNextSectionAsync().ConfigureAwait(false);
if (section == null)
{
break;
}
ContentDispositionHeaderValue contentDisposition;
var hasContentDispositionHeader = ContentDispositionHeaderValue.TryParse(section.ContentDisposition,
out contentDisposition);
if (hasContentDispositionHeader == false)
{
continue;
}
if (MultipartRequestHelper.HasFormDataContentDisposition(contentDisposition))
{
var key = HeaderUtilities.RemoveQuotes(contentDisposition.Name);
if (key != "importOptions")
{
continue;
}
var blittableJson = await context.ReadForMemoryAsync(section.Body, "importOptions");
smugglerOptions = JsonDeserializationServer.DatabaseSmugglerOptions(blittableJson);
continue;
}
if (MultipartRequestHelper.HasFileContentDisposition(contentDisposition) == false)
{
continue;
}
var stream = new GZipStream(section.Body, CompressionMode.Decompress);
var fileImport =
await DoImportInternal(context, stream, smugglerOptions, onProgress).ConfigureAwait(false);
result.DocumentsCount += fileImport.DocumentsCount;
result.Warnings.AddRange(fileImport.Warnings);
result.IdentitiesCount += fileImport.IdentitiesCount;
result.Message = fileImport.Message;
result.RevisionDocumentsCount += fileImport.RevisionDocumentsCount;
result.IndexesCount += fileImport.IndexesCount;
result.TransformersCount += fileImport.TransformersCount;
}
}
catch (Exception e)
{
result.Message = $"Error occured during export. Exception: {e.Message}";
result.Exception = e.ToString();
}
return (IOperationResult)result;
}));
}, operationId, token).ConfigureAwait(false);
WriteImportResult(context, sp, result, ResponseBodyStream());
}
}
public static void WriteIndexStats(this BlittableJsonTextWriter writer, JsonOperationContext context, IndexStats stats)
{
var djv = (DynamicJsonValue)TypeConverter.ToBlittableSupportedType(stats);
writer.WriteObject(context.ReadObject(djv, "index/stats"));
}
public Task GetClusterTopology()
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
var topology = ServerStore.GetClusterTopology(context);
var nodeTag = ServerStore.NodeTag;
if (topology.AllNodes.Count == 0)
{
var tag = ServerStore.NodeTag ?? "A";
var serverUrl = ServerStore.GetNodeHttpServerUrl(HttpContext.Request.GetClientRequestedNodeUrl());
topology = new ClusterTopology(
topology.TopologyId ?? "dummy",
new Dictionary <string, string>
{
[tag] = serverUrl
},
new Dictionary <string, string>(),
new Dictionary <string, string>(),
tag
);
nodeTag = tag;
}
else
{
var isClientIndependent = GetBoolValueQueryString("clientIndependent", false) ?? false;
if (isClientIndependent == false)
{
topology.ReplaceCurrentNodeUrlWithClientRequestedNodeUrlIfNecessary(ServerStore, HttpContext);
}
}
HttpContext.Response.StatusCode = (int)HttpStatusCode.OK;
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
var loadLicenseLimits = ServerStore.LoadLicenseLimits();
var nodeLicenseDetails = loadLicenseLimits == null ?
null : DynamicJsonValue.Convert(loadLicenseLimits.NodeLicenseDetails);
var json = new DynamicJsonValue
{
["Topology"] = topology.ToSortedJson(),
["Leader"] = ServerStore.LeaderTag,
["LeaderShipDuration"] = ServerStore.Engine.CurrentLeader?.LeaderShipDuration,
["CurrentState"] = ServerStore.CurrentRachisState,
["NodeTag"] = nodeTag,
["CurrentTerm"] = ServerStore.Engine.CurrentTerm,
["NodeLicenseDetails"] = nodeLicenseDetails,
[nameof(ServerStore.Engine.LastStateChangeReason)] = ServerStore.LastStateChangeReason()
};
var clusterErrors = ServerStore.GetClusterErrors();
if (clusterErrors.Count > 0)
{
json["Errors"] = clusterErrors;
}
var nodesStatues = ServerStore.GetNodesStatuses();
json["Status"] = DynamicJsonValue.Convert(nodesStatues);
context.Write(writer, json);
writer.Flush();
}
}
return(Task.CompletedTask);
}
private static void WriteIndexFieldOptions(this BlittableJsonTextWriter writer, JsonOperationContext context, IndexFieldOptions options, bool removeAnalyzers)
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(options.Analyzer));
if (string.IsNullOrWhiteSpace(options.Analyzer) == false && !removeAnalyzers)
{
writer.WriteString(options.Analyzer);
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(options.Indexing));
if (options.Indexing.HasValue)
{
writer.WriteString(options.Indexing.ToString());
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(options.Storage));
if (options.Storage.HasValue)
{
writer.WriteString(options.Storage.ToString());
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(options.Suggestions));
if (options.Suggestions.HasValue)
{
writer.WriteBool(options.Suggestions.Value);
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(options.TermVector));
if (options.TermVector.HasValue)
{
writer.WriteString(options.TermVector.ToString());
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(options.Spatial));
if (options.Spatial != null)
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(options.Spatial.Type));
writer.WriteString(options.Spatial.Type.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(options.Spatial.MaxTreeLevel));
writer.WriteInteger(options.Spatial.MaxTreeLevel);
writer.WriteComma();
writer.WritePropertyName(nameof(options.Spatial.MaxX));
LazyStringValue lazyStringValue;
using (lazyStringValue = context.GetLazyString(CharExtensions.ToInvariantString(options.Spatial.MaxX)))
writer.WriteDouble(new LazyNumberValue(lazyStringValue));
writer.WriteComma();
writer.WritePropertyName(nameof(options.Spatial.MaxY));
using (lazyStringValue = context.GetLazyString(CharExtensions.ToInvariantString(options.Spatial.MaxY)))
writer.WriteDouble(new LazyNumberValue(lazyStringValue));
writer.WriteComma();
writer.WritePropertyName(nameof(options.Spatial.MinX));
using (lazyStringValue = context.GetLazyString(CharExtensions.ToInvariantString(options.Spatial.MinX)))
writer.WriteDouble(new LazyNumberValue(lazyStringValue));
writer.WriteComma();
writer.WritePropertyName(nameof(options.Spatial.MinY));
using (lazyStringValue = context.GetLazyString(CharExtensions.ToInvariantString(options.Spatial.MinY)))
writer.WriteDouble(new LazyNumberValue(lazyStringValue));
writer.WriteComma();
writer.WritePropertyName(nameof(options.Spatial.Strategy));
writer.WriteString(options.Spatial.Strategy.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(options.Spatial.Units));
writer.WriteString(options.Spatial.Units.ToString());
writer.WriteEndObject();
}
else
{
writer.WriteNull();
}
writer.WriteEndObject();
}
public async Task Patch()
{
var id = GetQueryStringValueAndAssertIfSingleAndNotEmpty("id");
var isTest = GetBoolValueQueryString("test", required: false) ?? false;
var debugMode = GetBoolValueQueryString("debug", required: false) ?? isTest;
var skipPatchIfChangeVectorMismatch = GetBoolValueQueryString("skipPatchIfChangeVectorMismatch", required: false) ?? false;
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var request = context.Read(RequestBodyStream(), "ScriptedPatchRequest");
if (request.TryGet("Patch", out BlittableJsonReaderObject patchCmd) == false || patchCmd == null)
{
throw new ArgumentException("The 'Patch' field in the body request is mandatory");
}
var patch = PatchRequest.Parse(patchCmd, out var patchArgs);
PatchRequest patchIfMissing = null;
BlittableJsonReaderObject patchIfMissingArgs = null;
if (request.TryGet("PatchIfMissing", out BlittableJsonReaderObject patchIfMissingCmd) && patchIfMissingCmd != null)
{
patchIfMissing = PatchRequest.Parse(patchIfMissingCmd, out patchIfMissingArgs);
}
var changeVector = context.GetLazyString(GetStringFromHeaders("If-Match"));
using (var command = new PatchDocumentCommand(context,
id,
changeVector,
skipPatchIfChangeVectorMismatch,
(patch, patchArgs),
(patchIfMissing, patchIfMissingArgs),
Database,
isTest,
debugMode,
true
))
{
if (isTest == false)
{
await Database.TxMerger.Enqueue(command);
}
else
{
// PutDocument requires the write access to the docs storage
// testing patching is rare enough not to optimize it
using (context.OpenWriteTransaction())
{
command.Execute(context, null);
}
}
switch (command.PatchResult.Status)
{
case PatchStatus.DocumentDoesNotExist:
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return;
case PatchStatus.Created:
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
break;
case PatchStatus.Skipped:
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified;
return;
case PatchStatus.Patched:
case PatchStatus.NotModified:
HttpContext.Response.StatusCode = (int)HttpStatusCode.OK;
break;
default:
throw new ArgumentOutOfRangeException();
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(command.PatchResult.Status));
writer.WriteString(command.PatchResult.Status.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(command.PatchResult.ModifiedDocument));
writer.WriteObject(command.PatchResult.ModifiedDocument);
if (debugMode)
{
writer.WriteComma();
writer.WritePropertyName(nameof(command.PatchResult.OriginalDocument));
if (isTest)
{
writer.WriteObject(command.PatchResult.OriginalDocument);
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(command.PatchResult.Debug));
context.Write(writer, new DynamicJsonValue
{
["Info"] = new DynamicJsonArray(command.DebugOutput),
["Actions"] = command.DebugActions?.GetDebugActions()
});
}
writer.WriteEndObject();
}
}
}
}
public Task GetCollectionFields()
{
var collection = GetStringQueryString("collection", required: false);
var prefix = GetStringQueryString("prefix", required: false);
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
long totalResults;
string changeVector;
string etag = null;
if (string.IsNullOrEmpty(collection))
{
changeVector = DocumentsStorage.GetDatabaseChangeVector(context);
totalResults = Database.DocumentsStorage.GetNumberOfDocuments(context);
etag = $"{changeVector}/{totalResults}";
}
else
{
changeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context, collection);
totalResults = Database.DocumentsStorage.GetCollection(collection, context).Count;
if (changeVector != null)
{
etag = $"{changeVector}/{totalResults}";
}
}
if (etag != null && GetStringFromHeaders("If-None-Match") == etag)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified;
return(Task.CompletedTask);
}
HttpContext.Response.Headers["ETag"] = "\"" + etag + "\"";
if (_buffers == null)
{
_buffers = new BlittableJsonReaderObject.PropertiesInsertionBuffer();
}
var fields = new Dictionary <LazyStringValue, FieldType>(LazyStringValueComparer.Instance);
if (string.IsNullOrEmpty(collection))
{
foreach (var collectionStats in Database.DocumentsStorage.GetCollections(context))
{
FetchFieldsForCollection(context, collectionStats.Name, prefix, fields, _buffers);
}
}
else
{
FetchFieldsForCollection(context, collection, prefix, fields, _buffers);
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
var first = true;
foreach (var field in fields)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WritePropertyName(field.Key);
writer.WriteString(field.Value.ToString());
}
writer.WriteEndObject();
}
return(Task.CompletedTask);
}
}
public Task GetAll()
{
var start = GetStart();
var pageSize = GetPageSize();
var history = GetBoolValueQueryString("history", required: false) ?? false;
var running = GetBoolValueQueryString("running", required: false) ?? false;
var id = GetLongQueryString("id", required: false);
var name = GetStringQueryString("name", required: false);
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
IEnumerable <SubscriptionStorage.SubscriptionGeneralDataAndStats> subscriptions;
if (string.IsNullOrEmpty(name) && id == null)
{
subscriptions = running
? Database.SubscriptionStorage.GetAllRunningSubscriptions(context, history, start, pageSize)
: Database.SubscriptionStorage.GetAllSubscriptions(context, history, start, pageSize);
}
else
{
var subscription = running
? Database
.SubscriptionStorage
.GetRunningSubscription(context, id, name, history)
: Database
.SubscriptionStorage
.GetSubscription(context, id, name, history);
if (subscription == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return(Task.CompletedTask);
}
subscriptions = new[] { subscription };
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
var subscriptionsAsBlittable = subscriptions.Select(x => new DynamicJsonValue()
{
[nameof(SubscriptionState.SubscriptionId)] = x.SubscriptionId,
[nameof(SubscriptionState.SubscriptionName)] = x.SubscriptionName,
[nameof(SubscriptionState.ChangeVectorForNextBatchStartingPoint)] = x.ChangeVectorForNextBatchStartingPoint,
[nameof(SubscriptionState.Query)] = x.Query,
[nameof(SubscriptionState.Disabled)] = x.Disabled,
[nameof(SubscriptionState.LastClientConnectionTime)] = x.LastClientConnectionTime,
[nameof(SubscriptionState.LastBatchAckTime)] = x.LastBatchAckTime,
["Connection"] = GetSubscriptionConnectionDJV(x.Connection),
["RecentConnections"] = x.RecentConnections?.Select(r => new DynamicJsonValue()
{
["State"] = new DynamicJsonValue()
{
["LatestChangeVectorClientACKnowledged"] = r.SubscriptionState.ChangeVectorForNextBatchStartingPoint,
["Query"] = r.SubscriptionState.Query
},
["Connection"] = GetSubscriptionConnectionDJV(r)
}),
["FailedConnections"] = x.RecentRejectedConnections?.Select(r => new DynamicJsonValue()
{
["State"] = new DynamicJsonValue()
{
["LatestChangeVectorClientACKnowledged"] = r.SubscriptionState.ChangeVectorForNextBatchStartingPoint,
["Query"] = r.SubscriptionState.Query
},
["Connection"] = GetSubscriptionConnectionDJV(r)
}).ToList()
});
writer.WriteArray(context, "Results", subscriptionsAsBlittable, (w, c, subscription) =>
{
c.Write(w, subscription);
});
writer.WriteEndObject();
}
}
return(Task.CompletedTask);
}
public GraphQueryDetailedReporter(BlittableJsonTextWriter writer, DocumentsOperationContext ctx)
{
_writer = writer;
_ctx = ctx;
}
public StreamJsonDocumentQueryResultWriter(HttpResponse response, Stream stream, JsonOperationContext context)
{
_context = context;
_writer = new BlittableJsonTextWriter(context, stream);
_response = response;
}
private async Task RequestHandler(HttpContext context)
{
var requestHandlerContext = new RequestHandlerContext
{
HttpContext = context
};
Exception exception = null;
Stopwatch sp = null;
try
{
context.Response.StatusCode = (int)HttpStatusCode.OK;
context.Response.Headers["Content-Type"] = "application/json; charset=utf-8";
context.Response.Headers[Constants.Headers.ServerVersion] = RavenVersionAttribute.Instance.AssemblyVersion;
if (_server.ServerStore.Initialized == false)
{
await _server.ServerStore.InitializationCompleted.WaitAsync();
}
sp = Stopwatch.StartNew();
await _router.HandlePath(requestHandlerContext);
sp.Stop();
}
catch (Exception e)
{
sp?.Stop();
exception = e;
CheckVersionAndWrapException(context, ref e);
MaybeSetExceptionStatusCode(context, _server.ServerStore, e);
if (context.RequestAborted.IsCancellationRequested)
{
return;
}
using (_server.ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext ctx))
{
var djv = new DynamicJsonValue
{
[nameof(ExceptionDispatcher.ExceptionSchema.Url)] = $"{context.Request.Path}{context.Request.QueryString}",
[nameof(ExceptionDispatcher.ExceptionSchema.Type)] = e.GetType().FullName,
[nameof(ExceptionDispatcher.ExceptionSchema.Message)] = e.Message,
[nameof(ExceptionDispatcher.ExceptionSchema.Error)] = e.ToString()
};
#if EXCEPTION_ERROR_HUNT
var f = Guid.NewGuid() + ".error";
File.WriteAllText(f,
$"{context.Request.Path}{context.Request.QueryString}" + Environment.NewLine + errorString);
#endif
MaybeAddAdditionalExceptionData(djv, e);
using (var writer = new BlittableJsonTextWriter(ctx, context.Response.Body))
{
var json = ctx.ReadObject(djv, "exception");
writer.WriteObject(json);
}
#if EXCEPTION_ERROR_HUNT
File.Delete(f);
#endif
}
}
finally
{
// check if TW has clients
if (TrafficWatchManager.HasRegisteredClients)
{
var database = requestHandlerContext.Database?.Name;
LogTrafficWatch(context, sp?.ElapsedMilliseconds ?? 0, database);
}
if (sp != null && requestHandlerContext.HttpContext.Response.StatusCode != (int)HttpStatusCode.SwitchingProtocols) // exclude web sockets
{
var requestDuration = sp.ElapsedMilliseconds;
requestHandlerContext.RavenServer?.Metrics.Requests.UpdateDuration(requestDuration);
requestHandlerContext.Database?.Metrics.Requests.UpdateDuration(requestDuration);
}
if (_logger.IsInfoEnabled && SkipHttpLogging == false)
{
_logger.Info($"{context.Request.Method} {context.Request.Path.Value}{context.Request.QueryString.Value} - {context.Response.StatusCode} - {(sp?.ElapsedMilliseconds ?? 0):#,#;;0} ms", exception);
}
}
}
public Task PreviewCollection()
{
var start = GetStart();
var pageSize = GetPageSize();
var collection = GetStringQueryString("collection", required: false);
var bindings = GetStringValuesQueryString("binding", required: false);
var fullBindings = GetStringValuesQueryString("fullBinding", required: false);
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
Document[] documents;
HashSet <LazyStringValue> availableColumns;
HashSet <string> propertiesPreviewToSend;
HashSet <string> fullPropertiesToSend = new HashSet <string>(fullBindings);
long totalResults;
string changeVector;
string etag = null;
if (string.IsNullOrEmpty(collection))
{
changeVector = DocumentsStorage.GetDatabaseChangeVector(context);
totalResults = Database.DocumentsStorage.GetNumberOfDocuments(context);
etag = $"{changeVector}/{totalResults}";
}
else
{
changeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context.Transaction.InnerTransaction, context, collection);
totalResults = Database.DocumentsStorage.GetCollection(collection, context).Count;
if (changeVector != null)
{
etag = $"{changeVector}/{totalResults}";
}
}
if (etag != null && GetStringFromHeaders("If-None-Match") == etag)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified;
return(Task.CompletedTask);
}
HttpContext.Response.Headers["ETag"] = "\"" + etag + "\"";
if (string.IsNullOrEmpty(collection))
{
documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, start, pageSize).ToArray();
availableColumns = ExtractColumnNames(documents, context);
propertiesPreviewToSend = bindings.Count > 0 ? new HashSet <string>(bindings) : new HashSet <string>();
}
else
{
documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, collection, start, pageSize).ToArray();
availableColumns = ExtractColumnNames(documents, context);
propertiesPreviewToSend = bindings.Count > 0 ? new HashSet <string>(bindings) : availableColumns.Take(ColumnsSamplingLimit).Select(x => x.ToString()).ToHashSet();
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
var first = true;
foreach (var document in documents)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
using (document.Data)
{
WriteDocument(writer, context, document, propertiesPreviewToSend, fullPropertiesToSend);
}
}
writer.WriteEndArray();
writer.WriteComma();
writer.WritePropertyName("TotalResults");
writer.WriteInteger(totalResults);
writer.WriteComma();
writer.WriteArray("AvailableColumns", availableColumns);
writer.WriteEndObject();
}
return(Task.CompletedTask);
}
}
public async Task Create()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var json = await context.ReadForMemoryAsync(RequestBodyStream(), null);
var options = JsonDeserializationServer.SubscriptionCreationParams(json);
if (TrafficWatchManager.HasRegisteredClients)
{
AddStringToHttpContext(json.ToString(), TrafficWatchChangeType.Subscriptions);
}
var sub = SubscriptionConnection.ParseSubscriptionQuery(options.Query);
if (Enum.TryParse(
options.ChangeVector,
out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
{
switch (changeVectorSpecialValue)
{
case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:
options.ChangeVector = null;
break;
case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
options.ChangeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context, sub.Collection);
break;
}
}
var id = GetLongQueryString("id", required: false);
var disabled = GetBoolValueQueryString("disabled", required: false);
var mentor = options.MentorNode;
var subscriptionId = await Database.SubscriptionStorage.PutSubscription(options, id, disabled, mentor : mentor);
var name = options.Name ?? subscriptionId.ToString();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext serverContext))
using (serverContext.OpenReadTransaction())
{
// need to wait on the relevant remote node
var node = Database.SubscriptionStorage.GetResponsibleNode(serverContext, name);
if (node != null && node != ServerStore.NodeTag)
{
await WaitForExecutionOnSpecificNode(serverContext, ServerStore.GetClusterTopology(serverContext), node, subscriptionId);
}
}
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created; // Created
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, new DynamicJsonValue
{
["Name"] = name
});
}
}
}