private void GetDocuments(DocumentsOperationContext context, Transformer transformer, bool metadataOnly)
{
// everything here operates on all docs
var actualEtag = ComputeAllDocumentsEtag(context);
if (GetLongFromHeaders("If-None-Match") == actualEtag)
{
HttpContext.Response.StatusCode = 304;
return;
}
HttpContext.Response.Headers["ETag"] = "\"" + actualEtag + "\"";
var etag = GetLongQueryString("etag", false);
IEnumerable <Document> documents;
bool isLoadStartingWith = false;
if (etag != null)
{
documents = Database.DocumentsStorage.GetDocumentsFrom(context, etag.Value, GetStart(), GetPageSize());
}
else if (HttpContext.Request.Query.ContainsKey("startsWith"))
{
isLoadStartingWith = true;
documents = Database.DocumentsStorage.GetDocumentsStartingWith(context,
HttpContext.Request.Query["startsWith"],
HttpContext.Request.Query["matches"],
HttpContext.Request.Query["excludes"],
GetStart(),
GetPageSize()
);
}
else // recent docs
{
documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, GetStart(), GetPageSize());
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
if (isLoadStartingWith)
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
}
if (transformer != null)
{
var transformerParameters = GetTransformerParameters(context);
using (var scope = transformer.OpenTransformationScope(transformerParameters, null, Database.DocumentsStorage,
Database.TransformerStore, context))
{
writer.WriteDocuments(context, scope.Transform(documents).ToList(), metadataOnly);
}
}
else
{
writer.WriteDocuments(context, documents, metadataOnly);
}
if (isLoadStartingWith)
{
writer.WriteEndObject();
}
}
}
public async Task Put()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var id = GetQueryStringValueAndAssertIfSingleAndNotEmpty("id");
var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name");
var contentType = GetStringQueryString("contentType", false) ?? "";
AttachmentDetails result;
using (var streamsTempFile = Database.DocumentsStorage.AttachmentsStorage.GetTempFile("put"))
using (var stream = streamsTempFile.StartNewStream())
{
Stream requestBodyStream = RequestBodyStream();
string hash;
try
{
hash = await AttachmentsStorageHelper.CopyStreamToFileAndCalculateHash(context, requestBodyStream, stream, Database.DatabaseShutdown);
}
catch (Exception)
{
try
{
// if we failed to read the entire request body stream, we might leave
// data in the pipe still, this will cause us to read and discard the
// rest of the attachment stream and return the actual error to the caller
requestBodyStream.CopyTo(Stream.Null);
}
catch (Exception)
{
// we tried, but we can't clean the request, so let's just kill
// the connection
HttpContext.Abort();
}
throw;
}
var changeVector = context.GetLazyString(GetStringFromHeaders("If-Match"));
var cmd = new MergedPutAttachmentCommand
{
Database = Database,
ExpectedChangeVector = changeVector,
DocumentId = id,
Name = name,
Stream = stream,
Hash = hash,
ContentType = contentType
};
stream.Flush();
await Database.TxMerger.Enqueue(cmd);
cmd.ExceptionDispatchInfo?.Throw();
result = cmd.Result;
}
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(AttachmentDetails.ChangeVector));
writer.WriteString(result.ChangeVector);
writer.WriteComma();
writer.WritePropertyName(nameof(AttachmentDetails.Name));
writer.WriteString(result.Name);
writer.WriteComma();
writer.WritePropertyName(nameof(AttachmentDetails.DocumentId));
writer.WriteString(result.DocumentId);
writer.WriteComma();
writer.WritePropertyName(nameof(AttachmentDetails.ContentType));
writer.WriteString(result.ContentType);
writer.WriteComma();
writer.WritePropertyName(nameof(AttachmentDetails.Hash));
writer.WriteString(result.Hash);
writer.WriteComma();
writer.WritePropertyName(nameof(AttachmentDetails.Size));
writer.WriteInteger(result.Size);
writer.WriteEndObject();
}
}
}
public async Task Patch()
{
var id = GetQueryStringValueAndAssertIfSingleAndNotEmpty("id");
var isTest = GetBoolValueQueryString("test", required: false) ?? false;
var debugMode = GetBoolValueQueryString("debug", required: false) ?? isTest;
var skipPatchIfChangeVectorMismatch = GetBoolValueQueryString("skipPatchIfChangeVectorMismatch", required: false) ?? false;
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var request = context.Read(RequestBodyStream(), "ScriptedPatchRequest");
if (request.TryGet("Patch", out BlittableJsonReaderObject patchCmd) == false || patchCmd == null)
{
throw new ArgumentException("The 'Patch' field in the body request is mandatory");
}
var patch = PatchRequest.Parse(patchCmd, out var patchArgs);
PatchRequest patchIfMissing = null;
BlittableJsonReaderObject patchIfMissingArgs = null;
if (request.TryGet("PatchIfMissing", out BlittableJsonReaderObject patchIfMissingCmd) && patchIfMissingCmd != null)
{
patchIfMissing = PatchRequest.Parse(patchIfMissingCmd, out patchIfMissingArgs);
}
var changeVector = context.GetLazyString(GetStringFromHeaders("If-Match"));
using (var command = new PatchDocumentCommand(context,
id,
changeVector,
skipPatchIfChangeVectorMismatch,
(patch, patchArgs),
(patchIfMissing, patchIfMissingArgs),
Database,
isTest,
debugMode,
true
))
{
if (isTest == false)
{
await Database.TxMerger.Enqueue(command);
}
else
{
// PutDocument requires the write access to the docs storage
// testing patching is rare enough not to optimize it
using (context.OpenWriteTransaction())
{
command.Execute(context);
}
}
switch (command.PatchResult.Status)
{
case PatchStatus.DocumentDoesNotExist:
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return;
case PatchStatus.Created:
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
break;
case PatchStatus.Skipped:
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified;
return;
case PatchStatus.Patched:
case PatchStatus.NotModified:
HttpContext.Response.StatusCode = (int)HttpStatusCode.OK;
break;
default:
throw new ArgumentOutOfRangeException();
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(command.PatchResult.Status));
writer.WriteString(command.PatchResult.Status.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(command.PatchResult.ModifiedDocument));
writer.WriteObject(command.PatchResult.ModifiedDocument);
if (debugMode)
{
writer.WriteComma();
writer.WritePropertyName(nameof(command.PatchResult.OriginalDocument));
if (isTest)
{
writer.WriteObject(command.PatchResult.OriginalDocument);
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(command.PatchResult.Debug));
context.Write(writer, new DynamicJsonValue
{
["Info"] = new DynamicJsonArray(command.DebugOutput),
["Actions"] = command.DebugActions?.GetDebugActions()
});
}
writer.WriteEndObject();
}
}
}
}
public Task ExtractInfoFromZip()
{
AssertOnlyInSetupMode();
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var continueSetupInfoJson = context.ReadForMemory(RequestBodyStream(), "continue-setup-info"))
{
var continueSetupInfo = JsonDeserializationServer.ContinueSetupInfo(continueSetupInfoJson);
byte[] zipBytes;
try
{
zipBytes = Convert.FromBase64String(continueSetupInfo.Zip);
}
catch (Exception e)
{
throw new ArgumentException($"Unable to parse the {nameof(continueSetupInfo.Zip)} property, expected a Base64 value", e);
}
try
{
var urlByTag = new Dictionary <string, string>();
using (var ms = new MemoryStream(zipBytes))
using (var archive = new ZipArchive(ms, ZipArchiveMode.Read, false))
{
foreach (var entry in archive.Entries)
{
if (entry.Name.Equals("settings.json") == false)
{
continue;
}
var tag = entry.FullName[0].ToString();
using (var settingsJson = context.ReadForMemory(entry.Open(), "settings-json"))
if (settingsJson.TryGet(nameof(ConfigurationNodeInfo.PublicServerUrl), out string publicServerUrl))
{
urlByTag[tag] = publicServerUrl;
}
}
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartArray();
var first = true;
foreach (var node in urlByTag)
{
if (first == false)
{
writer.WriteComma();
}
writer.WriteStartObject();
writer.WritePropertyName(nameof(ConfigurationNodeInfo.Tag));
writer.WriteString(node.Key);
writer.WriteComma();
writer.WritePropertyName(nameof(ConfigurationNodeInfo.PublicServerUrl));
writer.WriteString(node.Value);
writer.WriteEndObject();
first = false;
}
writer.WriteEndArray();
}
}
catch (Exception e)
{
throw new InvalidOperationException("Unable to extract setup information from the zip file.", e);
}
}
return(Task.CompletedTask);
}
public Task GetCollectionFields()
{
var collection = GetStringQueryString("collection", required: false);
var prefix = GetStringQueryString("prefix", required: false);
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
long totalResults;
string changeVector;
string etag = null;
if (string.IsNullOrEmpty(collection))
{
changeVector = DocumentsStorage.GetDatabaseChangeVector(context);
totalResults = Database.DocumentsStorage.GetNumberOfDocuments(context);
etag = $"{changeVector}/{totalResults}";
}
else
{
changeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context, collection);
totalResults = Database.DocumentsStorage.GetCollection(collection, context).Count;
if (changeVector != null)
{
etag = $"{changeVector}/{totalResults}";
}
}
if (etag != null && GetStringFromHeaders("If-None-Match") == etag)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified;
return(Task.CompletedTask);
}
HttpContext.Response.Headers["ETag"] = "\"" + etag + "\"";
if (_buffers == null)
{
_buffers = new BlittableJsonReaderObject.PropertiesInsertionBuffer();
}
var fields = new Dictionary <LazyStringValue, FieldType>(LazyStringValueComparer.Instance);
if (string.IsNullOrEmpty(collection))
{
foreach (var collectionStats in Database.DocumentsStorage.GetCollections(context))
{
FetchFieldsForCollection(context, collectionStats.Name, prefix, fields, _buffers);
}
}
else
{
FetchFieldsForCollection(context, collection, prefix, fields, _buffers);
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
var first = true;
foreach (var field in fields)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WritePropertyName(field.Key);
writer.WriteString(field.Value.ToString());
}
writer.WriteEndObject();
}
return(Task.CompletedTask);
}
}
public Task GetAll()
{
var start = GetStart();
var pageSize = GetPageSize();
var history = GetBoolValueQueryString("history", required: false) ?? false;
var running = GetBoolValueQueryString("running", required: false) ?? false;
var id = GetLongQueryString("id", required: false);
var name = GetStringQueryString("name", required: false);
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
IEnumerable <SubscriptionStorage.SubscriptionGeneralDataAndStats> subscriptions;
if (string.IsNullOrEmpty(name) && id == null)
{
subscriptions = running
? Database.SubscriptionStorage.GetAllRunningSubscriptions(context, history, start, pageSize)
: Database.SubscriptionStorage.GetAllSubscriptions(context, history, start, pageSize);
}
else
{
var subscription = running
? Database
.SubscriptionStorage
.GetRunningSubscription(context, id, name, history)
: Database
.SubscriptionStorage
.GetSubscription(context, id, name, history);
if (subscription == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return(Task.CompletedTask);
}
subscriptions = new[] { subscription };
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
var subscriptionsAsBlittable = subscriptions.Select(x => new DynamicJsonValue()
{
[nameof(SubscriptionState.SubscriptionId)] = x.SubscriptionId,
[nameof(SubscriptionState.SubscriptionName)] = x.SubscriptionName,
[nameof(SubscriptionState.ChangeVectorForNextBatchStartingPoint)] = x.ChangeVectorForNextBatchStartingPoint,
[nameof(SubscriptionState.Query)] = x.Query,
[nameof(SubscriptionState.Disabled)] = x.Disabled,
[nameof(SubscriptionState.LastClientConnectionTime)] = x.LastClientConnectionTime,
[nameof(SubscriptionState.LastBatchAckTime)] = x.LastBatchAckTime,
["Connection"] = GetSubscriptionConnectionDJV(x.Connection),
["RecentConnections"] = x.RecentConnections?.Select(r => new DynamicJsonValue()
{
["State"] = new DynamicJsonValue()
{
["LatestChangeVectorClientACKnowledged"] = r.SubscriptionState.ChangeVectorForNextBatchStartingPoint,
["Query"] = r.SubscriptionState.Query
},
["Connection"] = GetSubscriptionConnectionDJV(r)
}),
["FailedConnections"] = x.RecentRejectedConnections?.Select(r => new DynamicJsonValue()
{
["State"] = new DynamicJsonValue()
{
["LatestChangeVectorClientACKnowledged"] = r.SubscriptionState.ChangeVectorForNextBatchStartingPoint,
["Query"] = r.SubscriptionState.Query
},
["Connection"] = GetSubscriptionConnectionDJV(r)
}).ToList()
});
writer.WriteArray(context, "Results", subscriptionsAsBlittable, (w, c, subscription) =>
{
c.Write(w, subscription);
});
writer.WriteEndObject();
}
}
return(Task.CompletedTask);
}
public Task GetIps()
{
AssertOnlyInSetupMode();
NetworkInterface[] netInterfaces = null;
try
{
netInterfaces = NetworkInterface.GetAllNetworkInterfaces();
}
catch (Exception)
{
// https://github.com/dotnet/corefx/issues/26476
// If GetAllNetworkInterfaces is not supported, we'll just return the default: 127.0.0.1
}
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("MachineName");
writer.WriteString(Environment.MachineName);
writer.WriteComma();
writer.WritePropertyName("NetworkInterfaces");
writer.WriteStartArray();
var first = true;
List <string> ips;
if (netInterfaces != null)
{
foreach (var netInterface in netInterfaces)
{
ips = netInterface.GetIPProperties().UnicastAddresses
.Where(x =>
{
// filter 169.254.xxx.xxx out, they are not meaningful for binding
if (x.Address.AddressFamily != AddressFamily.InterNetwork)
{
return(false);
}
var addressBytes = x.Address.GetAddressBytes();
// filter 127.xxx.xxx.xxx out, in docker only
if (SetupParameters.Get(ServerStore).IsDocker&& addressBytes[0] == 127)
{
return(false);
}
return(addressBytes[0] != 169 || addressBytes[1] != 254);
})
.Select(addr => addr.Address.ToString())
.ToList();
// If there's a hostname in the server url, add it to the list
if (SetupParameters.Get(ServerStore).DockerHostname != null && ips.Contains(SetupParameters.Get(ServerStore).DockerHostname) == false)
{
ips.Add(SetupParameters.Get(ServerStore).DockerHostname);
}
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WriteStartObject();
writer.WritePropertyName("Name");
writer.WriteString(netInterface.Name);
writer.WriteComma();
writer.WritePropertyName("Description");
writer.WriteString(netInterface.Description);
writer.WriteComma();
writer.WriteArray("Addresses", ips);
writer.WriteEndObject();
}
}
else
{
// https://github.com/dotnet/corefx/issues/26476
// If GetAllNetworkInterfaces is not supported, we'll just return the default: 127.0.0.1
ips = new List <string>
{
"127.0.0.1"
};
writer.WriteStartObject();
writer.WritePropertyName("Name");
writer.WriteString("Loopback Interface");
writer.WriteComma();
writer.WritePropertyName("Description");
writer.WriteString("Loopback Interface");
writer.WriteComma();
writer.WriteArray("Addresses", ips);
writer.WriteEndObject();
}
writer.WriteEndArray();
writer.WriteEndObject();
}
return(Task.CompletedTask);
}
private static void WriteMemoryStats(BlittableJsonTextWriter writer, JsonOperationContext context, bool includeThreads, bool includeMappings)
{
writer.WriteStartObject();
var memInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
long managedMemoryInBytes = AbstractLowMemoryMonitor.GetManagedMemoryInBytes();
long totalUnmanagedAllocations = NativeMemory.TotalAllocatedMemory;
var encryptionBuffers = EncryptionBuffersPool.Instance.GetStats();
var dirtyMemoryState = MemoryInformation.GetDirtyMemoryState();
var memoryUsageRecords = MemoryInformation.GetMemoryUsageRecords();
long totalMapping = 0;
var fileMappingByDir = new Dictionary <string, Dictionary <string, ConcurrentDictionary <IntPtr, long> > >();
var fileMappingSizesByDir = new Dictionary <string, long>();
foreach (var mapping in NativeMemory.FileMapping)
{
var dir = Path.GetDirectoryName(mapping.Key);
if (fileMappingByDir.TryGetValue(dir, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value) == false)
{
value = new Dictionary <string, ConcurrentDictionary <IntPtr, long> >();
fileMappingByDir[dir] = value;
}
value[mapping.Key] = mapping.Value.Value.Info;
foreach (var singleMapping in mapping.Value.Value.Info)
{
fileMappingSizesByDir.TryGetValue(dir, out long prevSize);
fileMappingSizesByDir[dir] = prevSize + singleMapping.Value;
totalMapping += singleMapping.Value;
}
}
var djv = new DynamicJsonValue
{
[nameof(MemoryInfo.PhysicalMemory)] = memInfo.TotalPhysicalMemory.ToString(),
[nameof(MemoryInfo.WorkingSet)] = memInfo.WorkingSet.ToString(),
[nameof(MemoryInfo.ManagedAllocations)] = Size.Humane(managedMemoryInBytes),
[nameof(MemoryInfo.UnmanagedAllocations)] = Size.Humane(totalUnmanagedAllocations),
[nameof(MemoryInfo.EncryptionBuffersInUse)] = Size.Humane(encryptionBuffers.CurrentlyInUseSize),
[nameof(MemoryInfo.EncryptionBuffersPool)] = Size.Humane(encryptionBuffers.TotalPoolSize),
[nameof(MemoryInfo.EncryptionLockedMemory)] = Size.Humane(Sodium.LockedBytes),
[nameof(MemoryInfo.MemoryMapped)] = Size.Humane(totalMapping),
[nameof(MemoryInfo.ScratchDirtyMemory)] = memInfo.TotalScratchDirtyMemory.ToString(),
[nameof(MemoryInfo.IsHighDirty)] = dirtyMemoryState.IsHighDirty,
[nameof(MemoryInfo.DirtyMemory)] = Size.Humane(dirtyMemoryState.TotalDirtyInBytes),
[nameof(MemoryInfo.AvailableMemory)] = Size.Humane(memInfo.AvailableMemory.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.AvailableMemoryForProcessing)] = memInfo.AvailableMemoryForProcessing.ToString(),
[nameof(MemoryInfo.HighMemLastOneMinute)] = memoryUsageRecords.High.LastOneMinute.ToString(),
[nameof(MemoryInfo.LowMemLastOneMinute)] = memoryUsageRecords.Low.LastOneMinute.ToString(),
[nameof(MemoryInfo.HighMemLastFiveMinute)] = memoryUsageRecords.High.LastFiveMinutes.ToString(),
[nameof(MemoryInfo.LowMemLastFiveMinute)] = memoryUsageRecords.Low.LastFiveMinutes.ToString(),
[nameof(MemoryInfo.HighMemSinceStartup)] = memoryUsageRecords.High.SinceStartup.ToString(),
[nameof(MemoryInfo.LowMemSinceStartup)] = memoryUsageRecords.Low.SinceStartup.ToString(),
};
writer.WritePropertyName(nameof(MemoryInformation));
context.Write(writer, djv);
writer.WriteComma();
writer.WritePropertyName("Threads");
writer.WriteStartArray();
WriteThreads(includeThreads, writer, context);
writer.WriteEndArray();
writer.WriteComma();
writer.WritePropertyName(nameof(MemoryInfo.Mappings));
writer.WriteStartArray();
WriteMappings(includeMappings, writer, context, fileMappingSizesByDir, fileMappingByDir);
writer.WriteEndArray();
writer.WriteEndObject();
}
public Task QueriesCacheList()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
var queryCache = Database.QueryMetadataCache.GetQueryCache();
var queriesList = new List <DynamicJsonValue>();
writer.WriteStartObject();
writer.WritePropertyName("TotalCachedQueries");
writer.WriteInteger(queryCache.Length);
writer.WriteComma();
foreach (var item in queryCache)
{
if (item != null)
{
var curDjvItem = new DynamicJsonValue();
queriesList.Add(curDjvItem);
curDjvItem[nameof(Queries.QueryMetadata.CreatedAt)] = item.CreatedAt;
curDjvItem[nameof(Queries.QueryMetadata.LastQueriedAt)] = item.LastQueriedAt;
if (item.IsGroupBy)
{
curDjvItem[nameof(Queries.QueryMetadata.IsGroupBy)] = true;
}
if (item.IsDistinct)
{
curDjvItem[nameof(Queries.QueryMetadata.IsDistinct)] = true;
}
if (item.HasFacet)
{
curDjvItem[nameof(Queries.QueryMetadata.HasFacet)] = true;
}
if (item.HasMoreLikeThis)
{
curDjvItem[nameof(Queries.QueryMetadata.HasMoreLikeThis)] = true;
}
if (item.HasSuggest)
{
curDjvItem[nameof(Queries.QueryMetadata.HasSuggest)] = true;
}
if (item.OrderBy != null)
{
curDjvItem["Sorted"] = true;
}
if (item.IsOptimizedSortOnly)
{
curDjvItem[nameof(Queries.QueryMetadata.IsOptimizedSortOnly)] = true;
}
if (item.HasCmpXchg)
{
curDjvItem[nameof(Queries.QueryMetadata.HasCmpXchg)] = true;
}
if (item.HasExplanations)
{
curDjvItem[nameof(Queries.QueryMetadata.HasExplanations)] = true;
}
if (item.HasIntersect)
{
curDjvItem[nameof(Queries.QueryMetadata.HasIntersect)] = true;
}
if (item.IsDynamic)
{
curDjvItem[nameof(Queries.QueryMetadata.IsDynamic)] = true;
}
if (item.SelectFields.Any(x => x.Function != null))
{
curDjvItem["IsJSProjection"] = true;
}
if (string.IsNullOrEmpty(item.CollectionName) == false)
{
curDjvItem[nameof(Queries.QueryMetadata.CollectionName)] = item.CollectionName;
}
if (string.IsNullOrEmpty(item.AutoIndexName) == false)
{
curDjvItem[nameof(Queries.QueryMetadata.AutoIndexName)] = item.AutoIndexName;
}
if (string.IsNullOrEmpty(item.IndexName) == false)
{
curDjvItem[nameof(Queries.QueryMetadata.IndexName)] = item.IndexName;
}
curDjvItem[nameof(Queries.QueryMetadata.QueryText)] = item.QueryText;
}
}
writer.WriteArray("Results", queriesList, context);
writer.WriteEndObject();
}
return(Task.CompletedTask);
}
public Task StreamDocsGet()
{
var transformerName = GetStringQueryString("transformer", required: false);
Transformer transformer = null;
if (string.IsNullOrEmpty(transformerName) == false)
{
transformer = Database.TransformerStore.GetTransformer(transformerName);
if (transformer == null)
{
throw new TransformerDoesNotExistsException("No transformer with the name: " + transformerName);
}
}
DocumentsOperationContext context;
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out context))
using (context.OpenReadTransaction())
{
var etag = GetLongQueryString("etag", false);
IEnumerable <Document> documents;
if (etag != null)
{
documents = Database.DocumentsStorage.GetDocumentsFrom(context, etag.Value, GetStart(), GetPageSize());
}
else if (HttpContext.Request.Query.ContainsKey("startsWith"))
{
documents = Database.DocumentsStorage.GetDocumentsStartingWith(context,
HttpContext.Request.Query["startsWith"],
HttpContext.Request.Query["matches"],
HttpContext.Request.Query["excludes"],
GetStart(),
GetPageSize()
);
}
else // recent docs
{
documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, GetStart(), GetPageSize());
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
if (transformer != null)
{
var transformerParameters = GetTransformerParameters(context);
using (var scope = transformer.OpenTransformationScope(transformerParameters, null, Database.DocumentsStorage,
Database.TransformerStore, context))
{
writer.WriteDocuments(context, scope.Transform(documents).ToList(), metadataOnly: false);
}
}
else
{
writer.WriteDocuments(context, documents, metadataOnly: false);
}
writer.WriteEndObject();
}
}
return(Task.CompletedTask);
}
public void FullBackupTo(string backupPath)
{
using (var file = new FileStream(backupPath, FileMode.Create))
using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true))
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
var databaseRecord = _serverStore.Cluster.ReadDatabase(context, Name);
Debug.Assert(databaseRecord != null);
var zipArchiveEntry = package.CreateEntry(RestoreSettings.SmugglerValuesFileName, CompressionLevel.Optimal);
using (var zipStream = zipArchiveEntry.Open())
{
var smugglerSource = new DatabaseSource(this, 0);
using (DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var smugglerDestination = new StreamDestination(zipStream, ctx, smugglerSource);
var databaseSmugglerOptionsServerSide = new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = DatabaseItemType.CmpXchg | DatabaseItemType.Identities
};
var smuggler = new DatabaseSmuggler(this,
smugglerSource,
smugglerDestination,
this.Time,
options: databaseSmugglerOptionsServerSide);
smuggler.Execute();
}
}
zipArchiveEntry = package.CreateEntry(RestoreSettings.SettingsFileName, CompressionLevel.Optimal);
using (var zipStream = zipArchiveEntry.Open())
using (var writer = new BlittableJsonTextWriter(context, zipStream))
{
//TODO: encrypt this file using the MasterKey
//http://issues.hibernatingrhinos.com/issue/RavenDB-7546
writer.WriteStartObject();
// save the database record
writer.WritePropertyName(nameof(RestoreSettings.DatabaseRecord));
var databaseRecordBlittable = EntityToBlittable.ConvertEntityToBlittable(databaseRecord, DocumentConventions.Default, context);
context.Write(writer, databaseRecordBlittable);
// save the database values (subscriptions, periodic backups statuses, etl states...)
writer.WriteComma();
writer.WritePropertyName(nameof(RestoreSettings.DatabaseValues));
writer.WriteStartObject();
var first = true;
var prefix = Helpers.ClusterStateMachineValuesPrefix(Name);
foreach (var keyValue in ClusterStateMachine.ReadValuesStartingWith(context, prefix))
{
if (first == false)
{
writer.WriteComma();
}
first = false;
var key = keyValue.Key.ToString().Substring(prefix.Length);
writer.WritePropertyName(key);
context.Write(writer, keyValue.Value);
}
writer.WriteEndObject();
// end of values
writer.WriteEndObject();
}
BackupMethods.Full.ToFile(GetAllStoragesEnvironmentInformation(), package);
file.Flush(true); // make sure that we fully flushed to disk
}
}
public Task GetIps()
{
AssertOnlyInSetupMode();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("MachineName");
writer.WriteString(Environment.MachineName);
writer.WriteComma();
writer.WritePropertyName("NetworkInterfaces");
writer.WriteStartArray();
var first = true;
foreach (var netInterface in NetworkInterface.GetAllNetworkInterfaces())
{
var ips = netInterface.GetIPProperties().UnicastAddresses
.Where(x =>
{
// filter 169.254.xxx.xxx out, they are not meaningful for binding
if (x.Address.AddressFamily != AddressFamily.InterNetwork)
{
return(false);
}
var addressBytes = x.Address.GetAddressBytes();
// filter 127.xxx.xxx.xxx out, in docker only
if (SetupParameters.Get(ServerStore).IsDocker&& addressBytes[0] == 127)
{
return(false);
}
return(addressBytes[0] != 169 || addressBytes[1] != 254);
})
.Select(addr => addr.Address.ToString())
.ToList();
// If there's a hostname in the server url, add it to the list
if (SetupParameters.Get(ServerStore).DockerHostname != null && ips.Contains(SetupParameters.Get(ServerStore).DockerHostname) == false)
{
ips.Add(SetupParameters.Get(ServerStore).DockerHostname);
}
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WriteStartObject();
writer.WritePropertyName("Name");
writer.WriteString(netInterface.Name);
writer.WriteComma();
writer.WritePropertyName("Description");
writer.WriteString(netInterface.Description);
writer.WriteComma();
writer.WriteArray("Addresses", ips);
writer.WriteEndObject();
}
writer.WriteEndArray();
writer.WriteEndObject();
}
return(Task.CompletedTask);
}
private async Task HandleRequestAsync(
BlittableJsonReaderObject request,
JsonOperationContext context,
MultiGetHttpResponseStream responseStream,
BlittableJsonTextWriter writer,
HttpContext httpContext,
HostString host,
string scheme,
LazyStringValue resultProperty,
LazyStringValue statusProperty,
LazyStringValue headersProperty,
StringBuilder trafficWatchStringBuilder)
{
writer.WriteStartObject();
if (request.TryGet(nameof(GetRequest.Url), out string url) == false || request.TryGet(nameof(GetRequest.Query), out string query) == false)
{
writer.WriteEndObject();
return;
}
if (request.TryGet(nameof(GetRequest.Method), out string method) == false || string.IsNullOrEmpty(method))
{
method = HttpMethod.Get.Method;
}
httpContext.Request.Method = method;
var routeInformation = Server.Router.GetRoute(method, url, out RouteMatch localMatch);
if (routeInformation == null)
{
HandleNoRoute(context, writer, method, url, query, statusProperty, resultProperty);
return;
}
var requestHandler = routeInformation.GetRequestHandler();
writer.WritePropertyName(resultProperty);
writer.Flush();
PrepareHttpContext(request, context, httpContext, method, query, host, scheme, trafficWatchStringBuilder, out var content);
var bytesWrittenBeforeRequest = responseStream.BytesWritten;
int statusCode;
try
{
if (Server.Configuration.Security.AuthenticationEnabled == false ||
Server.Router.TryAuthorize(routeInformation, httpContext, Database, out var status))
{
await requestHandler(new RequestHandlerContext
{
Database = Database,
RavenServer = Server,
RouteMatch = localMatch,
HttpContext = httpContext
});
}
if (bytesWrittenBeforeRequest == responseStream.BytesWritten)
{
writer.WriteNull();
}
statusCode = httpContext.Response.StatusCode == 0
? (int)HttpStatusCode.OK
: httpContext.Response.StatusCode;
}
catch (Exception e)
{
if (bytesWrittenBeforeRequest != responseStream.BytesWritten)
{
throw;
}
statusCode = (int)HttpStatusCode.InternalServerError;
HandleException(context, writer, e, url, query);
}
writer.WriteComma();
writer.WritePropertyName(statusProperty);
writer.WriteInteger(statusCode);
writer.WriteComma();
WriteHeaders(writer, httpContext, headersProperty);
writer.WriteEndObject();
trafficWatchStringBuilder?.Append(content).AppendLine();
}
public Task Patch()
{
var id = GetQueryStringValueAndAssertIfSingleAndNotEmpty("id");
var etag = GetLongFromHeaders("If-Match");
var isTestOnly = GetBoolValueQueryString("test", required: false) ?? false;
DocumentsOperationContext context;
using (ContextPool.AllocateOperationContext(out context))
{
var request = context.Read(RequestBodyStream(), "ScriptedPatchRequest");
BlittableJsonReaderObject patchCmd, patchIsMissingCmd;
if (request.TryGet("Patch", out patchCmd) == false)
{
throw new ArgumentException("The 'Patch' field in the body request is mandatory");
}
var patch = PatchRequest.Parse(patchCmd);
PatchRequest patchIfMissing = null;
if (request.TryGet("PatchIfMissing", out patchIsMissingCmd))
{
patchIfMissing = PatchRequest.Parse(patchCmd);
}
// TODO: In order to properly move this to the transaction merger, we need
// TODO: move a lot of the costs (such as script parsing) out, so we create
// TODO: an object that we'll apply, otherwise we'll slow down a lot the transactions
// TODO: just by doing the javascript parsing and preparing the engine
PatchResultData patchResult;
using (context.OpenWriteTransaction())
{
patchResult = Database.Patch.Apply(context, id, etag, patch, patchIfMissing, isTestOnly);
context.Transaction.Commit();
}
Debug.Assert(patchResult.PatchResult == PatchResult.Patched == isTestOnly == false);
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName(("Patched"));
writer.WriteBool(isTestOnly == false);
writer.WriteComma();
writer.WritePropertyName(("Debug"));
writer.WriteObject(patchResult.ModifiedDocument);
if (isTestOnly)
{
writer.WriteComma();
writer.WritePropertyName(("Document"));
writer.WriteObject(patchResult.OriginalDocument);
}
writer.WriteEndObject();
}
}
return(Task.CompletedTask);
}
public async Task TestJavaScriptIndex()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var input = await context.ReadForMemoryAsync(RequestBodyStream(), "TestJavaScriptIndex");
if (input.TryGet("Definition", out BlittableJsonReaderObject index) == false)
{
ThrowRequiredPropertyNameInRequest("Definition");
}
input.TryGet("Ids", out BlittableJsonReaderArray ids);
var indexDefinition = JsonDeserializationServer.IndexDefinition(index);
if (indexDefinition.Maps == null || indexDefinition.Maps.Count == 0)
{
throw new ArgumentException("Index must have a 'Maps' fields");
}
indexDefinition.Type = indexDefinition.DetectStaticIndexType();
if (indexDefinition.Type.IsJavaScript() == false)
{
throw new UnauthorizedAccessException("Testing indexes is only allowed for JavaScript indexes.");
}
var compiledIndex = new JavaScriptIndex(indexDefinition, Database.Configuration);
var inputSize = GetIntValueQueryString("inputSize", false) ?? defaultInputSizeForTestingJavaScriptIndex;
var collections = new HashSet <string>(compiledIndex.Maps.Keys);
var docsPerCollection = new Dictionary <string, List <DynamicBlittableJson> >();
using (context.OpenReadTransaction())
{
if (ids == null)
{
foreach (var collection in collections)
{
docsPerCollection.Add(collection,
Database.DocumentsStorage.GetDocumentsFrom(context, collection, 0, 0, inputSize).Select(d => new DynamicBlittableJson(d)).ToList());
}
}
else
{
var listOfIds = ids.Select(x => x.ToString());
var _ = new Reference <int>
{
Value = 0
};
var docs = Database.DocumentsStorage.GetDocuments(context, listOfIds, 0, int.MaxValue, _);
foreach (var doc in docs)
{
if (doc.TryGetMetadata(out var metadata) && metadata.TryGet(Constants.Documents.Metadata.Collection, out string collectionStr))
{
if (docsPerCollection.TryGetValue(collectionStr, out var listOfDocs) == false)
{
listOfDocs = docsPerCollection[collectionStr] = new List <DynamicBlittableJson>();
}
listOfDocs.Add(new DynamicBlittableJson(doc));
}
}
}
var mapRes = new List <ObjectInstance>();
//all maps
foreach (var ListOfFunctions in compiledIndex.Maps)
{
//multi maps per collection
foreach (var mapFunc in ListOfFunctions.Value)
{
if (docsPerCollection.TryGetValue(ListOfFunctions.Key, out var docs))
{
foreach (var res in mapFunc(docs))
{
mapRes.Add((ObjectInstance)res);
}
}
}
}
var first = true;
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("MapResults");
writer.WriteStartArray();
foreach (var mapResult in mapRes)
{
if (JavaScriptIndexUtils.StringifyObject(mapResult) is JsString jsStr)
{
if (first == false)
{
writer.WriteComma();
}
writer.WriteString(jsStr.ToString());
first = false;
}
}
writer.WriteEndArray();
if (indexDefinition.Reduce != null)
{
using (var bufferPool = new UnmanagedBuffersPoolWithLowMemoryHandling("JavaScriptIndexTest", Database.Name))
{
compiledIndex.SetBufferPoolForTestingPurposes(bufferPool);
compiledIndex.SetAllocatorForTestingPurposes(context.Allocator);
first = true;
writer.WritePropertyName("ReduceResults");
writer.WriteStartArray();
var reduceResults = compiledIndex.Reduce(mapRes.Select(mr => new DynamicBlittableJson(JsBlittableBridge.Translate(context, mr.Engine, mr))));
foreach (JsValue reduceResult in reduceResults)
{
if (JavaScriptIndexUtils.StringifyObject(reduceResult) is JsString jsStr)
{
if (first == false)
{
writer.WriteComma();
}
writer.WriteString(jsStr.ToString());
first = false;
}
}
}
writer.WriteEndArray();
}
writer.WriteEndObject();
}
}
}
}
public Task RunningQueries()
{
var indexes = Database
.IndexStore
.GetIndexes()
.ToList();
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
var isFirst = true;
foreach (var index in indexes)
{
if (isFirst == false)
{
writer.WriteComma();
}
isFirst = false;
writer.WritePropertyName(index.Name);
writer.WriteStartArray();
var isFirstInternal = true;
foreach (var query in index.CurrentlyRunningQueries)
{
if (isFirstInternal == false)
{
writer.WriteComma();
}
isFirstInternal = false;
writer.WriteStartObject();
writer.WritePropertyName((nameof(query.Duration)));
writer.WriteString(query.Duration.ToString());
writer.WriteComma();
writer.WritePropertyName((nameof(query.QueryId)));
writer.WriteInteger(query.QueryId);
writer.WriteComma();
writer.WritePropertyName((nameof(query.StartTime)));
writer.WriteDateTime(query.StartTime, isUtc: true);
writer.WriteComma();
writer.WritePropertyName((nameof(query.QueryInfo)));
writer.WriteIndexQuery(context, query.QueryInfo);
writer.WriteEndObject();
}
writer.WriteEndArray();
}
writer.WriteEndObject();
}
return(Task.CompletedTask);
}
public async Task Try()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
BlittableJsonReaderObject json = await context.ReadForMemoryAsync(RequestBodyStream(), null);
SubscriptionTryout tryout = JsonDeserializationServer.SubscriptionTryout(json);
SubscriptionConnection.ParsedSubscription sub = SubscriptionConnection.ParseSubscriptionQuery(tryout.Query);
SubscriptionPatchDocument patch = null;
if (string.IsNullOrEmpty(sub.Script) == false)
{
patch = new SubscriptionPatchDocument(sub.Script, sub.Functions);
}
if (sub.Collection == null)
{
throw new ArgumentException("Collection must be specified");
}
const int maxPageSize = 1024;
var pageSize = GetIntValueQueryString("pageSize") ?? 1;
if (pageSize > maxPageSize)
{
throw new ArgumentException($"Cannot gather more than {maxPageSize} results during tryouts, but requested number was {pageSize}.");
}
var state = new SubscriptionState
{
ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector,
Query = tryout.Query
};
var fetcher = new SubscriptionDocumentsFetcher(Database, int.MaxValue, -0x42,
new IPEndPoint(HttpContext.Connection.RemoteIpAddress, HttpContext.Connection.RemotePort), sub.Collection, sub.Revisions, state, patch);
var includeCmd = new IncludeDocumentsCommand(Database.DocumentsStorage, context, sub.Includes, isProjection: patch != null);
if (Enum.TryParse(
tryout.ChangeVector,
out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
{
switch (changeVectorSpecialValue)
{
case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:
case Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange:
state.ChangeVectorForNextBatchStartingPoint = null;
break;
case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
using (context.OpenReadTransaction())
{
state.ChangeVectorForNextBatchStartingPoint = Database.DocumentsStorage.GetLastDocumentChangeVector(context, sub.Collection);
}
break;
}
}
else
{
state.ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector;
}
var changeVector = state.ChangeVectorForNextBatchStartingPoint.ToChangeVector();
var cv = changeVector.FirstOrDefault(x => x.DbId == Database.DbBase64Id);
var sp = Stopwatch.StartNew();
var timeLimit = TimeSpan.FromSeconds(GetIntValueQueryString("timeLimit", false) ?? 15);
var startEtag = cv.Etag;
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
using (context.OpenReadTransaction())
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
var numberOfDocs = 0;
while (numberOfDocs == 0 && sp.Elapsed < timeLimit)
{
var first = true;
var lastEtag = startEtag;
foreach (var itemDetails in fetcher.GetDataToSend(context, includeCmd, startEtag))
{
if (itemDetails.Doc.Data != null)
{
using (itemDetails.Doc.Data)
{
includeCmd.Gather(itemDetails.Doc);
if (first == false)
{
writer.WriteComma();
}
if (itemDetails.Exception == null)
{
writer.WriteDocument(context, itemDetails.Doc, metadataOnly: false);
}
else
{
var documentWithException = new DocumentWithException
{
Exception = itemDetails.Exception.ToString(),
ChangeVector = itemDetails.Doc.ChangeVector,
Id = itemDetails.Doc.Id,
DocumentData = itemDetails.Doc.Data
};
writer.WriteObject(context.ReadObject(documentWithException.ToJson(), ""));
}
first = false;
if (++numberOfDocs >= pageSize)
{
break;
}
}
}
if (sp.Elapsed >= timeLimit)
{
break;
}
lastEtag = itemDetails.Doc.Etag;
}
if (startEtag == lastEtag)
{
break;
}
startEtag = lastEtag;
}
writer.WriteEndArray();
writer.WriteComma();
writer.WritePropertyName("Includes");
var includes = new List <Document>();
includeCmd.Fill(includes);
writer.WriteIncludes(context, includes);
writer.WriteEndObject();
}
}
}
private void WriteDocument(BlittableJsonTextWriter writer, JsonOperationContext context, Document document, HashSet <string> propertiesPreviewToSend, HashSet <string> fullPropertiesToSend)
{
writer.WriteStartObject();
document.Data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata);
bool first = true;
var arrayStubsJson = new DynamicJsonValue();
var objectStubsJson = new DynamicJsonValue();
var trimmedValue = new HashSet <LazyStringValue>();
var prop = new BlittableJsonReaderObject.PropertyDetails();
using (var buffers = document.Data.GetPropertiesByInsertionOrder())
{
for (int i = 0; i < buffers.Properties.Count; i++)
{
document.Data.GetPropertyByIndex(buffers.Properties.Array[i + buffers.Properties.Offset], ref prop);
var sendFull = fullPropertiesToSend.Contains(prop.Name);
if (sendFull || propertiesPreviewToSend.Contains(prop.Name))
{
var strategy = sendFull ? ValueWriteStrategy.Passthrough : FindWriteStrategy(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
if (strategy == ValueWriteStrategy.Passthrough || strategy == ValueWriteStrategy.Trim)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
}
switch (strategy)
{
case ValueWriteStrategy.Passthrough:
writer.WritePropertyName(prop.Name);
writer.WriteValue(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
break;
case ValueWriteStrategy.SubstituteWithArrayStub:
arrayStubsJson[prop.Name] = ((BlittableJsonReaderArray)prop.Value).Length;
break;
case ValueWriteStrategy.SubstituteWithObjectStub:
objectStubsJson[prop.Name] = ((BlittableJsonReaderObject)prop.Value).Count;
break;
case ValueWriteStrategy.Trim:
writer.WritePropertyName(prop.Name);
WriteTrimmedValue(writer, prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
trimmedValue.Add(prop.Name);
break;
}
}
}
}
if (first == false)
{
writer.WriteComma();
}
var extraMetadataProperties = new DynamicJsonValue
{
[ObjectStubsKey] = objectStubsJson,
[ArrayStubsKey] = arrayStubsJson,
[TrimmedValueKey] = new DynamicJsonArray(trimmedValue)
};
if (metadata != null)
{
metadata.Modifications = extraMetadataProperties;
metadata = context.ReadObject(metadata, document.Id);
}
else
{
metadata = context.ReadObject(extraMetadataProperties, document.Id);
}
writer.WriteMetadata(document, metadata);
writer.WriteEndObject();
}
public Task CommitNonLazyTx()
{
var modeStr = GetQueryStringValueAndAssertIfSingleAndNotEmpty("mode");
TransactionsMode mode;
if (Enum.TryParse(modeStr, true, out mode) == false)
{
throw new InvalidOperationException("Query string value 'mode' is not a valid mode: " + modeStr);
}
var configDuration = TimeSpan.FromMinutes(Database.Configuration.Storage.TransactionsModeDuration);
var duration = GetTimeSpanQueryString("duration", required: false) ?? configDuration;
JsonOperationContext context;
using (ContextPool.AllocateOperationContext(out context))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName(("Environments"));
writer.WriteStartArray();
bool first = true;
foreach (var storageEnvironment in Database.GetAllStoragesEnvironment())
{
if (storageEnvironment == null)
{
continue;
}
if (first == false)
{
writer.WriteComma();
}
first = false;
var result = storageEnvironment.Environment.SetTransactionMode(mode, duration);
switch (result)
{
case TransactionsModeResult.ModeAlreadySet:
context.Write(writer, new DynamicJsonValue
{
["Type"] = mode.ToString(),
["Path"] = storageEnvironment.Environment.Options.BasePath,
["Result"] = "Mode Already Set"
});
break;
case TransactionsModeResult.SetModeSuccessfully:
context.Write(writer, new DynamicJsonValue
{
["Type"] = mode.ToString(),
["Path"] = storageEnvironment.Environment.Options.BasePath,
["Result"] = "Mode Set Successfully"
});
break;
default:
throw new ArgumentOutOfRangeException("Result is unexpected value: " + result);
}
}
writer.WriteEndArray();
writer.WriteEndObject();
}
return(Task.CompletedTask);
}
public Task PreviewCollection()
{
var start = GetStart();
var pageSize = GetPageSize();
var collection = GetStringQueryString("collection", required: false);
var bindings = GetStringValuesQueryString("binding", required: false);
var fullBindings = GetStringValuesQueryString("fullBinding", required: false);
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
Document[] documents;
HashSet <LazyStringValue> availableColumns;
HashSet <string> propertiesPreviewToSend;
HashSet <string> fullPropertiesToSend = new HashSet <string>(fullBindings);
long totalResults;
string changeVector;
string etag = null;
if (string.IsNullOrEmpty(collection))
{
changeVector = DocumentsStorage.GetDatabaseChangeVector(context);
totalResults = Database.DocumentsStorage.GetNumberOfDocuments(context);
etag = $"{changeVector}/{totalResults}";
}
else
{
changeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context, collection);
totalResults = Database.DocumentsStorage.GetCollection(collection, context).Count;
if (changeVector != null)
{
etag = $"{changeVector}/{totalResults}";
}
}
if (etag != null && GetStringFromHeaders("If-None-Match") == etag)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified;
return(Task.CompletedTask);
}
HttpContext.Response.Headers["ETag"] = "\"" + etag + "\"";
if (string.IsNullOrEmpty(collection))
{
documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, start, pageSize).ToArray();
availableColumns = ExtractColumnNames(documents, context);
propertiesPreviewToSend = bindings.Count > 0 ? new HashSet <string>(bindings) : new HashSet <string>();
}
else
{
documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, collection, start, pageSize).ToArray();
availableColumns = ExtractColumnNames(documents, context);
propertiesPreviewToSend = bindings.Count > 0 ? new HashSet <string>(bindings) : availableColumns.Take(ColumnsSamplingLimit).Select(x => x.ToString()).ToHashSet();
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
var first = true;
foreach (var document in documents)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
using (document.Data)
{
WriteDocument(writer, context, document, propertiesPreviewToSend, fullPropertiesToSend);
}
}
writer.WriteEndArray();
writer.WriteComma();
writer.WritePropertyName("TotalResults");
writer.WriteInteger(totalResults);
writer.WriteComma();
writer.WriteArray("AvailableColumns", availableColumns);
writer.WriteEndObject();
}
return(Task.CompletedTask);
}
}
public Task GetHosts()
{
AssertOnlyInSetupMode();
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var certificateJson = context.ReadForMemory(RequestBodyStream(), "setup-certificate"))
{
var certDef = JsonDeserializationServer.CertificateDefinition(certificateJson);
X509Certificate2 certificate = null;
string cn;
try
{
certificate = certDef.Password == null
? new X509Certificate2(Convert.FromBase64String(certDef.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet)
: new X509Certificate2(Convert.FromBase64String(certDef.Certificate), certDef.Password, X509KeyStorageFlags.MachineKeySet);
cn = certificate.GetNameInfo(X509NameType.SimpleName, false);
}
catch (Exception e)
{
throw new BadRequestException($"Failed to extract the CN property from the certificate {certificate?.FriendlyName}. Maybe the password is wrong?", e);
}
if (cn == null)
{
throw new BadRequestException($"Failed to extract the CN property from the certificate. CN is null");
}
if (cn.LastIndexOf('*') > 0)
{
throw new NotSupportedException("The wildcard CN name contains a '*' which is not at the first character of the string. It is not supported in the Setup Wizard, you can do a manual setup instead.");
}
try
{
SecretProtection.ValidateKeyUsages("Setup Wizard", certificate);
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to load the uploaded certificate. Did you accidently upload a client certificate?", e);
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("CN");
writer.WriteString(cn);
writer.WriteComma();
writer.WritePropertyName("AlternativeNames");
writer.WriteStartArray();
var first = true;
foreach (var value in SetupManager.GetCertificateAlternativeNames(certificate))
{
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WriteString(value);
}
writer.WriteEndArray();
writer.WriteEndObject();
}
}
return(Task.CompletedTask);
}
public static void WriteIndexQuery(this BlittableJsonTextWriter writer, DocumentConventions conventions, JsonOperationContext context, IndexQuery query)
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(query.Query));
writer.WriteString(query.Query);
writer.WriteComma();
#pragma warning disable 618
if (query.PageSizeSet && query.PageSize >= 0)
{
writer.WritePropertyName(nameof(query.PageSize));
writer.WriteInteger(query.PageSize);
writer.WriteComma();
}
#pragma warning restore 618
if (query.WaitForNonStaleResults)
{
writer.WritePropertyName(nameof(query.WaitForNonStaleResults));
writer.WriteBool(query.WaitForNonStaleResults);
writer.WriteComma();
}
#pragma warning disable 618
if (query.Start > 0)
{
writer.WritePropertyName(nameof(query.Start));
writer.WriteInteger(query.Start);
writer.WriteComma();
}
#pragma warning restore 618
if (query.WaitForNonStaleResultsTimeout.HasValue)
{
writer.WritePropertyName(nameof(query.WaitForNonStaleResultsTimeout));
writer.WriteString(query.WaitForNonStaleResultsTimeout.Value.ToInvariantString());
writer.WriteComma();
}
if (query.DisableCaching)
{
writer.WritePropertyName(nameof(query.DisableCaching));
writer.WriteBool(query.DisableCaching);
writer.WriteComma();
}
if (query.SkipDuplicateChecking)
{
writer.WritePropertyName(nameof(query.SkipDuplicateChecking));
writer.WriteBool(query.SkipDuplicateChecking);
writer.WriteComma();
}
writer.WritePropertyName(nameof(query.QueryParameters));
if (query.QueryParameters != null)
{
writer.WriteObject(conventions.Serialization.DefaultConverter.ToBlittable(query.QueryParameters, context));
}
else
{
writer.WriteNull();
}
if (query.ProjectionBehavior.HasValue && query.ProjectionBehavior.Value != ProjectionBehavior.Default)
{
writer.WriteComma();
writer.WritePropertyName(nameof(query.ProjectionBehavior));
writer.WriteString(query.ProjectionBehavior.ToString());
}
writer.WriteEndObject();
}
public async Task PostMultiGet()
{
using (ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var input = await context.ReadForMemoryAsync(RequestBodyStream(), "multi_get");
if (input.TryGet("Requests", out BlittableJsonReaderArray requests) == false)
{
ThrowRequiredPropertyNameInRequest("Requests");
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
var resultProperty = context.GetLazyStringForFieldWithCaching(nameof(GetResponse.Result));
var statusProperty = context.GetLazyStringForFieldWithCaching(nameof(GetResponse.StatusCode));
var headersProperty = context.GetLazyStringForFieldWithCaching(nameof(GetResponse.Headers));
var features = new FeatureCollection(HttpContext.Features);
var responseStream = new MultiGetHttpResponseStream(ResponseBodyStream());
features.Set <IHttpResponseFeature>(new MultiGetHttpResponseFeature(responseStream));
var httpContext = new DefaultHttpContext(features);
for (int i = 0; i < requests.Length; i++)
{
var request = (BlittableJsonReaderObject)requests[i];
if (i != 0)
{
writer.WriteComma();
}
writer.WriteStartObject();
if (request.TryGet("Url", out string url) == false || request.TryGet("Query", out string query) == false)
{
writer.WriteEndObject();
continue;
}
if (request.TryGet("Method", out string method) == false || string.IsNullOrEmpty(method))
{
method = HttpMethod.Get.Method;
}
httpContext.Request.Method = method;
var routeInformation = Server.Router.GetRoute(method, url, out RouteMatch localMatch);
if (routeInformation == null)
{
writer.WritePropertyName(statusProperty);
writer.WriteInteger((int)HttpStatusCode.BadRequest);
writer.WritePropertyName(resultProperty);
context.Write(writer, new DynamicJsonValue
{
["Error"] = $"There is no handler for path: {method} {url}{query}"
});
writer.WriteEndObject();
continue;
}
var requestHandler = routeInformation.GetRequestHandler();
writer.WritePropertyName(resultProperty);
writer.Flush();
httpContext.Response.StatusCode = 0;
httpContext.Request.Headers.Clear();
httpContext.Response.Headers.Clear();
httpContext.Request.QueryString = new QueryString(query);
if (request.TryGet("Headers", out BlittableJsonReaderObject headers))
{
foreach (var header in headers.GetPropertyNames())
{
if (headers.TryGet(header, out string value) == false)
{
continue;
}
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
httpContext.Request.Headers.Add(header, value);
}
}
if (method == HttpMethod.Post.Method && request.TryGet("Content", out object content))
{
if (content is LazyStringValue)
{
var requestBody = GetRequestBody(content.ToString());
HttpContext.Response.RegisterForDispose(requestBody);
httpContext.Request.Body = requestBody;
}
else
{
var requestBody = new MemoryStream();
var contentWriter = new BlittableJsonTextWriter(context, requestBody);
context.Write(contentWriter, (BlittableJsonReaderObject)content);
contentWriter.Flush();
HttpContext.Response.RegisterForDispose(requestBody);
httpContext.Request.Body = requestBody;
httpContext.Request.Body.Position = 0;
}
}
var bytesWrittenBeforeRequest = responseStream.BytesWritten;
int statusCode;
try
{
await requestHandler(new RequestHandlerContext
{
Database = Database,
RavenServer = Server,
RouteMatch = localMatch,
HttpContext = httpContext
});
if (bytesWrittenBeforeRequest == responseStream.BytesWritten)
{
writer.WriteNull();
}
statusCode = httpContext.Response.StatusCode == 0
? (int)HttpStatusCode.OK
: httpContext.Response.StatusCode;
}
catch (Exception e)
{
if (bytesWrittenBeforeRequest != responseStream.BytesWritten)
{
throw;
}
statusCode = (int)HttpStatusCode.InternalServerError;
var djv = new DynamicJsonValue
{
[nameof(ExceptionDispatcher.ExceptionSchema.Url)] = $"{url}{query}",
[nameof(ExceptionDispatcher.ExceptionSchema.Type)] = e.GetType().FullName,
[nameof(ExceptionDispatcher.ExceptionSchema.Message)] = e.Message,
[nameof(ExceptionDispatcher.ExceptionSchema.Error)] = e.ToString()
};
using (var json = context.ReadObject(djv, "exception"))
writer.WriteObject(json);
}
writer.WriteComma();
writer.WritePropertyName(statusProperty);
writer.WriteInteger(statusCode);
writer.WriteComma();
writer.WritePropertyName(headersProperty);
writer.WriteStartObject();
bool headerStart = true;
foreach (var header in httpContext.Response.Headers)
{
foreach (var value in header.Value)
{
if (headerStart == false)
{
writer.WriteComma();
}
headerStart = false;
writer.WritePropertyName(header.Key);
writer.WriteString(value);
}
}
writer.WriteEndObject();
writer.WriteEndObject();
}
writer.WriteEndArray();
writer.WriteEndObject();
}
}
}
public Task Stats()
{
var documentId = GetStringQueryString("docId");
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var document = Database.DocumentsStorage.Get(context, documentId, DocumentFields.Data);
if (document == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return(Task.CompletedTask);
}
var timeSeriesNames = GetTimesSeriesNames(document);
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(TimeSeriesStatistics.DocumentId));
writer.WriteString(documentId);
writer.WriteComma();
writer.WritePropertyName(nameof(TimeSeriesStatistics.TimeSeries));
writer.WriteStartArray();
var first = true;
foreach (var tsName in timeSeriesNames)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
var stats = Database.DocumentsStorage.TimeSeriesStorage.Stats.GetStats(context, documentId, tsName);
writer.WriteStartObject();
writer.WritePropertyName(nameof(TimeSeriesItemDetail.Name));
writer.WriteString(tsName);
writer.WriteComma();
writer.WritePropertyName(nameof(TimeSeriesItemDetail.NumberOfEntries));
writer.WriteInteger(stats.Count);
writer.WriteComma();
writer.WritePropertyName(nameof(TimeSeriesItemDetail.StartDate));
writer.WriteDateTime(stats.Start, isUtc: true);
writer.WriteComma();
writer.WritePropertyName(nameof(TimeSeriesItemDetail.EndDate));
writer.WriteDateTime(stats.End, isUtc: true);
writer.WriteEndObject();
}
writer.WriteEndArray();
writer.WriteEndObject();
}
}
return(Task.CompletedTask);
}
public Task GetCertificates()
{
var thumbprint = GetStringQueryString("thumbprint", required: false);
var showSecondary = GetBoolValueQueryString("secondary", required: false) ?? false;
var start = GetStart();
var pageSize = GetPageSize();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
var certificates = new List <(string ItemName, BlittableJsonReaderObject Value)>();
try
{
if (string.IsNullOrEmpty(thumbprint))
{
if (ServerStore.CurrentRachisState == RachisState.Passive)
{
foreach (var localCertKey in ServerStore.Cluster.GetCertificateKeysFromLocalState(context))
{
var localCertificate = ServerStore.Cluster.GetLocalState(context, localCertKey);
if (localCertificate == null)
{
continue;
}
var def = JsonDeserializationServer.CertificateDefinition(localCertificate);
if (showSecondary || string.IsNullOrEmpty(def.CollectionPrimaryKey))
{
certificates.Add((localCertKey, localCertificate));
}
else
{
localCertificate.Dispose();
}
}
}
else
{
foreach (var item in ServerStore.Cluster.ItemsStartingWith(context, Constants.Certificates.Prefix, start, pageSize))
{
var def = JsonDeserializationServer.CertificateDefinition(item.Value);
if (showSecondary || string.IsNullOrEmpty(def.CollectionPrimaryKey))
{
certificates.Add(item);
}
else
{
item.Value.Dispose();
}
}
}
}
else
{
var key = Constants.Certificates.Prefix + thumbprint;
var certificate = ServerStore.CurrentRachisState == RachisState.Passive
? ServerStore.Cluster.GetLocalState(context, key)
: ServerStore.Cluster.Read(context, key);
if (certificate == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return(Task.CompletedTask);
}
var definition = JsonDeserializationServer.CertificateDefinition(certificate);
if (string.IsNullOrEmpty(definition.CollectionPrimaryKey) == false)
{
certificate = ServerStore.Cluster.Read(context, definition.CollectionPrimaryKey);
if (certificate == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return(Task.CompletedTask);
}
}
certificates.Add((key, certificate));
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WriteArray(context, "Results", certificates.ToArray(), (w, c, cert) =>
{
c.Write(w, cert.Value);
});
writer.WriteEndObject();
}
}
finally
{
foreach (var cert in certificates)
{
cert.Value?.Dispose();
}
}
}
return(Task.CompletedTask);
}
public RecoveryStatus Execute(CancellationToken ct)
{
var sw = new Stopwatch();
StorageEnvironment se = null;
sw.Start();
if (_copyOnWrite)
{
Console.WriteLine("Recovering journal files, this may take a while...");
try
{
se = new StorageEnvironment(_option);
Console.WriteLine(
$"Journal recovery has completed successfully within {sw.Elapsed.TotalSeconds:N1} seconds");
}
catch (Exception e)
{
Console.WriteLine($"Journal recovery failed, reason:{Environment.NewLine}{e}");
}
finally
{
se?.Dispose();
}
}
_option = StorageEnvironmentOptions.ForPath(Path.GetDirectoryName(_datafile));
var mem = Pager.AcquirePagePointer(null, 0);
long startOffset = (long)mem;
var fi = new FileInfo(_datafile);
var fileSize = fi.Length;
//making sure eof is page aligned
var eof = mem + (fileSize / _pageSize) * _pageSize;
DateTime lastProgressReport = DateTime.MinValue;
using (var destinationStream = File.OpenWrite(_output))
using (var logFile = File.CreateText(Path.Combine(Path.GetDirectoryName(_output), LogFileName)))
using (var gZipStream = new GZipStream(destinationStream, CompressionMode.Compress, true))
using (var context = new JsonOperationContext(_initialContextSize, _initialContextLongLivedSize))
using (var writer = new BlittableJsonTextWriter(context, gZipStream))
{
WriteSmugglerHeader(writer);
while (mem < eof)
{
try
{
if (ct.IsCancellationRequested)
{
logFile.WriteLine(
$"Cancellation requested while recovery was in position {GetFilePosition(startOffset, mem)}");
_cancellationRequested = true;
break;
}
var now = DateTime.UtcNow;
if ((now - lastProgressReport).TotalSeconds >= _progressIntervalInSeconds)
{
if (lastProgressReport != DateTime.MinValue)
{
Console.Clear();
Console.WriteLine("Press 'q' to quit the recovery process");
}
lastProgressReport = now;
var currPos = GetFilePosition(startOffset, mem);
var eofPos = GetFilePosition(startOffset, eof);
Console.WriteLine(
$"{now:hh:MM:ss}: Recovering page at position {currPos:#,#;;0}/{eofPos:#,#;;0} ({(double) currPos/eofPos:p}) - Last recovered doc is {_lastRecoveredDocumentKey}");
}
var pageHeader = (PageHeader *)mem;
//this page is not raw data section move on
if ((pageHeader->Flags).HasFlag(PageFlags.RawData) == false)
{
mem += _pageSize;
continue;
}
if (pageHeader->Flags.HasFlag(PageFlags.Single) &&
pageHeader->Flags.HasFlag(PageFlags.Overflow))
{
var message =
$"page #{pageHeader->PageNumber} (offset={GetFilePosition(startOffset, mem)}) has both Overflow and Single flag turned";
mem = PrintErrorAndAdvanceMem(message, mem, logFile);
continue;
}
//overflow page
if (pageHeader->Flags.HasFlag(PageFlags.Overflow))
{
var endOfOverflow = pageHeader +
Pager.GetNumberOfOverflowPages(pageHeader->OverflowSize) * _pageSize;
// the endOfOeverFlow can be equal to eof if the last page is overflow
if (endOfOverflow > eof)
{
var message =
$"Overflow page #{pageHeader->PageNumber} (offset={GetFilePosition(startOffset, mem)})" +
$" size exceeds the end of the file ([{(long) pageHeader}:{(long) endOfOverflow}])";
mem = PrintErrorAndAdvanceMem(message, mem, logFile);
continue;
}
//Should we increase the check size to page size (0=>_pageSize)?
if (pageHeader->OverflowSize <= 0)
{
var message =
$"Overflow page #{pageHeader->PageNumber} (offset={GetFilePosition(startOffset, mem)})" +
$" OverflowSize is not a positive number ({pageHeader->OverflowSize})";
mem = PrintErrorAndAdvanceMem(message, mem, logFile);
continue;
}
if (WriteDocument((byte *)pageHeader + sizeof(PageHeader), pageHeader->OverflowSize,
writer,
logFile, context, startOffset))
{
var numberOfPages = Pager.GetNumberOfOverflowPages(pageHeader->OverflowSize);
mem += numberOfPages * _pageSize;
}
else
//write document failed
{
mem += _pageSize;
}
continue;
}
// small raw data section
var rawHeader = (RawDataSmallPageHeader *)mem;
if (rawHeader->RawDataFlags.HasFlag(RawDataPageFlags.Header))
{
mem += _pageSize;
continue;
}
if (rawHeader->NextAllocation > _pageSize)
{
var message =
$"RawDataSmallPage #{rawHeader->PageNumber} at {GetFilePosition(startOffset, mem)} next allocation is larger than {_pageSize} bytes";
mem = PrintErrorAndAdvanceMem(message, mem, logFile);
continue;
}
for (var pos = sizeof(PageHeader); pos < rawHeader->NextAllocation;)
{
var debug = GetFilePosition(startOffset, mem);
var currMem = mem + pos;
var entry = (RawDataSection.RawDataEntrySizes *)currMem;
//this indicates that the current entry is invalid because it is outside the size of a page
if (pos > _pageSize)
{
var message =
$"RawDataSmallPage #{rawHeader->PageNumber} has an invalid entry at {GetFilePosition(startOffset, currMem)}";
mem = PrintErrorAndAdvanceMem(message, mem, logFile);
//we can't retrive entries past the invalid entry
break;
}
//Allocated size of entry exceed the bound of the page next allocation
if (entry->AllocatedSize + pos + sizeof(RawDataSection.RawDataEntrySizes) >
rawHeader->NextAllocation)
{
var message =
$"RawDataSmallPage #{rawHeader->PageNumber} has an invalid entry at {GetFilePosition(startOffset, currMem)}" +
"the allocated entry exceed the bound of the page next allocation.";
mem = PrintErrorAndAdvanceMem(message, mem, logFile);
//we can't retrive entries past the invalid entry
break;
}
if (entry->UsedSize > entry->AllocatedSize)
{
var message =
$"RawDataSmallPage #{rawHeader->PageNumber} has an invalid entry at {GetFilePosition(startOffset, currMem)}" +
"the size of the entry exceed the allocated size";
mem = PrintErrorAndAdvanceMem(message, mem, logFile);
//we can't retrive entries past the invalid entry
break;
}
pos += entry->AllocatedSize + sizeof(RawDataSection.RawDataEntrySizes);
if (entry->AllocatedSize == 0 || entry->UsedSize == -1)
{
continue;
}
if (
WriteDocument(currMem + sizeof(RawDataSection.RawDataEntrySizes), entry->UsedSize,
writer, logFile, context, startOffset) == false)
{
break;
}
}
mem += _pageSize;
}
catch (Exception e)
{
var message =
$"Unexpected exception at position {GetFilePosition(startOffset, mem)}:{Environment.NewLine} {e}";
mem = PrintErrorAndAdvanceMem(message, mem, logFile);
}
}
writer.WriteEndArray();
writer.WriteEndObject();
logFile.WriteLine(
$"Discovered a total of {_numberOfDocumentsRetrived:#,#;00} documents within {sw.Elapsed.TotalSeconds::#,#.#;;00} seconds.");
logFile.WriteLine($"Discovered a total of {_numberOfFaultedPages::#,#;00} faulted pages.");
}
if (_cancellationRequested)
{
return(RecoveryStatus.CancellationRequested);
public Task Debug()
{
var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name");
var index = Database.IndexStore.GetIndex(name);
if (index == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return(Task.CompletedTask);
}
var operation = GetStringQueryString("op");
using (ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
if (string.Equals(operation, "map-reduce-tree", StringComparison.OrdinalIgnoreCase))
{
if (index.Type.IsMapReduce() == false)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.BadRequest;
context.Write(writer, new DynamicJsonValue
{
["Error"] = $"{index.Name} is not map-reduce index"
});
return(Task.CompletedTask);
}
var docIds = GetStringValuesQueryString("docId", required: false);
using (index.GetReduceTree(docIds.ToArray(), out IEnumerable <ReduceTree> trees))
{
writer.WriteReduceTrees(trees);
}
return(Task.CompletedTask);
}
if (string.Equals(operation, "source-doc-ids", StringComparison.OrdinalIgnoreCase))
{
using (index.GetIdentifiersOfMappedDocuments(GetStringQueryString("startsWith", required: false), GetStart(), GetPageSize(), out IEnumerable <string> ids))
{
writer.WriteArrayOfResultsAndCount(ids);
}
return(Task.CompletedTask);
}
if (string.Equals(operation, "entries-fields", StringComparison.OrdinalIgnoreCase))
{
var fields = index.GetEntriesFields();
writer.WriteStartObject();
writer.WriteArray("Results", fields);
writer.WriteEndObject();
return(Task.CompletedTask);
}
throw new NotSupportedException($"{operation} is not supported");
}
}
public Task Stats()
{
var name = GetStringQueryString("name", required: false);
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
IndexStats[] indexStats;
using (context.OpenReadTransaction())
{
if (string.IsNullOrEmpty(name))
{
indexStats = Database.IndexStore
.GetIndexes()
.OrderBy(x => x.Name)
.Select(x =>
{
try
{
return(x.GetStats(calculateLag: true, calculateStaleness: true, documentsContext: context));
}
catch (Exception e)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations($"Failed to get stats of '{x.Name}' index", e);
}
try
{
Database.NotificationCenter.Add(AlertRaised.Create(Database.Name, $"Failed to get stats of '{x.Name}' index",
$"Exception was thrown on getting stats of '{x.Name}' index",
AlertType.Indexing_CouldNotGetStats, NotificationSeverity.Error, key: x.Name, details: new ExceptionDetails(e)));
}
catch (Exception addAlertException)
{
if (Logger.IsOperationsEnabled && addAlertException.IsOutOfMemory() == false && addAlertException.IsDiskFullException() == false)
{
Logger.Operations($"Failed to add alert when getting error on retrieving stats of '{x.Name}' index", addAlertException);
}
}
var state = x.State;
if (e.IsOutOfMemory() == false && e.IsDiskFullException() == false)
{
try
{
state = IndexState.Error;
x.SetState(state, inMemoryOnly: true);
}
catch (Exception ex)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations($"Failed to change state of '{x.Name}' index to error after encountering exception when getting its stats.",
ex);
}
}
}
return(new IndexStats
{
Name = x.Name,
Type = x.Type,
State = state,
Status = x.Status,
LockMode = x.Definition.LockMode,
Priority = x.Definition.Priority,
});
}
})
.ToArray();
}
else
{
var index = Database.IndexStore.GetIndex(name);
if (index == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return(Task.CompletedTask);
}
indexStats = new[] { index.GetStats(calculateLag: true, calculateStaleness: true, documentsContext: context) };
}
}
writer.WriteStartObject();
writer.WriteArray(context, "Results", indexStats, (w, c, stats) =>
{
w.WriteIndexStats(context, stats);
});
writer.WriteEndObject();
}
return(Task.CompletedTask);
}
/// <summary>
/// Iterates on a batch in document collection, process it and send documents if found any match
/// </summary>
/// <param name="docsContext"></param>
/// <param name="sendingCurrentBatchStopwatch"></param>
/// <returns>Whether succeeded finding any documents to send</returns>
private async Task <bool> TrySendingBatchToClient(DocumentsOperationContext docsContext, Stopwatch sendingCurrentBatchStopwatch)
{
bool anyDocumentsSentInCurrentIteration = false;
int docsToFlush = 0;
using (var writer = new BlittableJsonTextWriter(docsContext, _buffer))
{
using (docsContext.OpenReadTransaction())
{
foreach (var result in _documentsFetcher.GetDataToSend(docsContext, _startEtag))
{
_startEtag = result.Doc.Etag;
_lastChangeVector = string.IsNullOrEmpty(SubscriptionState.ChangeVectorForNextBatchStartingPoint)
? result.Doc.ChangeVector
: ChangeVectorUtils.MergeVectors(result.Doc.ChangeVector, SubscriptionState.ChangeVectorForNextBatchStartingPoint);
if (result.Doc.Data == null)
{
if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
await SendHeartBeat();
sendingCurrentBatchStopwatch.Restart();
}
continue;
}
anyDocumentsSentInCurrentIteration = true;
writer.WriteStartObject();
writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(TypeSegment));
writer.WriteValue(BlittableJsonToken.String, docsContext.GetLazyStringForFieldWithCaching(DataSegment));
writer.WriteComma();
writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(DataSegment));
result.Doc.EnsureMetadata();
if (result.Exception != null)
{
var metadata = result.Doc.Data[Client.Constants.Documents.Metadata.Key];
writer.WriteValue(BlittableJsonToken.StartObject,
docsContext.ReadObject(new DynamicJsonValue
{
[Client.Constants.Documents.Metadata.Key] = metadata
}, result.Doc.Id)
);
writer.WriteComma();
writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(ExceptionSegment));
writer.WriteValue(BlittableJsonToken.String, docsContext.GetLazyStringForFieldWithCaching(result.Exception.ToString()));
}
else
{
writer.WriteDocument(docsContext, result.Doc, metadataOnly: false);
}
writer.WriteEndObject();
docsToFlush++;
// perform flush for current batch after 1000ms of running or 1 MB
if (_buffer.Length > Constants.Size.Megabyte ||
sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
if (docsToFlush > 0)
{
await FlushDocsToClient(writer, docsToFlush);
docsToFlush = 0;
sendingCurrentBatchStopwatch.Restart();
}
else
{
await SendHeartBeat();
}
}
}
}
if (anyDocumentsSentInCurrentIteration)
{
docsContext.Write(writer, new DynamicJsonValue
{
[nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.EndOfBatch)
});
await FlushDocsToClient(writer, docsToFlush, true);
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Finished sending a batch with {docsToFlush} documents for subscription {Options.SubscriptionName}");
}
}
}
return(anyDocumentsSentInCurrentIteration);
}
public async Task Put()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var id = GetQueryStringValueAndAssertIfSingleAndNotEmpty("id");
var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name");
var contentType = GetStringQueryString("contentType", false) ?? "";
AttachmentDetails result;
using (var streamsTempFile = Database.DocumentsStorage.AttachmentsStorage.GetTempFile("put"))
using (var stream = streamsTempFile.StartNewStream())
{
var hash = await AttachmentsStorageHelper.CopyStreamToFileAndCalculateHash(context, RequestBodyStream(), stream, Database.DatabaseShutdown);
var changeVector = context.GetLazyString(GetStringQueryString("If-Match", false));
var cmd = new MergedPutAttachmentCommand
{
Database = Database,
ExpectedChangeVector = changeVector,
DocumentId = id,
Name = name,
Stream = stream,
Hash = hash,
ContentType = contentType
};
await Database.TxMerger.Enqueue(cmd);
cmd.ExceptionDispatchInfo?.Throw();
result = cmd.Result;
}
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(AttachmentDetails.ChangeVector));
writer.WriteString(result.ChangeVector);
writer.WriteComma();
writer.WritePropertyName(nameof(AttachmentDetails.Name));
writer.WriteString(result.Name);
writer.WriteComma();
writer.WritePropertyName(nameof(AttachmentDetails.DocumentId));
writer.WriteString(result.DocumentId);
writer.WriteComma();
writer.WritePropertyName(nameof(AttachmentDetails.ContentType));
writer.WriteString(result.ContentType);
writer.WriteComma();
writer.WritePropertyName(nameof(AttachmentDetails.Hash));
writer.WriteString(result.Hash);
writer.WriteComma();
writer.WritePropertyName(nameof(AttachmentDetails.Size));
writer.WriteInteger(result.Size);
writer.WriteEndObject();
}
}
}