public static async Task <ClusterTopologyResponse> TryLoadAsync(string topologyHash, DocumentConventions conventions, JsonOperationContext context)
{
try
{
if (conventions.DisableTopologyCache)
{
return(null);
}
var path = GetPath(topologyHash, conventions);
if (File.Exists(path) == false)
{
return(null);
}
using (var stream = SafeFileStream.Create(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var json = await context.ReadForMemoryAsync(stream, "raven-cluster-topology").ConfigureAwait(false))
{
return(JsonDeserializationClient.ClusterTopology(json));
}
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Could not understand the persisted cluster topology", e);
}
return(null);
}
}
private static async Task <BlittableJsonReaderObject> GetJson(JsonOperationContext context, HttpResponseMessage response, Stream stream)
{
BlittableJsonReaderObject json;
try
{
json = await context.ReadForMemoryAsync(stream, "error/response").ConfigureAwait(false);
}
catch (Exception e)
{
string content = null;
if (stream.CanSeek)
{
stream.Position = 0;
using (var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 4096, leaveOpen: true))
content = await reader.ReadToEndAsync().ConfigureAwait(false);
}
if (content != null)
{
content = $"Content: {content}";
}
throw new InvalidOperationException($"Cannot parse the '{response.StatusCode}' response. {content}", e);
}
return(json);
}
private async Task <int> SupervisorReadResponseAndGetVersionAsync(JsonOperationContext ctx, BlittableJsonTextWriter writer, Stream stream, string url, CancellationToken ct)
{
using (var responseJson = await ctx.ReadForMemoryAsync(stream, _readStatusUpdateDebugString + "/Read-Handshake-Response"))
{
var headerResponse = JsonDeserializationServer.TcpConnectionHeaderResponse(responseJson);
switch (headerResponse.Status)
{
case TcpConnectionStatus.Ok:
return(headerResponse.Version);
case TcpConnectionStatus.AuthorizationFailed:
throw new AuthorizationException(
$"Node with ClusterTag = {ClusterTag} replied to initial handshake with authorization failure {headerResponse.Message}");
case TcpConnectionStatus.TcpVersionMismatch:
if (headerResponse.Version != -1)
{
return(headerResponse.Version);
}
//Kindly request the server to drop the connection
WriteOperationHeaderToRemote(writer, headerResponse.Version, drop: true);
throw new InvalidOperationException($"Node with ClusterTag = {ClusterTag} replied to initial handshake with mismatching tcp version {headerResponse.Message}");
default:
throw new InvalidOperationException($"{url} replied with unknown status {headerResponse.Status}, message:{headerResponse.Message}");
}
}
}
private async Task <IndexQueryServerSide> GetIndexQuery(JsonOperationContext context, HttpMethod method, RequestTimeTracker tracker)
{
if (method == HttpMethod.Get)
{
return(IndexQueryServerSide.Create(HttpContext, GetStart(), GetPageSize(), context, tracker));
}
var json = await context.ReadForMemoryAsync(RequestBodyStream(), "index/query");
return(IndexQueryServerSide.Create(HttpContext, json, Database.QueryMetadataCache, tracker));
}
public static async Task <(List <Facet> Facets, long FacetsEtag)> ParseFromStringAsync(string facetsArrayAsString, JsonOperationContext context)
{
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(facetsArrayAsString)))
{
var input = await context.ReadForMemoryAsync(stream, "facets");
if (input.TryGet("Facets", out BlittableJsonReaderArray array) == false)
{
RequestHandler.ThrowRequiredPropertyNameInRequest("Facets");
}
return(ParseFromJson(array));
}
}
public async Task <BlittableJsonReaderObject> InvokeAndReadObjectAsync(RouteInformation route, JsonOperationContext context, Dictionary <string, Microsoft.Extensions.Primitives.StringValues> parameters = null)
{
var response = await InvokeAsync(route, parameters);
try
{
return(await context.ReadForMemoryAsync(response.Body, $"read/local endpoint/{route.Path}"));
}
catch (InvalidStartOfObjectException e)
{
//precaution, ideally this exception should never be thrown
throw new InvalidOperationException("Expected to find a blittable object as a result of debug endpoint, but found something else (see inner exception for details). This should be investigated as all RavenDB endpoints are supposed to return an object.", e);
}
}
private async Task <IndexQueryServerSide> GetIndexQuery(JsonOperationContext context, HttpMethod method, RequestTimeTracker tracker)
{
var addSpatialProperties = GetBoolValueQueryString("addSpatialProperties", required: false) ?? false;
var clientQueryId = GetStringQueryString("clientQueryId", required: false);
if (method == HttpMethod.Get)
{
return(await IndexQueryServerSide.CreateAsync(HttpContext, GetStart(), GetPageSize(), context, tracker, addSpatialProperties, clientQueryId));
}
var json = await context.ReadForMemoryAsync(RequestBodyStream(), "index/query");
return(IndexQueryServerSide.Create(HttpContext, json, Database.QueryMetadataCache, tracker, addSpatialProperties, clientQueryId, Database));
}
private static async Task <BlittableJsonReaderObject> GetJson(JsonOperationContext context, HttpResponseMessage response, Stream stream)
{
BlittableJsonReaderObject json;
try
{
json = await context.ReadForMemoryAsync(stream, "error/response").ConfigureAwait(false);
}
catch (Exception e)
{
stream.Position = 0;
throw new InvalidOperationException($"Cannot parse the {response.StatusCode} response. Content: {new StreamReader(stream).ReadToEnd()}", e);
}
return(json);
}
internal static async Task InvokeAndWriteToArchive(ZipArchive archive, JsonOperationContext jsonOperationContext, LocalEndpointClient localEndpointClient,
RouteInformation route, string path,
Dictionary <string, Microsoft.Extensions.Primitives.StringValues> endpointParameters = null,
CancellationToken token = default)
{
try
{
var response = await localEndpointClient.InvokeAsync(route, endpointParameters);
var entryName = DebugInfoPackageUtils.GetOutputPathFromRouteInformation(route, path, response.ContentType == "text/plain" ? "txt" : "json");
var entry = archive.CreateEntry(entryName);
entry.ExternalAttributes = ((int)(FilePermissions.S_IRUSR | FilePermissions.S_IWUSR)) << 16;
using (var entryStream = entry.Open())
{
if (response.ContentType == "text/plain")
{
await response.Body.CopyToAsync(entryStream, token);
}
else
{
await using (var writer = new AsyncBlittableJsonTextWriter(jsonOperationContext, entryStream))
{
var endpointOutput = await jsonOperationContext.ReadForMemoryAsync(response.Body, $"read/local endpoint/{route.Path}");
jsonOperationContext.Write(writer, endpointOutput);
await writer.FlushAsync();
}
}
await entryStream.FlushAsync(token);
}
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
//precaution, ideally this exception should never be thrown
if (e is InvalidStartOfObjectException)
{
e = new InvalidOperationException("Expected to find a blittable object as a result of debug endpoint, but found something else (see inner exception for details). This should be investigated as all RavenDB endpoints are supposed to return an object.", e);
}
await DebugInfoPackageUtils.WriteExceptionAsZipEntryAsync(e, archive, DebugInfoPackageUtils.GetOutputPathFromRouteInformation(route, path, null));
}
}
private static async Task ThrowServerError(JsonOperationContext context, HttpResponseMessage response)
{
using (var stream = await ReadAsStreamUncompressedAsync(response))
{
BlittableJsonReaderObject blittableJsonReaderObject;
try
{
blittableJsonReaderObject = await context.ReadForMemoryAsync(stream, "ErrorResponse");
}
catch (Exception e)
{
stream.Position = 0;
throw new InvalidOperationException(
$"Cannot parse the {response.StatusCode} response: {new StreamReader(stream).ReadToEnd()}", e);
}
stream.Position = 0;
using (blittableJsonReaderObject)
{
string error;
if (blittableJsonReaderObject.TryGet("Error", out error) == false)
{
throw new InvalidOperationException(
$"Doesn't know how to handle error: {response.StatusCode}, response: {new StreamReader(stream).ReadToEnd()}");
}
if (response.StatusCode == HttpStatusCode.BadRequest)
{
throw new BadRequestException(error + ". Response: " + blittableJsonReaderObject);
}
string indexDefinitionProperty;
if (blittableJsonReaderObject.TryGet(nameof(IndexCompilationException.IndexDefinitionProperty),
out indexDefinitionProperty))
{
var indexCompilationException = new IndexCompilationException(error);
blittableJsonReaderObject.TryGet(nameof(IndexCompilationException.IndexDefinitionProperty),
out indexCompilationException.IndexDefinitionProperty);
blittableJsonReaderObject.TryGet(nameof(IndexCompilationException.ProblematicText),
out indexCompilationException.ProblematicText);
throw indexCompilationException;
}
throw new InternalServerErrorException(error + ". Response: " + blittableJsonReaderObject);
}
}
}
public virtual async Task ProcessResponse(JsonOperationContext context, HttpCache cache, HttpResponseMessage response, string url)
{
using (response)
using (var stream = await response.Content.ReadAsStreamAsync())
{
// we intentionally don't dispose the reader here, we'll be using it
// in the command, any associated memory will be released on context reset
var blittableJsonReaderObject = await context.ReadForMemoryAsync(stream, "PutResult");
if (response.Headers.ETag != null)
{
long?etag = response.GetEtagHeader();
if (etag != null)
{
cache.Set(url, (long)etag, blittableJsonReaderObject);
}
}
SetResponse(blittableJsonReaderObject);
}
}
private async ValueTask <TcpConnectionHeaderMessage.NegotiationResponse> SupervisorReadResponseAndGetVersionAsync(JsonOperationContext ctx, AsyncBlittableJsonTextWriter writer, Stream stream, string url)
{
using (var responseJson = await ctx.ReadForMemoryAsync(stream, _readStatusUpdateDebugString + "/Read-Handshake-Response"))
{
var headerResponse = JsonDeserializationServer.TcpConnectionHeaderResponse(responseJson);
switch (headerResponse.Status)
{
case TcpConnectionStatus.Ok:
return(new TcpConnectionHeaderMessage.NegotiationResponse
{
Version = headerResponse.Version,
LicensedFeatures = headerResponse.LicensedFeatures
});
case TcpConnectionStatus.AuthorizationFailed:
throw new AuthorizationException(
$"Node with ClusterTag = {ClusterTag} replied to initial handshake with authorization failure {headerResponse.Message}");
case TcpConnectionStatus.TcpVersionMismatch:
if (headerResponse.Version != TcpNegotiation.OutOfRangeStatus)
{
return(new TcpConnectionHeaderMessage.NegotiationResponse
{
Version = headerResponse.Version,
LicensedFeatures = headerResponse.LicensedFeatures
});
}
//Kindly request the server to drop the connection
await WriteOperationHeaderToRemoteAsync(writer, headerResponse.Version, drop : true);
throw new InvalidOperationException($"Node with ClusterTag = {ClusterTag} replied to initial handshake with mismatching tcp version {headerResponse.Message}");
case TcpConnectionStatus.InvalidNetworkTopology:
throw new AuthorizationException(
$"Node with ClusterTag = {ClusterTag} replied to initial handshake with {nameof(TcpConnectionStatus.InvalidNetworkTopology)} error {headerResponse.Message}");
default:
throw new InvalidOperationException($"{url} replied with unknown status {headerResponse.Status}, message:{headerResponse.Message}");
}
}
}
private static async Task <Topology> TryLoadAsync(string path, JsonOperationContext context)
{
try
{
if (File.Exists(path) == false)
{
return(null);
}
using (var stream = SafeFileStream.Create(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var json = await context.ReadForMemoryAsync(stream, "raven-database-topology").ConfigureAwait(false))
{
return(JsonDeserializationClient.Topology(json));
}
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Could not understand the persisted database topology", e);
}
return(null);
}
}
protected async Task <RestoreSettings> SnapshotRestore(JsonOperationContext context, string backupPath,
Action <IOperationProgress> onProgress, RestoreResult restoreResult)
{
Debug.Assert(onProgress != null);
RestoreSettings restoreSettings = null;
var fullBackupPath = GetBackupPath(backupPath);
using (var zip = await GetZipArchiveForSnapshot(fullBackupPath))
{
var restorePath = new VoronPathSetting(RestoreFromConfiguration.DataDirectory);
if (Directory.Exists(restorePath.FullPath) == false)
{
Directory.CreateDirectory(restorePath.FullPath);
}
// validate free space
var snapshotSize = zip.Entries.Sum(entry => entry.Length);
BackupHelper.AssertFreeSpaceForSnapshot(restorePath.FullPath, snapshotSize, "restore a backup", Logger);
foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length)))
{
var directory = zipEntries.Key;
if (string.IsNullOrWhiteSpace(directory))
{
foreach (var zipEntry in zipEntries)
{
if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase))
{
await using (var entryStream = zipEntry.Open())
{
var snapshotEncryptionKey = RestoreFromConfiguration.EncryptionKey != null
? Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey)
: null;
await using (var stream = GetInputStream(entryStream, snapshotEncryptionKey))
{
var json = await context.ReadForMemoryAsync(stream, "read database settings for restore");
json.BlittableValidation();
restoreSettings = JsonDeserializationServer.RestoreSettings(json);
restoreSettings.DatabaseRecord.DatabaseName = RestoreFromConfiguration.DatabaseName;
DatabaseHelper.Validate(RestoreFromConfiguration.DatabaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration);
if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false)
{
throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!");
}
if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey)
{
throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!");
}
}
}
}
}
continue;
}
var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase)
? restorePath
: restorePath.Combine(directory);
BackupMethods.Full.Restore(
zipEntries,
restoreDirectory,
journalDir: null,
onProgress: message =>
{
restoreResult.AddInfo(message);
restoreResult.SnapshotRestore.ReadCount++;
onProgress.Invoke(restoreResult.Progress);
},
cancellationToken: _operationCancelToken.Token);
}
}
if (restoreSettings == null)
{
throw new InvalidDataException("Cannot restore the snapshot without the settings file!");
}
return(restoreSettings);
}
protected async Task <RestoreSettings> SnapshotRestore(JsonOperationContext context, string backupPath,
Action <IOperationProgress> onProgress, RestoreResult restoreResult)
{
Debug.Assert(onProgress != null);
RestoreSettings restoreSettings = null;
var fullBackupPath = GetBackupPath(backupPath);
using (var zip = await GetZipArchiveForSnapshot(fullBackupPath))
{
var restorePath = new VoronPathSetting(RestoreFromConfiguration.DataDirectory);
if (Directory.Exists(restorePath.FullPath) == false)
{
Directory.CreateDirectory(restorePath.FullPath);
}
// validate free space
var snapshotSize = zip.Entries.Sum(entry => entry.Length);
BackupHelper.AssertFreeSpaceForSnapshot(restorePath.FullPath, snapshotSize, "restore a backup", Logger);
foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length)))
{
var directory = zipEntries.Key;
if (string.IsNullOrWhiteSpace(directory))
{
foreach (var zipEntry in zipEntries)
{
if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase))
{
await using (var entryStream = zipEntry.Open())
{
var snapshotEncryptionKey = RestoreFromConfiguration.EncryptionKey != null
? Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey)
: null;
await using (var stream = GetInputStream(entryStream, snapshotEncryptionKey))
{
var json = await context.ReadForMemoryAsync(stream, "read database settings for restore");
json.BlittableValidation();
restoreSettings = JsonDeserializationServer.RestoreSettings(json);
restoreSettings.DatabaseRecord.DatabaseName = RestoreFromConfiguration.DatabaseName;
DatabaseHelper.Validate(RestoreFromConfiguration.DatabaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration);
if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false)
{
throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!");
}
if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey)
{
throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!");
}
}
}
}
}
continue;
}
var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase)
? restorePath
: restorePath.Combine(directory);
var isSubDirectory = PathUtil.IsSubDirectory(restoreDirectory.FullPath, restorePath.FullPath);
if (isSubDirectory == false)
{
var extensions = zipEntries
.Select(x => Path.GetExtension(x.Name))
.Distinct()
.ToArray();
if (extensions.Length != 1 || string.Equals(extensions[0], TableValueCompressor.CompressionRecoveryExtension, StringComparison.OrdinalIgnoreCase) == false)
{
throw new InvalidOperationException($"Encountered invalid directory '{directory}' in snapshot file with following file extensions: {string.Join(", ", extensions)}");
}
// this enables backward compatibility of snapshot backups with compression recovery files before fix was made in RavenDB-17173
// the underlying issue was that we were putting full path when compression recovery files were backed up using snapshot
// because of that the end restore directory was not a sub-directory of a restore path
// which could result in a file already exists exception
// since restoring of compression recovery files is not mandatory then it is safe to skip them
continue;
}
BackupMethods.Full.Restore(
zipEntries,
restoreDirectory,
journalDir: null,
onProgress: message =>
{
restoreResult.AddInfo(message);
restoreResult.SnapshotRestore.ReadCount++;
onProgress.Invoke(restoreResult.Progress);
},
cancellationToken: _operationCancelToken.Token);
}
}
if (restoreSettings == null)
{
throw new InvalidDataException("Cannot restore the snapshot without the settings file!");
}
return(restoreSettings);
}
private async Task <bool> HandleUnsuccessfulResponse <TResult>(ChoosenNode choosenNode, JsonOperationContext context, RavenCommand <TResult> command,
HttpResponseMessage response, string url)
{
switch (response.StatusCode)
{
case HttpStatusCode.NotFound:
command.SetResponse(null);
return(true);
case HttpStatusCode.Unauthorized:
case HttpStatusCode.PreconditionFailed:
if (string.IsNullOrEmpty(choosenNode.Node.ApiKey))
{
throw new UnauthorizedAccessException(
$"Got unauthorized response exception for {url}. Please specify an API Key.");
}
if (++command.AuthenticationRetries > 1)
{
throw new UnauthorizedAccessException(
$"Got unauthorized response exception for {url} after trying to authenticate using ApiKey.");
}
var oauthSource = response.Headers.GetFirstValue("OAuth-Source");
#if DEBUG && FIDDLER
// Make sure to avoid a cross DNS security issue, when running with Fiddler
if (string.IsNullOrEmpty(oauthSource) == false)
{
oauthSource = oauthSource.Replace("localhost:", "localhost.fiddler:");
}
#endif
await HandleUnauthorized(oauthSource, choosenNode.Node, context).ConfigureAwait(false);
await ExecuteAsync(choosenNode, context, command).ConfigureAwait(false);
return(true);
case HttpStatusCode.Forbidden:
throw new UnauthorizedAccessException(
$"Forbidan access to {url}. Make sure you're using the correct ApiKey.");
case HttpStatusCode.BadGateway:
case HttpStatusCode.ServiceUnavailable:
await HandleServerDown(choosenNode, context, command, null);
break;
case HttpStatusCode.Conflict:
// TODO: Conflict resolution
// current implementation is temporary
object message;
using (var stream = await response.Content.ReadAsStreamAsync())
{
var blittableJsonReaderObject = await context.ReadForMemoryAsync(stream, "PutResult");
blittableJsonReaderObject.TryGetMember("Message", out message);
}
throw new ConcurrencyException(message.ToString());
default:
await ThrowServerError(context, response);
break;
}
return(false);
}