public static async Task <ClusterTopologyResponse> TryLoadAsync(string topologyHash, DocumentConventions conventions, JsonOperationContext context) { try { if (conventions.DisableTopologyCache) { return(null); } var path = GetPath(topologyHash, conventions); if (File.Exists(path) == false) { return(null); } using (var stream = SafeFileStream.Create(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) using (var json = await context.ReadForMemoryAsync(stream, "raven-cluster-topology").ConfigureAwait(false)) { return(JsonDeserializationClient.ClusterTopology(json)); } } catch (Exception e) { if (_logger.IsInfoEnabled) { _logger.Info("Could not understand the persisted cluster topology", e); } return(null); } }
private static async Task <BlittableJsonReaderObject> GetJson(JsonOperationContext context, HttpResponseMessage response, Stream stream) { BlittableJsonReaderObject json; try { json = await context.ReadForMemoryAsync(stream, "error/response").ConfigureAwait(false); } catch (Exception e) { string content = null; if (stream.CanSeek) { stream.Position = 0; using (var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 4096, leaveOpen: true)) content = await reader.ReadToEndAsync().ConfigureAwait(false); } if (content != null) { content = $"Content: {content}"; } throw new InvalidOperationException($"Cannot parse the '{response.StatusCode}' response. {content}", e); } return(json); }
private async Task <int> SupervisorReadResponseAndGetVersionAsync(JsonOperationContext ctx, BlittableJsonTextWriter writer, Stream stream, string url, CancellationToken ct) { using (var responseJson = await ctx.ReadForMemoryAsync(stream, _readStatusUpdateDebugString + "/Read-Handshake-Response")) { var headerResponse = JsonDeserializationServer.TcpConnectionHeaderResponse(responseJson); switch (headerResponse.Status) { case TcpConnectionStatus.Ok: return(headerResponse.Version); case TcpConnectionStatus.AuthorizationFailed: throw new AuthorizationException( $"Node with ClusterTag = {ClusterTag} replied to initial handshake with authorization failure {headerResponse.Message}"); case TcpConnectionStatus.TcpVersionMismatch: if (headerResponse.Version != -1) { return(headerResponse.Version); } //Kindly request the server to drop the connection WriteOperationHeaderToRemote(writer, headerResponse.Version, drop: true); throw new InvalidOperationException($"Node with ClusterTag = {ClusterTag} replied to initial handshake with mismatching tcp version {headerResponse.Message}"); default: throw new InvalidOperationException($"{url} replied with unknown status {headerResponse.Status}, message:{headerResponse.Message}"); } } }
private async Task <IndexQueryServerSide> GetIndexQuery(JsonOperationContext context, HttpMethod method, RequestTimeTracker tracker) { if (method == HttpMethod.Get) { return(IndexQueryServerSide.Create(HttpContext, GetStart(), GetPageSize(), context, tracker)); } var json = await context.ReadForMemoryAsync(RequestBodyStream(), "index/query"); return(IndexQueryServerSide.Create(HttpContext, json, Database.QueryMetadataCache, tracker)); }
public static async Task <(List <Facet> Facets, long FacetsEtag)> ParseFromStringAsync(string facetsArrayAsString, JsonOperationContext context) { using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(facetsArrayAsString))) { var input = await context.ReadForMemoryAsync(stream, "facets"); if (input.TryGet("Facets", out BlittableJsonReaderArray array) == false) { RequestHandler.ThrowRequiredPropertyNameInRequest("Facets"); } return(ParseFromJson(array)); } }
public async Task <BlittableJsonReaderObject> InvokeAndReadObjectAsync(RouteInformation route, JsonOperationContext context, Dictionary <string, Microsoft.Extensions.Primitives.StringValues> parameters = null) { var response = await InvokeAsync(route, parameters); try { return(await context.ReadForMemoryAsync(response.Body, $"read/local endpoint/{route.Path}")); } catch (InvalidStartOfObjectException e) { //precaution, ideally this exception should never be thrown throw new InvalidOperationException("Expected to find a blittable object as a result of debug endpoint, but found something else (see inner exception for details). This should be investigated as all RavenDB endpoints are supposed to return an object.", e); } }
private async Task <IndexQueryServerSide> GetIndexQuery(JsonOperationContext context, HttpMethod method, RequestTimeTracker tracker) { var addSpatialProperties = GetBoolValueQueryString("addSpatialProperties", required: false) ?? false; var clientQueryId = GetStringQueryString("clientQueryId", required: false); if (method == HttpMethod.Get) { return(await IndexQueryServerSide.CreateAsync(HttpContext, GetStart(), GetPageSize(), context, tracker, addSpatialProperties, clientQueryId)); } var json = await context.ReadForMemoryAsync(RequestBodyStream(), "index/query"); return(IndexQueryServerSide.Create(HttpContext, json, Database.QueryMetadataCache, tracker, addSpatialProperties, clientQueryId, Database)); }
private static async Task <BlittableJsonReaderObject> GetJson(JsonOperationContext context, HttpResponseMessage response, Stream stream) { BlittableJsonReaderObject json; try { json = await context.ReadForMemoryAsync(stream, "error/response").ConfigureAwait(false); } catch (Exception e) { stream.Position = 0; throw new InvalidOperationException($"Cannot parse the {response.StatusCode} response. Content: {new StreamReader(stream).ReadToEnd()}", e); } return(json); }
internal static async Task InvokeAndWriteToArchive(ZipArchive archive, JsonOperationContext jsonOperationContext, LocalEndpointClient localEndpointClient, RouteInformation route, string path, Dictionary <string, Microsoft.Extensions.Primitives.StringValues> endpointParameters = null, CancellationToken token = default) { try { var response = await localEndpointClient.InvokeAsync(route, endpointParameters); var entryName = DebugInfoPackageUtils.GetOutputPathFromRouteInformation(route, path, response.ContentType == "text/plain" ? "txt" : "json"); var entry = archive.CreateEntry(entryName); entry.ExternalAttributes = ((int)(FilePermissions.S_IRUSR | FilePermissions.S_IWUSR)) << 16; using (var entryStream = entry.Open()) { if (response.ContentType == "text/plain") { await response.Body.CopyToAsync(entryStream, token); } else { await using (var writer = new AsyncBlittableJsonTextWriter(jsonOperationContext, entryStream)) { var endpointOutput = await jsonOperationContext.ReadForMemoryAsync(response.Body, $"read/local endpoint/{route.Path}"); jsonOperationContext.Write(writer, endpointOutput); await writer.FlushAsync(); } } await entryStream.FlushAsync(token); } } catch (OperationCanceledException) { throw; } catch (Exception e) { //precaution, ideally this exception should never be thrown if (e is InvalidStartOfObjectException) { e = new InvalidOperationException("Expected to find a blittable object as a result of debug endpoint, but found something else (see inner exception for details). This should be investigated as all RavenDB endpoints are supposed to return an object.", e); } await DebugInfoPackageUtils.WriteExceptionAsZipEntryAsync(e, archive, DebugInfoPackageUtils.GetOutputPathFromRouteInformation(route, path, null)); } }
private static async Task ThrowServerError(JsonOperationContext context, HttpResponseMessage response) { using (var stream = await ReadAsStreamUncompressedAsync(response)) { BlittableJsonReaderObject blittableJsonReaderObject; try { blittableJsonReaderObject = await context.ReadForMemoryAsync(stream, "ErrorResponse"); } catch (Exception e) { stream.Position = 0; throw new InvalidOperationException( $"Cannot parse the {response.StatusCode} response: {new StreamReader(stream).ReadToEnd()}", e); } stream.Position = 0; using (blittableJsonReaderObject) { string error; if (blittableJsonReaderObject.TryGet("Error", out error) == false) { throw new InvalidOperationException( $"Doesn't know how to handle error: {response.StatusCode}, response: {new StreamReader(stream).ReadToEnd()}"); } if (response.StatusCode == HttpStatusCode.BadRequest) { throw new BadRequestException(error + ". Response: " + blittableJsonReaderObject); } string indexDefinitionProperty; if (blittableJsonReaderObject.TryGet(nameof(IndexCompilationException.IndexDefinitionProperty), out indexDefinitionProperty)) { var indexCompilationException = new IndexCompilationException(error); blittableJsonReaderObject.TryGet(nameof(IndexCompilationException.IndexDefinitionProperty), out indexCompilationException.IndexDefinitionProperty); blittableJsonReaderObject.TryGet(nameof(IndexCompilationException.ProblematicText), out indexCompilationException.ProblematicText); throw indexCompilationException; } throw new InternalServerErrorException(error + ". Response: " + blittableJsonReaderObject); } } }
public virtual async Task ProcessResponse(JsonOperationContext context, HttpCache cache, HttpResponseMessage response, string url) { using (response) using (var stream = await response.Content.ReadAsStreamAsync()) { // we intentionally don't dispose the reader here, we'll be using it // in the command, any associated memory will be released on context reset var blittableJsonReaderObject = await context.ReadForMemoryAsync(stream, "PutResult"); if (response.Headers.ETag != null) { long?etag = response.GetEtagHeader(); if (etag != null) { cache.Set(url, (long)etag, blittableJsonReaderObject); } } SetResponse(blittableJsonReaderObject); } }
private async ValueTask <TcpConnectionHeaderMessage.NegotiationResponse> SupervisorReadResponseAndGetVersionAsync(JsonOperationContext ctx, AsyncBlittableJsonTextWriter writer, Stream stream, string url) { using (var responseJson = await ctx.ReadForMemoryAsync(stream, _readStatusUpdateDebugString + "/Read-Handshake-Response")) { var headerResponse = JsonDeserializationServer.TcpConnectionHeaderResponse(responseJson); switch (headerResponse.Status) { case TcpConnectionStatus.Ok: return(new TcpConnectionHeaderMessage.NegotiationResponse { Version = headerResponse.Version, LicensedFeatures = headerResponse.LicensedFeatures }); case TcpConnectionStatus.AuthorizationFailed: throw new AuthorizationException( $"Node with ClusterTag = {ClusterTag} replied to initial handshake with authorization failure {headerResponse.Message}"); case TcpConnectionStatus.TcpVersionMismatch: if (headerResponse.Version != TcpNegotiation.OutOfRangeStatus) { return(new TcpConnectionHeaderMessage.NegotiationResponse { Version = headerResponse.Version, LicensedFeatures = headerResponse.LicensedFeatures }); } //Kindly request the server to drop the connection await WriteOperationHeaderToRemoteAsync(writer, headerResponse.Version, drop : true); throw new InvalidOperationException($"Node with ClusterTag = {ClusterTag} replied to initial handshake with mismatching tcp version {headerResponse.Message}"); case TcpConnectionStatus.InvalidNetworkTopology: throw new AuthorizationException( $"Node with ClusterTag = {ClusterTag} replied to initial handshake with {nameof(TcpConnectionStatus.InvalidNetworkTopology)} error {headerResponse.Message}"); default: throw new InvalidOperationException($"{url} replied with unknown status {headerResponse.Status}, message:{headerResponse.Message}"); } } }
private static async Task <Topology> TryLoadAsync(string path, JsonOperationContext context) { try { if (File.Exists(path) == false) { return(null); } using (var stream = SafeFileStream.Create(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) using (var json = await context.ReadForMemoryAsync(stream, "raven-database-topology").ConfigureAwait(false)) { return(JsonDeserializationClient.Topology(json)); } } catch (Exception e) { if (_logger.IsInfoEnabled) { _logger.Info("Could not understand the persisted database topology", e); } return(null); } }
protected async Task <RestoreSettings> SnapshotRestore(JsonOperationContext context, string backupPath, Action <IOperationProgress> onProgress, RestoreResult restoreResult) { Debug.Assert(onProgress != null); RestoreSettings restoreSettings = null; var fullBackupPath = GetBackupPath(backupPath); using (var zip = await GetZipArchiveForSnapshot(fullBackupPath)) { var restorePath = new VoronPathSetting(RestoreFromConfiguration.DataDirectory); if (Directory.Exists(restorePath.FullPath) == false) { Directory.CreateDirectory(restorePath.FullPath); } // validate free space var snapshotSize = zip.Entries.Sum(entry => entry.Length); BackupHelper.AssertFreeSpaceForSnapshot(restorePath.FullPath, snapshotSize, "restore a backup", Logger); foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length))) { var directory = zipEntries.Key; if (string.IsNullOrWhiteSpace(directory)) { foreach (var zipEntry in zipEntries) { if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase)) { await using (var entryStream = zipEntry.Open()) { var snapshotEncryptionKey = RestoreFromConfiguration.EncryptionKey != null ? Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey) : null; await using (var stream = GetInputStream(entryStream, snapshotEncryptionKey)) { var json = await context.ReadForMemoryAsync(stream, "read database settings for restore"); json.BlittableValidation(); restoreSettings = JsonDeserializationServer.RestoreSettings(json); restoreSettings.DatabaseRecord.DatabaseName = RestoreFromConfiguration.DatabaseName; DatabaseHelper.Validate(RestoreFromConfiguration.DatabaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration); if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false) { throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!"); } if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey) { throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!"); } } } } } continue; } var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase) ? restorePath : restorePath.Combine(directory); BackupMethods.Full.Restore( zipEntries, restoreDirectory, journalDir: null, onProgress: message => { restoreResult.AddInfo(message); restoreResult.SnapshotRestore.ReadCount++; onProgress.Invoke(restoreResult.Progress); }, cancellationToken: _operationCancelToken.Token); } } if (restoreSettings == null) { throw new InvalidDataException("Cannot restore the snapshot without the settings file!"); } return(restoreSettings); }
protected async Task <RestoreSettings> SnapshotRestore(JsonOperationContext context, string backupPath, Action <IOperationProgress> onProgress, RestoreResult restoreResult) { Debug.Assert(onProgress != null); RestoreSettings restoreSettings = null; var fullBackupPath = GetBackupPath(backupPath); using (var zip = await GetZipArchiveForSnapshot(fullBackupPath)) { var restorePath = new VoronPathSetting(RestoreFromConfiguration.DataDirectory); if (Directory.Exists(restorePath.FullPath) == false) { Directory.CreateDirectory(restorePath.FullPath); } // validate free space var snapshotSize = zip.Entries.Sum(entry => entry.Length); BackupHelper.AssertFreeSpaceForSnapshot(restorePath.FullPath, snapshotSize, "restore a backup", Logger); foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length))) { var directory = zipEntries.Key; if (string.IsNullOrWhiteSpace(directory)) { foreach (var zipEntry in zipEntries) { if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase)) { await using (var entryStream = zipEntry.Open()) { var snapshotEncryptionKey = RestoreFromConfiguration.EncryptionKey != null ? Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey) : null; await using (var stream = GetInputStream(entryStream, snapshotEncryptionKey)) { var json = await context.ReadForMemoryAsync(stream, "read database settings for restore"); json.BlittableValidation(); restoreSettings = JsonDeserializationServer.RestoreSettings(json); restoreSettings.DatabaseRecord.DatabaseName = RestoreFromConfiguration.DatabaseName; DatabaseHelper.Validate(RestoreFromConfiguration.DatabaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration); if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false) { throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!"); } if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey) { throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!"); } } } } } continue; } var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase) ? restorePath : restorePath.Combine(directory); var isSubDirectory = PathUtil.IsSubDirectory(restoreDirectory.FullPath, restorePath.FullPath); if (isSubDirectory == false) { var extensions = zipEntries .Select(x => Path.GetExtension(x.Name)) .Distinct() .ToArray(); if (extensions.Length != 1 || string.Equals(extensions[0], TableValueCompressor.CompressionRecoveryExtension, StringComparison.OrdinalIgnoreCase) == false) { throw new InvalidOperationException($"Encountered invalid directory '{directory}' in snapshot file with following file extensions: {string.Join(", ", extensions)}"); } // this enables backward compatibility of snapshot backups with compression recovery files before fix was made in RavenDB-17173 // the underlying issue was that we were putting full path when compression recovery files were backed up using snapshot // because of that the end restore directory was not a sub-directory of a restore path // which could result in a file already exists exception // since restoring of compression recovery files is not mandatory then it is safe to skip them continue; } BackupMethods.Full.Restore( zipEntries, restoreDirectory, journalDir: null, onProgress: message => { restoreResult.AddInfo(message); restoreResult.SnapshotRestore.ReadCount++; onProgress.Invoke(restoreResult.Progress); }, cancellationToken: _operationCancelToken.Token); } } if (restoreSettings == null) { throw new InvalidDataException("Cannot restore the snapshot without the settings file!"); } return(restoreSettings); }
private async Task <bool> HandleUnsuccessfulResponse <TResult>(ChoosenNode choosenNode, JsonOperationContext context, RavenCommand <TResult> command, HttpResponseMessage response, string url) { switch (response.StatusCode) { case HttpStatusCode.NotFound: command.SetResponse(null); return(true); case HttpStatusCode.Unauthorized: case HttpStatusCode.PreconditionFailed: if (string.IsNullOrEmpty(choosenNode.Node.ApiKey)) { throw new UnauthorizedAccessException( $"Got unauthorized response exception for {url}. Please specify an API Key."); } if (++command.AuthenticationRetries > 1) { throw new UnauthorizedAccessException( $"Got unauthorized response exception for {url} after trying to authenticate using ApiKey."); } var oauthSource = response.Headers.GetFirstValue("OAuth-Source"); #if DEBUG && FIDDLER // Make sure to avoid a cross DNS security issue, when running with Fiddler if (string.IsNullOrEmpty(oauthSource) == false) { oauthSource = oauthSource.Replace("localhost:", "localhost.fiddler:"); } #endif await HandleUnauthorized(oauthSource, choosenNode.Node, context).ConfigureAwait(false); await ExecuteAsync(choosenNode, context, command).ConfigureAwait(false); return(true); case HttpStatusCode.Forbidden: throw new UnauthorizedAccessException( $"Forbidan access to {url}. Make sure you're using the correct ApiKey."); case HttpStatusCode.BadGateway: case HttpStatusCode.ServiceUnavailable: await HandleServerDown(choosenNode, context, command, null); break; case HttpStatusCode.Conflict: // TODO: Conflict resolution // current implementation is temporary object message; using (var stream = await response.Content.ReadAsStreamAsync()) { var blittableJsonReaderObject = await context.ReadForMemoryAsync(stream, "PutResult"); blittableJsonReaderObject.TryGetMember("Message", out message); } throw new ConcurrencyException(message.ToString()); default: await ThrowServerError(context, response); break; } return(false); }