protected async Task <RestoreSettings> SnapshotRestore(JsonOperationContext context, string backupPath, Action <IOperationProgress> onProgress, RestoreResult restoreResult) { Debug.Assert(onProgress != null); RestoreSettings restoreSettings = null; var fullBackupPath = GetBackupPath(backupPath); using (var zip = await GetZipArchiveForSnapshot(fullBackupPath)) { var restorePath = new VoronPathSetting(RestoreFromConfiguration.DataDirectory); if (Directory.Exists(restorePath.FullPath) == false) { Directory.CreateDirectory(restorePath.FullPath); } // validate free space var snapshotSize = zip.Entries.Sum(entry => entry.Length); BackupHelper.AssertFreeSpaceForSnapshot(restorePath.FullPath, snapshotSize, "restore a backup", Logger); foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length))) { var directory = zipEntries.Key; if (string.IsNullOrWhiteSpace(directory)) { foreach (var zipEntry in zipEntries) { if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase)) { await using (var entryStream = zipEntry.Open()) { var snapshotEncryptionKey = RestoreFromConfiguration.EncryptionKey != null ? Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey) : null; await using (var stream = GetInputStream(entryStream, snapshotEncryptionKey)) { var json = await context.ReadForMemoryAsync(stream, "read database settings for restore"); json.BlittableValidation(); restoreSettings = JsonDeserializationServer.RestoreSettings(json); restoreSettings.DatabaseRecord.DatabaseName = RestoreFromConfiguration.DatabaseName; DatabaseHelper.Validate(RestoreFromConfiguration.DatabaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration); if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false) { throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!"); } if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey) { throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!"); } } } } } continue; } var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase) ? restorePath : restorePath.Combine(directory); var isSubDirectory = PathUtil.IsSubDirectory(restoreDirectory.FullPath, restorePath.FullPath); if (isSubDirectory == false) { var extensions = zipEntries .Select(x => Path.GetExtension(x.Name)) .Distinct() .ToArray(); if (extensions.Length != 1 || string.Equals(extensions[0], TableValueCompressor.CompressionRecoveryExtension, StringComparison.OrdinalIgnoreCase) == false) { throw new InvalidOperationException($"Encountered invalid directory '{directory}' in snapshot file with following file extensions: {string.Join(", ", extensions)}"); } // this enables backward compatibility of snapshot backups with compression recovery files before fix was made in RavenDB-17173 // the underlying issue was that we were putting full path when compression recovery files were backed up using snapshot // because of that the end restore directory was not a sub-directory of a restore path // which could result in a file already exists exception // since restoring of compression recovery files is not mandatory then it is safe to skip them continue; } BackupMethods.Full.Restore( zipEntries, restoreDirectory, journalDir: null, onProgress: message => { restoreResult.AddInfo(message); restoreResult.SnapshotRestore.ReadCount++; onProgress.Invoke(restoreResult.Progress); }, cancellationToken: _operationCancelToken.Token); } } if (restoreSettings == null) { throw new InvalidDataException("Cannot restore the snapshot without the settings file!"); } return(restoreSettings); }
public Task ExtractInfoFromZip() { AssertOnlyInSetupMode(); using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) using (var continueSetupInfoJson = context.ReadForMemory(RequestBodyStream(), "continue-setup-info")) { var continueSetupInfo = JsonDeserializationServer.ContinueSetupInfo(continueSetupInfoJson); byte[] zipBytes; try { zipBytes = Convert.FromBase64String(continueSetupInfo.Zip); } catch (Exception e) { throw new ArgumentException($"Unable to parse the {nameof(continueSetupInfo.Zip)} property, expected a Base64 value", e); } try { var urlByTag = new Dictionary <string, string>(); using (var ms = new MemoryStream(zipBytes)) using (var archive = new ZipArchive(ms, ZipArchiveMode.Read, false)) { foreach (var entry in archive.Entries) { if (entry.Name.Equals("settings.json") == false) { continue; } var tag = entry.FullName[0].ToString(); using (var settingsJson = context.ReadForMemory(entry.Open(), "settings-json")) if (settingsJson.TryGet(nameof(ConfigurationNodeInfo.PublicServerUrl), out string publicServerUrl)) { urlByTag[tag] = publicServerUrl; } } } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartArray(); var first = true; foreach (var node in urlByTag) { if (first == false) { writer.WriteComma(); } writer.WriteStartObject(); writer.WritePropertyName(nameof(ConfigurationNodeInfo.Tag)); writer.WriteString(node.Key); writer.WriteComma(); writer.WritePropertyName(nameof(ConfigurationNodeInfo.PublicServerUrl)); writer.WriteString(node.Value); writer.WriteEndObject(); first = false; } writer.WriteEndArray(); } } catch (Exception e) { throw new InvalidOperationException("Unable to extract setup information from the zip file.", e); } } return(Task.CompletedTask); }
public Task GetHosts() { AssertOnlyInSetupMode(); using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) using (var certificateJson = context.ReadForMemory(RequestBodyStream(), "setup-certificate")) { var certDef = JsonDeserializationServer.CertificateDefinition(certificateJson); X509Certificate2 certificate = null; string cn; try { certificate = certDef.Password == null ? new X509Certificate2(Convert.FromBase64String(certDef.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet) : new X509Certificate2(Convert.FromBase64String(certDef.Certificate), certDef.Password, X509KeyStorageFlags.MachineKeySet); cn = certificate.GetNameInfo(X509NameType.SimpleName, false); } catch (Exception e) { throw new BadRequestException($"Failed to extract the CN property from the certificate {certificate?.FriendlyName}. Maybe the password is wrong?", e); } if (cn == null) { throw new BadRequestException($"Failed to extract the CN property from the certificate. CN is null"); } if (cn.LastIndexOf('*') > 0) { throw new NotSupportedException("The wildcard CN name contains a '*' which is not at the first character of the string. It is not supported in the Setup Wizard, you can do a manual setup instead."); } try { SecretProtection.ValidateKeyUsages("Setup Wizard", certificate); } catch (Exception e) { throw new InvalidOperationException($"Failed to load the uploaded certificate. Did you accidently upload a client certificate?", e); } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName("CN"); writer.WriteString(cn); writer.WriteComma(); writer.WritePropertyName("AlternativeNames"); writer.WriteStartArray(); var first = true; foreach (var value in SetupManager.GetCertificateAlternativeNames(certificate)) { if (first == false) { writer.WriteComma(); } first = false; writer.WriteString(value); } writer.WriteEndArray(); writer.WriteEndObject(); } } return(Task.CompletedTask); }
public async Task TestJavaScriptIndex() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var input = await context.ReadForMemoryAsync(RequestBodyStream(), "TestJavaScriptIndex"); if (input.TryGet("Definition", out BlittableJsonReaderObject index) == false) { ThrowRequiredPropertyNameInRequest("Definition"); } input.TryGet("Ids", out BlittableJsonReaderArray ids); var indexDefinition = JsonDeserializationServer.IndexDefinition(index); if (indexDefinition.Maps == null || indexDefinition.Maps.Count == 0) { throw new ArgumentException("Index must have a 'Maps' fields"); } indexDefinition.Type = indexDefinition.DetectStaticIndexType(); if (indexDefinition.Type.IsJavaScript() == false) { throw new UnauthorizedAccessException("Testing indexes is only allowed for JavaScript indexes."); } var compiledIndex = new JavaScriptIndex(indexDefinition, Database.Configuration); var inputSize = GetIntValueQueryString("inputSize", false) ?? defaultInputSizeForTestingJavaScriptIndex; var collections = new HashSet <string>(compiledIndex.Maps.Keys); var docsPerCollection = new Dictionary <string, List <DynamicBlittableJson> >(); using (context.OpenReadTransaction()) { if (ids == null) { foreach (var collection in collections) { docsPerCollection.Add(collection, Database.DocumentsStorage.GetDocumentsFrom(context, collection, 0, 0, inputSize).Select(d => new DynamicBlittableJson(d)).ToList()); } } else { var listOfIds = ids.Select(x => x.ToString()); var _ = new Reference <int> { Value = 0 }; var docs = Database.DocumentsStorage.GetDocuments(context, listOfIds, 0, int.MaxValue, _); foreach (var doc in docs) { if (doc.TryGetMetadata(out var metadata) && metadata.TryGet(Constants.Documents.Metadata.Collection, out string collectionStr)) { if (docsPerCollection.TryGetValue(collectionStr, out var listOfDocs) == false) { listOfDocs = docsPerCollection[collectionStr] = new List <DynamicBlittableJson>(); } listOfDocs.Add(new DynamicBlittableJson(doc)); } } } var mapRes = new List <ObjectInstance>(); //all maps foreach (var ListOfFunctions in compiledIndex.Maps) { //multi maps per collection foreach (var mapFunc in ListOfFunctions.Value) { if (docsPerCollection.TryGetValue(ListOfFunctions.Key, out var docs)) { foreach (var res in mapFunc(docs)) { mapRes.Add((ObjectInstance)res); } } } } var first = true; using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName("MapResults"); writer.WriteStartArray(); foreach (var mapResult in mapRes) { if (JavaScriptIndexUtils.StringifyObject(mapResult) is JsString jsStr) { if (first == false) { writer.WriteComma(); } writer.WriteString(jsStr.ToString()); first = false; } } writer.WriteEndArray(); if (indexDefinition.Reduce != null) { using (var bufferPool = new UnmanagedBuffersPoolWithLowMemoryHandling("JavaScriptIndexTest", Database.Name)) { compiledIndex.SetBufferPoolForTestingPurposes(bufferPool); compiledIndex.SetAllocatorForTestingPurposes(context.Allocator); first = true; writer.WritePropertyName("ReduceResults"); writer.WriteStartArray(); var reduceResults = compiledIndex.Reduce(mapRes.Select(mr => new DynamicBlittableJson(JsBlittableBridge.Translate(context, mr.Engine, mr)))); foreach (JsValue reduceResult in reduceResults) { if (JavaScriptIndexUtils.StringifyObject(reduceResult) is JsString jsStr) { if (first == false) { writer.WriteComma(); } writer.WriteString(jsStr.ToString()); first = false; } } } writer.WriteEndArray(); } writer.WriteEndObject(); } } } }
public async Task Create() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.OpenReadTransaction()) { var json = await context.ReadForMemoryAsync(RequestBodyStream(), null); var options = JsonDeserializationServer.SubscriptionCreationParams(json); if (TrafficWatchManager.HasRegisteredClients) { AddStringToHttpContext(json.ToString(), TrafficWatchChangeType.Subscriptions); } var sub = SubscriptionConnection.ParseSubscriptionQuery(options.Query); if (Enum.TryParse( options.ChangeVector, out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue)) { switch (changeVectorSpecialValue) { case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime: options.ChangeVector = null; break; case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument: options.ChangeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context, sub.Collection); break; } } var id = GetLongQueryString("id", required: false); var disabled = GetBoolValueQueryString("disabled", required: false); var mentor = options.MentorNode; var subscriptionId = await Database.SubscriptionStorage.PutSubscription(options, GetRaftRequestIdFromQuery(), id, disabled, mentor : mentor); var name = options.Name ?? subscriptionId.ToString(); using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext serverContext)) using (serverContext.OpenReadTransaction()) { // need to wait on the relevant remote node var node = Database.SubscriptionStorage.GetResponsibleNode(serverContext, name); if (node != null && node != ServerStore.NodeTag) { await WaitForExecutionOnSpecificNode(serverContext, ServerStore.GetClusterTopology(serverContext), node, subscriptionId); } } HttpContext.Response.StatusCode = (int)HttpStatusCode.Created; // Created using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue { ["Name"] = name }); } } }
public static IEnumerable <AbstractDashboardNotification> FetchDatabasesInfo(ServerStore serverStore, CancellationTokenSource cts) { var databasesInfo = new DatabasesInfo(); var indexingSpeed = new IndexingSpeed(); var trafficWatch = new TrafficWatch(); var drivesUsage = new DrivesUsage(); using (serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionContext)) using (transactionContext.OpenReadTransaction()) { foreach (var databaseTuple in serverStore.Cluster.ItemsStartingWith(transactionContext, Constants.Documents.Prefix, 0, int.MaxValue)) { var databaseName = databaseTuple.ItemName.Substring(3); if (cts.IsCancellationRequested) { yield break; } if (serverStore.DatabasesLandlord.DatabasesCache.TryGetValue(databaseName, out var databaseTask) == false) { // database does not exist in this server or disabled continue; } var databaseOnline = IsDatabaseOnline(databaseTask, out var database); if (databaseOnline == false) { var databaseRecord = serverStore.LoadDatabaseRecord(databaseName, out var _); if (databaseRecord == null) { // database doesn't exist continue; } var databaseInfoItem = new DatabaseInfoItem { Database = databaseName, Online = false }; DatabaseInfo databaseInfo = null; if (serverStore.DatabaseInfoCache.TryGet(databaseName, databaseInfoJson => databaseInfo = JsonDeserializationServer.DatabaseInfo(databaseInfoJson))) { Debug.Assert(databaseInfo != null); databaseInfoItem.DocumentsCount = databaseInfo.DocumentsCount ?? 0; databaseInfoItem.IndexesCount = databaseInfo.IndexesCount ?? databaseRecord.Indexes.Count; databaseInfoItem.ReplicationFactor = databaseRecord.Topology?.ReplicationFactor ?? databaseInfo.ReplicationFactor; databaseInfoItem.ErroredIndexesCount = databaseInfo.IndexingErrors ?? 0; } databasesInfo.Items.Add(databaseInfoItem); continue; } var indexingSpeedItem = new IndexingSpeedItem { Database = database.Name, IndexedPerSecond = database.Metrics.MapIndexes.IndexedPerSec.FiveSecondRate, MappedPerSecond = database.Metrics.MapReduceIndexes.MappedPerSec.FiveSecondRate, ReducedPerSecond = database.Metrics.MapReduceIndexes.ReducedPerSec.FiveSecondRate }; indexingSpeed.Items.Add(indexingSpeedItem); var replicationFactor = GetReplicationFactor(databaseTuple.Value); var documentsStorage = database.DocumentsStorage; var indexStorage = database.IndexStore; using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext)) using (documentsContext.OpenReadTransaction()) { var databaseInfoItem = new DatabaseInfoItem { Database = databaseName, DocumentsCount = documentsStorage.GetNumberOfDocuments(documentsContext), IndexesCount = database.IndexStore.Count, AlertsCount = database.NotificationCenter.GetAlertCount(), ReplicationFactor = replicationFactor, ErroredIndexesCount = indexStorage.GetIndexes().Count(index => index.GetErrorCount() > 0), Online = true }; databasesInfo.Items.Add(databaseInfoItem); } var trafficWatchItem = new TrafficWatchItem { Database = databaseName, RequestsPerSecond = (int)database.Metrics.Requests.RequestsPerSec.FiveSecondRate, WritesPerSecond = (int)database.Metrics.Docs.PutsPerSec.FiveSecondRate, WriteBytesPerSecond = database.Metrics.Docs.BytesPutsPerSec.FiveSecondRate }; trafficWatch.Items.Add(trafficWatchItem); foreach (var mountPointUsage in database.GetMountPointsUsage()) { if (cts.IsCancellationRequested) { yield break; } var mountPoint = mountPointUsage.DiskSpaceResult.DriveName; var usage = drivesUsage.Items.FirstOrDefault(x => x.MountPoint == mountPoint); if (usage == null) { usage = new MountPointUsage { MountPoint = mountPoint, VolumeLabel = mountPointUsage.DiskSpaceResult.VolumeLabel, FreeSpace = mountPointUsage.DiskSpaceResult.TotalFreeSpace.GetValue(SizeUnit.Bytes), TotalCapacity = mountPointUsage.DiskSpaceResult.TotalSize.GetValue(SizeUnit.Bytes) }; drivesUsage.Items.Add(usage); } var existingDatabaseUsage = usage.Items.FirstOrDefault(x => x.Database == databaseName); if (existingDatabaseUsage == null) { existingDatabaseUsage = new DatabaseDiskUsage { Database = databaseName }; usage.Items.Add(existingDatabaseUsage); } existingDatabaseUsage.Size += mountPointUsage.UsedSpace; } } } yield return(databasesInfo); yield return(indexingSpeed); yield return(trafficWatch); yield return(drivesUsage); }
internal ReplicationMessageReply HandleServerResponse(BlittableJsonReaderObject replicationBatchReplyMessage, bool allowNotify) { replicationBatchReplyMessage.BlittableValidation(); var replicationBatchReply = JsonDeserializationServer.ReplicationMessageReply(replicationBatchReplyMessage); if (replicationBatchReply.MessageType == "Processing") { return(null); } if (allowNotify == false && replicationBatchReply.MessageType == "Notify") { return(null); } DestinationDbId = replicationBatchReply.DatabaseId; switch (replicationBatchReply.Type) { case ReplicationMessageReply.ReplyType.Ok: UpdateDestinationChangeVector(replicationBatchReply); OnSuccessfulTwoWaysCommunication(); break; case ReplicationMessageReply.ReplyType.MissingAttachments: break; default: var msg = $"Received error from remote replication destination. Error received: {replicationBatchReply.Exception}"; if (_log.IsInfoEnabled) { _log.Info(msg); } break; } if (_log.IsInfoEnabled) { switch (replicationBatchReply.Type) { case ReplicationMessageReply.ReplyType.Ok: _log.Info( $"Received reply for replication batch from {Destination.FromString()}. New destination change vector is {LastAcceptedChangeVector}"); break; case ReplicationMessageReply.ReplyType.Error: _log.Info( $"Received reply for replication batch from {Destination.FromString()}. There has been a failure, error string received : {replicationBatchReply.Exception}"); throw new InvalidOperationException( $"Received failure reply for replication batch. Error string received = {replicationBatchReply.Exception}"); case ReplicationMessageReply.ReplyType.MissingAttachments: _log.Info($"Received reply for replication batch from {Destination.FromString()}. Destination is reporting missing attachments."); break; default: throw new ArgumentOutOfRangeException(nameof(replicationBatchReply), "Received reply for replication batch with unrecognized type... got " + replicationBatchReply.Type); } } return(replicationBatchReply); }
private async Task ListenToMaintenanceWorker() { bool needToWait = false; var onErrorDelayTime = _parent.Config.OnErrorDelayTime.AsTimeSpan; var receiveFromWorkerTimeout = _parent.Config.ReceiveFromWorkerTimeout.AsTimeSpan; TcpConnectionInfo tcpConnection = null; try { tcpConnection = await ReplicationUtils.GetTcpInfoAsync(Url, null, "Supervisor", _parent._server.Server.Certificate?.Certificate); } catch (Exception e) { if (_log.IsInfoEnabled) { _log.Info($"ClusterMaintenanceSupervisor() => Failed to add to cluster node key = {ClusterTag}", e); } } while (_token.IsCancellationRequested == false) { var internalTaskCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(_token); try { if (needToWait) { needToWait = false; // avoid tight loop if there was timeout / error await TimeoutManager.WaitFor(onErrorDelayTime, _token); tcpConnection = await ReplicationUtils.GetTcpInfoAsync(Url, null, "Supervisor", _parent._server.Server.Certificate.Certificate); } if (tcpConnection == null) { needToWait = true; continue; } var(tcpClient, connection) = await ConnectToClientNodeAsync(tcpConnection, _parent._server.Engine.TcpConnectionTimeout); using (tcpClient) using (_cts.Token.Register(tcpClient.Dispose)) using (connection) { while (_token.IsCancellationRequested == false) { using (_contextPool.AllocateOperationContext(out JsonOperationContext context)) { var readResponseTask = context.ReadForMemoryAsync(connection, _readStatusUpdateDebugString, internalTaskCancellationToken.Token); var timeout = TimeoutManager.WaitFor(receiveFromWorkerTimeout, _token); if (readResponseTask.IsCompleted == false && await Task.WhenAny(readResponseTask.AsTask(), timeout) == timeout) { if (_log.IsInfoEnabled) { _log.Info($"Timeout occurred while collecting info from {ClusterTag}"); } ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(), ClusterNodeStatusReport.ReportStatus.Timeout, null, DateTime.UtcNow, _lastSuccessfulReceivedReport); needToWait = true; internalTaskCancellationToken.Cancel(); try { await readResponseTask; } catch { // expecting and ignoring this error, we MUST wait // until the task is done before releasing the context that // it is using } break; } using (var statusUpdateJson = await readResponseTask) { var report = new Dictionary <string, DatabaseStatusReport>(); foreach (var property in statusUpdateJson.GetPropertyNames()) { var value = (BlittableJsonReaderObject)statusUpdateJson[property]; report.Add(property, JsonDeserializationServer.DatabaseStatusReport(value)); } ReceivedReport = new ClusterNodeStatusReport( report, ClusterNodeStatusReport.ReportStatus.Ok, null, DateTime.UtcNow, _lastSuccessfulReceivedReport); _lastSuccessfulReceivedReport = ReceivedReport; } } } } } catch (Exception e) { if (_log.IsInfoEnabled) { _log.Info($"Exception was thrown while collecting info from {ClusterTag}", e); } ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(), ClusterNodeStatusReport.ReportStatus.Error, e, DateTime.UtcNow, _lastSuccessfulReceivedReport); needToWait = true; } finally { internalTaskCancellationToken.Dispose(); } } }
public override int Execute(DocumentsOperationContext context) { if (Database.ServerStore.Configuration.Core.FeaturesAvailability == FeaturesAvailability.Stable) { FeaturesAvailabilityException.Throw("Cluster Transactions"); } var global = context.LastDatabaseChangeVector ?? (context.LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context)); var dbGrpId = Database.DatabaseGroupId; var current = ChangeVectorUtils.GetEtagById(global, dbGrpId); foreach (var command in _batch) { Replies.Add(command.Index, new DynamicJsonArray()); Reply = Replies[command.Index]; var commands = command.Commands; var count = command.PreviousCount; var options = Options[command.Index] = command.Options; if (options.WaitForIndexesTimeout != null) { ModifiedCollections = new HashSet <string>(); } if (commands != null) { foreach (BlittableJsonReaderObject blittableCommand in commands) { count++; var changeVector = $"RAFT:{count}-{dbGrpId}"; var cmd = JsonDeserializationServer.ClusterTransactionDataCommand(blittableCommand); switch (cmd.Type) { case CommandType.PUT: if (current < count) { // delete the document to avoid exception if we put new document in a different collection. // TODO: document this behavior using (DocumentIdWorker.GetSliceFromId(context, cmd.Id, out Slice lowerId)) { Database.DocumentsStorage.Delete(context, lowerId, cmd.Id, expectedChangeVector: null, nonPersistentFlags: NonPersistentDocumentFlags.SkipRevisionCreation); } var putResult = Database.DocumentsStorage.Put(context, cmd.Id, null, cmd.Document.Clone(context), changeVector: changeVector, flags: DocumentFlags.FromClusterTransaction); context.DocumentDatabase.HugeDocuments.AddIfDocIsHuge(cmd.Id, cmd.Document.Size); AddPutResult(putResult); } else { try { var item = Database.DocumentsStorage.GetDocumentOrTombstone(context, cmd.Id); if (item.Missing) { AddPutResult(new DocumentsStorage.PutOperationResults { ChangeVector = changeVector, Id = cmd.Id, LastModified = DateTime.UtcNow, Collection = Database.DocumentsStorage.ExtractCollectionName(context, cmd.Document) }); continue; } var collection = GetCollection(context, item); AddPutResult(new DocumentsStorage.PutOperationResults { ChangeVector = changeVector, Id = cmd.Id, Flags = item.Document?.Flags ?? item.Tombstone.Flags, LastModified = item.Document?.LastModified ?? item.Tombstone.LastModified, Collection = collection }); } catch (DocumentConflictException) { AddPutResult(new DocumentsStorage.PutOperationResults { ChangeVector = changeVector, Id = cmd.Id, Collection = GetFirstConflictCollection(context, cmd) }); } } break; case CommandType.DELETE: if (current < count) { using (DocumentIdWorker.GetSliceFromId(context, cmd.Id, out Slice lowerId)) { var deleteResult = Database.DocumentsStorage.Delete(context, lowerId, cmd.Id, null, changeVector: changeVector, documentFlags: DocumentFlags.FromClusterTransaction); AddDeleteResult(deleteResult, cmd.Id); } } else { try { var item = Database.DocumentsStorage.GetDocumentOrTombstone(context, cmd.Id); if (item.Missing) { AddDeleteResult(new DocumentsStorage.DeleteOperationResult { ChangeVector = changeVector, Collection = null }, cmd.Id); continue; } var collection = GetCollection(context, item); AddDeleteResult(new DocumentsStorage.DeleteOperationResult { ChangeVector = changeVector, Collection = collection }, cmd.Id); } catch (DocumentConflictException) { AddDeleteResult(new DocumentsStorage.DeleteOperationResult { ChangeVector = changeVector, Collection = GetFirstConflictCollection(context, cmd) }, cmd.Id); } } break; default: throw new NotSupportedException($"{cmd.Type} is not supported in {nameof(ClusterTransactionMergedCommand)}."); } } } if (context.LastDatabaseChangeVector == null) { context.LastDatabaseChangeVector = global; } var result = ChangeVectorUtils.TryUpdateChangeVector("RAFT", dbGrpId, count, context.LastDatabaseChangeVector); if (result.IsValid) { context.LastDatabaseChangeVector = result.ChangeVector; } } return(Reply.Count); }
private RestoreSettings SnapshotRestore( string backupPath, string dataDirectory, Action <IOperationProgress> onProgress, RestoreResult restoreResult) { Debug.Assert(onProgress != null); RestoreSettings restoreSettings = null; var voronBackupPath = new VoronPathSetting(backupPath); var voronDataDirectory = new VoronPathSetting(dataDirectory); using (var zip = ZipFile.Open(voronBackupPath.FullPath, ZipArchiveMode.Read, System.Text.Encoding.UTF8)) { foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length))) { var directory = zipEntries.Key; if (string.IsNullOrWhiteSpace(directory)) { foreach (var zipEntry in zipEntries) { if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase)) { using (var entryStream = zipEntry.Open()) using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) { var json = context.Read(entryStream, "read database settings for restore"); json.BlittableValidation(); restoreSettings = JsonDeserializationServer.RestoreSettings(json); restoreSettings.DatabaseRecord.DatabaseName = _restoreConfiguration.DatabaseName; DatabaseHelper.Validate(_restoreConfiguration.DatabaseName, restoreSettings.DatabaseRecord); if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false) { throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!"); } if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey) { throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!"); } } } } continue; } var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase) ? voronDataDirectory : voronDataDirectory.Combine(directory); BackupMethods.Full.Restore( zipEntries, restoreDirectory, journalDir: null, onProgress: message => { restoreResult.AddInfo(message); restoreResult.SnapshotRestore.ReadCount++; onProgress.Invoke(restoreResult.Progress); }, cancellationToken: _operationCancelToken.Token); } } if (restoreSettings == null) { throw new InvalidDataException("Cannot restore the snapshot without the settings file!"); } return(restoreSettings); }
public async Task SetupUnsecured() { AssertOnlyInSetupMode(); using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (var setupInfoJson = context.ReadForMemory(RequestBodyStream(), "setup-unsecured")) { // Making sure we don't have leftovers from previous setup try { using (var tx = context.OpenWriteTransaction()) { ServerStore.Engine.DeleteTopology(context); tx.Commit(); } } catch (Exception) { // ignored } var setupInfo = JsonDeserializationServer.UnsecuredSetupInfo(setupInfoJson); BlittableJsonReaderObject settingsJson; using (var fs = new FileStream(ServerStore.Configuration.ConfigPath, FileMode.Open, FileAccess.Read)) { settingsJson = context.ReadForMemory(fs, "settings-json"); } settingsJson.Modifications = new DynamicJsonValue(settingsJson) { [RavenConfiguration.GetKey(x => x.Licensing.EulaAccepted)] = true, [RavenConfiguration.GetKey(x => x.Core.SetupMode)] = nameof(SetupMode.Unsecured), [RavenConfiguration.GetKey(x => x.Security.UnsecuredAccessAllowed)] = nameof(UnsecuredAccessAddressRange.PublicNetwork) }; if (setupInfo.Port == 0) { setupInfo.Port = 8080; } settingsJson.Modifications[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = string.Join(";", setupInfo.Addresses.Select(ip => IpAddressToUrl(ip, setupInfo.Port))); if (setupInfo.TcpPort == 0) { setupInfo.TcpPort = 38888; } settingsJson.Modifications[RavenConfiguration.GetKey(x => x.Core.TcpServerUrls)] = string.Join(";", setupInfo.Addresses.Select(ip => IpAddressToUrl(ip, setupInfo.TcpPort, "tcp"))); if (setupInfo.EnableExperimentalFeatures) { settingsJson.Modifications[RavenConfiguration.GetKey(x => x.Core.FeaturesAvailability)] = FeaturesAvailability.Experimental; } if (!string.IsNullOrEmpty(setupInfo.LocalNodeTag)) { ServerStore.EnsureNotPassive(nodeTag: setupInfo.LocalNodeTag); } if (setupInfo.Environment != StudioConfiguration.StudioEnvironment.None) { var res = await ServerStore.PutValueInClusterAsync( new PutServerWideStudioConfigurationCommand(new ServerWideStudioConfiguration { Disabled = false, Environment = setupInfo.Environment }, RaftIdGenerator.DontCareId)); await ServerStore.Cluster.WaitForIndexNotification(res.Index); } var modifiedJsonObj = context.ReadObject(settingsJson, "modified-settings-json"); var indentedJson = SetupManager.IndentJsonString(modifiedJsonObj.ToString()); SetupManager.WriteSettingsJsonLocally(ServerStore.Configuration.ConfigPath, indentedJson); } NoContentStatus(); }
public async Task MigrateFromAnotherDatabase() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var blittable = await context.ReadForMemoryAsync(RequestBodyStream(), "migration-configuration"); var migrationConfiguration = JsonDeserializationServer.MigrationConfiguration(blittable); if (string.IsNullOrWhiteSpace(migrationConfiguration.MigratorFullPath)) { throw new ArgumentException("MigratorFullPath cannot be null or empty"); } if (migrationConfiguration.InputConfiguration == null) { throw new ArgumentException("InputConfiguration cannot be null"); } if (migrationConfiguration.InputConfiguration.TryGet("Command", out string command) == false) { throw new ArgumentException("Cannot find the Command property in the InputConfiguration"); } var migratorFile = ResolveMigratorPath(migrationConfiguration); if (command == "validateMigratorPath") { NoContentStatus(); return; } if (string.IsNullOrWhiteSpace(migrationConfiguration.DatabaseTypeName)) { throw new ArgumentException("DatabaseTypeName cannot be null or empty"); } var processStartInfo = new ProcessStartInfo { FileName = migratorFile.FullName, Arguments = $"{migrationConfiguration.DatabaseTypeName}", WorkingDirectory = migratorFile.Directory.FullName, CreateNoWindow = true, RedirectStandardOutput = true, RedirectStandardError = true, RedirectStandardInput = true, UseShellExecute = false }; Process process = null; try { process = Process.Start(processStartInfo); } catch (Exception e) { var killed = KillProcess(process); throw new InvalidOperationException($"Unable to execute Migrator. Process killed: {killed}" + Environment.NewLine + "Command was: " + Environment.NewLine + (processStartInfo.WorkingDirectory ?? Directory.GetCurrentDirectory()) + "> " + processStartInfo.FileName + " " + processStartInfo.Arguments, e); } await process.StandardInput.WriteLineAsync(migrationConfiguration.InputConfiguration.ToString()); var isExportCommand = command == "export"; if (isExportCommand == false) { var errorReadTask = ReadOutput(process.StandardError); var outputReadTask = ReadOutput(process.StandardOutput); await Task.WhenAll(new Task[] { errorReadTask, outputReadTask }).ConfigureAwait(false); Debug.Assert(process.HasExited, "Migrator is still running!"); if (process.ExitCode == -1) { await ExitWithException(errorReadTask, null).ConfigureAwait(false); } try { var line = await outputReadTask.ConfigureAwait(false); using (var sw = new StreamWriter(ResponseBodyStream())) { await sw.WriteAsync(line); } return; } catch (Exception e) { await ExitWithException(errorReadTask, e).ConfigureAwait(false); } } var operationId = GetLongQueryString("operationId", false) ?? Database.Operations.GetNextOperationId(); var token = CreateOperationToken(); var transformScript = migrationConfiguration.TransformScript; var t = Database.Operations.AddOperation(Database, $"Migration from: {migrationConfiguration.DatabaseTypeName}", Operations.OperationType.DatabaseMigration, onProgress => { return(Task.Run(async() => { var result = new SmugglerResult(); try { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext migrateContext)) { var options = new DatabaseSmugglerOptionsServerSide { TransformScript = transformScript }; DoImportInternal(migrateContext, process.StandardOutput.BaseStream, options, result, onProgress, token); } } catch (OperationCanceledException) { KillProcess(process); throw; } catch (ObjectDisposedException) { KillProcess(process); throw; }
public void IndexDefinitionSerialization() { var indexDefinition = new IndexDefinition(); #if FEATURE_TEST_INDEX indexDefinition.IsTestIndex = true; #endif indexDefinition.LockMode = IndexLockMode.LockedIgnore; indexDefinition.Maps = new HashSet <string> { "a", "b" }; indexDefinition.Name = "n1"; indexDefinition.Reduce = "c"; indexDefinition.Fields = new Dictionary <string, IndexFieldOptions> { { "f1", new IndexFieldOptions { Spatial = new SpatialOptions { Type = SpatialFieldType.Geography, Units = SpatialUnits.Miles, MinY = 3, MinX = 5, MaxY = 2, MaxX = 5, Strategy = SpatialSearchStrategy.QuadPrefixTree, MaxTreeLevel = 2 }, Indexing = FieldIndexing.No, Suggestions = true, Storage = FieldStorage.Yes, Analyzer = "a1", TermVector = FieldTermVector.WithPositionsAndOffsets } }, { "f2", new IndexFieldOptions { Spatial = new SpatialOptions { Type = SpatialFieldType.Cartesian, Units = SpatialUnits.Kilometers, MinY = 5, MinX = 2, MaxY = 9, MaxX = 3, Strategy = SpatialSearchStrategy.BoundingBox, MaxTreeLevel = 5 }, Indexing = FieldIndexing.Exact, Suggestions = false, Storage = FieldStorage.No, Analyzer = "a2", TermVector = FieldTermVector.WithPositions } } }; using (var context = JsonOperationContext.ShortTermSingleUse()) { var builder = indexDefinition.ToJson(); using (var json = context.ReadObject(builder, nameof(IndexDefinition))) { var newIndexDefinition = JsonDeserializationServer.IndexDefinition(json); Assert.True(indexDefinition.Equals(newIndexDefinition)); } } }
protected async Task <RestoreSettings> SnapshotRestore(JsonOperationContext context, string backupPath, Action <IOperationProgress> onProgress, RestoreResult restoreResult) { Debug.Assert(onProgress != null); RestoreSettings restoreSettings = null; var fullBackupPath = GetBackupPath(backupPath); using (var zip = await GetZipArchiveForSnapshot(fullBackupPath)) { foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length))) { var directory = zipEntries.Key; if (string.IsNullOrWhiteSpace(directory)) { foreach (var zipEntry in zipEntries) { if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase)) { using (var entryStream = zipEntry.Open()) { var stream = RestoreFromConfiguration.BackupEncryptionSettings?.EncryptionMode == EncryptionMode.UseDatabaseKey ? new DecryptingXChaCha20Oly1305Stream(entryStream, Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey)) : entryStream; var json = context.Read(stream, "read database settings for restore"); json.BlittableValidation(); restoreSettings = JsonDeserializationServer.RestoreSettings(json); restoreSettings.DatabaseRecord.DatabaseName = RestoreFromConfiguration.DatabaseName; DatabaseHelper.Validate(RestoreFromConfiguration.DatabaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration); if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false) { throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!"); } if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey) { throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!"); } } } } continue; } var voronDataDirectory = new VoronPathSetting(RestoreFromConfiguration.DataDirectory); var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase) ? voronDataDirectory : voronDataDirectory.Combine(directory); BackupMethods.Full.Restore( zipEntries, restoreDirectory, journalDir: null, onProgress: message => { restoreResult.AddInfo(message); restoreResult.SnapshotRestore.ReadCount++; onProgress.Invoke(restoreResult.Progress); }, cancellationToken: _operationCancelToken.Token); } } if (restoreSettings == null) { throw new InvalidDataException("Cannot restore the snapshot without the settings file!"); } return(restoreSettings); }
public async Task AddNode() { SetupCORSHeaders(); var nodeUrl = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url"); var tag = GetStringQueryString("tag", false); var watcher = GetBoolValueQueryString("watcher", false); var assignedCores = GetIntValueQueryString("assignedCores", false); if (assignedCores <= 0) { throw new ArgumentException("Assigned cores must be greater than 0!"); } nodeUrl = UrlHelper.TryGetLeftPart(nodeUrl); var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase); if (HttpContext.Request.IsHttps != remoteIsHttps) { throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP."); } NodeInfo nodeInfo; using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx)) using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.Certificate.Certificate)) { requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout; var infoCmd = new GetNodeInfoCommand(); try { await requestExecutor.ExecuteAsync(infoCmd, ctx); } catch (AllTopologyNodesDownException e) { throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e); } nodeInfo = infoCmd.Result; if (ServerStore.IsPassive() && nodeInfo.TopologyId != null) { throw new TopologyMismatchException("You can't add new node to an already existing cluster"); } } if (assignedCores != null && assignedCores > nodeInfo.NumberOfCores) { throw new ArgumentException("Cannot add node because the assigned cores is larger " + $"than the available cores on that machine: {nodeInfo.NumberOfCores}"); } ServerStore.EnsureNotPassive(); if (assignedCores == null) { assignedCores = ServerStore.LicenseManager.GetCoresToAssign(nodeInfo.NumberOfCores); } Debug.Assert(assignedCores <= nodeInfo.NumberOfCores); ServerStore.LicenseManager.AssertCanAddNode(nodeUrl, assignedCores.Value); if (ServerStore.IsLeader()) { using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx)) { string topologyId; ClusterTopology clusterTopology; using (ctx.OpenReadTransaction()) { clusterTopology = ServerStore.GetClusterTopology(ctx); topologyId = clusterTopology.TopologyId; } var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl); if (possibleNode.HasUrl) { throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}"); } if (nodeInfo.ServerId == ServerStore.GetServerId()) { throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because it's a synonym of the current node URL:{ServerStore.GetNodeHttpServerUrl()}"); } if (nodeInfo.TopologyId != null) { if (topologyId != nodeInfo.TopologyId) { throw new TopologyMismatchException( $"Adding a new node to cluster failed. The new node is already in another cluster. " + $"Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}"); } if (nodeInfo.CurrentState != RachisState.Passive) { throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster " + $"because it's already in the cluster under tag :{nodeInfo.NodeTag} " + $"and URL: {clusterTopology.GetUrlFromTag(nodeInfo.NodeTag)}"); } } var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? tag : nodeInfo.NodeTag; CertificateDefinition oldServerCert = null; X509Certificate2 certificate = null; if (remoteIsHttps) { if (nodeInfo.Certificate == null) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS"); } certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet); var now = DateTime.UtcNow; if (certificate.NotBefore.ToUniversalTime() > now) { // Because of time zone and time drift issues, we can't assume that the certificate generation will be // proper. Because of that, we allow tolerance of the NotBefore to be a bit earlier / later than the // current time. Clients may still fail to work with our certificate because of timing issues, // but the admin needs to setup time sync properly and there isn't much we can do at that point if ((certificate.NotBefore.ToUniversalTime() - now).TotalDays > 1) { throw new InvalidOperationException( $"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}"); } } if (certificate.NotAfter.ToUniversalTime() < now) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}"); } var expected = GetStringQueryString("expectedThumbprint", required: false); if (expected != null) { if (certificate.Thumbprint != expected) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'."); } } using (ctx.OpenReadTransaction()) { var key = Constants.Certificates.Prefix + certificate.Thumbprint; var readCert = ServerStore.Cluster.Read(ctx, key); if (readCert != null) { oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert); } } if (oldServerCert == null) { var certificateDefinition = new CertificateDefinition { Certificate = nodeInfo.Certificate, Thumbprint = certificate.Thumbprint, NotAfter = certificate.NotAfter, Name = "Server Certificate for " + nodeUrl, SecurityClearance = SecurityClearance.ClusterNode }; var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition)); await ServerStore.Cluster.WaitForIndexNotification(res.Index); } } await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : true, asWatcher : watcher ?? false); using (ctx.OpenReadTransaction()) { clusterTopology = ServerStore.GetClusterTopology(ctx); possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl); nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null; if (certificate != null) { var key = Constants.Certificates.Prefix + certificate.Thumbprint; var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.Read(ctx, key)); if (modifiedServerCert == null) { throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through."); } if (oldServerCert == null) { modifiedServerCert.Name = "Server certificate for Node " + nodeTag; } else { var value = "Node " + nodeTag; if (modifiedServerCert.Name.Contains(value) == false) { modifiedServerCert.Name += ", " + value; } } var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(key, modifiedServerCert)); await ServerStore.Cluster.WaitForIndexNotification(res.Index); } var nodeDetails = new NodeDetails { NodeTag = nodeTag, AssignedCores = assignedCores.Value, NumberOfCores = nodeInfo.NumberOfCores, InstalledMemoryInGb = nodeInfo.InstalledMemoryInGb, UsableMemoryInGb = nodeInfo.UsableMemoryInGb, BuildInfo = nodeInfo.BuildInfo, OsInfo = nodeInfo.OsInfo }; await ServerStore.LicenseManager.CalculateLicenseLimits(nodeDetails, forceFetchingNodeInfo : true, waitToUpdate : true); } NoContentStatus(); return; } } RedirectToLeader(); }
private async Task PutInternal(bool validatedAsAdmin) { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var createdIndexes = new List <(string Name, long RaftIndex)>(); var isReplicatedQueryString = GetStringQueryString("is-replicated", required: false); if (isReplicatedQueryString != null && bool.TryParse(isReplicatedQueryString, out var result) && result) { await HandleLegacyIndexesAsync(); return; } var input = await context.ReadForMemoryAsync(RequestBodyStream(), "Indexes"); if (input.TryGet("Indexes", out BlittableJsonReaderArray indexes) == false) { ThrowRequiredPropertyNameInRequest("Indexes"); } var raftRequestId = GetRaftRequestIdFromQuery(); foreach (BlittableJsonReaderObject indexToAdd in indexes) { var indexDefinition = JsonDeserializationServer.IndexDefinition(indexToAdd); indexDefinition.Name = indexDefinition.Name?.Trim(); var source = IsLocalRequest(HttpContext) ? Environment.MachineName : HttpContext.Connection.RemoteIpAddress.ToString(); if (LoggingSource.AuditLog.IsInfoEnabled) { var clientCert = GetCurrentCertificate(); var auditLog = LoggingSource.AuditLog.GetLogger(Database.Name, "Audit"); auditLog.Info($"Index {indexDefinition.Name} PUT by {clientCert?.Subject} {clientCert?.Thumbprint} with definition: {indexToAdd} from {source} at {DateTime.UtcNow}"); } if (indexDefinition.Maps == null || indexDefinition.Maps.Count == 0) { throw new ArgumentException("Index must have a 'Maps' fields"); } indexDefinition.Type = indexDefinition.DetectStaticIndexType(); // C# index using a non-admin endpoint if (indexDefinition.Type.IsJavaScript() == false && validatedAsAdmin == false) { throw new UnauthorizedAccessException($"Index {indexDefinition.Name} is a C# index but was sent through a non-admin endpoint using REST api, this is not allowed."); } if (indexDefinition.Name.StartsWith(Constants.Documents.Indexing.SideBySideIndexNamePrefix, StringComparison.OrdinalIgnoreCase)) { throw new ArgumentException( $"Index name must not start with '{Constants.Documents.Indexing.SideBySideIndexNamePrefix}'. Provided index name: '{indexDefinition.Name}'"); } var index = await Database.IndexStore.CreateIndexInternal(indexDefinition, $"{raftRequestId}/{indexDefinition.Name}", source); createdIndexes.Add((indexDefinition.Name, index)); } if (TrafficWatchManager.HasRegisteredClients) { AddStringToHttpContext(indexes.ToString(), TrafficWatchChangeType.Index); } HttpContext.Response.StatusCode = (int)HttpStatusCode.Created; await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WriteArray(context, "Results", createdIndexes, (w, c, index) => { w.WriteStartObject(); w.WritePropertyName(nameof(PutIndexResult.Index)); w.WriteString(index.Name); w.WriteComma(); w.WritePropertyName(nameof(PutIndexResult.RaftCommandIndex)); w.WriteInteger(index.RaftIndex); w.WriteEndObject(); }); writer.WriteEndObject(); } } }
public void AcceptIncomingConnection(TcpConnectionOptions tcpConnectionOptions) { ReplicationLatestEtagRequest getLatestEtagMessage; using (tcpConnectionOptions.ContextPool.AllocateOperationContext(out JsonOperationContext context)) using (var readerObject = context.ParseToMemory( tcpConnectionOptions.Stream, "IncomingReplication/get-last-etag-message read", BlittableJsonDocumentBuilder.UsageMode.None, tcpConnectionOptions.PinnedBuffer)) { getLatestEtagMessage = JsonDeserializationServer.ReplicationLatestEtagRequest(readerObject); if (_log.IsInfoEnabled) { _log.Info( $"GetLastEtag: {getLatestEtagMessage.SourceTag}({getLatestEtagMessage.SourceMachineName}) / {getLatestEtagMessage.SourceDatabaseName} ({getLatestEtagMessage.SourceDatabaseId}) - {getLatestEtagMessage.SourceUrl}"); } } var connectionInfo = IncomingConnectionInfo.FromGetLatestEtag(getLatestEtagMessage); try { AssertValidConnection(connectionInfo); } catch (Exception e) { if (_log.IsInfoEnabled) { _log.Info($"Connection from [{connectionInfo}] is rejected.", e); } var incomingConnectionRejectionInfos = _incomingRejectionStats.GetOrAdd(connectionInfo, _ => new ConcurrentQueue <IncomingConnectionRejectionInfo>()); incomingConnectionRejectionInfos.Enqueue(new IncomingConnectionRejectionInfo { Reason = e.ToString() }); try { tcpConnectionOptions.Dispose(); } catch { // do nothing } throw; } try { using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsOperationContext)) using (Database.ConfigurationStorage.ContextPool.AllocateOperationContext(out TransactionOperationContext configurationContext)) using (var writer = new BlittableJsonTextWriter(documentsOperationContext, tcpConnectionOptions.Stream)) using (documentsOperationContext.OpenReadTransaction()) using (configurationContext.OpenReadTransaction()) { var changeVector = DocumentsStorage.GetDatabaseChangeVector(documentsOperationContext); var lastEtagFromSrc = Database.DocumentsStorage.GetLastReplicateEtagFrom( documentsOperationContext, getLatestEtagMessage.SourceDatabaseId); if (_log.IsInfoEnabled) { _log.Info($"GetLastEtag response, last etag: {lastEtagFromSrc}"); } var response = new DynamicJsonValue { [nameof(ReplicationMessageReply.Type)] = "Ok", [nameof(ReplicationMessageReply.MessageType)] = ReplicationMessageType.Heartbeat, [nameof(ReplicationMessageReply.LastEtagAccepted)] = lastEtagFromSrc, [nameof(ReplicationMessageReply.NodeTag)] = _server.NodeTag, [nameof(ReplicationMessageReply.DatabaseChangeVector)] = changeVector }; documentsOperationContext.Write(writer, response); writer.Flush(); } } catch (Exception) { try { tcpConnectionOptions.Dispose(); } catch (Exception) { // do nothing } throw; } var newIncoming = new IncomingReplicationHandler( tcpConnectionOptions, getLatestEtagMessage, this); newIncoming.Failed += OnIncomingReceiveFailed; newIncoming.DocumentsReceived += OnIncomingReceiveSucceeded; if (_log.IsInfoEnabled) { _log.Info( $"Initialized document replication connection from {connectionInfo.SourceDatabaseName} located at {connectionInfo.SourceUrl}"); } // need to safeguard against two concurrent connection attempts var newConnection = _incoming.GetOrAdd(newIncoming.ConnectionInfo.SourceDatabaseId, newIncoming); if (newConnection == newIncoming) { newIncoming.Start(); IncomingReplicationAdded?.Invoke(newIncoming); ForceTryReconnectAll(); } else { newIncoming.Dispose(); } }
public async Task PostImportAsync() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { if (HttpContext.Request.HasFormContentType == false) { HttpContext.Response.StatusCode = (int)HttpStatusCode.BadRequest; // Bad request using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue { ["Type"] = "Error", ["Error"] = "This endpoint requires form content type" }); return; } } var operationId = GetLongQueryString("operationId"); var token = CreateOperationToken(); var result = new SmugglerResult(); await Database.Operations.AddOperation(Database, "Import to: " + Database.Name, Operations.OperationType.DatabaseImport, onProgress => { return(Task.Run(async() => { try { var boundary = MultipartRequestHelper.GetBoundary( MediaTypeHeaderValue.Parse(HttpContext.Request.ContentType), MultipartRequestHelper.MultipartBoundaryLengthLimit); var reader = new MultipartReader(boundary, HttpContext.Request.Body); DatabaseSmugglerOptionsServerSide options = null; while (true) { var section = await reader.ReadNextSectionAsync().ConfigureAwait(false); if (section == null) { break; } if (ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out ContentDispositionHeaderValue contentDisposition) == false) { continue; } if (MultipartRequestHelper.HasFormDataContentDisposition(contentDisposition)) { var key = HeaderUtilities.RemoveQuotes(contentDisposition.Name); if (key != "importOptions") { continue; } BlittableJsonReaderObject blittableJson; if (section.Headers.ContainsKey("Content-Encoding") && section.Headers["Content-Encoding"] == "gzip") { using (var gzipStream = new GZipStream(section.Body, CompressionMode.Decompress)) { blittableJson = await context.ReadForMemoryAsync(gzipStream, "importOptions"); } } else { blittableJson = await context.ReadForMemoryAsync(section.Body, "importOptions"); } options = JsonDeserializationServer.DatabaseSmugglerOptions(blittableJson); continue; } if (MultipartRequestHelper.HasFileContentDisposition(contentDisposition) == false) { continue; } var stream = new GZipStream(section.Body, CompressionMode.Decompress); DoImportInternal(context, stream, options, result, onProgress, token); } } catch (Exception e) { result.AddError($"Error occurred during import. Exception: {e.Message}"); throw; } return (IOperationResult)result; })); }, operationId, token).ConfigureAwait(false); WriteImportResult(context, result, ResponseBodyStream()); }
public async Task GetFolderPathOptions() { PeriodicBackupConnectionType connectionType; var type = GetStringValuesQueryString("type", false).FirstOrDefault(); if (type == null) { //Backward compatibility connectionType = PeriodicBackupConnectionType.Local; } else if (Enum.TryParse(type, out connectionType) == false) { throw new ArgumentException($"Query string '{type}' was not recognized as valid type"); } using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) { var folderPathOptions = new FolderPathOptions(); ; switch (connectionType) { case PeriodicBackupConnectionType.Local: var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false; var path = GetStringQueryString("path", required: false); folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration); break; case PeriodicBackupConnectionType.S3: var json = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format"); if (connectionType != PeriodicBackupConnectionType.Local && json == null) { throw new BadRequestException("No JSON was posted."); } var s3Settings = JsonDeserializationServer.S3Settings(json); if (s3Settings == null) { throw new BadRequestException("No S3Settings were found."); } if (string.IsNullOrWhiteSpace(s3Settings.AwsAccessKey) || string.IsNullOrWhiteSpace(s3Settings.AwsSecretKey) || string.IsNullOrWhiteSpace(s3Settings.BucketName) || string.IsNullOrWhiteSpace(s3Settings.AwsRegionName)) { break; } using (var client = new RavenAwsS3Client(s3Settings, ServerStore.Configuration.Backup)) { // fetching only the first 64 results for the auto complete var folders = await client.ListObjectsAsync(s3Settings.RemoteFolderName, "/", true, take : 64); if (folders != null) { foreach (var folder in folders.FileInfoDetails) { var fullPath = folder.FullPath; if (string.IsNullOrWhiteSpace(fullPath)) { continue; } folderPathOptions.List.Add(fullPath); } } } break; case PeriodicBackupConnectionType.Azure: var azureJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format"); if (connectionType != PeriodicBackupConnectionType.Local && azureJson == null) { throw new BadRequestException("No JSON was posted."); } var azureSettings = JsonDeserializationServer.AzureSettings(azureJson); if (azureSettings == null) { throw new BadRequestException("No AzureSettings were found."); } if (string.IsNullOrWhiteSpace(azureSettings.AccountName) || string.IsNullOrWhiteSpace(azureSettings.AccountKey) || string.IsNullOrWhiteSpace(azureSettings.StorageContainer)) { break; } using (var client = RavenAzureClient.Create(azureSettings, ServerStore.Configuration.Backup)) { var folders = (await client.ListBlobsAsync(azureSettings.RemoteFolderName, "/", true)); foreach (var folder in folders.List) { var fullPath = folder.Name; if (string.IsNullOrWhiteSpace(fullPath)) { continue; } folderPathOptions.List.Add(fullPath); } } break; case PeriodicBackupConnectionType.GoogleCloud: var googleCloudJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format"); if (connectionType != PeriodicBackupConnectionType.Local && googleCloudJson == null) { throw new BadRequestException("No JSON was posted."); } var googleCloudSettings = JsonDeserializationServer.GoogleCloudSettings(googleCloudJson); if (googleCloudSettings == null) { throw new BadRequestException("No AzureSettings were found."); } if (string.IsNullOrWhiteSpace(googleCloudSettings.BucketName) || string.IsNullOrWhiteSpace(googleCloudSettings.GoogleCredentialsJson)) { break; } using (var client = new RavenGoogleCloudClient(googleCloudSettings, ServerStore.Configuration.Backup)) { var folders = (await client.ListObjectsAsync(googleCloudSettings.RemoteFolderName)); var requestedPathLength = googleCloudSettings.RemoteFolderName.Split('/').Length; foreach (var folder in folders) { const char separator = '/'; var splitted = folder.Name.Split(separator); var result = string.Join(separator, splitted.Take(requestedPathLength)) + separator; if (string.IsNullOrWhiteSpace(result)) { continue; } folderPathOptions.List.Add(result); } } break; case PeriodicBackupConnectionType.FTP: case PeriodicBackupConnectionType.Glacier: throw new NotSupportedException(); default: throw new ArgumentOutOfRangeException(); } await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue { [nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List) }); } } }
private void UploadToServer(string backupPath, string folderName, string fileName) { var s3Settings = GetBackupConfigurationFromScript(_configuration.S3Settings, x => JsonDeserializationServer.S3Settings(x), settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForS3(settings, _database.Name)); var glacierSettings = GetBackupConfigurationFromScript(_configuration.GlacierSettings, x => JsonDeserializationServer.GlacierSettings(x), settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForGlacier(settings, _database.Name)); var azureSettings = GetBackupConfigurationFromScript(_configuration.AzureSettings, x => JsonDeserializationServer.AzureSettings(x), settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForAzure(settings, _database.Name)); var googleCloudSettings = GetBackupConfigurationFromScript(_configuration.GoogleCloudSettings, x => JsonDeserializationServer.GoogleCloudSettings(x), settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForGoogleCloud(settings, _database.Name)); var ftpSettings = GetBackupConfigurationFromScript(_configuration.FtpSettings, x => JsonDeserializationServer.FtpSettings(x), settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForFtp(settings, _database.Name)); TaskCancelToken.Token.ThrowIfCancellationRequested(); var uploaderSettings = new BackupUploaderSettings { S3Settings = s3Settings, GlacierSettings = glacierSettings, AzureSettings = azureSettings, GoogleCloudSettings = googleCloudSettings, FtpSettings = ftpSettings, BackupPath = backupPath, FolderName = folderName, FileName = fileName, DatabaseName = _database.Name, TaskName = _configuration.Name, BackupType = _configuration.BackupType }; var backupUploader = new BackupUploader(uploaderSettings, _retentionPolicyParameters, _logger, _backupResult, _onProgress, TaskCancelToken); backupUploader.Execute(); }
public IEnumerable <QueryResult> MoreLikeThis( IndexQueryServerSide query, IQueryResultRetriever retriever, DocumentsOperationContext context, CancellationToken token) { IDisposable releaseServerContext = null; IDisposable closeServerTransaction = null; TransactionOperationContext serverContext = null; MoreLikeThisQuery moreLikeThisQuery; try { if (query.Metadata.HasCmpXchg) { releaseServerContext = context.DocumentDatabase.ServerStore.ContextPool.AllocateOperationContext(out serverContext); closeServerTransaction = serverContext.OpenReadTransaction(); } using (closeServerTransaction) moreLikeThisQuery = QueryBuilder.BuildMoreLikeThisQuery(serverContext, context, query.Metadata, query.Metadata.Query.Where, query.QueryParameters, _analyzer, _queryBuilderFactories); } finally { releaseServerContext?.Dispose(); } var options = moreLikeThisQuery.Options != null?JsonDeserializationServer.MoreLikeThisOptions(moreLikeThisQuery.Options) : MoreLikeThisOptions.Default; HashSet <string> stopWords = null; if (string.IsNullOrWhiteSpace(options.StopWordsDocumentId) == false) { var stopWordsDoc = context.DocumentDatabase.DocumentsStorage.Get(context, options.StopWordsDocumentId); if (stopWordsDoc == null) { throw new InvalidOperationException($"Stop words document {options.StopWordsDocumentId} could not be found"); } if (stopWordsDoc.Data.TryGet(nameof(MoreLikeThisStopWords.StopWords), out BlittableJsonReaderArray value) && value != null) { stopWords = new HashSet <string>(StringComparer.OrdinalIgnoreCase); for (var i = 0; i < value.Length; i++) { stopWords.Add(value.GetStringByIndex(i)); } } } var ir = _searcher.IndexReader; var mlt = new RavenMoreLikeThis(ir, options, _state); int?baseDocId = null; if (moreLikeThisQuery.BaseDocument == null) { var td = _searcher.Search(moreLikeThisQuery.BaseDocumentQuery, 1, _state); // get the current Lucene docid for the given RavenDB doc ID if (td.ScoreDocs.Length == 0) { throw new InvalidOperationException("Given filtering expression did not yield any documents that could be used as a base of comparison"); } baseDocId = td.ScoreDocs[0].Doc; } if (stopWords != null) { mlt.SetStopWords(stopWords); } string[] fieldNames; if (options.Fields != null && options.Fields.Length > 0) { fieldNames = options.Fields; } else { fieldNames = ir.GetFieldNames(IndexReader.FieldOption.INDEXED) .Where(x => x != Constants.Documents.Indexing.Fields.DocumentIdFieldName && x != Constants.Documents.Indexing.Fields.SourceDocumentIdFieldName && x != Constants.Documents.Indexing.Fields.ReduceKeyHashFieldName) .ToArray(); } mlt.SetFieldNames(fieldNames); mlt.Analyzer = _analyzer; var pageSize = GetPageSize(_searcher, query.PageSize); Query mltQuery; if (baseDocId.HasValue) { mltQuery = mlt.Like(baseDocId.Value); } else { using (var blittableJson = ParseJsonStringIntoBlittable(moreLikeThisQuery.BaseDocument, context)) mltQuery = mlt.Like(blittableJson); } var tsdc = TopScoreDocCollector.Create(pageSize, true); if (moreLikeThisQuery.FilterQuery != null && moreLikeThisQuery.FilterQuery is MatchAllDocsQuery == false) { mltQuery = new BooleanQuery { { mltQuery, Occur.MUST }, { moreLikeThisQuery.FilterQuery, Occur.MUST } }; } _searcher.Search(mltQuery, tsdc, _state); var hits = tsdc.TopDocs().ScoreDocs; var ids = new HashSet <string>(StringComparer.OrdinalIgnoreCase); for (int i = 0; i < hits.Length; i++) { var hit = hits[i]; token.ThrowIfCancellationRequested(); if (hit.Doc == baseDocId) { continue; } var doc = _searcher.Doc(hit.Doc, _state); var id = doc.Get(Constants.Documents.Indexing.Fields.DocumentIdFieldName, _state) ?? doc.Get(Constants.Documents.Indexing.Fields.ReduceKeyHashFieldName, _state); if (id == null) { continue; } if (ids.Add(id) == false) { continue; } yield return(new QueryResult { Result = retriever.Get(doc, hit, _state) }); } }
public IOperationResult RunPeriodicBackup(Action <IOperationProgress> onProgress) { _onProgress = onProgress; AddInfo($"Started task: '{_configuration.Name}'"); var totalSw = Stopwatch.StartNew(); var operationCanceled = false; var runningBackupStatus = _periodicBackup.RunningBackupStatus = new PeriodicBackupStatus { TaskId = _configuration.TaskId, BackupType = _configuration.BackupType, LastEtag = _previousBackupStatus.LastEtag, LastRaftIndex = _previousBackupStatus.LastRaftIndex, LastFullBackup = _previousBackupStatus.LastFullBackup, LastIncrementalBackup = _previousBackupStatus.LastIncrementalBackup, LastFullBackupInternal = _previousBackupStatus.LastFullBackupInternal, LastIncrementalBackupInternal = _previousBackupStatus.LastIncrementalBackupInternal, IsFull = _isFullBackup, LocalBackup = _previousBackupStatus.LocalBackup, LastOperationId = _previousBackupStatus.LastOperationId, FolderName = _previousBackupStatus.FolderName, LastDatabaseChangeVector = _previousBackupStatus.LastDatabaseChangeVector }; try { if (runningBackupStatus.LocalBackup == null) { runningBackupStatus.LocalBackup = new LocalBackup(); } if (runningBackupStatus.LastRaftIndex == null) { runningBackupStatus.LastRaftIndex = new LastRaftIndex(); } if (_logger.IsInfoEnabled) { var fullBackupText = "a " + (_configuration.BackupType == BackupType.Backup ? "full backup" : "snapshot"); _logger.Info($"Creating {(_isFullBackup ? fullBackupText : "an incremental backup")}"); } var currentLastRaftIndex = GetDatabaseEtagForBackup(); if (_isFullBackup == false) { // if we come from old version the _previousBackupStatus won't have LastRaftIndex if (_previousBackupStatus.LastRaftIndex == null) { _previousBackupStatus.LastRaftIndex = new LastRaftIndex(); } // no-op if nothing has changed var(currentLastEtag, currentChangeVector) = _database.ReadLastEtagAndChangeVector(); // if we come from old version the _previousBackupStatus won't have LastRaftIndex if (_previousBackupStatus.LastRaftIndex == null) { _previousBackupStatus.LastRaftIndex = new LastRaftIndex(); } if (currentLastEtag == _previousBackupStatus.LastEtag && currentChangeVector == _previousBackupStatus.LastDatabaseChangeVector && currentLastRaftIndex == _previousBackupStatus.LastRaftIndex.LastEtag) { var message = $"Skipping incremental backup because no changes were made from last full backup on {_previousBackupStatus.LastFullBackup}."; if (_logger.IsInfoEnabled) { _logger.Info(message); } UpdateOperationId(runningBackupStatus); runningBackupStatus.LastIncrementalBackup = _periodicBackup.StartTimeInUtc; DatabaseSmuggler.EnsureProcessed(_backupResult); AddInfo(message); return(_backupResult); } } // update the local configuration before starting the local backup var localSettings = GetBackupConfigurationFromScript(_configuration.LocalSettings, x => JsonDeserializationServer.LocalSettings(x), settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForLocal(settings, _database.Name)); GenerateFolderNameAndBackupDirectory(localSettings, out var nowAsString, out var folderName, out var backupDirectory); var startDocumentEtag = _isFullBackup == false ? _previousBackupStatus.LastEtag : null; var startRaftIndex = _isFullBackup == false ? _previousBackupStatus.LastRaftIndex.LastEtag : null; var fileName = GetFileName(_isFullBackup, backupDirectory.FullPath, nowAsString, _configuration.BackupType, out string backupFilePath); var internalBackupResult = CreateLocalBackupOrSnapshot(runningBackupStatus, backupFilePath, startDocumentEtag, startRaftIndex); runningBackupStatus.LocalBackup.BackupDirectory = _backupToLocalFolder ? backupDirectory.FullPath : null; runningBackupStatus.LocalBackup.TempFolderUsed = _backupToLocalFolder == false; runningBackupStatus.IsFull = _isFullBackup; try { UploadToServer(backupFilePath, folderName, fileName); } finally { runningBackupStatus.UploadToS3 = _backupResult.S3Backup; runningBackupStatus.UploadToAzure = _backupResult.AzureBackup; runningBackupStatus.UploadToGoogleCloud = _backupResult.GoogleCloudBackup; runningBackupStatus.UploadToGlacier = _backupResult.GlacierBackup; runningBackupStatus.UploadToFtp = _backupResult.FtpBackup; // if user did not specify local folder we delete the temporary file if (_backupToLocalFolder == false) { DeleteFile(backupFilePath); } } UpdateOperationId(runningBackupStatus); runningBackupStatus.LastEtag = internalBackupResult.LastDocumentEtag; runningBackupStatus.LastDatabaseChangeVector = internalBackupResult.LastDatabaseChangeVector; runningBackupStatus.LastRaftIndex.LastEtag = internalBackupResult.LastRaftIndex; runningBackupStatus.FolderName = folderName; if (_isFullBackup) { runningBackupStatus.LastFullBackup = _periodicBackup.StartTimeInUtc; } else { runningBackupStatus.LastIncrementalBackup = _periodicBackup.StartTimeInUtc; } totalSw.Stop(); if (_logger.IsInfoEnabled) { var fullBackupText = "a " + (_configuration.BackupType == BackupType.Backup ? " full backup" : " snapshot"); _logger.Info($"Successfully created {(_isFullBackup ? fullBackupText : "an incremental backup")} " + $"in {totalSw.ElapsedMilliseconds:#,#;;0} ms"); } return(_backupResult); } catch (OperationCanceledException) { operationCanceled = TaskCancelToken.Token.IsCancellationRequested && _databaseShutdownCancellationToken.IsCancellationRequested; throw; } catch (ObjectDisposedException) { // shutting down, probably operationCanceled = true; throw; } catch (Exception e) { const string message = "Error when performing periodic backup"; runningBackupStatus.Error = new Error { Exception = e.ToString(), At = DateTime.UtcNow }; if (_logger.IsOperationsEnabled) { _logger.Operations(message, e); } _database.NotificationCenter.Add(AlertRaised.Create( _database.Name, $"Periodic Backup task: '{_periodicBackup.Configuration.Name}'", message, AlertType.PeriodicBackup, NotificationSeverity.Error, details: new ExceptionDetails(e))); throw; } finally { if (operationCanceled == false) { // whether we succeeded or not, // we need to update the last backup time to avoid // starting a new backup right after this one if (_isFullBackup) { runningBackupStatus.LastFullBackupInternal = _periodicBackup.StartTimeInUtc; } else { runningBackupStatus.LastIncrementalBackupInternal = _periodicBackup.StartTimeInUtc; } runningBackupStatus.NodeTag = _serverStore.NodeTag; runningBackupStatus.DurationInMs = totalSw.ElapsedMilliseconds; runningBackupStatus.Version = ++_previousBackupStatus.Version; _periodicBackup.BackupStatus = runningBackupStatus; // save the backup status WriteStatus(runningBackupStatus); } } }
private static void UpdateDatabaseInfo(RawDatabaseRecord databaseRecord, ServerStore serverStore, string databaseName, DrivesUsage existingDrivesUsage, DatabaseInfoItem databaseInfoItem) { DatabaseInfo databaseInfo = null; if (serverStore.DatabaseInfoCache.TryGet(databaseName, databaseInfoJson => databaseInfo = JsonDeserializationServer.DatabaseInfo(databaseInfoJson)) == false) { return; } Debug.Assert(databaseInfo != null); var databaseTopology = databaseRecord.Topology; var indexesCount = databaseRecord.CountOfIndexes; databaseInfoItem.DocumentsCount = databaseInfo.DocumentsCount ?? 0; databaseInfoItem.IndexesCount = databaseInfo.IndexesCount ?? indexesCount; databaseInfoItem.ReplicationFactor = databaseTopology?.ReplicationFactor ?? databaseInfo.ReplicationFactor; databaseInfoItem.ErroredIndexesCount = databaseInfo.IndexingErrors ?? 0; if (databaseInfo.MountPointsUsage == null) { return; } foreach (var mountPointUsage in databaseInfo.MountPointsUsage) { var driveName = mountPointUsage.DiskSpaceResult.DriveName; var diskSpaceResult = DiskSpaceChecker.GetDiskSpaceInfo( mountPointUsage.DiskSpaceResult.DriveName, new DriveInfoBase { DriveName = driveName }); if (diskSpaceResult != null) { // update the latest drive info mountPointUsage.DiskSpaceResult = new Client.ServerWide.Operations.DiskSpaceResult { DriveName = diskSpaceResult.DriveName, VolumeLabel = diskSpaceResult.VolumeLabel, TotalFreeSpaceInBytes = diskSpaceResult.TotalFreeSpace.GetValue(SizeUnit.Bytes), TotalSizeInBytes = diskSpaceResult.TotalSize.GetValue(SizeUnit.Bytes) }; } UpdateMountPoint(serverStore.Configuration.Storage, mountPointUsage, databaseName, existingDrivesUsage); } }
public async Task GetFolderPathOptions() { PeriodicBackupConnectionType connectionType; var type = GetStringValuesQueryString("type", false).FirstOrDefault(); if (type == null) { //Backward compatibility connectionType = PeriodicBackupConnectionType.Local; } else if (Enum.TryParse(type, out connectionType) == false) { throw new ArgumentException($"Query string '{type}' was not recognized as valid type"); } using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) { FolderPathOptions folderPathOptions; switch (connectionType) { case PeriodicBackupConnectionType.Local: var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false; var path = GetStringQueryString("path", required: false); folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration); break; case PeriodicBackupConnectionType.S3: var json = context.ReadForMemory(RequestBodyStream(), "studio-tasks/format"); if (connectionType != PeriodicBackupConnectionType.Local && json == null) { throw new BadRequestException("No JSON was posted."); } var s3Settings = JsonDeserializationServer.S3Settings(json); if (s3Settings == null) { throw new BadRequestException("No S3Settings were found."); } using (var client = new RavenAwsS3Client(s3Settings)) { // fetching only the first 64 results for the auto complete var folders = await client.ListObjectsAsync(s3Settings.RemoteFolderName, "/", true, 64); folderPathOptions = new FolderPathOptions(); foreach (var folder in folders.FileInfoDetails) { var fullPath = folder.FullPath; if (string.IsNullOrWhiteSpace(fullPath)) { continue; } folderPathOptions.List.Add(fullPath); } } break; case PeriodicBackupConnectionType.Glacier: case PeriodicBackupConnectionType.Azure: case PeriodicBackupConnectionType.GoogleCloud: case PeriodicBackupConnectionType.FTP: throw new NotSupportedException(); default: throw new ArgumentOutOfRangeException(); } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue { [nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List) }); } } }
public async Task Try() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { BlittableJsonReaderObject json = await context.ReadForMemoryAsync(RequestBodyStream(), null); SubscriptionTryout tryout = JsonDeserializationServer.SubscriptionTryout(json); SubscriptionConnection.ParsedSubscription sub = SubscriptionConnection.ParseSubscriptionQuery(tryout.Query); SubscriptionPatchDocument patch = null; if (string.IsNullOrEmpty(sub.Script) == false) { patch = new SubscriptionPatchDocument(sub.Script, sub.Functions); } if (sub.Collection == null) { throw new ArgumentException("Collection must be specified"); } const int maxPageSize = 1024; var pageSize = GetIntValueQueryString("pageSize") ?? 1; if (pageSize > maxPageSize) { throw new ArgumentException($"Cannot gather more than {maxPageSize} results during tryouts, but requested number was {pageSize}."); } var state = new SubscriptionState { ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector, Query = tryout.Query }; var fetcher = new SubscriptionDocumentsFetcher(Database, int.MaxValue, -0x42, new IPEndPoint(HttpContext.Connection.RemoteIpAddress, HttpContext.Connection.RemotePort), sub.Collection, sub.Revisions, state, patch); var includeCmd = new IncludeDocumentsCommand(Database.DocumentsStorage, context, sub.Includes, isProjection: patch != null); if (Enum.TryParse( tryout.ChangeVector, out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue)) { switch (changeVectorSpecialValue) { case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime: case Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange: state.ChangeVectorForNextBatchStartingPoint = null; break; case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument: using (context.OpenReadTransaction()) { state.ChangeVectorForNextBatchStartingPoint = Database.DocumentsStorage.GetLastDocumentChangeVector(context, sub.Collection); } break; } } else { state.ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector; } var changeVector = state.ChangeVectorForNextBatchStartingPoint.ToChangeVector(); var cv = changeVector.FirstOrDefault(x => x.DbId == Database.DbBase64Id); var sp = Stopwatch.StartNew(); var timeLimit = TimeSpan.FromSeconds(GetIntValueQueryString("timeLimit", false) ?? 15); var startEtag = cv.Etag; using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) using (context.OpenReadTransaction()) { writer.WriteStartObject(); writer.WritePropertyName("Results"); writer.WriteStartArray(); var numberOfDocs = 0; while (numberOfDocs == 0 && sp.Elapsed < timeLimit) { var first = true; var lastEtag = startEtag; foreach (var itemDetails in fetcher.GetDataToSend(context, includeCmd, startEtag)) { if (itemDetails.Doc.Data != null) { using (itemDetails.Doc.Data) { includeCmd.Gather(itemDetails.Doc); if (first == false) { writer.WriteComma(); } if (itemDetails.Exception == null) { writer.WriteDocument(context, itemDetails.Doc, metadataOnly: false); } else { var documentWithException = new DocumentWithException { Exception = itemDetails.Exception.ToString(), ChangeVector = itemDetails.Doc.ChangeVector, Id = itemDetails.Doc.Id, DocumentData = itemDetails.Doc.Data }; writer.WriteObject(context.ReadObject(documentWithException.ToJson(), "")); } first = false; if (++numberOfDocs >= pageSize) { break; } } } if (sp.Elapsed >= timeLimit) { break; } lastEtag = itemDetails.Doc.Etag; } if (startEtag == lastEtag) { break; } startEtag = lastEtag; } writer.WriteEndArray(); writer.WriteComma(); writer.WritePropertyName("Includes"); var includes = new List <Document>(); includeCmd.Fill(includes); writer.WriteIncludes(context, includes); writer.WriteEndObject(); } } }
private async Task <PingResult> PingOnce(string url) { var sp = Stopwatch.StartNew(); var result = new PingResult { Url = url }; using (var cts = new CancellationTokenSource(ServerStore.Engine.TcpConnectionTimeout)) { var info = await ReplicationUtils.GetTcpInfoAsync(url, null, "PingTest", ServerStore.Engine.ClusterCertificate, cts.Token); result.TcpInfoTime = sp.ElapsedMilliseconds; using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) { var log = new List <string>(); using (await TcpUtils.ConnectSecuredTcpSocket(info, ServerStore.Engine.ClusterCertificate, Server.CipherSuitesPolicy, TcpConnectionHeaderMessage.OperationTypes.Ping, NegotiationCallback, context, ServerStore.Engine.TcpConnectionTimeout, log, cts.Token)) { } result.Log = log; async Task <TcpConnectionHeaderMessage.SupportedFeatures> NegotiationCallback(string curUrl, TcpConnectionInfo tcpInfo, Stream stream, JsonOperationContext ctx, List <string> logs = null) { try { var msg = new DynamicJsonValue { [nameof(TcpConnectionHeaderMessage.DatabaseName)] = null, [nameof(TcpConnectionHeaderMessage.Operation)] = TcpConnectionHeaderMessage.OperationTypes.Ping, [nameof(TcpConnectionHeaderMessage.OperationVersion)] = -1, [nameof(TcpConnectionHeaderMessage.ServerId)] = tcpInfo.ServerId }; await using (var writer = new AsyncBlittableJsonTextWriter(ctx, stream)) using (var msgJson = ctx.ReadObject(msg, "message")) { result.SendTime = sp.ElapsedMilliseconds; logs?.Add($"message sent to url {curUrl} at {result.SendTime} ms."); ctx.Write(writer, msgJson); } using (var rawResponse = await ctx.ReadForMemoryAsync(stream, "cluster-ConnectToPeer-header-response")) { TcpConnectionHeaderResponse response = JsonDeserializationServer.TcpConnectionHeaderResponse(rawResponse); result.ReceiveTime = sp.ElapsedMilliseconds; logs?.Add($"response received from url {curUrl} at {result.ReceiveTime} ms."); switch (response.Status) { case TcpConnectionStatus.Ok: result.Error = null; logs?.Add($"Successfully negotiated with {url}."); break; case TcpConnectionStatus.AuthorizationFailed: result.Error = $"Connection to {url} failed because of authorization failure: {response.Message}"; logs?.Add(result.Error); throw new AuthorizationException(result.Error); case TcpConnectionStatus.TcpVersionMismatch: result.Error = $"Connection to {url} failed because of mismatching tcp version: {response.Message}"; logs?.Add(result.Error); throw new AuthorizationException(result.Error); case TcpConnectionStatus.InvalidNetworkTopology: result.Error = $"Connection to {url} failed because of {nameof(TcpConnectionStatus.InvalidNetworkTopology)} error: {response.Message}"; logs?.Add(result.Error); throw new InvalidNetworkTopologyException(result.Error); } } } catch (Exception e) { result.Error = e.ToString(); logs?.Add($"Error occurred while attempting to negotiate with the server. {e.Message}"); throw; } return(null); } } } return(result); }
public async Task UserDomains() { AssertOnlyInSetupMode(); using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) { var json = context.Read(RequestBodyStream(), "license activation"); var licenseInfo = JsonDeserializationServer.LicenseInfo(json); var content = new StringContent(JsonConvert.SerializeObject(licenseInfo), Encoding.UTF8, "application/json"); try { string error = null; object result = null; string responseString = null; string errorMessage = null; try { var response = await ApiHttpClient.Instance.PostAsync("/api/v1/dns-n-cert/user-domains", content).ConfigureAwait(false); HttpContext.Response.StatusCode = (int)response.StatusCode; responseString = await response.Content.ReadAsStringAsync().ConfigureAwait(false); if (response.IsSuccessStatusCode == false) { error = responseString; errorMessage = GeneralDomainRegistrationError; } else { result = JsonConvert.DeserializeObject <JObject>(responseString); } } catch (Exception e) { result = responseString; HttpContext.Response.StatusCode = (int)HttpStatusCode.InternalServerError; error = e.ToString(); errorMessage = DomainRegistrationServiceUnreachableError; } if (error != null) { JToken errorJToken = null; if (responseString != null) { JsonConvert.DeserializeObject <JObject>(responseString).TryGetValue("Error", out errorJToken); } using (var streamWriter = new StreamWriter(ResponseBodyStream())) { new JsonSerializer().Serialize(streamWriter, new { Message = errorMessage, Response = result, Error = errorJToken ?? error }); streamWriter.Flush(); } return; } var results = JsonConvert.DeserializeObject <UserDomainsResult>(responseString); var fullResult = new UserDomainsAndLicenseInfo { UserDomainsWithIps = new UserDomainsWithIps { Emails = results.Emails, RootDomains = results.RootDomains, Domains = new Dictionary <string, List <SubDomainAndIps> >() } }; foreach (var domain in results.Domains) { var list = new List <SubDomainAndIps>(); foreach (var subDomain in domain.Value) { try { list.Add(new SubDomainAndIps { SubDomain = subDomain, // The ip list will be populated on the next call (/setup/populate-ips), when we know which root domain the user selected }); } catch (Exception) { continue; } } fullResult.UserDomainsWithIps.Domains.Add(domain.Key, list); } var licenseStatus = await SetupManager .GetUpdatedLicenseStatus(ServerStore, licenseInfo.License) .ConfigureAwait(false); fullResult.MaxClusterSize = licenseStatus.MaxClusterSize; fullResult.LicenseType = licenseStatus.Type; using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { var blittable = EntityToBlittable.ConvertCommandToBlittable(fullResult, context); context.Write(writer, blittable); } } catch (LicenseExpiredException) { throw; } catch (Exception e) { throw new InvalidOperationException(GeneralDomainRegistrationError, e); } } }
public async Task AddNode() { var nodeUrl = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url"); var tag = GetStringQueryString("tag", false); var watcher = GetBoolValueQueryString("watcher", false); var raftRequestId = GetRaftRequestIdFromQuery(); var maxUtilizedCores = GetIntValueQueryString("maxUtilizedCores", false); if (maxUtilizedCores != null && maxUtilizedCores <= 0) { throw new ArgumentException("Max utilized cores cores must be greater than 0"); } nodeUrl = nodeUrl.Trim(); if (Uri.IsWellFormedUriString(nodeUrl, UriKind.Absolute) == false) { throw new InvalidOperationException($"Given node URL '{nodeUrl}' is not in a correct format."); } nodeUrl = UrlHelper.TryGetLeftPart(nodeUrl); var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase); if (HttpContext.Request.IsHttps != remoteIsHttps) { throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP."); } tag = tag?.Trim(); NodeInfo nodeInfo; using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx)) using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.Certificate.Certificate)) { requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout; // test connection to remote. var result = await ServerStore.TestConnectionToRemote(nodeUrl, database : null); if (result.Success == false) { throw new InvalidOperationException(result.Error); } // test connection from remote to destination result = await ServerStore.TestConnectionFromRemote(requestExecutor, ctx, nodeUrl); if (result.Success == false) { throw new InvalidOperationException(result.Error); } var infoCmd = new GetNodeInfoCommand(); try { await requestExecutor.ExecuteAsync(infoCmd, ctx); } catch (AllTopologyNodesDownException e) { throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e); } nodeInfo = infoCmd.Result; if (SchemaUpgrader.CurrentVersion.ServerVersion != nodeInfo.ServerSchemaVersion) { var nodesVersion = nodeInfo.ServerSchemaVersion == 0 ? "Pre 4.2 version" : nodeInfo.ServerSchemaVersion.ToString(); throw new InvalidOperationException($"Can't add node with mismatched storage schema version.{Environment.NewLine}" + $"My version is {SchemaUpgrader.CurrentVersion.ServerVersion}, while node's version is {nodesVersion}"); } if (ServerStore.IsPassive() && nodeInfo.TopologyId != null) { throw new TopologyMismatchException("You can't add new node to an already existing cluster"); } } if (ServerStore.ValidateFixedPort && nodeInfo.HasFixedPort == false) { throw new InvalidOperationException($"Failed to add node '{nodeUrl}' to cluster. " + $"Node '{nodeUrl}' has port '0' in 'Configuration.Core.ServerUrls' setting. " + "Adding a node with non fixed port is forbidden. Define a fixed port for the node to enable cluster creation."); } ServerStore.EnsureNotPassive(); ServerStore.LicenseManager.AssertCanAddNode(); if (ServerStore.IsLeader()) { using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx)) { var clusterTopology = ServerStore.GetClusterTopology(); var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl); if (possibleNode.HasUrl) { throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}"); } if (nodeInfo.ServerId == ServerStore.GetServerId()) { throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because it's a synonym of the current node URL:{ServerStore.GetNodeHttpServerUrl()}"); } if (nodeInfo.TopologyId != null) { AssertCanAddNodeWithTopologyId(clusterTopology, nodeInfo, nodeUrl); } var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? tag : nodeInfo.NodeTag; CertificateDefinition oldServerCert = null; X509Certificate2 certificate = null; if (remoteIsHttps) { if (nodeInfo.Certificate == null) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS"); } certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet); var now = DateTime.UtcNow; if (certificate.NotBefore.ToUniversalTime() > now) { // Because of time zone and time drift issues, we can't assume that the certificate generation will be // proper. Because of that, we allow tolerance of the NotBefore to be a bit earlier / later than the // current time. Clients may still fail to work with our certificate because of timing issues, // but the admin needs to setup time sync properly and there isn't much we can do at that point if ((certificate.NotBefore.ToUniversalTime() - now).TotalDays > 1) { throw new InvalidOperationException( $"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}"); } } if (certificate.NotAfter.ToUniversalTime() < now) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}"); } var expected = GetStringQueryString("expectedThumbprint", required: false); if (expected != null) { if (certificate.Thumbprint != expected) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'."); } } // if it's the same server certificate as our own, we don't want to add it to the cluster if (certificate.Thumbprint != Server.Certificate.Certificate.Thumbprint) { using (ctx.OpenReadTransaction()) { var readCert = ServerStore.Cluster.GetCertificateByThumbprint(ctx, certificate.Thumbprint); if (readCert != null) { oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert); } } if (oldServerCert == null) { var certificateDefinition = new CertificateDefinition { Certificate = nodeInfo.Certificate, Thumbprint = certificate.Thumbprint, PublicKeyPinningHash = certificate.GetPublicKeyPinningHash(), NotAfter = certificate.NotAfter, Name = "Server Certificate for " + nodeUrl, SecurityClearance = SecurityClearance.ClusterNode }; var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(certificate.Thumbprint, certificateDefinition, $"{raftRequestId}/put-new-certificate")); await ServerStore.Cluster.WaitForIndexNotification(res.Index); } } } await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : true, asWatcher : watcher ?? false); using (ctx.OpenReadTransaction()) { clusterTopology = ServerStore.GetClusterTopology(ctx); possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl); nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null; if (certificate != null && certificate.Thumbprint != Server.Certificate.Certificate.Thumbprint) { var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.GetCertificateByThumbprint(ctx, certificate.Thumbprint)); if (modifiedServerCert == null) { throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through."); } if (oldServerCert == null) { modifiedServerCert.Name = "Server certificate for Node " + nodeTag; } else { var value = "Node " + nodeTag; if (modifiedServerCert.Name.Contains(value) == false) { modifiedServerCert.Name += ", " + value; } } var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(certificate.Thumbprint, modifiedServerCert, $"{raftRequestId}/put-modified-certificate")); await ServerStore.Cluster.WaitForIndexNotification(res.Index); } var detailsPerNode = new DetailsPerNode { MaxUtilizedCores = maxUtilizedCores, NumberOfCores = nodeInfo.NumberOfCores, InstalledMemoryInGb = nodeInfo.InstalledMemoryInGb, UsableMemoryInGb = nodeInfo.UsableMemoryInGb, BuildInfo = nodeInfo.BuildInfo, OsInfo = nodeInfo.OsInfo }; var maxCores = ServerStore.LicenseManager.LicenseStatus.MaxCores; try { await ServerStore.PutNodeLicenseLimitsAsync(nodeTag, detailsPerNode, maxCores, $"{raftRequestId}/put-license-limits"); } catch { // we'll retry this again later } } NoContentStatus(); return; } } RedirectToLeader(); }
public Task SetupUnsecured() { AssertOnlyInSetupMode(); using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (var setupInfoJson = context.ReadForMemory(RequestBodyStream(), "setup-unsecured")) { // Making sure we don't have leftovers from previous setup try { using (var tx = context.OpenWriteTransaction()) { ServerStore.Engine.DeleteTopology(context); tx.Commit(); } } catch (Exception) { // ignored } var setupInfo = JsonDeserializationServer.UnsecuredSetupInfo(setupInfoJson); BlittableJsonReaderObject settingsJson; using (var fs = new FileStream(ServerStore.Configuration.ConfigPath, FileMode.Open, FileAccess.Read)) { settingsJson = context.ReadForMemory(fs, "settings-json"); } settingsJson.Modifications = new DynamicJsonValue(settingsJson) { [RavenConfiguration.GetKey(x => x.Licensing.EulaAccepted)] = true, [RavenConfiguration.GetKey(x => x.Core.SetupMode)] = nameof(SetupMode.Unsecured), [RavenConfiguration.GetKey(x => x.Security.UnsecuredAccessAllowed)] = nameof(UnsecuredAccessAddressRange.PublicNetwork) }; if (setupInfo.Port == 0) { setupInfo.Port = 8080; } settingsJson.Modifications[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = string.Join(";", setupInfo.Addresses.Select(ip => IpAddressToUrl(ip, setupInfo.Port))); if (setupInfo.TcpPort == 0) { setupInfo.TcpPort = 38888; } settingsJson.Modifications[RavenConfiguration.GetKey(x => x.Core.TcpServerUrls)] = string.Join(";", setupInfo.Addresses.Select(ip => IpAddressToUrl(ip, setupInfo.TcpPort, "tcp"))); if (setupInfo.EnableExperimentalFeatures) { settingsJson.Modifications[RavenConfiguration.GetKey(x => x.Core.FeaturesAvailability)] = FeaturesAvailability.Experimental; } var modifiedJsonObj = context.ReadObject(settingsJson, "modified-settings-json"); var indentedJson = SetupManager.IndentJsonString(modifiedJsonObj.ToString()); SetupManager.WriteSettingsJsonLocally(ServerStore.Configuration.ConfigPath, indentedJson); } return(NoContent()); }
public async Task Try() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { BlittableJsonReaderObject json = await context.ReadForMemoryAsync(RequestBodyStream(), null); SubscriptionTryout tryout = JsonDeserializationServer.SubscriptionTryout(json); SubscriptionConnection.ParsedSubscription sub = SubscriptionConnection.ParseSubscriptionQuery(tryout.Query); SubscriptionPatchDocument patch = null; if (string.IsNullOrEmpty(sub.Script) == false) { patch = new SubscriptionPatchDocument(sub.Script, sub.Functions); } if (sub.Collection == null) { throw new ArgumentException("Collection must be specified"); } var pageSize = GetIntValueQueryString("pageSize") ?? 1; var state = new SubscriptionState { ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector, Query = tryout.Query }; var fetcher = new SubscriptionDocumentsFetcher(Database, pageSize, -0x42, new IPEndPoint(HttpContext.Connection.RemoteIpAddress, HttpContext.Connection.RemotePort), sub.Collection, sub.Revisions, state, patch); var includeCmd = new IncludeDocumentsCommand(Database.DocumentsStorage, context, sub.Includes); if (Enum.TryParse( tryout.ChangeVector, out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue)) { switch (changeVectorSpecialValue) { case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime: case Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange: state.ChangeVectorForNextBatchStartingPoint = null; break; case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument: state.ChangeVectorForNextBatchStartingPoint = Database.DocumentsStorage.GetLastDocumentChangeVector(context, sub.Collection); break; } } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName("Results"); writer.WriteStartArray(); using (context.OpenReadTransaction()) { var first = true; foreach (var itemDetails in fetcher.GetDataToSend(context, includeCmd, 0)) { if (itemDetails.Doc.Data == null) { continue; } includeCmd.Gather(itemDetails.Doc); if (first == false) { writer.WriteComma(); } if (itemDetails.Exception == null) { writer.WriteDocument(context, itemDetails.Doc, metadataOnly: false); } else { var docWithExcepton = new DocumentWithException { Exception = itemDetails.Exception.ToString(), ChangeVector = itemDetails.Doc.ChangeVector, Id = itemDetails.Doc.Id, DocumentData = itemDetails.Doc.Data }; writer.WriteObject(context.ReadObject(docWithExcepton.ToJson(), "")); } first = false; } writer.WriteEndArray(); writer.WriteComma(); writer.WritePropertyName("Includes"); var includes = new List <Document>(); includeCmd.Fill(includes); writer.WriteIncludes(context, includes); writer.WriteEndObject(); } } } }