Ejemplo n.º 1
0
        protected async Task <RestoreSettings> SnapshotRestore(JsonOperationContext context, string backupPath,
                                                               Action <IOperationProgress> onProgress, RestoreResult restoreResult)
        {
            Debug.Assert(onProgress != null);

            RestoreSettings restoreSettings = null;

            var fullBackupPath = GetBackupPath(backupPath);

            using (var zip = await GetZipArchiveForSnapshot(fullBackupPath))
            {
                var restorePath = new VoronPathSetting(RestoreFromConfiguration.DataDirectory);
                if (Directory.Exists(restorePath.FullPath) == false)
                {
                    Directory.CreateDirectory(restorePath.FullPath);
                }

                // validate free space
                var snapshotSize = zip.Entries.Sum(entry => entry.Length);
                BackupHelper.AssertFreeSpaceForSnapshot(restorePath.FullPath, snapshotSize, "restore a backup", Logger);

                foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length)))
                {
                    var directory = zipEntries.Key;

                    if (string.IsNullOrWhiteSpace(directory))
                    {
                        foreach (var zipEntry in zipEntries)
                        {
                            if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase))
                            {
                                await using (var entryStream = zipEntry.Open())
                                {
                                    var snapshotEncryptionKey = RestoreFromConfiguration.EncryptionKey != null
                                        ? Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey)
                                        : null;

                                    await using (var stream = GetInputStream(entryStream, snapshotEncryptionKey))
                                    {
                                        var json = await context.ReadForMemoryAsync(stream, "read database settings for restore");

                                        json.BlittableValidation();

                                        restoreSettings = JsonDeserializationServer.RestoreSettings(json);

                                        restoreSettings.DatabaseRecord.DatabaseName = RestoreFromConfiguration.DatabaseName;
                                        DatabaseHelper.Validate(RestoreFromConfiguration.DatabaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration);

                                        if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false)
                                        {
                                            throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!");
                                        }

                                        if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey)
                                        {
                                            throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!");
                                        }
                                    }
                                }
                            }
                        }

                        continue;
                    }

                    var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase)
                        ? restorePath
                        : restorePath.Combine(directory);

                    var isSubDirectory = PathUtil.IsSubDirectory(restoreDirectory.FullPath, restorePath.FullPath);
                    if (isSubDirectory == false)
                    {
                        var extensions = zipEntries
                                         .Select(x => Path.GetExtension(x.Name))
                                         .Distinct()
                                         .ToArray();

                        if (extensions.Length != 1 || string.Equals(extensions[0], TableValueCompressor.CompressionRecoveryExtension, StringComparison.OrdinalIgnoreCase) == false)
                        {
                            throw new InvalidOperationException($"Encountered invalid directory '{directory}' in snapshot file with following file extensions: {string.Join(", ", extensions)}");
                        }

                        // this enables backward compatibility of snapshot backups with compression recovery files before fix was made in RavenDB-17173
                        // the underlying issue was that we were putting full path when compression recovery files were backed up using snapshot
                        // because of that the end restore directory was not a sub-directory of a restore path
                        // which could result in a file already exists exception
                        // since restoring of compression recovery files is not mandatory then it is safe to skip them
                        continue;
                    }

                    BackupMethods.Full.Restore(
                        zipEntries,
                        restoreDirectory,
                        journalDir: null,
                        onProgress: message =>
                    {
                        restoreResult.AddInfo(message);
                        restoreResult.SnapshotRestore.ReadCount++;
                        onProgress.Invoke(restoreResult.Progress);
                    },
                        cancellationToken: _operationCancelToken.Token);
                }
            }

            if (restoreSettings == null)
            {
                throw new InvalidDataException("Cannot restore the snapshot without the settings file!");
            }

            return(restoreSettings);
        }
Ejemplo n.º 2
0
        public Task ExtractInfoFromZip()
        {
            AssertOnlyInSetupMode();

            using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
                using (var continueSetupInfoJson = context.ReadForMemory(RequestBodyStream(), "continue-setup-info"))
                {
                    var    continueSetupInfo = JsonDeserializationServer.ContinueSetupInfo(continueSetupInfoJson);
                    byte[] zipBytes;
                    try
                    {
                        zipBytes = Convert.FromBase64String(continueSetupInfo.Zip);
                    }
                    catch (Exception e)
                    {
                        throw new ArgumentException($"Unable to parse the {nameof(continueSetupInfo.Zip)} property, expected a Base64 value", e);
                    }

                    try
                    {
                        var urlByTag = new Dictionary <string, string>();

                        using (var ms = new MemoryStream(zipBytes))
                            using (var archive = new ZipArchive(ms, ZipArchiveMode.Read, false))
                            {
                                foreach (var entry in archive.Entries)
                                {
                                    if (entry.Name.Equals("settings.json") == false)
                                    {
                                        continue;
                                    }

                                    var tag = entry.FullName[0].ToString();

                                    using (var settingsJson = context.ReadForMemory(entry.Open(), "settings-json"))
                                        if (settingsJson.TryGet(nameof(ConfigurationNodeInfo.PublicServerUrl), out string publicServerUrl))
                                        {
                                            urlByTag[tag] = publicServerUrl;
                                        }
                                }
                            }

                        using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                        {
                            writer.WriteStartArray();
                            var first = true;

                            foreach (var node in urlByTag)
                            {
                                if (first == false)
                                {
                                    writer.WriteComma();
                                }

                                writer.WriteStartObject();
                                writer.WritePropertyName(nameof(ConfigurationNodeInfo.Tag));
                                writer.WriteString(node.Key);
                                writer.WriteComma();
                                writer.WritePropertyName(nameof(ConfigurationNodeInfo.PublicServerUrl));
                                writer.WriteString(node.Value);
                                writer.WriteEndObject();

                                first = false;
                            }

                            writer.WriteEndArray();
                        }
                    }
                    catch (Exception e)
                    {
                        throw new InvalidOperationException("Unable to extract setup information from the zip file.", e);
                    }
                }

            return(Task.CompletedTask);
        }
Ejemplo n.º 3
0
        public Task GetHosts()
        {
            AssertOnlyInSetupMode();

            using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
                using (var certificateJson = context.ReadForMemory(RequestBodyStream(), "setup-certificate"))
                {
                    var certDef = JsonDeserializationServer.CertificateDefinition(certificateJson);

                    X509Certificate2 certificate = null;
                    string           cn;

                    try
                    {
                        certificate = certDef.Password == null
                        ? new X509Certificate2(Convert.FromBase64String(certDef.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet)
                        : new X509Certificate2(Convert.FromBase64String(certDef.Certificate), certDef.Password, X509KeyStorageFlags.MachineKeySet);

                        cn = certificate.GetNameInfo(X509NameType.SimpleName, false);
                    }
                    catch (Exception e)
                    {
                        throw new BadRequestException($"Failed to extract the CN property from the certificate {certificate?.FriendlyName}. Maybe the password is wrong?", e);
                    }

                    if (cn == null)
                    {
                        throw new BadRequestException($"Failed to extract the CN property from the certificate. CN is null");
                    }

                    if (cn.LastIndexOf('*') > 0)
                    {
                        throw new NotSupportedException("The wildcard CN name contains a '*' which is not at the first character of the string. It is not supported in the Setup Wizard, you can do a manual setup instead.");
                    }

                    try
                    {
                        SecretProtection.ValidateKeyUsages("Setup Wizard", certificate);
                    }
                    catch (Exception e)
                    {
                        throw new InvalidOperationException($"Failed to load the uploaded certificate. Did you accidently upload a client certificate?", e);
                    }

                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        writer.WriteStartObject();
                        writer.WritePropertyName("CN");
                        writer.WriteString(cn);
                        writer.WriteComma();
                        writer.WritePropertyName("AlternativeNames");
                        writer.WriteStartArray();

                        var first = true;
                        foreach (var value in SetupManager.GetCertificateAlternativeNames(certificate))
                        {
                            if (first == false)
                            {
                                writer.WriteComma();
                            }
                            first = false;

                            writer.WriteString(value);
                        }

                        writer.WriteEndArray();

                        writer.WriteEndObject();
                    }
                }

            return(Task.CompletedTask);
        }
Ejemplo n.º 4
0
        public async Task TestJavaScriptIndex()
        {
            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
            {
                var input = await context.ReadForMemoryAsync(RequestBodyStream(), "TestJavaScriptIndex");

                if (input.TryGet("Definition", out BlittableJsonReaderObject index) == false)
                {
                    ThrowRequiredPropertyNameInRequest("Definition");
                }

                input.TryGet("Ids", out BlittableJsonReaderArray ids);

                var indexDefinition = JsonDeserializationServer.IndexDefinition(index);

                if (indexDefinition.Maps == null || indexDefinition.Maps.Count == 0)
                {
                    throw new ArgumentException("Index must have a 'Maps' fields");
                }

                indexDefinition.Type = indexDefinition.DetectStaticIndexType();

                if (indexDefinition.Type.IsJavaScript() == false)
                {
                    throw new UnauthorizedAccessException("Testing indexes is only allowed for JavaScript indexes.");
                }

                var compiledIndex = new JavaScriptIndex(indexDefinition, Database.Configuration);

                var inputSize         = GetIntValueQueryString("inputSize", false) ?? defaultInputSizeForTestingJavaScriptIndex;
                var collections       = new HashSet <string>(compiledIndex.Maps.Keys);
                var docsPerCollection = new Dictionary <string, List <DynamicBlittableJson> >();
                using (context.OpenReadTransaction())
                {
                    if (ids == null)
                    {
                        foreach (var collection in collections)
                        {
                            docsPerCollection.Add(collection,
                                                  Database.DocumentsStorage.GetDocumentsFrom(context, collection, 0, 0, inputSize).Select(d => new DynamicBlittableJson(d)).ToList());
                        }
                    }
                    else
                    {
                        var listOfIds = ids.Select(x => x.ToString());
                        var _         = new Reference <int>
                        {
                            Value = 0
                        };
                        var docs = Database.DocumentsStorage.GetDocuments(context, listOfIds, 0, int.MaxValue, _);
                        foreach (var doc in docs)
                        {
                            if (doc.TryGetMetadata(out var metadata) && metadata.TryGet(Constants.Documents.Metadata.Collection, out string collectionStr))
                            {
                                if (docsPerCollection.TryGetValue(collectionStr, out var listOfDocs) == false)
                                {
                                    listOfDocs = docsPerCollection[collectionStr] = new List <DynamicBlittableJson>();
                                }
                                listOfDocs.Add(new DynamicBlittableJson(doc));
                            }
                        }
                    }

                    var mapRes = new List <ObjectInstance>();
                    //all maps
                    foreach (var ListOfFunctions in compiledIndex.Maps)
                    {
                        //multi maps per collection
                        foreach (var mapFunc in ListOfFunctions.Value)
                        {
                            if (docsPerCollection.TryGetValue(ListOfFunctions.Key, out var docs))
                            {
                                foreach (var res in mapFunc(docs))
                                {
                                    mapRes.Add((ObjectInstance)res);
                                }
                            }
                        }
                    }
                    var first = true;
                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        writer.WriteStartObject();
                        writer.WritePropertyName("MapResults");
                        writer.WriteStartArray();
                        foreach (var mapResult in mapRes)
                        {
                            if (JavaScriptIndexUtils.StringifyObject(mapResult) is JsString jsStr)
                            {
                                if (first == false)
                                {
                                    writer.WriteComma();
                                }
                                writer.WriteString(jsStr.ToString());
                                first = false;
                            }
                        }
                        writer.WriteEndArray();
                        if (indexDefinition.Reduce != null)
                        {
                            using (var bufferPool = new UnmanagedBuffersPoolWithLowMemoryHandling("JavaScriptIndexTest", Database.Name))
                            {
                                compiledIndex.SetBufferPoolForTestingPurposes(bufferPool);
                                compiledIndex.SetAllocatorForTestingPurposes(context.Allocator);
                                first = true;
                                writer.WritePropertyName("ReduceResults");
                                writer.WriteStartArray();

                                var reduceResults = compiledIndex.Reduce(mapRes.Select(mr => new DynamicBlittableJson(JsBlittableBridge.Translate(context, mr.Engine, mr))));

                                foreach (JsValue reduceResult in reduceResults)
                                {
                                    if (JavaScriptIndexUtils.StringifyObject(reduceResult) is JsString jsStr)
                                    {
                                        if (first == false)
                                        {
                                            writer.WriteComma();
                                        }

                                        writer.WriteString(jsStr.ToString());
                                        first = false;
                                    }
                                }
                            }

                            writer.WriteEndArray();
                        }
                        writer.WriteEndObject();
                    }
                }
            }
        }
Ejemplo n.º 5
0
        public async Task Create()
        {
            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
                using (context.OpenReadTransaction())
                {
                    var json = await context.ReadForMemoryAsync(RequestBodyStream(), null);

                    var options = JsonDeserializationServer.SubscriptionCreationParams(json);

                    if (TrafficWatchManager.HasRegisteredClients)
                    {
                        AddStringToHttpContext(json.ToString(), TrafficWatchChangeType.Subscriptions);
                    }

                    var sub = SubscriptionConnection.ParseSubscriptionQuery(options.Query);

                    if (Enum.TryParse(
                            options.ChangeVector,
                            out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
                    {
                        switch (changeVectorSpecialValue)
                        {
                        case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:

                            options.ChangeVector = null;
                            break;

                        case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
                            options.ChangeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context, sub.Collection);
                            break;
                        }
                    }
                    var id             = GetLongQueryString("id", required: false);
                    var disabled       = GetBoolValueQueryString("disabled", required: false);
                    var mentor         = options.MentorNode;
                    var subscriptionId = await Database.SubscriptionStorage.PutSubscription(options, GetRaftRequestIdFromQuery(), id, disabled, mentor : mentor);

                    var name = options.Name ?? subscriptionId.ToString();

                    using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext serverContext))
                        using (serverContext.OpenReadTransaction())
                        {
                            // need to wait on the relevant remote node
                            var node = Database.SubscriptionStorage.GetResponsibleNode(serverContext, name);
                            if (node != null && node != ServerStore.NodeTag)
                            {
                                await WaitForExecutionOnSpecificNode(serverContext, ServerStore.GetClusterTopology(serverContext), node, subscriptionId);
                            }
                        }

                    HttpContext.Response.StatusCode = (int)HttpStatusCode.Created; // Created

                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        context.Write(writer, new DynamicJsonValue
                        {
                            ["Name"] = name
                        });
                    }
                }
        }
Ejemplo n.º 6
0
        public static IEnumerable <AbstractDashboardNotification> FetchDatabasesInfo(ServerStore serverStore, CancellationTokenSource cts)
        {
            var databasesInfo = new DatabasesInfo();
            var indexingSpeed = new IndexingSpeed();
            var trafficWatch  = new TrafficWatch();
            var drivesUsage   = new DrivesUsage();

            using (serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionContext))
                using (transactionContext.OpenReadTransaction())
                {
                    foreach (var databaseTuple in serverStore.Cluster.ItemsStartingWith(transactionContext, Constants.Documents.Prefix, 0, int.MaxValue))
                    {
                        var databaseName = databaseTuple.ItemName.Substring(3);
                        if (cts.IsCancellationRequested)
                        {
                            yield break;
                        }

                        if (serverStore.DatabasesLandlord.DatabasesCache.TryGetValue(databaseName, out var databaseTask) == false)
                        {
                            // database does not exist in this server or disabled
                            continue;
                        }

                        var databaseOnline = IsDatabaseOnline(databaseTask, out var database);
                        if (databaseOnline == false)
                        {
                            var databaseRecord = serverStore.LoadDatabaseRecord(databaseName, out var _);
                            if (databaseRecord == null)
                            {
                                // database doesn't exist
                                continue;
                            }

                            var databaseInfoItem = new DatabaseInfoItem
                            {
                                Database = databaseName,
                                Online   = false
                            };

                            DatabaseInfo databaseInfo = null;
                            if (serverStore.DatabaseInfoCache.TryGet(databaseName,
                                                                     databaseInfoJson => databaseInfo = JsonDeserializationServer.DatabaseInfo(databaseInfoJson)))
                            {
                                Debug.Assert(databaseInfo != null);

                                databaseInfoItem.DocumentsCount      = databaseInfo.DocumentsCount ?? 0;
                                databaseInfoItem.IndexesCount        = databaseInfo.IndexesCount ?? databaseRecord.Indexes.Count;
                                databaseInfoItem.ReplicationFactor   = databaseRecord.Topology?.ReplicationFactor ?? databaseInfo.ReplicationFactor;
                                databaseInfoItem.ErroredIndexesCount = databaseInfo.IndexingErrors ?? 0;
                            }

                            databasesInfo.Items.Add(databaseInfoItem);
                            continue;
                        }

                        var indexingSpeedItem = new IndexingSpeedItem
                        {
                            Database         = database.Name,
                            IndexedPerSecond = database.Metrics.MapIndexes.IndexedPerSec.FiveSecondRate,
                            MappedPerSecond  = database.Metrics.MapReduceIndexes.MappedPerSec.FiveSecondRate,
                            ReducedPerSecond = database.Metrics.MapReduceIndexes.ReducedPerSec.FiveSecondRate
                        };
                        indexingSpeed.Items.Add(indexingSpeedItem);

                        var replicationFactor = GetReplicationFactor(databaseTuple.Value);
                        var documentsStorage  = database.DocumentsStorage;
                        var indexStorage      = database.IndexStore;
                        using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext))
                            using (documentsContext.OpenReadTransaction())
                            {
                                var databaseInfoItem = new DatabaseInfoItem
                                {
                                    Database            = databaseName,
                                    DocumentsCount      = documentsStorage.GetNumberOfDocuments(documentsContext),
                                    IndexesCount        = database.IndexStore.Count,
                                    AlertsCount         = database.NotificationCenter.GetAlertCount(),
                                    ReplicationFactor   = replicationFactor,
                                    ErroredIndexesCount = indexStorage.GetIndexes().Count(index => index.GetErrorCount() > 0),
                                    Online = true
                                };
                                databasesInfo.Items.Add(databaseInfoItem);
                            }

                        var trafficWatchItem = new TrafficWatchItem
                        {
                            Database            = databaseName,
                            RequestsPerSecond   = (int)database.Metrics.Requests.RequestsPerSec.FiveSecondRate,
                            WritesPerSecond     = (int)database.Metrics.Docs.PutsPerSec.FiveSecondRate,
                            WriteBytesPerSecond = database.Metrics.Docs.BytesPutsPerSec.FiveSecondRate
                        };
                        trafficWatch.Items.Add(trafficWatchItem);

                        foreach (var mountPointUsage in database.GetMountPointsUsage())
                        {
                            if (cts.IsCancellationRequested)
                            {
                                yield break;
                            }

                            var mountPoint = mountPointUsage.DiskSpaceResult.DriveName;
                            var usage      = drivesUsage.Items.FirstOrDefault(x => x.MountPoint == mountPoint);
                            if (usage == null)
                            {
                                usage = new MountPointUsage
                                {
                                    MountPoint    = mountPoint,
                                    VolumeLabel   = mountPointUsage.DiskSpaceResult.VolumeLabel,
                                    FreeSpace     = mountPointUsage.DiskSpaceResult.TotalFreeSpace.GetValue(SizeUnit.Bytes),
                                    TotalCapacity = mountPointUsage.DiskSpaceResult.TotalSize.GetValue(SizeUnit.Bytes)
                                };
                                drivesUsage.Items.Add(usage);
                            }

                            var existingDatabaseUsage = usage.Items.FirstOrDefault(x => x.Database == databaseName);
                            if (existingDatabaseUsage == null)
                            {
                                existingDatabaseUsage = new DatabaseDiskUsage
                                {
                                    Database = databaseName
                                };
                                usage.Items.Add(existingDatabaseUsage);
                            }

                            existingDatabaseUsage.Size += mountPointUsage.UsedSpace;
                        }
                    }
                }

            yield return(databasesInfo);

            yield return(indexingSpeed);

            yield return(trafficWatch);

            yield return(drivesUsage);
        }
        internal ReplicationMessageReply HandleServerResponse(BlittableJsonReaderObject replicationBatchReplyMessage, bool allowNotify)
        {
            replicationBatchReplyMessage.BlittableValidation();
            var replicationBatchReply = JsonDeserializationServer.ReplicationMessageReply(replicationBatchReplyMessage);

            if (replicationBatchReply.MessageType == "Processing")
            {
                return(null);
            }

            if (allowNotify == false && replicationBatchReply.MessageType == "Notify")
            {
                return(null);
            }

            DestinationDbId = replicationBatchReply.DatabaseId;

            switch (replicationBatchReply.Type)
            {
            case ReplicationMessageReply.ReplyType.Ok:
                UpdateDestinationChangeVector(replicationBatchReply);
                OnSuccessfulTwoWaysCommunication();
                break;

            case ReplicationMessageReply.ReplyType.MissingAttachments:
                break;

            default:
                var msg = $"Received error from remote replication destination. Error received: {replicationBatchReply.Exception}";
                if (_log.IsInfoEnabled)
                {
                    _log.Info(msg);
                }
                break;
            }

            if (_log.IsInfoEnabled)
            {
                switch (replicationBatchReply.Type)
                {
                case ReplicationMessageReply.ReplyType.Ok:
                    _log.Info(
                        $"Received reply for replication batch from {Destination.FromString()}. New destination change vector is {LastAcceptedChangeVector}");
                    break;

                case ReplicationMessageReply.ReplyType.Error:
                    _log.Info(
                        $"Received reply for replication batch from {Destination.FromString()}. There has been a failure, error string received : {replicationBatchReply.Exception}");
                    throw new InvalidOperationException(
                              $"Received failure reply for replication batch. Error string received = {replicationBatchReply.Exception}");

                case ReplicationMessageReply.ReplyType.MissingAttachments:
                    _log.Info($"Received reply for replication batch from {Destination.FromString()}. Destination is reporting missing attachments.");
                    break;

                default:
                    throw new ArgumentOutOfRangeException(nameof(replicationBatchReply),
                                                          "Received reply for replication batch with unrecognized type... got " +
                                                          replicationBatchReply.Type);
                }
            }
            return(replicationBatchReply);
        }
            private async Task ListenToMaintenanceWorker()
            {
                bool needToWait               = false;
                var  onErrorDelayTime         = _parent.Config.OnErrorDelayTime.AsTimeSpan;
                var  receiveFromWorkerTimeout = _parent.Config.ReceiveFromWorkerTimeout.AsTimeSpan;

                TcpConnectionInfo tcpConnection = null;

                try
                {
                    tcpConnection = await ReplicationUtils.GetTcpInfoAsync(Url, null, "Supervisor",
                                                                           _parent._server.Server.Certificate?.Certificate);
                }
                catch (Exception e)
                {
                    if (_log.IsInfoEnabled)
                    {
                        _log.Info($"ClusterMaintenanceSupervisor() => Failed to add to cluster node key = {ClusterTag}", e);
                    }
                }
                while (_token.IsCancellationRequested == false)
                {
                    var internalTaskCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(_token);
                    try
                    {
                        if (needToWait)
                        {
                            needToWait = false; // avoid tight loop if there was timeout / error
                            await TimeoutManager.WaitFor(onErrorDelayTime, _token);

                            tcpConnection = await ReplicationUtils.GetTcpInfoAsync(Url, null, "Supervisor",
                                                                                   _parent._server.Server.Certificate.Certificate);
                        }

                        if (tcpConnection == null)
                        {
                            needToWait = true;
                            continue;
                        }
                        var(tcpClient, connection) = await ConnectToClientNodeAsync(tcpConnection, _parent._server.Engine.TcpConnectionTimeout);

                        using (tcpClient)
                            using (_cts.Token.Register(tcpClient.Dispose))
                                using (connection)
                                {
                                    while (_token.IsCancellationRequested == false)
                                    {
                                        using (_contextPool.AllocateOperationContext(out JsonOperationContext context))
                                        {
                                            var readResponseTask = context.ReadForMemoryAsync(connection, _readStatusUpdateDebugString, internalTaskCancellationToken.Token);
                                            var timeout          = TimeoutManager.WaitFor(receiveFromWorkerTimeout, _token);

                                            if (readResponseTask.IsCompleted == false &&
                                                await Task.WhenAny(readResponseTask.AsTask(), timeout) == timeout)
                                            {
                                                if (_log.IsInfoEnabled)
                                                {
                                                    _log.Info($"Timeout occurred while collecting info from {ClusterTag}");
                                                }
                                                ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
                                                                                             ClusterNodeStatusReport.ReportStatus.Timeout,
                                                                                             null,
                                                                                             DateTime.UtcNow,
                                                                                             _lastSuccessfulReceivedReport);
                                                needToWait = true;
                                                internalTaskCancellationToken.Cancel();
                                                try
                                                {
                                                    await readResponseTask;
                                                }
                                                catch
                                                {
                                                    // expecting and ignoring this error, we MUST wait
                                                    // until the task is done before releasing the context that
                                                    // it is using
                                                }
                                                break;
                                            }

                                            using (var statusUpdateJson = await readResponseTask)
                                            {
                                                var report = new Dictionary <string, DatabaseStatusReport>();
                                                foreach (var property in statusUpdateJson.GetPropertyNames())
                                                {
                                                    var value = (BlittableJsonReaderObject)statusUpdateJson[property];
                                                    report.Add(property, JsonDeserializationServer.DatabaseStatusReport(value));
                                                }

                                                ReceivedReport = new ClusterNodeStatusReport(
                                                    report,
                                                    ClusterNodeStatusReport.ReportStatus.Ok,
                                                    null,
                                                    DateTime.UtcNow,
                                                    _lastSuccessfulReceivedReport);
                                                _lastSuccessfulReceivedReport = ReceivedReport;
                                            }
                                        }
                                    }
                                }
                    }
                    catch (Exception e)
                    {
                        if (_log.IsInfoEnabled)
                        {
                            _log.Info($"Exception was thrown while collecting info from {ClusterTag}", e);
                        }
                        ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
                                                                     ClusterNodeStatusReport.ReportStatus.Error,
                                                                     e,
                                                                     DateTime.UtcNow,
                                                                     _lastSuccessfulReceivedReport);
                        needToWait = true;
                    }
                    finally
                    {
                        internalTaskCancellationToken.Dispose();
                    }
                }
            }
Ejemplo n.º 9
0
            public override int Execute(DocumentsOperationContext context)
            {
                if (Database.ServerStore.Configuration.Core.FeaturesAvailability == FeaturesAvailability.Stable)
                {
                    FeaturesAvailabilityException.Throw("Cluster Transactions");
                }
                var global = context.LastDatabaseChangeVector ??
                             (context.LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context));
                var dbGrpId = Database.DatabaseGroupId;
                var current = ChangeVectorUtils.GetEtagById(global, dbGrpId);

                foreach (var command in _batch)
                {
                    Replies.Add(command.Index, new DynamicJsonArray());
                    Reply = Replies[command.Index];

                    var commands = command.Commands;
                    var count    = command.PreviousCount;
                    var options  = Options[command.Index] = command.Options;

                    if (options.WaitForIndexesTimeout != null)
                    {
                        ModifiedCollections = new HashSet <string>();
                    }

                    if (commands != null)
                    {
                        foreach (BlittableJsonReaderObject blittableCommand in commands)
                        {
                            count++;
                            var changeVector = $"RAFT:{count}-{dbGrpId}";

                            var cmd = JsonDeserializationServer.ClusterTransactionDataCommand(blittableCommand);

                            switch (cmd.Type)
                            {
                            case CommandType.PUT:
                                if (current < count)
                                {
                                    // delete the document to avoid exception if we put new document in a different collection.
                                    // TODO: document this behavior
                                    using (DocumentIdWorker.GetSliceFromId(context, cmd.Id, out Slice lowerId))
                                    {
                                        Database.DocumentsStorage.Delete(context, lowerId, cmd.Id, expectedChangeVector: null,
                                                                         nonPersistentFlags: NonPersistentDocumentFlags.SkipRevisionCreation);
                                    }

                                    var putResult = Database.DocumentsStorage.Put(context, cmd.Id, null, cmd.Document.Clone(context), changeVector: changeVector,
                                                                                  flags: DocumentFlags.FromClusterTransaction);
                                    context.DocumentDatabase.HugeDocuments.AddIfDocIsHuge(cmd.Id, cmd.Document.Size);
                                    AddPutResult(putResult);
                                }
                                else
                                {
                                    try
                                    {
                                        var item = Database.DocumentsStorage.GetDocumentOrTombstone(context, cmd.Id);
                                        if (item.Missing)
                                        {
                                            AddPutResult(new DocumentsStorage.PutOperationResults
                                            {
                                                ChangeVector = changeVector,
                                                Id           = cmd.Id,
                                                LastModified = DateTime.UtcNow,
                                                Collection   = Database.DocumentsStorage.ExtractCollectionName(context, cmd.Document)
                                            });
                                            continue;
                                        }
                                        var collection = GetCollection(context, item);
                                        AddPutResult(new DocumentsStorage.PutOperationResults
                                        {
                                            ChangeVector = changeVector,
                                            Id           = cmd.Id,
                                            Flags        = item.Document?.Flags ?? item.Tombstone.Flags,
                                            LastModified = item.Document?.LastModified ?? item.Tombstone.LastModified,
                                            Collection   = collection
                                        });
                                    }
                                    catch (DocumentConflictException)
                                    {
                                        AddPutResult(new DocumentsStorage.PutOperationResults
                                        {
                                            ChangeVector = changeVector,
                                            Id           = cmd.Id,
                                            Collection   = GetFirstConflictCollection(context, cmd)
                                        });
                                    }
                                }

                                break;

                            case CommandType.DELETE:
                                if (current < count)
                                {
                                    using (DocumentIdWorker.GetSliceFromId(context, cmd.Id, out Slice lowerId))
                                    {
                                        var deleteResult = Database.DocumentsStorage.Delete(context, lowerId, cmd.Id, null, changeVector: changeVector,
                                                                                            documentFlags: DocumentFlags.FromClusterTransaction);
                                        AddDeleteResult(deleteResult, cmd.Id);
                                    }
                                }
                                else
                                {
                                    try
                                    {
                                        var item = Database.DocumentsStorage.GetDocumentOrTombstone(context, cmd.Id);
                                        if (item.Missing)
                                        {
                                            AddDeleteResult(new DocumentsStorage.DeleteOperationResult
                                            {
                                                ChangeVector = changeVector,
                                                Collection   = null
                                            }, cmd.Id);
                                            continue;
                                        }
                                        var collection = GetCollection(context, item);
                                        AddDeleteResult(new DocumentsStorage.DeleteOperationResult
                                        {
                                            ChangeVector = changeVector,
                                            Collection   = collection
                                        }, cmd.Id);
                                    }
                                    catch (DocumentConflictException)
                                    {
                                        AddDeleteResult(new DocumentsStorage.DeleteOperationResult
                                        {
                                            ChangeVector = changeVector,
                                            Collection   = GetFirstConflictCollection(context, cmd)
                                        }, cmd.Id);
                                    }
                                }
                                break;

                            default:
                                throw new NotSupportedException($"{cmd.Type} is not supported in {nameof(ClusterTransactionMergedCommand)}.");
                            }
                        }
                    }

                    if (context.LastDatabaseChangeVector == null)
                    {
                        context.LastDatabaseChangeVector = global;
                    }

                    var result = ChangeVectorUtils.TryUpdateChangeVector("RAFT", dbGrpId, count, context.LastDatabaseChangeVector);
                    if (result.IsValid)
                    {
                        context.LastDatabaseChangeVector = result.ChangeVector;
                    }
                }

                return(Reply.Count);
            }
Ejemplo n.º 10
0
        private RestoreSettings SnapshotRestore(
            string backupPath,
            string dataDirectory,
            Action <IOperationProgress> onProgress,
            RestoreResult restoreResult)
        {
            Debug.Assert(onProgress != null);

            RestoreSettings restoreSettings = null;

            var voronBackupPath    = new VoronPathSetting(backupPath);
            var voronDataDirectory = new VoronPathSetting(dataDirectory);

            using (var zip = ZipFile.Open(voronBackupPath.FullPath, ZipArchiveMode.Read, System.Text.Encoding.UTF8))
            {
                foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length)))
                {
                    var directory = zipEntries.Key;

                    if (string.IsNullOrWhiteSpace(directory))
                    {
                        foreach (var zipEntry in zipEntries)
                        {
                            if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase))
                            {
                                using (var entryStream = zipEntry.Open())
                                    using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                                    {
                                        var json = context.Read(entryStream, "read database settings for restore");
                                        json.BlittableValidation();

                                        restoreSettings = JsonDeserializationServer.RestoreSettings(json);

                                        restoreSettings.DatabaseRecord.DatabaseName = _restoreConfiguration.DatabaseName;
                                        DatabaseHelper.Validate(_restoreConfiguration.DatabaseName, restoreSettings.DatabaseRecord);

                                        if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false)
                                        {
                                            throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!");
                                        }

                                        if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey)
                                        {
                                            throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!");
                                        }
                                    }
                            }
                        }

                        continue;
                    }

                    var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase)
                        ? voronDataDirectory
                        : voronDataDirectory.Combine(directory);

                    BackupMethods.Full.Restore(
                        zipEntries,
                        restoreDirectory,
                        journalDir: null,
                        onProgress: message =>
                    {
                        restoreResult.AddInfo(message);
                        restoreResult.SnapshotRestore.ReadCount++;
                        onProgress.Invoke(restoreResult.Progress);
                    },
                        cancellationToken: _operationCancelToken.Token);
                }
            }

            if (restoreSettings == null)
            {
                throw new InvalidDataException("Cannot restore the snapshot without the settings file!");
            }

            return(restoreSettings);
        }
Ejemplo n.º 11
0
        public async Task SetupUnsecured()
        {
            AssertOnlyInSetupMode();

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                using (var setupInfoJson = context.ReadForMemory(RequestBodyStream(), "setup-unsecured"))
                {
                    // Making sure we don't have leftovers from previous setup
                    try
                    {
                        using (var tx = context.OpenWriteTransaction())
                        {
                            ServerStore.Engine.DeleteTopology(context);
                            tx.Commit();
                        }
                    }
                    catch (Exception)
                    {
                        // ignored
                    }

                    var setupInfo = JsonDeserializationServer.UnsecuredSetupInfo(setupInfoJson);

                    BlittableJsonReaderObject settingsJson;
                    using (var fs = new FileStream(ServerStore.Configuration.ConfigPath, FileMode.Open, FileAccess.Read))
                    {
                        settingsJson = context.ReadForMemory(fs, "settings-json");
                    }

                    settingsJson.Modifications = new DynamicJsonValue(settingsJson)
                    {
                        [RavenConfiguration.GetKey(x => x.Licensing.EulaAccepted)]          = true,
                        [RavenConfiguration.GetKey(x => x.Core.SetupMode)]                  = nameof(SetupMode.Unsecured),
                        [RavenConfiguration.GetKey(x => x.Security.UnsecuredAccessAllowed)] = nameof(UnsecuredAccessAddressRange.PublicNetwork)
                    };

                    if (setupInfo.Port == 0)
                    {
                        setupInfo.Port = 8080;
                    }

                    settingsJson.Modifications[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = string.Join(";", setupInfo.Addresses.Select(ip => IpAddressToUrl(ip, setupInfo.Port)));

                    if (setupInfo.TcpPort == 0)
                    {
                        setupInfo.TcpPort = 38888;
                    }

                    settingsJson.Modifications[RavenConfiguration.GetKey(x => x.Core.TcpServerUrls)] = string.Join(";", setupInfo.Addresses.Select(ip => IpAddressToUrl(ip, setupInfo.TcpPort, "tcp")));

                    if (setupInfo.EnableExperimentalFeatures)
                    {
                        settingsJson.Modifications[RavenConfiguration.GetKey(x => x.Core.FeaturesAvailability)] = FeaturesAvailability.Experimental;
                    }

                    if (!string.IsNullOrEmpty(setupInfo.LocalNodeTag))
                    {
                        ServerStore.EnsureNotPassive(nodeTag: setupInfo.LocalNodeTag);
                    }

                    if (setupInfo.Environment != StudioConfiguration.StudioEnvironment.None)
                    {
                        var res = await ServerStore.PutValueInClusterAsync(
                            new PutServerWideStudioConfigurationCommand(new ServerWideStudioConfiguration {
                            Disabled = false, Environment = setupInfo.Environment
                        },
                                                                        RaftIdGenerator.DontCareId));

                        await ServerStore.Cluster.WaitForIndexNotification(res.Index);
                    }

                    var modifiedJsonObj = context.ReadObject(settingsJson, "modified-settings-json");

                    var indentedJson = SetupManager.IndentJsonString(modifiedJsonObj.ToString());
                    SetupManager.WriteSettingsJsonLocally(ServerStore.Configuration.ConfigPath, indentedJson);
                }

            NoContentStatus();
        }
Ejemplo n.º 12
0
        public async Task MigrateFromAnotherDatabase()
        {
            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
            {
                var blittable = await context.ReadForMemoryAsync(RequestBodyStream(), "migration-configuration");

                var migrationConfiguration = JsonDeserializationServer.MigrationConfiguration(blittable);

                if (string.IsNullOrWhiteSpace(migrationConfiguration.MigratorFullPath))
                {
                    throw new ArgumentException("MigratorFullPath cannot be null or empty");
                }

                if (migrationConfiguration.InputConfiguration == null)
                {
                    throw new ArgumentException("InputConfiguration cannot be null");
                }

                if (migrationConfiguration.InputConfiguration.TryGet("Command", out string command) == false)
                {
                    throw new ArgumentException("Cannot find the Command property in the InputConfiguration");
                }

                var migratorFile = ResolveMigratorPath(migrationConfiguration);
                if (command == "validateMigratorPath")
                {
                    NoContentStatus();
                    return;
                }

                if (string.IsNullOrWhiteSpace(migrationConfiguration.DatabaseTypeName))
                {
                    throw new ArgumentException("DatabaseTypeName cannot be null or empty");
                }

                var processStartInfo = new ProcessStartInfo
                {
                    FileName               = migratorFile.FullName,
                    Arguments              = $"{migrationConfiguration.DatabaseTypeName}",
                    WorkingDirectory       = migratorFile.Directory.FullName,
                    CreateNoWindow         = true,
                    RedirectStandardOutput = true,
                    RedirectStandardError  = true,
                    RedirectStandardInput  = true,
                    UseShellExecute        = false
                };

                Process process = null;
                try
                {
                    process = Process.Start(processStartInfo);
                }
                catch (Exception e)
                {
                    var killed = KillProcess(process);
                    throw new InvalidOperationException($"Unable to execute Migrator. Process killed: {killed}" + Environment.NewLine +
                                                        "Command was: " + Environment.NewLine +
                                                        (processStartInfo.WorkingDirectory ?? Directory.GetCurrentDirectory()) + "> "
                                                        + processStartInfo.FileName + " " + processStartInfo.Arguments, e);
                }

                await process.StandardInput.WriteLineAsync(migrationConfiguration.InputConfiguration.ToString());

                var isExportCommand = command == "export";
                if (isExportCommand == false)
                {
                    var errorReadTask  = ReadOutput(process.StandardError);
                    var outputReadTask = ReadOutput(process.StandardOutput);

                    await Task.WhenAll(new Task[] { errorReadTask, outputReadTask }).ConfigureAwait(false);

                    Debug.Assert(process.HasExited, "Migrator is still running!");

                    if (process.ExitCode == -1)
                    {
                        await ExitWithException(errorReadTask, null).ConfigureAwait(false);
                    }

                    try
                    {
                        var line = await outputReadTask.ConfigureAwait(false);

                        using (var sw = new StreamWriter(ResponseBodyStream()))
                        {
                            await sw.WriteAsync(line);
                        }

                        return;
                    }
                    catch (Exception e)
                    {
                        await ExitWithException(errorReadTask, e).ConfigureAwait(false);
                    }
                }

                var operationId     = GetLongQueryString("operationId", false) ?? Database.Operations.GetNextOperationId();
                var token           = CreateOperationToken();
                var transformScript = migrationConfiguration.TransformScript;

                var t = Database.Operations.AddOperation(Database, $"Migration from: {migrationConfiguration.DatabaseTypeName}",
                                                         Operations.OperationType.DatabaseMigration,
                                                         onProgress =>
                {
                    return(Task.Run(async() =>
                    {
                        var result = new SmugglerResult();

                        try
                        {
                            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext migrateContext))
                            {
                                var options = new DatabaseSmugglerOptionsServerSide
                                {
                                    TransformScript = transformScript
                                };
                                DoImportInternal(migrateContext, process.StandardOutput.BaseStream, options, result, onProgress, token);
                            }
                        }
                        catch (OperationCanceledException)
                        {
                            KillProcess(process);
                            throw;
                        }
                        catch (ObjectDisposedException)
                        {
                            KillProcess(process);
                            throw;
                        }
Ejemplo n.º 13
0
        public void IndexDefinitionSerialization()
        {
            var indexDefinition = new IndexDefinition();

#if FEATURE_TEST_INDEX
            indexDefinition.IsTestIndex = true;
#endif
            indexDefinition.LockMode = IndexLockMode.LockedIgnore;
            indexDefinition.Maps     = new HashSet <string>
            {
                "a",
                "b"
            };
            indexDefinition.Name   = "n1";
            indexDefinition.Reduce = "c";
            indexDefinition.Fields = new Dictionary <string, IndexFieldOptions>
            {
                { "f1", new IndexFieldOptions
                  {
                      Spatial = new SpatialOptions
                      {
                          Type         = SpatialFieldType.Geography,
                          Units        = SpatialUnits.Miles,
                          MinY         = 3,
                          MinX         = 5,
                          MaxY         = 2,
                          MaxX         = 5,
                          Strategy     = SpatialSearchStrategy.QuadPrefixTree,
                          MaxTreeLevel = 2
                      },
                      Indexing    = FieldIndexing.No,
                      Suggestions = true,
                      Storage     = FieldStorage.Yes,
                      Analyzer    = "a1",
                      TermVector  = FieldTermVector.WithPositionsAndOffsets
                  } },
                { "f2", new IndexFieldOptions
                  {
                      Spatial = new SpatialOptions
                      {
                          Type         = SpatialFieldType.Cartesian,
                          Units        = SpatialUnits.Kilometers,
                          MinY         = 5,
                          MinX         = 2,
                          MaxY         = 9,
                          MaxX         = 3,
                          Strategy     = SpatialSearchStrategy.BoundingBox,
                          MaxTreeLevel = 5
                      },
                      Indexing    = FieldIndexing.Exact,
                      Suggestions = false,
                      Storage     = FieldStorage.No,
                      Analyzer    = "a2",
                      TermVector  = FieldTermVector.WithPositions
                  } }
            };

            using (var context = JsonOperationContext.ShortTermSingleUse())
            {
                var builder = indexDefinition.ToJson();
                using (var json = context.ReadObject(builder, nameof(IndexDefinition)))
                {
                    var newIndexDefinition = JsonDeserializationServer.IndexDefinition(json);

                    Assert.True(indexDefinition.Equals(newIndexDefinition));
                }
            }
        }
Ejemplo n.º 14
0
        protected async Task <RestoreSettings> SnapshotRestore(JsonOperationContext context, string backupPath,
                                                               Action <IOperationProgress> onProgress, RestoreResult restoreResult)
        {
            Debug.Assert(onProgress != null);

            RestoreSettings restoreSettings = null;

            var fullBackupPath = GetBackupPath(backupPath);

            using (var zip = await GetZipArchiveForSnapshot(fullBackupPath))
            {
                foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length)))
                {
                    var directory = zipEntries.Key;

                    if (string.IsNullOrWhiteSpace(directory))
                    {
                        foreach (var zipEntry in zipEntries)
                        {
                            if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase))
                            {
                                using (var entryStream = zipEntry.Open())
                                {
                                    var stream = RestoreFromConfiguration.BackupEncryptionSettings?.EncryptionMode == EncryptionMode.UseDatabaseKey ?
                                                 new DecryptingXChaCha20Oly1305Stream(entryStream, Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey))
                                        : entryStream;

                                    var json = context.Read(stream, "read database settings for restore");
                                    json.BlittableValidation();

                                    restoreSettings = JsonDeserializationServer.RestoreSettings(json);

                                    restoreSettings.DatabaseRecord.DatabaseName = RestoreFromConfiguration.DatabaseName;
                                    DatabaseHelper.Validate(RestoreFromConfiguration.DatabaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration);

                                    if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false)
                                    {
                                        throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!");
                                    }

                                    if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey)
                                    {
                                        throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!");
                                    }
                                }
                            }
                        }

                        continue;
                    }

                    var voronDataDirectory = new VoronPathSetting(RestoreFromConfiguration.DataDirectory);
                    var restoreDirectory   = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase)
                        ? voronDataDirectory
                        : voronDataDirectory.Combine(directory);

                    BackupMethods.Full.Restore(
                        zipEntries,
                        restoreDirectory,
                        journalDir: null,
                        onProgress: message =>
                    {
                        restoreResult.AddInfo(message);
                        restoreResult.SnapshotRestore.ReadCount++;
                        onProgress.Invoke(restoreResult.Progress);
                    },
                        cancellationToken: _operationCancelToken.Token);
                }
            }

            if (restoreSettings == null)
            {
                throw new InvalidDataException("Cannot restore the snapshot without the settings file!");
            }

            return(restoreSettings);
        }
Ejemplo n.º 15
0
        public async Task AddNode()
        {
            SetupCORSHeaders();

            var nodeUrl       = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url");
            var tag           = GetStringQueryString("tag", false);
            var watcher       = GetBoolValueQueryString("watcher", false);
            var assignedCores = GetIntValueQueryString("assignedCores", false);

            if (assignedCores <= 0)
            {
                throw new ArgumentException("Assigned cores must be greater than 0!");
            }

            nodeUrl = UrlHelper.TryGetLeftPart(nodeUrl);
            var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);

            if (HttpContext.Request.IsHttps != remoteIsHttps)
            {
                throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
            }

            NodeInfo nodeInfo;

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
                using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.Certificate.Certificate))
                {
                    requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;

                    var infoCmd = new GetNodeInfoCommand();
                    try
                    {
                        await requestExecutor.ExecuteAsync(infoCmd, ctx);
                    }
                    catch (AllTopologyNodesDownException e)
                    {
                        throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
                    }

                    nodeInfo = infoCmd.Result;

                    if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
                    {
                        throw new TopologyMismatchException("You can't add new node to an already existing cluster");
                    }
                }

            if (assignedCores != null && assignedCores > nodeInfo.NumberOfCores)
            {
                throw new ArgumentException("Cannot add node because the assigned cores is larger " +
                                            $"than the available cores on that machine: {nodeInfo.NumberOfCores}");
            }

            ServerStore.EnsureNotPassive();

            if (assignedCores == null)
            {
                assignedCores = ServerStore.LicenseManager.GetCoresToAssign(nodeInfo.NumberOfCores);
            }

            Debug.Assert(assignedCores <= nodeInfo.NumberOfCores);

            ServerStore.LicenseManager.AssertCanAddNode(nodeUrl, assignedCores.Value);

            if (ServerStore.IsLeader())
            {
                using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
                {
                    string          topologyId;
                    ClusterTopology clusterTopology;
                    using (ctx.OpenReadTransaction())
                    {
                        clusterTopology = ServerStore.GetClusterTopology(ctx);
                        topologyId      = clusterTopology.TopologyId;
                    }

                    var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
                    if (possibleNode.HasUrl)
                    {
                        throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
                    }

                    if (nodeInfo.ServerId == ServerStore.GetServerId())
                    {
                        throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because it's a synonym of the current node URL:{ServerStore.GetNodeHttpServerUrl()}");
                    }

                    if (nodeInfo.TopologyId != null)
                    {
                        if (topologyId != nodeInfo.TopologyId)
                        {
                            throw new TopologyMismatchException(
                                      $"Adding a new node to cluster failed. The new node is already in another cluster. " +
                                      $"Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}");
                        }

                        if (nodeInfo.CurrentState != RachisState.Passive)
                        {
                            throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster " +
                                                                $"because it's already in the cluster under tag :{nodeInfo.NodeTag} " +
                                                                $"and URL: {clusterTopology.GetUrlFromTag(nodeInfo.NodeTag)}");
                        }
                    }

                    var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? tag : nodeInfo.NodeTag;
                    CertificateDefinition oldServerCert = null;
                    X509Certificate2      certificate   = null;

                    if (remoteIsHttps)
                    {
                        if (nodeInfo.Certificate == null)
                        {
                            throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS");
                        }

                        certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet);

                        var now = DateTime.UtcNow;
                        if (certificate.NotBefore.ToUniversalTime() > now)
                        {
                            // Because of time zone and time drift issues, we can't assume that the certificate generation will be
                            // proper. Because of that, we allow tolerance of the NotBefore to be a bit earlier / later than the
                            // current time. Clients may still fail to work with our certificate because of timing issues,
                            // but the admin needs to setup time sync properly and there isn't much we can do at that point
                            if ((certificate.NotBefore.ToUniversalTime() - now).TotalDays > 1)
                            {
                                throw new InvalidOperationException(
                                          $"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
                            }
                        }

                        if (certificate.NotAfter.ToUniversalTime() < now)
                        {
                            throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
                        }

                        var expected = GetStringQueryString("expectedThumbprint", required: false);
                        if (expected != null)
                        {
                            if (certificate.Thumbprint != expected)
                            {
                                throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
                            }
                        }

                        using (ctx.OpenReadTransaction())
                        {
                            var key      = Constants.Certificates.Prefix + certificate.Thumbprint;
                            var readCert = ServerStore.Cluster.Read(ctx, key);
                            if (readCert != null)
                            {
                                oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert);
                            }
                        }

                        if (oldServerCert == null)
                        {
                            var certificateDefinition = new CertificateDefinition
                            {
                                Certificate       = nodeInfo.Certificate,
                                Thumbprint        = certificate.Thumbprint,
                                NotAfter          = certificate.NotAfter,
                                Name              = "Server Certificate for " + nodeUrl,
                                SecurityClearance = SecurityClearance.ClusterNode
                            };

                            var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition));

                            await ServerStore.Cluster.WaitForIndexNotification(res.Index);
                        }
                    }

                    await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : true, asWatcher : watcher ?? false);

                    using (ctx.OpenReadTransaction())
                    {
                        clusterTopology = ServerStore.GetClusterTopology(ctx);
                        possibleNode    = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
                        nodeTag         = possibleNode.HasUrl ? possibleNode.NodeTag : null;

                        if (certificate != null)
                        {
                            var key = Constants.Certificates.Prefix + certificate.Thumbprint;

                            var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.Read(ctx, key));

                            if (modifiedServerCert == null)
                            {
                                throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through.");
                            }

                            if (oldServerCert == null)
                            {
                                modifiedServerCert.Name = "Server certificate for Node " + nodeTag;
                            }
                            else
                            {
                                var value = "Node " + nodeTag;
                                if (modifiedServerCert.Name.Contains(value) == false)
                                {
                                    modifiedServerCert.Name += ", " + value;
                                }
                            }

                            var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(key, modifiedServerCert));

                            await ServerStore.Cluster.WaitForIndexNotification(res.Index);
                        }

                        var nodeDetails = new NodeDetails
                        {
                            NodeTag             = nodeTag,
                            AssignedCores       = assignedCores.Value,
                            NumberOfCores       = nodeInfo.NumberOfCores,
                            InstalledMemoryInGb = nodeInfo.InstalledMemoryInGb,
                            UsableMemoryInGb    = nodeInfo.UsableMemoryInGb,
                            BuildInfo           = nodeInfo.BuildInfo,
                            OsInfo = nodeInfo.OsInfo
                        };
                        await ServerStore.LicenseManager.CalculateLicenseLimits(nodeDetails, forceFetchingNodeInfo : true, waitToUpdate : true);
                    }

                    NoContentStatus();
                    return;
                }
            }
            RedirectToLeader();
        }
Ejemplo n.º 16
0
        private async Task PutInternal(bool validatedAsAdmin)
        {
            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
            {
                var createdIndexes = new List <(string Name, long RaftIndex)>();

                var isReplicatedQueryString = GetStringQueryString("is-replicated", required: false);
                if (isReplicatedQueryString != null && bool.TryParse(isReplicatedQueryString, out var result) && result)
                {
                    await HandleLegacyIndexesAsync();

                    return;
                }

                var input = await context.ReadForMemoryAsync(RequestBodyStream(), "Indexes");

                if (input.TryGet("Indexes", out BlittableJsonReaderArray indexes) == false)
                {
                    ThrowRequiredPropertyNameInRequest("Indexes");
                }
                var raftRequestId = GetRaftRequestIdFromQuery();
                foreach (BlittableJsonReaderObject indexToAdd in indexes)
                {
                    var indexDefinition = JsonDeserializationServer.IndexDefinition(indexToAdd);
                    indexDefinition.Name = indexDefinition.Name?.Trim();

                    var source = IsLocalRequest(HttpContext) ? Environment.MachineName : HttpContext.Connection.RemoteIpAddress.ToString();

                    if (LoggingSource.AuditLog.IsInfoEnabled)
                    {
                        var clientCert = GetCurrentCertificate();

                        var auditLog = LoggingSource.AuditLog.GetLogger(Database.Name, "Audit");
                        auditLog.Info($"Index {indexDefinition.Name} PUT by {clientCert?.Subject} {clientCert?.Thumbprint} with definition: {indexToAdd} from {source} at {DateTime.UtcNow}");
                    }

                    if (indexDefinition.Maps == null || indexDefinition.Maps.Count == 0)
                    {
                        throw new ArgumentException("Index must have a 'Maps' fields");
                    }

                    indexDefinition.Type = indexDefinition.DetectStaticIndexType();

                    // C# index using a non-admin endpoint
                    if (indexDefinition.Type.IsJavaScript() == false && validatedAsAdmin == false)
                    {
                        throw new UnauthorizedAccessException($"Index {indexDefinition.Name} is a C# index but was sent through a non-admin endpoint using REST api, this is not allowed.");
                    }

                    if (indexDefinition.Name.StartsWith(Constants.Documents.Indexing.SideBySideIndexNamePrefix, StringComparison.OrdinalIgnoreCase))
                    {
                        throw new ArgumentException(
                                  $"Index name must not start with '{Constants.Documents.Indexing.SideBySideIndexNamePrefix}'. Provided index name: '{indexDefinition.Name}'");
                    }

                    var index = await Database.IndexStore.CreateIndexInternal(indexDefinition, $"{raftRequestId}/{indexDefinition.Name}", source);

                    createdIndexes.Add((indexDefinition.Name, index));
                }
                if (TrafficWatchManager.HasRegisteredClients)
                {
                    AddStringToHttpContext(indexes.ToString(), TrafficWatchChangeType.Index);
                }

                HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;

                await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
                {
                    writer.WriteStartObject();

                    writer.WriteArray(context, "Results", createdIndexes, (w, c, index) =>
                    {
                        w.WriteStartObject();
                        w.WritePropertyName(nameof(PutIndexResult.Index));
                        w.WriteString(index.Name);
                        w.WriteComma();
                        w.WritePropertyName(nameof(PutIndexResult.RaftCommandIndex));
                        w.WriteInteger(index.RaftIndex);
                        w.WriteEndObject();
                    });

                    writer.WriteEndObject();
                }
            }
        }
Ejemplo n.º 17
0
        public void AcceptIncomingConnection(TcpConnectionOptions tcpConnectionOptions)
        {
            ReplicationLatestEtagRequest getLatestEtagMessage;

            using (tcpConnectionOptions.ContextPool.AllocateOperationContext(out JsonOperationContext context))
                using (var readerObject = context.ParseToMemory(
                           tcpConnectionOptions.Stream,
                           "IncomingReplication/get-last-etag-message read",
                           BlittableJsonDocumentBuilder.UsageMode.None,
                           tcpConnectionOptions.PinnedBuffer))
                {
                    getLatestEtagMessage = JsonDeserializationServer.ReplicationLatestEtagRequest(readerObject);
                    if (_log.IsInfoEnabled)
                    {
                        _log.Info(
                            $"GetLastEtag: {getLatestEtagMessage.SourceTag}({getLatestEtagMessage.SourceMachineName}) / {getLatestEtagMessage.SourceDatabaseName} ({getLatestEtagMessage.SourceDatabaseId}) - {getLatestEtagMessage.SourceUrl}");
                    }
                }

            var connectionInfo = IncomingConnectionInfo.FromGetLatestEtag(getLatestEtagMessage);

            try
            {
                AssertValidConnection(connectionInfo);
            }
            catch (Exception e)
            {
                if (_log.IsInfoEnabled)
                {
                    _log.Info($"Connection from [{connectionInfo}] is rejected.", e);
                }

                var incomingConnectionRejectionInfos = _incomingRejectionStats.GetOrAdd(connectionInfo,
                                                                                        _ => new ConcurrentQueue <IncomingConnectionRejectionInfo>());
                incomingConnectionRejectionInfos.Enqueue(new IncomingConnectionRejectionInfo {
                    Reason = e.ToString()
                });

                try
                {
                    tcpConnectionOptions.Dispose();
                }
                catch
                {
                    // do nothing
                }

                throw;
            }

            try
            {
                using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsOperationContext))
                    using (Database.ConfigurationStorage.ContextPool.AllocateOperationContext(out TransactionOperationContext configurationContext))
                        using (var writer = new BlittableJsonTextWriter(documentsOperationContext, tcpConnectionOptions.Stream))
                            using (documentsOperationContext.OpenReadTransaction())
                                using (configurationContext.OpenReadTransaction())
                                {
                                    var changeVector = DocumentsStorage.GetDatabaseChangeVector(documentsOperationContext);

                                    var lastEtagFromSrc = Database.DocumentsStorage.GetLastReplicateEtagFrom(
                                        documentsOperationContext, getLatestEtagMessage.SourceDatabaseId);
                                    if (_log.IsInfoEnabled)
                                    {
                                        _log.Info($"GetLastEtag response, last etag: {lastEtagFromSrc}");
                                    }
                                    var response = new DynamicJsonValue
                                    {
                                        [nameof(ReplicationMessageReply.Type)]                 = "Ok",
                                        [nameof(ReplicationMessageReply.MessageType)]          = ReplicationMessageType.Heartbeat,
                                        [nameof(ReplicationMessageReply.LastEtagAccepted)]     = lastEtagFromSrc,
                                        [nameof(ReplicationMessageReply.NodeTag)]              = _server.NodeTag,
                                        [nameof(ReplicationMessageReply.DatabaseChangeVector)] = changeVector
                                    };

                                    documentsOperationContext.Write(writer, response);
                                    writer.Flush();
                                }
            }
            catch (Exception)
            {
                try
                {
                    tcpConnectionOptions.Dispose();
                }

                catch (Exception)
                {
                    // do nothing
                }
                throw;
            }

            var newIncoming = new IncomingReplicationHandler(
                tcpConnectionOptions,
                getLatestEtagMessage,
                this);

            newIncoming.Failed            += OnIncomingReceiveFailed;
            newIncoming.DocumentsReceived += OnIncomingReceiveSucceeded;

            if (_log.IsInfoEnabled)
            {
                _log.Info(
                    $"Initialized document replication connection from {connectionInfo.SourceDatabaseName} located at {connectionInfo.SourceUrl}");
            }

            // need to safeguard against two concurrent connection attempts
            var newConnection = _incoming.GetOrAdd(newIncoming.ConnectionInfo.SourceDatabaseId, newIncoming);

            if (newConnection == newIncoming)
            {
                newIncoming.Start();
                IncomingReplicationAdded?.Invoke(newIncoming);
                ForceTryReconnectAll();
            }
            else
            {
                newIncoming.Dispose();
            }
        }
Ejemplo n.º 18
0
        public async Task PostImportAsync()
        {
            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
            {
                if (HttpContext.Request.HasFormContentType == false)
                {
                    HttpContext.Response.StatusCode = (int)HttpStatusCode.BadRequest; // Bad request
                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        context.Write(writer, new DynamicJsonValue
                        {
                            ["Type"]  = "Error",
                            ["Error"] = "This endpoint requires form content type"
                        });
                        return;
                    }
                }

                var operationId = GetLongQueryString("operationId");
                var token       = CreateOperationToken();

                var result = new SmugglerResult();
                await Database.Operations.AddOperation(Database, "Import to: " + Database.Name,
                                                       Operations.OperationType.DatabaseImport,
                                                       onProgress =>
                {
                    return(Task.Run(async() =>
                    {
                        try
                        {
                            var boundary = MultipartRequestHelper.GetBoundary(
                                MediaTypeHeaderValue.Parse(HttpContext.Request.ContentType),
                                MultipartRequestHelper.MultipartBoundaryLengthLimit);
                            var reader = new MultipartReader(boundary, HttpContext.Request.Body);
                            DatabaseSmugglerOptionsServerSide options = null;

                            while (true)
                            {
                                var section = await reader.ReadNextSectionAsync().ConfigureAwait(false);
                                if (section == null)
                                {
                                    break;
                                }

                                if (ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out ContentDispositionHeaderValue contentDisposition) == false)
                                {
                                    continue;
                                }

                                if (MultipartRequestHelper.HasFormDataContentDisposition(contentDisposition))
                                {
                                    var key = HeaderUtilities.RemoveQuotes(contentDisposition.Name);
                                    if (key != "importOptions")
                                    {
                                        continue;
                                    }

                                    BlittableJsonReaderObject blittableJson;
                                    if (section.Headers.ContainsKey("Content-Encoding") && section.Headers["Content-Encoding"] == "gzip")
                                    {
                                        using (var gzipStream = new GZipStream(section.Body, CompressionMode.Decompress))
                                        {
                                            blittableJson = await context.ReadForMemoryAsync(gzipStream, "importOptions");
                                        }
                                    }
                                    else
                                    {
                                        blittableJson = await context.ReadForMemoryAsync(section.Body, "importOptions");
                                    }

                                    options = JsonDeserializationServer.DatabaseSmugglerOptions(blittableJson);
                                    continue;
                                }

                                if (MultipartRequestHelper.HasFileContentDisposition(contentDisposition) == false)
                                {
                                    continue;
                                }

                                var stream = new GZipStream(section.Body, CompressionMode.Decompress);
                                DoImportInternal(context, stream, options, result, onProgress, token);
                            }
                        }
                        catch (Exception e)
                        {
                            result.AddError($"Error occurred during import. Exception: {e.Message}");
                            throw;
                        }

                        return (IOperationResult)result;
                    }));
                }, operationId, token).ConfigureAwait(false);

                WriteImportResult(context, result, ResponseBodyStream());
            }
Ejemplo n.º 19
0
        public async Task GetFolderPathOptions()
        {
            PeriodicBackupConnectionType connectionType;
            var type = GetStringValuesQueryString("type", false).FirstOrDefault();

            if (type == null)
            {
                //Backward compatibility
                connectionType = PeriodicBackupConnectionType.Local;
            }
            else if (Enum.TryParse(type, out connectionType) == false)
            {
                throw new ArgumentException($"Query string '{type}' was not recognized as valid type");
            }

            using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
            {
                var folderPathOptions = new FolderPathOptions();
                ;
                switch (connectionType)
                {
                case PeriodicBackupConnectionType.Local:
                    var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false;
                    var path           = GetStringQueryString("path", required: false);
                    folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration);
                    break;

                case PeriodicBackupConnectionType.S3:
                    var json = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");

                    if (connectionType != PeriodicBackupConnectionType.Local && json == null)
                    {
                        throw new BadRequestException("No JSON was posted.");
                    }

                    var s3Settings = JsonDeserializationServer.S3Settings(json);
                    if (s3Settings == null)
                    {
                        throw new BadRequestException("No S3Settings were found.");
                    }

                    if (string.IsNullOrWhiteSpace(s3Settings.AwsAccessKey) ||
                        string.IsNullOrWhiteSpace(s3Settings.AwsSecretKey) ||
                        string.IsNullOrWhiteSpace(s3Settings.BucketName) ||
                        string.IsNullOrWhiteSpace(s3Settings.AwsRegionName))
                    {
                        break;
                    }

                    using (var client = new RavenAwsS3Client(s3Settings, ServerStore.Configuration.Backup))
                    {
                        // fetching only the first 64 results for the auto complete
                        var folders = await client.ListObjectsAsync(s3Settings.RemoteFolderName, "/", true, take : 64);

                        if (folders != null)
                        {
                            foreach (var folder in folders.FileInfoDetails)
                            {
                                var fullPath = folder.FullPath;
                                if (string.IsNullOrWhiteSpace(fullPath))
                                {
                                    continue;
                                }

                                folderPathOptions.List.Add(fullPath);
                            }
                        }
                    }
                    break;

                case PeriodicBackupConnectionType.Azure:
                    var azureJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");

                    if (connectionType != PeriodicBackupConnectionType.Local && azureJson == null)
                    {
                        throw new BadRequestException("No JSON was posted.");
                    }

                    var azureSettings = JsonDeserializationServer.AzureSettings(azureJson);
                    if (azureSettings == null)
                    {
                        throw new BadRequestException("No AzureSettings were found.");
                    }

                    if (string.IsNullOrWhiteSpace(azureSettings.AccountName) ||
                        string.IsNullOrWhiteSpace(azureSettings.AccountKey) ||
                        string.IsNullOrWhiteSpace(azureSettings.StorageContainer))
                    {
                        break;
                    }

                    using (var client = RavenAzureClient.Create(azureSettings, ServerStore.Configuration.Backup))
                    {
                        var folders = (await client.ListBlobsAsync(azureSettings.RemoteFolderName, "/", true));

                        foreach (var folder in folders.List)
                        {
                            var fullPath = folder.Name;
                            if (string.IsNullOrWhiteSpace(fullPath))
                            {
                                continue;
                            }

                            folderPathOptions.List.Add(fullPath);
                        }
                    }
                    break;

                case PeriodicBackupConnectionType.GoogleCloud:
                    var googleCloudJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");

                    if (connectionType != PeriodicBackupConnectionType.Local && googleCloudJson == null)
                    {
                        throw new BadRequestException("No JSON was posted.");
                    }

                    var googleCloudSettings = JsonDeserializationServer.GoogleCloudSettings(googleCloudJson);
                    if (googleCloudSettings == null)
                    {
                        throw new BadRequestException("No AzureSettings were found.");
                    }

                    if (string.IsNullOrWhiteSpace(googleCloudSettings.BucketName) ||
                        string.IsNullOrWhiteSpace(googleCloudSettings.GoogleCredentialsJson))
                    {
                        break;
                    }

                    using (var client = new RavenGoogleCloudClient(googleCloudSettings, ServerStore.Configuration.Backup))
                    {
                        var folders             = (await client.ListObjectsAsync(googleCloudSettings.RemoteFolderName));
                        var requestedPathLength = googleCloudSettings.RemoteFolderName.Split('/').Length;

                        foreach (var folder in folders)
                        {
                            const char separator = '/';
                            var        splitted  = folder.Name.Split(separator);
                            var        result    = string.Join(separator, splitted.Take(requestedPathLength)) + separator;

                            if (string.IsNullOrWhiteSpace(result))
                            {
                                continue;
                            }

                            folderPathOptions.List.Add(result);
                        }
                    }
                    break;

                case PeriodicBackupConnectionType.FTP:
                case PeriodicBackupConnectionType.Glacier:
                    throw new NotSupportedException();

                default:
                    throw new ArgumentOutOfRangeException();
                }

                await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
                {
                    context.Write(writer, new DynamicJsonValue
                    {
                        [nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List)
                    });
                }
            }
        }
Ejemplo n.º 20
0
        private void UploadToServer(string backupPath, string folderName, string fileName)
        {
            var s3Settings = GetBackupConfigurationFromScript(_configuration.S3Settings, x => JsonDeserializationServer.S3Settings(x),
                                                              settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForS3(settings, _database.Name));
            var glacierSettings = GetBackupConfigurationFromScript(_configuration.GlacierSettings, x => JsonDeserializationServer.GlacierSettings(x),
                                                                   settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForGlacier(settings, _database.Name));
            var azureSettings = GetBackupConfigurationFromScript(_configuration.AzureSettings, x => JsonDeserializationServer.AzureSettings(x),
                                                                 settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForAzure(settings, _database.Name));
            var googleCloudSettings = GetBackupConfigurationFromScript(_configuration.GoogleCloudSettings, x => JsonDeserializationServer.GoogleCloudSettings(x),
                                                                       settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForGoogleCloud(settings, _database.Name));
            var ftpSettings = GetBackupConfigurationFromScript(_configuration.FtpSettings, x => JsonDeserializationServer.FtpSettings(x),
                                                               settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForFtp(settings, _database.Name));

            TaskCancelToken.Token.ThrowIfCancellationRequested();

            var uploaderSettings = new BackupUploaderSettings
            {
                S3Settings          = s3Settings,
                GlacierSettings     = glacierSettings,
                AzureSettings       = azureSettings,
                GoogleCloudSettings = googleCloudSettings,
                FtpSettings         = ftpSettings,

                BackupPath   = backupPath,
                FolderName   = folderName,
                FileName     = fileName,
                DatabaseName = _database.Name,
                TaskName     = _configuration.Name,

                BackupType = _configuration.BackupType
            };

            var backupUploader = new BackupUploader(uploaderSettings, _retentionPolicyParameters, _logger, _backupResult, _onProgress, TaskCancelToken);

            backupUploader.Execute();
        }
Ejemplo n.º 21
0
        public IEnumerable <QueryResult> MoreLikeThis(
            IndexQueryServerSide query,
            IQueryResultRetriever retriever,
            DocumentsOperationContext context,
            CancellationToken token)
        {
            IDisposable releaseServerContext          = null;
            IDisposable closeServerTransaction        = null;
            TransactionOperationContext serverContext = null;
            MoreLikeThisQuery           moreLikeThisQuery;

            try
            {
                if (query.Metadata.HasCmpXchg)
                {
                    releaseServerContext   = context.DocumentDatabase.ServerStore.ContextPool.AllocateOperationContext(out serverContext);
                    closeServerTransaction = serverContext.OpenReadTransaction();
                }

                using (closeServerTransaction)
                    moreLikeThisQuery = QueryBuilder.BuildMoreLikeThisQuery(serverContext, context, query.Metadata, query.Metadata.Query.Where, query.QueryParameters, _analyzer, _queryBuilderFactories);
            }
            finally
            {
                releaseServerContext?.Dispose();
            }

            var options = moreLikeThisQuery.Options != null?JsonDeserializationServer.MoreLikeThisOptions(moreLikeThisQuery.Options) : MoreLikeThisOptions.Default;

            HashSet <string> stopWords = null;

            if (string.IsNullOrWhiteSpace(options.StopWordsDocumentId) == false)
            {
                var stopWordsDoc = context.DocumentDatabase.DocumentsStorage.Get(context, options.StopWordsDocumentId);
                if (stopWordsDoc == null)
                {
                    throw new InvalidOperationException($"Stop words document {options.StopWordsDocumentId} could not be found");
                }

                if (stopWordsDoc.Data.TryGet(nameof(MoreLikeThisStopWords.StopWords), out BlittableJsonReaderArray value) && value != null)
                {
                    stopWords = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
                    for (var i = 0; i < value.Length; i++)
                    {
                        stopWords.Add(value.GetStringByIndex(i));
                    }
                }
            }

            var ir  = _searcher.IndexReader;
            var mlt = new RavenMoreLikeThis(ir, options, _state);

            int?baseDocId = null;

            if (moreLikeThisQuery.BaseDocument == null)
            {
                var td = _searcher.Search(moreLikeThisQuery.BaseDocumentQuery, 1, _state);

                // get the current Lucene docid for the given RavenDB doc ID
                if (td.ScoreDocs.Length == 0)
                {
                    throw new InvalidOperationException("Given filtering expression did not yield any documents that could be used as a base of comparison");
                }

                baseDocId = td.ScoreDocs[0].Doc;
            }

            if (stopWords != null)
            {
                mlt.SetStopWords(stopWords);
            }

            string[] fieldNames;
            if (options.Fields != null && options.Fields.Length > 0)
            {
                fieldNames = options.Fields;
            }
            else
            {
                fieldNames = ir.GetFieldNames(IndexReader.FieldOption.INDEXED)
                             .Where(x => x != Constants.Documents.Indexing.Fields.DocumentIdFieldName && x != Constants.Documents.Indexing.Fields.SourceDocumentIdFieldName && x != Constants.Documents.Indexing.Fields.ReduceKeyHashFieldName)
                             .ToArray();
            }

            mlt.SetFieldNames(fieldNames);
            mlt.Analyzer = _analyzer;

            var pageSize = GetPageSize(_searcher, query.PageSize);

            Query mltQuery;

            if (baseDocId.HasValue)
            {
                mltQuery = mlt.Like(baseDocId.Value);
            }
            else
            {
                using (var blittableJson = ParseJsonStringIntoBlittable(moreLikeThisQuery.BaseDocument, context))
                    mltQuery = mlt.Like(blittableJson);
            }

            var tsdc = TopScoreDocCollector.Create(pageSize, true);

            if (moreLikeThisQuery.FilterQuery != null && moreLikeThisQuery.FilterQuery is MatchAllDocsQuery == false)
            {
                mltQuery = new BooleanQuery
                {
                    { mltQuery, Occur.MUST },
                    { moreLikeThisQuery.FilterQuery, Occur.MUST }
                };
            }

            _searcher.Search(mltQuery, tsdc, _state);
            var hits = tsdc.TopDocs().ScoreDocs;

            var ids = new HashSet <string>(StringComparer.OrdinalIgnoreCase);

            for (int i = 0; i < hits.Length; i++)
            {
                var hit = hits[i];
                token.ThrowIfCancellationRequested();

                if (hit.Doc == baseDocId)
                {
                    continue;
                }

                var doc = _searcher.Doc(hit.Doc, _state);
                var id  = doc.Get(Constants.Documents.Indexing.Fields.DocumentIdFieldName, _state) ?? doc.Get(Constants.Documents.Indexing.Fields.ReduceKeyHashFieldName, _state);
                if (id == null)
                {
                    continue;
                }

                if (ids.Add(id) == false)
                {
                    continue;
                }

                yield return(new QueryResult
                {
                    Result = retriever.Get(doc, hit, _state)
                });
            }
        }
Ejemplo n.º 22
0
        public IOperationResult RunPeriodicBackup(Action <IOperationProgress> onProgress)
        {
            _onProgress = onProgress;
            AddInfo($"Started task: '{_configuration.Name}'");

            var totalSw           = Stopwatch.StartNew();
            var operationCanceled = false;

            var runningBackupStatus = _periodicBackup.RunningBackupStatus = new PeriodicBackupStatus
            {
                TaskId                        = _configuration.TaskId,
                BackupType                    = _configuration.BackupType,
                LastEtag                      = _previousBackupStatus.LastEtag,
                LastRaftIndex                 = _previousBackupStatus.LastRaftIndex,
                LastFullBackup                = _previousBackupStatus.LastFullBackup,
                LastIncrementalBackup         = _previousBackupStatus.LastIncrementalBackup,
                LastFullBackupInternal        = _previousBackupStatus.LastFullBackupInternal,
                LastIncrementalBackupInternal = _previousBackupStatus.LastIncrementalBackupInternal,
                IsFull                        = _isFullBackup,
                LocalBackup                   = _previousBackupStatus.LocalBackup,
                LastOperationId               = _previousBackupStatus.LastOperationId,
                FolderName                    = _previousBackupStatus.FolderName,
                LastDatabaseChangeVector      = _previousBackupStatus.LastDatabaseChangeVector
            };

            try
            {
                if (runningBackupStatus.LocalBackup == null)
                {
                    runningBackupStatus.LocalBackup = new LocalBackup();
                }

                if (runningBackupStatus.LastRaftIndex == null)
                {
                    runningBackupStatus.LastRaftIndex = new LastRaftIndex();
                }

                if (_logger.IsInfoEnabled)
                {
                    var fullBackupText = "a " + (_configuration.BackupType == BackupType.Backup ? "full backup" : "snapshot");
                    _logger.Info($"Creating {(_isFullBackup ? fullBackupText : "an incremental backup")}");
                }

                var currentLastRaftIndex = GetDatabaseEtagForBackup();

                if (_isFullBackup == false)
                {
                    // if we come from old version the _previousBackupStatus won't have LastRaftIndex
                    if (_previousBackupStatus.LastRaftIndex == null)
                    {
                        _previousBackupStatus.LastRaftIndex = new LastRaftIndex();
                    }

                    // no-op if nothing has changed
                    var(currentLastEtag, currentChangeVector) = _database.ReadLastEtagAndChangeVector();

                    // if we come from old version the _previousBackupStatus won't have LastRaftIndex
                    if (_previousBackupStatus.LastRaftIndex == null)
                    {
                        _previousBackupStatus.LastRaftIndex = new LastRaftIndex();
                    }

                    if (currentLastEtag == _previousBackupStatus.LastEtag &&
                        currentChangeVector == _previousBackupStatus.LastDatabaseChangeVector &&
                        currentLastRaftIndex == _previousBackupStatus.LastRaftIndex.LastEtag)
                    {
                        var message = $"Skipping incremental backup because no changes were made from last full backup on {_previousBackupStatus.LastFullBackup}.";

                        if (_logger.IsInfoEnabled)
                        {
                            _logger.Info(message);
                        }

                        UpdateOperationId(runningBackupStatus);
                        runningBackupStatus.LastIncrementalBackup = _periodicBackup.StartTimeInUtc;
                        DatabaseSmuggler.EnsureProcessed(_backupResult);
                        AddInfo(message);

                        return(_backupResult);
                    }
                }

                // update the local configuration before starting the local backup
                var localSettings = GetBackupConfigurationFromScript(_configuration.LocalSettings, x => JsonDeserializationServer.LocalSettings(x),
                                                                     settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForLocal(settings, _database.Name));

                GenerateFolderNameAndBackupDirectory(localSettings, out var nowAsString, out var folderName, out var backupDirectory);
                var startDocumentEtag = _isFullBackup == false ? _previousBackupStatus.LastEtag : null;
                var startRaftIndex    = _isFullBackup == false ? _previousBackupStatus.LastRaftIndex.LastEtag : null;

                var fileName             = GetFileName(_isFullBackup, backupDirectory.FullPath, nowAsString, _configuration.BackupType, out string backupFilePath);
                var internalBackupResult = CreateLocalBackupOrSnapshot(runningBackupStatus, backupFilePath, startDocumentEtag, startRaftIndex);

                runningBackupStatus.LocalBackup.BackupDirectory = _backupToLocalFolder ? backupDirectory.FullPath : null;
                runningBackupStatus.LocalBackup.TempFolderUsed  = _backupToLocalFolder == false;
                runningBackupStatus.IsFull = _isFullBackup;

                try
                {
                    UploadToServer(backupFilePath, folderName, fileName);
                }
                finally
                {
                    runningBackupStatus.UploadToS3          = _backupResult.S3Backup;
                    runningBackupStatus.UploadToAzure       = _backupResult.AzureBackup;
                    runningBackupStatus.UploadToGoogleCloud = _backupResult.GoogleCloudBackup;
                    runningBackupStatus.UploadToGlacier     = _backupResult.GlacierBackup;
                    runningBackupStatus.UploadToFtp         = _backupResult.FtpBackup;

                    // if user did not specify local folder we delete the temporary file
                    if (_backupToLocalFolder == false)
                    {
                        DeleteFile(backupFilePath);
                    }
                }

                UpdateOperationId(runningBackupStatus);
                runningBackupStatus.LastEtag = internalBackupResult.LastDocumentEtag;
                runningBackupStatus.LastDatabaseChangeVector = internalBackupResult.LastDatabaseChangeVector;
                runningBackupStatus.LastRaftIndex.LastEtag   = internalBackupResult.LastRaftIndex;
                runningBackupStatus.FolderName = folderName;

                if (_isFullBackup)
                {
                    runningBackupStatus.LastFullBackup = _periodicBackup.StartTimeInUtc;
                }
                else
                {
                    runningBackupStatus.LastIncrementalBackup = _periodicBackup.StartTimeInUtc;
                }

                totalSw.Stop();

                if (_logger.IsInfoEnabled)
                {
                    var fullBackupText = "a " + (_configuration.BackupType == BackupType.Backup ? " full backup" : " snapshot");
                    _logger.Info($"Successfully created {(_isFullBackup ? fullBackupText : "an incremental backup")} " +
                                 $"in {totalSw.ElapsedMilliseconds:#,#;;0} ms");
                }

                return(_backupResult);
            }
            catch (OperationCanceledException)
            {
                operationCanceled = TaskCancelToken.Token.IsCancellationRequested &&
                                    _databaseShutdownCancellationToken.IsCancellationRequested;
                throw;
            }
            catch (ObjectDisposedException)
            {
                // shutting down, probably
                operationCanceled = true;
                throw;
            }
            catch (Exception e)
            {
                const string message = "Error when performing periodic backup";

                runningBackupStatus.Error = new Error
                {
                    Exception = e.ToString(),
                    At        = DateTime.UtcNow
                };

                if (_logger.IsOperationsEnabled)
                {
                    _logger.Operations(message, e);
                }

                _database.NotificationCenter.Add(AlertRaised.Create(
                                                     _database.Name,
                                                     $"Periodic Backup task: '{_periodicBackup.Configuration.Name}'",
                                                     message,
                                                     AlertType.PeriodicBackup,
                                                     NotificationSeverity.Error,
                                                     details: new ExceptionDetails(e)));

                throw;
            }
            finally
            {
                if (operationCanceled == false)
                {
                    // whether we succeeded or not,
                    // we need to update the last backup time to avoid
                    // starting a new backup right after this one
                    if (_isFullBackup)
                    {
                        runningBackupStatus.LastFullBackupInternal = _periodicBackup.StartTimeInUtc;
                    }
                    else
                    {
                        runningBackupStatus.LastIncrementalBackupInternal = _periodicBackup.StartTimeInUtc;
                    }

                    runningBackupStatus.NodeTag      = _serverStore.NodeTag;
                    runningBackupStatus.DurationInMs = totalSw.ElapsedMilliseconds;
                    runningBackupStatus.Version      = ++_previousBackupStatus.Version;

                    _periodicBackup.BackupStatus = runningBackupStatus;

                    // save the backup status
                    WriteStatus(runningBackupStatus);
                }
            }
        }
        private static void UpdateDatabaseInfo(RawDatabaseRecord databaseRecord, ServerStore serverStore, string databaseName, DrivesUsage existingDrivesUsage,
                                               DatabaseInfoItem databaseInfoItem)
        {
            DatabaseInfo databaseInfo = null;

            if (serverStore.DatabaseInfoCache.TryGet(databaseName,
                                                     databaseInfoJson => databaseInfo = JsonDeserializationServer.DatabaseInfo(databaseInfoJson)) == false)
            {
                return;
            }

            Debug.Assert(databaseInfo != null);
            var databaseTopology = databaseRecord.Topology;
            var indexesCount     = databaseRecord.CountOfIndexes;

            databaseInfoItem.DocumentsCount      = databaseInfo.DocumentsCount ?? 0;
            databaseInfoItem.IndexesCount        = databaseInfo.IndexesCount ?? indexesCount;
            databaseInfoItem.ReplicationFactor   = databaseTopology?.ReplicationFactor ?? databaseInfo.ReplicationFactor;
            databaseInfoItem.ErroredIndexesCount = databaseInfo.IndexingErrors ?? 0;

            if (databaseInfo.MountPointsUsage == null)
            {
                return;
            }

            foreach (var mountPointUsage in databaseInfo.MountPointsUsage)
            {
                var driveName       = mountPointUsage.DiskSpaceResult.DriveName;
                var diskSpaceResult = DiskSpaceChecker.GetDiskSpaceInfo(
                    mountPointUsage.DiskSpaceResult.DriveName,
                    new DriveInfoBase
                {
                    DriveName = driveName
                });

                if (diskSpaceResult != null)
                {
                    // update the latest drive info
                    mountPointUsage.DiskSpaceResult = new Client.ServerWide.Operations.DiskSpaceResult
                    {
                        DriveName             = diskSpaceResult.DriveName,
                        VolumeLabel           = diskSpaceResult.VolumeLabel,
                        TotalFreeSpaceInBytes = diskSpaceResult.TotalFreeSpace.GetValue(SizeUnit.Bytes),
                        TotalSizeInBytes      = diskSpaceResult.TotalSize.GetValue(SizeUnit.Bytes)
                    };
                }

                UpdateMountPoint(serverStore.Configuration.Storage, mountPointUsage, databaseName, existingDrivesUsage);
            }
        }
Ejemplo n.º 24
0
        public async Task GetFolderPathOptions()
        {
            PeriodicBackupConnectionType connectionType;
            var type = GetStringValuesQueryString("type", false).FirstOrDefault();

            if (type == null)
            {
                //Backward compatibility
                connectionType = PeriodicBackupConnectionType.Local;
            }
            else if (Enum.TryParse(type, out connectionType) == false)
            {
                throw new ArgumentException($"Query string '{type}' was not recognized as valid type");
            }

            using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
            {
                FolderPathOptions folderPathOptions;
                switch (connectionType)
                {
                case PeriodicBackupConnectionType.Local:
                    var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false;
                    var path           = GetStringQueryString("path", required: false);
                    folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration);

                    break;

                case PeriodicBackupConnectionType.S3:
                    var json = context.ReadForMemory(RequestBodyStream(), "studio-tasks/format");
                    if (connectionType != PeriodicBackupConnectionType.Local && json == null)
                    {
                        throw new BadRequestException("No JSON was posted.");
                    }

                    var s3Settings = JsonDeserializationServer.S3Settings(json);
                    if (s3Settings == null)
                    {
                        throw new BadRequestException("No S3Settings were found.");
                    }

                    using (var client = new RavenAwsS3Client(s3Settings))
                    {
                        // fetching only the first 64 results for the auto complete
                        var folders = await client.ListObjectsAsync(s3Settings.RemoteFolderName, "/", true, 64);

                        folderPathOptions = new FolderPathOptions();
                        foreach (var folder in folders.FileInfoDetails)
                        {
                            var fullPath = folder.FullPath;
                            if (string.IsNullOrWhiteSpace(fullPath))
                            {
                                continue;
                            }

                            folderPathOptions.List.Add(fullPath);
                        }
                    }
                    break;

                case PeriodicBackupConnectionType.Glacier:
                case PeriodicBackupConnectionType.Azure:
                case PeriodicBackupConnectionType.GoogleCloud:
                case PeriodicBackupConnectionType.FTP:
                    throw new NotSupportedException();

                default:
                    throw new ArgumentOutOfRangeException();
                }


                using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                {
                    context.Write(writer, new DynamicJsonValue
                    {
                        [nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List)
                    });
                }
            }
        }
Ejemplo n.º 25
0
        public async Task Try()
        {
            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
            {
                BlittableJsonReaderObject json = await context.ReadForMemoryAsync(RequestBodyStream(), null);

                SubscriptionTryout tryout = JsonDeserializationServer.SubscriptionTryout(json);

                SubscriptionConnection.ParsedSubscription sub = SubscriptionConnection.ParseSubscriptionQuery(tryout.Query);
                SubscriptionPatchDocument patch = null;
                if (string.IsNullOrEmpty(sub.Script) == false)
                {
                    patch = new SubscriptionPatchDocument(sub.Script, sub.Functions);
                }

                if (sub.Collection == null)
                {
                    throw new ArgumentException("Collection must be specified");
                }

                const int maxPageSize = 1024;
                var       pageSize    = GetIntValueQueryString("pageSize") ?? 1;
                if (pageSize > maxPageSize)
                {
                    throw new ArgumentException($"Cannot gather more than {maxPageSize} results during tryouts, but requested number was {pageSize}.");
                }

                var state = new SubscriptionState
                {
                    ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector,
                    Query = tryout.Query
                };

                var fetcher = new SubscriptionDocumentsFetcher(Database, int.MaxValue, -0x42,
                                                               new IPEndPoint(HttpContext.Connection.RemoteIpAddress, HttpContext.Connection.RemotePort), sub.Collection, sub.Revisions, state, patch);

                var includeCmd = new IncludeDocumentsCommand(Database.DocumentsStorage, context, sub.Includes, isProjection: patch != null);

                if (Enum.TryParse(
                        tryout.ChangeVector,
                        out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
                {
                    switch (changeVectorSpecialValue)
                    {
                    case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:
                    case Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange:
                        state.ChangeVectorForNextBatchStartingPoint = null;
                        break;

                    case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
                        using (context.OpenReadTransaction())
                        {
                            state.ChangeVectorForNextBatchStartingPoint = Database.DocumentsStorage.GetLastDocumentChangeVector(context, sub.Collection);
                        }
                        break;
                    }
                }
                else
                {
                    state.ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector;
                }

                var changeVector = state.ChangeVectorForNextBatchStartingPoint.ToChangeVector();
                var cv           = changeVector.FirstOrDefault(x => x.DbId == Database.DbBase64Id);

                var sp        = Stopwatch.StartNew();
                var timeLimit = TimeSpan.FromSeconds(GetIntValueQueryString("timeLimit", false) ?? 15);
                var startEtag = cv.Etag;

                using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    using (context.OpenReadTransaction())
                    {
                        writer.WriteStartObject();
                        writer.WritePropertyName("Results");
                        writer.WriteStartArray();
                        var numberOfDocs = 0;
                        while (numberOfDocs == 0 && sp.Elapsed < timeLimit)
                        {
                            var first    = true;
                            var lastEtag = startEtag;
                            foreach (var itemDetails in fetcher.GetDataToSend(context, includeCmd, startEtag))
                            {
                                if (itemDetails.Doc.Data != null)
                                {
                                    using (itemDetails.Doc.Data)
                                    {
                                        includeCmd.Gather(itemDetails.Doc);

                                        if (first == false)
                                        {
                                            writer.WriteComma();
                                        }

                                        if (itemDetails.Exception == null)
                                        {
                                            writer.WriteDocument(context, itemDetails.Doc, metadataOnly: false);
                                        }
                                        else
                                        {
                                            var documentWithException = new DocumentWithException
                                            {
                                                Exception    = itemDetails.Exception.ToString(),
                                                ChangeVector = itemDetails.Doc.ChangeVector,
                                                Id           = itemDetails.Doc.Id,
                                                DocumentData = itemDetails.Doc.Data
                                            };
                                            writer.WriteObject(context.ReadObject(documentWithException.ToJson(), ""));
                                        }

                                        first = false;

                                        if (++numberOfDocs >= pageSize)
                                        {
                                            break;
                                        }
                                    }
                                }

                                if (sp.Elapsed >= timeLimit)
                                {
                                    break;
                                }

                                lastEtag = itemDetails.Doc.Etag;
                            }

                            if (startEtag == lastEtag)
                            {
                                break;
                            }

                            startEtag = lastEtag;
                        }

                        writer.WriteEndArray();
                        writer.WriteComma();
                        writer.WritePropertyName("Includes");
                        var includes = new List <Document>();
                        includeCmd.Fill(includes);
                        writer.WriteIncludes(context, includes);
                        writer.WriteEndObject();
                    }
            }
        }
Ejemplo n.º 26
0
        private async Task <PingResult> PingOnce(string url)
        {
            var sp     = Stopwatch.StartNew();
            var result = new PingResult
            {
                Url = url
            };

            using (var cts = new CancellationTokenSource(ServerStore.Engine.TcpConnectionTimeout))
            {
                var info = await ReplicationUtils.GetTcpInfoAsync(url, null, "PingTest", ServerStore.Engine.ClusterCertificate, cts.Token);

                result.TcpInfoTime = sp.ElapsedMilliseconds;

                using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
                {
                    var log = new List <string>();

                    using (await TcpUtils.ConnectSecuredTcpSocket(info, ServerStore.Engine.ClusterCertificate, Server.CipherSuitesPolicy,
                                                                  TcpConnectionHeaderMessage.OperationTypes.Ping, NegotiationCallback, context, ServerStore.Engine.TcpConnectionTimeout, log, cts.Token))
                    {
                    }

                    result.Log = log;

                    async Task <TcpConnectionHeaderMessage.SupportedFeatures> NegotiationCallback(string curUrl, TcpConnectionInfo tcpInfo, Stream stream,
                                                                                                  JsonOperationContext ctx, List <string> logs = null)
                    {
                        try
                        {
                            var msg = new DynamicJsonValue
                            {
                                [nameof(TcpConnectionHeaderMessage.DatabaseName)]     = null,
                                [nameof(TcpConnectionHeaderMessage.Operation)]        = TcpConnectionHeaderMessage.OperationTypes.Ping,
                                [nameof(TcpConnectionHeaderMessage.OperationVersion)] = -1,
                                [nameof(TcpConnectionHeaderMessage.ServerId)]         = tcpInfo.ServerId
                            };

                            await using (var writer = new AsyncBlittableJsonTextWriter(ctx, stream))
                                using (var msgJson = ctx.ReadObject(msg, "message"))
                                {
                                    result.SendTime = sp.ElapsedMilliseconds;
                                    logs?.Add($"message sent to url {curUrl} at {result.SendTime} ms.");
                                    ctx.Write(writer, msgJson);
                                }

                            using (var rawResponse = await ctx.ReadForMemoryAsync(stream, "cluster-ConnectToPeer-header-response"))
                            {
                                TcpConnectionHeaderResponse response = JsonDeserializationServer.TcpConnectionHeaderResponse(rawResponse);
                                result.ReceiveTime = sp.ElapsedMilliseconds;
                                logs?.Add($"response received from url {curUrl} at {result.ReceiveTime} ms.");

                                switch (response.Status)
                                {
                                case TcpConnectionStatus.Ok:
                                    result.Error = null;
                                    logs?.Add($"Successfully negotiated with {url}.");
                                    break;

                                case TcpConnectionStatus.AuthorizationFailed:
                                    result.Error = $"Connection to {url} failed because of authorization failure: {response.Message}";
                                    logs?.Add(result.Error);
                                    throw new AuthorizationException(result.Error);

                                case TcpConnectionStatus.TcpVersionMismatch:
                                    result.Error = $"Connection to {url} failed because of mismatching tcp version: {response.Message}";
                                    logs?.Add(result.Error);
                                    throw new AuthorizationException(result.Error);

                                case TcpConnectionStatus.InvalidNetworkTopology:
                                    result.Error = $"Connection to {url} failed because of {nameof(TcpConnectionStatus.InvalidNetworkTopology)} error: {response.Message}";
                                    logs?.Add(result.Error);
                                    throw new InvalidNetworkTopologyException(result.Error);
                                }
                            }
                        }
                        catch (Exception e)
                        {
                            result.Error = e.ToString();
                            logs?.Add($"Error occurred while attempting to negotiate with the server. {e.Message}");
                            throw;
                        }

                        return(null);
                    }
                }
            }
            return(result);
        }
Ejemplo n.º 27
0
        public async Task UserDomains()
        {
            AssertOnlyInSetupMode();

            using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
            {
                var json        = context.Read(RequestBodyStream(), "license activation");
                var licenseInfo = JsonDeserializationServer.LicenseInfo(json);

                var content = new StringContent(JsonConvert.SerializeObject(licenseInfo), Encoding.UTF8, "application/json");
                try
                {
                    string error          = null;
                    object result         = null;
                    string responseString = null;
                    string errorMessage   = null;

                    try
                    {
                        var response = await ApiHttpClient.Instance.PostAsync("/api/v1/dns-n-cert/user-domains", content).ConfigureAwait(false);

                        HttpContext.Response.StatusCode = (int)response.StatusCode;
                        responseString = await response.Content.ReadAsStringAsync().ConfigureAwait(false);

                        if (response.IsSuccessStatusCode == false)
                        {
                            error        = responseString;
                            errorMessage = GeneralDomainRegistrationError;
                        }
                        else
                        {
                            result = JsonConvert.DeserializeObject <JObject>(responseString);
                        }
                    }
                    catch (Exception e)
                    {
                        result = responseString;
                        HttpContext.Response.StatusCode = (int)HttpStatusCode.InternalServerError;
                        error        = e.ToString();
                        errorMessage = DomainRegistrationServiceUnreachableError;
                    }

                    if (error != null)
                    {
                        JToken errorJToken = null;
                        if (responseString != null)
                        {
                            JsonConvert.DeserializeObject <JObject>(responseString).TryGetValue("Error", out errorJToken);
                        }

                        using (var streamWriter = new StreamWriter(ResponseBodyStream()))
                        {
                            new JsonSerializer().Serialize(streamWriter, new
                            {
                                Message  = errorMessage,
                                Response = result,
                                Error    = errorJToken ?? error
                            });

                            streamWriter.Flush();
                        }

                        return;
                    }

                    var results = JsonConvert.DeserializeObject <UserDomainsResult>(responseString);

                    var fullResult = new UserDomainsAndLicenseInfo
                    {
                        UserDomainsWithIps = new UserDomainsWithIps
                        {
                            Emails      = results.Emails,
                            RootDomains = results.RootDomains,
                            Domains     = new Dictionary <string, List <SubDomainAndIps> >()
                        }
                    };

                    foreach (var domain in results.Domains)
                    {
                        var list = new List <SubDomainAndIps>();
                        foreach (var subDomain in domain.Value)
                        {
                            try
                            {
                                list.Add(new SubDomainAndIps
                                {
                                    SubDomain = subDomain,
                                    // The ip list will be populated on the next call (/setup/populate-ips), when we know which root domain the user selected
                                });
                            }
                            catch (Exception)
                            {
                                continue;
                            }
                        }

                        fullResult.UserDomainsWithIps.Domains.Add(domain.Key, list);
                    }

                    var licenseStatus = await SetupManager
                                        .GetUpdatedLicenseStatus(ServerStore, licenseInfo.License)
                                        .ConfigureAwait(false);

                    fullResult.MaxClusterSize = licenseStatus.MaxClusterSize;
                    fullResult.LicenseType    = licenseStatus.Type;

                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        var blittable = EntityToBlittable.ConvertCommandToBlittable(fullResult, context);
                        context.Write(writer, blittable);
                    }
                }
                catch (LicenseExpiredException)
                {
                    throw;
                }
                catch (Exception e)
                {
                    throw new InvalidOperationException(GeneralDomainRegistrationError, e);
                }
            }
        }
Ejemplo n.º 28
0
        public async Task AddNode()
        {
            var nodeUrl       = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url");
            var tag           = GetStringQueryString("tag", false);
            var watcher       = GetBoolValueQueryString("watcher", false);
            var raftRequestId = GetRaftRequestIdFromQuery();

            var maxUtilizedCores = GetIntValueQueryString("maxUtilizedCores", false);

            if (maxUtilizedCores != null && maxUtilizedCores <= 0)
            {
                throw new ArgumentException("Max utilized cores cores must be greater than 0");
            }

            nodeUrl = nodeUrl.Trim();
            if (Uri.IsWellFormedUriString(nodeUrl, UriKind.Absolute) == false)
            {
                throw new InvalidOperationException($"Given node URL '{nodeUrl}' is not in a correct format.");
            }

            nodeUrl = UrlHelper.TryGetLeftPart(nodeUrl);
            var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);

            if (HttpContext.Request.IsHttps != remoteIsHttps)
            {
                throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
            }

            tag = tag?.Trim();

            NodeInfo nodeInfo;

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
                using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.Certificate.Certificate))
                {
                    requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;

                    // test connection to remote.
                    var result = await ServerStore.TestConnectionToRemote(nodeUrl, database : null);

                    if (result.Success == false)
                    {
                        throw new InvalidOperationException(result.Error);
                    }

                    // test connection from remote to destination
                    result = await ServerStore.TestConnectionFromRemote(requestExecutor, ctx, nodeUrl);

                    if (result.Success == false)
                    {
                        throw new InvalidOperationException(result.Error);
                    }

                    var infoCmd = new GetNodeInfoCommand();
                    try
                    {
                        await requestExecutor.ExecuteAsync(infoCmd, ctx);
                    }
                    catch (AllTopologyNodesDownException e)
                    {
                        throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
                    }

                    nodeInfo = infoCmd.Result;

                    if (SchemaUpgrader.CurrentVersion.ServerVersion != nodeInfo.ServerSchemaVersion)
                    {
                        var nodesVersion = nodeInfo.ServerSchemaVersion == 0 ? "Pre 4.2 version" : nodeInfo.ServerSchemaVersion.ToString();
                        throw new InvalidOperationException($"Can't add node with mismatched storage schema version.{Environment.NewLine}" +
                                                            $"My version is {SchemaUpgrader.CurrentVersion.ServerVersion}, while node's version is {nodesVersion}");
                    }

                    if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
                    {
                        throw new TopologyMismatchException("You can't add new node to an already existing cluster");
                    }
                }

            if (ServerStore.ValidateFixedPort && nodeInfo.HasFixedPort == false)
            {
                throw new InvalidOperationException($"Failed to add node '{nodeUrl}' to cluster. " +
                                                    $"Node '{nodeUrl}' has port '0' in 'Configuration.Core.ServerUrls' setting. " +
                                                    "Adding a node with non fixed port is forbidden. Define a fixed port for the node to enable cluster creation.");
            }

            ServerStore.EnsureNotPassive();

            ServerStore.LicenseManager.AssertCanAddNode();

            if (ServerStore.IsLeader())
            {
                using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
                {
                    var clusterTopology = ServerStore.GetClusterTopology();

                    var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
                    if (possibleNode.HasUrl)
                    {
                        throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
                    }

                    if (nodeInfo.ServerId == ServerStore.GetServerId())
                    {
                        throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because it's a synonym of the current node URL:{ServerStore.GetNodeHttpServerUrl()}");
                    }

                    if (nodeInfo.TopologyId != null)
                    {
                        AssertCanAddNodeWithTopologyId(clusterTopology, nodeInfo, nodeUrl);
                    }

                    var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? tag : nodeInfo.NodeTag;
                    CertificateDefinition oldServerCert = null;
                    X509Certificate2      certificate   = null;

                    if (remoteIsHttps)
                    {
                        if (nodeInfo.Certificate == null)
                        {
                            throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS");
                        }

                        certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet);

                        var now = DateTime.UtcNow;
                        if (certificate.NotBefore.ToUniversalTime() > now)
                        {
                            // Because of time zone and time drift issues, we can't assume that the certificate generation will be
                            // proper. Because of that, we allow tolerance of the NotBefore to be a bit earlier / later than the
                            // current time. Clients may still fail to work with our certificate because of timing issues,
                            // but the admin needs to setup time sync properly and there isn't much we can do at that point
                            if ((certificate.NotBefore.ToUniversalTime() - now).TotalDays > 1)
                            {
                                throw new InvalidOperationException(
                                          $"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
                            }
                        }

                        if (certificate.NotAfter.ToUniversalTime() < now)
                        {
                            throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
                        }

                        var expected = GetStringQueryString("expectedThumbprint", required: false);
                        if (expected != null)
                        {
                            if (certificate.Thumbprint != expected)
                            {
                                throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
                            }
                        }

                        // if it's the same server certificate as our own, we don't want to add it to the cluster
                        if (certificate.Thumbprint != Server.Certificate.Certificate.Thumbprint)
                        {
                            using (ctx.OpenReadTransaction())
                            {
                                var readCert = ServerStore.Cluster.GetCertificateByThumbprint(ctx, certificate.Thumbprint);
                                if (readCert != null)
                                {
                                    oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert);
                                }
                            }

                            if (oldServerCert == null)
                            {
                                var certificateDefinition = new CertificateDefinition
                                {
                                    Certificate          = nodeInfo.Certificate,
                                    Thumbprint           = certificate.Thumbprint,
                                    PublicKeyPinningHash = certificate.GetPublicKeyPinningHash(),
                                    NotAfter             = certificate.NotAfter,
                                    Name = "Server Certificate for " + nodeUrl,
                                    SecurityClearance = SecurityClearance.ClusterNode
                                };

                                var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(certificate.Thumbprint, certificateDefinition,
                                                                                                             $"{raftRequestId}/put-new-certificate"));

                                await ServerStore.Cluster.WaitForIndexNotification(res.Index);
                            }
                        }
                    }

                    await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : true, asWatcher : watcher ?? false);

                    using (ctx.OpenReadTransaction())
                    {
                        clusterTopology = ServerStore.GetClusterTopology(ctx);
                        possibleNode    = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
                        nodeTag         = possibleNode.HasUrl ? possibleNode.NodeTag : null;

                        if (certificate != null && certificate.Thumbprint != Server.Certificate.Certificate.Thumbprint)
                        {
                            var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.GetCertificateByThumbprint(ctx, certificate.Thumbprint));

                            if (modifiedServerCert == null)
                            {
                                throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through.");
                            }

                            if (oldServerCert == null)
                            {
                                modifiedServerCert.Name = "Server certificate for Node " + nodeTag;
                            }
                            else
                            {
                                var value = "Node " + nodeTag;
                                if (modifiedServerCert.Name.Contains(value) == false)
                                {
                                    modifiedServerCert.Name += ", " + value;
                                }
                            }

                            var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(certificate.Thumbprint, modifiedServerCert, $"{raftRequestId}/put-modified-certificate"));

                            await ServerStore.Cluster.WaitForIndexNotification(res.Index);
                        }

                        var detailsPerNode = new DetailsPerNode
                        {
                            MaxUtilizedCores    = maxUtilizedCores,
                            NumberOfCores       = nodeInfo.NumberOfCores,
                            InstalledMemoryInGb = nodeInfo.InstalledMemoryInGb,
                            UsableMemoryInGb    = nodeInfo.UsableMemoryInGb,
                            BuildInfo           = nodeInfo.BuildInfo,
                            OsInfo = nodeInfo.OsInfo
                        };

                        var maxCores = ServerStore.LicenseManager.LicenseStatus.MaxCores;

                        try
                        {
                            await ServerStore.PutNodeLicenseLimitsAsync(nodeTag, detailsPerNode, maxCores, $"{raftRequestId}/put-license-limits");
                        }
                        catch
                        {
                            // we'll retry this again later
                        }
                    }

                    NoContentStatus();
                    return;
                }
            }
            RedirectToLeader();
        }
Ejemplo n.º 29
0
        public Task SetupUnsecured()
        {
            AssertOnlyInSetupMode();

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                using (var setupInfoJson = context.ReadForMemory(RequestBodyStream(), "setup-unsecured"))
                {
                    // Making sure we don't have leftovers from previous setup
                    try
                    {
                        using (var tx = context.OpenWriteTransaction())
                        {
                            ServerStore.Engine.DeleteTopology(context);
                            tx.Commit();
                        }
                    }
                    catch (Exception)
                    {
                        // ignored
                    }

                    var setupInfo = JsonDeserializationServer.UnsecuredSetupInfo(setupInfoJson);

                    BlittableJsonReaderObject settingsJson;
                    using (var fs = new FileStream(ServerStore.Configuration.ConfigPath, FileMode.Open, FileAccess.Read))
                    {
                        settingsJson = context.ReadForMemory(fs, "settings-json");
                    }

                    settingsJson.Modifications = new DynamicJsonValue(settingsJson)
                    {
                        [RavenConfiguration.GetKey(x => x.Licensing.EulaAccepted)]          = true,
                        [RavenConfiguration.GetKey(x => x.Core.SetupMode)]                  = nameof(SetupMode.Unsecured),
                        [RavenConfiguration.GetKey(x => x.Security.UnsecuredAccessAllowed)] = nameof(UnsecuredAccessAddressRange.PublicNetwork)
                    };

                    if (setupInfo.Port == 0)
                    {
                        setupInfo.Port = 8080;
                    }

                    settingsJson.Modifications[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = string.Join(";", setupInfo.Addresses.Select(ip => IpAddressToUrl(ip, setupInfo.Port)));

                    if (setupInfo.TcpPort == 0)
                    {
                        setupInfo.TcpPort = 38888;
                    }

                    settingsJson.Modifications[RavenConfiguration.GetKey(x => x.Core.TcpServerUrls)] = string.Join(";", setupInfo.Addresses.Select(ip => IpAddressToUrl(ip, setupInfo.TcpPort, "tcp")));

                    if (setupInfo.EnableExperimentalFeatures)
                    {
                        settingsJson.Modifications[RavenConfiguration.GetKey(x => x.Core.FeaturesAvailability)] = FeaturesAvailability.Experimental;
                    }

                    var modifiedJsonObj = context.ReadObject(settingsJson, "modified-settings-json");

                    var indentedJson = SetupManager.IndentJsonString(modifiedJsonObj.ToString());
                    SetupManager.WriteSettingsJsonLocally(ServerStore.Configuration.ConfigPath, indentedJson);
                }

            return(NoContent());
        }
Ejemplo n.º 30
0
        public async Task Try()
        {
            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
            {
                BlittableJsonReaderObject json = await context.ReadForMemoryAsync(RequestBodyStream(), null);

                SubscriptionTryout tryout = JsonDeserializationServer.SubscriptionTryout(json);

                SubscriptionConnection.ParsedSubscription sub = SubscriptionConnection.ParseSubscriptionQuery(tryout.Query);
                SubscriptionPatchDocument patch = null;
                if (string.IsNullOrEmpty(sub.Script) == false)
                {
                    patch = new SubscriptionPatchDocument(sub.Script, sub.Functions);
                }

                if (sub.Collection == null)
                {
                    throw new ArgumentException("Collection must be specified");
                }

                var pageSize = GetIntValueQueryString("pageSize") ?? 1;

                var state = new SubscriptionState
                {
                    ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector,
                    Query = tryout.Query
                };

                var fetcher = new SubscriptionDocumentsFetcher(Database, pageSize, -0x42,
                                                               new IPEndPoint(HttpContext.Connection.RemoteIpAddress, HttpContext.Connection.RemotePort), sub.Collection, sub.Revisions, state, patch);

                var includeCmd = new IncludeDocumentsCommand(Database.DocumentsStorage, context, sub.Includes);

                if (Enum.TryParse(
                        tryout.ChangeVector,
                        out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
                {
                    switch (changeVectorSpecialValue)
                    {
                    case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:
                    case Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange:
                        state.ChangeVectorForNextBatchStartingPoint = null;
                        break;

                    case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
                        state.ChangeVectorForNextBatchStartingPoint = Database.DocumentsStorage.GetLastDocumentChangeVector(context, sub.Collection);
                        break;
                    }
                }

                using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                {
                    writer.WriteStartObject();
                    writer.WritePropertyName("Results");
                    writer.WriteStartArray();

                    using (context.OpenReadTransaction())
                    {
                        var first = true;

                        foreach (var itemDetails in fetcher.GetDataToSend(context, includeCmd, 0))
                        {
                            if (itemDetails.Doc.Data == null)
                            {
                                continue;
                            }

                            includeCmd.Gather(itemDetails.Doc);

                            if (first == false)
                            {
                                writer.WriteComma();
                            }

                            if (itemDetails.Exception == null)
                            {
                                writer.WriteDocument(context, itemDetails.Doc, metadataOnly: false);
                            }
                            else
                            {
                                var docWithExcepton = new DocumentWithException
                                {
                                    Exception    = itemDetails.Exception.ToString(),
                                    ChangeVector = itemDetails.Doc.ChangeVector,
                                    Id           = itemDetails.Doc.Id,
                                    DocumentData = itemDetails.Doc.Data
                                };
                                writer.WriteObject(context.ReadObject(docWithExcepton.ToJson(), ""));
                            }

                            first = false;
                        }

                        writer.WriteEndArray();
                        writer.WriteComma();
                        writer.WritePropertyName("Includes");
                        var includes = new List <Document>();
                        includeCmd.Fill(includes);
                        writer.WriteIncludes(context, includes);
                        writer.WriteEndObject();
                    }
                }
            }
        }