public Task GetPeriodicBackup()
{
var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name");
if (TryGetAllowedDbs(name, out var _, requireAdmin: false) == false)
{
return(Task.CompletedTask);
}
var taskId = GetLongQueryString("taskId", required: true);
if (taskId == 0)
{
throw new ArgumentException("Task ID cannot be 0");
}
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
var databaseRecord = ServerStore.Cluster.ReadDatabase(context, name, out _);
var periodicBackup = databaseRecord.PeriodicBackups.FirstOrDefault(x => x.TaskId == taskId);
if (periodicBackup == null)
{
throw new InvalidOperationException($"Periodic backup task ID: {taskId} doesn't exist");
}
var databaseRecordBlittable = EntityToBlittable.ConvertEntityToBlittable(periodicBackup, DocumentConventions.Default, context);
context.Write(writer, databaseRecordBlittable);
writer.Flush();
}
return(Task.CompletedTask);
}
public PutIndexesCommand(DocumentConventions conventions, JsonOperationContext context, IndexDefinition[] indexesToAdd)
{
if (conventions == null)
{
throw new ArgumentNullException(nameof(conventions));
}
if (indexesToAdd == null)
{
throw new ArgumentNullException(nameof(indexesToAdd));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_indexToAdd = new BlittableJsonReaderObject[indexesToAdd.Length];
for (var i = 0; i < indexesToAdd.Length; i++)
{
if (indexesToAdd[i].Name == null)
{
throw new ArgumentNullException(nameof(IndexDefinition.Name));
}
_indexToAdd[i] = EntityToBlittable.ConvertEntityToBlittable(indexesToAdd[i], conventions, context);
}
}
public PatchCommand(DocumentConventions conventions, JsonOperationContext context, string id, string changeVector, PatchRequest patch, PatchRequest patchIfMissing, bool skipPatchIfChangeVectorMismatch, bool returnDebugInformation, bool test)
{
if (conventions == null)
{
throw new ArgumentNullException(nameof(conventions));
}
if (patch == null)
{
throw new ArgumentNullException(nameof(patch));
}
if (string.IsNullOrWhiteSpace(patch.Script))
{
throw new ArgumentNullException(nameof(patch.Script));
}
if (patchIfMissing != null && string.IsNullOrWhiteSpace(patchIfMissing.Script))
{
throw new ArgumentNullException(nameof(patchIfMissing.Script));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_id = id ?? throw new ArgumentNullException(nameof(id));
_changeVector = changeVector;
_patch = EntityToBlittable.ConvertEntityToBlittable(new
{
Patch = patch,
PatchIfMissing = patchIfMissing
}, conventions, context);
_skipPatchIfChangeVectorMismatch = skipPatchIfChangeVectorMismatch;
_returnDebugInformation = returnDebugInformation;
_test = test;
}
public Task PopulateIps()
{
AssertOnlyInSetupMode();
var rootDomain = GetQueryStringValueAndAssertIfSingleAndNotEmpty("rootDomain");
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var userDomainsWithIpsJson = context.ReadForMemory(RequestBodyStream(), "setup-secured"))
{
var userDomainsWithIps = JsonDeserializationServer.UserDomainsWithIps(userDomainsWithIpsJson);
foreach (var domain in userDomainsWithIps.Domains)
{
foreach (var subDomain in domain.Value)
{
try
{
subDomain.Ips = Dns.GetHostAddresses(subDomain.SubDomain + "." + rootDomain).Select(ip => ip.ToString()).ToList();
}
catch (Exception)
{
continue;
}
}
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
var blittable = EntityToBlittable.ConvertEntityToBlittable(userDomainsWithIps, DocumentConventions.Default, context);
context.Write(writer, blittable);
}
}
return(Task.CompletedTask);
}
private async Task SaveLastState(long operationId)
{
var retries = 0;
using (Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
while (retries++ < 15)
{
var operationState = await GetOperationState(DatabaseName, operationId, context);
if (operationState == null)
{
return;
}
if (operationState.TryGet("Status", out OperationStatus operationStatus) == false)
{
return;
}
if (operationStatus == OperationStatus.InProgress)
{
await Task.Delay(1000, CancelToken.Token);
continue;
}
if (operationStatus == OperationStatus.Canceled || operationStatus == OperationStatus.Faulted)
{
throw new InvalidOperationException("Couldn't get last operation state because the " +
$"operation state is {operationStatus.ToString()} " +
"although the operation was completed successfully");
}
if (operationState.TryGet("Result", out BlittableJsonReaderObject smugglerResultBlittable) == false)
{
return;
}
var smugglerResult = JsonDeserializationClient.SmugglerResult(smugglerResultBlittable);
if (smugglerResult == null)
{
return;
}
var importInfo = new ImportInfo
{
LastEtag = smugglerResult.GetLastEtag() + 1,
ServerUrl = ServerUrl,
DatabaseName = DatabaseName
};
var importInfoBlittable = EntityToBlittable.ConvertEntityToBlittable(importInfo, DocumentConventions.Default, context);
await SaveLastOperationState(importInfoBlittable);
return;
}
}
}
protected async Task SaveLastOperationState(LastEtagsInfo lastEtagsInfo)
{
using (Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
var operationStateBlittable = EntityToBlittable.ConvertEntityToBlittable(lastEtagsInfo, DocumentConventions.Default, context);
await SaveLastOperationState(operationStateBlittable);
}
}
public async Task StoreAsync(object entity, string id, IMetadataDictionary metadata = null)
{
VerifyValidId(id);
if (_stream == null)
{
await WaitForId().ConfigureAwait(false);
await EnsureStream().ConfigureAwait(false);
}
var docInfo = new DocumentInfo
{
Collection = _requestExecutor.Conventions.GetCollectionName(entity)
};
JsonOperationContext tempContext;
using (_requestExecutor.ContextPool.AllocateOperationContext(out tempContext))
{
if (metadata != null)
{
docInfo.MetadataInstance = metadata;
docInfo.Metadata = EntityToBlittable.ConvertEntityToBlittable(metadata, _requestExecutor.Conventions, tempContext);
}
using (var doc = EntityToBlittable.ConvertEntityToBlittable(entity, _requestExecutor.Conventions, tempContext, docInfo))
{
if (_first == false)
{
_jsonWriter.WriteComma();
}
_first = false;
var cmd = new DynamicJsonValue
{
[nameof(PutCommandDataWithBlittableJson.Type)] = "PUT",
[nameof(PutCommandDataWithBlittableJson.Id)] = id,
[nameof(PutCommandDataWithBlittableJson.Document)] = doc
};
try
{
tempContext.Write(_jsonWriter, cmd);
}
catch (Exception e)
{
var error = await GetExceptionFromOperation().ConfigureAwait(false);
if (error != null)
{
throw error;
}
await ThrowOnUnavailableStream(id, e).ConfigureAwait(false);
}
}
}
}
private static async Task <long> SetupRevisions(Raven.Server.ServerWide.ServerStore serverStore, string database, RevisionsConfiguration configuration)
{
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configurationJson = EntityToBlittable.ConvertEntityToBlittable(configuration, DocumentConventions.Default, context);
(long etag, _) = await serverStore.ModifyDatabaseRevisions(context, database, configurationJson);
return(etag);
}
}
public async Task <PutResult> PutAsync(string id, string changeVector, object data, Dictionary <string, object> metadata = null, CancellationToken cancellationToken = default(CancellationToken))
{
if (id == null)
{
throw new ArgumentNullException(nameof(id));
}
if (data == null)
{
throw new ArgumentNullException(nameof(data));
}
var documentInfo = new DocumentInfo
{
Id = id,
ChangeVector = changeVector
};
using (var session = _store.OpenSession())
{
var documentJson = data as BlittableJsonReaderObject;
var metadataJson = metadata != null
? EntityToBlittable.ConvertEntityToBlittable(metadata, session.Advanced.DocumentStore.Conventions, session.Advanced.Context)
: null;
if (documentJson == null)
{
if (metadataJson != null)
{
documentInfo.Metadata = metadataJson;
documentInfo.MetadataInstance = new MetadataAsDictionary(metadata);
}
documentJson = session.Advanced.EntityToBlittable.ConvertEntityToBlittable(data, documentInfo);
}
else
{
if (metadataJson != null)
{
documentJson.Modifications = new DynamicJsonValue(documentJson)
{
[Constants.Documents.Metadata.Key] = metadataJson
};
documentJson = session.Advanced.Context.ReadObject(documentJson, id);
}
}
var command = new PutDocumentCommand(id, changeVector, documentJson);
await RequestExecutor.ExecuteAsync(command, Context, token : cancellationToken);
return(command.Result);
}
}
public void FullBackupTo(string backupPath)
{
using (var file = new FileStream(backupPath, FileMode.Create))
using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true))
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
var databaseRecord = _serverStore.Cluster.ReadDatabase(context, Name);
Debug.Assert(databaseRecord != null);
var zipArchiveEntry = package.CreateEntry(RestoreSettings.FileName, CompressionLevel.Optimal);
using (var zipStream = zipArchiveEntry.Open())
using (var writer = new BlittableJsonTextWriter(context, zipStream))
{
//TODO: encrypt this file using the MasterKey
//http://issues.hibernatingrhinos.com/issue/RavenDB-7546
writer.WriteStartObject();
// save the database record
writer.WritePropertyName(nameof(RestoreSettings.DatabaseRecord));
var databaseRecordBlittable = EntityToBlittable.ConvertEntityToBlittable(databaseRecord, DocumentConventions.Default, context);
context.Write(writer, databaseRecordBlittable);
// save the database values (subscriptions, periodic backups statuses, etl states...)
writer.WriteComma();
writer.WritePropertyName(nameof(RestoreSettings.DatabaseValues));
writer.WriteStartObject();
var first = true;
foreach (var keyValue in ClusterStateMachine.ReadValuesStartingWith(context,
Helpers.ClusterStateMachineValuesPrefix(Name)))
{
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WritePropertyName(keyValue.Key.ToString());
context.Write(writer, keyValue.Value);
}
writer.WriteEndObject();
// end of dictionary
writer.WriteEndObject();
}
BackupMethods.Full.ToFile(GetAllStoragesEnvironmentInformation(), package);
file.Flush(true); // make sure that we fully flushed to disk
}
}
public PatchByCollectionCommand CreateRequest(string collectionName, PatchRequest patch, DocumentStore documentStore)
{
var entityToBlittable = new EntityToBlittable(null);
var requestData = entityToBlittable.ConvertEntityToBlittable(patch, documentStore.Conventions, _context);
return(new PatchByCollectionCommand()
{
Script = requestData,
Context = _context,
CollectionName = collectionName
});
}
public void Can_serialize_with_default_enum_value()
{
var user = new Document
{
Name = "john"
};
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var blittable = EntityToBlittable.ConvertEntityToBlittable(user, DocumentConventions.Default, context);
Assert.Equal("{\"Name\":\"john\",\"Type\":0}", blittable.ToString());
}
}
public PutTransformerCommand CreateRequest(DocumentConvention documentConvention, string transformerName, TransformerDefinition transformerDefinition)
{
var entityToBlittable = new EntityToBlittable(null);
var transformerDefinitionAsBlittable = entityToBlittable.ConvertEntityToBlittable(transformerDefinition,
documentConvention, _context);
return(new PutTransformerCommand()
{
TransformerDefinition = transformerDefinitionAsBlittable,
Context = _context,
TransformerName = transformerName
});
}
public override DynamicJsonValue ToJson(JsonOperationContext context)
{
return(new DynamicJsonValue
{
["Type"] = nameof(AddDatabaseCommand),
[nameof(Name)] = Name,
[nameof(Record)] = EntityToBlittable.ConvertEntityToBlittable(Record, DocumentConventions.Default, context),
[nameof(RaftCommandIndex)] = RaftCommandIndex,
[nameof(Encrypted)] = Encrypted,
[nameof(DatabaseValues)] = EntityToBlittable.ConvertEntityToBlittable(DatabaseValues, DocumentConventions.Default, context),
[nameof(IsRestore)] = IsRestore
});
}
public PutIndexCommand CreateRequest(DocumentConvention documentConvention, string indexName, IndexDefinition indexDefinition)
{
var entityToBlittable = new EntityToBlittable(null);
var indexDefinitionAsBlittable = entityToBlittable.ConvertEntityToBlittable(indexDefinition,
documentConvention, _context);
return(new PutIndexCommand()
{
IndexDefinition = indexDefinitionAsBlittable,
Context = _context,
IndexName = indexName
});
}
public CreateDatabaseCommand(DocumentConventions conventions, JsonOperationContext context, DatabaseRecord databaseRecord,
CreateDatabaseOperationWithoutNameValidation createDatabaseOperation)
{
if (conventions == null)
{
throw new ArgumentNullException(nameof(conventions));
}
_context = context ?? throw new ArgumentNullException(nameof(context));
_createDatabaseOperation = createDatabaseOperation;
_databaseName = databaseRecord?.DatabaseName ?? throw new ArgumentNullException(nameof(databaseRecord));
_databaseDocument = EntityToBlittable.ConvertEntityToBlittable(databaseRecord, conventions, context);
}
public async Task <TResult> RawDeleteJsonAsync <TResult>(string url, object payload)
where TResult : BlittableJsonReaderBase
{
using (var session = _store.OpenSession())
{
var payloadJson = EntityToBlittable.ConvertEntityToBlittable(payload, session.Advanced.DocumentStore.Conventions, session.Advanced.Context);
var command = new JsonCommandWithPayload <TResult>(url, HttpMethod.Delete, payloadJson);
await RequestExecutor.ExecuteAsync(command, Context);
return(command.Result);
}
}
private void WriteToServer(string database, Stream serverStream)
{
var streamNetworkBuffer = serverStream;
var writeToStreamBuffer = new byte[32 * 1024];
var header = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new TcpConnectionHeaderMessage
{
DatabaseName = database,
Operation = TcpConnectionHeaderMessage.OperationTypes.BulkInsert
}));
streamNetworkBuffer.Write(header, 0, header.Length);
while (_documents.IsCompleted == false)
{
_cts.Token.ThrowIfCancellationRequested();
Tuple <object, string> doc;
try
{
doc = _documents.Take();
}
catch (InvalidOperationException)
{
break;
}
var needToThrottle = _throttlingEvent.Wait(0) == false;
_jsonOperationContext.ResetAndRenew();
var documentInfo = new DocumentInfo();
var tag = _store.Conventions.GetDynamicTagName(doc.Item1);
var metadata = new DynamicJsonValue();
if (tag != null)
{
metadata[Constants.Headers.RavenEntityName] = tag;
}
metadata[Constants.Metadata.Id] = doc.Item2;
documentInfo.Metadata = _jsonOperationContext.ReadObject(metadata, doc.Item2);
var data = _entityToBlittable.ConvertEntityToBlittable(doc.Item1, _store.Conventions, _jsonOperationContext, documentInfo);
WriteVariableSizeInt(streamNetworkBuffer, data.Size);
WriteToStream(streamNetworkBuffer, data, writeToStreamBuffer);
if (needToThrottle)
{
streamNetworkBuffer.Flush();
_throttlingEvent.Wait(500);
}
}
streamNetworkBuffer.WriteByte(0); //done
streamNetworkBuffer.Flush();
}
public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url)
{
url = $"{node.Url}/admin/database-restore";
var request = new HttpRequestMessage
{
Method = HttpMethod.Post,
Content = new BlittableJsonContent(stream =>
{
var config = EntityToBlittable.ConvertEntityToBlittable(_restoreConfiguration, DocumentConventions.Default, ctx);
ctx.Write(stream, config);
})
};
return(request);
}
public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url)
{
url = $"{node.Url}/databases/{node.Database}/subscriptions";
var request = new HttpRequestMessage
{
Method = HttpMethod.Put,
Content = new BlittableJsonContent(stream =>
{
ctx.Write(stream,
EntityToBlittable.ConvertEntityToBlittable(_options, DocumentConventions.Default, ctx));
})
};
return(request);
}
public PatchByIndexCommand CreateRequest(string indexName, IndexQuery queryToUpdate, QueryOperationOptions options,
PatchRequest patch, DocumentStore documentStore)
{
var entityToBlittable = new EntityToBlittable(null);
var requestData = entityToBlittable.ConvertEntityToBlittable(patch, documentStore.Conventions, _context);
return(new PatchByIndexCommand()
{
Script = requestData,
IndexName = indexName,
QueryToUpdate = queryToUpdate,
Options = options,
Context = _context
});
}
public async Task ExecuteJsonAsync(string url, HttpMethod method, object payload)
{
using (var session = _store.OpenSession())
{
BlittableJsonReaderObject payloadJson = null;
if (payload != null)
{
payloadJson = EntityToBlittable.ConvertEntityToBlittable(payload, session.Advanced.DocumentStore.Conventions, session.Advanced.Context);
}
var command = new JsonCommandWithPayload <BlittableJsonReaderObject>(url, method, payloadJson);
await RequestExecutor.ExecuteAsync(command, Context);
}
}
protected void SetLicenseLimitResponse(LicenseLimit licenseLimit)
{
if (licenseLimit == null)
{
throw new ArgumentNullException(nameof(licenseLimit));
}
HttpContext.Response.StatusCode = (int)HttpStatusCode.PaymentRequired;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
var blittable = EntityToBlittable.ConvertEntityToBlittable(licenseLimit, DocumentConventions.Default, context);
context.Write(writer, blittable);
writer.Flush();
}
}
public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url)
{
url = $"{node.Url}/databases/{node.Database}/admin/etl?id={_taskId}";
var request = new HttpRequestMessage
{
Method = HttpMethod.Put,
Content = new BlittableJsonContent(stream =>
{
var config = EntityToBlittable.ConvertEntityToBlittable(_configuration, _conventions, ctx);
ctx.Write(stream, config);
})
};
return(request);
}
public CompactDatabaseCommand(DocumentConventions conventions, JsonOperationContext context, CompactSettings compactSettings)
{
if (conventions == null)
{
throw new ArgumentNullException(nameof(conventions));
}
if (compactSettings == null)
{
throw new ArgumentNullException(nameof(compactSettings));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_compactSettings = EntityToBlittable.ConvertEntityToBlittable(compactSettings, conventions, context);
}
public SetIndexPriorityCommand(DocumentConventions conventions, JsonOperationContext context, Parameters parameters)
{
if (conventions == null)
{
throw new ArgumentNullException(nameof(conventions));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (parameters == null)
{
throw new ArgumentNullException(nameof(parameters));
}
_parameters = EntityToBlittable.ConvertEntityToBlittable(parameters, conventions, context);
}
public DeleteRevisionsCommand(DocumentConventions conventions, JsonOperationContext context, AdminRevisionsHandler.Parameters parameters)
{
if (conventions == null)
{
throw new ArgumentNullException(nameof(conventions));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (parameters == null)
{
throw new ArgumentNullException(nameof(parameters));
}
_parameters = EntityToBlittable.ConvertEntityToBlittable(parameters, conventions, context);
}
public PutServerWideClientConfigurationCommand(DocumentConventions conventions, JsonOperationContext context, ClientConfiguration configuration)
{
if (conventions == null)
{
throw new ArgumentNullException(nameof(conventions));
}
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_configuration = EntityToBlittable.ConvertEntityToBlittable(configuration, conventions, context);
}
public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url)
{
url = $"{node.Url}/admin/databases?name={_databaseName}";
url += "&replicationFactor=" + _replicationFactor;
var databaseDocument = EntityToBlittable.ConvertEntityToBlittable(_databaseRecord, _conventions, ctx);
var request = new HttpRequestMessage
{
Method = HttpMethod.Put,
Content = new BlittableJsonContent(stream =>
{
ctx.Write(stream, databaseDocument);
})
};
return(request);
}
public ToggleDatabaseStateCommand(DocumentConventions conventions, JsonOperationContext context, Parameters parameters, bool disable)
{
if (conventions == null)
{
throw new ArgumentNullException(nameof(conventions));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (parameters == null)
{
throw new ArgumentNullException(nameof(parameters));
}
_disable = disable;
_parameters = EntityToBlittable.ConvertEntityToBlittable(parameters, conventions, context);
}
