private async Task MigrateDatabase(long operationId, ImportInfo importInfo)
{
var startDocumentEtag = importInfo?.LastEtag ?? 0;
var url = $"{ServerUrl}/databases/{DatabaseName}/smuggler/export?operationId={operationId}&startEtag={startDocumentEtag}";
var json = JsonConvert.SerializeObject(new DatabaseSmugglerOptionsServerSide());
var content = new StringContent(json, Encoding.UTF8, "application/json");
var request = new HttpRequestMessage(HttpMethod.Post, url)
{
Content = content
};
var response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, CancelToken.Token);
if (response.IsSuccessStatusCode == false)
{
var responseString = await response.Content.ReadAsStringAsync();
throw new InvalidOperationException($"Failed to export database from server: {ServerUrl}, " +
$"status code: {response.StatusCode}, " +
$"error: {responseString}");
}
using (var responseStream = await response.Content.ReadAsStreamAsync())
using (var stream = new GZipStream(responseStream, mode: CompressionMode.Decompress))
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var source = new StreamSource(stream, context))
{
var destination = new DatabaseDestination(Database);
var options = new DatabaseSmugglerOptionsServerSide();
var smuggler = new Documents.DatabaseSmuggler(Database, source, destination, Database.Time, options, Result, OnProgress, CancelToken.Token);
smuggler.Execute();
}
}
private async Task SaveLastState(long operationId)
{
var retries = 0;
using (Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
while (retries++ < 15)
{
var operationState = await GetOperationState(DatabaseName, operationId, context);
if (operationState == null)
{
return;
}
if (operationState.TryGet("Status", out OperationStatus operationStatus) == false)
{
return;
}
if (operationStatus == OperationStatus.InProgress)
{
await Task.Delay(1000, CancelToken.Token);
continue;
}
if (operationStatus == OperationStatus.Canceled || operationStatus == OperationStatus.Faulted)
{
throw new InvalidOperationException("Couldn't get last operation state because the " +
$"operation state is {operationStatus.ToString()} " +
"although the operation was completed successfully");
}
if (operationState.TryGet("Result", out BlittableJsonReaderObject smugglerResultBlittable) == false)
{
return;
}
var smugglerResult = JsonDeserializationClient.SmugglerResult(smugglerResultBlittable);
if (smugglerResult == null)
{
return;
}
var importInfo = new ImportInfo
{
LastEtag = smugglerResult.GetLastEtag() + 1,
ServerUrl = ServerUrl,
DatabaseName = DatabaseName
};
var importInfoBlittable = EntityToBlittable.ConvertCommandToBlittable(importInfo, context);
await SaveLastOperationState(importInfoBlittable);
return;
}
}
}
private async Task MigrateDatabase(long operationId, ImportInfo importInfo)
{
var startDocumentEtag = importInfo?.LastEtag ?? 0;
var url = $"{Options.ServerUrl}/databases/{Options.DatabaseName}/smuggler/export?operationId={operationId}&startEtag={startDocumentEtag}";
var databaseSmugglerOptionsServerSide = new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = Options.OperateOnTypes,
RemoveAnalyzers = Options.RemoveAnalyzers
};
if (importInfo != null)
{
databaseSmugglerOptionsServerSide.OperateOnTypes |= DatabaseItemType.Tombstones;
databaseSmugglerOptionsServerSide.OperateOnTypes |= DatabaseItemType.CompareExchangeTombstones;
}
var json = JsonConvert.SerializeObject(databaseSmugglerOptionsServerSide);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var request = new HttpRequestMessage(HttpMethod.Post, url)
{
Content = content
};
var response = await Parameters.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, Parameters.CancelToken.Token);
if (response.IsSuccessStatusCode == false)
{
var responseString = await response.Content.ReadAsStringAsync();
throw new InvalidOperationException($"Failed to export database from server: {Options.ServerUrl}, " +
$"status code: {response.StatusCode}, " +
$"error: {responseString}");
}
await using (var responseStream = await response.Content.ReadAsStreamAsync())
await using (var stream = new GZipStream(responseStream, mode: CompressionMode.Decompress))
using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var source = new StreamSource(stream, context, Parameters.Database))
{
var destination = new DatabaseDestination(Parameters.Database);
var options = new DatabaseSmugglerOptionsServerSide
{
TransformScript = Options.TransformScript,
OperateOnTypes = Options.OperateOnTypes,
OperateOnDatabaseRecordTypes = Options.OperateOnDatabaseRecordTypes
};
var smuggler = new Documents.DatabaseSmuggler(Parameters.Database, source, destination, Parameters.Database.Time, options, Parameters.Result, Parameters.OnProgress, Parameters.CancelToken.Token);
await smuggler.ExecuteAsync();
}
}
private async Task SaveLastState(long operationId, ImportInfo previousImportInfo)
{
var retries = 0;
using (Parameters.Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
while (retries++ < 15)
{
var operationState = await GetOperationState(Options.DatabaseName, operationId, context);
if (operationState == null)
{
return;
}
if (operationState.TryGet("Status", out OperationStatus operationStatus) == false)
{
return;
}
if (operationStatus == OperationStatus.InProgress)
{
await Task.Delay(1000, Parameters.CancelToken.Token);
continue;
}
if (operationStatus == OperationStatus.Canceled || operationStatus == OperationStatus.Faulted)
{
throw new InvalidOperationException("Couldn't get last operation state because the " +
$"operation state is {operationStatus.ToString()} " +
"although the operation was completed successfully");
}
if (operationState.TryGet("Result", out BlittableJsonReaderObject smugglerResultBlittable) == false)
{
return;
}
var smugglerResult = JsonDeserializationClient.SmugglerResult(smugglerResultBlittable);
if (smugglerResult == null)
{
return;
}
if ((_buildVersion >= 40000 && _buildVersion < 41000) || _buildVersion == 40)
{
// prevent NRE, counter were added in 4.1
smugglerResult.Counters = new SmugglerProgressBase.CountsWithLastEtag();
}
if ((_buildVersion >= 40000 && _buildVersion < 50000) || (_buildVersion >= 40 && _buildVersion < 50))
{
// prevent NRE, time series were added in 5.0
smugglerResult.TimeSeries = new SmugglerProgressBase.CountsWithLastEtag();
}
var importInfo = new ImportInfo
{
LastEtag = Math.Max(previousImportInfo?.LastEtag ?? 0, smugglerResult.GetLastEtag() + 1),
LastRaftIndex = Math.Max(previousImportInfo?.LastRaftIndex ?? 0, smugglerResult.GetLastRaftIndex() + 1),
ServerUrl = Options.ServerUrl,
DatabaseName = Options.DatabaseName
};
var importInfoBlittable = DocumentConventions.DefaultForServer.Serialization.DefaultConverter.ToBlittable(importInfo, context);
await SaveLastOperationState(importInfoBlittable);
return;
}
}
}