public async Task InitializeAsync(DatabaseSmugglerOptions options, CancellationToken cancellationToken)
{
_sources.Clear();
if (File.Exists(_path))
{
_sources.Add(await CreateSourceAsync(options, _path, cancellationToken).ConfigureAwait(false));
return;
}
var files = Directory.GetFiles(Path.GetFullPath(_path))
.Where(file => ".ravendb-incremental-dump".Equals(Path.GetExtension(file), StringComparison.CurrentCultureIgnoreCase))
.OrderBy(File.GetLastWriteTimeUtc)
.ToArray();
if (files.Length == 0)
{
return;
}
var optionsWithoutIndexesAndTransformers = options.Clone();
optionsWithoutIndexesAndTransformers.OperateOnTypes &= ~(DatabaseItemType.Indexes | DatabaseItemType.Transformers);
for (var i = 0; i < files.Length - 1; i++)
{
var path = Path.Combine(_path, files[i]);
_sources.Add(await CreateSourceAsync(optionsWithoutIndexesAndTransformers, path, cancellationToken).ConfigureAwait(false));
}
_sources.Add(await CreateSourceAsync(options, Path.Combine(_path, files.Last()), cancellationToken).ConfigureAwait(false));
}
public async Task InitializeAsync(DatabaseSmugglerOptions options, CancellationToken cancellationToken)
{
_typeIndex = 0;
_options = options;
await InitializeBatchSizeAsync(_database, _options).ConfigureAwait(false);
}
public DocumentSmuggler(DatabaseSmugglerOptions options, DatabaseSmugglerNotifications notifications, IDatabaseSmugglerSource source, IDatabaseSmugglerDestination destination, DatabaseLastEtagsInfo maxEtags)
: base(options, notifications, source, destination)
{
_maxEtags = maxEtags;
_patcher = new SmugglerJintHelper();
_patcher.Initialize(options);
}
public DatabaseSmuggler(DatabaseSmugglerOptions options, IDatabaseSmugglerSource source, IDatabaseSmugglerDestination destination)
{
_options = options;
_source = source;
_destination = destination;
_notifications = new DatabaseSmugglerNotifications();
}
protected async Task InitializeBatchSizeAsync(DocumentStore store, DatabaseSmugglerOptions options)
{
if (store.HasJsonRequestFactory == false)
{
return;
}
var url = store.Url.ForDatabase(store.DefaultDatabase) + "/debug/config";
try
{
using (var request = store.JsonRequestFactory.CreateHttpJsonRequest(new CreateHttpJsonRequestParams(null, url, HttpMethods.Get, store.DatabaseCommands.PrimaryCredentials, store.Conventions)))
{
var configuration = (RavenJObject)await request.ReadResponseJsonAsync().ConfigureAwait(false);
var maxNumberOfItemsToProcessInSingleBatch = configuration["Core"].Value <int>("MaxNumberOfItemsToProcessInSingleBatch");
if (maxNumberOfItemsToProcessInSingleBatch <= 0)
{
return;
}
var current = options.BatchSize;
options.BatchSize = Math.Min(current, maxNumberOfItemsToProcessInSingleBatch);
}
}
catch (ErrorResponseException e)
{
if (e.StatusCode == HttpStatusCode.Forbidden) // let it continue with the user defined batch size
{
return;
}
throw;
}
}
protected SmugglerBase(DatabaseSmugglerOptions options, DatabaseSmugglerNotifications notifications, IDatabaseSmugglerSource source, IDatabaseSmugglerDestination destination)
{
Options = options;
Notifications = notifications;
Source = source;
Destination = destination;
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
_currentTypeIndex = 0;
if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Counters))
{
_returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context);
_disposeTransaction = _context.OpenReadTransaction();
LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction);
}
if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Identities))
{
_returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext);
_disposeServerTransaction = _serverContext.OpenReadTransaction();
}
buildVersion = ServerVersion.Build;
return(new DisposableAction(() =>
{
_disposeServerTransaction?.Dispose();
_returnServerContext?.Dispose();
_disposeTransaction?.Dispose();
_returnContext?.Dispose();
}));
}
protected Task InitializeBatchSizeAsync(DocumentDatabase database, DatabaseSmugglerOptions options)
{
var current = options.BatchSize;
options.BatchSize = Math.Min(current, database.Configuration.Core.MaxNumberOfItemsToProcessInSingleBatch);
return(new CompletedTask());
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
_result = result;
_returnBuffer = _context.GetManagedBuffer(out _buffer);
_state = new JsonParserState();
_parser = new UnmanagedJsonParser(_context, _state, "file");
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json.", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start object, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
buildVersion = ReadBuildVersion();
_buildVersionType = BuildVersion.Type(buildVersion);
_readLegacyEtag = options.ReadLegacyEtag;
return(new DisposableAction(() =>
{
_parser.Dispose();
_returnBuffer.Dispose();
_returnWriteBuffer.Dispose();
}));
}
public async Task <DatabaseSmugglerOperationState> LoadOperationStateAsync(DatabaseSmugglerOptions options, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(_options.ContinuationToken) == false)
{
var continuationDocId = "Raven/Smuggler/Continuation/" + _options.ContinuationToken;
try
{
var continuationDocument = await _store
.AsyncDatabaseCommands
.GetAsync(continuationDocId, cancellationToken)
.ConfigureAwait(false);
if (continuationDocument != null)
{
return(continuationDocument.DataAsJson.JsonDeserialization <DatabaseSmugglerOperationState>());
}
}
catch (Exception e)
{
if (options.IgnoreErrorsAndContinue == false)
{
throw;
}
_notifications.ShowProgress("Failed loading continuation state. Message: {0}", e.Message);
}
}
return(null);
}
public DatabaseSmugglerImportOptions(DatabaseSmugglerOptions options)
{
IncludeExpired = options.IncludeExpired;
MaxStepsForTransformScript = options.MaxStepsForTransformScript;
OperateOnTypes = options.OperateOnTypes;
RemoveAnalyzers = options.RemoveAnalyzers;
TransformScript = options.TransformScript;
}
public SmugglerPatcher(DatabaseSmugglerOptions options, DocumentDatabase database)
{
if (string.IsNullOrWhiteSpace(options.TransformScript))
{
throw new InvalidOperationException("Cannot create a patcher with empty transform script.");
}
_options = options;
_database = database;
}
private IOperationResult ExportDatabaseInternal(DatabaseSmugglerOptions options, Action <IOperationProgress> onProgress, DocumentsOperationContext context, OperationCancelToken token)
{
using (token)
{
var source = new DatabaseSource(Database, 0);
var destination = new StreamDestination(ResponseBodyStream(), context, source);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, onProgress: onProgress, token: token.Token);
return(smuggler.Execute());
}
}
public override Task SaveOperationStateAsync(DatabaseSmugglerOptions options, DatabaseSmugglerOperationState state, CancellationToken cancellationToken)
{
if (_options.Incremental)
{
var etagFileLocation = Path.Combine(_path, IncrementalExportStateFile);
WriteLastEtagsToFile(state, etagFileLocation);
}
return(new CompletedTask());
}
public virtual Task InitializeAsync(DatabaseSmugglerOptions options, DatabaseSmugglerNotifications notifications, CancellationToken cancellationToken)
{
_gZipStream = new GZipStream(_stream, CompressionMode.Compress, leaveOpen: true);
_streamWriter = new StreamWriter(_gZipStream);
_writer = new JsonTextWriter(_streamWriter)
{
Formatting = Formatting.Indented
};
_writer.WriteStartObject();
return(new CompletedTask());
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
buildVersion = 40;
_reader = new StreamReader(_stream);
_csvReader = new CsvReader(_reader);
_csvReader.Configuration.Delimiter = ",";
return(new DisposableAction(() =>
{
_reader.Dispose();
_csvReader.Dispose();
}));
}
private static async Task <IDatabaseSmugglerSource> CreateSourceAsync(DatabaseSmugglerOptions options, string path, CancellationToken cancellationToken)
{
var source = new DatabaseSmugglerStreamSource(File.OpenRead(path), leaveOpen: false)
{
DisplayName = Path.GetFileName(path)
};
await source
.InitializeAsync(options, cancellationToken)
.ConfigureAwait(false);
return(source);
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
_currentTypeIndex = 0;
_returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context);
_disposeTransaction = _context.OpenReadTransaction();
buildVersion = ServerVersion.Build;
return(new DisposableAction(() =>
{
_disposeTransaction.Dispose();
_returnContext.Dispose();
}));
}
public HttpResponseMessage ValidateExportOptions([FromBody] DatabaseSmugglerOptions smugglerOptions)
{
try
{
new SmugglerJintHelper().Initialize(smugglerOptions);
}
catch (Exception e)
{
throw new InvalidDataException("Incorrect transform script", e);
}
return(GetEmptyMessage(HttpStatusCode.NoContent));
}
private async Task <ImportResult> DoImportInternal(DocumentsOperationContext context, Stream stream,
DatabaseSmugglerOptions smugglerOptions, Action <IOperationProgress> onProgress)
{
try
{
var importer = new SmugglerImporter(Database, smugglerOptions);
return(await importer.Import(context, stream, onProgress));
}
finally
{
stream.Dispose();
}
}
public async Task InitializeAsync(DatabaseSmugglerOptions options, CancellationToken cancellationToken)
{
_typeIndex = 0;
_options = options;
_store = _storeFactory();
if (_ownsStore)
{
_store.Initialize(ensureDatabaseExists: false);
}
await ServerValidation.ValidateThatServerIsUpAndDatabaseExistsAsync(_store, cancellationToken).ConfigureAwait(false);
await InitializeBatchSizeAsync(_store, _options).ConfigureAwait(false);
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
buildVersion = ServerVersion.DevBuildNumber;
_reader = new StreamReader(_stream);
_csvReader = new CsvReader(_reader, _csvHelperConfig);
_result = result;
return(new DisposableAction(() =>
{
_reader.Dispose();
_csvReader.Dispose();
}));
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, long buildVersion)
{
_gzipStream = new GZipStream(_stream, CompressionMode.Compress, leaveOpen: true);
_writer = new BlittableJsonTextWriter(_context, _gzipStream);
_writer.WriteStartObject();
_writer.WritePropertyName("BuildVersion");
_writer.WriteInteger(buildVersion);
return(new DisposableAction(() =>
{
_writer.WriteEndObject();
_writer.Dispose();
_gzipStream.Dispose();
}));
}
public async Task InitializeAsync(DatabaseSmugglerOptions options, DatabaseSmugglerNotifications notifications, CancellationToken cancellationToken)
{
_globalOptions = options;
_notifications = notifications;
_store = _storeFactory();
if (_ownsStore)
{
_store.Initialize(ensureDatabaseExists: false);
}
_store.JsonRequestFactory.DisableRequestCompression = _options.DisableCompression;
await ServerValidation.ValidateThatServerIsUpAndDatabaseExistsAsync(_store, cancellationToken).ConfigureAwait(false);
await InitializeBatchSizeAsync(_store, _globalOptions).ConfigureAwait(false);
}
private void ImportDatabase(DocumentStore docStore, string database)
{
var smugglerOpts = new DatabaseSmugglerOptions()
{
Database = database
};
if (DatabaseDumpFilePath != null)
{
AsyncHelpers.RunSync(() => docStore.Smuggler
.ImportAsync(smugglerOpts, DatabaseDumpFilePath));
}
else if (DatabaseDumpFileStream != null)
{
AsyncHelpers.RunSync(() => docStore.Smuggler
.ImportAsync(smugglerOpts, DatabaseDumpFileStream));
}
}
public void Initialize(DatabaseSmugglerOptions options)
{
if (string.IsNullOrEmpty(options?.TransformScript))
{
return;
}
jint = new Engine(cfg =>
{
cfg.AllowDebuggerStatement(false);
cfg.MaxStatements(options.MaxStepsForTransformScript);
});
jint.Execute(string.Format(@"
function Transform(docInner){{
return ({0}).apply(this, [docInner]);
}};", options.TransformScript));
}
public DatabaseSmugglerRemoteDocumentActions(DatabaseSmugglerOptions globalOptions, DatabaseSmugglerRemoteDestinationOptions options, DatabaseSmugglerNotifications notifications, DocumentStore store)
{
_options = options;
_notifications = notifications;
_bulkInsert = store.BulkInsert(store.DefaultDatabase, new BulkInsertOptions
{
BatchSize = globalOptions.BatchSize,
OverwriteExisting = true,
Compression = options.DisableCompression ? BulkInsertCompression.None : BulkInsertCompression.GZip,
ChunkedBulkInsertOptions = new ChunkedBulkInsertOptions
{
MaxChunkVolumeInBytes = options.TotalDocumentSizeInChunkLimitInBytes,
MaxDocumentsPerChunk = options.ChunkSize
}
});
_notifications.OnDocumentRead += DocumentFound;
_timeSinceLastWrite = Stopwatch.StartNew();
}
public override async Task InitializeAsync(DatabaseSmugglerOptions options, DatabaseSmugglerNotifications notifications, CancellationToken cancellationToken)
{
var filePath = _path;
if (_options.Incremental)
{
if (Directory.Exists(_path) == false)
{
if (File.Exists(_path))
{
filePath = Path.GetDirectoryName(_path) ?? _path;
}
else
{
Directory.CreateDirectory(_path);
}
}
filePath = Path.Combine(filePath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm-0", CultureInfo.InvariantCulture) + ".ravendb-incremental-dump");
if (File.Exists(filePath))
{
var counter = 1;
while (true)
{
filePath = Path.Combine(Path.GetDirectoryName(filePath), SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + "-" + counter + ".ravendb-incremental-dump");
if (File.Exists(filePath) == false)
{
break;
}
counter++;
}
}
}
_outputFilePath = filePath;
_stream = File.Create(filePath);
await base.InitializeAsync(options, notifications, cancellationToken).ConfigureAwait(false);
}
public Task InitializeAsync(DatabaseSmugglerOptions options, CancellationToken cancellationToken)
{
_options = options;
try
{
_stream.Position = 0;
_sizeStream = new CountingStream(new GZipStream(_stream, CompressionMode.Decompress));
_streamReader = new StreamReader(_sizeStream);
_reader = new RavenJsonTextReader(_streamReader);
}
catch (Exception e)
{
if (e is InvalidDataException == false)
{
throw;
}
_stream.Seek(0, SeekOrigin.Begin);
_sizeStream = new CountingStream(_stream);
_streamReader = new StreamReader(_sizeStream);
_reader = new JsonTextReader(_streamReader);
}
if (_reader.Read() == false)
{
return(new CompletedTask());
}
if (_reader.TokenType != JsonToken.StartObject)
{
throw new InvalidDataException("StartObject was expected");
}
return(new CompletedTask());
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
_currentTypeIndex = 0;
if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.CounterGroups))
{
_returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context);
_disposeTransaction = _context.OpenReadTransaction();
LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction);
LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(_disposeTransaction.InnerTransaction);
}
if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Identities) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchangeTombstones) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Subscriptions))
{
_returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext);
_disposeServerTransaction = _serverContext.OpenReadTransaction();
using (var rawRecord = _database.ServerStore.Cluster.ReadRawDatabaseRecord(_serverContext, _database.Name))
{
LastRaftIndex = rawRecord.EtagForBackup;
}
}
buildVersion = ServerVersion.Build;
return(new DisposableAction(() =>
{
_disposeServerTransaction?.Dispose();
_returnServerContext?.Dispose();
_disposeTransaction?.Dispose();
_returnContext?.Dispose();
}));
}
