public async Task InitializeAsync(DatabaseSmugglerOptions options, CancellationToken cancellationToken) { _sources.Clear(); if (File.Exists(_path)) { _sources.Add(await CreateSourceAsync(options, _path, cancellationToken).ConfigureAwait(false)); return; } var files = Directory.GetFiles(Path.GetFullPath(_path)) .Where(file => ".ravendb-incremental-dump".Equals(Path.GetExtension(file), StringComparison.CurrentCultureIgnoreCase)) .OrderBy(File.GetLastWriteTimeUtc) .ToArray(); if (files.Length == 0) { return; } var optionsWithoutIndexesAndTransformers = options.Clone(); optionsWithoutIndexesAndTransformers.OperateOnTypes &= ~(DatabaseItemType.Indexes | DatabaseItemType.Transformers); for (var i = 0; i < files.Length - 1; i++) { var path = Path.Combine(_path, files[i]); _sources.Add(await CreateSourceAsync(optionsWithoutIndexesAndTransformers, path, cancellationToken).ConfigureAwait(false)); } _sources.Add(await CreateSourceAsync(options, Path.Combine(_path, files.Last()), cancellationToken).ConfigureAwait(false)); }
public async Task InitializeAsync(DatabaseSmugglerOptions options, CancellationToken cancellationToken) { _typeIndex = 0; _options = options; await InitializeBatchSizeAsync(_database, _options).ConfigureAwait(false); }
public DocumentSmuggler(DatabaseSmugglerOptions options, DatabaseSmugglerNotifications notifications, IDatabaseSmugglerSource source, IDatabaseSmugglerDestination destination, DatabaseLastEtagsInfo maxEtags) : base(options, notifications, source, destination) { _maxEtags = maxEtags; _patcher = new SmugglerJintHelper(); _patcher.Initialize(options); }
public DatabaseSmuggler(DatabaseSmugglerOptions options, IDatabaseSmugglerSource source, IDatabaseSmugglerDestination destination) { _options = options; _source = source; _destination = destination; _notifications = new DatabaseSmugglerNotifications(); }
protected async Task InitializeBatchSizeAsync(DocumentStore store, DatabaseSmugglerOptions options) { if (store.HasJsonRequestFactory == false) { return; } var url = store.Url.ForDatabase(store.DefaultDatabase) + "/debug/config"; try { using (var request = store.JsonRequestFactory.CreateHttpJsonRequest(new CreateHttpJsonRequestParams(null, url, HttpMethods.Get, store.DatabaseCommands.PrimaryCredentials, store.Conventions))) { var configuration = (RavenJObject)await request.ReadResponseJsonAsync().ConfigureAwait(false); var maxNumberOfItemsToProcessInSingleBatch = configuration["Core"].Value <int>("MaxNumberOfItemsToProcessInSingleBatch"); if (maxNumberOfItemsToProcessInSingleBatch <= 0) { return; } var current = options.BatchSize; options.BatchSize = Math.Min(current, maxNumberOfItemsToProcessInSingleBatch); } } catch (ErrorResponseException e) { if (e.StatusCode == HttpStatusCode.Forbidden) // let it continue with the user defined batch size { return; } throw; } }
protected SmugglerBase(DatabaseSmugglerOptions options, DatabaseSmugglerNotifications notifications, IDatabaseSmugglerSource source, IDatabaseSmugglerDestination destination) { Options = options; Notifications = notifications; Source = source; Destination = destination; }
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion) { _currentTypeIndex = 0; if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) || options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) || options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) || options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) || options.OperateOnTypes.HasFlag(DatabaseItemType.Counters)) { _returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context); _disposeTransaction = _context.OpenReadTransaction(); LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction); } if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) || options.OperateOnTypes.HasFlag(DatabaseItemType.Identities)) { _returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext); _disposeServerTransaction = _serverContext.OpenReadTransaction(); } buildVersion = ServerVersion.Build; return(new DisposableAction(() => { _disposeServerTransaction?.Dispose(); _returnServerContext?.Dispose(); _disposeTransaction?.Dispose(); _returnContext?.Dispose(); })); }
protected Task InitializeBatchSizeAsync(DocumentDatabase database, DatabaseSmugglerOptions options) { var current = options.BatchSize; options.BatchSize = Math.Min(current, database.Configuration.Core.MaxNumberOfItemsToProcessInSingleBatch); return(new CompletedTask()); }
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion) { _result = result; _returnBuffer = _context.GetManagedBuffer(out _buffer); _state = new JsonParserState(); _parser = new UnmanagedJsonParser(_context, _state, "file"); if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json.", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartObject) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start object, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } buildVersion = ReadBuildVersion(); _buildVersionType = BuildVersion.Type(buildVersion); _readLegacyEtag = options.ReadLegacyEtag; return(new DisposableAction(() => { _parser.Dispose(); _returnBuffer.Dispose(); _returnWriteBuffer.Dispose(); })); }
public async Task <DatabaseSmugglerOperationState> LoadOperationStateAsync(DatabaseSmugglerOptions options, CancellationToken cancellationToken) { if (string.IsNullOrWhiteSpace(_options.ContinuationToken) == false) { var continuationDocId = "Raven/Smuggler/Continuation/" + _options.ContinuationToken; try { var continuationDocument = await _store .AsyncDatabaseCommands .GetAsync(continuationDocId, cancellationToken) .ConfigureAwait(false); if (continuationDocument != null) { return(continuationDocument.DataAsJson.JsonDeserialization <DatabaseSmugglerOperationState>()); } } catch (Exception e) { if (options.IgnoreErrorsAndContinue == false) { throw; } _notifications.ShowProgress("Failed loading continuation state. Message: {0}", e.Message); } } return(null); }
public DatabaseSmugglerImportOptions(DatabaseSmugglerOptions options) { IncludeExpired = options.IncludeExpired; MaxStepsForTransformScript = options.MaxStepsForTransformScript; OperateOnTypes = options.OperateOnTypes; RemoveAnalyzers = options.RemoveAnalyzers; TransformScript = options.TransformScript; }
public SmugglerPatcher(DatabaseSmugglerOptions options, DocumentDatabase database) { if (string.IsNullOrWhiteSpace(options.TransformScript)) { throw new InvalidOperationException("Cannot create a patcher with empty transform script."); } _options = options; _database = database; }
private IOperationResult ExportDatabaseInternal(DatabaseSmugglerOptions options, Action <IOperationProgress> onProgress, DocumentsOperationContext context, OperationCancelToken token) { using (token) { var source = new DatabaseSource(Database, 0); var destination = new StreamDestination(ResponseBodyStream(), context, source); var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, onProgress: onProgress, token: token.Token); return(smuggler.Execute()); } }
public override Task SaveOperationStateAsync(DatabaseSmugglerOptions options, DatabaseSmugglerOperationState state, CancellationToken cancellationToken) { if (_options.Incremental) { var etagFileLocation = Path.Combine(_path, IncrementalExportStateFile); WriteLastEtagsToFile(state, etagFileLocation); } return(new CompletedTask()); }
public virtual Task InitializeAsync(DatabaseSmugglerOptions options, DatabaseSmugglerNotifications notifications, CancellationToken cancellationToken) { _gZipStream = new GZipStream(_stream, CompressionMode.Compress, leaveOpen: true); _streamWriter = new StreamWriter(_gZipStream); _writer = new JsonTextWriter(_streamWriter) { Formatting = Formatting.Indented }; _writer.WriteStartObject(); return(new CompletedTask()); }
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion) { buildVersion = 40; _reader = new StreamReader(_stream); _csvReader = new CsvReader(_reader); _csvReader.Configuration.Delimiter = ","; return(new DisposableAction(() => { _reader.Dispose(); _csvReader.Dispose(); })); }
private static async Task <IDatabaseSmugglerSource> CreateSourceAsync(DatabaseSmugglerOptions options, string path, CancellationToken cancellationToken) { var source = new DatabaseSmugglerStreamSource(File.OpenRead(path), leaveOpen: false) { DisplayName = Path.GetFileName(path) }; await source .InitializeAsync(options, cancellationToken) .ConfigureAwait(false); return(source); }
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion) { _currentTypeIndex = 0; _returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context); _disposeTransaction = _context.OpenReadTransaction(); buildVersion = ServerVersion.Build; return(new DisposableAction(() => { _disposeTransaction.Dispose(); _returnContext.Dispose(); })); }
public HttpResponseMessage ValidateExportOptions([FromBody] DatabaseSmugglerOptions smugglerOptions) { try { new SmugglerJintHelper().Initialize(smugglerOptions); } catch (Exception e) { throw new InvalidDataException("Incorrect transform script", e); } return(GetEmptyMessage(HttpStatusCode.NoContent)); }
private async Task <ImportResult> DoImportInternal(DocumentsOperationContext context, Stream stream, DatabaseSmugglerOptions smugglerOptions, Action <IOperationProgress> onProgress) { try { var importer = new SmugglerImporter(Database, smugglerOptions); return(await importer.Import(context, stream, onProgress)); } finally { stream.Dispose(); } }
public async Task InitializeAsync(DatabaseSmugglerOptions options, CancellationToken cancellationToken) { _typeIndex = 0; _options = options; _store = _storeFactory(); if (_ownsStore) { _store.Initialize(ensureDatabaseExists: false); } await ServerValidation.ValidateThatServerIsUpAndDatabaseExistsAsync(_store, cancellationToken).ConfigureAwait(false); await InitializeBatchSizeAsync(_store, _options).ConfigureAwait(false); }
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion) { buildVersion = ServerVersion.DevBuildNumber; _reader = new StreamReader(_stream); _csvReader = new CsvReader(_reader, _csvHelperConfig); _result = result; return(new DisposableAction(() => { _reader.Dispose(); _csvReader.Dispose(); })); }
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, long buildVersion) { _gzipStream = new GZipStream(_stream, CompressionMode.Compress, leaveOpen: true); _writer = new BlittableJsonTextWriter(_context, _gzipStream); _writer.WriteStartObject(); _writer.WritePropertyName("BuildVersion"); _writer.WriteInteger(buildVersion); return(new DisposableAction(() => { _writer.WriteEndObject(); _writer.Dispose(); _gzipStream.Dispose(); })); }
public async Task InitializeAsync(DatabaseSmugglerOptions options, DatabaseSmugglerNotifications notifications, CancellationToken cancellationToken) { _globalOptions = options; _notifications = notifications; _store = _storeFactory(); if (_ownsStore) { _store.Initialize(ensureDatabaseExists: false); } _store.JsonRequestFactory.DisableRequestCompression = _options.DisableCompression; await ServerValidation.ValidateThatServerIsUpAndDatabaseExistsAsync(_store, cancellationToken).ConfigureAwait(false); await InitializeBatchSizeAsync(_store, _globalOptions).ConfigureAwait(false); }
private void ImportDatabase(DocumentStore docStore, string database) { var smugglerOpts = new DatabaseSmugglerOptions() { Database = database }; if (DatabaseDumpFilePath != null) { AsyncHelpers.RunSync(() => docStore.Smuggler .ImportAsync(smugglerOpts, DatabaseDumpFilePath)); } else if (DatabaseDumpFileStream != null) { AsyncHelpers.RunSync(() => docStore.Smuggler .ImportAsync(smugglerOpts, DatabaseDumpFileStream)); } }
public void Initialize(DatabaseSmugglerOptions options) { if (string.IsNullOrEmpty(options?.TransformScript)) { return; } jint = new Engine(cfg => { cfg.AllowDebuggerStatement(false); cfg.MaxStatements(options.MaxStepsForTransformScript); }); jint.Execute(string.Format(@" function Transform(docInner){{ return ({0}).apply(this, [docInner]); }};", options.TransformScript)); }
public DatabaseSmugglerRemoteDocumentActions(DatabaseSmugglerOptions globalOptions, DatabaseSmugglerRemoteDestinationOptions options, DatabaseSmugglerNotifications notifications, DocumentStore store) { _options = options; _notifications = notifications; _bulkInsert = store.BulkInsert(store.DefaultDatabase, new BulkInsertOptions { BatchSize = globalOptions.BatchSize, OverwriteExisting = true, Compression = options.DisableCompression ? BulkInsertCompression.None : BulkInsertCompression.GZip, ChunkedBulkInsertOptions = new ChunkedBulkInsertOptions { MaxChunkVolumeInBytes = options.TotalDocumentSizeInChunkLimitInBytes, MaxDocumentsPerChunk = options.ChunkSize } }); _notifications.OnDocumentRead += DocumentFound; _timeSinceLastWrite = Stopwatch.StartNew(); }
public override async Task InitializeAsync(DatabaseSmugglerOptions options, DatabaseSmugglerNotifications notifications, CancellationToken cancellationToken) { var filePath = _path; if (_options.Incremental) { if (Directory.Exists(_path) == false) { if (File.Exists(_path)) { filePath = Path.GetDirectoryName(_path) ?? _path; } else { Directory.CreateDirectory(_path); } } filePath = Path.Combine(filePath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm-0", CultureInfo.InvariantCulture) + ".ravendb-incremental-dump"); if (File.Exists(filePath)) { var counter = 1; while (true) { filePath = Path.Combine(Path.GetDirectoryName(filePath), SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + "-" + counter + ".ravendb-incremental-dump"); if (File.Exists(filePath) == false) { break; } counter++; } } } _outputFilePath = filePath; _stream = File.Create(filePath); await base.InitializeAsync(options, notifications, cancellationToken).ConfigureAwait(false); }
public Task InitializeAsync(DatabaseSmugglerOptions options, CancellationToken cancellationToken) { _options = options; try { _stream.Position = 0; _sizeStream = new CountingStream(new GZipStream(_stream, CompressionMode.Decompress)); _streamReader = new StreamReader(_sizeStream); _reader = new RavenJsonTextReader(_streamReader); } catch (Exception e) { if (e is InvalidDataException == false) { throw; } _stream.Seek(0, SeekOrigin.Begin); _sizeStream = new CountingStream(_stream); _streamReader = new StreamReader(_sizeStream); _reader = new JsonTextReader(_streamReader); } if (_reader.Read() == false) { return(new CompletedTask()); } if (_reader.TokenType != JsonToken.StartObject) { throw new InvalidDataException("StartObject was expected"); } return(new CompletedTask()); }
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion) { _currentTypeIndex = 0; if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) || options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) || options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) || options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) || options.OperateOnTypes.HasFlag(DatabaseItemType.CounterGroups)) { _returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context); _disposeTransaction = _context.OpenReadTransaction(); LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction); LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(_disposeTransaction.InnerTransaction); } if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) || options.OperateOnTypes.HasFlag(DatabaseItemType.Identities) || options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchangeTombstones) || options.OperateOnTypes.HasFlag(DatabaseItemType.Subscriptions)) { _returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext); _disposeServerTransaction = _serverContext.OpenReadTransaction(); using (var rawRecord = _database.ServerStore.Cluster.ReadRawDatabaseRecord(_serverContext, _database.Name)) { LastRaftIndex = rawRecord.EtagForBackup; } } buildVersion = ServerVersion.Build; return(new DisposableAction(() => { _disposeServerTransaction?.Dispose(); _returnServerContext?.Dispose(); _disposeTransaction?.Dispose(); _returnContext?.Dispose(); })); }