コード例 #1
0
ファイル: CsvStreamSource.cs プロジェクト: xaimaran/ravendb
 public CsvStreamSource(DocumentDatabase database, Stream stream, DocumentsOperationContext context, string collection)
 {
     _database    = database;
     _stream      = stream;
     _context     = context;
     _currentType = DatabaseItemType.Documents;
     _collection  = collection;
 }
コード例 #2
0
 protected AbstractMigrator(MigratorOptions options)
 {
     MigrationStateKey = options.MigrationStateKey;
     ServerUrl         = options.ServerUrl;
     DatabaseName      = options.DatabaseName;
     HttpClient        = options.HttpClient;
     OperateOnTypes    = options.OperateOnTypes;
     RemoveAnalyzers   = options.RemoveAnalyzers;
     ImportRavenFs     = options.ImportRavenFs;
     Result            = options.Result;
     OnProgress        = options.OnProgress;
     Database          = options.Database;
     CancelToken       = options.CancelToken;
 }
コード例 #3
0
 protected AbstractMigrator(MigratorOptions options)
 {
     MigrationStateKey = options.MigrationStateKey;
     ServerUrl         = options.ServerUrl;
     DatabaseName      = options.DatabaseName;
     HttpClient        = options.HttpClient;
     ApiKey            = options.ApiKey;
     EnableBasicAuthenticationOverUnsecuredHttp = options.EnableBasicAuthenticationOverUnsecuredHttp;
     SkipServerCertificateValidation            = options.SkipServerCertificateValidation;
     OperateOnTypes  = options.OperateOnTypes;
     RemoveAnalyzers = options.RemoveAnalyzers;
     ImportRavenFs   = options.ImportRavenFs;
     Result          = options.Result;
     OnProgress      = options.OnProgress;
     Database        = options.Database;
     CancelToken     = options.CancelToken;
 }
コード例 #4
0
        public long SkipType(DatabaseItemType type)
        {
            switch (type)
            {
            case DatabaseItemType.None:
                return(0);

            case DatabaseItemType.Documents:
            case DatabaseItemType.RevisionDocuments:
            case DatabaseItemType.Tombstones:
            case DatabaseItemType.Indexes:
            case DatabaseItemType.Identities:
                return(SkipArray());

            default:
                throw new ArgumentOutOfRangeException(nameof(type), type, null);
            }
        }
コード例 #5
0
ファイル: CsvStreamSource.cs プロジェクト: rubyzhang/ravendb
        public CsvStreamSource(DocumentDatabase database, Stream stream, DocumentsOperationContext context, string collection, CsvImportOptions csvConfig)
        {
            _database    = database;
            _stream      = stream;
            _context     = context;
            _currentType = DatabaseItemType.Documents;
            _collection  = collection;

            _csvHelperConfig = new CsvConfiguration(CultureInfo.InvariantCulture);

            _csvHelperConfig.Delimiter     = csvConfig.Delimiter;
            _csvHelperConfig.Quote         = csvConfig.Quote;
            _csvHelperConfig.TrimOptions   = csvConfig.TrimOptions;
            _csvHelperConfig.AllowComments = csvConfig.AllowComments;
            if (csvConfig.AllowComments && csvConfig.Comment != null)
            {
                _csvHelperConfig.Comment = (char)csvConfig.Comment;
            }

            _csvHelperConfig.BadDataFound = null;
        }
コード例 #6
0
ファイル: IdentitySmuggler.cs プロジェクト: rstonkus/ravendb
        private static bool FilterIdentity(string indentityName, DatabaseItemType operateOnTypes)
        {
            if ("Raven/Etag".Equals(indentityName, StringComparison.InvariantCultureIgnoreCase))
            {
                return(false);
            }

            if ("IndexId".Equals(indentityName, StringComparison.InvariantCultureIgnoreCase))
            {
                return(false);
            }

            if (Constants.RavenSubscriptionsPrefix.Equals(indentityName, StringComparison.OrdinalIgnoreCase))
            {
                return(false);
            }

            if (operateOnTypes.HasFlag(DatabaseItemType.Documents))
            {
                return(true);
            }

            return(false);
        }
コード例 #7
0
 public SmugglerMetadataModifier(DatabaseItemType operateOnTypes)
 {
     _operateOnTypes = operateOnTypes;
 }
コード例 #8
0
 public long SkipType(DatabaseItemType type, Action <long> onSkipped)
 {
     return(0); // no-op
 }
コード例 #9
0
        public async Task <HttpResponseMessage> ImportDatabase(int batchSize, bool includeExpiredDocuments, bool stripReplicationInformation, bool shouldDisableVersioningBundle, DatabaseItemType operateOnTypes, string filtersPipeDelimited, string transformScript)
        {
            if (!Request.Content.IsMimeMultipartContent())
            {
                throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
            }

            string tempPath     = Database.Configuration.Core.TempPath;
            var    fullTempPath = tempPath + Constants.TempUploadsDirectoryName;

            if (File.Exists(fullTempPath))
            {
                File.Delete(fullTempPath);
            }
            if (Directory.Exists(fullTempPath) == false)
            {
                Directory.CreateDirectory(fullTempPath);
            }

            var streamProvider = new MultipartFileStreamProvider(fullTempPath);
            await Request.Content.ReadAsMultipartAsync(streamProvider).ConfigureAwait(false);

            var uploadedFilePath = streamProvider.FileData[0].LocalFileName;

            string fileName    = null;
            var    fileContent = streamProvider.Contents.SingleOrDefault();

            if (fileContent != null)
            {
                fileName = fileContent.Headers.ContentDisposition.FileName.Replace("\"", string.Empty);
            }

            var status = new ImportOperationStatus();
            var cts    = new CancellationTokenSource();

            var task = Task.Run(async() =>
            {
                try
                {
                    using (var fileStream = File.Open(uploadedFilePath, FileMode.Open, FileAccess.Read))
                    {
                        var smugglerOptions                           = new DatabaseSmugglerOptions();
                        smugglerOptions.BatchSize                     = batchSize;
                        smugglerOptions.ShouldExcludeExpired          = !includeExpiredDocuments;
                        smugglerOptions.StripReplicationInformation   = stripReplicationInformation;
                        smugglerOptions.ShouldDisableVersioningBundle = shouldDisableVersioningBundle;
                        smugglerOptions.OperateOnTypes                = operateOnTypes;
                        smugglerOptions.TransformScript               = transformScript;

                        // Filters are passed in without the aid of the model binder. Instead, we pass in a list of FilterSettings using a string like this: pathHere;;;valueHere;;;true|||againPathHere;;;anotherValue;;;false
                        // Why? Because I don't see a way to pass a list of a values to a WebAPI method that accepts a file upload, outside of passing in a simple string value and parsing it ourselves.
                        if (filtersPipeDelimited != null)
                        {
                            smugglerOptions.Filters.AddRange(filtersPipeDelimited
                                                             .Split(new[] { "|||" }, StringSplitOptions.RemoveEmptyEntries)
                                                             .Select(f => f.Split(new[] { ";;;" }, StringSplitOptions.RemoveEmptyEntries))
                                                             .Select(o => new FilterSetting {
                                Path = o[0], Values = new List <string> {
                                    o[1]
                                }, ShouldMatch = bool.Parse(o[2])
                            }));
                        }

                        var smuggler = new DatabaseSmuggler(smugglerOptions, new DatabaseSmugglerStreamSource(fileStream), new DatabaseSmugglerEmbeddedDestination(Database));
                        smuggler.Notifications.OnProgress += (sender, message) => status.LastProgress = message;

                        await smuggler.ExecuteAsync(cts.Token).ConfigureAwait(false);
                    }
                }
                catch (Exception e)
                {
                    status.Faulted = true;
                    status.State   = RavenJObject.FromObject(new
                    {
                        Error = e.ToString()
                    });
                    if (cts.Token.IsCancellationRequested)
                    {
                        status.State = RavenJObject.FromObject(new { Error = "Task was cancelled" });
                        cts.Token.ThrowIfCancellationRequested(); //needed for displaying the task status as canceled and not faulted
                    }

                    if (e is InvalidDataException)
                    {
                        status.ExceptionDetails = e.Message;
                    }
                    else if (e is JsonReaderException)
                    {
                        status.ExceptionDetails = "Failed to load JSON Data. Please make sure you are importing .ravendump file, exported by smuggler (aka database export). If you are importing a .ravnedump file then the file may be corrupted";
                    }
                    else if (e is OperationVetoedException && e.Message.Contains(VersioningPutTrigger.CreationOfHistoricalRevisionIsNotAllowed))
                    {
                        status.ExceptionDetails = "You are trying to import historical documents while the versioning bundle is enabled. " +
                                                  "The versioning bundle is enabled. You should disable versioning during import. " +
                                                  "Please mark the checkbox 'Disable versioning bundle during import' at Import Database: Advanced settings before importing";
                    }
                    else
                    {
                        status.ExceptionDetails = e.ToString();
                    }
                    throw;
                }
                finally
                {
                    status.Completed = true;
                    File.Delete(uploadedFilePath);
                }
            }, cts.Token);

            long id;

            Database.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription
            {
                StartTime = SystemTime.UtcNow,
                TaskType  = TaskActions.PendingTaskType.ImportDatabase,
                Payload   = fileName,
            }, out id, cts);

            return(GetMessageWithObject(new
            {
                OperationId = id
            }, HttpStatusCode.Accepted));
        }
コード例 #10
0
 public long SkipType(DatabaseItemType type, Action <long> onSkipped, CancellationToken token)
 {
     return(0);
 }
コード例 #11
0
 public BlittableMetadataModifier(JsonOperationContext context, bool legacyImport, bool readLegacyEtag, DatabaseItemType operateOnTypes)
 {
     _ctx = context;
     ReadFirstEtagOfLegacyRevision = legacyImport;
     ReadLegacyEtag = readLegacyEtag;
     OperateOnTypes = operateOnTypes;
 }
コード例 #12
0
 public CreateSampleDataCommand(DatabaseItemType operateOnTypes)
 {
     _operateOnTypes = operateOnTypes;
 }
コード例 #13
0
 public CreateSampleDataOperation(DatabaseItemType operateOnTypes = DatabaseItemType.Documents)
 {
     _operateOnTypes = operateOnTypes;
 }
コード例 #14
0
        private static async Task <Operation> Migrate(DocumentStore @from, DocumentStore to, DatabaseItemType exclude = DatabaseItemType.None)
        {
            using var client = new HttpClient();
            var url = Uri.EscapeUriString($"{to.Urls.First()}/admin/remote-server/build/version?serverUrl={@from.Urls.First()}");
            var rawVersionRespond = (await client.GetAsync(url)).Content.ReadAsStringAsync().Result;
            var versionRespond    = JsonConvert.DeserializeObject <BuildInfo>(rawVersionRespond);

            var configuration = new SingleDatabaseMigrationConfiguration
            {
                ServerUrl         = @from.Urls.First(),
                BuildVersion      = versionRespond.BuildVersion,
                BuildMajorVersion = versionRespond.MajorVersion,
                MigrationSettings = new DatabaseMigrationSettings
                {
                    DatabaseName = @from.Database, OperateOnTypes = DatabaseSmugglerOptions.DefaultOperateOnTypes & ~exclude
                }
            };

            var serializeObject      = JsonConvert.SerializeObject(configuration, new StringEnumConverter());
            var data                 = new StringContent(serializeObject, Encoding.UTF8, "application/json");
            var rawOperationIdResult = await client.PostAsync($"{to.Urls.First()}/databases/{to.Database}/admin/smuggler/migrate/ravendb", data);

            var operationIdResult = JsonConvert.DeserializeObject <OperationIdResult>(rawOperationIdResult.Content.ReadAsStringAsync().Result);

            return(new Operation(to.GetRequestExecutor(), () => to.Changes(), to.Conventions, operationIdResult.OperationId));
        }
コード例 #15
0
ファイル: DatabaseSmuggler.cs プロジェクト: xaimaran/ravendb
        private void ProcessType(DatabaseItemType type, SmugglerResult result, BuildVersionType buildType, bool ensureStepsProcessed = true)
        {
            if ((_options.OperateOnTypes & type) != type)
            {
                switch (type)
                {
                case DatabaseItemType.LegacyDocumentDeletions:
                    // process only those when we are processing documents
                    if ((_options.OperateOnTypes & DatabaseItemType.Documents) != DatabaseItemType.Documents)
                    {
                        SkipType(type, result, ensureStepsProcessed);
                        return;
                    }
                    break;

                case DatabaseItemType.LegacyAttachments:
                case DatabaseItemType.LegacyAttachmentDeletions:
                    // we cannot skip those?
                    break;

                default:
                    SkipType(type, result, ensureStepsProcessed);
                    return;
                }
            }

            result.AddInfo($"Started processing {type}.");
            _onProgress.Invoke(result.Progress);

            SmugglerProgressBase.Counts counts;
            switch (type)
            {
            case DatabaseItemType.DatabaseRecord:
                counts = ProcessDatabaseRecord(result);
                break;

            case DatabaseItemType.Documents:
                counts = ProcessDocuments(result, buildType);
                break;

            case DatabaseItemType.RevisionDocuments:
                counts = ProcessRevisionDocuments(result);
                break;

            case DatabaseItemType.Tombstones:
                counts = ProcessTombstones(result);
                break;

            case DatabaseItemType.Conflicts:
                counts = ProcessConflicts(result);
                break;

            case DatabaseItemType.Indexes:
                counts = ProcessIndexes(result);
                break;

            case DatabaseItemType.Identities:
                counts = ProcessIdentities(result);
                break;

            case DatabaseItemType.LegacyAttachments:
                counts = ProcessLegacyAttachments(result);
                break;

            case DatabaseItemType.LegacyDocumentDeletions:
                counts = ProcessLegacyDocumentDeletions(result);
                break;

            case DatabaseItemType.LegacyAttachmentDeletions:
                counts = ProcessLegacyAttachmentDeletions(result);
                break;

            case DatabaseItemType.CompareExchange:
                counts = ProcessCompareExchange(result);
                break;

            case DatabaseItemType.Counters:
                counts = ProcessCounters(result);
                break;

            default:
                throw new ArgumentOutOfRangeException(nameof(type), type, null);
            }

            counts.Processed = true;

            if (counts is SmugglerProgressBase.CountsWithLastEtag countsWithEtag)
            {
                countsWithEtag.Attachments.Processed = true;
            }

            result.AddInfo($"Finished processing {type}. {counts}");
            _onProgress.Invoke(result.Progress);
        }
コード例 #16
0
ファイル: CsvStreamSource.cs プロジェクト: rubyzhang/ravendb
 public Task <long> SkipTypeAsync(DatabaseItemType type, Action <long> onSkipped, CancellationToken token)
 {
     return(Task.FromResult(0L));
 }
コード例 #17
0
ファイル: DatabaseSmuggler.cs プロジェクト: xaimaran/ravendb
        private void SkipType(DatabaseItemType type, SmugglerResult result, bool ensureStepProcessed = true)
        {
            result.AddInfo($"Skipping '{type}' processing.");
            _onProgress.Invoke(result.Progress);

            SmugglerProgressBase.Counts counts;
            switch (type)
            {
            case DatabaseItemType.DatabaseRecord:
                counts = result.DatabaseRecord;
                break;

            case DatabaseItemType.Documents:
                counts = result.Documents;
                break;

            case DatabaseItemType.RevisionDocuments:
                counts = result.RevisionDocuments;
                break;

            case DatabaseItemType.Tombstones:
                counts = result.Tombstones;
                break;

            case DatabaseItemType.Conflicts:
                counts = result.Conflicts;
                break;

            case DatabaseItemType.Indexes:
                counts = result.Indexes;
                break;

            case DatabaseItemType.Identities:
                counts = result.Identities;
                break;

            case DatabaseItemType.CompareExchange:
                counts = result.CompareExchange;
                break;

            case DatabaseItemType.Counters:
                counts = result.Counters;
                break;

            case DatabaseItemType.LegacyDocumentDeletions:
                counts = new SmugglerProgressBase.Counts();
                break;

            default:
                throw new ArgumentOutOfRangeException(nameof(type), type, null);
            }

            void OnSkipped(long skipped)
            {
                if (type == DatabaseItemType.Documents)
                {
                    result.Documents.SkippedCount = skipped;
                }

                if (skipped % 10000 != 0)
                {
                    return;
                }

                result.AddInfo($"Skipped {skipped:#,#;;0} {type.ToString().ToLowerInvariant()}");
                _onProgress.Invoke(result.Progress);
            }

            var numberOfItemsSkipped = _source.SkipType(type, onSkipped: OnSkipped);

            if (ensureStepProcessed == false)
            {
                return;
            }

            counts.Skipped   = true;
            counts.Processed = true;

            if (numberOfItemsSkipped > 0)
            {
                counts.ReadCount = numberOfItemsSkipped;
                result.AddInfo($"Skipped '{type}' processing. Skipped {numberOfItemsSkipped:#,#;;0} items.");
            }
            else
            {
                result.AddInfo($"Skipped '{type}' processing.");
            }

            _onProgress.Invoke(result.Progress);
        }
コード例 #18
0
ファイル: RavenTestBase.Samples.cs プロジェクト: ikvm/ravendb
 public void CreateNorthwindDatabase(IDocumentStore store, DatabaseItemType operateOnTypes = DatabaseItemType.Documents)
 {
     store.Maintenance.Send(new CreateSampleDataOperation(operateOnTypes));
 }