public CsvStreamSource(DocumentDatabase database, Stream stream, DocumentsOperationContext context, string collection)
{
_database = database;
_stream = stream;
_context = context;
_currentType = DatabaseItemType.Documents;
_collection = collection;
}
protected AbstractMigrator(MigratorOptions options)
{
MigrationStateKey = options.MigrationStateKey;
ServerUrl = options.ServerUrl;
DatabaseName = options.DatabaseName;
HttpClient = options.HttpClient;
OperateOnTypes = options.OperateOnTypes;
RemoveAnalyzers = options.RemoveAnalyzers;
ImportRavenFs = options.ImportRavenFs;
Result = options.Result;
OnProgress = options.OnProgress;
Database = options.Database;
CancelToken = options.CancelToken;
}
protected AbstractMigrator(MigratorOptions options)
{
MigrationStateKey = options.MigrationStateKey;
ServerUrl = options.ServerUrl;
DatabaseName = options.DatabaseName;
HttpClient = options.HttpClient;
ApiKey = options.ApiKey;
EnableBasicAuthenticationOverUnsecuredHttp = options.EnableBasicAuthenticationOverUnsecuredHttp;
SkipServerCertificateValidation = options.SkipServerCertificateValidation;
OperateOnTypes = options.OperateOnTypes;
RemoveAnalyzers = options.RemoveAnalyzers;
ImportRavenFs = options.ImportRavenFs;
Result = options.Result;
OnProgress = options.OnProgress;
Database = options.Database;
CancelToken = options.CancelToken;
}
public long SkipType(DatabaseItemType type)
{
switch (type)
{
case DatabaseItemType.None:
return(0);
case DatabaseItemType.Documents:
case DatabaseItemType.RevisionDocuments:
case DatabaseItemType.Tombstones:
case DatabaseItemType.Indexes:
case DatabaseItemType.Identities:
return(SkipArray());
default:
throw new ArgumentOutOfRangeException(nameof(type), type, null);
}
}
public CsvStreamSource(DocumentDatabase database, Stream stream, DocumentsOperationContext context, string collection, CsvImportOptions csvConfig)
{
_database = database;
_stream = stream;
_context = context;
_currentType = DatabaseItemType.Documents;
_collection = collection;
_csvHelperConfig = new CsvConfiguration(CultureInfo.InvariantCulture);
_csvHelperConfig.Delimiter = csvConfig.Delimiter;
_csvHelperConfig.Quote = csvConfig.Quote;
_csvHelperConfig.TrimOptions = csvConfig.TrimOptions;
_csvHelperConfig.AllowComments = csvConfig.AllowComments;
if (csvConfig.AllowComments && csvConfig.Comment != null)
{
_csvHelperConfig.Comment = (char)csvConfig.Comment;
}
_csvHelperConfig.BadDataFound = null;
}
private static bool FilterIdentity(string indentityName, DatabaseItemType operateOnTypes)
{
if ("Raven/Etag".Equals(indentityName, StringComparison.InvariantCultureIgnoreCase))
{
return(false);
}
if ("IndexId".Equals(indentityName, StringComparison.InvariantCultureIgnoreCase))
{
return(false);
}
if (Constants.RavenSubscriptionsPrefix.Equals(indentityName, StringComparison.OrdinalIgnoreCase))
{
return(false);
}
if (operateOnTypes.HasFlag(DatabaseItemType.Documents))
{
return(true);
}
return(false);
}
public SmugglerMetadataModifier(DatabaseItemType operateOnTypes)
{
_operateOnTypes = operateOnTypes;
}
public long SkipType(DatabaseItemType type, Action <long> onSkipped)
{
return(0); // no-op
}
public async Task <HttpResponseMessage> ImportDatabase(int batchSize, bool includeExpiredDocuments, bool stripReplicationInformation, bool shouldDisableVersioningBundle, DatabaseItemType operateOnTypes, string filtersPipeDelimited, string transformScript)
{
if (!Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
string tempPath = Database.Configuration.Core.TempPath;
var fullTempPath = tempPath + Constants.TempUploadsDirectoryName;
if (File.Exists(fullTempPath))
{
File.Delete(fullTempPath);
}
if (Directory.Exists(fullTempPath) == false)
{
Directory.CreateDirectory(fullTempPath);
}
var streamProvider = new MultipartFileStreamProvider(fullTempPath);
await Request.Content.ReadAsMultipartAsync(streamProvider).ConfigureAwait(false);
var uploadedFilePath = streamProvider.FileData[0].LocalFileName;
string fileName = null;
var fileContent = streamProvider.Contents.SingleOrDefault();
if (fileContent != null)
{
fileName = fileContent.Headers.ContentDisposition.FileName.Replace("\"", string.Empty);
}
var status = new ImportOperationStatus();
var cts = new CancellationTokenSource();
var task = Task.Run(async() =>
{
try
{
using (var fileStream = File.Open(uploadedFilePath, FileMode.Open, FileAccess.Read))
{
var smugglerOptions = new DatabaseSmugglerOptions();
smugglerOptions.BatchSize = batchSize;
smugglerOptions.ShouldExcludeExpired = !includeExpiredDocuments;
smugglerOptions.StripReplicationInformation = stripReplicationInformation;
smugglerOptions.ShouldDisableVersioningBundle = shouldDisableVersioningBundle;
smugglerOptions.OperateOnTypes = operateOnTypes;
smugglerOptions.TransformScript = transformScript;
// Filters are passed in without the aid of the model binder. Instead, we pass in a list of FilterSettings using a string like this: pathHere;;;valueHere;;;true|||againPathHere;;;anotherValue;;;false
// Why? Because I don't see a way to pass a list of a values to a WebAPI method that accepts a file upload, outside of passing in a simple string value and parsing it ourselves.
if (filtersPipeDelimited != null)
{
smugglerOptions.Filters.AddRange(filtersPipeDelimited
.Split(new[] { "|||" }, StringSplitOptions.RemoveEmptyEntries)
.Select(f => f.Split(new[] { ";;;" }, StringSplitOptions.RemoveEmptyEntries))
.Select(o => new FilterSetting {
Path = o[0], Values = new List <string> {
o[1]
}, ShouldMatch = bool.Parse(o[2])
}));
}
var smuggler = new DatabaseSmuggler(smugglerOptions, new DatabaseSmugglerStreamSource(fileStream), new DatabaseSmugglerEmbeddedDestination(Database));
smuggler.Notifications.OnProgress += (sender, message) => status.LastProgress = message;
await smuggler.ExecuteAsync(cts.Token).ConfigureAwait(false);
}
}
catch (Exception e)
{
status.Faulted = true;
status.State = RavenJObject.FromObject(new
{
Error = e.ToString()
});
if (cts.Token.IsCancellationRequested)
{
status.State = RavenJObject.FromObject(new { Error = "Task was cancelled" });
cts.Token.ThrowIfCancellationRequested(); //needed for displaying the task status as canceled and not faulted
}
if (e is InvalidDataException)
{
status.ExceptionDetails = e.Message;
}
else if (e is JsonReaderException)
{
status.ExceptionDetails = "Failed to load JSON Data. Please make sure you are importing .ravendump file, exported by smuggler (aka database export). If you are importing a .ravnedump file then the file may be corrupted";
}
else if (e is OperationVetoedException && e.Message.Contains(VersioningPutTrigger.CreationOfHistoricalRevisionIsNotAllowed))
{
status.ExceptionDetails = "You are trying to import historical documents while the versioning bundle is enabled. " +
"The versioning bundle is enabled. You should disable versioning during import. " +
"Please mark the checkbox 'Disable versioning bundle during import' at Import Database: Advanced settings before importing";
}
else
{
status.ExceptionDetails = e.ToString();
}
throw;
}
finally
{
status.Completed = true;
File.Delete(uploadedFilePath);
}
}, cts.Token);
long id;
Database.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription
{
StartTime = SystemTime.UtcNow,
TaskType = TaskActions.PendingTaskType.ImportDatabase,
Payload = fileName,
}, out id, cts);
return(GetMessageWithObject(new
{
OperationId = id
}, HttpStatusCode.Accepted));
}
public long SkipType(DatabaseItemType type, Action <long> onSkipped, CancellationToken token)
{
return(0);
}
public BlittableMetadataModifier(JsonOperationContext context, bool legacyImport, bool readLegacyEtag, DatabaseItemType operateOnTypes)
{
_ctx = context;
ReadFirstEtagOfLegacyRevision = legacyImport;
ReadLegacyEtag = readLegacyEtag;
OperateOnTypes = operateOnTypes;
}
public CreateSampleDataCommand(DatabaseItemType operateOnTypes)
{
_operateOnTypes = operateOnTypes;
}
public CreateSampleDataOperation(DatabaseItemType operateOnTypes = DatabaseItemType.Documents)
{
_operateOnTypes = operateOnTypes;
}
private static async Task <Operation> Migrate(DocumentStore @from, DocumentStore to, DatabaseItemType exclude = DatabaseItemType.None)
{
using var client = new HttpClient();
var url = Uri.EscapeUriString($"{to.Urls.First()}/admin/remote-server/build/version?serverUrl={@from.Urls.First()}");
var rawVersionRespond = (await client.GetAsync(url)).Content.ReadAsStringAsync().Result;
var versionRespond = JsonConvert.DeserializeObject <BuildInfo>(rawVersionRespond);
var configuration = new SingleDatabaseMigrationConfiguration
{
ServerUrl = @from.Urls.First(),
BuildVersion = versionRespond.BuildVersion,
BuildMajorVersion = versionRespond.MajorVersion,
MigrationSettings = new DatabaseMigrationSettings
{
DatabaseName = @from.Database, OperateOnTypes = DatabaseSmugglerOptions.DefaultOperateOnTypes & ~exclude
}
};
var serializeObject = JsonConvert.SerializeObject(configuration, new StringEnumConverter());
var data = new StringContent(serializeObject, Encoding.UTF8, "application/json");
var rawOperationIdResult = await client.PostAsync($"{to.Urls.First()}/databases/{to.Database}/admin/smuggler/migrate/ravendb", data);
var operationIdResult = JsonConvert.DeserializeObject <OperationIdResult>(rawOperationIdResult.Content.ReadAsStringAsync().Result);
return(new Operation(to.GetRequestExecutor(), () => to.Changes(), to.Conventions, operationIdResult.OperationId));
}
private void ProcessType(DatabaseItemType type, SmugglerResult result, BuildVersionType buildType, bool ensureStepsProcessed = true)
{
if ((_options.OperateOnTypes & type) != type)
{
switch (type)
{
case DatabaseItemType.LegacyDocumentDeletions:
// process only those when we are processing documents
if ((_options.OperateOnTypes & DatabaseItemType.Documents) != DatabaseItemType.Documents)
{
SkipType(type, result, ensureStepsProcessed);
return;
}
break;
case DatabaseItemType.LegacyAttachments:
case DatabaseItemType.LegacyAttachmentDeletions:
// we cannot skip those?
break;
default:
SkipType(type, result, ensureStepsProcessed);
return;
}
}
result.AddInfo($"Started processing {type}.");
_onProgress.Invoke(result.Progress);
SmugglerProgressBase.Counts counts;
switch (type)
{
case DatabaseItemType.DatabaseRecord:
counts = ProcessDatabaseRecord(result);
break;
case DatabaseItemType.Documents:
counts = ProcessDocuments(result, buildType);
break;
case DatabaseItemType.RevisionDocuments:
counts = ProcessRevisionDocuments(result);
break;
case DatabaseItemType.Tombstones:
counts = ProcessTombstones(result);
break;
case DatabaseItemType.Conflicts:
counts = ProcessConflicts(result);
break;
case DatabaseItemType.Indexes:
counts = ProcessIndexes(result);
break;
case DatabaseItemType.Identities:
counts = ProcessIdentities(result);
break;
case DatabaseItemType.LegacyAttachments:
counts = ProcessLegacyAttachments(result);
break;
case DatabaseItemType.LegacyDocumentDeletions:
counts = ProcessLegacyDocumentDeletions(result);
break;
case DatabaseItemType.LegacyAttachmentDeletions:
counts = ProcessLegacyAttachmentDeletions(result);
break;
case DatabaseItemType.CompareExchange:
counts = ProcessCompareExchange(result);
break;
case DatabaseItemType.Counters:
counts = ProcessCounters(result);
break;
default:
throw new ArgumentOutOfRangeException(nameof(type), type, null);
}
counts.Processed = true;
if (counts is SmugglerProgressBase.CountsWithLastEtag countsWithEtag)
{
countsWithEtag.Attachments.Processed = true;
}
result.AddInfo($"Finished processing {type}. {counts}");
_onProgress.Invoke(result.Progress);
}
public Task <long> SkipTypeAsync(DatabaseItemType type, Action <long> onSkipped, CancellationToken token)
{
return(Task.FromResult(0L));
}
private void SkipType(DatabaseItemType type, SmugglerResult result, bool ensureStepProcessed = true)
{
result.AddInfo($"Skipping '{type}' processing.");
_onProgress.Invoke(result.Progress);
SmugglerProgressBase.Counts counts;
switch (type)
{
case DatabaseItemType.DatabaseRecord:
counts = result.DatabaseRecord;
break;
case DatabaseItemType.Documents:
counts = result.Documents;
break;
case DatabaseItemType.RevisionDocuments:
counts = result.RevisionDocuments;
break;
case DatabaseItemType.Tombstones:
counts = result.Tombstones;
break;
case DatabaseItemType.Conflicts:
counts = result.Conflicts;
break;
case DatabaseItemType.Indexes:
counts = result.Indexes;
break;
case DatabaseItemType.Identities:
counts = result.Identities;
break;
case DatabaseItemType.CompareExchange:
counts = result.CompareExchange;
break;
case DatabaseItemType.Counters:
counts = result.Counters;
break;
case DatabaseItemType.LegacyDocumentDeletions:
counts = new SmugglerProgressBase.Counts();
break;
default:
throw new ArgumentOutOfRangeException(nameof(type), type, null);
}
void OnSkipped(long skipped)
{
if (type == DatabaseItemType.Documents)
{
result.Documents.SkippedCount = skipped;
}
if (skipped % 10000 != 0)
{
return;
}
result.AddInfo($"Skipped {skipped:#,#;;0} {type.ToString().ToLowerInvariant()}");
_onProgress.Invoke(result.Progress);
}
var numberOfItemsSkipped = _source.SkipType(type, onSkipped: OnSkipped);
if (ensureStepProcessed == false)
{
return;
}
counts.Skipped = true;
counts.Processed = true;
if (numberOfItemsSkipped > 0)
{
counts.ReadCount = numberOfItemsSkipped;
result.AddInfo($"Skipped '{type}' processing. Skipped {numberOfItemsSkipped:#,#;;0} items.");
}
else
{
result.AddInfo($"Skipped '{type}' processing.");
}
_onProgress.Invoke(result.Progress);
}
public void CreateNorthwindDatabase(IDocumentStore store, DatabaseItemType operateOnTypes = DatabaseItemType.Documents)
{
store.Maintenance.Send(new CreateSampleDataOperation(operateOnTypes));
}