public async Task InitializeAsync(FileSystemSmugglerOptions options, CancellationToken cancellationToken)
{
if (File.Exists(path))
{
sources.Add(await CreateSourceAsync(options, path, cancellationToken).ConfigureAwait(false));
return;
}
// incremental
var directory = new DirectoryInfo(path);
if (!directory.Exists)
{
throw new InvalidOperationException($"The directory '{path}' does not exists.");
}
var files = Directory.GetFiles(directory.FullName)
.Where(file => Path.GetExtension(file).Equals(".ravenfs-incremental-dump", StringComparison.CurrentCultureIgnoreCase))
.OrderBy(x => File.GetLastWriteTimeUtc(x))
.ToArray();
if (files.Length == 0)
{
return;
}
foreach (string filename in files)
{
var filePath = Path.Combine(path, filename);
sources.Add(await CreateSourceAsync(options, filePath, cancellationToken).ConfigureAwait(false));
}
}
protected SmugglerBase(IFileSystemSmugglerSource source, IFileSystemSmugglerDestination destination, FileSystemSmugglerOptions options, FileSystemSmugglerNotifications notifications)
{
Notifications = notifications;
Source = source;
Destination = destination;
Options = options;
}
public async Task InitializeAsync(FileSystemSmugglerOptions options, CancellationToken cancellationToken)
{
filesStore = FileStoreHelper.CreateStore(connectionOptions);
await ServerValidation.ValidateThatServerIsUpAndFileSystemExists(connectionOptions, filesStore).ConfigureAwait(false);
await ServerValidation.DetectServerSupportedFeatures(connectionOptions).ConfigureAwait(false); // TODO arek - merge those 2 methods into single one
}
public Task InitializeAsync(FileSystemSmugglerOptions options, FileSystemSmugglerNotifications notifications, CancellationToken cancellationToken)
{
// We use PositionWrapperStream due to:
// http://connect.microsoft.com/VisualStudio/feedbackdetail/view/816411/ziparchive-shouldnt-read-the-position-of-non-seekable-streams
positionStream = new PositionWrapperStream(stream, leaveOpen: true);
archive = new ZipArchive(positionStream, ZipArchiveMode.Create, leaveOpen: true);
return(new CompletedTask());
}
private static async Task <StreamSmugglingSource> CreateSourceAsync(FileSystemSmugglerOptions options, string path, CancellationToken cancellationToken)
{
var source = new StreamSmugglingSource(File.OpenRead(path))
{
// TODO arek DisplayName = Path.GetFileName(path)
};
await source
.InitializeAsync(options, cancellationToken)
.ConfigureAwait(false);
return(source);
}
public Task InitializeAsync(FileSystemSmugglerOptions options, FileSystemSmugglerNotifications notifications, CancellationToken cancellationToken)
{
if (incremental)
{
if (Directory.Exists(path) == false)
{
if (File.Exists(path))
{
path = Path.GetDirectoryName(path) ?? path;
}
else
{
Directory.CreateDirectory(path);
}
}
if (options.StartFilesEtag == Etag.Empty)
{
var lastExportedEtags = ReadLastEtagsFromIncrementalExportFile(path);
if (lastExportedEtags != null)
{
options.StartFilesEtag = lastExportedEtags.LastFileEtag;
options.StartFilesDeletionEtag = lastExportedEtags.LastDeletedFileEtag;
}
}
path = Path.Combine(path, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm-0", CultureInfo.InvariantCulture) + ".ravenfs-incremental-dump");
if (File.Exists(path))
{
var counter = 1;
while (true)
{
path = Path.Combine(Path.GetDirectoryName(path), SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + "-" + counter + ".ravenfs-incremental-dump");
if (File.Exists(path) == false)
{
break;
}
counter++;
}
}
}
streamDestination = new StreamSmugglingDestination(File.Create(path));
return(streamDestination.InitializeAsync(options, notifications, cancellationToken));
}
public Task InitializeAsync(FileSystemSmugglerOptions options, CancellationToken cancellationToken)
{
this.options = options; // TODO arek - verify if options are really necessary in smuggling source
try
{
archive = new ZipArchive(stream, ZipArchiveMode.Read);
}
catch (InvalidDataException e)
{
throw new InvalidDataException("Invalid file system export file", e);
}
zipEntries = archive.Entries.ToDictionary(x => x.FullName);
return(new CompletedTask());
}
private void DefineFilesystemOptionSet(OptionSet optionSet, FileSystemSmugglerOptions options, FilesConnectionStringOptions source, FilesConnectionStringOptions destination)
{
optionSet.OnWarning += s => ConsoleHelper.WriteLineWithColor(ConsoleColor.Yellow, s);
// TODO arek
//optionSet.Add("timeout:", OptionCategory.SmugglerFileSystem, "The timeout to use for requests", s => options.Timeout = TimeSpan.FromMilliseconds(int.Parse(s)));
//optionSet.Add("incremental", OptionCategory.SmugglerFileSystem, "States usage of incremental operations", _ => options.Incremental = true);
databaseOptionSet.Add("disable-versioning-during-import", OptionCategory.SmugglerFileSystem, "Disables versioning for the duration of the import", _ => options.ShouldDisableVersioningBundle = true);
optionSet.Add("u|user|username:"******"The username to use when the filesystem requires the client to authenticate.", value => GetCredentials(source).UserName = value);
optionSet.Add("u2|user2|username2:", OptionCategory.SmugglerFileSystem, "The username to use when the filesystem requires the client to authenticate. This parameter is used only in the between operation.", value => GetCredentials(destination).UserName = value);
optionSet.Add("p|pass|password:"******"The password to use when the filesystem requires the client to authenticate.", value => GetCredentials(source).Password = value);
optionSet.Add("p2|pass2|password2:", OptionCategory.SmugglerFileSystem, "The password to use when the filesystem requires the client to authenticate. This parameter is used only in the between operation.", value => GetCredentials(destination).Password = value);
optionSet.Add("domain:", OptionCategory.SmugglerFileSystem, "The domain to use when the filesystem requires the client to authenticate.", value => GetCredentials(source).Domain = value);
optionSet.Add("domain2:", OptionCategory.SmugglerFileSystem, "The domain to use when the filesystem requires the client to authenticate. This parameter is used only in the between operation.", value => GetCredentials(destination).Domain = value);
optionSet.Add("key|api-key|apikey:", OptionCategory.SmugglerFileSystem, "The API-key to use, when using OAuth.", value => source.ApiKey = value);
optionSet.Add("key2|api-key2|apikey2:", OptionCategory.SmugglerFileSystem, "The API-key to use, when using OAuth. This parameter is used only in the between operation.", value => destination.ApiKey = value);
optionSet.Add("f|filesystem:", OptionCategory.SmugglerFileSystem, "The filesystem to operate on. If no specified, the operations will be on the default filesystem.", value => source.DefaultFileSystem = value);
optionSet.Add("f2|filesystem2:", OptionCategory.SmugglerFileSystem, "The filesystem to export to. If no specified, the operations will be on the default filesystem. This parameter is used only in the between operation.", value => destination.DefaultFileSystem = value);
}
public FileSmuggler(IFileSystemSmugglerSource source, IFileSystemSmugglerDestination destination, FileSystemSmugglerOptions options, FileSystemSmugglerNotifications notifications, Etag maxEtag)
: base(source, destination, options, notifications)
{
this.maxEtag = maxEtag;
}
public FileSystemSmuggler(FileSystemSmugglerOptions options)
{
this.options = options;
Notifications = new FileSystemSmugglerNotifications();
}
public async Task <HttpResponseMessage> ImportFilesystem(int batchSize, bool stripReplicationInformation, bool shouldDisableVersioningBundle)
{
if (!Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
string tempPath = FileSystem.Configuration.Core.TempPath;
var fullTempPath = tempPath + Constants.TempUploadsDirectoryName;
if (File.Exists(fullTempPath))
{
File.Delete(fullTempPath);
}
if (Directory.Exists(fullTempPath) == false)
{
Directory.CreateDirectory(fullTempPath);
}
var streamProvider = new MultipartFileStreamProvider(fullTempPath);
await Request.Content.ReadAsMultipartAsync(streamProvider).ConfigureAwait(false);
var uploadedFilePath = streamProvider.FileData[0].LocalFileName;
string fileName = null;
var fileContent = streamProvider.Contents.SingleOrDefault();
if (fileContent != null)
{
fileName = fileContent.Headers.ContentDisposition.FileName.Replace("\"", string.Empty);
}
var status = new ImportOperationStatus();
var cts = new CancellationTokenSource();
var task = Task.Run(async() =>
{
try
{
using (var fileStream = File.Open(uploadedFilePath, FileMode.Open, FileAccess.Read))
{
var options = new FileSystemSmugglerOptions
{
BatchSize = batchSize,
StripReplicationInformation = stripReplicationInformation,
ShouldDisableVersioningBundle = shouldDisableVersioningBundle
};
var smuggler = new FileSystemSmuggler(options);
smuggler.Notifications.OnProgress += (sender, message) => status.LastProgress = message;
await smuggler.ExecuteAsync(new StreamSmugglingSource(fileStream), new EmbeddedSmugglingDestination(FileSystem), cts.Token).ConfigureAwait(false);
}
}
catch (Exception e)
{
status.Faulted = true;
status.State = RavenJObject.FromObject(new
{
Error = e.ToString()
});
if (cts.Token.IsCancellationRequested)
{
status.State = RavenJObject.FromObject(new { Error = "Task was cancelled" });
cts.Token.ThrowIfCancellationRequested(); //needed for displaying the task status as canceled and not faulted
}
if (e is InvalidDataException)
{
status.ExceptionDetails = e.Message;
}
else if (e is OperationVetoedException && e.Message.Contains(VersioningTriggerActions.CreationOfHistoricalRevisionIsNotAllowed))
{
status.ExceptionDetails = "You are trying to import historical documents while the versioning bundle is enabled. " +
"You should disable versioning during such import. Please mark the checkbox 'Disable versioning bundle during import' at Import File System";
}
else
{
status.ExceptionDetails = e.ToString();
}
throw;
}
finally
{
status.Completed = true;
File.Delete(uploadedFilePath);
}
}, cts.Token);
long id;
FileSystem.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription
{
StartTime = SystemTime.UtcNow,
TaskType = TaskActions.PendingTaskType.ImportFileSystem,
Payload = fileName,
}, out id, cts);
return(GetMessageWithObject(new
{
OperationId = id
}, HttpStatusCode.Accepted));
}
public Task InitializeAsync(FileSystemSmugglerOptions options, CancellationToken cancellationToken)
{
return(new CompletedTask());
}
public ConfigurationSmuggler(IFileSystemSmugglerSource source, IFileSystemSmugglerDestination destination, FileSystemSmugglerOptions options, FileSystemSmugglerNotifications notifications)
: base(source, destination, options, notifications)
{
}
