public override async Task SmuggleAsync(FileSystemSmugglerOperationState state, CancellationToken cancellationToken)
{
var written = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
Notifications.ShowProgress("Exporting Configurations");
using (var writer = Destination.WriteConfigurations())
{
while (true)
{
bool hasConfigs = false;
foreach (var configItem in await Source.GetConfigurations(written, Options.BatchSize).ConfigureAwait(false))
{
if (internalConfigs.IsMatch(configItem.Key))
{
continue;
}
if (Options.StripReplicationInformation)
{
if (configItem.Key.Equals(SynchronizationConstants.RavenSynchronizationVersionHiLo, StringComparison.OrdinalIgnoreCase))
{
continue;
}
}
hasConfigs = true;
var config = configItem.Value;
if (string.Equals(configItem.Key, SynchronizationConstants.RavenSynchronizationDestinations, StringComparison.OrdinalIgnoreCase))
{
config = DisableSynchronizationDestinations(config);
}
await writer.WriteConfigurationAsync(configItem.Key, config).ConfigureAwait(false);
written++;
if (written % 100 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
Notifications.ShowProgress("Exported {0} configurations. ", written);
lastReport = SystemTime.UtcNow;
}
}
if (hasConfigs == false)
{
break;
}
}
}
Notifications.ShowProgress("Done with exporting configurations");
}
public Task AfterExecuteAsync(FileSystemSmugglerOperationState state)
{
if (incremental)
{
WriteLastEtagsToFile(state, Path.GetDirectoryName(path));
}
state.OutputPath = path;
return(new CompletedTask());
}
public static void WriteLastEtagsToFile(FileSystemSmugglerOperationState state, string backupPath)
{
var etagFileLocation = Path.Combine(backupPath, IncrementalExportStateFile);
using (var streamWriter = new StreamWriter(File.Create(etagFileLocation)))
{
new RavenJObject
{
{ "LastFileEtag", state.LastFileEtag.ToString() },
{ "LastDeletedFileEtag", state.LastDeletedFileEtag.ToString() },
}.WriteTo(new JsonTextWriter(streamWriter));
}
}
public async Task <FileSystemSmugglerOperationState> ExecuteAsync(IFileSystemSmugglerSource source, IFileSystemSmugglerDestination destination, CancellationToken cancellationToken = default(CancellationToken))
{
using (source) // TODO arek
using (destination)
{
FileSystemSmugglerOperationState state = null;
try
{
await source
.InitializeAsync(options, cancellationToken)
.ConfigureAwait(false);
await destination
.InitializeAsync(options, Notifications, cancellationToken)
.ConfigureAwait(false);
state = new FileSystemSmugglerOperationState
{
LastFileEtag = options.StartFilesEtag,
LastDeletedFileEtag = options.StartFilesDeletionEtag,
};
Debug.Assert(state.LastFileEtag != null); //TODO arek - for refactoring purposes
Debug.Assert(state.LastDeletedFileEtag != null);
await ProcessAsync(source, destination, state, cancellationToken).ConfigureAwait(false);
await destination.AfterExecuteAsync(state).ConfigureAwait(false);
return(state);
}
catch (SmugglerException e)
{
// TODO arek
//source.OnException(e);
destination.OnException(e);
throw;
}
}
}
public override async Task SmuggleAsync(FileSystemSmugglerOperationState state, CancellationToken cancellationToken)
{
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
Notifications.ShowProgress("Exporting Files");
using (var writer = Destination.WriteFiles())
{
try
{
while (true)
{
bool hasDocs = false;
using (var files = await Source.GetFilesAsync(state.LastFileEtag, Options.BatchSize, cancellationToken).ConfigureAwait(false))
{
while (await files.MoveNextAsync().ConfigureAwait(false))
{
hasDocs = true;
var file = files.Current;
if (file.IsTombstone)
{
state.LastFileEtag = file.Etag;
continue;
}
var tempLastEtag = file.Etag; // TODO arek - why do we need temp variable
if (maxEtag != null && tempLastEtag.CompareTo(maxEtag) > 0)
{
break;
}
if (Options.StripReplicationInformation)
{
file.Metadata = StripReplicationInformationFromMetadata(file.Metadata);
}
if (Options.ShouldDisableVersioningBundle)
{
file.Metadata = DisableVersioning(file.Metadata);
}
using (var fileStream = await Source.DownloadFileAsync(file).ConfigureAwait(false))
{
await writer.WriteFileAsync(file, fileStream).ConfigureAwait(false);
}
state.LastFileEtag = tempLastEtag;
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
//TODO: Show also the MB/sec and total GB exported. // TODO arek
Notifications.ShowProgress("Exported {0} files. ", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
if (hasDocs == false)
{
break;
}
}
}
catch (Exception e)
{
Notifications.ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
Notifications.ShowProgress("Done with reading files, total: {0}, lastEtag: {1}", totalCount, state.LastFileEtag);
throw new SmugglerException(e.Message, e)
{
LastEtag = state.LastFileEtag
};
}
}
Notifications.ShowProgress("Done with reading files, total: {0}, lastEtag: {1}", totalCount, state.LastFileEtag);
}
private async Task ProcessAsync(IFileSystemSmugglerSource source, IFileSystemSmugglerDestination destination, FileSystemSmugglerOperationState state, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(source.DisplayName) == false)
{
Notifications.ShowProgress("Processing source: {0}", source.DisplayName);
}
var maxEtags = await source
.FetchCurrentMaxEtagsAsync()
.ConfigureAwait(false);
foreach (var type in source.GetItemsToSmuggle())
{
switch (type)
{
case SmuggleType.None:
return;
case SmuggleType.File:
await new FileSmuggler(source, destination, options, Notifications, maxEtags.LastFileEtag).SmuggleAsync(state, cancellationToken).ConfigureAwait(false);
continue;
case SmuggleType.Configuration:
await new ConfigurationSmuggler(source, destination, options, Notifications).SmuggleAsync(state, cancellationToken).ConfigureAwait(false);
continue;
default:
throw new NotSupportedException(type.ToString());
}
}
}
public Task AfterExecuteAsync(FileSystemSmugglerOperationState state)
{
return(new CompletedTask());
}
public abstract Task SmuggleAsync(FileSystemSmugglerOperationState state, CancellationToken cancellationToken);
