public override async Task<OperationState> ExportData(SmugglerExportOptions<RavenConnectionStringOptions> exportOptions)
{
using (store = CreateStore(exportOptions.From))
{
return await base.ExportData(exportOptions);
}
}
public override async Task<ExportDataResult> ExportData(SmugglerExportOptions exportOptions)
{
using (store = CreateStore(exportOptions.From))
{
return await base.ExportData(exportOptions);
}
}
public async void Sample()
{
#region smuggler_api_1
// export Documents, Indexes, Attachments and Transformers
// to dump.raven file
// from Northwind database
// found on http://localhost:8080 server
SmugglerDatabaseApi smugglerApi = new SmugglerDatabaseApi(new SmugglerDatabaseOptions
{
OperateOnTypes =
ItemType.Documents | ItemType.Indexes | ItemType.Attachments | ItemType.Transformers,
Incremental = false
});
SmugglerExportOptions<RavenConnectionStringOptions> exportOptions = new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = "dump.raven",
From = new RavenConnectionStringOptions
{
DefaultDatabase = "Northwind",
Url = "http://localhost:8080"
}
};
await smugglerApi.ExportData(exportOptions);
#endregion
}
public override async Task<ExportFilesResult> ExportData(SmugglerExportOptions<FilesConnectionStringOptions> exportOptions)
{
if (exportOptions.From == null)
throw new ArgumentNullException("exportOptions");
using (primaryStore = await CreateStore(exportOptions.From))
using (documentStore = CreateDocumentStore(exportOptions.From))
{
Operations = new SmugglerRemoteFilesOperations(() => primaryStore, () => documentStore);
return await base.ExportData(exportOptions);
}
}
public virtual async Task<ExportDataResult> ExportData(SmugglerExportOptions exportOptions)
{
Operations.Configure(SmugglerOptions);
Operations.Initialize(SmugglerOptions);
var result = new ExportDataResult
{
FilePath = exportOptions.ToFile,
LastAttachmentsEtag = SmugglerOptions.StartAttachmentsEtag,
LastDocsEtag = SmugglerOptions.StartDocsEtag,
LastDocDeleteEtag = SmugglerOptions.StartDocsDeletionEtag,
LastAttachmentsDeleteEtag = SmugglerOptions.StartAttachmentsDeletionEtag
};
if (SmugglerOptions.Incremental)
{
if (Directory.Exists(result.FilePath) == false)
{
if (File.Exists(result.FilePath))
result.FilePath = Path.GetDirectoryName(result.FilePath) ?? result.FilePath;
else
Directory.CreateDirectory(result.FilePath);
}
if (SmugglerOptions.StartDocsEtag == Etag.Empty && SmugglerOptions.StartAttachmentsEtag == Etag.Empty)
{
ReadLastEtagsFromFile(result);
}
result.FilePath = Path.Combine(result.FilePath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + ".ravendb-incremental-dump");
if (File.Exists(result.FilePath))
{
var counter = 1;
while (true)
{
// ReSharper disable once AssignNullToNotNullAttribute
result.FilePath = Path.Combine(Path.GetDirectoryName(result.FilePath), SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + " - " + counter + ".ravendb-incremental-dump");
if (File.Exists(result.FilePath) == false)
break;
counter++;
}
}
}
SmugglerExportException lastException = null;
bool ownedStream = exportOptions.ToStream == null;
var stream = exportOptions.ToStream ?? File.Create(result.FilePath);
try
{
await DetectServerSupportedFeatures(exportOptions.From);
}
catch (WebException e)
{
Operations.ShowProgress("Failed to query server for supported features. Reason : " + e.Message);
SetLegacyMode(); //could not detect supported features, then run in legacy mode
// lastException = new SmugglerExportException
// {
// LastEtag = Etag.Empty,
// File = ownedStream ? result.FilePath : null
// };
}
try
{
using (var gZipStream = new GZipStream(stream, CompressionMode.Compress,
leaveOpen: true))
using (var streamWriter = new StreamWriter(gZipStream))
{
var jsonWriter = new JsonTextWriter(streamWriter)
{
Formatting = Formatting.Indented
};
jsonWriter.WriteStartObject();
jsonWriter.WritePropertyName("Indexes");
jsonWriter.WriteStartArray();
if (SmugglerOptions.OperateOnTypes.HasFlag(ItemType.Indexes))
{
await ExportIndexes(exportOptions.From, jsonWriter);
}
jsonWriter.WriteEndArray();
// used to synchronize max returned values for put/delete operations
var maxEtags = Operations.FetchCurrentMaxEtags();
jsonWriter.WritePropertyName("Docs");
jsonWriter.WriteStartArray();
if (SmugglerOptions.OperateOnTypes.HasFlag(ItemType.Documents))
{
try
{
result.LastDocsEtag = await ExportDocuments(exportOptions.From, jsonWriter, result.LastDocsEtag, maxEtags.LastDocsEtag);
}
catch (SmugglerExportException e)
{
result.LastDocsEtag = e.LastEtag;
e.File = ownedStream ? result.FilePath : null;
lastException = e;
}
}
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("Attachments");
jsonWriter.WriteStartArray();
if (SmugglerOptions.OperateOnTypes.HasFlag(ItemType.Attachments) && lastException == null)
{
try
{
result.LastAttachmentsEtag = await ExportAttachments(exportOptions.From, jsonWriter, result.LastAttachmentsEtag, maxEtags.LastAttachmentsEtag);
}
catch (SmugglerExportException e)
{
result.LastAttachmentsEtag = e.LastEtag;
e.File = ownedStream ? result.FilePath : null;
lastException = e;
}
}
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("Transformers");
jsonWriter.WriteStartArray();
if (SmugglerOptions.OperateOnTypes.HasFlag(ItemType.Transformers) && lastException == null)
{
await ExportTransformers(exportOptions.From, jsonWriter);
}
jsonWriter.WriteEndArray();
if (SmugglerOptions.ExportDeletions)
{
await ExportDeletions(jsonWriter, result, maxEtags);
}
await ExportIdentities(jsonWriter, SmugglerOptions.OperateOnTypes);
jsonWriter.WriteEndObject();
streamWriter.Flush();
}
if (SmugglerOptions.Incremental)
WriteLastEtagsToFile(result, result.FilePath);
if (SmugglerOptions.ExportDeletions)
{
Operations.PurgeTombstones(result);
}
if (lastException != null)
throw lastException;
return result;
}
finally
{
if (ownedStream && stream != null)
stream.Dispose();
}
}
public virtual async Task <ExportFilesResult> ExportData(SmugglerExportOptions <FilesConnectionStringOptions> exportOptions)
{
Operations.Configure(Options);
Operations.Initialize(Options);
var result = new ExportFilesResult
{
FilePath = exportOptions.ToFile,
LastFileEtag = Options.StartFilesEtag,
LastDeletedFileEtag = Options.StartFilesDeletionEtag,
};
result.FilePath = Path.GetFullPath(result.FilePath);
if (Options.Incremental)
{
if (Directory.Exists(result.FilePath) == false)
{
if (File.Exists(result.FilePath))
{
result.FilePath = Path.GetDirectoryName(result.FilePath) ?? result.FilePath;
}
else
{
Directory.CreateDirectory(result.FilePath);
}
}
if (Options.StartFilesEtag == Etag.Empty)
{
ReadLastEtagsFromFile(result);
}
result.FilePath = Path.Combine(result.FilePath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + ".ravenfs-incremental-dump");
if (File.Exists(result.FilePath))
{
var counter = 1;
while (true)
{
result.FilePath = Path.Combine(Path.GetDirectoryName(result.FilePath), SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + " - " + counter + ".ravenfs-incremental-dump");
if (File.Exists(result.FilePath) == false)
{
break;
}
counter++;
}
}
}
SmugglerExportException lastException = null;
try
{
await DetectServerSupportedFeatures(exportOptions.From);
}
catch (WebException e)
{
throw new SmugglerExportException("Failed to query server for supported features. Reason : " + e.Message)
{
LastEtag = Etag.Empty,
File = result.FilePath
};
}
if (string.IsNullOrWhiteSpace(result.FilePath))
{
throw new SmugglerException("Output directory cannot be null, empty or whitespace.");
}
var stream = File.Create(result.FilePath);
try
{
// used to synchronize max returned values for put/delete operations
var maxEtags = Operations.FetchCurrentMaxEtags();
try
{
await ExportFiles(exportOptions, stream, result.LastFileEtag, maxEtags.LastFileEtag);
}
catch (SmugglerExportException ex)
{
result.LastFileEtag = ex.LastEtag;
ex.File = result.FilePath;
lastException = ex;
}
catch (Exception)
{
throw;
}
if (Options.Incremental)
{
WriteLastEtagsToFile(result, Path.GetDirectoryName(result.FilePath));
}
if (lastException != null)
{
throw lastException;
}
return(result);
}
finally
{
stream.Dispose();
}
}
private async Task <Etag> ExportFiles(SmugglerExportOptions <FilesConnectionStringOptions> options, Stream stream, Etag lastEtag, Etag maxEtag)
{
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
Operations.ShowProgress("Exporting Files");
using (var archive = new ZipArchive(stream, ZipArchiveMode.Create))
{
var metadataList = new List <FileContainer>();
Exception exceptionHappened = null;
try
{
using (var files = await Operations.GetFiles(options.From, lastEtag, Math.Min(Options.BatchSize, int.MaxValue)))
{
while (await files.MoveNextAsync())
{
var file = files.Current;
var tempLastEtag = file.Etag;
if (maxEtag != null && tempLastEtag.CompareTo(maxEtag) > 0)
{
break;
}
// Write the metadata (which includes the stream size and file container name)
var fileContainer = new FileContainer
{
Key = Path.Combine(file.Directory.TrimStart('/'), file.Name),
Metadata = file.Metadata,
};
ZipArchiveEntry fileToStore = archive.CreateEntry(fileContainer.Key);
using (var fileStream = await Operations.DownloadFile(file))
using (var zipStream = fileToStore.Open())
{
await fileStream.CopyToAsync(zipStream).ConfigureAwait(false);
}
metadataList.Add(fileContainer);
lastEtag = tempLastEtag;
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
//TODO: Show also the MB/sec and total GB exported.
Operations.ShowProgress("Exported {0} files. ", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
}
catch (Exception e)
{
Operations.ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
Operations.ShowProgress("Done with reading files, total: {0}, lastEtag: {1}", totalCount, lastEtag);
exceptionHappened = new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
var metadataEntry = archive.CreateEntry(".metadata");
using (var metadataStream = metadataEntry.Open())
using (var writer = new StreamWriter(metadataStream))
{
foreach (var item in metadataList)
{
writer.WriteLine(RavenJObject.FromObject(item));
}
}
if (exceptionHappened != null)
{
throw exceptionHappened;
}
Operations.ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
}
public virtual async Task <ExportFilesResult> ExportData(SmugglerExportOptions <FilesConnectionStringOptions> exportOptions)
{
Operations.Configure(Options);
Operations.Initialize(Options);
var result = new ExportFilesResult
{
FilePath = exportOptions.ToFile,
LastFileEtag = Options.StartFilesEtag,
LastDeletedFileEtag = Options.StartFilesDeletionEtag,
};
if (result.FilePath != null)
{
result.FilePath = Path.GetFullPath(result.FilePath);
}
if (Options.Incremental)
{
if (Directory.Exists(result.FilePath) == false)
{
if (File.Exists(result.FilePath))
{
result.FilePath = Path.GetDirectoryName(result.FilePath) ?? result.FilePath;
}
else
{
Directory.CreateDirectory(result.FilePath);
}
}
if (Options.StartFilesEtag == Etag.Empty)
{
ReadLastEtagsFromFile(result);
}
result.FilePath = Path.Combine(result.FilePath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm-0", CultureInfo.InvariantCulture) + ".ravenfs-incremental-dump");
if (File.Exists(result.FilePath))
{
var counter = 1;
while (true)
{
result.FilePath = Path.Combine(Path.GetDirectoryName(result.FilePath), SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + "-" + counter + ".ravenfs-incremental-dump");
if (File.Exists(result.FilePath) == false)
{
break;
}
counter++;
}
}
}
SmugglerExportException lastException = null;
bool ownedStream = exportOptions.ToStream == null;
var stream = exportOptions.ToStream ?? File.Create(result.FilePath);
try
{
await DetectServerSupportedFeatures(exportOptions.From).ConfigureAwait(false);
}
catch (WebException e)
{
throw new SmugglerExportException("Failed to query server for supported features. Reason : " + e.Message)
{
LastEtag = Etag.Empty,
File = result.FilePath
};
}
try
{
// used to synchronize max returned values for put/delete operations
var maxEtags = Operations.FetchCurrentMaxEtags();
try
{
// We use PositionWrapperStream due to:
// http://connect.microsoft.com/VisualStudio/feedbackdetail/view/816411/ziparchive-shouldnt-read-the-position-of-non-seekable-streams
using (var positionStream = new PositionWrapperStream(stream, leaveOpen: true))
using (var archive = new ZipArchive(positionStream, ZipArchiveMode.Create, leaveOpen: true))
{
await ExportFiles(archive, result.LastFileEtag, maxEtags.LastFileEtag).ConfigureAwait(false);
await ExportConfigurations(archive).ConfigureAwait(false);
}
}
catch (SmugglerExportException ex)
{
result.LastFileEtag = ex.LastEtag;
ex.File = result.FilePath;
lastException = ex;
}
if (Options.Incremental)
{
WriteLastEtagsToFile(result, Path.GetDirectoryName(result.FilePath));
}
if (lastException != null)
{
throw lastException;
}
return(result);
}
finally
{
if (ownedStream && stream != null)
{
stream.Dispose();
}
}
}
public virtual async Task <ExportDataResult> ExportData(SmugglerExportOptions exportOptions, SmugglerOptions options)
{
SetSmugglerOptions(options);
var result = new ExportDataResult
{
FilePath = exportOptions.ToFile,
LastAttachmentsEtag = options.StartAttachmentsEtag,
LastDocsEtag = options.StartDocsEtag,
LastDocDeleteEtag = options.StartDocsDeletionEtag,
LastAttachmentsDeleteEtag = options.StartAttachmentsDeletionEtag
};
if (options.Incremental)
{
if (Directory.Exists(result.FilePath) == false)
{
if (File.Exists(result.FilePath))
{
result.FilePath = Path.GetDirectoryName(result.FilePath) ?? result.FilePath;
}
else
{
Directory.CreateDirectory(result.FilePath);
}
}
if (options.StartDocsEtag == Etag.Empty && options.StartAttachmentsEtag == Etag.Empty)
{
ReadLastEtagsFromFile(result);
}
result.FilePath = Path.Combine(result.FilePath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + ".ravendb-incremental-dump");
if (File.Exists(result.FilePath))
{
var counter = 1;
while (true)
{
// ReSharper disable once AssignNullToNotNullAttribute
result.FilePath = Path.Combine(Path.GetDirectoryName(result.FilePath), SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + " - " + counter + ".ravendb-incremental-dump");
if (File.Exists(result.FilePath) == false)
{
break;
}
counter++;
}
}
}
SmugglerExportException lastException = null;
bool ownedStream = exportOptions.ToStream == null;
var stream = exportOptions.ToStream ?? File.Create(result.FilePath);
try
{
await DetectServerSupportedFeatures(exportOptions.From);
}
catch (WebException e)
{
ShowProgress("Failed to query server for supported features. Reason : " + e.Message);
SetLegacyMode(); //could not detect supported features, then run in legacy mode
// lastException = new SmugglerExportException
// {
// LastEtag = Etag.Empty,
// File = ownedStream ? result.FilePath : null
// };
}
try
{
using (var gZipStream = new GZipStream(stream, CompressionMode.Compress,
leaveOpen: true))
using (var streamWriter = new StreamWriter(gZipStream))
{
var jsonWriter = new JsonTextWriter(streamWriter)
{
Formatting = Formatting.Indented
};
jsonWriter.WriteStartObject();
jsonWriter.WritePropertyName("Indexes");
jsonWriter.WriteStartArray();
if (options.OperateOnTypes.HasFlag(ItemType.Indexes))
{
await ExportIndexes(exportOptions.From, jsonWriter);
}
jsonWriter.WriteEndArray();
// used to synchronize max returned values for put/delete operations
var maxEtags = FetchCurrentMaxEtags();
jsonWriter.WritePropertyName("Docs");
jsonWriter.WriteStartArray();
if (options.OperateOnTypes.HasFlag(ItemType.Documents))
{
try
{
result.LastDocsEtag = await ExportDocuments(exportOptions.From, options, jsonWriter, result.LastDocsEtag, maxEtags.LastDocsEtag);
}
catch (SmugglerExportException e)
{
result.LastDocsEtag = e.LastEtag;
e.File = ownedStream ? result.FilePath : null;
lastException = e;
}
}
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("Attachments");
jsonWriter.WriteStartArray();
if (options.OperateOnTypes.HasFlag(ItemType.Attachments) && lastException == null)
{
try
{
result.LastAttachmentsEtag = await ExportAttachments(exportOptions.From, jsonWriter, result.LastAttachmentsEtag, maxEtags.LastAttachmentsEtag);
}
catch (SmugglerExportException e)
{
result.LastAttachmentsEtag = e.LastEtag;
e.File = ownedStream ? result.FilePath : null;
lastException = e;
}
}
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("Transformers");
jsonWriter.WriteStartArray();
if (options.OperateOnTypes.HasFlag(ItemType.Transformers) && lastException == null)
{
await ExportTransformers(exportOptions.From, jsonWriter);
}
jsonWriter.WriteEndArray();
if (options.ExportDeletions)
{
ExportDeletions(jsonWriter, options, result, maxEtags);
}
jsonWriter.WriteEndObject();
streamWriter.Flush();
}
if (options.Incremental)
{
WriteLastEtagsToFile(result, result.FilePath);
}
if (options.ExportDeletions)
{
PurgeTombstones(result);
}
if (lastException != null)
{
throw lastException;
}
return(result);
}
finally
{
if (ownedStream && stream != null)
{
stream.Dispose();
}
}
}
public async Task ShouldExportAndImportConfigurations()
{
using (var exportStream = new MemoryStream())
{
int countOfConfigurations;
using (var store = NewStore())
{
for (int i = 0; i < 100; i++)
{
await store.AsyncFilesCommands.Configuration.SetKeyAsync("items/" + i, new RavenJObject
{
{
"test", "value"
},
{
"test-array", new RavenJArray
{
"item-1", "item-2", "item-3"
}
}
});
}
countOfConfigurations = (await store.AsyncFilesCommands.Configuration.GetKeyNamesAsync(0, 200)).Length;
var exportOptions = new SmugglerExportOptions<FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = store.Url,
DefaultFileSystem = store.DefaultFileSystem
},
ToStream = exportStream
};
await new SmugglerFilesApi().ExportData(exportOptions);
}
using (var import = NewStore(1))
{
exportStream.Position = 0;
var importOptions = new SmugglerImportOptions<FilesConnectionStringOptions>
{
FromStream = exportStream,
To = new FilesConnectionStringOptions()
{
Url = import.Url,
DefaultFileSystem = import.DefaultFileSystem
}
};
await new SmugglerFilesApi().ImportData(importOptions);
Assert.Equal(countOfConfigurations, (await import.AsyncFilesCommands.Configuration.GetKeyNamesAsync(0, 200)).Length);
for (int i = 0; i < 100; i++)
{
Assert.NotNull(await import.AsyncFilesCommands.Configuration.GetKeyAsync<RavenJObject>("items/" + i));
}
}
}
}
public async Task ExportShouldDisableSynchronizationDestinations()
{
using (var exportStream = new MemoryStream())
using (var exportStore = NewStore())
using (var importStore = NewStore(1))
{
await exportStore.AsyncFilesCommands.Synchronization.SetDestinationsAsync(new SynchronizationDestination()
{
ServerUrl = "http://sample.com",
FileSystem = "Sample",
Enabled = true
});
var exportOptions = new SmugglerExportOptions<FilesConnectionStringOptions>
{
From = new FilesConnectionStringOptions
{
Url = exportStore.Url,
DefaultFileSystem = exportStore.DefaultFileSystem
},
ToStream = exportStream
};
await new SmugglerFilesApi().ExportData(exportOptions);
exportStream.Position = 0;
var importOptions = new SmugglerImportOptions<FilesConnectionStringOptions>
{
FromStream = exportStream,
To = new FilesConnectionStringOptions()
{
Url = importStore.Url,
DefaultFileSystem = importStore.DefaultFileSystem
}
};
await new SmugglerFilesApi().ImportData(importOptions);
var destinations = await importStore.AsyncFilesCommands.Synchronization.GetDestinationsAsync();
Assert.Equal(1, destinations.Length);
Assert.Equal("http://sample.com/fs/Sample", destinations[0].Url);
Assert.Equal("Sample", destinations[0].FileSystem);
Assert.False(destinations[0].Enabled);
}
}
public async void Sample3()
{
#region smuggler_api_4
using (EmbeddableDocumentStore store = new EmbeddableDocumentStore
{
DefaultDatabase = "Northwind"
})
{
store.Initialize();
var dataDumper = new DatabaseDataDumper(
store.DocumentDatabase,
new SmugglerDatabaseOptions
{
OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments | ItemType.Transformers,
Incremental = false
});
SmugglerExportOptions<RavenConnectionStringOptions> exportOptions = new SmugglerExportOptions<RavenConnectionStringOptions>
{
From = new EmbeddedRavenConnectionStringOptions(),
ToFile = "dump.raven"
};
await dataDumper.ExportData(exportOptions);
}
#endregion
}
