public static void ReadLastEtagsFromFile(ExportDataResult result)
{
var log = LogManager.GetCurrentClassLogger();
var etagFileLocation = Path.Combine(result.FilePath, IncrementalExportStateFile);
if (!File.Exists(etagFileLocation))
{
return;
}
using (var streamReader = new StreamReader(new FileStream(etagFileLocation, FileMode.Open)))
using (var jsonReader = new JsonTextReader(streamReader))
{
RavenJObject ravenJObject;
try
{
ravenJObject = RavenJObject.Load(jsonReader);
}
catch (Exception e)
{
log.WarnException("Could not parse etag document from file : " + etagFileLocation + ", ignoring, will start from scratch", e);
return;
}
result.LastDocsEtag = Etag.Parse(ravenJObject.Value <string>("LastDocEtag"));
result.LastAttachmentsEtag = Etag.Parse(ravenJObject.Value <string>("LastAttachmentEtag"));
result.LastDocDeleteEtag = Etag.Parse(ravenJObject.Value <string>("LastDocDeleteEtag") ?? Etag.Empty.ToString());
result.LastAttachmentsDeleteEtag =
Etag.Parse(ravenJObject.Value <string>("LastAttachentsDeleteEtag") ?? Etag.Empty.ToString());
}
}
protected override void PurgeTombstones(ExportDataResult result)
{
database.TransactionalStorage.Batch(accessor =>
{
// since remove all before is inclusive, but we want last etag for function FetchCurrentMaxEtags we modify ranges
accessor.Lists.RemoveAllBefore(Constants.RavenPeriodicBackupsDocsTombstones, result.LastDocDeleteEtag.IncrementBy(-1));
accessor.Lists.RemoveAllBefore(Constants.RavenPeriodicBackupsAttachmentsTombstones, result.LastAttachmentsDeleteEtag.IncrementBy(-1));
});
}
/// <summary>
///
/// </summary>
/// <param name="jsonWriter"></param>
/// <param name="options"></param>
/// <param name="result"></param>
/// <param name="maxEtags">Max etags are inclusive</param>
protected async override void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result, LastEtagsInfo maxEtags)
{
jsonWriter.WritePropertyName("DocsDeletions");
jsonWriter.WriteStartArray();
result.LastDocDeleteEtag = await ExportDocumentsDeletion(options, jsonWriter, result.LastDocDeleteEtag, maxEtags.LastDocDeleteEtag.IncrementBy(1));
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("AttachmentsDeletions");
jsonWriter.WriteStartArray();
result.LastAttachmentsDeleteEtag = await ExportAttachmentsDeletion(options, jsonWriter, result.LastAttachmentsDeleteEtag, maxEtags.LastAttachmentsDeleteEtag.IncrementBy(1));
jsonWriter.WriteEndArray();
}
public static void WriteLastEtagsToFile(ExportDataResult result, string backupPath)
{
// ReSharper disable once AssignNullToNotNullAttribute
var etagFileLocation = Path.Combine(Path.GetDirectoryName(backupPath), IncrementalExportStateFile);
using (var streamWriter = new StreamWriter(File.Create(etagFileLocation)))
{
new RavenJObject
{
{ "LastDocEtag", result.LastDocsEtag.ToString() },
{ "LastAttachmentEtag", result.LastAttachmentsEtag.ToString() },
{ "LastDocDeleteEtag", result.LastDocDeleteEtag.ToString() },
{ "LastAttachentsDeleteEtag", result.LastAttachmentsDeleteEtag.ToString() }
}.WriteTo(new JsonTextWriter(streamWriter));
streamWriter.Flush();
}
}
public virtual async Task <ExportDataResult> ExportData(SmugglerExportOptions exportOptions, SmugglerOptions options)
{
SetSmugglerOptions(options);
var result = new ExportDataResult
{
FilePath = exportOptions.ToFile,
LastAttachmentsEtag = options.StartAttachmentsEtag,
LastDocsEtag = options.StartDocsEtag,
LastDocDeleteEtag = options.StartDocsDeletionEtag,
LastAttachmentsDeleteEtag = options.StartAttachmentsDeletionEtag
};
if (options.Incremental)
{
if (Directory.Exists(result.FilePath) == false)
{
if (File.Exists(result.FilePath))
{
result.FilePath = Path.GetDirectoryName(result.FilePath) ?? result.FilePath;
}
else
{
Directory.CreateDirectory(result.FilePath);
}
}
if (options.StartDocsEtag == Etag.Empty && options.StartAttachmentsEtag == Etag.Empty)
{
ReadLastEtagsFromFile(result);
}
result.FilePath = Path.Combine(result.FilePath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + ".ravendb-incremental-dump");
if (File.Exists(result.FilePath))
{
var counter = 1;
while (true)
{
// ReSharper disable once AssignNullToNotNullAttribute
result.FilePath = Path.Combine(Path.GetDirectoryName(result.FilePath), SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + " - " + counter + ".ravendb-incremental-dump");
if (File.Exists(result.FilePath) == false)
{
break;
}
counter++;
}
}
}
SmugglerExportException lastException = null;
bool ownedStream = exportOptions.ToStream == null;
var stream = exportOptions.ToStream ?? File.Create(result.FilePath);
try
{
await DetectServerSupportedFeatures(exportOptions.From);
}
catch (WebException e)
{
ShowProgress("Failed to query server for supported features. Reason : " + e.Message);
SetLegacyMode(); //could not detect supported features, then run in legacy mode
// lastException = new SmugglerExportException
// {
// LastEtag = Etag.Empty,
// File = ownedStream ? result.FilePath : null
// };
}
try
{
using (var gZipStream = new GZipStream(stream, CompressionMode.Compress,
leaveOpen: true))
using (var streamWriter = new StreamWriter(gZipStream))
{
var jsonWriter = new JsonTextWriter(streamWriter)
{
Formatting = Formatting.Indented
};
jsonWriter.WriteStartObject();
jsonWriter.WritePropertyName("Indexes");
jsonWriter.WriteStartArray();
if (options.OperateOnTypes.HasFlag(ItemType.Indexes))
{
await ExportIndexes(exportOptions.From, jsonWriter);
}
jsonWriter.WriteEndArray();
// used to synchronize max returned values for put/delete operations
var maxEtags = FetchCurrentMaxEtags();
jsonWriter.WritePropertyName("Docs");
jsonWriter.WriteStartArray();
if (options.OperateOnTypes.HasFlag(ItemType.Documents))
{
try
{
result.LastDocsEtag = await ExportDocuments(exportOptions.From, options, jsonWriter, result.LastDocsEtag, maxEtags.LastDocsEtag);
}
catch (SmugglerExportException e)
{
result.LastDocsEtag = e.LastEtag;
e.File = ownedStream ? result.FilePath : null;
lastException = e;
}
}
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("Attachments");
jsonWriter.WriteStartArray();
if (options.OperateOnTypes.HasFlag(ItemType.Attachments) && lastException == null)
{
try
{
result.LastAttachmentsEtag = await ExportAttachments(exportOptions.From, jsonWriter, result.LastAttachmentsEtag, maxEtags.LastAttachmentsEtag);
}
catch (SmugglerExportException e)
{
result.LastAttachmentsEtag = e.LastEtag;
e.File = ownedStream ? result.FilePath : null;
lastException = e;
}
}
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("Transformers");
jsonWriter.WriteStartArray();
if (options.OperateOnTypes.HasFlag(ItemType.Transformers) && lastException == null)
{
await ExportTransformers(exportOptions.From, jsonWriter);
}
jsonWriter.WriteEndArray();
if (options.ExportDeletions)
{
ExportDeletions(jsonWriter, options, result, maxEtags);
}
jsonWriter.WriteEndObject();
streamWriter.Flush();
}
if (options.Incremental)
{
WriteLastEtagsToFile(result, result.FilePath);
}
if (options.ExportDeletions)
{
PurgeTombstones(result);
}
if (lastException != null)
{
throw lastException;
}
return(result);
}
finally
{
if (ownedStream && stream != null)
{
stream.Dispose();
}
}
}
protected abstract void PurgeTombstones(ExportDataResult result);
protected abstract void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result, LastEtagsInfo maxEtagsToFetch);
public static void ReadLastEtagsFromFile(ExportDataResult result)
{
var log = LogManager.GetCurrentClassLogger();
var etagFileLocation = Path.Combine(result.FilePath, IncrementalExportStateFile);
if (!File.Exists(etagFileLocation))
return;
using (var streamReader = new StreamReader(new FileStream(etagFileLocation, FileMode.Open)))
using (var jsonReader = new JsonTextReader(streamReader))
{
RavenJObject ravenJObject;
try
{
ravenJObject = RavenJObject.Load(jsonReader);
}
catch (Exception e)
{
log.WarnException("Could not parse etag document from file : " + etagFileLocation + ", ignoring, will start from scratch", e);
return;
}
result.LastDocsEtag = Etag.Parse(ravenJObject.Value<string>("LastDocEtag"));
result.LastAttachmentsEtag = Etag.Parse(ravenJObject.Value<string>("LastAttachmentEtag"));
result.LastDocDeleteEtag = Etag.Parse(ravenJObject.Value<string>("LastDocDeleteEtag") ?? Etag.Empty.ToString());
result.LastAttachmentsDeleteEtag =
Etag.Parse(ravenJObject.Value<string>("LastAttachentsDeleteEtag") ?? Etag.Empty.ToString());
}
}
public new void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result, LastEtagsInfo maxEtags)
{
base.ExportDeletions(jsonWriter, options, result, maxEtags);
}
protected override void PurgeTombstones(ExportDataResult result)
{
throw new NotImplementedException("Purge tombstones is not supported for Command Line Smuggler");
}
public async Task CanHandleAttachmentExceptionsGracefully_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
var server = GetNewServer();
var allowDownload = false;
var forwarder = new ProxyServer(8070, 8079)
{
VetoTransfer = (totalRead, buffer) =>
{
var payload = System.Text.Encoding.UTF8.GetString(buffer.Array, buffer.Offset, buffer.Count);
return payload.Contains("GET /static/users/678 ") && allowDownload == false;
}
};
try
{
string databaseName;
using (var store = new DocumentStore
{
Url = "http://localhost:8079"
})
{
databaseName = store.DefaultDatabase;
store.Initialize();
InsertAttachments(store, 2000);
}
var options = new SmugglerOptions
{
Limit = 1500,
Incremental = true
};
var dumper = new SmugglerApi();
var allAttachments = new List<RavenJObject>();
ExportDataResult exportResult = null;
try
{
exportResult = dumper.ExportData(new SmugglerExportOptions
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8070",
DefaultDatabase = databaseName,
}
}, options).Result;
Assert.False(true, "Previous op should throw.");
}
catch (AggregateException e)
{
var inner = (SmugglerExportException)e.ExtractSingleInnerException();
exportResult = new ExportDataResult
{
FilePath = inner.File
};
}
allowDownload = true;
using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileStream, CompressionMode.Decompress))
{
var chunk1 = RavenJToken.TryLoad(stream) as RavenJObject;
var att1 = chunk1["Attachments"] as RavenJArray;
allAttachments.AddRange(att1.Values<RavenJObject>());
}
exportResult = await dumper.ExportData(new SmugglerExportOptions
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8070",
DefaultDatabase = databaseName,
}
}, options);
using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileStream, CompressionMode.Decompress))
{
var chunk2 = RavenJToken.TryLoad(stream) as RavenJObject;
var attr2 = chunk2["Attachments"] as RavenJArray;
allAttachments.AddRange(attr2.Values<RavenJObject>());
}
Assert.Equal(2000, allAttachments.Count());
IOExtensions.DeleteDirectory(backupPath);
}
finally
{
forwarder.Dispose();
server.Dispose();
}
}
public async Task CanHandleDocumentExceptionsGracefully_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
var server = GetNewServer(databaseName: Constants.SystemDatabase);
var alreadyReset = false;
var forwarder = new ProxyServer(8070, 8079)
{
VetoTransfer = (totalRead, buffer) =>
{
if (alreadyReset == false && totalRead > 25000)
{
alreadyReset = true;
return true;
}
return false;
}
};
try
{
string databaseName;
using (var store = new DocumentStore
{
Url = "http://localhost:8079"
})
{
databaseName = store.DefaultDatabase;
store.Initialize();
InsertUsers(store, 0, 2000);
}
var options = new SmugglerOptions
{
Limit = 1900,
Incremental = true
};
var dumper = new SmugglerApi();
var allDocs = new List<RavenJObject>();
ExportDataResult exportResult = null;
try
{
exportResult = await dumper.ExportData(new SmugglerExportOptions
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8070",
DefaultDatabase = databaseName,
}
}, options);
Assert.False(true, "Previous op should throw.");
}
catch (SmugglerExportException e)
{
exportResult = new ExportDataResult
{
FilePath = e.File
};
}
using (var fileSteam = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileSteam, CompressionMode.Decompress))
{
var chunk1 = RavenJToken.TryLoad(stream) as RavenJObject;
var doc1 = chunk1["Docs"] as RavenJArray;
allDocs.AddRange(doc1.Values<RavenJObject>());
}
exportResult = await dumper.ExportData(new SmugglerExportOptions
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8070",
DefaultDatabase = databaseName,
}
}, options);
using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileStream, CompressionMode.Decompress))
{
var chunk2 = RavenJToken.TryLoad(stream) as RavenJObject;
var doc2 = chunk2["Docs"] as RavenJArray;
allDocs.AddRange(doc2.Values<RavenJObject>());
}
Assert.Equal(2000, allDocs.Count(d => (d.Value<string>("Name") ?? String.Empty).StartsWith("User")));
}
finally
{
forwarder.Dispose();
server.Dispose();
IOExtensions.DeleteDirectory(backupPath);
}
}
public virtual async Task<ExportDataResult> ExportData(SmugglerExportOptions exportOptions, SmugglerOptions options)
{
SetSmugglerOptions(options);
var result = new ExportDataResult
{
FilePath = exportOptions.ToFile,
LastAttachmentsEtag = options.StartAttachmentsEtag,
LastDocsEtag = options.StartDocsEtag,
LastDocDeleteEtag = options.StartDocsDeletionEtag,
LastAttachmentsDeleteEtag = options.StartAttachmentsDeletionEtag
};
if (options.Incremental)
{
if (Directory.Exists(result.FilePath) == false)
{
if (File.Exists(result.FilePath))
result.FilePath = Path.GetDirectoryName(result.FilePath) ?? result.FilePath;
else
Directory.CreateDirectory(result.FilePath);
}
if (options.StartDocsEtag == Etag.Empty && options.StartAttachmentsEtag == Etag.Empty)
{
ReadLastEtagsFromFile(result);
}
result.FilePath = Path.Combine(result.FilePath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + ".ravendb-incremental-dump");
if (File.Exists(result.FilePath))
{
var counter = 1;
while (true)
{
// ReSharper disable once AssignNullToNotNullAttribute
result.FilePath = Path.Combine(Path.GetDirectoryName(result.FilePath), SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + " - " + counter + ".ravendb-incremental-dump");
if (File.Exists(result.FilePath) == false)
break;
counter++;
}
}
}
SmugglerExportException lastException = null;
bool ownedStream = exportOptions.ToStream == null;
var stream = exportOptions.ToStream ?? File.Create(result.FilePath);
try
{
await DetectServerSupportedFeatures(exportOptions.From);
}
catch (WebException e)
{
ShowProgress("Failed to query server for supported features. Reason : " + e.Message);
SetLegacyMode(); //could not detect supported features, then run in legacy mode
// lastException = new SmugglerExportException
// {
// LastEtag = Etag.Empty,
// File = ownedStream ? result.FilePath : null
// };
}
try
{
using (var gZipStream = new GZipStream(stream, CompressionMode.Compress,
leaveOpen: true))
using (var streamWriter = new StreamWriter(gZipStream))
{
var jsonWriter = new JsonTextWriter(streamWriter)
{
Formatting = Formatting.Indented
};
jsonWriter.WriteStartObject();
jsonWriter.WritePropertyName("Indexes");
jsonWriter.WriteStartArray();
if (options.OperateOnTypes.HasFlag(ItemType.Indexes))
{
await ExportIndexes(exportOptions.From,jsonWriter);
}
jsonWriter.WriteEndArray();
// used to synchronize max returned values for put/delete operations
var maxEtags = FetchCurrentMaxEtags();
jsonWriter.WritePropertyName("Docs");
jsonWriter.WriteStartArray();
if (options.OperateOnTypes.HasFlag(ItemType.Documents))
{
try
{
result.LastDocsEtag = await ExportDocuments(exportOptions.From,options, jsonWriter, result.LastDocsEtag, maxEtags.LastDocsEtag);
}
catch (SmugglerExportException e)
{
result.LastDocsEtag = e.LastEtag;
e.File = ownedStream ? result.FilePath : null;
lastException = e;
}
}
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("Attachments");
jsonWriter.WriteStartArray();
if (options.OperateOnTypes.HasFlag(ItemType.Attachments) && lastException == null)
{
try
{
result.LastAttachmentsEtag = await ExportAttachments(exportOptions.From, jsonWriter, result.LastAttachmentsEtag, maxEtags.LastAttachmentsEtag);
}
catch (SmugglerExportException e)
{
result.LastAttachmentsEtag = e.LastEtag;
e.File = ownedStream ? result.FilePath : null;
lastException = e;
}
}
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("Transformers");
jsonWriter.WriteStartArray();
if (options.OperateOnTypes.HasFlag(ItemType.Transformers) && lastException == null)
{
await ExportTransformers(exportOptions.From,jsonWriter);
}
jsonWriter.WriteEndArray();
if (options.ExportDeletions)
{
ExportDeletions(jsonWriter, options, result, maxEtags);
}
jsonWriter.WriteEndObject();
streamWriter.Flush();
}
if (options.Incremental)
WriteLastEtagsToFile(result, result.FilePath);
if (options.ExportDeletions)
{
PurgeTombstones(result);
}
if (lastException != null)
throw lastException;
return result;
}
finally
{
if (ownedStream && stream != null)
stream.Dispose();
}
}
protected abstract void PurgeTombstones(ExportDataResult result);
protected abstract void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result, LastEtagsInfo maxEtagsToFetch);
public static void WriteLastEtagsToFile(ExportDataResult result, string backupPath)
{
// ReSharper disable once AssignNullToNotNullAttribute
var etagFileLocation = Path.Combine(Path.GetDirectoryName(backupPath), IncrementalExportStateFile);
using (var streamWriter = new StreamWriter(File.Create(etagFileLocation)))
{
new RavenJObject
{
{"LastDocEtag", result.LastDocsEtag.ToString()},
{"LastAttachmentEtag", result.LastAttachmentsEtag.ToString()},
{"LastDocDeleteEtag", result.LastDocDeleteEtag.ToString()},
{"LastAttachentsDeleteEtag", result.LastAttachmentsDeleteEtag.ToString()}
}.WriteTo(new JsonTextWriter(streamWriter));
streamWriter.Flush();
}
}
public void CanProperlyReadLastEtagUsingPreviousFormat()
{
var backupPath = NewDataPath("BackupFolder", forceCreateDir:true);
var etagFileLocation = Path.Combine(Path.GetDirectoryName(backupPath), "IncrementalExport.state.json");
using (var streamWriter = new StreamWriter(File.Create(etagFileLocation)))
{
new RavenJObject
{
{"LastDocEtag", Etag.Parse("00000000-0000-0000-0000-000000000001").ToString()},
{"LastAttachmentEtag", Etag.Parse("00000000-0000-0000-0000-000000000002").ToString()}
}.WriteTo(new JsonTextWriter(streamWriter));
streamWriter.Flush();
}
var result = new ExportDataResult
{
FilePath = backupPath
};
SmugglerApiBase.ReadLastEtagsFromFile(result);
Assert.Equal("00000000-0000-0000-0000-000000000001", result.LastDocsEtag.ToString());
Assert.Equal("00000000-0000-0000-0000-000000000002", result.LastAttachmentsEtag.ToString());
Assert.Equal(Etag.Empty, result.LastDocDeleteEtag);
Assert.Equal(Etag.Empty, result.LastAttachmentsDeleteEtag);
}
protected override void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result,
LastEtagsInfo maxEtagsToFetch)
{
throw new NotImplementedException("Exporting deletions is not supported for Command Line Smuggler");
}
public async Task DataDumperExportHandlesMaxEtagCorrectly()
{
using (var store = NewDocumentStore())
{
using (var session = store.OpenSession())
{
for (var i = 0; i < 10; i++)
{
session.Store(new User { Name = "oren #" + (i+1) });
}
session.SaveChanges();
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.DocumentDatabase)
{
SmugglerOptions = new SmugglerOptions()
};
var startEtag = store.DocumentDatabase.Statistics.LastDocEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(new SmugglerOptions(), writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", exportedDocs.Last().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.DocumentDatabase)
{
SmugglerOptions = new SmugglerOptions()
};
var startEtag = store.DocumentDatabase.Statistics.LastDocEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(new SmugglerOptions(), writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", exportedDocs.Last().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", lastEtag.ToString());
}
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.PutAttachment("attach/" + (i+1), null, new MemoryStream(new [] { (byte)i }), new RavenJObject());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.DocumentDatabase)
{
SmugglerOptions = new SmugglerOptions()
};
var startEtag = store.DocumentDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", exportedAttachments.Last().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.DocumentDatabase)
{
SmugglerOptions = new SmugglerOptions()
};
var startEtag = store.DocumentDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", exportedAttachments.Last().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", lastEtag.ToString());
}
WaitForIndexing(store);
store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery()
{
Query = "Tag:Users"
}).WaitForCompletion();
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.DeleteAttachment("attach/" + (i+1), null);
}
Etag user6DeletionEtag = null, user9DeletionEtag = null, attach5DeletionEtag = null, attach7DeletionEtag = null;
WaitForUserToContinueTheTest(store);
store.DocumentDatabase.TransactionalStorage.Batch(accessor =>
{
user6DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/6").Etag;
user9DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/9").Etag;
attach5DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/5").Etag;
attach7DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/7").Etag;
});
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.DocumentDatabase)
{
SmugglerOptions = new SmugglerOptions()
};
writer.WriteStartObject();
var lastEtags = new LastEtagsInfo();
var exportResult = new ExportDataResult
{
LastDocDeleteEtag = user6DeletionEtag,
LastAttachmentsDeleteEtag = attach5DeletionEtag
};
lastEtags.LastDocDeleteEtag = user9DeletionEtag;
lastEtags.LastAttachmentsDeleteEtag = attach7DeletionEtag;
dumper.ExportDeletions(writer, new SmugglerOptions(), exportResult, lastEtags);
writer.WriteEndObject();
writer.Flush();
// read exported content
var exportJson = RavenJObject.Parse(textStream.GetStringBuilder().ToString());
var docsKeys =
exportJson.Value<RavenJArray>("DocsDeletions").Select(x => x.Value<string>("Key")).ToArray();
var attachmentsKeys =
exportJson.Value<RavenJArray>("AttachmentsDeletions")
.Select(x => x.Value<string>("Key"))
.ToArray();
Assert.Equal(new [] { "users/7", "users/8", "users/9" }, docsKeys);
Assert.Equal(new [] { "attach/6", "attach/7" }, attachmentsKeys);
}
}
}
public abstract Task ExportDeletions(JsonTextWriter jsonWriter, ExportDataResult result, LastEtagsInfo maxEtagsToFetch);