public void CanProperlyReadLastEtagUsingPreviousFormat()
{
var backupPath = NewDataPath("BackupFolder", forceCreateDir: true);
var etagFileLocation = Path.Combine(Path.GetDirectoryName(backupPath), "IncrementalExport.state.json");
using (var streamWriter = new StreamWriter(File.Create(etagFileLocation)))
{
new RavenJObject
{
{ "LastDocEtag", Etag.Parse("00000000-0000-0000-0000-000000000001").ToString() },
{ "LastAttachmentEtag", Etag.Parse("00000000-0000-0000-0000-000000000002").ToString() }
}.WriteTo(new JsonTextWriter(streamWriter));
streamWriter.Flush();
}
var result = new ExportDataResult
{
FilePath = backupPath
};
SmugglerApiBase.ReadLastEtagsFromFile(result);
Assert.Equal("00000000-0000-0000-0000-000000000001", result.LastDocsEtag.ToString());
Assert.Equal("00000000-0000-0000-0000-000000000002", result.LastAttachmentsEtag.ToString());
Assert.Equal(Etag.Empty, result.LastDocDeleteEtag);
Assert.Equal(Etag.Empty, result.LastAttachmentsDeleteEtag);
}
public void PurgeTombstones(ExportDataResult result)
{
database.TransactionalStorage.Batch(accessor =>
{
// since remove all before is inclusive, but we want last etag for function FetchCurrentMaxEtags we modify ranges
accessor.Lists.RemoveAllBefore(Constants.RavenPeriodicExportsDocsTombstones, result.LastDocDeleteEtag.IncrementBy(-1));
accessor.Lists.RemoveAllBefore(Constants.RavenPeriodicExportsAttachmentsTombstones, result.LastAttachmentsDeleteEtag.IncrementBy(-1));
});
}
public override async Task ExportDeletions(JsonTextWriter jsonWriter, ExportDataResult result, LastEtagsInfo maxEtagsToFetch)
{
jsonWriter.WritePropertyName("DocsDeletions");
jsonWriter.WriteStartArray();
result.LastDocDeleteEtag = await Operations.ExportDocumentsDeletion(jsonWriter, result.LastDocDeleteEtag, maxEtagsToFetch.LastDocDeleteEtag.IncrementBy(1));
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("AttachmentsDeletions");
jsonWriter.WriteStartArray();
result.LastAttachmentsDeleteEtag = await Operations.ExportAttachmentsDeletion(jsonWriter, result.LastAttachmentsDeleteEtag, maxEtagsToFetch.LastAttachmentsDeleteEtag.IncrementBy(1));
jsonWriter.WriteEndArray();
}
public override async Task ExportDeletions(JsonTextWriter jsonWriter, ExportDataResult result, LastEtagsInfo maxEtagsToFetch)
{
jsonWriter.WritePropertyName("DocsDeletions");
jsonWriter.WriteStartArray();
result.LastDocDeleteEtag = await Operations.ExportDocumentsDeletion(jsonWriter, result.LastDocDeleteEtag, maxEtagsToFetch.LastDocDeleteEtag.IncrementBy(1));
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("AttachmentsDeletions");
jsonWriter.WriteStartArray();
result.LastAttachmentsDeleteEtag = await Operations.ExportAttachmentsDeletion(jsonWriter, result.LastAttachmentsDeleteEtag, maxEtagsToFetch.LastAttachmentsDeleteEtag.IncrementBy(1));
jsonWriter.WriteEndArray();
}
public async Task CanHandleDocumentExceptionsGracefully_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
var server = GetNewServer(databaseName: Constants.SystemDatabase);
var alreadyReset = false;
var forwarder = new ProxyServer(8070, 8079)
{
VetoTransfer = (totalRead, buffer) =>
{
if (alreadyReset == false && totalRead > 25000)
{
alreadyReset = true;
return true;
}
return false;
}
};
try
{
string databaseName;
using (var store = new DocumentStore
{
Url = "http://localhost:8079"
})
{
databaseName = store.DefaultDatabase;
store.Initialize();
InsertUsers(store, 0, 2000);
}
var dumper = new SmugglerApi {SmugglerOptions = {Limit = 1900, Incremental = true}};
var allDocs = new List<RavenJObject>();
ExportDataResult exportResult = null;
try
{
exportResult = await dumper.ExportData(new SmugglerExportOptions
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8070",
DefaultDatabase = databaseName,
}
});
Assert.False(true, "Previous op should throw.");
}
catch (SmugglerExportException e)
{
exportResult = new ExportDataResult
{
FilePath = e.File
};
}
using (var fileSteam = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileSteam, CompressionMode.Decompress))
{
var chunk1 = RavenJToken.TryLoad(stream) as RavenJObject;
var doc1 = chunk1["Docs"] as RavenJArray;
allDocs.AddRange(doc1.Values<RavenJObject>());
}
exportResult = await dumper.ExportData(new SmugglerExportOptions
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8070",
DefaultDatabase = databaseName,
}
});
using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileStream, CompressionMode.Decompress))
{
var chunk2 = RavenJToken.TryLoad(stream) as RavenJObject;
var doc2 = chunk2["Docs"] as RavenJArray;
allDocs.AddRange(doc2.Values<RavenJObject>());
}
Assert.Equal(2000, allDocs.Count(d => (d.Value<string>("Name") ?? String.Empty).StartsWith("User")));
}
finally
{
forwarder.Dispose();
server.Dispose();
IOExtensions.DeleteDirectory(backupPath);
}
}
public virtual async Task<ExportDataResult> ExportData(SmugglerExportOptions exportOptions)
{
Operations.Configure(SmugglerOptions);
Operations.Initialize(SmugglerOptions);
var result = new ExportDataResult
{
FilePath = exportOptions.ToFile,
LastAttachmentsEtag = SmugglerOptions.StartAttachmentsEtag,
LastDocsEtag = SmugglerOptions.StartDocsEtag,
LastDocDeleteEtag = SmugglerOptions.StartDocsDeletionEtag,
LastAttachmentsDeleteEtag = SmugglerOptions.StartAttachmentsDeletionEtag
};
if (SmugglerOptions.Incremental)
{
if (Directory.Exists(result.FilePath) == false)
{
if (File.Exists(result.FilePath))
result.FilePath = Path.GetDirectoryName(result.FilePath) ?? result.FilePath;
else
Directory.CreateDirectory(result.FilePath);
}
if (SmugglerOptions.StartDocsEtag == Etag.Empty && SmugglerOptions.StartAttachmentsEtag == Etag.Empty)
{
ReadLastEtagsFromFile(result);
}
result.FilePath = Path.Combine(result.FilePath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + ".ravendb-incremental-dump");
if (File.Exists(result.FilePath))
{
var counter = 1;
while (true)
{
// ReSharper disable once AssignNullToNotNullAttribute
result.FilePath = Path.Combine(Path.GetDirectoryName(result.FilePath), SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + " - " + counter + ".ravendb-incremental-dump");
if (File.Exists(result.FilePath) == false)
break;
counter++;
}
}
}
SmugglerExportException lastException = null;
bool ownedStream = exportOptions.ToStream == null;
var stream = exportOptions.ToStream ?? File.Create(result.FilePath);
try
{
await DetectServerSupportedFeatures(exportOptions.From);
}
catch (WebException e)
{
Operations.ShowProgress("Failed to query server for supported features. Reason : " + e.Message);
SetLegacyMode(); //could not detect supported features, then run in legacy mode
// lastException = new SmugglerExportException
// {
// LastEtag = Etag.Empty,
// File = ownedStream ? result.FilePath : null
// };
}
try
{
using (var gZipStream = new GZipStream(stream, CompressionMode.Compress,
leaveOpen: true))
using (var streamWriter = new StreamWriter(gZipStream))
{
var jsonWriter = new JsonTextWriter(streamWriter)
{
Formatting = Formatting.Indented
};
jsonWriter.WriteStartObject();
jsonWriter.WritePropertyName("Indexes");
jsonWriter.WriteStartArray();
if (SmugglerOptions.OperateOnTypes.HasFlag(ItemType.Indexes))
{
await ExportIndexes(exportOptions.From, jsonWriter);
}
jsonWriter.WriteEndArray();
// used to synchronize max returned values for put/delete operations
var maxEtags = Operations.FetchCurrentMaxEtags();
jsonWriter.WritePropertyName("Docs");
jsonWriter.WriteStartArray();
if (SmugglerOptions.OperateOnTypes.HasFlag(ItemType.Documents))
{
try
{
result.LastDocsEtag = await ExportDocuments(exportOptions.From, jsonWriter, result.LastDocsEtag, maxEtags.LastDocsEtag);
}
catch (SmugglerExportException e)
{
result.LastDocsEtag = e.LastEtag;
e.File = ownedStream ? result.FilePath : null;
lastException = e;
}
}
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("Attachments");
jsonWriter.WriteStartArray();
if (SmugglerOptions.OperateOnTypes.HasFlag(ItemType.Attachments) && lastException == null)
{
try
{
result.LastAttachmentsEtag = await ExportAttachments(exportOptions.From, jsonWriter, result.LastAttachmentsEtag, maxEtags.LastAttachmentsEtag);
}
catch (SmugglerExportException e)
{
result.LastAttachmentsEtag = e.LastEtag;
e.File = ownedStream ? result.FilePath : null;
lastException = e;
}
}
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("Transformers");
jsonWriter.WriteStartArray();
if (SmugglerOptions.OperateOnTypes.HasFlag(ItemType.Transformers) && lastException == null)
{
await ExportTransformers(exportOptions.From, jsonWriter);
}
jsonWriter.WriteEndArray();
if (SmugglerOptions.ExportDeletions)
{
await ExportDeletions(jsonWriter, result, maxEtags);
}
await ExportIdentities(jsonWriter, SmugglerOptions.OperateOnTypes);
jsonWriter.WriteEndObject();
streamWriter.Flush();
}
if (SmugglerOptions.Incremental)
WriteLastEtagsToFile(result, result.FilePath);
if (SmugglerOptions.ExportDeletions)
{
Operations.PurgeTombstones(result);
}
if (lastException != null)
throw lastException;
return result;
}
finally
{
if (ownedStream && stream != null)
stream.Dispose();
}
}
public abstract Task ExportDeletions(JsonTextWriter jsonWriter, ExportDataResult result, LastEtagsInfo maxEtagsToFetch);
public void PurgeTombstones(ExportDataResult result)
{
database.TransactionalStorage.Batch(accessor =>
{
// since remove all before is inclusive, but we want last etag for function FetchCurrentMaxEtags we modify ranges
accessor.Lists.RemoveAllBefore(Constants.RavenPeriodicExportsDocsTombstones, result.LastDocDeleteEtag.IncrementBy(-1));
accessor.Lists.RemoveAllBefore(Constants.RavenPeriodicExportsAttachmentsTombstones, result.LastAttachmentsDeleteEtag.IncrementBy(-1));
});
}
protected override void PurgeTombstones(ExportDataResult result)
{
throw new NotImplementedException();
}
public FileContentResult ExportProfile(string startDate, string endDate)
{
try
{
List <ExperienceProfileDetail> export = new List <ExperienceProfileDetail>();
string[] paramiters = { WebVisit.DefaultFacetKey, LocaleInfo.DefaultFacetKey, IpInfo.DefaultFacetKey };
var interactionsobj = new RelatedInteractionsExpandOptions(paramiters);
DateTime sdate = DateTime.Now;
DateTime edate = DateTime.Now;
if (!string.IsNullOrEmpty(startDate) && DateTime.TryParse(startDate, out sdate))
{
interactionsobj.StartDateTime = sdate.AddDays(1).ToUniversalTime();
}
else
{
interactionsobj.StartDateTime = DateTime.UtcNow.AddDays(-30);
}
if (!string.IsNullOrEmpty(endDate) && DateTime.TryParse(endDate, out edate))
{
interactionsobj.EndDateTime = edate.AddDays(1).ToUniversalTime();
}
else
{
interactionsobj.EndDateTime = DateTime.UtcNow;
}
interactionsobj.Limit = int.MaxValue;
ExportDataResult exportResult;
using (Sitecore.XConnect.Client.XConnectClient client = SitecoreXConnectClientConfiguration.GetClient())
{
List <Contact> contactList = new List <Contact>();
bool includeAnonymous = false;
var settingvalue = Sitecore.Configuration.Settings.GetSetting("IncludeAnonymous");
bool.TryParse(settingvalue, out includeAnonymous);
var Contactsid = client.Contacts.Where(d => d.Interactions.Any(i => i.StartDateTime >= interactionsobj.StartDateTime && i.EndDateTime <= interactionsobj.EndDateTime)).AsAsyncQueryable();
if (!includeAnonymous)
{
Contactsid = Contactsid.Where(c => c.Identifiers.Any(t => t.IdentifierType == Sitecore.XConnect.ContactIdentifierType.Known));
}
contactList = Contactsid.ToList().Result;
var references = new List <IEntityReference <Sitecore.XConnect.Contact> >();
references.AddRange(contactList);
var contacts = client.Get <Contact>(references, new Sitecore.XConnect.ContactExpandOptions(PersonalInformation.DefaultFacetKey)
{
Interactions = interactionsobj
}.Expand <EmailAddressList>().Expand <AddressList>().Expand <PhoneNumberList>());
exportResult = new ExportDataResult()
{
Content = GenerateFileContent(contacts),
FileName = GenerateFileName(interactionsobj.StartDateTime.Value, interactionsobj.EndDateTime.Value),
MediaType = "application/octet-stream"
};
}
FileContentResult fileresult;
if (exportResult != null)
{
fileresult = new FileContentResult(exportResult.Content, exportResult.MediaType);
fileresult.FileDownloadName = exportResult.FileName;
}
else
{
fileresult = new FileContentResult(null, "application/octet-stream")
{
FileDownloadName = "NoData.csv"
};
}
return(fileresult);
}
catch (Exception ex)
{
Log.Error("ERROR IN EXPORT PROFILE GETDATA:", ex.Message);
return(new FileContentResult(null, "application/octet-stream")
{
FileDownloadName = "NoData.csv"
});
}
}
public async Task CanHandleAttachmentExceptionsGracefully_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
var server = GetNewServer();
var allowDownload = false;
var forwarder = new ProxyServer(8070, 8079)
{
VetoTransfer = (totalRead, buffer) =>
{
var payload = System.Text.Encoding.UTF8.GetString(buffer.Array, buffer.Offset, buffer.Count);
return payload.Contains("GET /static/users/678 ") && allowDownload == false;
}
};
try
{
string databaseName;
using (var store = new DocumentStore
{
Url = "http://localhost:8079"
})
{
databaseName = store.DefaultDatabase;
store.Initialize();
InsertAttachments(store, 2000);
}
var dumper = new SmugglerApi {SmugglerOptions = {Limit = 1500, Incremental = true}};
var allAttachments = new List<RavenJObject>();
ExportDataResult exportResult = null;
try
{
exportResult = dumper.ExportData(new SmugglerExportOptions
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8070",
DefaultDatabase = databaseName,
}
}).Result;
Assert.False(true, "Previous op should throw.");
}
catch (AggregateException e)
{
var inner = (SmugglerExportException)e.ExtractSingleInnerException();
exportResult = new ExportDataResult
{
FilePath = inner.File
};
}
allowDownload = true;
using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileStream, CompressionMode.Decompress))
{
var chunk1 = RavenJToken.TryLoad(stream) as RavenJObject;
var att1 = chunk1["Attachments"] as RavenJArray;
allAttachments.AddRange(att1.Values<RavenJObject>());
}
exportResult = await dumper.ExportData(new SmugglerExportOptions
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8070",
DefaultDatabase = databaseName,
}
});
using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileStream, CompressionMode.Decompress))
{
var chunk2 = RavenJToken.TryLoad(stream) as RavenJObject;
var attr2 = chunk2["Attachments"] as RavenJArray;
allAttachments.AddRange(attr2.Values<RavenJObject>());
}
Assert.Equal(2000, allAttachments.Count());
}
finally
{
IOExtensions.DeleteDirectory(backupPath);
forwarder.Dispose();
server.Dispose();
}
}
public async Task CanHandleDocumentExceptionsGracefully_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
var server = GetNewServer(databaseName: Constants.SystemDatabase);
var alreadyReset = false;
var port = 8070;
var forwarder = new ProxyServer(ref port, 8079)
{
VetoTransfer = (totalRead, buffer) =>
{
if (alreadyReset == false && totalRead > 25000)
{
alreadyReset = true;
return(true);
}
return(false);
}
};
try
{
string databaseName;
using (var store = new DocumentStore
{
Url = "http://localhost:8079"
})
{
databaseName = store.DefaultDatabase;
store.Initialize();
InsertUsers(store, 0, 2000);
}
var dumper = new SmugglerDatabaseApi {
Options = { Limit = 1900, Incremental = true }
};
var allDocs = new List <RavenJObject>();
ExportDataResult exportResult = null;
try
{
exportResult = await dumper.ExportData(new SmugglerExportOptions <RavenConnectionStringOptions>
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:" + port,
DefaultDatabase = databaseName,
}
});
Assert.False(true, "Previous op should throw.");
}
catch (SmugglerExportException e)
{
exportResult = new ExportDataResult
{
FilePath = e.File
};
}
using (var fileSteam = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileSteam, CompressionMode.Decompress))
{
var chunk1 = RavenJToken.TryLoad(stream) as RavenJObject;
var doc1 = chunk1["Docs"] as RavenJArray;
allDocs.AddRange(doc1.Values <RavenJObject>());
}
exportResult = await dumper.ExportData(new SmugglerExportOptions <RavenConnectionStringOptions>
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8070",
DefaultDatabase = databaseName,
}
});
using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileStream, CompressionMode.Decompress))
{
var chunk2 = RavenJToken.TryLoad(stream) as RavenJObject;
var doc2 = chunk2["Docs"] as RavenJArray;
allDocs.AddRange(doc2.Values <RavenJObject>());
}
Assert.Equal(2000, allDocs.Count(d => (d.Value <string>("Name") ?? String.Empty).StartsWith("User")));
}
finally
{
forwarder.Dispose();
server.Dispose();
IOExtensions.DeleteDirectory(backupPath);
}
}
public void PurgeTombstones(ExportDataResult result)
{
throw new NotImplementedException("Purge tombstones is not supported for Command Line Smuggler");
}
public new void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result, LastEtagsInfo maxEtags)
{
base.ExportDeletions(jsonWriter, options, result, maxEtags);
}
public async Task DataDumperExportHandlesMaxEtagCorrectly()
{
using (var store = NewDocumentStore())
{
using (var session = store.OpenSession())
{
for (var i = 0; i < 10; i++)
{
session.Store(new User { Name = "oren #" + (i+1) });
}
session.SaveChanges();
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", exportedDocs.Last().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", exportedDocs.Last().Value<RavenJObject>("@metadata").Value<string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", lastEtag.ToString());
}
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.PutAttachment("attach/" + (i+1), null, new MemoryStream(new [] { (byte)i }), new RavenJObject());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", exportedAttachments.Last().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", exportedAttachments.Last().Value<string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", lastEtag.ToString());
}
WaitForIndexing(store);
store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery()
{
Query = "Tag:Users"
}).WaitForCompletion();
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.DeleteAttachment("attach/" + (i+1), null);
}
Etag user6DeletionEtag = null, user9DeletionEtag = null, attach5DeletionEtag = null, attach7DeletionEtag = null;
WaitForUserToContinueTheTest(store);
store.SystemDatabase.TransactionalStorage.Batch(accessor =>
{
user6DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/6").Etag;
user9DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/9").Etag;
attach5DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/5").Etag;
attach7DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/7").Etag;
});
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
writer.WriteStartObject();
var lastEtags = new LastEtagsInfo();
var exportResult = new ExportDataResult
{
LastDocDeleteEtag = user6DeletionEtag,
LastAttachmentsDeleteEtag = attach5DeletionEtag
};
lastEtags.LastDocDeleteEtag = user9DeletionEtag;
lastEtags.LastAttachmentsDeleteEtag = attach7DeletionEtag;
dumper.ExportDeletions(writer, exportResult, lastEtags).Wait();
writer.WriteEndObject();
writer.Flush();
// read exported content
var exportJson = RavenJObject.Parse(textStream.GetStringBuilder().ToString());
var docsKeys =
exportJson.Value<RavenJArray>("DocsDeletions").Select(x => x.Value<string>("Key")).ToArray();
var attachmentsKeys =
exportJson.Value<RavenJArray>("AttachmentsDeletions")
.Select(x => x.Value<string>("Key"))
.ToArray();
Assert.Equal(new [] { "users/7", "users/8", "users/9" }, docsKeys);
Assert.Equal(new [] { "attach/6", "attach/7" }, attachmentsKeys);
}
}
}
public override Task ExportDeletions(JsonTextWriter jsonWriter, ExportDataResult result, LastEtagsInfo maxEtags)
{
return base.ExportDeletions(jsonWriter, result, maxEtags);
}
public void CanProperlyReadLastEtagUsingPreviousFormat()
{
var backupPath = NewDataPath("BackupFolder", forceCreateDir:true);
var etagFileLocation = Path.Combine(Path.GetDirectoryName(backupPath), "IncrementalExport.state.json");
using (var streamWriter = new StreamWriter(File.Create(etagFileLocation)))
{
new RavenJObject
{
{"LastDocEtag", Etag.Parse("00000000-0000-0000-0000-000000000001").ToString()},
{"LastAttachmentEtag", Etag.Parse("00000000-0000-0000-0000-000000000002").ToString()}
}.WriteTo(new JsonTextWriter(streamWriter));
streamWriter.Flush();
}
var result = new ExportDataResult
{
FilePath = backupPath
};
SmugglerDatabaseApiBase.ReadLastEtagsFromFile(result);
Assert.Equal("00000000-0000-0000-0000-000000000001", result.LastDocsEtag.ToString());
Assert.Equal("00000000-0000-0000-0000-000000000002", result.LastAttachmentsEtag.ToString());
Assert.Equal(Etag.Empty, result.LastDocDeleteEtag);
Assert.Equal(Etag.Empty, result.LastAttachmentsDeleteEtag);
}
public void PurgeTombstones(ExportDataResult result)
{
throw new NotImplementedException("Purge tombstones is not supported for Command Line Smuggler");
}
public override Task ExportDeletions(JsonTextWriter jsonWriter, ExportDataResult result, LastEtagsInfo maxEtags)
{
return(base.ExportDeletions(jsonWriter, result, maxEtags));
}
public async Task CanHandleAttachmentExceptionsGracefully_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
var server = GetNewServer();
var allowDownload = false;
var port = 8070;
var forwarder = new ProxyServer(ref port, 8079)
{
VetoTransfer = (totalRead, buffer) =>
{
var payload = System.Text.Encoding.UTF8.GetString(buffer.Array, buffer.Offset, buffer.Count);
return(payload.Contains("GET /static/users/678 ") && allowDownload == false);
}
};
try
{
string databaseName;
using (var store = new DocumentStore
{
Url = "http://localhost:8079"
})
{
databaseName = store.DefaultDatabase;
store.Initialize();
InsertAttachments(store, 2000);
}
var dumper = new SmugglerDatabaseApi {
Options = { Limit = 1500, Incremental = true }
};
var allAttachments = new List <RavenJObject>();
ExportDataResult exportResult = null;
try
{
exportResult = dumper.ExportData(new SmugglerExportOptions <RavenConnectionStringOptions>
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:" + port,
DefaultDatabase = databaseName,
}
}).Result;
Assert.False(true, "Previous op should throw.");
}
catch (AggregateException e)
{
var inner = (SmugglerExportException)e.ExtractSingleInnerException();
exportResult = new ExportDataResult
{
FilePath = inner.File
};
}
allowDownload = true;
using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileStream, CompressionMode.Decompress))
{
var chunk1 = RavenJToken.TryLoad(stream) as RavenJObject;
var att1 = chunk1["Attachments"] as RavenJArray;
allAttachments.AddRange(att1.Values <RavenJObject>());
}
exportResult = await dumper.ExportData(new SmugglerExportOptions <RavenConnectionStringOptions>
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8070",
DefaultDatabase = databaseName,
}
});
using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileStream, CompressionMode.Decompress))
{
var chunk2 = RavenJToken.TryLoad(stream) as RavenJObject;
var attr2 = chunk2["Attachments"] as RavenJArray;
allAttachments.AddRange(attr2.Values <RavenJObject>());
}
Assert.Equal(2000, allAttachments.Count());
}
finally
{
IOExtensions.DeleteDirectory(backupPath);
forwarder.Dispose();
server.Dispose();
}
}
public async Task DataDumperExportHandlesMaxEtagCorrectly()
{
using (var store = NewDocumentStore())
{
using (var session = store.OpenSession())
{
for (var i = 0; i < 10; i++)
{
session.Store(new User {
Name = "oren #" + (i + 1)
});
}
session.SaveChanges();
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", exportedDocs.Last().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", exportedDocs.Last().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", lastEtag.ToString());
}
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.PutAttachment("attach/" + (i + 1), null, new MemoryStream(new [] { (byte)i }), new RavenJObject());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", exportedAttachments.Last().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", exportedAttachments.Last().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", lastEtag.ToString());
}
WaitForIndexing(store);
store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery()
{
Query = "Tag:Users"
}).WaitForCompletion();
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.DeleteAttachment("attach/" + (i + 1), null);
}
Etag user6DeletionEtag = null, user9DeletionEtag = null, attach5DeletionEtag = null, attach7DeletionEtag = null;
WaitForUserToContinueTheTest(store);
store.SystemDatabase.TransactionalStorage.Batch(accessor =>
{
user6DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/6").Etag;
user9DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/9").Etag;
attach5DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/5").Etag;
attach7DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/7").Etag;
});
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
writer.WriteStartObject();
var lastEtags = new LastEtagsInfo();
var exportResult = new ExportDataResult
{
LastDocDeleteEtag = user6DeletionEtag,
LastAttachmentsDeleteEtag = attach5DeletionEtag
};
lastEtags.LastDocDeleteEtag = user9DeletionEtag;
lastEtags.LastAttachmentsDeleteEtag = attach7DeletionEtag;
dumper.ExportDeletions(writer, exportResult, lastEtags).Wait();
writer.WriteEndObject();
writer.Flush();
// read exported content
var exportJson = RavenJObject.Parse(textStream.GetStringBuilder().ToString());
var docsKeys =
exportJson.Value <RavenJArray>("DocsDeletions").Select(x => x.Value <string>("Key")).ToArray();
var attachmentsKeys =
exportJson.Value <RavenJArray>("AttachmentsDeletions")
.Select(x => x.Value <string>("Key"))
.ToArray();
Assert.Equal(new [] { "users/7", "users/8", "users/9" }, docsKeys);
Assert.Equal(new [] { "attach/6", "attach/7" }, attachmentsKeys);
}
}
}
public override Task ExportDeletions(JsonTextWriter jsonWriter, ExportDataResult result, LastEtagsInfo maxEtagsToFetch)
{
throw new NotSupportedException("Exporting deletions is not supported for Command Line Smuggler");
}
public override Task ExportDeletions(JsonTextWriter jsonWriter, ExportDataResult result, LastEtagsInfo maxEtagsToFetch)
{
throw new NotSupportedException("Exporting deletions is not supported for Command Line Smuggler");
}
protected override void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result,
LastEtagsInfo maxEtagsToFetch)
{
throw new NotImplementedException("Export Deletions is not supported for command line smuggler");
}
public static void ReadLastEtagsFromFile(ExportDataResult result)
{
var log = LogManager.GetCurrentClassLogger();
var etagFileLocation = Path.Combine(result.FilePath, IncrementalExportStateFile);
if (!File.Exists(etagFileLocation))
return;
using (var streamReader = new StreamReader(new FileStream(etagFileLocation, FileMode.Open)))
using (var jsonReader = new JsonTextReader(streamReader))
{
RavenJObject ravenJObject;
try
{
ravenJObject = RavenJObject.Load(jsonReader);
}
catch (Exception e)
{
log.WarnException("Could not parse etag document from file : " + etagFileLocation + ", ignoring, will start from scratch", e);
return;
}
result.LastDocsEtag = Etag.Parse(ravenJObject.Value<string>("LastDocEtag"));
result.LastAttachmentsEtag = Etag.Parse(ravenJObject.Value<string>("LastAttachmentEtag"));
result.LastDocDeleteEtag = Etag.Parse(ravenJObject.Value<string>("LastDocDeleteEtag") ?? Etag.Empty.ToString());
result.LastAttachmentsDeleteEtag =
Etag.Parse(ravenJObject.Value<string>("LastAttachmentsDeleteEtag") ?? Etag.Empty.ToString());
}
}
public static void WriteLastEtagsToFile(ExportDataResult result, string backupPath)
{
// ReSharper disable once AssignNullToNotNullAttribute
var etagFileLocation = Path.Combine(Path.GetDirectoryName(backupPath), IncrementalExportStateFile);
using (var streamWriter = new StreamWriter(File.Create(etagFileLocation)))
{
new RavenJObject
{
{"LastDocEtag", result.LastDocsEtag.ToString()},
{"LastAttachmentEtag", result.LastAttachmentsEtag.ToString()},
{"LastDocDeleteEtag", result.LastDocDeleteEtag.ToString()},
{"LastAttachmentsDeleteEtag", result.LastAttachmentsDeleteEtag.ToString()}
}.WriteTo(new JsonTextWriter(streamWriter));
streamWriter.Flush();
}
}
/// <summary>
///
/// </summary>
/// <param name="jsonWriter"></param>
/// <param name="options"></param>
/// <param name="result"></param>
/// <param name="maxEtags">Max etags are inclusive</param>
protected async override void ExportDeletions(JsonTextWriter jsonWriter, SmugglerOptions options, ExportDataResult result, LastEtagsInfo maxEtags)
{
jsonWriter.WritePropertyName("DocsDeletions");
jsonWriter.WriteStartArray();
result.LastDocDeleteEtag = await ExportDocumentsDeletion(options, jsonWriter, result.LastDocDeleteEtag, maxEtags.LastDocDeleteEtag.IncrementBy(1));
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("AttachmentsDeletions");
jsonWriter.WriteStartArray();
result.LastAttachmentsDeleteEtag = await ExportAttachmentsDeletion(options, jsonWriter, result.LastAttachmentsDeleteEtag, maxEtags.LastAttachmentsDeleteEtag.IncrementBy(1));
jsonWriter.WriteEndArray();
}
