public ExportResult Export(DocumentsOperationContext context, Stream destinationStream, Action <IOperationProgress> onProgress = null)
{
var result = new ExportResult();
var progress = new IndeterminateProgress();
using (var gZipStream = new GZipStream(destinationStream, CompressionMode.Compress, leaveOpen: true))
using (var writer = new BlittableJsonTextWriter(context, gZipStream))
{
writer.WriteStartObject();
writer.WritePropertyName(("BuildVersion"));
writer.WriteInteger(40000);
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents))
{
progress.Progress = "Exporting Documents";
onProgress?.Invoke(progress);
writer.WriteComma();
writer.WritePropertyName(("Docs"));
IEnumerable <Document> documents = Options.CollectionsToExport.Count != 0 ?
_database.DocumentsStorage.GetDocumentsFrom(context, Options.CollectionsToExport, StartDocsEtag ?? 0, int.MaxValue) :
_database.DocumentsStorage.GetDocumentsFrom(context, StartDocsEtag ?? 0, 0, int.MaxValue);
writer.WriteStartArray();
PatchDocument patch = null;
PatchRequest patchRequest = null;
if (string.IsNullOrWhiteSpace(Options.TransformScript) == false)
{
patch = new PatchDocument(context.DocumentDatabase);
patchRequest = new PatchRequest
{
Script = Options.TransformScript
};
}
bool first = true;
foreach (var document in documents)
{
if (document == null)
{
continue;
}
if (Options.IncludeExpired == false && document.Expired(_database.Time.GetUtcNow()))
{
continue;
}
var patchResult = patch?.Apply(context, document, patchRequest);
if (patchResult != null && patchResult.ModifiedDocument.Equals(document.Data) == false)
{
document.Data = patchResult.ModifiedDocument;
}
using (document.Data)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
document.EnsureMetadata();
context.Write(writer, document.Data);
result.LastDocsEtag = document.Etag;
}
result.ExportedDocuments++;
}
writer.WriteEndArray();
}
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments))
{
var versioningStorage = _database.BundleLoader.VersioningStorage;
if (versioningStorage != null)
{
writer.WriteComma();
writer.WritePropertyName("RevisionDocuments");
writer.WriteStartArray();
var first = true;
var revisionDocuments = Options.RevisionDocumentsLimit.HasValue
? versioningStorage.GetRevisionsAfter(context, StartRevisionDocumentsEtag ?? 0, Options.RevisionDocumentsLimit.Value)
: versioningStorage.GetRevisionsAfter(context, StartRevisionDocumentsEtag ?? 0);
foreach (var revisionDocument in revisionDocuments)
{
if (revisionDocument == null)
{
continue;
}
using (revisionDocument.Data)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
revisionDocument.EnsureMetadata();
context.Write(writer, revisionDocument.Data);
result.LastRevisionDocumentsEtag = revisionDocument.Etag;
}
}
writer.WriteEndArray();
}
}
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Indexes))
{
progress.Progress = "Exporting Indexes";
onProgress?.Invoke(progress);
writer.WriteComma();
writer.WritePropertyName("Indexes");
writer.WriteStartArray();
var isFirst = true;
foreach (var index in _database.IndexStore.GetIndexes())
{
if (isFirst == false)
{
writer.WriteComma();
}
isFirst = false;
IndexProcessor.Export(writer, index, context, Options.RemoveAnalyzers);
}
writer.WriteEndArray();
}
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Transformers))
{
progress.Progress = "Exporting Transformers";
onProgress?.Invoke(progress);
writer.WriteComma();
writer.WritePropertyName(("Transformers"));
writer.WriteStartArray();
var isFirst = true;
foreach (var transformer in _database.TransformerStore.GetTransformers())
{
if (isFirst == false)
{
writer.WriteComma();
}
isFirst = false;
TransformerProcessor.Export(writer, transformer, context);
}
writer.WriteEndArray();
}
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Identities))
{
progress.Progress = "Exporting Identities";
onProgress?.Invoke(progress);
writer.WriteComma();
writer.WritePropertyName(("Identities"));
writer.WriteStartArray();
var identities = _database.DocumentsStorage.GetIdentities(context);
var first = true;
foreach (var identity in identities)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WriteStartObject();
writer.WritePropertyName(("Key"));
writer.WriteString((identity.Key));
writer.WriteComma();
writer.WritePropertyName(("Value"));
writer.WriteString((identity.Value.ToString()));
writer.WriteEndObject();
}
writer.WriteEndArray();
}
writer.WriteEndObject();
progress.Progress = $"Finish Exported database to {Options.FileName}. Exported {result.ExportedDocuments}";
onProgress?.Invoke(progress);
}
return(result);
}
public void ExecuteReplicationOnce()
{
_orderedReplicaItems.Clear();
try
{
var sp = Stopwatch.StartNew();
var timeout = Debugger.IsAttached ? 60 * 1000 : 1000;
var configurationContext = _parent._configurationContext;
using (configurationContext.OpenReadTransaction())
{
while (sp.ElapsedMilliseconds < timeout)
{
LastEtag = _parent._lastSentIndexOrTransformerEtag;
_parent.CancellationToken.ThrowIfCancellationRequested();
var indexAndTransformerMetadata = _parent._database.IndexMetadataPersistence.GetAfter(
configurationContext.Transaction.InnerTransaction,
configurationContext, LastEtag + 1, 0, 1024);
using (var stream = new MemoryStream())
{
foreach (var item in indexAndTransformerMetadata)
{
_parent.CancellationToken.ThrowIfCancellationRequested();
stream.Position = 0;
using (var writer = new BlittableJsonTextWriter(configurationContext, stream))
{
switch (item.Type)
{
case IndexEntryType.Index:
var index = _parent._database.IndexStore.GetIndex(item.Id);
if (index == null) //precaution
{
throw new InvalidDataException(
$"Index with name {item.Name} has metadata, but is not at the index store. This is not supposed to happen and is likely a bug.");
}
try
{
IndexProcessor.Export(writer, index, configurationContext, false);
}
catch (InvalidOperationException e)
{
if (_log.IsInfoEnabled)
{
_log.Info(
$"Failed to export index definition for replication. Index name = {item.Name}",
e);
}
}
break;
case IndexEntryType.Transformer:
var transformer = _parent._database.TransformerStore.GetTransformer(item.Id);
if (transformer == null) //precaution
{
throw new InvalidDataException(
$"Transformer with name {item.Name} has metadata, but is not at the transformer store. This is not supposed to happen and is likely a bug.");
}
try
{
TransformerProcessor.Export(writer, transformer,
configurationContext);
}
catch (InvalidOperationException e)
{
if (_log.IsInfoEnabled)
{
_log.Info(
$"Failed to export transformer definition for replication. Transformer name = {item.Name}",
e);
}
}
break;
default:
throw new ArgumentOutOfRangeException(nameof(item),
"Unexpected item type in index/transformer metadata. This is not supposed to happen.");
}
writer.Flush();
stream.Position = 0;
var newItem = new ReplicationBatchIndexItem
{
Name = item.Name,
ChangeVector = item.ChangeVector,
Etag = item.Etag,
Type = (int)item.Type,
Definition =
configurationContext.ReadForMemory(stream,
"Index/Transformer Replication - Reading definition into memory")
};
AddReplicationItemToBatch(newItem);
}
}
}
// if we are at the end, we are done
if (LastEtag <=
_parent._database.IndexMetadataPersistence.ReadLastEtag(
configurationContext.Transaction.InnerTransaction))
{
break;
}
}
if (_log.IsInfoEnabled)
{
_log.Info(
$"Found {_orderedReplicaItems.Count:#,#;;0} indexes/transformers to replicate to {_parent.Destination.Database} @ {_parent.Destination.Url} in {sp.ElapsedMilliseconds:#,#;;0} ms.");
}
_parent.CancellationToken.ThrowIfCancellationRequested();
try
{
using (_parent._documentsContext.OpenReadTransaction())
SendIndexTransformerBatch();
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info("Failed to send index/transformer replication batch", e);
}
throw;
}
}
}
finally
{
//release memory at the end of the operation
foreach (var item in _orderedReplicaItems)
{
item.Value.Definition.Dispose();
}
_orderedReplicaItems.Clear();
}
}