public async Task <ImportResult> Import(DocumentsOperationContext context, Stream stream, Action <IOperationProgress> onProgress = null)
{
var result = new ImportResult();
var progress = new IndeterminateProgress();
var state = new JsonParserState();
JsonOperationContext.ManagedPinnedBuffer buffer;
using (context.GetManagedBuffer(out buffer))
using (var parser = new UnmanagedJsonParser(context, state, "fileName"))
{
var operateOnType = "__top_start_object";
var buildVersion = 0L;
var identities = new Dictionary <string, long>();
VersioningStorage versioningStorage = null;
while (true)
{
if (parser.Read() == false)
{
var read = await stream.ReadAsync(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length);
if (read == 0)
{
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
break;
}
parser.SetBuffer(buffer, read);
continue;
}
switch (state.CurrentTokenType)
{
case JsonParserToken.String:
unsafe
{
operateOnType =
new LazyStringValue(null, state.StringBuffer, state.StringSize, context).ToString();
}
break;
case JsonParserToken.Integer:
switch (operateOnType)
{
case "BuildVersion":
buildVersion = state.Long;
break;
}
break;
case JsonParserToken.StartObject:
if (operateOnType == "__top_start_object")
{
operateOnType = null;
break;
}
context.CachedProperties.NewDocument();
var builder = new BlittableJsonDocumentBuilder(_batchPutCommand.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "ImportObject", parser, state);
builder.ReadNestedObject();
while (builder.Read() == false)
{
var read = await stream.ReadAsync(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(buffer, read);
}
builder.FinalizeDocument();
if (operateOnType == "Docs" && Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents))
{
progress.Progress = "Importing Documents";
onProgress?.Invoke(progress);
PatchDocument patch = null;
PatchRequest patchRequest = null;
if (string.IsNullOrWhiteSpace(Options.TransformScript) == false)
{
patch = new PatchDocument(context.DocumentDatabase);
patchRequest = new PatchRequest
{
Script = Options.TransformScript
};
}
result.DocumentsCount++;
var reader = builder.CreateReader();
var document = new Document
{
Data = reader,
};
if (Options.IncludeExpired == false && document.Expired(_database.Time.GetUtcNow()))
{
continue;
}
TransformScriptOrDisableVersioningIfNeeded(context, patch, reader, document,
patchRequest);
_batchPutCommand.Add(document.Data);
if (result.DocumentsCount % 1000 == 0)
{
progress.Progress = $"Imported {result.DocumentsCount} documents";
onProgress?.Invoke(progress);
}
await HandleBatchOfDocuments(context, parser, buildVersion).ConfigureAwait(false);
}
else if (operateOnType == "RevisionDocuments" &&
Options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments))
{
if (versioningStorage == null)
{
break;
}
result.RevisionDocumentsCount++;
var reader = builder.CreateReader();
_batchPutCommand.Add(reader);
await HandleBatchOfDocuments(context, parser, buildVersion).ConfigureAwait(false);;
}
else
{
using (builder)
{
switch (operateOnType)
{
case "Attachments":
result.Warnings.Add("Attachments are not supported anymore. Use RavenFS isntead. Skipping.");
break;
case "Indexes":
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Indexes) == false)
{
continue;
}
result.IndexesCount++;
progress.Progress = "importing Indexes";
onProgress?.Invoke(progress);
try
{
IndexProcessor.Import(builder, _database, buildVersion, Options.RemoveAnalyzers);
}
catch (Exception e)
{
result.Warnings.Add($"Could not import index. Message: {e.Message}");
}
break;
case "Transformers":
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Transformers) == false)
{
continue;
}
result.TransformersCount++;
progress.Progress = "Importing Transformers";
onProgress?.Invoke(progress);
try
{
TransformerProcessor.Import(builder, _database, buildVersion);
}
catch (Exception e)
{
result.Warnings.Add($"Could not import transformer. Message: {e.Message}");
}
break;
case "Identities":
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Identities))
{
result.IdentitiesCount++;
progress.Progress = "Importing Identities";
onProgress?.Invoke(progress);
using (var reader = builder.CreateReader())
{
try
{
string identityKey, identityValueString;
long identityValue;
if (reader.TryGet("Key", out identityKey) == false || reader.TryGet("Value", out identityValueString) == false || long.TryParse(identityValueString, out identityValue) == false)
{
result.Warnings.Add($"Cannot import the following identity: '{reader}'. Skipping.");
}
else
{
identities[identityKey] = identityValue;
}
}
catch (Exception e)
{
result.Warnings.Add($"Cannot import the following identity: '{reader}'. Error: {e}. Skipping.");
}
}
}
break;
default:
result.Warnings.Add(
$"The following type is not recognized: '{operateOnType}'. Skipping.");
break;
}
}
}
break;
case JsonParserToken.StartArray:
switch (operateOnType)
{
case "RevisionDocuments":
// We are taking a reference here since the documents import can activate or disable the versioning.
// We hold a local copy because the user can disable the bundle during the import process, exteranly.
// In this case we want to continue to import the revisions documents.
versioningStorage = _database.BundleLoader.VersioningStorage;
_batchPutCommand.IsRevision = true;
break;
}
break;
case JsonParserToken.EndArray:
switch (operateOnType)
{
case "Docs":
await FinishBatchOfDocuments();
_batchPutCommand = new MergedBatchPutCommand(_database, buildVersion);
break;
case "RevisionDocuments":
await FinishBatchOfDocuments();
break;
case "Identities":
if (identities.Count > 0)
{
using (var tx = context.OpenWriteTransaction())
{
_database.DocumentsStorage.UpdateIdentities(context, identities);
tx.Commit();
}
}
identities = null;
break;
}
break;
}
}
}
return(result);
}
public ExportResult Export(DocumentsOperationContext context, Stream destinationStream, Action <IOperationProgress> onProgress = null)
{
var result = new ExportResult();
var progress = new IndeterminateProgress();
using (var gZipStream = new GZipStream(destinationStream, CompressionMode.Compress, leaveOpen: true))
using (var writer = new BlittableJsonTextWriter(context, gZipStream))
{
writer.WriteStartObject();
writer.WritePropertyName(("BuildVersion"));
writer.WriteInteger(40000);
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents))
{
progress.Progress = "Exporting Documents";
onProgress?.Invoke(progress);
writer.WriteComma();
writer.WritePropertyName(("Docs"));
IEnumerable <Document> documents = Options.CollectionsToExport.Count != 0 ?
_database.DocumentsStorage.GetDocumentsFrom(context, Options.CollectionsToExport, StartDocsEtag ?? 0, int.MaxValue) :
_database.DocumentsStorage.GetDocumentsFrom(context, StartDocsEtag ?? 0, 0, int.MaxValue);
writer.WriteStartArray();
PatchDocument patch = null;
PatchRequest patchRequest = null;
if (string.IsNullOrWhiteSpace(Options.TransformScript) == false)
{
patch = new PatchDocument(context.DocumentDatabase);
patchRequest = new PatchRequest
{
Script = Options.TransformScript
};
}
bool first = true;
foreach (var document in documents)
{
if (document == null)
{
continue;
}
if (Options.IncludeExpired == false && document.Expired(_database.Time.GetUtcNow()))
{
continue;
}
var patchResult = patch?.Apply(context, document, patchRequest);
if (patchResult != null && patchResult.ModifiedDocument.Equals(document.Data) == false)
{
document.Data = patchResult.ModifiedDocument;
}
using (document.Data)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
document.EnsureMetadata();
context.Write(writer, document.Data);
result.LastDocsEtag = document.Etag;
}
result.ExportedDocuments++;
}
writer.WriteEndArray();
}
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments))
{
var versioningStorage = _database.BundleLoader.VersioningStorage;
if (versioningStorage != null)
{
writer.WriteComma();
writer.WritePropertyName("RevisionDocuments");
writer.WriteStartArray();
var first = true;
var revisionDocuments = Options.RevisionDocumentsLimit.HasValue
? versioningStorage.GetRevisionsAfter(context, StartRevisionDocumentsEtag ?? 0, Options.RevisionDocumentsLimit.Value)
: versioningStorage.GetRevisionsAfter(context, StartRevisionDocumentsEtag ?? 0);
foreach (var revisionDocument in revisionDocuments)
{
if (revisionDocument == null)
{
continue;
}
using (revisionDocument.Data)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
revisionDocument.EnsureMetadata();
context.Write(writer, revisionDocument.Data);
result.LastRevisionDocumentsEtag = revisionDocument.Etag;
}
}
writer.WriteEndArray();
}
}
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Indexes))
{
progress.Progress = "Exporting Indexes";
onProgress?.Invoke(progress);
writer.WriteComma();
writer.WritePropertyName("Indexes");
writer.WriteStartArray();
var isFirst = true;
foreach (var index in _database.IndexStore.GetIndexes())
{
if (isFirst == false)
{
writer.WriteComma();
}
isFirst = false;
IndexProcessor.Export(writer, index, context, Options.RemoveAnalyzers);
}
writer.WriteEndArray();
}
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Transformers))
{
progress.Progress = "Exporting Transformers";
onProgress?.Invoke(progress);
writer.WriteComma();
writer.WritePropertyName(("Transformers"));
writer.WriteStartArray();
var isFirst = true;
foreach (var transformer in _database.TransformerStore.GetTransformers())
{
if (isFirst == false)
{
writer.WriteComma();
}
isFirst = false;
TransformerProcessor.Export(writer, transformer, context);
}
writer.WriteEndArray();
}
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Identities))
{
progress.Progress = "Exporting Identities";
onProgress?.Invoke(progress);
writer.WriteComma();
writer.WritePropertyName(("Identities"));
writer.WriteStartArray();
var identities = _database.DocumentsStorage.GetIdentities(context);
var first = true;
foreach (var identity in identities)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WriteStartObject();
writer.WritePropertyName(("Key"));
writer.WriteString((identity.Key));
writer.WriteComma();
writer.WritePropertyName(("Value"));
writer.WriteString((identity.Value.ToString()));
writer.WriteEndObject();
}
writer.WriteEndArray();
}
writer.WriteEndObject();
progress.Progress = $"Finish Exported database to {Options.FileName}. Exported {result.ExportedDocuments}";
onProgress?.Invoke(progress);
}
return(result);
}
