protected void RecordActionForDebug(string actionName, string documentKey, RavenJObject documentData, RavenJObject documentMetadata)
{
if (debugMode == false)
return;
var actions = DebugActions[actionName] as RavenJArray;
switch (actionName)
{
case "LoadDocument":
case "DeleteDocument":
if (actions == null)
DebugActions[actionName] = actions = new RavenJArray();
actions.Add(documentKey);
break;
case "PutDocument":
if (actions == null)
DebugActions[actionName] = actions = new RavenJArray();
actions.Add(new RavenJObject
{
{ "Key", documentKey },
{ "Data", documentData },
{ "Metadata", documentMetadata }
});
break;
default:
throw new NotSupportedException("Cannot record action: " + actionName);
}
}
private static void ModifyResult(RavenJObject result)
{
var tags = result.Value<string>("Tags");
if (tags != null)
{
result["Tags"] =
new RavenJArray(tags.Split(new[] {' ', ',', ';'}, StringSplitOptions.RemoveEmptyEntries));
}
else
{
result["Tags"] = new RavenJArray();
}
var deps = result.Value<string>("Dependencies");
if (deps != null)
{
result["Dependencies"] =
new RavenJArray(deps.Split(new[] {'|'}, StringSplitOptions.RemoveEmptyEntries)
.Select(s =>
{
var strings = s.Split(':');
return RavenJObject.FromObject(new {Package = strings[0], Version = strings[1]});
}));
}
result["PackageId"] = result["Id"];
result.Remove("Id");
result.Remove("__metadata");
result.Remove("DownloadCount");
}
public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation)
{
if (key.StartsWith("Raven/", StringComparison.OrdinalIgnoreCase) && // we don't deal with system documents
key.StartsWith("Raven/Hilo/", StringComparison.OrdinalIgnoreCase) == false) // except for hilos
return;
using (Database.DisableAllTriggersForCurrentThread())
{
var documentMetadata = GetDocumentMetadata(key);
if (documentMetadata != null)
{
RavenJArray history = new RavenJArray(ReplicationData.GetHistory(documentMetadata));
metadata[Constants.RavenReplicationHistory] = history;
if (documentMetadata.ContainsKey(Constants.RavenReplicationVersion) &&
documentMetadata.ContainsKey(Constants.RavenReplicationSource))
{
history.Add(new RavenJObject
{
{Constants.RavenReplicationVersion, documentMetadata[Constants.RavenReplicationVersion]},
{Constants.RavenReplicationSource, documentMetadata[Constants.RavenReplicationSource]}
});
}
while (history.Length > Constants.ChangeHistoryLength)
{
history.RemoveAt(0);
}
}
metadata[Constants.RavenReplicationVersion] = RavenJToken.FromObject(HiLo.NextId());
metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(Database.TransactionalStorage.Id);
}
}
/// <summary>
/// Adds the cascade delete reference.
/// </summary>
/// <param name="session">The session.</param>
/// <param name="entity">The entity.</param>
/// <param name="documentKeys">The document keys.</param>
public static void AddCascadeDeleteReference(this IAdvancedDocumentSessionOperations session, object entity, params string[] documentKeys)
{
var metadata = session.GetMetadataFor(entity);
if(metadata == null)
{
throw new InvalidOperationException("The entity must be tracked in the session before calling this method.");
}
if(documentKeys.Length == 0)
{
throw new ArgumentException("At least one document key must be specified.");
}
const string metadataKey = "Raven-Cascade-Delete-Documents";
RavenJToken token;
if(!metadata.TryGetValue(metadataKey, out token))
{
token = new RavenJArray();
}
var list = (RavenJArray)token;
foreach(var documentKey in documentKeys.Where(key => !list.Contains(key)))
{
list.Add(documentKey);
}
metadata[metadataKey] = list;
}
public override void Respond(IHttpContext context)
{
RavenJArray itemsToLoad;
if(context.Request.HttpMethod == "POST")
itemsToLoad = context.ReadJsonArray();
else
itemsToLoad = new RavenJArray(context.Request.QueryString.GetValues("id"));
var result = new MultiLoadResult();
var loadedIds = new HashSet<string>();
var includes = context.Request.QueryString.GetValues("include") ?? new string[0];
var transactionInformation = GetRequestTransaction(context);
var includedEtags = new List<byte>();
Database.TransactionalStorage.Batch(actions =>
{
foreach (RavenJToken item in itemsToLoad)
{
var value = item.Value<string>();
if(loadedIds.Add(value)==false)
continue;
var documentByKey = Database.Get(value, transactionInformation);
if (documentByKey == null)
continue;
result.Results.Add(documentByKey.ToJson());
if (documentByKey.Etag != null)
{
includedEtags.AddRange(documentByKey.Etag.Value.ToByteArray());
}
includedEtags.Add((documentByKey.NonAuthoritativeInformation ?? false) ? (byte)0 : (byte)1);
}
var addIncludesCommand = new AddIncludesCommand(Database, transactionInformation, (etag, includedDoc) =>
{
includedEtags.AddRange(etag.ToByteArray());
result.Includes.Add(includedDoc);
}, includes, loadedIds);
foreach (var item in result.Results.Where(item => item != null))
{
addIncludesCommand.Execute(item);
}
});
Guid computedEtag;
using (var md5 = MD5.Create())
{
var computeHash = md5.ComputeHash(includedEtags.ToArray());
computedEtag = new Guid(computeHash);
}
if (context.MatchEtag(computedEtag))
{
context.SetStatusToNotModified();
return;
}
context.WriteETag(computedEtag);
context.WriteJson(result);
}
public override void OnPut(string key, Stream data, RavenJObject metadata)
{
if (key.StartsWith("Raven/")) // we don't deal with system attachment
return;
using (Database.DisableAllTriggersForCurrentThread())
{
var attachmentMetadata = GetAttachmentMetadata(key);
if (attachmentMetadata != null)
{
RavenJArray history = new RavenJArray(metadata.Value<RavenJArray>(Constants.RavenReplicationHistory));
metadata[Constants.RavenReplicationHistory] = history;
if (attachmentMetadata.ContainsKey(Constants.RavenReplicationVersion) &&
attachmentMetadata.ContainsKey(Constants.RavenReplicationSource))
{
history.Add(new RavenJObject
{
{Constants.RavenReplicationVersion, attachmentMetadata[Constants.RavenReplicationVersion]},
{Constants.RavenReplicationSource, attachmentMetadata[Constants.RavenReplicationSource]}
});
}
if (history.Length > Constants.ChangeHistoryLength)
{
history.RemoveAt(0);
}
}
metadata[Constants.RavenReplicationVersion] = RavenJToken.FromObject(HiLo.NextId());
metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(Database.TransactionalStorage.Id);
}
}
public ReplicationTopologyDiscoverer(DocumentDatabase database, RavenJArray @from, int ttl, ILog log)
{
this.database = database;
this.ttl = ttl;
this.log = log;
this.@from = @from;
requestFactory = new HttpRavenRequestFactory();
}
private bool TryHandleArrayValue(int index, Dictionary<string, object> result, KeyValuePair<string, RavenJToken> prop)
{
var arrays = new List<RavenJArray>
{
(RavenJArray)prop.Value
};
for (var i = 0; i < docs.Length; i++)
{
if (i == index)
continue;
RavenJToken token;
if (docs[i].TryGetValue(prop.Key, out token) && token.Type != JTokenType.Array)
return false;
if (token == null)
continue;
if (token.IsSnapshot)
token = token.CreateSnapshot();
arrays.Add((RavenJArray)token);
}
var mergedArray = new RavenJArray();
while (arrays.Count > 0)
{
var set = new HashSet<RavenJToken>(RavenJTokenEqualityComparer.Default);
for (var i = 0; i < arrays.Count; i++)
{
if (arrays[i].Length == 0)
{
arrays.RemoveAt(i);
i -= 1;
continue;
}
set.Add(arrays[i][0]);
arrays[i].RemoveAt(0);
}
foreach (var ravenJToken in set)
{
mergedArray.Add(ravenJToken);
}
}
if (RavenJTokenEqualityComparer.Default.Equals(mergedArray, prop.Value))
{
result.Add(prop.Key, mergedArray);
return true;
}
result.Add(prop.Key, new ArrayWithWarning(mergedArray));
return true;
}
private void Initialize()
{
if (mappedTypes != null)
return;
lock (padlock)
{
if (mappedTypes != null)
return;
var collectionData = new CollectionData();
var jsonDoc = store.DatabaseCommands.Get(collectionNamesDocId);
if (jsonDoc != null)
{
var collectionNames = jsonDoc.DataAsJson["Collections"] as RavenJArray;
foreach (RavenJValue value in collectionNames)
{
collectionData.Collections.Add(value.Value as string);
}
}
if (timeoutsEnabled)
{
MapTypeToCollectionName(typeof(TimeoutPersisters.RavenDB.TimeoutData), collectionData);
}
if (sagasEnabled)
{
foreach (var sagaType in types.Where(IsSagaEntity))
{
MapTypeToCollectionName(sagaType, collectionData);
}
}
if (collectionData.Changed)
{
var newDoc = new RavenJObject();
var list = new RavenJArray();
foreach (var name in collectionData.Collections)
{
list.Add(new RavenJValue(name));
}
newDoc["EndpointName"] = this.endpointName;
newDoc["EndpointName"] = this.endpointName;
newDoc["Collections"] = list;
var metadata = new RavenJObject();
store.DatabaseCommands.Put(collectionNamesDocId, null, newDoc, metadata);
}
// Completes initialization
this.mappedTypes = collectionData.Mappings;
}
}
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, bool allowStale, Func<string, TransactionInformation, object> batchOperation)
{
var array = new RavenJArray();
var bulkIndexQuery = new IndexQuery
{
Query = indexQuery.Query,
Start = indexQuery.Start,
Cutoff = indexQuery.Cutoff,
WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow,
PageSize = int.MaxValue,
FieldsToFetch = new[] { Constants.DocumentIdFieldName },
SortedFields = indexQuery.SortedFields,
HighlighterPreTags = indexQuery.HighlighterPreTags,
HighlighterPostTags = indexQuery.HighlighterPostTags,
HighlightedFields = indexQuery.HighlightedFields,
SortHints = indexQuery.SortHints
};
bool stale;
var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);
if (stale && allowStale == false)
{
throw new InvalidOperationException(
"Bulk operation cancelled because the index is stale and allowStale is false");
}
var token = tokenSource.Token;
const int batchSize = 1024;
using (var enumerator = queryResults.GetEnumerator())
{
while (true)
{
if (timeout != null)
timeout.Delay();
var batchCount = 0;
token.ThrowIfCancellationRequested();
database.TransactionalStorage.Batch(actions =>
{
while (batchCount < batchSize && enumerator.MoveNext())
{
batchCount++;
var result = batchOperation(enumerator.Current, transactionInformation);
array.Add(RavenJObject.FromObject(result));
}
});
if (batchCount < batchSize) break;
}
}
return array;
}
public override void OnPut(string key, Stream data, RavenJObject metadata)
{
using (Database.DisableAllTriggersForCurrentThread())
{
metadata.Remove(Constants.RavenReplicationConflict);// you can't put conflicts
var oldVersion = Database.Attachments.GetStatic(key);
if (oldVersion == null)
return;
if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null)
return;
var history = new RavenJArray(ReplicationData.GetHistory(metadata));
metadata[Constants.RavenReplicationHistory] = history;
var ravenJTokenEqualityComparer = new RavenJTokenEqualityComparer();
// this is a conflict document, holding document keys in the
// values of the properties
var conflictData = oldVersion.Data().ToJObject();
var conflicts = conflictData.Value<RavenJArray>("Conflicts");
if (conflicts == null)
return;
foreach (var prop in conflicts)
{
var id = prop.Value<string>();
Attachment attachment = Database.Attachments.GetStatic(id);
if(attachment == null)
continue;
Database.Attachments.DeleteStatic(id, null);
// add the conflict history to the mix, so we make sure that we mark that we resolved the conflict
var conflictHistory = new RavenJArray(ReplicationData.GetHistory(attachment.Metadata));
conflictHistory.Add(new RavenJObject
{
{Constants.RavenReplicationVersion, attachment.Metadata[Constants.RavenReplicationVersion]},
{Constants.RavenReplicationSource, attachment.Metadata[Constants.RavenReplicationSource]}
});
foreach (var item in conflictHistory)
{
if (history.Any(x => ravenJTokenEqualityComparer.Equals(x, item)))
continue;
history.Add(item);
}
}
}
}
public override IEnumerable<Row> Execute(IEnumerable<Row> rows)
{
int count = 0;
foreach (var commentsForPosts in rows.Partition(Constants.BatchSize))
{
var cmds = new List<ICommandData>();
foreach (var commentsForPost in commentsForPosts.GroupBy(row => row["PostId"]))
{
var comments = new RavenJArray();
foreach (var row in commentsForPost)
{
comments.Add(new RavenJObject
{
{"Score", new RavenJValue(row["Score"])},
{"CreationDate", new RavenJValue(row["CreationDate"])},
{"Text", new RavenJValue(row["Text"])},
{"UserId", new RavenJValue(row["UserId"])}
});
}
cmds.Add(new PatchCommandData
{
Key = "posts/" + commentsForPost.Key,
Patches = new[]
{
new PatchRequest
{
Name = "Comments",
Type = PatchCommandType.Set,
Value = comments
},
}
});
}
count++;
WriteCommandsTo("Comments #" + count.ToString("00000") + ".json", cmds);
}
yield break;
}
private static void WriteJsonArray(RavenJArray array, StringWriter sw, int margin, int indent = 0)
{
sw.WriteLine('[');
indent += 1;
var isFirstItem = true;
foreach (var token in array.Values())
{
if (isFirstItem)
isFirstItem = false;
else
sw.WriteLine(',');
Indent(sw, indent);
WriteValue(token, sw, margin, indent);
}
sw.WriteLine();
indent -= 1;
Indent(sw, indent);
sw.Write(']');
}
public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation)
{
using (Database.DisableAllTriggersForCurrentThread())
{
metadata.Remove(Constants.RavenReplicationConflict);// you can't put conflicts
var oldVersion = Database.Get(key, transactionInformation);
if (oldVersion == null)
return;
if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null)
return;
RavenJArray history = new RavenJArray(ReplicationData.GetHistory(metadata));
metadata[Constants.RavenReplicationHistory] = history;
var ravenJTokenEqualityComparer = new RavenJTokenEqualityComparer();
// this is a conflict document, holding document keys in the
// values of the properties
var conflicts = oldVersion.DataAsJson.Value<RavenJArray>("Conflicts");
if(conflicts == null)
return;
foreach (var prop in conflicts)
{
RavenJObject deletedMetadata;
Database.Delete(prop.Value<string>(), null, transactionInformation, out deletedMetadata);
// add the conflict history to the mix, so we make sure that we mark that we resolved the conflict
var conflictHistory = new RavenJArray(ReplicationData.GetHistory(deletedMetadata));
conflictHistory.Add(new RavenJObject
{
{Constants.RavenReplicationVersion, deletedMetadata[Constants.RavenReplicationVersion]},
{Constants.RavenReplicationSource, deletedMetadata[Constants.RavenReplicationSource]}
});
foreach (var item in conflictHistory)
{
if(history.Any(x=>ravenJTokenEqualityComparer.Equals(x, item)))
continue;
history.Add(item);
}
}
}
}
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, bool allowStale, Func<string, TransactionInformation, object> batchOperation)
{
var array = new RavenJArray();
var bulkIndexQuery = new IndexQuery
{
Query = indexQuery.Query,
Start = indexQuery.Start,
Cutoff = indexQuery.Cutoff,
PageSize = int.MaxValue,
FieldsToFetch = new[] { Constants.DocumentIdFieldName },
SortedFields = indexQuery.SortedFields
};
bool stale;
var queryResults = database.QueryDocumentIds(index, bulkIndexQuery, out stale);
if (stale && allowStale == false)
{
throw new InvalidOperationException(
"Bulk operation cancelled because the index is stale and allowStale is false");
}
const int batchSize = 1024;
using (var enumerator = queryResults.GetEnumerator())
{
while (true)
{
var batchCount = 0;
database.TransactionalStorage.Batch(actions =>
{
while (batchCount < batchSize && enumerator.MoveNext())
{
batchCount++;
var result = batchOperation(enumerator.Current, transactionInformation);
array.Add(RavenJObject.FromObject(result, JsonExtensions.CreateDefaultJsonSerializer()));
}
});
if (batchCount < batchSize) break;
}
}
return array;
}
private static void WriteJsonArray(RavenJArray array, CountingWriter sw, int width, int numberOfLines)
{
sw.WriteLine("[");
sw.PushIndent();
var isFirstItem = true;
foreach (var token in array.Values())
{
if (sw.LinesWritten >= numberOfLines)
break;
if (isFirstItem)
isFirstItem = false;
else
sw.WriteLine(",");
WriteValue(token, sw, width, numberOfLines);
}
sw.WriteLine("");
sw.PopIndent();
sw.Write("]");
}
public void ForEach(RavenJToken result, RavenJToken item, Action<PathPart, RavenJToken, RavenJToken> action)
{
if (string.IsNullOrEmpty(FinalName) == false)
{
action(this, item, result);
return;
}
RavenJToken newResult = GetTheNewResultOrWireTheDefault(result);
if (item == null)
{
foreach (var pathPart in Items)
pathPart.Value.ForEach(newResult, null, action);
return;
}
if (item is RavenJArray == false)
{
foreach (var pathPart in Items)
pathPart.Value.ForEach(newResult, item.SelectToken(pathPart.Key), action);
}
else
{
var jArray = newResult as RavenJArray;
if (jArray == null)
{
jArray = new RavenJArray();
((RavenJObject)result)[Name] = jArray;
}
foreach (var subItem in item.Values<RavenJToken>())
{
newResult = new RavenJObject();
jArray.Add(newResult);
foreach (var pathPart in Items)
{
pathPart.Value.ForEach(newResult, subItem.SelectToken(pathPart.Key), action);
}
}
}
}
public async Task<HttpResponseMessage> ReplicationTopologyDiscover()
{
var ttlAsString = GetQueryStringValue("ttl");
int ttl;
RavenJArray from;
if (string.IsNullOrEmpty(ttlAsString))
{
ttl = 10;
from = new RavenJArray();
}
else
{
ttl = int.Parse(ttlAsString);
from = await ReadJsonArrayAsync();
}
var replicationSchemaDiscoverer = new ReplicationTopologyDiscoverer(Database, from, ttl, Log);
var node = replicationSchemaDiscoverer.Discover();
return GetMessageWithObject(node);
}
public static RavenJToken MinimizeToken(RavenJToken obj, int depth = 0)
{
switch (obj.Type)
{
case JTokenType.Array:
var array = new RavenJArray();
foreach (var item in ((RavenJArray)obj))
{
array.Add(MinimizeToken(item, depth + 1));
}
return array;
case JTokenType.Object:
var ravenJObject = ((RavenJObject)obj);
if (ravenJObject.ContainsKey(Constants.Metadata) == false)
{
// this might be a wrapper object, let check for first level arrays
if (depth == 0)
{
var newRootObj = new RavenJObject();
foreach (var prop in ravenJObject)
{
newRootObj[prop.Key] = prop.Value.Type == JTokenType.Array ?
MinimizeToken(prop.Value, depth + 1) :
prop.Value;
}
return newRootObj;
}
return obj;
}
var newObj = new RavenJObject();
newObj[Constants.Metadata] = ravenJObject[Constants.Metadata];
return newObj;
default:
return obj;
}
}
/// <summary>
/// Filters the headers from unwanted headers
/// </summary>
/// <param name="self">The self.</param>
/// <param name="isServerDocument">if set to <c>true</c> [is server document].</param>
/// <returns></returns>public static RavenJObject FilterHeaders(this System.Collections.Specialized.NameValueCollection self, bool isServerDocument)
public static RavenJObject FilterHeaders(this NameValueCollection self, bool isServerDocument)
{
var metadata = new RavenJObject(StringComparer.InvariantCultureIgnoreCase);
foreach (string header in self)
{
try
{
if(header.StartsWith("Temp"))
continue;
if (HeadersToIgnoreClient.Contains(header))
continue;
if (isServerDocument && HeadersToIgnoreServerDocument.Contains(header))
continue;
var values = self.GetValues(header);
var headerName = CaptureHeaderName(header);
// TODO: Can values be null?
if (values.Length == 1)
metadata[headerName] = GetValue(values[0]);
else
metadata[headerName] = new RavenJArray(values.Select(GetValue));
}
catch (Exception exc)
{
throw new JsonReaderException(string.Concat("Unable to Filter Header: ", header), exc);
}
}
return metadata;
}
public static IEnumerable <dynamic> GetDocumentsFromString(string json)
{
return(RavenJArray.Parse(json).Cast <RavenJObject>().Select(JsonToExpando.Convert));
}
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, BulkOperationOptions options, Func<string, TransactionInformation, object> batchOperation)
{
options = options ?? new BulkOperationOptions();
var array = new RavenJArray();
var bulkIndexQuery = new IndexQuery
{
Query = indexQuery.Query,
Start = indexQuery.Start,
Cutoff = indexQuery.Cutoff ?? SystemTime.UtcNow,
WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow,
PageSize = int.MaxValue,
FieldsToFetch = new[] { Constants.DocumentIdFieldName },
SortedFields = indexQuery.SortedFields,
HighlighterPreTags = indexQuery.HighlighterPreTags,
HighlighterPostTags = indexQuery.HighlighterPostTags,
HighlightedFields = indexQuery.HighlightedFields,
SortHints = indexQuery.SortHints
};
bool stale;
var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);
if (stale && options.AllowStale == false)
{
if (options.StaleTimeout != null)
{
var staleWaitTimeout = Stopwatch.StartNew();
while (stale && staleWaitTimeout.Elapsed < options.StaleTimeout)
{
queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);
if(stale)
SystemTime.Wait(100);
}
}
if (stale)
{
if (options.StaleTimeout != null)
throw new InvalidOperationException("Bulk operation cancelled because the index is stale and StaleTimout of " + options.StaleTimeout + "passed");
throw new InvalidOperationException("Bulk operation cancelled because the index is stale and allowStale is false");
}
}
var token = tokenSource.Token;
const int batchSize = 1024;
int maxOpsPerSec = options.MaxOpsPerSec ?? int.MaxValue;
using (var enumerator = queryResults.GetEnumerator())
{
var duration = Stopwatch.StartNew();
var operations = 0;
while (true)
{
database.WorkContext.UpdateFoundWork();
if (timeout != null)
timeout.Delay();
var batchCount = 0;
var shouldWaitNow = false;
token.ThrowIfCancellationRequested();
using (database.DocumentLock.Lock())
{
database.TransactionalStorage.Batch(actions =>
{
while (batchCount < batchSize && enumerator.MoveNext())
{
batchCount++;
operations++;
var result = batchOperation(enumerator.Current, transactionInformation);
if(options.RetrieveDetails)
array.Add(RavenJObject.FromObject(result));
if (operations >= maxOpsPerSec && duration.ElapsedMilliseconds < 1000)
{
shouldWaitNow = true;
break;
}
}
});
if (shouldWaitNow)
{
SystemTime.Wait(500);
operations = 0;
duration.Restart();
continue;
}
}
if (batchCount < batchSize) break;
}
}
return array;
}
private JsValue ToJsArray(Engine engine, RavenJArray array)
{
var elements = new JsValue[array.Length];
for (var i = 0; i < array.Length; i++)
{
elements[i] = ToJsInstance(engine, array[i]);
}
return engine.Array.Construct(elements);
}
public override void Respond(IHttpContext context)
{
RavenJArray itemsToLoad;
if (context.Request.HttpMethod == "POST")
{
itemsToLoad = context.ReadJsonArray();
}
else
{
itemsToLoad = new RavenJArray(context.Request.QueryString.GetValues("id"));
}
var result = new MultiLoadResult();
var loadedIds = new HashSet <string>();
var includes = context.Request.QueryString.GetValues("include") ?? new string[0];
var transactionInformation = GetRequestTransaction(context);
var includedEtags = new List <byte>();
Database.TransactionalStorage.Batch(actions =>
{
var addIncludesCommand = new AddIncludesCommand(Database, transactionInformation, (etag, includedDoc) =>
{
includedEtags.AddRange(etag.ToByteArray());
result.Includes.Add(includedDoc);
}, includes, loadedIds);
foreach (RavenJToken item in itemsToLoad)
{
var value = item.Value <string>();
if (loadedIds.Add(value) == false)
{
continue;
}
var documentByKey = Database.Get(value, transactionInformation);
if (documentByKey == null)
{
continue;
}
result.Results.Add(documentByKey.ToJson());
if (documentByKey.Etag != null)
{
includedEtags.AddRange(documentByKey.Etag.Value.ToByteArray());
}
includedEtags.Add((documentByKey.NonAuthoritativeInformation ?? false) ? (byte)0 : (byte)1);
addIncludesCommand.Execute(documentByKey.DataAsJson);
}
});
Guid computedEtag;
using (var md5 = MD5.Create())
{
var computeHash = md5.ComputeHash(includedEtags.ToArray());
computedEtag = new Guid(computeHash);
}
if (context.MatchEtag(computedEtag))
{
context.SetStatusToNotModified();
return;
}
context.Response.AddHeader("ETag", computedEtag.ToString());
context.WriteJson(result);
}
private bool TryReplicationDocuments(ReplicationStrategy destination, RavenJArray jsonDocuments, out string lastError)
{
try
{
log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination);
var url = destination.ConnectionStringOptions.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl()
+ "&dbid=" + docDb.TransactionalStorage.Id +
"&count=" + jsonDocuments.Length;
var sp = Stopwatch.StartNew();
using (HttpRavenRequestFactory.Expect100Continue(destination.ConnectionStringOptions.Url))
{
var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions, GetRequestBuffering(destination));
request.Write(jsonDocuments);
request.ExecuteRequest(docDb.WorkContext.CancellationToken);
log.Info("Replicated {0} documents to {1} in {2:#,#;;0} ms", jsonDocuments.Length, destination, sp.ElapsedMilliseconds);
lastError = "";
return true;
}
}
catch (WebException e)
{
HandleRequestBufferingErrors(e, destination);
var response = e.Response as HttpWebResponse;
if (response != null)
{
var responseStream = response.GetResponseStream();
if (responseStream != null)
{
using (var streamReader = new StreamReader(responseStream))
{
var error = streamReader.ReadToEnd();
log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
}
}
else
{
log.WarnException("Replication to " + destination + " had failed", e);
}
}
else
{
log.WarnException("Replication to " + destination + " had failed", e);
}
lastError = e.Message;
return false;
}
catch (Exception e)
{
log.WarnException("Replication to " + destination + " had failed", e);
lastError = e.Message;
return false;
}
}
public ArrayWithWarning(RavenJArray mergedArray)
{
MergedArray = mergedArray;
}
protected override Guid FlushBatch(List <RavenJObject> batch)
{
var sw = Stopwatch.StartNew();
var commands = new RavenJArray();
foreach (var doc in batch)
{
var metadata = doc.Value <RavenJObject>("@metadata");
doc.Remove("@metadata");
commands.Add(new RavenJObject
{
{ "Method", "PUT" },
{ "Document", doc },
{ "Metadata", metadata },
{ "Key", metadata.Value <string>("@id") }
});
}
var retries = retriesCount;
HttpRavenRequest request = null;
BatchResult[] results;
while (true)
{
try
{
request = CreateRequest("/bulk_docs", "POST");
request.Write(commands);
results = request.ExecuteRequest <BatchResult[]>();
sw.Stop();
break;
}
catch (Exception e)
{
if (--retries == 0 || request == null)
{
throw;
}
sw.Stop();
LastRequestErrored = true;
ShowProgress("Error flushing to database, remaining attempts {0} - time {2:#,#} ms, will retry [{3:#,#.##;;0} kb compressed to {4:#,#.##;;0} kb]. Error: {1}",
retriesCount - retries, e, sw.ElapsedMilliseconds,
(double)request.NumberOfBytesWrittenUncompressed / 1024,
(double)request.NumberOfBytesWrittenCompressed / 1024);
}
}
total += batch.Count;
ShowProgress("{2,5:#,#}: Wrote {0:#,#;;0} in {1,6:#,#;;0} ms ({6:0.00} ms per doc) (total of {3:#,#;;0}) documents [{4:#,#.##;;0} kb compressed to {5:#,#.##;;0} kb]",
batch.Count, sw.ElapsedMilliseconds, ++count, total,
(double)request.NumberOfBytesWrittenUncompressed / 1024,
(double)request.NumberOfBytesWrittenCompressed / 1024,
Math.Round((double)sw.ElapsedMilliseconds / Math.Max(1, batch.Count), 2));
batch.Clear();
if (results.Length == 0)
{
return(Guid.Empty);
}
return(results.Last().Etag.Value);
}
private bool TryHandleArrayValue(int index, Dictionary <string, object> result, KeyValuePair <string, RavenJToken> prop)
{
var arrays = new List <RavenJArray>
{
(RavenJArray)prop.Value
};
for (var i = 0; i < docs.Length; i++)
{
if (i == index)
{
continue;
}
RavenJToken token;
if (docs[i].TryGetValue(prop.Key, out token) && token.Type != JTokenType.Array)
{
return(false);
}
if (token == null)
{
continue;
}
if (token.IsSnapshot)
{
token = token.CreateSnapshot();
}
arrays.Add((RavenJArray)token);
}
var mergedArray = new RavenJArray();
while (arrays.Count > 0)
{
var set = new HashSet <RavenJToken>(ravenJTokenEqualityComparer);
for (var i = 0; i < arrays.Count; i++)
{
if (arrays[i].Length == 0)
{
arrays.RemoveAt(i);
i -= 1;
continue;
}
set.Add(arrays[i][0]);
arrays[i].RemoveAt(0);
}
foreach (var ravenJToken in set)
{
mergedArray.Add(ravenJToken);
}
}
if (ravenJTokenEqualityComparer.Equals(mergedArray, prop.Value))
{
result.Add(prop.Key, mergedArray);
return(true);
}
result.Add(prop.Key, new ArrayWithWarning(mergedArray));
return(true);
}
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource)
{
RavenJArray jsonDocuments = null;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag)
.Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs
.Where(x => x.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source
.Where(x=> x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict
.Select(x=>
{
DocumentRetriever.EnsureIdInMetadata(x);
return x;
})
.Take(100)
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return jsonDocuments;
}
private async Task <HttpResponseMessage> GetQueriesResponse(bool isGet)
{
RavenJArray itemsToLoad;
if (isGet == false)
{
try
{
itemsToLoad = await ReadJsonArrayAsync().ConfigureAwait(false);
}
catch (InvalidOperationException e)
{
if (Log.IsDebugEnabled)
{
Log.DebugException("Failed to deserialize query request.", e);
}
return(GetMessageWithObject(new
{
Message = "Could not understand json, please check its validity."
}, (HttpStatusCode)422)); //http code 422 - Unprocessable entity
}
catch (InvalidDataException e)
{
if (Log.IsDebugEnabled)
{
Log.DebugException("Failed to deserialize query request.", e);
}
return(GetMessageWithObject(new
{
e.Message
}, (HttpStatusCode)422)); //http code 422 - Unprocessable entity
}
AddRequestTraceInfo(sb =>
{
foreach (var item in itemsToLoad)
{
sb.Append("\t").Append(item).AppendLine();
}
});
}
else
{
itemsToLoad = new RavenJArray(GetQueryStringValues("id").Cast <object>());
}
var result = new MultiLoadResult();
var loadedIds = new HashSet <string>();
var includedIds = new HashSet <string>();
var includes = GetQueryStringValues("include") ?? new string[0];
var transformer = GetQueryStringValue("transformer") ?? GetQueryStringValue("resultTransformer");
var transformerParameters = this.ExtractTransformerParameters();
var includedEtags = new List <byte>();
if (string.IsNullOrEmpty(transformer) == false)
{
var transformerDef = Database.IndexDefinitionStorage.GetTransformer(transformer);
if (transformerDef == null)
{
return(GetMessageWithObject(new { Error = "No such transformer: " + transformer }, HttpStatusCode.BadRequest));
}
includedEtags.AddRange(transformerDef.GetHashCodeBytes());
}
Database.TransactionalStorage.Batch(actions =>
{
foreach (RavenJToken item in itemsToLoad)
{
var value = item.Value <string>();
if (loadedIds.Add(value) == false)
{
continue;
}
var documentByKey = string.IsNullOrEmpty(transformer)
? Database.Documents.Get(value)
: Database.Documents.GetWithTransformer(value, transformer, transformerParameters, out includedIds);
if (documentByKey == null)
{
if (ClientIsV3OrHigher(Request))
{
result.Results.Add(null);
}
continue;
}
result.Results.Add(documentByKey.ToJson());
if (documentByKey.Etag != null)
{
includedEtags.AddRange(documentByKey.Etag.ToByteArray());
}
// TODO: Revise this.
includedEtags.Add((false) ? (byte)0 : (byte)1);
}
var addIncludesCommand = new AddIncludesCommand(Database, (etag, includedDoc) =>
{
includedEtags.AddRange(etag.ToByteArray());
result.Includes.Add(includedDoc);
}, includes, loadedIds);
foreach (var item in result.Results.Where(item => item != null))
{
addIncludesCommand.Execute(item);
}
});
foreach (var includedId in includedIds)
{
var doc = Database.Documents.Get(includedId);
if (doc == null)
{
continue;
}
includedEtags.AddRange(doc.Etag.ToByteArray());
result.Includes.Add(doc.ToJson());
}
var computeHash = Hashing.Metro128.Calculate(includedEtags.ToArray());
Etag computedEtag = Etag.FromHash(computeHash);
if (MatchEtag(computedEtag))
{
return(GetEmptyMessage(HttpStatusCode.NotModified));
}
var msg = GetMessageWithObject(result);
WriteETag(computedEtag, msg);
AddRequestTraceInfo(sb => sb.Append("Results count: {0}, includes count: {1}", result.Results.Count, result.Includes.Count).AppendLine());
return(msg);
}
/* Checks for updates and returns release informations when there is newer one.
* Returns null if there is no newer release.
* An existing last checked release will be discarded.
* Throws UpdaterException if error occurs.
*/
public static Release CheckForUpdates()
{
if (Latest != null)
{
if (Latest.IsDownloading)
{
throw new UpdaterException("Download already in progress");
}
Latest.Dispose();
}
Latest = null;
IsChecked = IsInstalled = false;
var webClient = new UpdaterWebClient();
webClient.Encoding = Encoding.UTF8;
try
{
string json = webClient.DownloadString(GitAPILatestReleaseURL);
RavenJArray releases = RavenJArray.Parse(json);
if (releases.Length < 1)
{
throw new UpdaterException("No release found");
}
RavenJObject latest = (RavenJObject)releases[0];
RavenJArray assets = (RavenJArray)latest["assets"];
if (assets.Length < 1)
{
throw new UpdaterException("Package for release is missing");
}
string current = GetCurrentVersion();
string tag = latest["tag_name"].Value <string>();
if (tag == current)
{
IsChecked = true;
return(null);
}
string url = ((RavenJObject)assets[0])["browser_download_url"].Value <string>();
IsChecked = true;
Latest = new Release
{
Name = latest["name"].Value <string>(),
Description = latest["body"].Value <string>(),
Version = tag,
URI = new Uri(url)
};
}
catch (WebException e)
{
if (e.Status == WebExceptionStatus.ProtocolError)
{
throw new UpdaterException("HTTP " + ((int)((HttpWebResponse)e.Response).StatusCode) + " " + ((HttpWebResponse)e.Response).StatusDescription);
}
else
{
throw new UpdaterException(e.Message, e);
}
}
catch (Exception e)
{
throw new UpdaterException(e.Message, e);
}
return(Latest);
}
public static RavenJArray ReadJsonArray(this IHttpContext context)
{
using (var streamReader = new StreamReader(context.Request.InputStream, GetRequestEncoding(context)))
using (var jsonReader = new JsonTextReader(streamReader))
return(RavenJArray.Load(jsonReader));
}
private Tuple <RavenJArray, Guid> GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray attachments = null;
Guid lastAttachmentEtag = Guid.Empty;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int attachmentSinceLastEtag = 0;
List <AttachmentInformation> attachmentsToReplicate;
List <AttachmentInformation> filteredAttachmentsToReplicate;
lastAttachmentEtag = destinationsReplicationInformationForSource.LastAttachmentEtag;
while (true)
{
attachmentsToReplicate = actions.Attachments.GetAttachmentsAfter(lastAttachmentEtag, 100, 1024 * 1024 * 10)
.Concat(actions.Lists.Read(Constants.RavenReplicationAttachmentsTombstones, lastAttachmentEtag, 100)
.Select(x => new AttachmentInformation
{
Key = x.Key,
Etag = x.Etag,
Metadata = x.Data,
Size = 0,
}))
.OrderBy(x => x.Etag)
.ToList();
filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList();
attachmentSinceLastEtag += attachmentsToReplicate.Count;
if (attachmentsToReplicate.Count == 0 ||
filteredAttachmentsToReplicate.Count != 0)
{
break;
}
AttachmentInformation jsonDocument = attachmentsToReplicate.Last();
Guid attachmentEtag = jsonDocument.Etag;
log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag);
lastAttachmentEtag = attachmentEtag;
}
log.Debug(() =>
{
if (attachmentSinceLastEtag == 0)
{
return(string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag));
}
if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count)
{
return(string.Format("Replicating {0} attachments [>{1}] to {2}.",
attachmentSinceLastEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination));
}
var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key);
return(string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
attachmentSinceLastEtag,
filteredAttachmentsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag));
});
attachments = new RavenJArray(filteredAttachmentsToReplicate
.Select(x =>
{
var data = new byte[0];
if (x.Size > 0)
{
data = actions.Attachments.GetAttachment(x.Key).Data().ReadData();
}
return(new RavenJObject
{
{ "@metadata", x.Metadata },
{ "@id", x.Key },
{ "@etag", x.Etag.ToByteArray() },
{ "data", data }
});
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return(Tuple.Create(attachments, lastAttachmentEtag));
}
public static RavenJArray ToJArray <T>(IEnumerable <T> result)
{
return((RavenJArray)RavenJArray.FromObject(result, CreateDefaultJsonSerializer()));
}
public async Task DataDumperExportHandlesMaxEtagCorrectly()
{
using (var store = NewDocumentStore())
{
using (var session = store.OpenSession())
{
for (var i = 0; i < 10; i++)
{
session.Store(new User {
Name = "oren #" + (i + 1)
});
}
session.SaveChanges();
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", exportedDocs.Last().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-000000000008", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportDocuments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedDocs.Count());
Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", exportedDocs.Last().Value <RavenJObject>("@metadata").Value <string>("@etag"));
Assert.Equal("01000000-0000-0001-0000-00000000000B", lastEtag.ToString());
}
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.PutAttachment("attach/" + (i + 1), null, new MemoryStream(new [] { (byte)i }), new RavenJObject());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
var endEtag = startEtag.IncrementBy(2);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, endEtag);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(2, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", exportedAttachments.Last().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-000000000007", lastEtag.ToString());
}
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
writer.WriteStartArray();
var lastEtag = await dumper.ExportAttachments(writer, startEtag, null);
writer.WriteEndArray();
writer.Flush();
// read exported content
var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
Assert.Equal(5, exportedAttachments.Count());
Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", exportedAttachments.Last().Value <string>("Etag"));
Assert.Equal("02000000-0000-0001-0000-00000000000A", lastEtag.ToString());
}
WaitForIndexing(store);
store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery()
{
Query = "Tag:Users"
}).WaitForCompletion();
for (var i = 0; i < 10; i++)
{
store.DatabaseCommands.DeleteAttachment("attach/" + (i + 1), null);
}
Etag user6DeletionEtag = null, user9DeletionEtag = null, attach5DeletionEtag = null, attach7DeletionEtag = null;
WaitForUserToContinueTheTest(store);
store.SystemDatabase.TransactionalStorage.Batch(accessor =>
{
user6DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/6").Etag;
user9DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/9").Etag;
attach5DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/5").Etag;
attach7DeletionEtag =
accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/7").Etag;
});
using (var textStream = new StringWriter())
using (var writer = new JsonTextWriter(textStream))
{
var dumper = new CustomDataDumper(store.SystemDatabase);
writer.WriteStartObject();
var lastEtags = new LastEtagsInfo();
var exportResult = new ExportDataResult
{
LastDocDeleteEtag = user6DeletionEtag,
LastAttachmentsDeleteEtag = attach5DeletionEtag
};
lastEtags.LastDocDeleteEtag = user9DeletionEtag;
lastEtags.LastAttachmentsDeleteEtag = attach7DeletionEtag;
dumper.ExportDeletions(writer, exportResult, lastEtags).Wait();
writer.WriteEndObject();
writer.Flush();
// read exported content
var exportJson = RavenJObject.Parse(textStream.GetStringBuilder().ToString());
var docsKeys =
exportJson.Value <RavenJArray>("DocsDeletions").Select(x => x.Value <string>("Key")).ToArray();
var attachmentsKeys =
exportJson.Value <RavenJArray>("AttachmentsDeletions")
.Select(x => x.Value <string>("Key"))
.ToArray();
Assert.Equal(new [] { "users/7", "users/8", "users/9" }, docsKeys);
Assert.Equal(new [] { "attach/6", "attach/7" }, attachmentsKeys);
}
}
}
/// <summary>
/// Deserialize a <param name="self"/> to a list of instances of<typeparam name="T"/>
/// </summary>
public static T[] JsonDeserialization <T>(this RavenJArray self)
{
var serializer = CreateDefaultJsonSerializer();
return(self.Select(x => (T)serializer.Deserialize(new RavenJTokenReader(x), typeof(T))).ToArray());
}
public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation)
{
using (Database.DisableAllTriggersForCurrentThread())
{
metadata.Remove(Constants.RavenReplicationConflict); // you can't put conflicts
var oldVersion = Database.Documents.Get(key, transactionInformation);
if (oldVersion == null)
{
return;
}
if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null)
{
return;
}
var history = new RavenJArray();
metadata[Constants.RavenReplicationHistory] = history;
// this is a conflict document, holding document keys in the
// values of the properties
var conflicts = oldVersion.DataAsJson.Value <RavenJArray>("Conflicts");
if (conflicts == null)
{
return;
}
var list = new List <RavenJArray>
{
new RavenJArray(ReplicationData.GetHistory(metadata)) // first item to interleave
};
foreach (var prop in conflicts)
{
RavenJObject deletedMetadata;
Database.Documents.Delete(prop.Value <string>(), null, transactionInformation, out deletedMetadata);
if (deletedMetadata != null)
{
var conflictHistory = new RavenJArray(ReplicationData.GetHistory(deletedMetadata));
conflictHistory.Add(new RavenJObject
{
{ Constants.RavenReplicationVersion, deletedMetadata[Constants.RavenReplicationVersion] },
{ Constants.RavenReplicationSource, deletedMetadata[Constants.RavenReplicationSource] }
});
list.Add(conflictHistory);
}
}
int index = 0;
bool added = true;
while (added) // interleave the history from all conflicts
{
added = false;
foreach (var deletedMetadata in list)
{
// add the conflict history to the mix, so we make sure that we mark that we resolved the conflict
if (index < deletedMetadata.Length)
{
history.Add(deletedMetadata[index]);
added = true;
}
}
index++;
}
while (history.Length > Constants.ChangeHistoryLength)
{
history.RemoveAt(0);
}
}
}
private void ReplicateDelete(string id, RavenJObject metadata, TExternal incoming)
{
TInternal existingItem;
Etag existingEtag;
bool deleted;
var existingMetadata = TryGetExisting(id, out existingItem, out existingEtag, out deleted);
if (existingMetadata == null)
{
log.Debug("Replicating deleted item {0} from {1} that does not exist, ignoring", id, Src);
return;
}
if (existingMetadata.Value <bool>(Constants.RavenDeleteMarker)) //deleted locally as well
{
log.Debug("Replicating deleted item {0} from {1} that was deleted locally. Merging histories", id, Src);
var existingHistory = new RavenJArray(ReplicationData.GetHistory(existingMetadata));
var newHistory = new RavenJArray(ReplicationData.GetHistory(metadata));
foreach (var item in newHistory)
{
existingHistory.Add(item);
}
if (metadata.ContainsKey(Constants.RavenReplicationVersion) &&
metadata.ContainsKey(Constants.RavenReplicationSource))
{
existingHistory.Add(new RavenJObject
{
{ Constants.RavenReplicationVersion, metadata[Constants.RavenReplicationVersion] },
{ Constants.RavenReplicationSource, metadata[Constants.RavenReplicationSource] }
});
}
while (existingHistory.Length > Constants.ChangeHistoryLength)
{
existingHistory.RemoveAt(0);
}
MarkAsDeleted(id, metadata);
return;
}
if (Historian.IsDirectChildOfCurrent(metadata, existingMetadata)) // not modified
{
log.Debug("Delete of existing item {0} was replicated successfully from {1}", id, Src);
DeleteItem(id, existingEtag);
MarkAsDeleted(id, metadata);
return;
}
CreatedConflict createdConflict;
if (existingMetadata.Value <bool>(Constants.RavenReplicationConflict)) // locally conflicted
{
log.Debug("Replicating deleted item {0} from {1} that is already conflicted, adding to conflicts.", id, Src);
var savedConflictedItemId = SaveConflictedItem(id, metadata, incoming, existingEtag);
createdConflict = AppendToCurrentItemConflicts(id, savedConflictedItemId, existingMetadata, existingItem);
}
else
{
var newConflictId = SaveConflictedItem(id, metadata, incoming, existingEtag);
log.Debug("Existing item {0} is in conflict with replicated delete from {1}, marking item as conflicted", id, Src);
// we have a new conflict move the existing doc to a conflict and create a conflict document
var existingDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(existingEtag);
createdConflict = CreateConflict(id, newConflictId, existingDocumentConflictId, existingItem, existingMetadata);
}
Database.TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() =>
Database.RaiseNotifications(new ReplicationConflictNotification()
{
Id = id,
Etag = createdConflict.Etag,
Conflicts = createdConflict.ConflictedIds,
ItemType = ReplicationConflictTypes.DocumentReplicationConflict,
OperationType = ReplicationOperationTypes.Delete
}));
}
private bool TryReplicationAttachments(ReplicationStrategy destination, RavenJArray jsonAttachments, out string errorMessage)
{
try
{
var url = destination.ConnectionStringOptions.Url + "/replication/replicateAttachments?from=" +
UrlEncodedServerUrl() + "&dbid=" + docDb.TransactionalStorage.Id;
var sp = Stopwatch.StartNew();
using (HttpRavenRequestFactory.Expect100Continue(destination.ConnectionStringOptions.Url))
{
var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions, GetRequestBuffering(destination));
request.WriteBson(jsonAttachments);
request.ExecuteRequest(docDb.WorkContext.CancellationToken);
log.Info("Replicated {0} attachments to {1} in {2:#,#;;0} ms", jsonAttachments.Length, destination, sp.ElapsedMilliseconds);
errorMessage = "";
return true;
}
}
catch (WebException e)
{
HandleRequestBufferingErrors(e, destination);
var response = e.Response as HttpWebResponse;
if (response != null)
{
using (var streamReader = new StreamReader(response.GetResponseStreamWithHttpDecompression()))
{
var error = streamReader.ReadToEnd();
try
{
var ravenJObject = RavenJObject.Parse(error);
log.WarnException("Replication to " + destination + " had failed\r\n" + ravenJObject.Value<string>("Error"), e);
errorMessage = error;
return false;
}
catch (Exception)
{
}
log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
errorMessage = error;
}
}
else
{
log.WarnException("Replication to " + destination + " had failed", e);
errorMessage = e.Message;
}
return false;
}
catch (Exception e)
{
log.WarnException("Replication to " + destination + " had failed", e);
errorMessage = e.Message;
return false;
}
}
public DestinationStats()
{
LastStats = new RavenJArray();
}
public static RavenJObject[] ReadAllEntriesFromIndex(IndexReader reader)
{
if (reader.MaxDoc > 512 * 1024)
{
throw new InvalidOperationException("Refusing to extract all index entires from an index with " + reader.MaxDoc +
" entries, because of the probable time / memory costs associated with that." +
Environment.NewLine +
"Viewing Index Entries are a debug tool, and should not be used on indexes of this size. You might want to try Luke, instead.");
}
var results = new RavenJObject[reader.MaxDoc];
using (var termDocs = reader.TermDocs())
using (var termEnum = reader.Terms())
{
while (termEnum.Next())
{
var term = termEnum.Term;
if (term == null)
break;
var text = term.Text;
termDocs.Seek(termEnum);
for (int i = 0; i < termEnum.DocFreq() && termDocs.Next(); i++)
{
RavenJObject result = results[termDocs.Doc];
if (result == null)
results[termDocs.Doc] = result = new RavenJObject();
var propertyName = term.Field;
if (propertyName.EndsWith("_ConvertToJson") ||
propertyName.EndsWith("_IsArray"))
continue;
if (result.ContainsKey(propertyName))
{
switch (result[propertyName].Type)
{
case JTokenType.Array:
((RavenJArray)result[propertyName]).Add(text);
break;
case JTokenType.String:
result[propertyName] = new RavenJArray
{
result[propertyName],
text
};
break;
default:
throw new ArgumentException("No idea how to handle " + result[propertyName].Type);
}
}
else
{
result[propertyName] = text;
}
}
}
}
return results;
}
private bool TryReplicationDocuments(RavenConnectionStringOptions destination, RavenJArray jsonDocuments)
{
try
{
log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination);
var request = (HttpWebRequest)WebRequest.Create(destination.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl());
request.UseDefaultCredentials = true;
request.ContentType = "application/json; charset=utf-8";
request.Credentials = destination.Credentials ?? CredentialCache.DefaultNetworkCredentials;
request.Method = "POST";
using (var stream = request.GetRequestStream())
using (var streamWriter = new StreamWriter(stream, Encoding.UTF8))
{
jsonDocuments.WriteTo(new JsonTextWriter(streamWriter));
streamWriter.Flush();
stream.Flush();
}
using (request.GetResponse())
{
log.Info("Replicated {0} documents to {1}", jsonDocuments.Length, destination);
}
return(true);
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null)
{
using (var streamReader = new StreamReader(response.GetResponseStream()))
{
var error = streamReader.ReadToEnd();
log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
}
}
else
{
log.WarnException("Replication to " + destination + " had failed", e);
}
return(false);
}
catch (Exception e)
{
log.WarnException("Replication to " + destination + " had failed", e);
return(false);
}
}
public override void OnPut(string key, RavenJObject jsonReplicationDocument, RavenJObject metadata, TransactionInformation transactionInformation)
{
if (key.StartsWith("Raven/", StringComparison.OrdinalIgnoreCase) && // we don't deal with system documents
key.StartsWith("Raven/Hilo/", StringComparison.OrdinalIgnoreCase) == false) // except for hilos
{
return;
}
using (Database.DisableAllTriggersForCurrentThread())
{
var documentMetadata = GetDocumentMetadata(key);
if (documentMetadata != null)
{
var history = new RavenJArray(ReplicationData.GetHistory(documentMetadata));
if (documentMetadata.ContainsKey(Constants.RavenReplicationMergedHistory) == false)
{
if (documentMetadata.ContainsKey(Constants.RavenReplicationVersion) &&
documentMetadata.ContainsKey(Constants.RavenReplicationSource))
{
history.Add(new RavenJObject
{
{ Constants.RavenReplicationVersion, documentMetadata[Constants.RavenReplicationVersion] },
{ Constants.RavenReplicationSource, documentMetadata[Constants.RavenReplicationSource] }
});
}
else
{
history.Add(new RavenJObject
{
{ Constants.RavenReplicationVersion, 0 },
{ Constants.RavenReplicationSource, RavenJToken.FromObject(Database.TransactionalStorage.Id) }
});
}
var sources = new HashSet <RavenJToken>(RavenJTokenEqualityComparer.Default);
int pos = history.Length - 1;
for (; pos >= 0; pos--)
{
var source = ((RavenJObject)history[pos])[Constants.RavenReplicationSource];
if (sources.Contains(source))
{
history.RemoveAt(pos);
continue;
}
sources.Add(source);
}
metadata[Constants.RavenReplicationMergedHistory] = true;
metadata[Constants.RavenReplicationHistory] = history;
}
//If we have the flag we must have Constants.RavenReplicationVersion and Constants.RavenReplicationSource too
//Here we assume that the replication history is in the form of a "sorted dictionary" so we just need to remove
//the entry with the current source id and insert the new version at the end of the history.
else
{
int i = history.Length - 1;
for (; i >= 0; i--)
{
var currentEntry = history[i];
if (RavenJTokenEqualityComparer.Default.Equals(((RavenJObject)currentEntry)
[Constants.RavenReplicationSource], documentMetadata[Constants.RavenReplicationSource]))
{
break;
}
}
if (i != -1)
{
history.RemoveAt(i);
}
history.Add(new RavenJObject
{
{ Constants.RavenReplicationVersion, documentMetadata[Constants.RavenReplicationVersion] },
{ Constants.RavenReplicationSource, documentMetadata[Constants.RavenReplicationSource] }
});
metadata[Constants.RavenReplicationHistory] = history;
}
}
metadata[Constants.RavenReplicationVersion] = RavenJToken.FromObject(ReplicationHiLo.NextId(Database));
metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(Database.TransactionalStorage.Id);
}
}
private RavenJToken ToRavenJToken(JsInstance v)
{
switch (v.Class)
{
case JsInstance.TYPE_OBJECT:
case JsInstance.CLASS_OBJECT:
return(ToRavenJObject((JsObject)v));
case JsInstance.CLASS_DATE:
var dt = (DateTime)v.Value;
return(new RavenJValue(dt));
case JsInstance.TYPE_NUMBER:
case JsInstance.CLASS_NUMBER:
var num = (double)v.Value;
var integer = Math.Truncate(num);
if (Math.Abs(num - integer) < double.Epsilon)
{
return(new RavenJValue((long)integer));
}
return(new RavenJValue(num));
case JsInstance.TYPE_STRING:
case JsInstance.TYPE_BOOLEAN:
case JsInstance.CLASS_STRING:
case JsInstance.CLASS_BOOLEAN:
return(new RavenJValue(v.Value));
case JsInstance.CLASS_NULL:
case JsInstance.TYPE_NULL:
return(RavenJValue.Null);
case JsInstance.CLASS_UNDEFINED:
case JsInstance.TYPE_UNDEFINED:
return(RavenJValue.Null);
case JsInstance.CLASS_ARRAY:
var jsArray = ((JsArray)v);
var rja = new RavenJArray();
for (int i = 0; i < jsArray.Length; i++)
{
var jsInstance = jsArray.get(i);
var ravenJToken = ToRavenJToken(jsInstance);
if (ravenJToken == null)
{
continue;
}
rja.Add(ravenJToken);
}
return(rja);
case JsInstance.CLASS_REGEXP:
case JsInstance.CLASS_ERROR:
case JsInstance.CLASS_ARGUMENTS:
case JsInstance.CLASS_DESCRIPTOR:
case JsInstance.CLASS_FUNCTION:
return(null);
default:
throw new NotSupportedException(v.Class);
}
}
private RavenJToken ToRavenJToken(JsValue v, string propertyKey)
{
if (v.IsBoolean())
return new RavenJValue(v.AsBoolean());
if (v.IsString())
{
const string RavenDataByteArrayToBase64 = "raven-data:byte[];base64,";
var value = v.AsString();
if (value != null && value.StartsWith(RavenDataByteArrayToBase64))
{
value = value.Remove(0, RavenDataByteArrayToBase64.Length);
var byteArray = Convert.FromBase64String(value);
return new RavenJValue(byteArray);
}
return new RavenJValue(value);
}
if (v.IsNumber())
{
var num = v.AsNumber();
KeyValuePair<RavenJValue, JsValue> property;
if (propertiesByValue.TryGetValue(propertyKey, out property))
{
var originalValue = property.Key;
if (originalValue.Type == JTokenType.Float)
return new RavenJValue(num);
if (originalValue.Type == JTokenType.Integer)
{
// If the current value is exactly as the original value, we can return the original value before we made the JS conversion,
// which will convert a Int64 to jsFloat.
var originalJsValue = property.Value;
if (originalJsValue.IsNumber() && Math.Abs(num - originalJsValue.AsNumber()) < double.Epsilon)
return originalValue;
return new RavenJValue((long)num);
}
}
// If we don't have the type, assume that if the number ending with ".0" it actually an integer.
var integer = Math.Truncate(num);
if (Math.Abs(num - integer) < double.Epsilon)
return new RavenJValue((long)integer);
return new RavenJValue(num);
}
if (v.IsNull())
return RavenJValue.Null;
if (v.IsUndefined())
return RavenJValue.Null;
if (v.IsArray())
{
var jsArray = v.AsArray();
var rja = new RavenJArray();
foreach (var property in jsArray.Properties)
{
if (property.Key == "length")
continue;
var jsInstance = property.Value.Value;
if (!jsInstance.HasValue)
continue;
var ravenJToken = ToRavenJToken(jsInstance.Value, CreatePropertyKey(property.Key, propertyKey));
if (ravenJToken == null)
continue;
rja.Add(ravenJToken);
}
return rja;
}
if (v.IsObject())
return ToRavenJObject(v, propertyKey);
if (v.IsRegExp())
return null;
throw new NotSupportedException(v.Type.ToString());
}
private static RavenJObject CreateJsonDocumentFromLuceneDocument(Document document)
{
var field = document.GetField(Constants.ReduceValueFieldName);
if (field != null)
{
return(RavenJObject.Parse(field.StringValue));
}
var ravenJObject = new RavenJObject();
var fields = document.GetFields();
var arrayMarkers = fields
.Where(x => x.Name.EndsWith("_IsArray"))
.Select(x => x.Name)
.ToList();
foreach (var fieldable in fields)
{
var stringValue = GetStringValue(fieldable);
var isArrayMarker = fieldable.Name.EndsWith("_IsArray");
var isArray = !isArrayMarker && arrayMarkers.Contains(fieldable.Name + "_IsArray");
RavenJToken token;
var isJson = RavenJToken.TryParse(stringValue, out token);
RavenJToken value;
if (ravenJObject.TryGetValue(fieldable.Name, out value) == false)
{
if (isArray)
{
ravenJObject[fieldable.Name] = new RavenJArray {
isJson?token : stringValue
}
}
;
else if (isArrayMarker)
{
var fieldName = fieldable.Name.Substring(0, fieldable.Name.Length - 8);
ravenJObject[fieldable.Name] = isJson ? token : stringValue;
ravenJObject[fieldName] = new RavenJArray();
}
else
{
ravenJObject[fieldable.Name] = isJson ? token : stringValue;
}
}
else
{
var ravenJArray = value as RavenJArray;
if (ravenJArray != null)
{
ravenJArray.Add(isJson ? token : stringValue);
}
else
{
ravenJArray = new RavenJArray {
value, isJson ? token : stringValue
};
ravenJObject[fieldable.Name] = ravenJArray;
}
}
}
return(ravenJObject);
}
protected override Guid FlushBatch(List<RavenJObject> batch)
{
var sw = Stopwatch.StartNew();
var commands = new RavenJArray();
foreach (var doc in batch)
{
var metadata = doc.Value<RavenJObject>("@metadata");
doc.Remove("@metadata");
commands.Add(new RavenJObject
{
{"Method", "PUT"},
{"Document", doc},
{"Metadata", metadata},
{"Key", metadata.Value<string>("@id")}
});
}
var retries = retriesCount;
HttpRavenRequest request = null;
BatchResult[] results;
while (true)
{
try
{
request = CreateRequest("/bulk_docs", "POST");
request.Write(commands);
results = request.ExecuteRequest<BatchResult[]>();
sw.Stop();
break;
}
catch (Exception e)
{
if (--retries == 0 || request == null)
throw;
sw.Stop();
LastRequestErrored = true;
ShowProgress("Error flushing to database, remaining attempts {0} - time {2:#,#} ms, will retry [{3:#,#.##;;0} kb compressed to {4:#,#.##;;0} kb]. Error: {1}",
retriesCount - retries, e, sw.ElapsedMilliseconds,
(double)request.NumberOfBytesWrittenUncompressed / 1024,
(double)request.NumberOfBytesWrittenCompressed / 1024);
}
}
total += batch.Count;
ShowProgress("{2,5:#,#}: Wrote {0:#,#;;0} in {1,6:#,#;;0} ms ({6:0.00} ms per doc) (total of {3:#,#;;0}) documents [{4:#,#.##;;0} kb compressed to {5:#,#.##;;0} kb]",
batch.Count, sw.ElapsedMilliseconds, ++count, total,
(double)request.NumberOfBytesWrittenUncompressed / 1024,
(double)request.NumberOfBytesWrittenCompressed / 1024,
Math.Round((double)sw.ElapsedMilliseconds / Math.Max(1, batch.Count), 2));
batch.Clear();
if (results.Length == 0)
return Guid.Empty;
return results.Last().Etag.Value;
}
private RavenJToken ToRavenJToken(JsValue v, string propertyKey, bool recursiveCall)
{
if (v.IsBoolean())
{
return(new RavenJValue(v.AsBoolean()));
}
if (v.IsString())
{
const string RavenDataByteArrayToBase64 = "raven-data:byte[];base64,";
var valueAsObject = v.ToObject();
var value = valueAsObject != null?valueAsObject.ToString() : null;
if (value != null && value.StartsWith(RavenDataByteArrayToBase64))
{
value = value.Remove(0, RavenDataByteArrayToBase64.Length);
var byteArray = Convert.FromBase64String(value);
return(new RavenJValue(byteArray));
}
return(new RavenJValue(value));
}
if (v.IsNumber())
{
var num = v.AsNumber();
KeyValuePair <RavenJValue, JsValue> property;
if (propertiesByValue.TryGetValue(propertyKey, out property))
{
var originalValue = property.Key;
if (originalValue.Type == JTokenType.Float)
{
return(new RavenJValue(num));
}
if (originalValue.Type == JTokenType.Integer)
{
// If the current value is exactly as the original value, we can return the original value before we made the JS conversion,
// which will convert a Int64 to jsFloat.
var originalJsValue = property.Value;
if (originalJsValue.IsNumber() && Math.Abs(num - originalJsValue.AsNumber()) < double.Epsilon)
{
return(originalValue);
}
return(new RavenJValue((long)num));
}
}
// If we don't have the type, assume that if the number ending with ".0" it actually an integer.
var integer = Math.Truncate(num);
if (Math.Abs(num - integer) < double.Epsilon)
{
return(new RavenJValue((long)integer));
}
return(new RavenJValue(num));
}
if (v.IsNull())
{
return(RavenJValue.Null);
}
if (v.IsUndefined())
{
return(RavenJValue.Null);
}
if (v.IsArray())
{
var jsArray = v.AsArray();
var rja = new RavenJArray();
foreach (var property in jsArray.GetOwnProperties())
{
if (InheritedProperties.Contains(property.Key))
{
continue;
}
var jsInstance = property.Value.Value;
if (!jsInstance.HasValue)
{
continue;
}
var ravenJToken = ToRavenJToken(jsInstance.Value, CreatePropertyKey(property.Key, propertyKey), recursiveCall);
if (ravenJToken == null)
{
continue;
}
rja.Add(ravenJToken);
}
return(rja);
}
if (v.IsObject())
{
return(ToRavenJObject(v, propertyKey, recursiveCall));
}
if (v.IsRegExp())
{
return(null);
}
throw new NotSupportedException(v.Type.ToString());
}
private bool TryReplicationDocuments(RavenConnectionStringOptions destination, RavenJArray jsonDocuments)
{
try
{
log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination);
var request = (HttpWebRequest)WebRequest.Create(destination.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl());
request.UseDefaultCredentials = true;
request.ContentType = "application/json; charset=utf-8";
request.Credentials = destination.Credentials ?? CredentialCache.DefaultNetworkCredentials;
request.Method = "POST";
using (var stream = request.GetRequestStream())
using (var streamWriter = new StreamWriter(stream, Encoding.UTF8))
{
jsonDocuments.WriteTo(new JsonTextWriter(streamWriter));
streamWriter.Flush();
stream.Flush();
}
using (request.GetResponse())
{
log.Info("Replicated {0} documents to {1}", jsonDocuments.Length, destination);
}
return true;
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null)
{
using (var streamReader = new StreamReader(response.GetResponseStream()))
{
var error = streamReader.ReadToEnd();
log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
}
}
else
{
log.WarnException("Replication to " + destination + " had failed", e);
}
return false;
}
catch (Exception e)
{
log.WarnException("Replication to " + destination + " had failed", e);
return false;
}
}
public static RavenJObject[] ReadAllEntriesFromIndex(IndexReader reader)
{
if (reader.MaxDoc > 512 * 1024)
{
throw new InvalidOperationException("Refusing to extract all index entires from an index with " + reader.MaxDoc +
" entries, because of the probable time / memory costs associated with that." +
Environment.NewLine +
"Viewing Index Entries are a debug tool, and should not be used on indexes of this size. You might want to try Luke, instead.");
}
var results = new RavenJObject[reader.MaxDoc];
using (var termDocs = reader.TermDocs())
using (var termEnum = reader.Terms())
{
while (termEnum.Next())
{
var term = termEnum.Term;
if (term == null)
{
break;
}
var text = term.Text;
termDocs.Seek(termEnum);
for (int i = 0; i < termEnum.DocFreq() && termDocs.Next(); i++)
{
RavenJObject result = results[termDocs.Doc];
if (result == null)
{
results[termDocs.Doc] = result = new RavenJObject();
}
var propertyName = term.Field;
if (propertyName.EndsWith("_ConvertToJson") ||
propertyName.EndsWith("_IsArray"))
{
continue;
}
if (result.ContainsKey(propertyName))
{
switch (result[propertyName].Type)
{
case JTokenType.Array:
((RavenJArray)result[propertyName]).Add(text);
break;
case JTokenType.String:
result[propertyName] = new RavenJArray
{
result[propertyName],
text
};
break;
default:
throw new ArgumentException("No idea how to handle " + result[propertyName].Type);
}
}
else
{
result[propertyName] = text;
}
}
}
}
return(results);
}
private RavenJArray GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource)
{
RavenJArray jsonAttachments = null;
try
{
string destinationInstanceId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonAttachments = new RavenJArray(actions.Attachments.GetAttachmentsAfter(destinationsReplicationInformationForSource.LastAttachmentEtag)
.Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs
.Where(x => x.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) != destinationInstanceId) // prevent replicating back to source
.Where(x => x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict
.Take(100)
.Select(x => new RavenJObject
{
{"@metadata", x.Metadata},
{"@id", x.Key},
{"@etag", x.Etag.ToByteArray()},
{"data", actions.Attachments.GetAttachment(x.Key).Data}
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return jsonAttachments;
}
protected override async Task <Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag)
{
var totalCount = 0;
while (true)
{
try
{
if (SmugglerOptions.Limit - totalCount <= 0)
{
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
var maxRecords = Math.Min(SmugglerOptions.Limit - totalCount, SmugglerOptions.BatchSize);
RavenJArray attachmentInfo = null;
var request = CreateRequest("/static/?pageSize=" + maxRecords + "&etag=" + lastEtag);
request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader)));
if (attachmentInfo.Length == 0)
{
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length,
totalCount);
foreach (var item in attachmentInfo)
{
ShowProgress("Downloading attachment: {0}", item.Value <string>("Key"));
byte[] attachmentData = null;
var requestData = CreateRequest("/static/" + item.Value <string>("Key"));
requestData.ExecuteRequest(reader => attachmentData = reader.ReadData());
new RavenJObject
{
{ "Data", attachmentData },
{ "Metadata", item.Value <RavenJObject>("Metadata") },
{ "Key", item.Value <string>("Key") }
}
.WriteTo(jsonWriter);
totalCount++;
lastEtag = Etag.Parse(item.Value <string>("Etag"));
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
