public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation)
{
if (key.StartsWith("Raven/", StringComparison.OrdinalIgnoreCase) && // we don't deal with system documents
key.StartsWith("Raven/Hilo/", StringComparison.OrdinalIgnoreCase) == false) // except for hilos
return;
using (Database.DisableAllTriggersForCurrentThread())
{
var documentMetadata = GetDocumentMetadata(key);
if (documentMetadata != null)
{
RavenJArray history = new RavenJArray(ReplicationData.GetHistory(documentMetadata));
metadata[Constants.RavenReplicationHistory] = history;
if (documentMetadata.ContainsKey(Constants.RavenReplicationVersion) &&
documentMetadata.ContainsKey(Constants.RavenReplicationSource))
{
history.Add(new RavenJObject
{
{Constants.RavenReplicationVersion, documentMetadata[Constants.RavenReplicationVersion]},
{Constants.RavenReplicationSource, documentMetadata[Constants.RavenReplicationSource]}
});
}
while (history.Length > Constants.ChangeHistoryLength)
{
history.RemoveAt(0);
}
}
metadata[Constants.RavenReplicationVersion] = RavenJToken.FromObject(HiLo.NextId());
metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(Database.TransactionalStorage.Id);
}
}
public override void OnPut(string key, Stream data, RavenJObject metadata)
{
if (key.StartsWith("Raven/")) // we don't deal with system attachment
return;
using (Database.DisableAllTriggersForCurrentThread())
{
var attachmentMetadata = GetAttachmentMetadata(key);
if (attachmentMetadata != null)
{
RavenJArray history = new RavenJArray(metadata.Value<RavenJArray>(Constants.RavenReplicationHistory));
metadata[Constants.RavenReplicationHistory] = history;
if (attachmentMetadata.ContainsKey(Constants.RavenReplicationVersion) &&
attachmentMetadata.ContainsKey(Constants.RavenReplicationSource))
{
history.Add(new RavenJObject
{
{Constants.RavenReplicationVersion, attachmentMetadata[Constants.RavenReplicationVersion]},
{Constants.RavenReplicationSource, attachmentMetadata[Constants.RavenReplicationSource]}
});
}
if (history.Length > Constants.ChangeHistoryLength)
{
history.RemoveAt(0);
}
}
metadata[Constants.RavenReplicationVersion] = RavenJToken.FromObject(HiLo.NextId());
metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(Database.TransactionalStorage.Id);
}
}
public override void OnPut(string key, Stream data, RavenJObject metadata)
{
using (Database.DisableAllTriggersForCurrentThread())
{
metadata.Remove(Constants.RavenReplicationConflict);// you can't put conflicts
var oldVersion = Database.Attachments.GetStatic(key);
if (oldVersion == null)
return;
if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null)
return;
var history = new RavenJArray(ReplicationData.GetHistory(metadata));
metadata[Constants.RavenReplicationHistory] = history;
var ravenJTokenEqualityComparer = new RavenJTokenEqualityComparer();
// this is a conflict document, holding document keys in the
// values of the properties
var conflictData = oldVersion.Data().ToJObject();
var conflicts = conflictData.Value<RavenJArray>("Conflicts");
if (conflicts == null)
return;
foreach (var prop in conflicts)
{
var id = prop.Value<string>();
Attachment attachment = Database.Attachments.GetStatic(id);
if(attachment == null)
continue;
Database.Attachments.DeleteStatic(id, null);
// add the conflict history to the mix, so we make sure that we mark that we resolved the conflict
var conflictHistory = new RavenJArray(ReplicationData.GetHistory(attachment.Metadata));
conflictHistory.Add(new RavenJObject
{
{Constants.RavenReplicationVersion, attachment.Metadata[Constants.RavenReplicationVersion]},
{Constants.RavenReplicationSource, attachment.Metadata[Constants.RavenReplicationSource]}
});
foreach (var item in conflictHistory)
{
if (history.Any(x => ravenJTokenEqualityComparer.Equals(x, item)))
continue;
history.Add(item);
}
}
}
}
public override IEnumerable<Row> Execute(IEnumerable<Row> rows)
{
int count = 0;
foreach (var commentsForPosts in rows.Partition(Constants.BatchSize))
{
var cmds = new List<ICommandData>();
foreach (var commentsForPost in commentsForPosts.GroupBy(row => row["PostId"]))
{
var comments = new RavenJArray();
foreach (var row in commentsForPost)
{
comments.Add(new RavenJObject
{
{"Score", new RavenJValue(row["Score"])},
{"CreationDate", new RavenJValue(row["CreationDate"])},
{"Text", new RavenJValue(row["Text"])},
{"UserId", new RavenJValue(row["UserId"])}
});
}
cmds.Add(new PatchCommandData
{
Key = "posts/" + commentsForPost.Key,
Patches = new[]
{
new PatchRequest
{
Name = "Comments",
Type = PatchCommandType.Set,
Value = comments
},
}
});
}
count++;
WriteCommandsTo("Comments #" + count.ToString("00000") + ".json", cmds);
}
yield break;
}
public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation)
{
using (Database.DisableAllTriggersForCurrentThread())
{
metadata.Remove(Constants.RavenReplicationConflict);// you can't put conflicts
var oldVersion = Database.Get(key, transactionInformation);
if (oldVersion == null)
return;
if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null)
return;
RavenJArray history = new RavenJArray(ReplicationData.GetHistory(metadata));
metadata[Constants.RavenReplicationHistory] = history;
var ravenJTokenEqualityComparer = new RavenJTokenEqualityComparer();
// this is a conflict document, holding document keys in the
// values of the properties
var conflicts = oldVersion.DataAsJson.Value<RavenJArray>("Conflicts");
if(conflicts == null)
return;
foreach (var prop in conflicts)
{
RavenJObject deletedMetadata;
Database.Delete(prop.Value<string>(), null, transactionInformation, out deletedMetadata);
// add the conflict history to the mix, so we make sure that we mark that we resolved the conflict
var conflictHistory = new RavenJArray(ReplicationData.GetHistory(deletedMetadata));
conflictHistory.Add(new RavenJObject
{
{Constants.RavenReplicationVersion, deletedMetadata[Constants.RavenReplicationVersion]},
{Constants.RavenReplicationSource, deletedMetadata[Constants.RavenReplicationSource]}
});
foreach (var item in conflictHistory)
{
if(history.Any(x=>ravenJTokenEqualityComparer.Equals(x, item)))
continue;
history.Add(item);
}
}
}
}
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, bool allowStale, Func<string, TransactionInformation, object> batchOperation)
{
var array = new RavenJArray();
var bulkIndexQuery = new IndexQuery
{
Query = indexQuery.Query,
Start = indexQuery.Start,
Cutoff = indexQuery.Cutoff,
PageSize = int.MaxValue,
FieldsToFetch = new[] { Constants.DocumentIdFieldName },
SortedFields = indexQuery.SortedFields
};
bool stale;
var queryResults = database.QueryDocumentIds(index, bulkIndexQuery, out stale);
if (stale && allowStale == false)
{
throw new InvalidOperationException(
"Bulk operation cancelled because the index is stale and allowStale is false");
}
const int batchSize = 1024;
using (var enumerator = queryResults.GetEnumerator())
{
while (true)
{
var batchCount = 0;
database.TransactionalStorage.Batch(actions =>
{
while (batchCount < batchSize && enumerator.MoveNext())
{
batchCount++;
var result = batchOperation(enumerator.Current, transactionInformation);
array.Add(RavenJObject.FromObject(result, JsonExtensions.CreateDefaultJsonSerializer()));
}
});
if (batchCount < batchSize) break;
}
}
return array;
}
public static RavenJToken MinimizeToken(RavenJToken obj, int depth = 0)
{
switch (obj.Type)
{
case JTokenType.Array:
var array = new RavenJArray();
foreach (var item in ((RavenJArray)obj))
{
array.Add(MinimizeToken(item, depth + 1));
}
return array;
case JTokenType.Object:
var ravenJObject = ((RavenJObject)obj);
if (ravenJObject.ContainsKey(Constants.Metadata) == false)
{
// this might be a wrapper object, let check for first level arrays
if (depth == 0)
{
var newRootObj = new RavenJObject();
foreach (var prop in ravenJObject)
{
newRootObj[prop.Key] = prop.Value.Type == JTokenType.Array ?
MinimizeToken(prop.Value, depth + 1) :
prop.Value;
}
return newRootObj;
}
return obj;
}
var newObj = new RavenJObject();
newObj[Constants.Metadata] = ravenJObject[Constants.Metadata];
return newObj;
default:
return obj;
}
}
public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation)
{
using (Database.DisableAllTriggersForCurrentThread())
{
if (metadata.Remove(Constants.RavenReplicationConflictSkipResolution))
{
if (key.IndexOf("/conflicts/", StringComparison.OrdinalIgnoreCase) == -1)
{
metadata["@Http-Status-Code"] = 409;
metadata["@Http-Status-Description"] = "Conflict";
}
return;
}
metadata.Remove(Constants.RavenReplicationConflict);// you can't put conflicts
var oldVersion = Database.Documents.Get(key, transactionInformation);
if (oldVersion == null)
return;
if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null)
return;
var history = new RavenJArray();
metadata[Constants.RavenReplicationHistory] = history;
// this is a conflict document, holding document keys in the
// values of the properties
var conflicts = oldVersion.DataAsJson.Value<RavenJArray>("Conflicts");
if(conflicts == null)
return;
var list = new List<RavenJArray>
{
new RavenJArray(ReplicationData.GetHistory(metadata)) // first item to interleave
};
foreach (var prop in conflicts)
{
RavenJObject deletedMetadata;
Database.Documents.Delete(prop.Value<string>(), null, transactionInformation, out deletedMetadata);
if (deletedMetadata != null)
{
var conflictHistory = new RavenJArray(ReplicationData.GetHistory(deletedMetadata));
conflictHistory.Add(new RavenJObject
{
{Constants.RavenReplicationVersion, deletedMetadata[Constants.RavenReplicationVersion]},
{Constants.RavenReplicationSource, deletedMetadata[Constants.RavenReplicationSource]}
});
list.Add(conflictHistory);
}
}
int index = 0;
bool added = true;
while (added) // interleave the history from all conflicts
{
added = false;
foreach (var deletedMetadata in list)
{
// add the conflict history to the mix, so we make sure that we mark that we resolved the conflict
if (index < deletedMetadata.Length)
{
history.Add(deletedMetadata[index]);
added = true;
}
}
index++;
}
while (history.Length > Constants.ChangeHistoryLength)
{
history.RemoveAt(0);
}
}
}
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, int start, int pageSize)
{
var list = new RavenJArray();
TransactionalStorage.Batch(actions =>
{
var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start)
.Take(pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents)
{
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
list.Add(document.ToJson());
}
});
return list;
}
private void FlushBatch(List<RavenJObject> batch)
{
var sw = Stopwatch.StartNew();
var commands = new RavenJArray();
foreach (var doc in batch)
{
var metadata = doc.Value<RavenJObject>("@metadata");
doc.Remove("@metadata");
commands.Add(new RavenJObject
{
{"Method", "PUT"},
{"Document", doc},
{"Metadata", metadata},
{"Key", metadata.Value<string>("@id")}
});
}
var request = CreateRequest("bulk_docs", "POST");
var size = request.Write(commands);
request.ExecuteRequest();
Console.Write("Wrote {0} documents", batch.Count, sw.ElapsedMilliseconds);
if (size > 0)
Console.Write(" [{0:#,#;;0} kb]", Math.Round((double) size/1024, 2));
Console.WriteLine(" in {0:#,#;;0} ms", sw.ElapsedMilliseconds);
batch.Clear();
}
protected override void FlushBatch(List<RavenJObject> batch)
{
var sw = Stopwatch.StartNew();
var commands = new RavenJArray();
foreach (var doc in batch)
{
var metadata = doc.Value<RavenJObject>("@metadata");
doc.Remove("@metadata");
commands.Add(new RavenJObject
{
{"Method", "PUT"},
{"Document", doc},
{"Metadata", metadata},
{"Key", metadata.Value<string>("@id")}
});
}
var request = CreateRequest("/bulk_docs", "POST");
request.Write(commands);
request.ExecuteRequest();
ShowProgress("Wrote {0} documents in {1}", batch.Count, sw.ElapsedMilliseconds);
ShowProgress(" in {0:#,#;;0} ms", sw.ElapsedMilliseconds);
batch.Clear();
}
private void AddValue(PatchRequest patchCmd, string propName, RavenJToken token)
{
EnsurePreviousValueMatchCurrentValue(patchCmd, token);
if (token == null)
{
token = new RavenJArray();
document[propName] = token;
}
var array = GetArray(token, propName);
array = new RavenJArray(array);
document[propName] = array;
array.Add(patchCmd.Value);
}
private RavenJToken ToRavenJToken(JsInstance v)
{
switch (v.Class)
{
case JsInstance.TYPE_OBJECT:
case JsInstance.CLASS_OBJECT:
return(ToRavenJObject((JsObject)v));
case JsInstance.CLASS_DATE:
var dt = (DateTime)v.Value;
return(new RavenJValue(dt));
case JsInstance.TYPE_NUMBER:
case JsInstance.CLASS_NUMBER:
var num = (double)v.Value;
KeyValuePair <RavenJValue, object> property;
if (propertiesByValue.TryGetValue(v, out property))
{
var originalValue = property.Key;
if (originalValue.Type == JTokenType.Float)
{
return(new RavenJValue(num));
}
if (originalValue.Type == JTokenType.Integer)
{
// If the current value is exactly as the original value, we can return the original value before we made the JS conversion,
// which will convert a Int64 to jsFloat.
var originalJsValue = property.Value;
if (originalJsValue is double && Math.Abs(num - (double)originalJsValue) < double.Epsilon)
{
return(originalValue);
}
return(new RavenJValue((long)num));
}
}
// If we don't have the type, assume that if the number ending with ".0" it actually an integer.
var integer = Math.Truncate(num);
if (Math.Abs(num - integer) < double.Epsilon)
{
return(new RavenJValue((long)integer));
}
return(new RavenJValue(num));
case JsInstance.TYPE_STRING:
case JsInstance.TYPE_BOOLEAN:
case JsInstance.CLASS_STRING:
case JsInstance.CLASS_BOOLEAN:
return(new RavenJValue(v.Value));
case JsInstance.CLASS_NULL:
case JsInstance.TYPE_NULL:
return(RavenJValue.Null);
case JsInstance.CLASS_UNDEFINED:
case JsInstance.TYPE_UNDEFINED:
return(RavenJValue.Null);
case JsInstance.CLASS_ARRAY:
var jsArray = ((JsArray)v);
var rja = new RavenJArray();
for (int i = 0; i < jsArray.Length; i++)
{
var jsInstance = jsArray.get(i);
var ravenJToken = ToRavenJToken(jsInstance);
if (ravenJToken == null)
{
continue;
}
rja.Add(ravenJToken);
}
return(rja);
case JsInstance.CLASS_REGEXP:
case JsInstance.CLASS_ERROR:
case JsInstance.CLASS_ARGUMENTS:
case JsInstance.CLASS_DESCRIPTOR:
case JsInstance.CLASS_FUNCTION:
return(null);
default:
throw new NotSupportedException(v.Class);
}
}
public override void OnPut(string key, Stream data, RavenJObject metadata)
{
if (key.StartsWith("Raven/")) // we don't deal with system attachment
{
return;
}
using (Database.DisableAllTriggersForCurrentThread())
{
var attachmentMetadata = GetAttachmentMetadata(key);
if (attachmentMetadata != null)
{
var history = new RavenJArray(ReplicationData.GetHistory(attachmentMetadata));
if (attachmentMetadata.ContainsKey(Constants.RavenReplicationMergedHistory) == false)
{
if (attachmentMetadata.ContainsKey(Constants.RavenReplicationVersion) &&
attachmentMetadata.ContainsKey(Constants.RavenReplicationSource))
{
history.Add(new RavenJObject
{
{ Constants.RavenReplicationVersion, attachmentMetadata[Constants.RavenReplicationVersion] },
{ Constants.RavenReplicationSource, attachmentMetadata[Constants.RavenReplicationSource] }
});
}
else
{
history.Add(new RavenJObject
{
{ Constants.RavenReplicationVersion, 0 },
{ Constants.RavenReplicationSource, RavenJToken.FromObject(Database.TransactionalStorage.Id) }
});
}
var sources = new HashSet <RavenJToken>(RavenJTokenEqualityComparer.Default);
int pos = history.Length - 1;
for (; pos >= 0; pos--)
{
var source = ((RavenJObject)history[pos])[Constants.RavenReplicationSource];
if (sources.Contains(source))
{
history.RemoveAt(pos);
continue;
}
sources.Add(source);
}
metadata[Constants.RavenReplicationMergedHistory] = true;
metadata[Constants.RavenReplicationHistory] = history;
}
//If we have the flag we must have Constants.RavenReplicationVersion and Constants.RavenReplicationSource too
//Here we assume that the replication history is in the form of a "sorted dictionary" so we just need to remove
//the entry with the current source id and insert the new version at the end of the history.
else
{
int i = history.Length - 1;
for (; i >= 0; i--)
{
var currentEntry = history[i];
if (RavenJTokenEqualityComparer.Default.Equals(((RavenJObject)currentEntry)
[Constants.RavenReplicationSource], attachmentMetadata[Constants.RavenReplicationSource]))
{
break;
}
}
if (i != -1)
{
history.RemoveAt(i);
}
history.Add(new RavenJObject
{
{ Constants.RavenReplicationVersion, attachmentMetadata[Constants.RavenReplicationVersion] },
{ Constants.RavenReplicationSource, attachmentMetadata[Constants.RavenReplicationSource] }
});
metadata[Constants.RavenReplicationHistory] = history;
}
}
metadata[Constants.RavenReplicationVersion] = RavenJToken.FromObject(ReplicationHiLo.NextId(Database));
metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(Database.TransactionalStorage.Id);
}
}
public override void OnPut(string key, RavenJObject jsonReplicationDocument, RavenJObject metadata, TransactionInformation transactionInformation)
{
using (Database.DisableAllTriggersForCurrentThread())
{
if (metadata.Remove(Constants.RavenReplicationConflictSkipResolution))
{
if (key.IndexOf("/conflicts/", StringComparison.OrdinalIgnoreCase) == -1)
{
metadata["@Http-Status-Code"] = 409;
metadata["@Http-Status-Description"] = "Conflict";
}
return;
}
metadata.Remove(Constants.RavenReplicationConflict);// you can't put conflicts
metadata.Remove(Constants.RavenReplicationConflictDocument);
var oldVersion = Database.Documents.Get(key, transactionInformation);
if (oldVersion == null)
{
return;
}
if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null)
{
return;
}
var history = new RavenJArray();
metadata[Constants.RavenReplicationHistory] = history;
// this is a conflict document, holding document keys in the
// values of the properties
var conflicts = oldVersion.DataAsJson.Value <RavenJArray>("Conflicts");
if (conflicts == null)
{
return;
}
Dictionary <string, RavenJObject> conflictHistoryAsDictionary = new Dictionary <string, RavenJObject>();
foreach (var prop in conflicts)
{
RavenJObject deletedMetadata;
Database.Documents.Delete(prop.Value <string>(), null, transactionInformation, out deletedMetadata);
if (deletedMetadata != null)
{
var conflictHistory = new RavenJArray(ReplicationData.GetOrCreateHistory(deletedMetadata));
conflictHistory.Add(new RavenJObject
{
{ Constants.RavenReplicationVersion, deletedMetadata[Constants.RavenReplicationVersion] },
{ Constants.RavenReplicationSource, deletedMetadata[Constants.RavenReplicationSource] }
});
Historian.MergeSingleHistory(conflictHistory, conflictHistoryAsDictionary, key);
}
}
metadata[Constants.RavenReplicationHistory] = new RavenJArray(conflictHistoryAsDictionary.Values);
metadata[Constants.RavenReplicationMergedHistory] = true;
}
}
private RavenJToken ToRavenJToken(JsInstance v, string propertyName)
{
switch (v.Class)
{
case JsInstance.TYPE_OBJECT:
case JsInstance.CLASS_OBJECT:
return ToRavenJObject((JsObject)v);
case JsInstance.CLASS_DATE:
var dt = (DateTime)v.Value;
return new RavenJValue(dt);
case JsInstance.TYPE_NUMBER:
case JsInstance.CLASS_NUMBER:
var num = (double)v.Value;
JTokenType type;
if (propertiesTypeByName.TryGetValue(propertyName, out type))
{
if (type == JTokenType.Float)
return new RavenJValue(num);
if (type == JTokenType.Integer)
return new RavenJValue((long) num);
}
// If we don't have the type, assume that if the number ending with ".0" it actually an integer.
var integer = Math.Truncate(num);
if (Math.Abs(num - integer) < double.Epsilon)
return new RavenJValue((long)integer);
return new RavenJValue(num);
case JsInstance.TYPE_STRING:
case JsInstance.TYPE_BOOLEAN:
case JsInstance.CLASS_STRING:
case JsInstance.CLASS_BOOLEAN:
return new RavenJValue(v.Value);
case JsInstance.CLASS_NULL:
case JsInstance.TYPE_NULL:
return RavenJValue.Null;
case JsInstance.CLASS_UNDEFINED:
case JsInstance.TYPE_UNDEFINED:
return RavenJValue.Null;
case JsInstance.CLASS_ARRAY:
var jsArray = ((JsArray)v);
var rja = new RavenJArray();
for (int i = 0; i < jsArray.Length; i++)
{
var jsInstance = jsArray.get(i);
var ravenJToken = ToRavenJToken(jsInstance, propertyName);
if (ravenJToken == null)
continue;
rja.Add(ravenJToken);
}
return rja;
case JsInstance.CLASS_REGEXP:
case JsInstance.CLASS_ERROR:
case JsInstance.CLASS_ARGUMENTS:
case JsInstance.CLASS_DESCRIPTOR:
case JsInstance.CLASS_FUNCTION:
return null;
default:
throw new NotSupportedException(v.Class);
}
}
protected override Guid FlushBatch(List <RavenJObject> batch)
{
var sw = Stopwatch.StartNew();
var commands = new RavenJArray();
foreach (var doc in batch)
{
var metadata = doc.Value <RavenJObject>("@metadata");
doc.Remove("@metadata");
commands.Add(new RavenJObject
{
{ "Method", "PUT" },
{ "Document", doc },
{ "Metadata", metadata },
{ "Key", metadata.Value <string>("@id") }
});
}
var retries = retriesCount;
HttpRavenRequest request = null;
BatchResult[] results;
while (true)
{
try
{
request = CreateRequest("/bulk_docs", "POST");
request.Write(commands);
results = request.ExecuteRequest <BatchResult[]>();
sw.Stop();
break;
}
catch (Exception e)
{
if (--retries == 0 || request == null)
{
throw;
}
sw.Stop();
LastRequestErrored = true;
ShowProgress("Error flushing to database, remaining attempts {0} - time {2:#,#} ms, will retry [{3:#,#.##;;0} kb compressed to {4:#,#.##;;0} kb]. Error: {1}",
retriesCount - retries, e, sw.ElapsedMilliseconds,
(double)request.NumberOfBytesWrittenUncompressed / 1024,
(double)request.NumberOfBytesWrittenCompressed / 1024);
}
}
total += batch.Count;
ShowProgress("{2,5:#,#}: Wrote {0:#,#;;0} in {1,6:#,#;;0} ms ({6:0.00} ms per doc) (total of {3:#,#;;0}) documents [{4:#,#.##;;0} kb compressed to {5:#,#.##;;0} kb]",
batch.Count, sw.ElapsedMilliseconds, ++count, total,
(double)request.NumberOfBytesWrittenUncompressed / 1024,
(double)request.NumberOfBytesWrittenCompressed / 1024,
Math.Round((double)sw.ElapsedMilliseconds / Math.Max(1, batch.Count), 2));
batch.Clear();
if (results.Length == 0)
{
return(Guid.Empty);
}
return(results.Last().Etag.Value);
}
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start,
int pageSize, CancellationToken token, ref int nextStart,
string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null,
string skipAfter = null)
{
var list = new RavenJArray();
GetDocumentsWithIdStartingWith(idPrefix, matches, exclude, start, pageSize, token, ref nextStart, doc => list.Add(doc.ToJson()),
transformer, transformerParameters, skipAfter);
return(list);
}
private RavenJToken ToRavenJToken(JsInstance v, string propertyName)
{
switch (v.Class)
{
case JsInstance.TYPE_OBJECT:
case JsInstance.CLASS_OBJECT:
return(ToRavenJObject((JsObject)v));
case JsInstance.CLASS_DATE:
var dt = (DateTime)v.Value;
return(new RavenJValue(dt));
case JsInstance.TYPE_NUMBER:
case JsInstance.CLASS_NUMBER:
var num = (double)v.Value;
JTokenType type;
if (propertiesTypeByName.TryGetValue(propertyName, out type))
{
if (type == JTokenType.Float)
{
return(new RavenJValue(num));
}
if (type == JTokenType.Integer)
{
return(new RavenJValue((long)num));
}
}
// If we don't have the type, assume that if the number ending with ".0" it actually an integer.
var integer = Math.Truncate(num);
if (Math.Abs(num - integer) < double.Epsilon)
{
return(new RavenJValue((long)integer));
}
return(new RavenJValue(num));
case JsInstance.TYPE_STRING:
case JsInstance.TYPE_BOOLEAN:
case JsInstance.CLASS_STRING:
case JsInstance.CLASS_BOOLEAN:
return(new RavenJValue(v.Value));
case JsInstance.CLASS_NULL:
case JsInstance.TYPE_NULL:
return(RavenJValue.Null);
case JsInstance.CLASS_UNDEFINED:
case JsInstance.TYPE_UNDEFINED:
return(RavenJValue.Null);
case JsInstance.CLASS_ARRAY:
var jsArray = ((JsArray)v);
var rja = new RavenJArray();
for (int i = 0; i < jsArray.Length; i++)
{
var jsInstance = jsArray.get(i);
var ravenJToken = ToRavenJToken(jsInstance, propertyName);
if (ravenJToken == null)
{
continue;
}
rja.Add(ravenJToken);
}
return(rja);
case JsInstance.CLASS_REGEXP:
case JsInstance.CLASS_ERROR:
case JsInstance.CLASS_ARGUMENTS:
case JsInstance.CLASS_DESCRIPTOR:
case JsInstance.CLASS_FUNCTION:
return(null);
default:
throw new NotSupportedException(v.Class);
}
}
public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation)
{
using (Database.DisableAllTriggersForCurrentThread())
{
metadata.Remove(Constants.RavenReplicationConflict); // you can't put conflicts
var oldVersion = Database.Get(key, transactionInformation);
if (oldVersion == null)
{
return;
}
if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null)
{
return;
}
var history = new RavenJArray();
metadata[Constants.RavenReplicationHistory] = history;
var ravenJTokenEqualityComparer = new RavenJTokenEqualityComparer();
// this is a conflict document, holding document keys in the
// values of the properties
var conflicts = oldVersion.DataAsJson.Value <RavenJArray>("Conflicts");
if (conflicts == null)
{
return;
}
var list = new List <RavenJArray>
{
new RavenJArray(ReplicationData.GetHistory(metadata)) // first item to interleave
};
foreach (var prop in conflicts)
{
RavenJObject deletedMetadata;
Database.Delete(prop.Value <string>(), null, transactionInformation, out deletedMetadata);
if (deletedMetadata != null)
{
var conflictHistory = new RavenJArray(ReplicationData.GetHistory(deletedMetadata));
conflictHistory.Add(new RavenJObject
{
{ Constants.RavenReplicationVersion, deletedMetadata[Constants.RavenReplicationVersion] },
{ Constants.RavenReplicationSource, deletedMetadata[Constants.RavenReplicationSource] }
});
list.Add(conflictHistory);
}
}
int index = 0;
bool added = true;
while (added) // interleave the history from all conflicts
{
added = false;
foreach (var deletedMetadata in list)
{
// add the conflict history to the mix, so we make sure that we mark that we resolved the conflict
if (index < deletedMetadata.Length)
{
history.Add(deletedMetadata[index]);
added = true;
}
}
index++;
}
while (history.Length > Constants.ChangeHistoryLength)
{
history.RemoveAt(0);
}
}
}
private RavenJToken ToRavenJToken(JsValue v, string propertyKey)
{
if (v.IsBoolean())
return new RavenJValue(v.AsBoolean());
if (v.IsString())
{
const string RavenDataByteArrayToBase64 = "raven-data:byte[];base64,";
var value = v.AsString();
if (value != null && value.StartsWith(RavenDataByteArrayToBase64))
{
value = value.Remove(0, RavenDataByteArrayToBase64.Length);
var byteArray = Convert.FromBase64String(value);
return new RavenJValue(byteArray);
}
return new RavenJValue(value);
}
if (v.IsNumber())
{
var num = v.AsNumber();
KeyValuePair<RavenJValue, JsValue> property;
if (propertiesByValue.TryGetValue(propertyKey, out property))
{
var originalValue = property.Key;
if (originalValue.Type == JTokenType.Float)
return new RavenJValue(num);
if (originalValue.Type == JTokenType.Integer)
{
// If the current value is exactly as the original value, we can return the original value before we made the JS conversion,
// which will convert a Int64 to jsFloat.
var originalJsValue = property.Value;
if (originalJsValue.IsNumber() && Math.Abs(num - originalJsValue.AsNumber()) < double.Epsilon)
return originalValue;
return new RavenJValue((long)num);
}
}
// If we don't have the type, assume that if the number ending with ".0" it actually an integer.
var integer = Math.Truncate(num);
if (Math.Abs(num - integer) < double.Epsilon)
return new RavenJValue((long)integer);
return new RavenJValue(num);
}
if (v.IsNull())
return RavenJValue.Null;
if (v.IsUndefined())
return RavenJValue.Null;
if (v.IsArray())
{
var jsArray = v.AsArray();
var rja = new RavenJArray();
foreach (var property in jsArray.Properties)
{
if (property.Key == "length")
continue;
var jsInstance = property.Value.Value;
if (!jsInstance.HasValue)
continue;
var ravenJToken = ToRavenJToken(jsInstance.Value, CreatePropertyKey(property.Key, propertyKey));
if (ravenJToken == null)
continue;
rja.Add(ravenJToken);
}
return rja;
}
if (v.IsObject())
return ToRavenJObject(v, propertyKey);
if (v.IsRegExp())
return null;
throw new NotSupportedException(v.Type.ToString());
}
private RavenJToken ToRavenJToken(JsValue v, string propertyKey, bool recursiveCall)
{
if (v.IsBoolean())
{
return(new RavenJValue(v.AsBoolean()));
}
if (v.IsString())
{
const string RavenDataByteArrayToBase64 = "raven-data:byte[];base64,";
var value = v.AsString();
if (value != null && value.StartsWith(RavenDataByteArrayToBase64))
{
value = value.Remove(0, RavenDataByteArrayToBase64.Length);
var byteArray = Convert.FromBase64String(value);
return(new RavenJValue(byteArray));
}
return(new RavenJValue(value));
}
if (v.IsNumber())
{
var num = v.AsNumber();
KeyValuePair <RavenJValue, JsValue> property;
if (propertiesByValue.TryGetValue(propertyKey, out property))
{
var originalValue = property.Key;
if (originalValue.Type == JTokenType.Float)
{
return(new RavenJValue(num));
}
if (originalValue.Type == JTokenType.Integer)
{
// If the current value is exactly as the original value, we can return the original value before we made the JS conversion,
// which will convert a Int64 to jsFloat.
var originalJsValue = property.Value;
if (originalJsValue.IsNumber() && Math.Abs(num - originalJsValue.AsNumber()) < double.Epsilon)
{
return(originalValue);
}
return(new RavenJValue((long)num));
}
}
// If we don't have the type, assume that if the number ending with ".0" it actually an integer.
var integer = Math.Truncate(num);
if (Math.Abs(num - integer) < double.Epsilon)
{
return(new RavenJValue((long)integer));
}
return(new RavenJValue(num));
}
if (v.IsNull())
{
return(RavenJValue.Null);
}
if (v.IsUndefined())
{
return(RavenJValue.Null);
}
if (v.IsArray())
{
var jsArray = v.AsArray();
var rja = new RavenJArray();
foreach (var property in jsArray.Properties)
{
if (property.Key == "length")
{
continue;
}
var jsInstance = property.Value.Value;
if (!jsInstance.HasValue)
{
continue;
}
var ravenJToken = ToRavenJToken(jsInstance.Value, CreatePropertyKey(property.Key, propertyKey), recursiveCall);
if (ravenJToken == null)
{
continue;
}
rja.Add(ravenJToken);
}
return(rja);
}
if (v.IsObject())
{
return(ToRavenJObject(v, propertyKey, recursiveCall));
}
if (v.IsRegExp())
{
return(null);
}
throw new NotSupportedException(v.Type.ToString());
}
public RavenJArray GetDocuments(int start, int pageSize, Guid? etag)
{
var list = new RavenJArray();
TransactionalStorage.Batch(actions =>
{
while (true)
{
var documents = etag == null ?
actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) :
actions.Documents.GetDocumentsAfter(etag.Value, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
if(etag != null)
etag = doc.Etag;
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
list.Add(document.ToJson());
}
if (list.Length != 0 || docCount == 0)
break;
start += docCount;
}
});
return list;
}
private static RavenJToken ToRavenJToken(JsInstance v)
{
switch (v.Class)
{
case JsInstance.TYPE_OBJECT:
case JsInstance.CLASS_OBJECT:
return(ToRavenJObject((JsObject)v));
case JsInstance.CLASS_DATE:
var dt = (DateTime)v.Value;
return(new RavenJValue(dt));
case JsInstance.TYPE_NUMBER:
case JsInstance.CLASS_NUMBER:
var num = (double)v.Value;
var integer = Math.Truncate(num);
if (Math.Abs(num - integer) < double.Epsilon)
{
return(new RavenJValue((long)integer));
}
return(new RavenJValue(num));
case JsInstance.TYPE_STRING:
case JsInstance.TYPE_BOOLEAN:
case JsInstance.CLASS_STRING:
case JsInstance.CLASS_BOOLEAN:
return(new RavenJValue(v.Value));
case JsInstance.CLASS_NULL:
case JsInstance.TYPE_NULL:
return(RavenJValue.Null);
case JsInstance.CLASS_UNDEFINED:
case JsInstance.TYPE_UNDEFINED:
return(RavenJValue.Null);
case JsInstance.CLASS_ARRAY:
var jsArray = ((JsArray)v);
var rja = new RavenJArray();
for (int i = 0; i < jsArray.Length; i++)
{
var jsInstance = jsArray.get(i);
var ravenJToken = ToRavenJToken(jsInstance);
if (ravenJToken == null)
{
continue;
}
rja.Add(ravenJToken);
}
return(rja);
case JsInstance.CLASS_REGEXP:
case JsInstance.CLASS_ERROR:
case JsInstance.CLASS_ARGUMENTS:
case JsInstance.CLASS_DESCRIPTOR:
case JsInstance.CLASS_FUNCTION:
return(null);
default:
throw new NotSupportedException(v.Class);
}
}
private RavenJArray GetAttachments(int start, Etag etag, int maxRecords)
{
var array = new RavenJArray();
var attachmentInfos = database.Attachments.GetAttachments(start, maxRecords, etag, null, 1024 * 1024 * 10);
foreach (var attachmentInfo in attachmentInfos)
{
var attachment = database.Attachments.GetStatic(attachmentInfo.Key);
if (attachment == null)
return null;
var data = attachment.Data;
attachment.Data = () =>
{
var memoryStream = new MemoryStream();
database.TransactionalStorage.Batch(accessor => data().CopyTo(memoryStream));
memoryStream.Position = 0;
return memoryStream;
};
var bytes = attachment.Data().ReadData();
array.Add(
new RavenJObject
{
{"Data", bytes},
{"Metadata", attachmentInfo.Metadata},
{"Key", attachmentInfo.Key},
{"Etag", new RavenJValue(attachmentInfo.Etag.ToString())}
});
}
return array;
}
public RavenJArray GetDocumentsAsJson(int start, int pageSize, Etag etag, CancellationToken token)
{
var list = new RavenJArray();
GetDocuments(start, pageSize, etag, token, doc => { list.Add(doc.ToJson()); return true; });
return list;
}
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, BulkOperationOptions options, Func <string, TransactionInformation, object> batchOperation, Action <BulkOperationProgress> reportProgress = null)
{
options = options ?? new BulkOperationOptions();
var array = new RavenJArray();
var bulkIndexQuery = new IndexQuery
{
Query = indexQuery.Query,
Start = indexQuery.Start,
Cutoff = indexQuery.Cutoff ?? SystemTime.UtcNow,
WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow,
PageSize = int.MaxValue,
FieldsToFetch = new[] { Constants.DocumentIdFieldName },
SortedFields = indexQuery.SortedFields,
HighlighterPreTags = indexQuery.HighlighterPreTags,
HighlighterPostTags = indexQuery.HighlighterPostTags,
HighlightedFields = indexQuery.HighlightedFields,
SortHints = indexQuery.SortHints,
HighlighterKeyName = indexQuery.HighlighterKeyName,
TransformerParameters = indexQuery.TransformerParameters,
ResultsTransformer = indexQuery.ResultsTransformer
};
var operationProgress = new BulkOperationProgress();
bool stale;
var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);
operationProgress.TotalEntries = queryResults.Count;
if (stale && options.AllowStale == false)
{
if (options.StaleTimeout != null)
{
var staleWaitTimeout = Stopwatch.StartNew();
while (stale && staleWaitTimeout.Elapsed < options.StaleTimeout)
{
queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);
operationProgress.TotalEntries = queryResults.Count;
if (stale)
{
SystemTime.Wait(100);
}
}
}
if (stale)
{
if (options.StaleTimeout != null)
{
throw new InvalidOperationException("Bulk operation cancelled because the index is stale and StaleTimout of " + options.StaleTimeout + " passed");
}
throw new InvalidOperationException("Bulk operation cancelled because the index is stale and allowStale is false");
}
}
var token = tokenSource.Token;
const int batchSize = 1024;
int maxOpsPerSec = options.MaxOpsPerSec ?? int.MaxValue;
using (var enumerator = queryResults.GetEnumerator())
{
var duration = Stopwatch.StartNew();
var operations = 0;
while (true)
{
database.WorkContext.UpdateFoundWork();
if (timeout != null)
{
timeout.Delay();
}
var batchCount = 0;
var shouldWaitNow = false;
token.ThrowIfCancellationRequested();
using (database.DocumentLock.Lock())
{
database.TransactionalStorage.Batch(actions =>
{
while (batchCount < batchSize && enumerator.MoveNext())
{
token.ThrowIfCancellationRequested();
batchCount++;
operations++;
var result = batchOperation(enumerator.Current, transactionInformation);
if (options.RetrieveDetails)
{
array.Add(RavenJObject.FromObject(result));
}
operationProgress.ProcessedEntries++;
reportProgress?.Invoke(operationProgress);
if (operations >= maxOpsPerSec && duration.ElapsedMilliseconds < 1000)
{
shouldWaitNow = true;
break;
}
}
});
}
if (shouldWaitNow)
{
SystemTime.Wait(500);
operations = 0;
duration.Restart();
continue;
}
if (batchCount < batchSize)
{
break;
}
}
}
return(array);
}
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start,
int pageSize, CancellationToken token, ref int nextStart,
string transformer = null, Dictionary<string, RavenJToken> transformerParameters = null,
string skipAfter = null)
{
var list = new RavenJArray();
GetDocumentsWithIdStartingWith(idPrefix, matches, exclude, start, pageSize, token, ref nextStart, doc => list.Add(doc.ToJson()),
transformer, transformerParameters, skipAfter);
return list;
}
private void Initialize()
{
if (mappedTypes != null)
return;
lock (padlock)
{
if (mappedTypes != null)
return;
var collectionData = new CollectionData();
var jsonDoc = store.DatabaseCommands.Get(collectionNamesDocId);
if (jsonDoc != null)
{
var collectionNames = jsonDoc.DataAsJson["Collections"] as RavenJArray;
foreach (RavenJValue value in collectionNames)
{
collectionData.Collections.Add(value.Value as string);
}
}
if (timeoutsEnabled)
{
MapTypeToCollectionName(typeof(TimeoutPersisters.RavenDB.TimeoutData), collectionData);
}
if (sagasEnabled)
{
foreach (var sagaType in types.Where(IsSagaEntity))
{
MapTypeToCollectionName(sagaType, collectionData);
}
}
if (collectionData.Changed)
{
var newDoc = new RavenJObject();
var list = new RavenJArray();
foreach (var name in collectionData.Collections)
{
list.Add(new RavenJValue(name));
}
newDoc["EndpointName"] = this.endpointName;
newDoc["EndpointName"] = this.endpointName;
newDoc["Collections"] = list;
var metadata = new RavenJObject();
store.DatabaseCommands.Put(collectionNamesDocId, null, newDoc, metadata);
}
// Completes initialization
this.mappedTypes = collectionData.Mappings;
}
}
private bool TryHandleArrayValue(int index, Dictionary <string, object> result, KeyValuePair <string, RavenJToken> prop)
{
var arrays = new List <RavenJArray>
{
(RavenJArray)prop.Value
};
for (var i = 0; i < docs.Length; i++)
{
if (i == index)
{
continue;
}
RavenJToken token;
if (docs[i].TryGetValue(prop.Key, out token) && token.Type != JTokenType.Array)
{
return(false);
}
if (token == null)
{
continue;
}
if (token.IsSnapshot)
{
token = token.CreateSnapshot();
}
arrays.Add((RavenJArray)token);
}
var mergedArray = new RavenJArray();
while (arrays.Count > 0)
{
var set = new HashSet <RavenJToken>(RavenJTokenEqualityComparer.Default);
for (var i = 0; i < arrays.Count; i++)
{
if (arrays[i].Length == 0)
{
arrays.RemoveAt(i);
i -= 1;
continue;
}
set.Add(arrays[i][0]);
arrays[i].RemoveAt(0);
}
foreach (var ravenJToken in set)
{
mergedArray.Add(ravenJToken);
}
}
if (RavenJTokenEqualityComparer.Default.Equals(mergedArray, prop.Value))
{
result.Add(prop.Key, mergedArray);
return(true);
}
result.Add(prop.Key, new ArrayWithWarning(mergedArray));
return(true);
}
private RavenJToken ToRavenJToken(JsInstance v)
{
switch (v.Class)
{
case JsInstance.TYPE_OBJECT:
case JsInstance.CLASS_OBJECT:
return ToRavenJObject((JsObject) v);
case JsInstance.CLASS_DATE:
var dt = (DateTime) v.Value;
return new RavenJValue(dt);
case JsInstance.TYPE_NUMBER:
case JsInstance.CLASS_NUMBER:
var num = (double) v.Value;
KeyValuePair<RavenJValue, object> property;
if (propertiesByValue.TryGetValue(v, out property))
{
var originalValue = property.Key;
if (originalValue.Type == JTokenType.Float)
return new RavenJValue(num);
if (originalValue.Type == JTokenType.Integer)
{
// If the current value is exactly as the original value, we can return the original value before we made the JS conversion,
// which will convert a Int64 to jsFloat.
var originalJsValue = property.Value;
if (originalJsValue is double && Math.Abs(num - (double)originalJsValue) < double.Epsilon)
return originalValue;
return new RavenJValue((long) num);
}
}
// If we don't have the type, assume that if the number ending with ".0" it actually an integer.
var integer = Math.Truncate(num);
if (Math.Abs(num - integer) < double.Epsilon)
return new RavenJValue((long) integer);
return new RavenJValue(num);
case JsInstance.TYPE_STRING:
case JsInstance.CLASS_STRING:
{
const string ravenDataByteArrayToBase64 = "raven-data:byte[];base64,";
var value = v.Value as string;
if (value != null && value.StartsWith(ravenDataByteArrayToBase64))
{
value = value.Remove(0, ravenDataByteArrayToBase64.Length);
var byteArray = Convert.FromBase64String(value);
return new RavenJValue(byteArray);
}
return new RavenJValue(v.Value);
}
case JsInstance.TYPE_BOOLEAN:
case JsInstance.CLASS_BOOLEAN:
return new RavenJValue(v.Value);
case JsInstance.CLASS_NULL:
case JsInstance.TYPE_NULL:
return RavenJValue.Null;
case JsInstance.CLASS_UNDEFINED:
case JsInstance.TYPE_UNDEFINED:
return RavenJValue.Null;
case JsInstance.CLASS_ARRAY:
var jsArray = ((JsArray) v);
var rja = new RavenJArray();
for (int i = 0; i < jsArray.Length; i++)
{
var jsInstance = jsArray.get(i);
var ravenJToken = ToRavenJToken(jsInstance);
if (ravenJToken == null)
continue;
rja.Add(ravenJToken);
}
return rja;
case JsInstance.CLASS_REGEXP:
case JsInstance.CLASS_ERROR:
case JsInstance.CLASS_ARGUMENTS:
case JsInstance.CLASS_DESCRIPTOR:
case JsInstance.CLASS_FUNCTION:
return null;
default:
throw new NotSupportedException(v.Class);
}
}
private bool TryHandleArrayValue(int index, Dictionary<string, object> result, KeyValuePair<string, RavenJToken> prop)
{
var arrays = new List<RavenJArray>
{
(RavenJArray)prop.Value
};
for (var i = 0; i < docs.Length; i++)
{
if (i == index)
continue;
RavenJToken token;
if (docs[i].TryGetValue(prop.Key, out token) && token.Type != JTokenType.Array)
return false;
if (token == null)
continue;
if (token.IsSnapshot)
token = token.CreateSnapshot();
arrays.Add((RavenJArray)token);
}
var mergedArray = new RavenJArray();
while (arrays.Count > 0)
{
var set = new HashSet<RavenJToken>(RavenJTokenEqualityComparer.Default);
for (var i = 0; i < arrays.Count; i++)
{
if (arrays[i].Length == 0)
{
arrays.RemoveAt(i);
i -= 1;
continue;
}
set.Add(arrays[i][0]);
arrays[i].RemoveAt(0);
}
foreach (var ravenJToken in set)
{
mergedArray.Add(ravenJToken);
}
}
if (RavenJTokenEqualityComparer.Default.Equals(mergedArray, prop.Value))
{
result.Add(prop.Key, mergedArray);
return true;
}
result.Add(prop.Key, new ArrayWithWarning(mergedArray));
return true;
}
public RavenJArray GetDocuments(int start, int pageSize, Guid? etag)
{
var list = new RavenJArray();
TransactionalStorage.Batch(actions =>
{
IEnumerable<JsonDocument> documents;
if (etag == null)
documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start);
else
documents = actions.Documents.GetDocumentsAfter(etag.Value);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents.Take(pageSize))
{
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
list.Add(document.ToJson());
}
});
return list;
}
public async Task<RavenJArray> GetTransformers(RavenConnectionStringOptions src, int start)
{
if (isTransformersSupported() == false)
return new RavenJArray();
var transformers = await Store.AsyncDatabaseCommands.GetTransformersAsync(start, Options.BatchSize);
var result = new RavenJArray();
foreach (var transformer in transformers)
{
result.Add(new RavenJObject
{
{ "name", transformer.Name },
{ "definition", RavenJObject.FromObject(transformer) }
});
}
return result;
}
private static void FlushBatch(string instanceUrl, List<RavenJObject> batch)
{
var sw = Stopwatch.StartNew();
long size;
using (var webClient = new WebClient())
{
webClient.Headers.Add("Content-Type", "application/json; charset=utf-8");
webClient.UseDefaultCredentials = true;
webClient.Credentials = CredentialCache.DefaultNetworkCredentials;
using (var stream = new MemoryStream())
{
using (var streamWriter = new StreamWriter(stream, Encoding.UTF8))
using (var jsonTextWriter = new JsonTextWriter(streamWriter))
{
var commands = new RavenJArray();
foreach (var doc in batch)
{
var metadata = doc.Value<RavenJObject>("@metadata");
doc.Remove("@metadata");
commands.Add(new RavenJObject
{
{"Method", "PUT"},
{"Document", doc},
{"Metadata", metadata},
{"Key", metadata.Value<string>("@id")}
});
}
commands.WriteTo(jsonTextWriter);
jsonTextWriter.Flush();
streamWriter.Flush();
stream.Flush();
size = stream.Length;
using (var netStream = webClient.OpenWrite(instanceUrl + "bulk_docs", "POST"))
{
stream.WriteTo(netStream);
netStream.Flush();
}
}
}
}
Console.WriteLine("Wrote {0} documents [{1:#,#} kb] in {2:#,#} ms",
batch.Count, Math.Round((double)size / 1024, 2), sw.ElapsedMilliseconds);
batch.Clear();
}
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, BulkOperationOptions options, Func<string, TransactionInformation, object> batchOperation)
{
options = options ?? new BulkOperationOptions();
var array = new RavenJArray();
var bulkIndexQuery = new IndexQuery
{
Query = indexQuery.Query,
Start = indexQuery.Start,
Cutoff = indexQuery.Cutoff ?? SystemTime.UtcNow,
WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow,
PageSize = int.MaxValue,
FieldsToFetch = new[] { Constants.DocumentIdFieldName },
SortedFields = indexQuery.SortedFields,
HighlighterPreTags = indexQuery.HighlighterPreTags,
HighlighterPostTags = indexQuery.HighlighterPostTags,
HighlightedFields = indexQuery.HighlightedFields,
SortHints = indexQuery.SortHints
};
bool stale;
var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);
if (stale && options.AllowStale == false)
{
if (options.StaleTimeout != null)
{
var staleWaitTimeout = Stopwatch.StartNew();
while (stale && staleWaitTimeout.Elapsed < options.StaleTimeout)
{
queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);
if(stale)
SystemTime.Wait(100);
}
}
if (stale)
{
if (options.StaleTimeout != null)
throw new InvalidOperationException("Bulk operation cancelled because the index is stale and StaleTimout of " + options.StaleTimeout + "passed");
throw new InvalidOperationException("Bulk operation cancelled because the index is stale and allowStale is false");
}
}
var token = tokenSource.Token;
const int batchSize = 1024;
int maxOpsPerSec = options.MaxOpsPerSec ?? int.MaxValue;
using (var enumerator = queryResults.GetEnumerator())
{
var duration = Stopwatch.StartNew();
var operations = 0;
while (true)
{
database.WorkContext.UpdateFoundWork();
if (timeout != null)
timeout.Delay();
var batchCount = 0;
var shouldWaitNow = false;
token.ThrowIfCancellationRequested();
using (database.DocumentLock.Lock())
{
database.TransactionalStorage.Batch(actions =>
{
while (batchCount < batchSize && enumerator.MoveNext())
{
batchCount++;
operations++;
var result = batchOperation(enumerator.Current, transactionInformation);
if(options.RetrieveDetails)
array.Add(RavenJObject.FromObject(result));
if (operations >= maxOpsPerSec && duration.ElapsedMilliseconds < 1000)
{
shouldWaitNow = true;
break;
}
}
});
if (shouldWaitNow)
{
SystemTime.Wait(500);
operations = 0;
duration.Restart();
continue;
}
}
if (batchCount < batchSize) break;
}
}
return array;
}
public async Task<RavenJArray> GetIndexes(RavenConnectionStringOptions src, int totalCount)
{
var indexes = await Store.AsyncDatabaseCommands.GetIndexesAsync(totalCount, Options.BatchSize);
var result = new RavenJArray();
foreach (var index in indexes)
{
result.Add(new RavenJObject
{
{ "name", index.Name },
{ "definition", RavenJObject.FromObject(index) }
});
}
return (RavenJArray)RavenJToken.FromObject(result);
}
private RavenJArray GetAttachments(int start)
{
var array = new RavenJArray();
var attachmentInfos = _documentDatabase.GetAttachments(start, 128, null,null,50000);
foreach (var attachmentInfo in attachmentInfos)
{
var attachment = _store.DatabaseCommands.GetAttachment(attachmentInfo.Key);
var bytes = StreamToBytes(attachment.Data());
// Based on Raven.Smuggler.Api.ExportAttachments from build 888.
array.Add(
new RavenJObject
{
{"Data", bytes},
{"Metadata", attachmentInfo.Metadata},
{"Key", attachmentInfo.Key}
});
}
return array;
}
private RavenJToken ToRavenJToken(JsInstance v)
{
switch (v.Class)
{
case JsInstance.TYPE_OBJECT:
case JsInstance.CLASS_OBJECT:
return ToRavenJObject((JsObject)v);
case JsInstance.CLASS_DATE:
var dt = (DateTime)v.Value;
return new RavenJValue(dt);
case JsInstance.TYPE_NUMBER:
case JsInstance.CLASS_NUMBER:
var num = (double)v.Value;
var integer = Math.Truncate(num);
if (Math.Abs(num - integer) < double.Epsilon)
return new RavenJValue((long)integer);
return new RavenJValue(num);
case JsInstance.TYPE_STRING:
case JsInstance.TYPE_BOOLEAN:
case JsInstance.CLASS_STRING:
case JsInstance.CLASS_BOOLEAN:
return new RavenJValue(v.Value);
case JsInstance.CLASS_NULL:
case JsInstance.TYPE_NULL:
return RavenJValue.Null;
case JsInstance.CLASS_UNDEFINED:
case JsInstance.TYPE_UNDEFINED:
return RavenJValue.Null;
case JsInstance.CLASS_ARRAY:
var jsArray = ((JsArray)v);
var rja = new RavenJArray();
for (int i = 0; i < jsArray.Length; i++)
{
var jsInstance = jsArray.get(i);
var ravenJToken = ToRavenJToken(jsInstance);
if (ravenJToken == null)
continue;
rja.Add(ravenJToken);
}
return rja;
case JsInstance.CLASS_REGEXP:
case JsInstance.CLASS_ERROR:
case JsInstance.CLASS_ARGUMENTS:
case JsInstance.CLASS_DESCRIPTOR:
case JsInstance.CLASS_FUNCTION:
return null;
default:
throw new NotSupportedException(v.Class);
}
}
private void ReplicateDelete(string id, RavenJObject metadata, TExternal incoming)
{
TInternal existingItem;
Guid existingEtag;
var existingMetadata = TryGetExisting(id, out existingItem, out existingEtag);
if (existingMetadata == null)
{
log.Debug("Replicating deleted item {0} from {1} that does not exist, ignoring", id, Src);
return;
}
if (existingMetadata.Value <bool>(Constants.RavenReplicationConflict)) // locally conflicted
{
log.Debug("Replicating deleted item {0} from {1} that is already conflicted, adding to conflicts.", id, Src);
var savedConflictedItemId = SaveConflictedItem(id, metadata, incoming, existingEtag);
AppendToCurrentItemConflicts(id, savedConflictedItemId, existingMetadata, existingItem);
return;
}
if (existingMetadata.Value <bool>(Constants.RavenDeleteMarker)) //deleted locally as well
{
log.Debug("Replicating deleted item {0} from {1} that was deleted locally. Merging histories", id, Src);
var existingHistory = new RavenJArray(existingMetadata.Value <RavenJArray>(Constants.RavenReplicationHistory));
var newHistory = new RavenJArray(metadata.Value <RavenJArray>(Constants.RavenReplicationHistory));
foreach (var item in newHistory)
{
existingHistory.Add(item);
}
if (metadata.ContainsKey(Constants.RavenReplicationVersion) &&
metadata.ContainsKey(Constants.RavenReplicationSource))
{
existingHistory.Add(new RavenJObject
{
{ Constants.RavenReplicationVersion, metadata[Constants.RavenReplicationVersion] },
{ Constants.RavenReplicationSource, metadata[Constants.RavenReplicationSource] }
});
}
while (existingHistory.Length > Constants.ChangeHistoryLength)
{
existingHistory.RemoveAt(0);
}
MarkAsDeleted(id, metadata);
return;
}
if (Historian.IsDirectChildOfCurrent(metadata, existingMetadata)) // not modified
{
log.Debug("Delete of existing item {0} was replicated successfully from {1}", id, Src);
DeleteItem(id, existingEtag);
MarkAsDeleted(id, metadata);
return;
}
Database.TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() =>
Database.RaiseNotifications(new DocumentChangeNotification
{
Id = id,
Type = DocumentChangeTypes.ReplicationConflict
}));
var newConflictId = SaveConflictedItem(id, metadata, incoming, existingEtag);
log.Debug("Existing item {0} is in conflict with replicated delete from {1}, marking item as conflicted", id, Src);
// we have a new conflict move the existing doc to a conflict and create a conflict document
var existingDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(existingEtag);
CreateConflict(id, newConflictId, existingDocumentConflictId, existingItem, existingMetadata);
}
private RavenJToken ToRavenJToken(JsValue v, string propertyKey, bool recursiveCall)
{
if (v.IsBoolean())
{
return(new RavenJValue(v.AsBoolean()));
}
if (v.IsString())
{
const string RavenDataByteArrayToBase64 = "raven-data:byte[];base64,";
var valueAsObject = v.ToObject();
var value = valueAsObject != null?valueAsObject.ToString() : null;
if (value != null && value.StartsWith(RavenDataByteArrayToBase64))
{
value = value.Remove(0, RavenDataByteArrayToBase64.Length);
var byteArray = Convert.FromBase64String(value);
return(new RavenJValue(byteArray));
}
return(new RavenJValue(value));
}
if (v.IsNumber())
{
var num = v.AsNumber();
KeyValuePair <RavenJValue, JsValue> property;
if (propertiesByValue.TryGetValue(propertyKey, out property))
{
var originalValue = property.Key;
if (originalValue.Type == JTokenType.Float ||
originalValue.Type == JTokenType.Integer)
{
// If the current value is exactly as the original value, we can return the original value before we made the JS conversion,
// which will convert a Int64 to jsFloat.
var originalJsValue = property.Value;
if (originalJsValue.IsNumber() && Math.Abs(num - originalJsValue.AsNumber()) < double.Epsilon)
{
return(originalValue);
}
//We might have change the type of num from Integer to long in the script by design
//Making sure the number isn't a real float before returning it as integer
if (originalValue.Type == JTokenType.Integer && (Math.Abs(num - Math.Floor(num)) <= double.Epsilon || Math.Abs(num - Math.Ceiling(num)) <= double.Epsilon))
{
return(new RavenJValue((long)num));
}
return(new RavenJValue(num));//float
}
}
// If we don't have the type, assume that if the number ending with ".0" it actually an integer.
var integer = Math.Truncate(num);
if (Math.Abs(num - integer) < double.Epsilon)
{
return(new RavenJValue((long)integer));
}
return(new RavenJValue(num));
}
if (v.IsNull())
{
return(RavenJValue.Null);
}
if (v.IsUndefined())
{
return(RavenJValue.Null);
}
if (v.IsArray())
{
var jsArray = v.AsArray();
var rja = new RavenJArray();
foreach (var property in jsArray.GetOwnProperties())
{
if (InheritedProperties.Contains(property.Key))
{
continue;
}
var jsInstance = property.Value.Value;
if (!jsInstance.HasValue)
{
continue;
}
var ravenJToken = ToRavenJToken(jsInstance.Value, propertyKey + "[" + property.Key + "]", recursiveCall);
if (ravenJToken == null)
{
continue;
}
rja.Add(ravenJToken);
}
return(rja);
}
if (v.IsDate())
{
return(new RavenJValue(v.AsDate().ToDateTime()));
}
if (v.IsObject())
{
return(ToRavenJObject(v, propertyKey, recursiveCall));
}
if (v.IsRegExp())
{
return(null);
}
throw new NotSupportedException(v.Type.ToString());
}
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, string matches, int start, int pageSize)
{
if (idPrefix == null)
throw new ArgumentNullException("idPrefix");
idPrefix = idPrefix.Trim();
var list = new RavenJArray();
TransactionalStorage.Batch(actions =>
{
while (true)
{
int docCount = 0;
var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize);
var documentRetriever = new DocumentRetriever(actions, ReadTriggers);
foreach (var doc in documents)
{
docCount++;
if (WildcardMatcher.Matches(matches, doc.Key.Substring(idPrefix.Length)) == false)
continue;
DocumentRetriever.EnsureIdInMetadata(doc);
var document = documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Load);
if (document == null)
continue;
list.Add(document.ToJson());
}
if (list.Length != 0 || docCount == 0)
break;
start += docCount;
}
});
return list;
}
protected override Guid FlushBatch(List<RavenJObject> batch)
{
var sw = Stopwatch.StartNew();
var commands = new RavenJArray();
foreach (var doc in batch)
{
var metadata = doc.Value<RavenJObject>("@metadata");
doc.Remove("@metadata");
commands.Add(new RavenJObject
{
{"Method", "PUT"},
{"Document", doc},
{"Metadata", metadata},
{"Key", metadata.Value<string>("@id")}
});
}
var retries = retriesCount;
HttpRavenRequest request = null;
BatchResult[] results;
while (true)
{
try
{
request = CreateRequest("/bulk_docs", "POST");
request.Write(commands);
results = request.ExecuteRequest<BatchResult[]>();
sw.Stop();
break;
}
catch (Exception e)
{
if (--retries == 0 || request == null)
throw;
sw.Stop();
LastRequestErrored = true;
ShowProgress("Error flushing to database, remaining attempts {0} - time {2:#,#} ms, will retry [{3:#,#.##;;0} kb compressed to {4:#,#.##;;0} kb]. Error: {1}",
retriesCount - retries, e, sw.ElapsedMilliseconds,
(double)request.NumberOfBytesWrittenUncompressed / 1024,
(double)request.NumberOfBytesWrittenCompressed / 1024);
}
}
total += batch.Count;
ShowProgress("{2,5:#,#}: Wrote {0:#,#;;0} in {1,6:#,#;;0} ms ({6:0.00} ms per doc) (total of {3:#,#;;0}) documents [{4:#,#.##;;0} kb compressed to {5:#,#.##;;0} kb]",
batch.Count, sw.ElapsedMilliseconds, ++count, total,
(double)request.NumberOfBytesWrittenUncompressed / 1024,
(double)request.NumberOfBytesWrittenCompressed / 1024,
Math.Round((double)sw.ElapsedMilliseconds / Math.Max(1, batch.Count), 2));
batch.Clear();
if (results.Length == 0)
return Guid.Empty;
return results.Last().Etag.Value;
}
public override void OnPut(string key, RavenJObject jsonReplicationDocument, RavenJObject metadata)
{
using (Database.DisableAllTriggersForCurrentThread())
{
if (metadata.Remove(Constants.RavenReplicationConflictSkipResolution))
{
if (key.IndexOf("/conflicts/", StringComparison.OrdinalIgnoreCase) == -1)
{
metadata["@Http-Status-Code"] = 409;
metadata["@Http-Status-Description"] = "Conflict";
}
return;
}
metadata.Remove(Constants.RavenReplicationConflict);// you can't put conflicts
var oldVersion = Database.Documents.Get(key);
if (oldVersion == null)
{
return;
}
if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null)
{
return;
}
var history = new RavenJArray();
metadata[Constants.RavenReplicationHistory] = history;
// this is a conflict document, holding document keys in the
// values of the properties
var conflicts = oldVersion.DataAsJson.Value <RavenJArray>("Conflicts");
if (conflicts == null)
{
return;
}
var list = new List <RavenJArray>
{
new RavenJArray(ReplicationData.GetHistory(metadata)) // first item to interleave
};
foreach (var prop in conflicts)
{
RavenJObject deletedMetadata;
Database.Documents.Delete(prop.Value <string>(), null, out deletedMetadata);
if (deletedMetadata != null)
{
var conflictHistory = new RavenJArray(ReplicationData.GetHistory(deletedMetadata));
conflictHistory.Add(new RavenJObject
{
{ Constants.RavenReplicationVersion, deletedMetadata[Constants.RavenReplicationVersion] },
{ Constants.RavenReplicationSource, deletedMetadata[Constants.RavenReplicationSource] }
});
list.Add(conflictHistory);
}
}
int index = 0;
bool added = true;
while (added) // interleave the history from all conflicts
{
added = false;
foreach (var deletedMetadata in list)
{
// add the conflict history to the mix, so we make sure that we mark that we resolved the conflict
if (index < deletedMetadata.Length)
{
history.Add(deletedMetadata[index]);
added = true;
}
}
index++;
}
while (history.Length > Constants.ChangeHistoryLength)
{
history.RemoveAt(0);
}
}
}
