public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation) { if (key.StartsWith("Raven/", StringComparison.OrdinalIgnoreCase) && // we don't deal with system documents key.StartsWith("Raven/Hilo/", StringComparison.OrdinalIgnoreCase) == false) // except for hilos return; using (Database.DisableAllTriggersForCurrentThread()) { var documentMetadata = GetDocumentMetadata(key); if (documentMetadata != null) { RavenJArray history = new RavenJArray(ReplicationData.GetHistory(documentMetadata)); metadata[Constants.RavenReplicationHistory] = history; if (documentMetadata.ContainsKey(Constants.RavenReplicationVersion) && documentMetadata.ContainsKey(Constants.RavenReplicationSource)) { history.Add(new RavenJObject { {Constants.RavenReplicationVersion, documentMetadata[Constants.RavenReplicationVersion]}, {Constants.RavenReplicationSource, documentMetadata[Constants.RavenReplicationSource]} }); } while (history.Length > Constants.ChangeHistoryLength) { history.RemoveAt(0); } } metadata[Constants.RavenReplicationVersion] = RavenJToken.FromObject(HiLo.NextId()); metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(Database.TransactionalStorage.Id); } }
public override void OnPut(string key, Stream data, RavenJObject metadata) { if (key.StartsWith("Raven/")) // we don't deal with system attachment return; using (Database.DisableAllTriggersForCurrentThread()) { var attachmentMetadata = GetAttachmentMetadata(key); if (attachmentMetadata != null) { RavenJArray history = new RavenJArray(metadata.Value<RavenJArray>(Constants.RavenReplicationHistory)); metadata[Constants.RavenReplicationHistory] = history; if (attachmentMetadata.ContainsKey(Constants.RavenReplicationVersion) && attachmentMetadata.ContainsKey(Constants.RavenReplicationSource)) { history.Add(new RavenJObject { {Constants.RavenReplicationVersion, attachmentMetadata[Constants.RavenReplicationVersion]}, {Constants.RavenReplicationSource, attachmentMetadata[Constants.RavenReplicationSource]} }); } if (history.Length > Constants.ChangeHistoryLength) { history.RemoveAt(0); } } metadata[Constants.RavenReplicationVersion] = RavenJToken.FromObject(HiLo.NextId()); metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(Database.TransactionalStorage.Id); } }
public override void OnPut(string key, Stream data, RavenJObject metadata) { using (Database.DisableAllTriggersForCurrentThread()) { metadata.Remove(Constants.RavenReplicationConflict);// you can't put conflicts var oldVersion = Database.Attachments.GetStatic(key); if (oldVersion == null) return; if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null) return; var history = new RavenJArray(ReplicationData.GetHistory(metadata)); metadata[Constants.RavenReplicationHistory] = history; var ravenJTokenEqualityComparer = new RavenJTokenEqualityComparer(); // this is a conflict document, holding document keys in the // values of the properties var conflictData = oldVersion.Data().ToJObject(); var conflicts = conflictData.Value<RavenJArray>("Conflicts"); if (conflicts == null) return; foreach (var prop in conflicts) { var id = prop.Value<string>(); Attachment attachment = Database.Attachments.GetStatic(id); if(attachment == null) continue; Database.Attachments.DeleteStatic(id, null); // add the conflict history to the mix, so we make sure that we mark that we resolved the conflict var conflictHistory = new RavenJArray(ReplicationData.GetHistory(attachment.Metadata)); conflictHistory.Add(new RavenJObject { {Constants.RavenReplicationVersion, attachment.Metadata[Constants.RavenReplicationVersion]}, {Constants.RavenReplicationSource, attachment.Metadata[Constants.RavenReplicationSource]} }); foreach (var item in conflictHistory) { if (history.Any(x => ravenJTokenEqualityComparer.Equals(x, item))) continue; history.Add(item); } } } }
public override IEnumerable<Row> Execute(IEnumerable<Row> rows) { int count = 0; foreach (var commentsForPosts in rows.Partition(Constants.BatchSize)) { var cmds = new List<ICommandData>(); foreach (var commentsForPost in commentsForPosts.GroupBy(row => row["PostId"])) { var comments = new RavenJArray(); foreach (var row in commentsForPost) { comments.Add(new RavenJObject { {"Score", new RavenJValue(row["Score"])}, {"CreationDate", new RavenJValue(row["CreationDate"])}, {"Text", new RavenJValue(row["Text"])}, {"UserId", new RavenJValue(row["UserId"])} }); } cmds.Add(new PatchCommandData { Key = "posts/" + commentsForPost.Key, Patches = new[] { new PatchRequest { Name = "Comments", Type = PatchCommandType.Set, Value = comments }, } }); } count++; WriteCommandsTo("Comments #" + count.ToString("00000") + ".json", cmds); } yield break; }
public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation) { using (Database.DisableAllTriggersForCurrentThread()) { metadata.Remove(Constants.RavenReplicationConflict);// you can't put conflicts var oldVersion = Database.Get(key, transactionInformation); if (oldVersion == null) return; if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null) return; RavenJArray history = new RavenJArray(ReplicationData.GetHistory(metadata)); metadata[Constants.RavenReplicationHistory] = history; var ravenJTokenEqualityComparer = new RavenJTokenEqualityComparer(); // this is a conflict document, holding document keys in the // values of the properties var conflicts = oldVersion.DataAsJson.Value<RavenJArray>("Conflicts"); if(conflicts == null) return; foreach (var prop in conflicts) { RavenJObject deletedMetadata; Database.Delete(prop.Value<string>(), null, transactionInformation, out deletedMetadata); // add the conflict history to the mix, so we make sure that we mark that we resolved the conflict var conflictHistory = new RavenJArray(ReplicationData.GetHistory(deletedMetadata)); conflictHistory.Add(new RavenJObject { {Constants.RavenReplicationVersion, deletedMetadata[Constants.RavenReplicationVersion]}, {Constants.RavenReplicationSource, deletedMetadata[Constants.RavenReplicationSource]} }); foreach (var item in conflictHistory) { if(history.Any(x=>ravenJTokenEqualityComparer.Equals(x, item))) continue; history.Add(item); } } } }
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, bool allowStale, Func<string, TransactionInformation, object> batchOperation) { var array = new RavenJArray(); var bulkIndexQuery = new IndexQuery { Query = indexQuery.Query, Start = indexQuery.Start, Cutoff = indexQuery.Cutoff, PageSize = int.MaxValue, FieldsToFetch = new[] { Constants.DocumentIdFieldName }, SortedFields = indexQuery.SortedFields }; bool stale; var queryResults = database.QueryDocumentIds(index, bulkIndexQuery, out stale); if (stale && allowStale == false) { throw new InvalidOperationException( "Bulk operation cancelled because the index is stale and allowStale is false"); } const int batchSize = 1024; using (var enumerator = queryResults.GetEnumerator()) { while (true) { var batchCount = 0; database.TransactionalStorage.Batch(actions => { while (batchCount < batchSize && enumerator.MoveNext()) { batchCount++; var result = batchOperation(enumerator.Current, transactionInformation); array.Add(RavenJObject.FromObject(result, JsonExtensions.CreateDefaultJsonSerializer())); } }); if (batchCount < batchSize) break; } } return array; }
public static RavenJToken MinimizeToken(RavenJToken obj, int depth = 0) { switch (obj.Type) { case JTokenType.Array: var array = new RavenJArray(); foreach (var item in ((RavenJArray)obj)) { array.Add(MinimizeToken(item, depth + 1)); } return array; case JTokenType.Object: var ravenJObject = ((RavenJObject)obj); if (ravenJObject.ContainsKey(Constants.Metadata) == false) { // this might be a wrapper object, let check for first level arrays if (depth == 0) { var newRootObj = new RavenJObject(); foreach (var prop in ravenJObject) { newRootObj[prop.Key] = prop.Value.Type == JTokenType.Array ? MinimizeToken(prop.Value, depth + 1) : prop.Value; } return newRootObj; } return obj; } var newObj = new RavenJObject(); newObj[Constants.Metadata] = ravenJObject[Constants.Metadata]; return newObj; default: return obj; } }
public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation) { using (Database.DisableAllTriggersForCurrentThread()) { if (metadata.Remove(Constants.RavenReplicationConflictSkipResolution)) { if (key.IndexOf("/conflicts/", StringComparison.OrdinalIgnoreCase) == -1) { metadata["@Http-Status-Code"] = 409; metadata["@Http-Status-Description"] = "Conflict"; } return; } metadata.Remove(Constants.RavenReplicationConflict);// you can't put conflicts var oldVersion = Database.Documents.Get(key, transactionInformation); if (oldVersion == null) return; if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null) return; var history = new RavenJArray(); metadata[Constants.RavenReplicationHistory] = history; // this is a conflict document, holding document keys in the // values of the properties var conflicts = oldVersion.DataAsJson.Value<RavenJArray>("Conflicts"); if(conflicts == null) return; var list = new List<RavenJArray> { new RavenJArray(ReplicationData.GetHistory(metadata)) // first item to interleave }; foreach (var prop in conflicts) { RavenJObject deletedMetadata; Database.Documents.Delete(prop.Value<string>(), null, transactionInformation, out deletedMetadata); if (deletedMetadata != null) { var conflictHistory = new RavenJArray(ReplicationData.GetHistory(deletedMetadata)); conflictHistory.Add(new RavenJObject { {Constants.RavenReplicationVersion, deletedMetadata[Constants.RavenReplicationVersion]}, {Constants.RavenReplicationSource, deletedMetadata[Constants.RavenReplicationSource]} }); list.Add(conflictHistory); } } int index = 0; bool added = true; while (added) // interleave the history from all conflicts { added = false; foreach (var deletedMetadata in list) { // add the conflict history to the mix, so we make sure that we mark that we resolved the conflict if (index < deletedMetadata.Length) { history.Add(deletedMetadata[index]); added = true; } } index++; } while (history.Length > Constants.ChangeHistoryLength) { history.RemoveAt(0); } } }
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, int start, int pageSize) { var list = new RavenJArray(); TransactionalStorage.Batch(actions => { var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start) .Take(pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents) { DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; list.Add(document.ToJson()); } }); return list; }
private void FlushBatch(List<RavenJObject> batch) { var sw = Stopwatch.StartNew(); var commands = new RavenJArray(); foreach (var doc in batch) { var metadata = doc.Value<RavenJObject>("@metadata"); doc.Remove("@metadata"); commands.Add(new RavenJObject { {"Method", "PUT"}, {"Document", doc}, {"Metadata", metadata}, {"Key", metadata.Value<string>("@id")} }); } var request = CreateRequest("bulk_docs", "POST"); var size = request.Write(commands); request.ExecuteRequest(); Console.Write("Wrote {0} documents", batch.Count, sw.ElapsedMilliseconds); if (size > 0) Console.Write(" [{0:#,#;;0} kb]", Math.Round((double) size/1024, 2)); Console.WriteLine(" in {0:#,#;;0} ms", sw.ElapsedMilliseconds); batch.Clear(); }
protected override void FlushBatch(List<RavenJObject> batch) { var sw = Stopwatch.StartNew(); var commands = new RavenJArray(); foreach (var doc in batch) { var metadata = doc.Value<RavenJObject>("@metadata"); doc.Remove("@metadata"); commands.Add(new RavenJObject { {"Method", "PUT"}, {"Document", doc}, {"Metadata", metadata}, {"Key", metadata.Value<string>("@id")} }); } var request = CreateRequest("/bulk_docs", "POST"); request.Write(commands); request.ExecuteRequest(); ShowProgress("Wrote {0} documents in {1}", batch.Count, sw.ElapsedMilliseconds); ShowProgress(" in {0:#,#;;0} ms", sw.ElapsedMilliseconds); batch.Clear(); }
private void AddValue(PatchRequest patchCmd, string propName, RavenJToken token) { EnsurePreviousValueMatchCurrentValue(patchCmd, token); if (token == null) { token = new RavenJArray(); document[propName] = token; } var array = GetArray(token, propName); array = new RavenJArray(array); document[propName] = array; array.Add(patchCmd.Value); }
private RavenJToken ToRavenJToken(JsInstance v) { switch (v.Class) { case JsInstance.TYPE_OBJECT: case JsInstance.CLASS_OBJECT: return(ToRavenJObject((JsObject)v)); case JsInstance.CLASS_DATE: var dt = (DateTime)v.Value; return(new RavenJValue(dt)); case JsInstance.TYPE_NUMBER: case JsInstance.CLASS_NUMBER: var num = (double)v.Value; KeyValuePair <RavenJValue, object> property; if (propertiesByValue.TryGetValue(v, out property)) { var originalValue = property.Key; if (originalValue.Type == JTokenType.Float) { return(new RavenJValue(num)); } if (originalValue.Type == JTokenType.Integer) { // If the current value is exactly as the original value, we can return the original value before we made the JS conversion, // which will convert a Int64 to jsFloat. var originalJsValue = property.Value; if (originalJsValue is double && Math.Abs(num - (double)originalJsValue) < double.Epsilon) { return(originalValue); } return(new RavenJValue((long)num)); } } // If we don't have the type, assume that if the number ending with ".0" it actually an integer. var integer = Math.Truncate(num); if (Math.Abs(num - integer) < double.Epsilon) { return(new RavenJValue((long)integer)); } return(new RavenJValue(num)); case JsInstance.TYPE_STRING: case JsInstance.TYPE_BOOLEAN: case JsInstance.CLASS_STRING: case JsInstance.CLASS_BOOLEAN: return(new RavenJValue(v.Value)); case JsInstance.CLASS_NULL: case JsInstance.TYPE_NULL: return(RavenJValue.Null); case JsInstance.CLASS_UNDEFINED: case JsInstance.TYPE_UNDEFINED: return(RavenJValue.Null); case JsInstance.CLASS_ARRAY: var jsArray = ((JsArray)v); var rja = new RavenJArray(); for (int i = 0; i < jsArray.Length; i++) { var jsInstance = jsArray.get(i); var ravenJToken = ToRavenJToken(jsInstance); if (ravenJToken == null) { continue; } rja.Add(ravenJToken); } return(rja); case JsInstance.CLASS_REGEXP: case JsInstance.CLASS_ERROR: case JsInstance.CLASS_ARGUMENTS: case JsInstance.CLASS_DESCRIPTOR: case JsInstance.CLASS_FUNCTION: return(null); default: throw new NotSupportedException(v.Class); } }
public override void OnPut(string key, Stream data, RavenJObject metadata) { if (key.StartsWith("Raven/")) // we don't deal with system attachment { return; } using (Database.DisableAllTriggersForCurrentThread()) { var attachmentMetadata = GetAttachmentMetadata(key); if (attachmentMetadata != null) { var history = new RavenJArray(ReplicationData.GetHistory(attachmentMetadata)); if (attachmentMetadata.ContainsKey(Constants.RavenReplicationMergedHistory) == false) { if (attachmentMetadata.ContainsKey(Constants.RavenReplicationVersion) && attachmentMetadata.ContainsKey(Constants.RavenReplicationSource)) { history.Add(new RavenJObject { { Constants.RavenReplicationVersion, attachmentMetadata[Constants.RavenReplicationVersion] }, { Constants.RavenReplicationSource, attachmentMetadata[Constants.RavenReplicationSource] } }); } else { history.Add(new RavenJObject { { Constants.RavenReplicationVersion, 0 }, { Constants.RavenReplicationSource, RavenJToken.FromObject(Database.TransactionalStorage.Id) } }); } var sources = new HashSet <RavenJToken>(RavenJTokenEqualityComparer.Default); int pos = history.Length - 1; for (; pos >= 0; pos--) { var source = ((RavenJObject)history[pos])[Constants.RavenReplicationSource]; if (sources.Contains(source)) { history.RemoveAt(pos); continue; } sources.Add(source); } metadata[Constants.RavenReplicationMergedHistory] = true; metadata[Constants.RavenReplicationHistory] = history; } //If we have the flag we must have Constants.RavenReplicationVersion and Constants.RavenReplicationSource too //Here we assume that the replication history is in the form of a "sorted dictionary" so we just need to remove //the entry with the current source id and insert the new version at the end of the history. else { int i = history.Length - 1; for (; i >= 0; i--) { var currentEntry = history[i]; if (RavenJTokenEqualityComparer.Default.Equals(((RavenJObject)currentEntry) [Constants.RavenReplicationSource], attachmentMetadata[Constants.RavenReplicationSource])) { break; } } if (i != -1) { history.RemoveAt(i); } history.Add(new RavenJObject { { Constants.RavenReplicationVersion, attachmentMetadata[Constants.RavenReplicationVersion] }, { Constants.RavenReplicationSource, attachmentMetadata[Constants.RavenReplicationSource] } }); metadata[Constants.RavenReplicationHistory] = history; } } metadata[Constants.RavenReplicationVersion] = RavenJToken.FromObject(ReplicationHiLo.NextId(Database)); metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(Database.TransactionalStorage.Id); } }
public override void OnPut(string key, RavenJObject jsonReplicationDocument, RavenJObject metadata, TransactionInformation transactionInformation) { using (Database.DisableAllTriggersForCurrentThread()) { if (metadata.Remove(Constants.RavenReplicationConflictSkipResolution)) { if (key.IndexOf("/conflicts/", StringComparison.OrdinalIgnoreCase) == -1) { metadata["@Http-Status-Code"] = 409; metadata["@Http-Status-Description"] = "Conflict"; } return; } metadata.Remove(Constants.RavenReplicationConflict);// you can't put conflicts metadata.Remove(Constants.RavenReplicationConflictDocument); var oldVersion = Database.Documents.Get(key, transactionInformation); if (oldVersion == null) { return; } if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null) { return; } var history = new RavenJArray(); metadata[Constants.RavenReplicationHistory] = history; // this is a conflict document, holding document keys in the // values of the properties var conflicts = oldVersion.DataAsJson.Value <RavenJArray>("Conflicts"); if (conflicts == null) { return; } Dictionary <string, RavenJObject> conflictHistoryAsDictionary = new Dictionary <string, RavenJObject>(); foreach (var prop in conflicts) { RavenJObject deletedMetadata; Database.Documents.Delete(prop.Value <string>(), null, transactionInformation, out deletedMetadata); if (deletedMetadata != null) { var conflictHistory = new RavenJArray(ReplicationData.GetOrCreateHistory(deletedMetadata)); conflictHistory.Add(new RavenJObject { { Constants.RavenReplicationVersion, deletedMetadata[Constants.RavenReplicationVersion] }, { Constants.RavenReplicationSource, deletedMetadata[Constants.RavenReplicationSource] } }); Historian.MergeSingleHistory(conflictHistory, conflictHistoryAsDictionary, key); } } metadata[Constants.RavenReplicationHistory] = new RavenJArray(conflictHistoryAsDictionary.Values); metadata[Constants.RavenReplicationMergedHistory] = true; } }
private RavenJToken ToRavenJToken(JsInstance v, string propertyName) { switch (v.Class) { case JsInstance.TYPE_OBJECT: case JsInstance.CLASS_OBJECT: return ToRavenJObject((JsObject)v); case JsInstance.CLASS_DATE: var dt = (DateTime)v.Value; return new RavenJValue(dt); case JsInstance.TYPE_NUMBER: case JsInstance.CLASS_NUMBER: var num = (double)v.Value; JTokenType type; if (propertiesTypeByName.TryGetValue(propertyName, out type)) { if (type == JTokenType.Float) return new RavenJValue(num); if (type == JTokenType.Integer) return new RavenJValue((long) num); } // If we don't have the type, assume that if the number ending with ".0" it actually an integer. var integer = Math.Truncate(num); if (Math.Abs(num - integer) < double.Epsilon) return new RavenJValue((long)integer); return new RavenJValue(num); case JsInstance.TYPE_STRING: case JsInstance.TYPE_BOOLEAN: case JsInstance.CLASS_STRING: case JsInstance.CLASS_BOOLEAN: return new RavenJValue(v.Value); case JsInstance.CLASS_NULL: case JsInstance.TYPE_NULL: return RavenJValue.Null; case JsInstance.CLASS_UNDEFINED: case JsInstance.TYPE_UNDEFINED: return RavenJValue.Null; case JsInstance.CLASS_ARRAY: var jsArray = ((JsArray)v); var rja = new RavenJArray(); for (int i = 0; i < jsArray.Length; i++) { var jsInstance = jsArray.get(i); var ravenJToken = ToRavenJToken(jsInstance, propertyName); if (ravenJToken == null) continue; rja.Add(ravenJToken); } return rja; case JsInstance.CLASS_REGEXP: case JsInstance.CLASS_ERROR: case JsInstance.CLASS_ARGUMENTS: case JsInstance.CLASS_DESCRIPTOR: case JsInstance.CLASS_FUNCTION: return null; default: throw new NotSupportedException(v.Class); } }
protected override Guid FlushBatch(List <RavenJObject> batch) { var sw = Stopwatch.StartNew(); var commands = new RavenJArray(); foreach (var doc in batch) { var metadata = doc.Value <RavenJObject>("@metadata"); doc.Remove("@metadata"); commands.Add(new RavenJObject { { "Method", "PUT" }, { "Document", doc }, { "Metadata", metadata }, { "Key", metadata.Value <string>("@id") } }); } var retries = retriesCount; HttpRavenRequest request = null; BatchResult[] results; while (true) { try { request = CreateRequest("/bulk_docs", "POST"); request.Write(commands); results = request.ExecuteRequest <BatchResult[]>(); sw.Stop(); break; } catch (Exception e) { if (--retries == 0 || request == null) { throw; } sw.Stop(); LastRequestErrored = true; ShowProgress("Error flushing to database, remaining attempts {0} - time {2:#,#} ms, will retry [{3:#,#.##;;0} kb compressed to {4:#,#.##;;0} kb]. Error: {1}", retriesCount - retries, e, sw.ElapsedMilliseconds, (double)request.NumberOfBytesWrittenUncompressed / 1024, (double)request.NumberOfBytesWrittenCompressed / 1024); } } total += batch.Count; ShowProgress("{2,5:#,#}: Wrote {0:#,#;;0} in {1,6:#,#;;0} ms ({6:0.00} ms per doc) (total of {3:#,#;;0}) documents [{4:#,#.##;;0} kb compressed to {5:#,#.##;;0} kb]", batch.Count, sw.ElapsedMilliseconds, ++count, total, (double)request.NumberOfBytesWrittenUncompressed / 1024, (double)request.NumberOfBytesWrittenCompressed / 1024, Math.Round((double)sw.ElapsedMilliseconds / Math.Max(1, batch.Count), 2)); batch.Clear(); if (results.Length == 0) { return(Guid.Empty); } return(results.Last().Etag.Value); }
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize, CancellationToken token, ref int nextStart, string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null, string skipAfter = null) { var list = new RavenJArray(); GetDocumentsWithIdStartingWith(idPrefix, matches, exclude, start, pageSize, token, ref nextStart, doc => list.Add(doc.ToJson()), transformer, transformerParameters, skipAfter); return(list); }
private RavenJToken ToRavenJToken(JsInstance v, string propertyName) { switch (v.Class) { case JsInstance.TYPE_OBJECT: case JsInstance.CLASS_OBJECT: return(ToRavenJObject((JsObject)v)); case JsInstance.CLASS_DATE: var dt = (DateTime)v.Value; return(new RavenJValue(dt)); case JsInstance.TYPE_NUMBER: case JsInstance.CLASS_NUMBER: var num = (double)v.Value; JTokenType type; if (propertiesTypeByName.TryGetValue(propertyName, out type)) { if (type == JTokenType.Float) { return(new RavenJValue(num)); } if (type == JTokenType.Integer) { return(new RavenJValue((long)num)); } } // If we don't have the type, assume that if the number ending with ".0" it actually an integer. var integer = Math.Truncate(num); if (Math.Abs(num - integer) < double.Epsilon) { return(new RavenJValue((long)integer)); } return(new RavenJValue(num)); case JsInstance.TYPE_STRING: case JsInstance.TYPE_BOOLEAN: case JsInstance.CLASS_STRING: case JsInstance.CLASS_BOOLEAN: return(new RavenJValue(v.Value)); case JsInstance.CLASS_NULL: case JsInstance.TYPE_NULL: return(RavenJValue.Null); case JsInstance.CLASS_UNDEFINED: case JsInstance.TYPE_UNDEFINED: return(RavenJValue.Null); case JsInstance.CLASS_ARRAY: var jsArray = ((JsArray)v); var rja = new RavenJArray(); for (int i = 0; i < jsArray.Length; i++) { var jsInstance = jsArray.get(i); var ravenJToken = ToRavenJToken(jsInstance, propertyName); if (ravenJToken == null) { continue; } rja.Add(ravenJToken); } return(rja); case JsInstance.CLASS_REGEXP: case JsInstance.CLASS_ERROR: case JsInstance.CLASS_ARGUMENTS: case JsInstance.CLASS_DESCRIPTOR: case JsInstance.CLASS_FUNCTION: return(null); default: throw new NotSupportedException(v.Class); } }
public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation) { using (Database.DisableAllTriggersForCurrentThread()) { metadata.Remove(Constants.RavenReplicationConflict); // you can't put conflicts var oldVersion = Database.Get(key, transactionInformation); if (oldVersion == null) { return; } if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null) { return; } var history = new RavenJArray(); metadata[Constants.RavenReplicationHistory] = history; var ravenJTokenEqualityComparer = new RavenJTokenEqualityComparer(); // this is a conflict document, holding document keys in the // values of the properties var conflicts = oldVersion.DataAsJson.Value <RavenJArray>("Conflicts"); if (conflicts == null) { return; } var list = new List <RavenJArray> { new RavenJArray(ReplicationData.GetHistory(metadata)) // first item to interleave }; foreach (var prop in conflicts) { RavenJObject deletedMetadata; Database.Delete(prop.Value <string>(), null, transactionInformation, out deletedMetadata); if (deletedMetadata != null) { var conflictHistory = new RavenJArray(ReplicationData.GetHistory(deletedMetadata)); conflictHistory.Add(new RavenJObject { { Constants.RavenReplicationVersion, deletedMetadata[Constants.RavenReplicationVersion] }, { Constants.RavenReplicationSource, deletedMetadata[Constants.RavenReplicationSource] } }); list.Add(conflictHistory); } } int index = 0; bool added = true; while (added) // interleave the history from all conflicts { added = false; foreach (var deletedMetadata in list) { // add the conflict history to the mix, so we make sure that we mark that we resolved the conflict if (index < deletedMetadata.Length) { history.Add(deletedMetadata[index]); added = true; } } index++; } while (history.Length > Constants.ChangeHistoryLength) { history.RemoveAt(0); } } }
private RavenJToken ToRavenJToken(JsValue v, string propertyKey) { if (v.IsBoolean()) return new RavenJValue(v.AsBoolean()); if (v.IsString()) { const string RavenDataByteArrayToBase64 = "raven-data:byte[];base64,"; var value = v.AsString(); if (value != null && value.StartsWith(RavenDataByteArrayToBase64)) { value = value.Remove(0, RavenDataByteArrayToBase64.Length); var byteArray = Convert.FromBase64String(value); return new RavenJValue(byteArray); } return new RavenJValue(value); } if (v.IsNumber()) { var num = v.AsNumber(); KeyValuePair<RavenJValue, JsValue> property; if (propertiesByValue.TryGetValue(propertyKey, out property)) { var originalValue = property.Key; if (originalValue.Type == JTokenType.Float) return new RavenJValue(num); if (originalValue.Type == JTokenType.Integer) { // If the current value is exactly as the original value, we can return the original value before we made the JS conversion, // which will convert a Int64 to jsFloat. var originalJsValue = property.Value; if (originalJsValue.IsNumber() && Math.Abs(num - originalJsValue.AsNumber()) < double.Epsilon) return originalValue; return new RavenJValue((long)num); } } // If we don't have the type, assume that if the number ending with ".0" it actually an integer. var integer = Math.Truncate(num); if (Math.Abs(num - integer) < double.Epsilon) return new RavenJValue((long)integer); return new RavenJValue(num); } if (v.IsNull()) return RavenJValue.Null; if (v.IsUndefined()) return RavenJValue.Null; if (v.IsArray()) { var jsArray = v.AsArray(); var rja = new RavenJArray(); foreach (var property in jsArray.Properties) { if (property.Key == "length") continue; var jsInstance = property.Value.Value; if (!jsInstance.HasValue) continue; var ravenJToken = ToRavenJToken(jsInstance.Value, CreatePropertyKey(property.Key, propertyKey)); if (ravenJToken == null) continue; rja.Add(ravenJToken); } return rja; } if (v.IsObject()) return ToRavenJObject(v, propertyKey); if (v.IsRegExp()) return null; throw new NotSupportedException(v.Type.ToString()); }
private RavenJToken ToRavenJToken(JsValue v, string propertyKey, bool recursiveCall) { if (v.IsBoolean()) { return(new RavenJValue(v.AsBoolean())); } if (v.IsString()) { const string RavenDataByteArrayToBase64 = "raven-data:byte[];base64,"; var value = v.AsString(); if (value != null && value.StartsWith(RavenDataByteArrayToBase64)) { value = value.Remove(0, RavenDataByteArrayToBase64.Length); var byteArray = Convert.FromBase64String(value); return(new RavenJValue(byteArray)); } return(new RavenJValue(value)); } if (v.IsNumber()) { var num = v.AsNumber(); KeyValuePair <RavenJValue, JsValue> property; if (propertiesByValue.TryGetValue(propertyKey, out property)) { var originalValue = property.Key; if (originalValue.Type == JTokenType.Float) { return(new RavenJValue(num)); } if (originalValue.Type == JTokenType.Integer) { // If the current value is exactly as the original value, we can return the original value before we made the JS conversion, // which will convert a Int64 to jsFloat. var originalJsValue = property.Value; if (originalJsValue.IsNumber() && Math.Abs(num - originalJsValue.AsNumber()) < double.Epsilon) { return(originalValue); } return(new RavenJValue((long)num)); } } // If we don't have the type, assume that if the number ending with ".0" it actually an integer. var integer = Math.Truncate(num); if (Math.Abs(num - integer) < double.Epsilon) { return(new RavenJValue((long)integer)); } return(new RavenJValue(num)); } if (v.IsNull()) { return(RavenJValue.Null); } if (v.IsUndefined()) { return(RavenJValue.Null); } if (v.IsArray()) { var jsArray = v.AsArray(); var rja = new RavenJArray(); foreach (var property in jsArray.Properties) { if (property.Key == "length") { continue; } var jsInstance = property.Value.Value; if (!jsInstance.HasValue) { continue; } var ravenJToken = ToRavenJToken(jsInstance.Value, CreatePropertyKey(property.Key, propertyKey), recursiveCall); if (ravenJToken == null) { continue; } rja.Add(ravenJToken); } return(rja); } if (v.IsObject()) { return(ToRavenJObject(v, propertyKey, recursiveCall)); } if (v.IsRegExp()) { return(null); } throw new NotSupportedException(v.Type.ToString()); }
public RavenJArray GetDocuments(int start, int pageSize, Guid? etag) { var list = new RavenJArray(); TransactionalStorage.Batch(actions => { while (true) { var documents = etag == null ? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) : actions.Documents.GetDocumentsAfter(etag.Value, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); int docCount = 0; foreach (var doc in documents) { docCount++; if(etag != null) etag = doc.Etag; DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; list.Add(document.ToJson()); } if (list.Length != 0 || docCount == 0) break; start += docCount; } }); return list; }
private static RavenJToken ToRavenJToken(JsInstance v) { switch (v.Class) { case JsInstance.TYPE_OBJECT: case JsInstance.CLASS_OBJECT: return(ToRavenJObject((JsObject)v)); case JsInstance.CLASS_DATE: var dt = (DateTime)v.Value; return(new RavenJValue(dt)); case JsInstance.TYPE_NUMBER: case JsInstance.CLASS_NUMBER: var num = (double)v.Value; var integer = Math.Truncate(num); if (Math.Abs(num - integer) < double.Epsilon) { return(new RavenJValue((long)integer)); } return(new RavenJValue(num)); case JsInstance.TYPE_STRING: case JsInstance.TYPE_BOOLEAN: case JsInstance.CLASS_STRING: case JsInstance.CLASS_BOOLEAN: return(new RavenJValue(v.Value)); case JsInstance.CLASS_NULL: case JsInstance.TYPE_NULL: return(RavenJValue.Null); case JsInstance.CLASS_UNDEFINED: case JsInstance.TYPE_UNDEFINED: return(RavenJValue.Null); case JsInstance.CLASS_ARRAY: var jsArray = ((JsArray)v); var rja = new RavenJArray(); for (int i = 0; i < jsArray.Length; i++) { var jsInstance = jsArray.get(i); var ravenJToken = ToRavenJToken(jsInstance); if (ravenJToken == null) { continue; } rja.Add(ravenJToken); } return(rja); case JsInstance.CLASS_REGEXP: case JsInstance.CLASS_ERROR: case JsInstance.CLASS_ARGUMENTS: case JsInstance.CLASS_DESCRIPTOR: case JsInstance.CLASS_FUNCTION: return(null); default: throw new NotSupportedException(v.Class); } }
private RavenJArray GetAttachments(int start, Etag etag, int maxRecords) { var array = new RavenJArray(); var attachmentInfos = database.Attachments.GetAttachments(start, maxRecords, etag, null, 1024 * 1024 * 10); foreach (var attachmentInfo in attachmentInfos) { var attachment = database.Attachments.GetStatic(attachmentInfo.Key); if (attachment == null) return null; var data = attachment.Data; attachment.Data = () => { var memoryStream = new MemoryStream(); database.TransactionalStorage.Batch(accessor => data().CopyTo(memoryStream)); memoryStream.Position = 0; return memoryStream; }; var bytes = attachment.Data().ReadData(); array.Add( new RavenJObject { {"Data", bytes}, {"Metadata", attachmentInfo.Metadata}, {"Key", attachmentInfo.Key}, {"Etag", new RavenJValue(attachmentInfo.Etag.ToString())} }); } return array; }
public RavenJArray GetDocumentsAsJson(int start, int pageSize, Etag etag, CancellationToken token) { var list = new RavenJArray(); GetDocuments(start, pageSize, etag, token, doc => { list.Add(doc.ToJson()); return true; }); return list; }
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, BulkOperationOptions options, Func <string, TransactionInformation, object> batchOperation, Action <BulkOperationProgress> reportProgress = null) { options = options ?? new BulkOperationOptions(); var array = new RavenJArray(); var bulkIndexQuery = new IndexQuery { Query = indexQuery.Query, Start = indexQuery.Start, Cutoff = indexQuery.Cutoff ?? SystemTime.UtcNow, WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow, PageSize = int.MaxValue, FieldsToFetch = new[] { Constants.DocumentIdFieldName }, SortedFields = indexQuery.SortedFields, HighlighterPreTags = indexQuery.HighlighterPreTags, HighlighterPostTags = indexQuery.HighlighterPostTags, HighlightedFields = indexQuery.HighlightedFields, SortHints = indexQuery.SortHints, HighlighterKeyName = indexQuery.HighlighterKeyName, TransformerParameters = indexQuery.TransformerParameters, ResultsTransformer = indexQuery.ResultsTransformer }; var operationProgress = new BulkOperationProgress(); bool stale; var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale); operationProgress.TotalEntries = queryResults.Count; if (stale && options.AllowStale == false) { if (options.StaleTimeout != null) { var staleWaitTimeout = Stopwatch.StartNew(); while (stale && staleWaitTimeout.Elapsed < options.StaleTimeout) { queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale); operationProgress.TotalEntries = queryResults.Count; if (stale) { SystemTime.Wait(100); } } } if (stale) { if (options.StaleTimeout != null) { throw new InvalidOperationException("Bulk operation cancelled because the index is stale and StaleTimout of " + options.StaleTimeout + " passed"); } throw new InvalidOperationException("Bulk operation cancelled because the index is stale and allowStale is false"); } } var token = tokenSource.Token; const int batchSize = 1024; int maxOpsPerSec = options.MaxOpsPerSec ?? int.MaxValue; using (var enumerator = queryResults.GetEnumerator()) { var duration = Stopwatch.StartNew(); var operations = 0; while (true) { database.WorkContext.UpdateFoundWork(); if (timeout != null) { timeout.Delay(); } var batchCount = 0; var shouldWaitNow = false; token.ThrowIfCancellationRequested(); using (database.DocumentLock.Lock()) { database.TransactionalStorage.Batch(actions => { while (batchCount < batchSize && enumerator.MoveNext()) { token.ThrowIfCancellationRequested(); batchCount++; operations++; var result = batchOperation(enumerator.Current, transactionInformation); if (options.RetrieveDetails) { array.Add(RavenJObject.FromObject(result)); } operationProgress.ProcessedEntries++; reportProgress?.Invoke(operationProgress); if (operations >= maxOpsPerSec && duration.ElapsedMilliseconds < 1000) { shouldWaitNow = true; break; } } }); } if (shouldWaitNow) { SystemTime.Wait(500); operations = 0; duration.Restart(); continue; } if (batchCount < batchSize) { break; } } } return(array); }
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize, CancellationToken token, ref int nextStart, string transformer = null, Dictionary<string, RavenJToken> transformerParameters = null, string skipAfter = null) { var list = new RavenJArray(); GetDocumentsWithIdStartingWith(idPrefix, matches, exclude, start, pageSize, token, ref nextStart, doc => list.Add(doc.ToJson()), transformer, transformerParameters, skipAfter); return list; }
private void Initialize() { if (mappedTypes != null) return; lock (padlock) { if (mappedTypes != null) return; var collectionData = new CollectionData(); var jsonDoc = store.DatabaseCommands.Get(collectionNamesDocId); if (jsonDoc != null) { var collectionNames = jsonDoc.DataAsJson["Collections"] as RavenJArray; foreach (RavenJValue value in collectionNames) { collectionData.Collections.Add(value.Value as string); } } if (timeoutsEnabled) { MapTypeToCollectionName(typeof(TimeoutPersisters.RavenDB.TimeoutData), collectionData); } if (sagasEnabled) { foreach (var sagaType in types.Where(IsSagaEntity)) { MapTypeToCollectionName(sagaType, collectionData); } } if (collectionData.Changed) { var newDoc = new RavenJObject(); var list = new RavenJArray(); foreach (var name in collectionData.Collections) { list.Add(new RavenJValue(name)); } newDoc["EndpointName"] = this.endpointName; newDoc["EndpointName"] = this.endpointName; newDoc["Collections"] = list; var metadata = new RavenJObject(); store.DatabaseCommands.Put(collectionNamesDocId, null, newDoc, metadata); } // Completes initialization this.mappedTypes = collectionData.Mappings; } }
private bool TryHandleArrayValue(int index, Dictionary <string, object> result, KeyValuePair <string, RavenJToken> prop) { var arrays = new List <RavenJArray> { (RavenJArray)prop.Value }; for (var i = 0; i < docs.Length; i++) { if (i == index) { continue; } RavenJToken token; if (docs[i].TryGetValue(prop.Key, out token) && token.Type != JTokenType.Array) { return(false); } if (token == null) { continue; } if (token.IsSnapshot) { token = token.CreateSnapshot(); } arrays.Add((RavenJArray)token); } var mergedArray = new RavenJArray(); while (arrays.Count > 0) { var set = new HashSet <RavenJToken>(RavenJTokenEqualityComparer.Default); for (var i = 0; i < arrays.Count; i++) { if (arrays[i].Length == 0) { arrays.RemoveAt(i); i -= 1; continue; } set.Add(arrays[i][0]); arrays[i].RemoveAt(0); } foreach (var ravenJToken in set) { mergedArray.Add(ravenJToken); } } if (RavenJTokenEqualityComparer.Default.Equals(mergedArray, prop.Value)) { result.Add(prop.Key, mergedArray); return(true); } result.Add(prop.Key, new ArrayWithWarning(mergedArray)); return(true); }
private RavenJToken ToRavenJToken(JsInstance v) { switch (v.Class) { case JsInstance.TYPE_OBJECT: case JsInstance.CLASS_OBJECT: return ToRavenJObject((JsObject) v); case JsInstance.CLASS_DATE: var dt = (DateTime) v.Value; return new RavenJValue(dt); case JsInstance.TYPE_NUMBER: case JsInstance.CLASS_NUMBER: var num = (double) v.Value; KeyValuePair<RavenJValue, object> property; if (propertiesByValue.TryGetValue(v, out property)) { var originalValue = property.Key; if (originalValue.Type == JTokenType.Float) return new RavenJValue(num); if (originalValue.Type == JTokenType.Integer) { // If the current value is exactly as the original value, we can return the original value before we made the JS conversion, // which will convert a Int64 to jsFloat. var originalJsValue = property.Value; if (originalJsValue is double && Math.Abs(num - (double)originalJsValue) < double.Epsilon) return originalValue; return new RavenJValue((long) num); } } // If we don't have the type, assume that if the number ending with ".0" it actually an integer. var integer = Math.Truncate(num); if (Math.Abs(num - integer) < double.Epsilon) return new RavenJValue((long) integer); return new RavenJValue(num); case JsInstance.TYPE_STRING: case JsInstance.CLASS_STRING: { const string ravenDataByteArrayToBase64 = "raven-data:byte[];base64,"; var value = v.Value as string; if (value != null && value.StartsWith(ravenDataByteArrayToBase64)) { value = value.Remove(0, ravenDataByteArrayToBase64.Length); var byteArray = Convert.FromBase64String(value); return new RavenJValue(byteArray); } return new RavenJValue(v.Value); } case JsInstance.TYPE_BOOLEAN: case JsInstance.CLASS_BOOLEAN: return new RavenJValue(v.Value); case JsInstance.CLASS_NULL: case JsInstance.TYPE_NULL: return RavenJValue.Null; case JsInstance.CLASS_UNDEFINED: case JsInstance.TYPE_UNDEFINED: return RavenJValue.Null; case JsInstance.CLASS_ARRAY: var jsArray = ((JsArray) v); var rja = new RavenJArray(); for (int i = 0; i < jsArray.Length; i++) { var jsInstance = jsArray.get(i); var ravenJToken = ToRavenJToken(jsInstance); if (ravenJToken == null) continue; rja.Add(ravenJToken); } return rja; case JsInstance.CLASS_REGEXP: case JsInstance.CLASS_ERROR: case JsInstance.CLASS_ARGUMENTS: case JsInstance.CLASS_DESCRIPTOR: case JsInstance.CLASS_FUNCTION: return null; default: throw new NotSupportedException(v.Class); } }
private bool TryHandleArrayValue(int index, Dictionary<string, object> result, KeyValuePair<string, RavenJToken> prop) { var arrays = new List<RavenJArray> { (RavenJArray)prop.Value }; for (var i = 0; i < docs.Length; i++) { if (i == index) continue; RavenJToken token; if (docs[i].TryGetValue(prop.Key, out token) && token.Type != JTokenType.Array) return false; if (token == null) continue; if (token.IsSnapshot) token = token.CreateSnapshot(); arrays.Add((RavenJArray)token); } var mergedArray = new RavenJArray(); while (arrays.Count > 0) { var set = new HashSet<RavenJToken>(RavenJTokenEqualityComparer.Default); for (var i = 0; i < arrays.Count; i++) { if (arrays[i].Length == 0) { arrays.RemoveAt(i); i -= 1; continue; } set.Add(arrays[i][0]); arrays[i].RemoveAt(0); } foreach (var ravenJToken in set) { mergedArray.Add(ravenJToken); } } if (RavenJTokenEqualityComparer.Default.Equals(mergedArray, prop.Value)) { result.Add(prop.Key, mergedArray); return true; } result.Add(prop.Key, new ArrayWithWarning(mergedArray)); return true; }
public RavenJArray GetDocuments(int start, int pageSize, Guid? etag) { var list = new RavenJArray(); TransactionalStorage.Batch(actions => { IEnumerable<JsonDocument> documents; if (etag == null) documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start); else documents = actions.Documents.GetDocumentsAfter(etag.Value); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents.Take(pageSize)) { DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; list.Add(document.ToJson()); } }); return list; }
public async Task<RavenJArray> GetTransformers(RavenConnectionStringOptions src, int start) { if (isTransformersSupported() == false) return new RavenJArray(); var transformers = await Store.AsyncDatabaseCommands.GetTransformersAsync(start, Options.BatchSize); var result = new RavenJArray(); foreach (var transformer in transformers) { result.Add(new RavenJObject { { "name", transformer.Name }, { "definition", RavenJObject.FromObject(transformer) } }); } return result; }
private static void FlushBatch(string instanceUrl, List<RavenJObject> batch) { var sw = Stopwatch.StartNew(); long size; using (var webClient = new WebClient()) { webClient.Headers.Add("Content-Type", "application/json; charset=utf-8"); webClient.UseDefaultCredentials = true; webClient.Credentials = CredentialCache.DefaultNetworkCredentials; using (var stream = new MemoryStream()) { using (var streamWriter = new StreamWriter(stream, Encoding.UTF8)) using (var jsonTextWriter = new JsonTextWriter(streamWriter)) { var commands = new RavenJArray(); foreach (var doc in batch) { var metadata = doc.Value<RavenJObject>("@metadata"); doc.Remove("@metadata"); commands.Add(new RavenJObject { {"Method", "PUT"}, {"Document", doc}, {"Metadata", metadata}, {"Key", metadata.Value<string>("@id")} }); } commands.WriteTo(jsonTextWriter); jsonTextWriter.Flush(); streamWriter.Flush(); stream.Flush(); size = stream.Length; using (var netStream = webClient.OpenWrite(instanceUrl + "bulk_docs", "POST")) { stream.WriteTo(netStream); netStream.Flush(); } } } } Console.WriteLine("Wrote {0} documents [{1:#,#} kb] in {2:#,#} ms", batch.Count, Math.Round((double)size / 1024, 2), sw.ElapsedMilliseconds); batch.Clear(); }
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, BulkOperationOptions options, Func<string, TransactionInformation, object> batchOperation) { options = options ?? new BulkOperationOptions(); var array = new RavenJArray(); var bulkIndexQuery = new IndexQuery { Query = indexQuery.Query, Start = indexQuery.Start, Cutoff = indexQuery.Cutoff ?? SystemTime.UtcNow, WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow, PageSize = int.MaxValue, FieldsToFetch = new[] { Constants.DocumentIdFieldName }, SortedFields = indexQuery.SortedFields, HighlighterPreTags = indexQuery.HighlighterPreTags, HighlighterPostTags = indexQuery.HighlighterPostTags, HighlightedFields = indexQuery.HighlightedFields, SortHints = indexQuery.SortHints }; bool stale; var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale); if (stale && options.AllowStale == false) { if (options.StaleTimeout != null) { var staleWaitTimeout = Stopwatch.StartNew(); while (stale && staleWaitTimeout.Elapsed < options.StaleTimeout) { queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale); if(stale) SystemTime.Wait(100); } } if (stale) { if (options.StaleTimeout != null) throw new InvalidOperationException("Bulk operation cancelled because the index is stale and StaleTimout of " + options.StaleTimeout + "passed"); throw new InvalidOperationException("Bulk operation cancelled because the index is stale and allowStale is false"); } } var token = tokenSource.Token; const int batchSize = 1024; int maxOpsPerSec = options.MaxOpsPerSec ?? int.MaxValue; using (var enumerator = queryResults.GetEnumerator()) { var duration = Stopwatch.StartNew(); var operations = 0; while (true) { database.WorkContext.UpdateFoundWork(); if (timeout != null) timeout.Delay(); var batchCount = 0; var shouldWaitNow = false; token.ThrowIfCancellationRequested(); using (database.DocumentLock.Lock()) { database.TransactionalStorage.Batch(actions => { while (batchCount < batchSize && enumerator.MoveNext()) { batchCount++; operations++; var result = batchOperation(enumerator.Current, transactionInformation); if(options.RetrieveDetails) array.Add(RavenJObject.FromObject(result)); if (operations >= maxOpsPerSec && duration.ElapsedMilliseconds < 1000) { shouldWaitNow = true; break; } } }); if (shouldWaitNow) { SystemTime.Wait(500); operations = 0; duration.Restart(); continue; } } if (batchCount < batchSize) break; } } return array; }
public async Task<RavenJArray> GetIndexes(RavenConnectionStringOptions src, int totalCount) { var indexes = await Store.AsyncDatabaseCommands.GetIndexesAsync(totalCount, Options.BatchSize); var result = new RavenJArray(); foreach (var index in indexes) { result.Add(new RavenJObject { { "name", index.Name }, { "definition", RavenJObject.FromObject(index) } }); } return (RavenJArray)RavenJToken.FromObject(result); }
private RavenJArray GetAttachments(int start) { var array = new RavenJArray(); var attachmentInfos = _documentDatabase.GetAttachments(start, 128, null,null,50000); foreach (var attachmentInfo in attachmentInfos) { var attachment = _store.DatabaseCommands.GetAttachment(attachmentInfo.Key); var bytes = StreamToBytes(attachment.Data()); // Based on Raven.Smuggler.Api.ExportAttachments from build 888. array.Add( new RavenJObject { {"Data", bytes}, {"Metadata", attachmentInfo.Metadata}, {"Key", attachmentInfo.Key} }); } return array; }
private RavenJToken ToRavenJToken(JsInstance v) { switch (v.Class) { case JsInstance.TYPE_OBJECT: case JsInstance.CLASS_OBJECT: return ToRavenJObject((JsObject)v); case JsInstance.CLASS_DATE: var dt = (DateTime)v.Value; return new RavenJValue(dt); case JsInstance.TYPE_NUMBER: case JsInstance.CLASS_NUMBER: var num = (double)v.Value; var integer = Math.Truncate(num); if (Math.Abs(num - integer) < double.Epsilon) return new RavenJValue((long)integer); return new RavenJValue(num); case JsInstance.TYPE_STRING: case JsInstance.TYPE_BOOLEAN: case JsInstance.CLASS_STRING: case JsInstance.CLASS_BOOLEAN: return new RavenJValue(v.Value); case JsInstance.CLASS_NULL: case JsInstance.TYPE_NULL: return RavenJValue.Null; case JsInstance.CLASS_UNDEFINED: case JsInstance.TYPE_UNDEFINED: return RavenJValue.Null; case JsInstance.CLASS_ARRAY: var jsArray = ((JsArray)v); var rja = new RavenJArray(); for (int i = 0; i < jsArray.Length; i++) { var jsInstance = jsArray.get(i); var ravenJToken = ToRavenJToken(jsInstance); if (ravenJToken == null) continue; rja.Add(ravenJToken); } return rja; case JsInstance.CLASS_REGEXP: case JsInstance.CLASS_ERROR: case JsInstance.CLASS_ARGUMENTS: case JsInstance.CLASS_DESCRIPTOR: case JsInstance.CLASS_FUNCTION: return null; default: throw new NotSupportedException(v.Class); } }
private void ReplicateDelete(string id, RavenJObject metadata, TExternal incoming) { TInternal existingItem; Guid existingEtag; var existingMetadata = TryGetExisting(id, out existingItem, out existingEtag); if (existingMetadata == null) { log.Debug("Replicating deleted item {0} from {1} that does not exist, ignoring", id, Src); return; } if (existingMetadata.Value <bool>(Constants.RavenReplicationConflict)) // locally conflicted { log.Debug("Replicating deleted item {0} from {1} that is already conflicted, adding to conflicts.", id, Src); var savedConflictedItemId = SaveConflictedItem(id, metadata, incoming, existingEtag); AppendToCurrentItemConflicts(id, savedConflictedItemId, existingMetadata, existingItem); return; } if (existingMetadata.Value <bool>(Constants.RavenDeleteMarker)) //deleted locally as well { log.Debug("Replicating deleted item {0} from {1} that was deleted locally. Merging histories", id, Src); var existingHistory = new RavenJArray(existingMetadata.Value <RavenJArray>(Constants.RavenReplicationHistory)); var newHistory = new RavenJArray(metadata.Value <RavenJArray>(Constants.RavenReplicationHistory)); foreach (var item in newHistory) { existingHistory.Add(item); } if (metadata.ContainsKey(Constants.RavenReplicationVersion) && metadata.ContainsKey(Constants.RavenReplicationSource)) { existingHistory.Add(new RavenJObject { { Constants.RavenReplicationVersion, metadata[Constants.RavenReplicationVersion] }, { Constants.RavenReplicationSource, metadata[Constants.RavenReplicationSource] } }); } while (existingHistory.Length > Constants.ChangeHistoryLength) { existingHistory.RemoveAt(0); } MarkAsDeleted(id, metadata); return; } if (Historian.IsDirectChildOfCurrent(metadata, existingMetadata)) // not modified { log.Debug("Delete of existing item {0} was replicated successfully from {1}", id, Src); DeleteItem(id, existingEtag); MarkAsDeleted(id, metadata); return; } Database.TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() => Database.RaiseNotifications(new DocumentChangeNotification { Id = id, Type = DocumentChangeTypes.ReplicationConflict })); var newConflictId = SaveConflictedItem(id, metadata, incoming, existingEtag); log.Debug("Existing item {0} is in conflict with replicated delete from {1}, marking item as conflicted", id, Src); // we have a new conflict move the existing doc to a conflict and create a conflict document var existingDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(existingEtag); CreateConflict(id, newConflictId, existingDocumentConflictId, existingItem, existingMetadata); }
private RavenJToken ToRavenJToken(JsValue v, string propertyKey, bool recursiveCall) { if (v.IsBoolean()) { return(new RavenJValue(v.AsBoolean())); } if (v.IsString()) { const string RavenDataByteArrayToBase64 = "raven-data:byte[];base64,"; var valueAsObject = v.ToObject(); var value = valueAsObject != null?valueAsObject.ToString() : null; if (value != null && value.StartsWith(RavenDataByteArrayToBase64)) { value = value.Remove(0, RavenDataByteArrayToBase64.Length); var byteArray = Convert.FromBase64String(value); return(new RavenJValue(byteArray)); } return(new RavenJValue(value)); } if (v.IsNumber()) { var num = v.AsNumber(); KeyValuePair <RavenJValue, JsValue> property; if (propertiesByValue.TryGetValue(propertyKey, out property)) { var originalValue = property.Key; if (originalValue.Type == JTokenType.Float || originalValue.Type == JTokenType.Integer) { // If the current value is exactly as the original value, we can return the original value before we made the JS conversion, // which will convert a Int64 to jsFloat. var originalJsValue = property.Value; if (originalJsValue.IsNumber() && Math.Abs(num - originalJsValue.AsNumber()) < double.Epsilon) { return(originalValue); } //We might have change the type of num from Integer to long in the script by design //Making sure the number isn't a real float before returning it as integer if (originalValue.Type == JTokenType.Integer && (Math.Abs(num - Math.Floor(num)) <= double.Epsilon || Math.Abs(num - Math.Ceiling(num)) <= double.Epsilon)) { return(new RavenJValue((long)num)); } return(new RavenJValue(num));//float } } // If we don't have the type, assume that if the number ending with ".0" it actually an integer. var integer = Math.Truncate(num); if (Math.Abs(num - integer) < double.Epsilon) { return(new RavenJValue((long)integer)); } return(new RavenJValue(num)); } if (v.IsNull()) { return(RavenJValue.Null); } if (v.IsUndefined()) { return(RavenJValue.Null); } if (v.IsArray()) { var jsArray = v.AsArray(); var rja = new RavenJArray(); foreach (var property in jsArray.GetOwnProperties()) { if (InheritedProperties.Contains(property.Key)) { continue; } var jsInstance = property.Value.Value; if (!jsInstance.HasValue) { continue; } var ravenJToken = ToRavenJToken(jsInstance.Value, propertyKey + "[" + property.Key + "]", recursiveCall); if (ravenJToken == null) { continue; } rja.Add(ravenJToken); } return(rja); } if (v.IsDate()) { return(new RavenJValue(v.AsDate().ToDateTime())); } if (v.IsObject()) { return(ToRavenJObject(v, propertyKey, recursiveCall)); } if (v.IsRegExp()) { return(null); } throw new NotSupportedException(v.Type.ToString()); }
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, string matches, int start, int pageSize) { if (idPrefix == null) throw new ArgumentNullException("idPrefix"); idPrefix = idPrefix.Trim(); var list = new RavenJArray(); TransactionalStorage.Batch(actions => { while (true) { int docCount = 0; var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents) { docCount++; if (WildcardMatcher.Matches(matches, doc.Key.Substring(idPrefix.Length)) == false) continue; DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; list.Add(document.ToJson()); } if (list.Length != 0 || docCount == 0) break; start += docCount; } }); return list; }
protected override Guid FlushBatch(List<RavenJObject> batch) { var sw = Stopwatch.StartNew(); var commands = new RavenJArray(); foreach (var doc in batch) { var metadata = doc.Value<RavenJObject>("@metadata"); doc.Remove("@metadata"); commands.Add(new RavenJObject { {"Method", "PUT"}, {"Document", doc}, {"Metadata", metadata}, {"Key", metadata.Value<string>("@id")} }); } var retries = retriesCount; HttpRavenRequest request = null; BatchResult[] results; while (true) { try { request = CreateRequest("/bulk_docs", "POST"); request.Write(commands); results = request.ExecuteRequest<BatchResult[]>(); sw.Stop(); break; } catch (Exception e) { if (--retries == 0 || request == null) throw; sw.Stop(); LastRequestErrored = true; ShowProgress("Error flushing to database, remaining attempts {0} - time {2:#,#} ms, will retry [{3:#,#.##;;0} kb compressed to {4:#,#.##;;0} kb]. Error: {1}", retriesCount - retries, e, sw.ElapsedMilliseconds, (double)request.NumberOfBytesWrittenUncompressed / 1024, (double)request.NumberOfBytesWrittenCompressed / 1024); } } total += batch.Count; ShowProgress("{2,5:#,#}: Wrote {0:#,#;;0} in {1,6:#,#;;0} ms ({6:0.00} ms per doc) (total of {3:#,#;;0}) documents [{4:#,#.##;;0} kb compressed to {5:#,#.##;;0} kb]", batch.Count, sw.ElapsedMilliseconds, ++count, total, (double)request.NumberOfBytesWrittenUncompressed / 1024, (double)request.NumberOfBytesWrittenCompressed / 1024, Math.Round((double)sw.ElapsedMilliseconds / Math.Max(1, batch.Count), 2)); batch.Clear(); if (results.Length == 0) return Guid.Empty; return results.Last().Etag.Value; }
public override void OnPut(string key, RavenJObject jsonReplicationDocument, RavenJObject metadata) { using (Database.DisableAllTriggersForCurrentThread()) { if (metadata.Remove(Constants.RavenReplicationConflictSkipResolution)) { if (key.IndexOf("/conflicts/", StringComparison.OrdinalIgnoreCase) == -1) { metadata["@Http-Status-Code"] = 409; metadata["@Http-Status-Description"] = "Conflict"; } return; } metadata.Remove(Constants.RavenReplicationConflict);// you can't put conflicts var oldVersion = Database.Documents.Get(key); if (oldVersion == null) { return; } if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null) { return; } var history = new RavenJArray(); metadata[Constants.RavenReplicationHistory] = history; // this is a conflict document, holding document keys in the // values of the properties var conflicts = oldVersion.DataAsJson.Value <RavenJArray>("Conflicts"); if (conflicts == null) { return; } var list = new List <RavenJArray> { new RavenJArray(ReplicationData.GetHistory(metadata)) // first item to interleave }; foreach (var prop in conflicts) { RavenJObject deletedMetadata; Database.Documents.Delete(prop.Value <string>(), null, out deletedMetadata); if (deletedMetadata != null) { var conflictHistory = new RavenJArray(ReplicationData.GetHistory(deletedMetadata)); conflictHistory.Add(new RavenJObject { { Constants.RavenReplicationVersion, deletedMetadata[Constants.RavenReplicationVersion] }, { Constants.RavenReplicationSource, deletedMetadata[Constants.RavenReplicationSource] } }); list.Add(conflictHistory); } } int index = 0; bool added = true; while (added) // interleave the history from all conflicts { added = false; foreach (var deletedMetadata in list) { // add the conflict history to the mix, so we make sure that we mark that we resolved the conflict if (index < deletedMetadata.Length) { history.Add(deletedMetadata[index]); added = true; } } index++; } while (history.Length > Constants.ChangeHistoryLength) { history.RemoveAt(0); } } }