protected void RecordActionForDebug(string actionName, string documentKey, RavenJObject documentData, RavenJObject documentMetadata)
        {
            if (debugMode == false)
                return;

            var actions = DebugActions[actionName] as RavenJArray;

            switch (actionName)
            {
                case "LoadDocument":
                case "DeleteDocument":
                    if (actions == null)
                        DebugActions[actionName] = actions = new RavenJArray();

                    actions.Add(documentKey);
                    break;
                case "PutDocument":
                    if (actions == null)
                        DebugActions[actionName] = actions = new RavenJArray();

                    actions.Add(new RavenJObject
                                {
                                    { "Key", documentKey }, 
                                    { "Data", documentData }, 
                                    { "Metadata", documentMetadata }
                                });
                    break;
                default:
                    throw new NotSupportedException("Cannot record action: " + actionName);
            }
        }
Ejemplo n.º 2
0
		private static void ModifyResult(RavenJObject result)
		{
			var tags = result.Value<string>("Tags");
			if (tags != null)
			{
				result["Tags"] =
					new RavenJArray(tags.Split(new[] {' ', ',', ';'}, StringSplitOptions.RemoveEmptyEntries));
			}
			else
			{
				result["Tags"] = new RavenJArray();
			}
			var deps = result.Value<string>("Dependencies");
			if (deps != null)
			{
				result["Dependencies"] =
					new RavenJArray(deps.Split(new[] {'|'}, StringSplitOptions.RemoveEmptyEntries)
						                .Select(s =>
							                {
								                var strings = s.Split(':');
								                return RavenJObject.FromObject(new {Package = strings[0], Version = strings[1]});
							                }));
			}
			result["PackageId"] = result["Id"];
			result.Remove("Id");
			result.Remove("__metadata");
			result.Remove("DownloadCount");
		}
Ejemplo n.º 3
0
		public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation)
		{
			if (key.StartsWith("Raven/", StringComparison.OrdinalIgnoreCase) && // we don't deal with system documents
				key.StartsWith("Raven/Hilo/", StringComparison.OrdinalIgnoreCase) == false) // except for hilos
				return;
			using (Database.DisableAllTriggersForCurrentThread())
			{
				var documentMetadata = GetDocumentMetadata(key);
				if (documentMetadata != null)
				{
					RavenJArray history = new RavenJArray(ReplicationData.GetHistory(documentMetadata));
					metadata[Constants.RavenReplicationHistory] = history;

					if (documentMetadata.ContainsKey(Constants.RavenReplicationVersion) && 
						documentMetadata.ContainsKey(Constants.RavenReplicationSource))
					{
						history.Add(new RavenJObject
						{
							{Constants.RavenReplicationVersion, documentMetadata[Constants.RavenReplicationVersion]},
							{Constants.RavenReplicationSource, documentMetadata[Constants.RavenReplicationSource]}
						});
					}

					while (history.Length > Constants.ChangeHistoryLength)
					{
						history.RemoveAt(0);
					}
				}

				metadata[Constants.RavenReplicationVersion] = RavenJToken.FromObject(HiLo.NextId());
				metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(Database.TransactionalStorage.Id);
			}
		}
Ejemplo n.º 4
0
        /// <summary>
        /// Adds the cascade delete reference.
        /// </summary>
        /// <param name="session">The session.</param>
        /// <param name="entity">The entity.</param>
        /// <param name="documentKeys">The document keys.</param>
        public static void AddCascadeDeleteReference(this IAdvancedDocumentSessionOperations session, object entity, params string[] documentKeys)
        {
            var metadata = session.GetMetadataFor(entity);

            if(metadata == null)
            {
                throw new InvalidOperationException("The entity must be tracked in the session before calling this method.");
            }

            if(documentKeys.Length == 0)
            {
                throw new ArgumentException("At least one document key must be specified.");
            }

            const string metadataKey = "Raven-Cascade-Delete-Documents";
            RavenJToken token;

            if(!metadata.TryGetValue(metadataKey, out token))
            {
                token = new RavenJArray();
            }

            var list = (RavenJArray)token;
            foreach(var documentKey in documentKeys.Where(key => !list.Contains(key)))
            {
                list.Add(documentKey);
            }

            metadata[metadataKey] = list;
        }
Ejemplo n.º 5
0
		public override void Respond(IHttpContext context)
		{
			RavenJArray itemsToLoad;
			if(context.Request.HttpMethod == "POST")
				itemsToLoad = context.ReadJsonArray();
			else
				itemsToLoad = new RavenJArray(context.Request.QueryString.GetValues("id"));
			var result = new MultiLoadResult();
			var loadedIds = new HashSet<string>();
			var includes = context.Request.QueryString.GetValues("include") ?? new string[0];
			var transactionInformation = GetRequestTransaction(context);
		    var includedEtags = new List<byte>();
			Database.TransactionalStorage.Batch(actions =>
			{
				foreach (RavenJToken item in itemsToLoad)
				{
					var value = item.Value<string>();
					if(loadedIds.Add(value)==false)
						continue;
					var documentByKey = Database.Get(value, transactionInformation);
					if (documentByKey == null)
						continue;
					result.Results.Add(documentByKey.ToJson());

					if (documentByKey.Etag != null)
					{
						includedEtags.AddRange(documentByKey.Etag.Value.ToByteArray());
					}
					includedEtags.Add((documentByKey.NonAuthoritativeInformation ?? false) ? (byte)0 : (byte)1);
				}

				var addIncludesCommand = new AddIncludesCommand(Database, transactionInformation, (etag, includedDoc) =>
				{
					includedEtags.AddRange(etag.ToByteArray());
					result.Includes.Add(includedDoc);
				}, includes, loadedIds);

				foreach (var item in result.Results.Where(item => item != null))
				{
					addIncludesCommand.Execute(item);
				}
			});

			Guid computedEtag;

			using (var md5 = MD5.Create())
			{
				var computeHash = md5.ComputeHash(includedEtags.ToArray());
				computedEtag = new Guid(computeHash);
			}

			if (context.MatchEtag(computedEtag))
			{
				context.SetStatusToNotModified();
				return;
			}

			context.WriteETag(computedEtag);
			context.WriteJson(result);
		}
Ejemplo n.º 6
0
		public override void OnPut(string key, Stream data, RavenJObject metadata)
		{
			if (key.StartsWith("Raven/")) // we don't deal with system attachment
				return;
			using (Database.DisableAllTriggersForCurrentThread())
			{
				var attachmentMetadata = GetAttachmentMetadata(key);
				if (attachmentMetadata != null)
				{
					RavenJArray history = new RavenJArray(metadata.Value<RavenJArray>(Constants.RavenReplicationHistory));
					metadata[Constants.RavenReplicationHistory] = history;

					if (attachmentMetadata.ContainsKey(Constants.RavenReplicationVersion) &&
						attachmentMetadata.ContainsKey(Constants.RavenReplicationSource))
					{
						history.Add(new RavenJObject
						{
							{Constants.RavenReplicationVersion, attachmentMetadata[Constants.RavenReplicationVersion]},
							{Constants.RavenReplicationSource, attachmentMetadata[Constants.RavenReplicationSource]}
						});
					}

					if (history.Length > Constants.ChangeHistoryLength)
					{
						history.RemoveAt(0);
					}
				}

				metadata[Constants.RavenReplicationVersion] = RavenJToken.FromObject(HiLo.NextId());
				metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(Database.TransactionalStorage.Id);
			}
		}
		public ReplicationTopologyDiscoverer(DocumentDatabase database, RavenJArray @from, int ttl, ILog log)
		{
			this.database = database;
			this.ttl = ttl;
			this.log = log;
			this.@from = @from;
			requestFactory = new HttpRavenRequestFactory();
		}
Ejemplo n.º 8
0
        private bool TryHandleArrayValue(int index, Dictionary<string, object> result, KeyValuePair<string, RavenJToken> prop)
        {
            var arrays = new List<RavenJArray>
            {
                (RavenJArray)prop.Value
            };

            for (var i = 0; i < docs.Length; i++)
            {
                if (i == index)
                    continue;

                RavenJToken token;
                if (docs[i].TryGetValue(prop.Key, out token) && token.Type != JTokenType.Array)
                    return false;
                if (token == null)
                    continue;
                if (token.IsSnapshot)
                    token = token.CreateSnapshot();
                arrays.Add((RavenJArray)token);
            }

            var mergedArray = new RavenJArray();
            while (arrays.Count > 0)
            {
                var set = new HashSet<RavenJToken>(RavenJTokenEqualityComparer.Default);
                for (var i = 0; i < arrays.Count; i++)
                {
                    if (arrays[i].Length == 0)
                    {
                        arrays.RemoveAt(i);
                        i -= 1;
                        continue;
                    }
                    set.Add(arrays[i][0]);
                    arrays[i].RemoveAt(0);
                }

                foreach (var ravenJToken in set)
                {
                    mergedArray.Add(ravenJToken);
                }
            }

            if (RavenJTokenEqualityComparer.Default.Equals(mergedArray, prop.Value))
            {
                result.Add(prop.Key, mergedArray);
                return true;
            }

            result.Add(prop.Key, new ArrayWithWarning(mergedArray));
            return true;
        }
        private void Initialize()
        {
            if (mappedTypes != null)
                return;

            lock (padlock)
            {
                if (mappedTypes != null)
                    return;

                var collectionData = new CollectionData();

                var jsonDoc = store.DatabaseCommands.Get(collectionNamesDocId);
                if (jsonDoc != null)
                {
                    var collectionNames = jsonDoc.DataAsJson["Collections"] as RavenJArray;
                    foreach (RavenJValue value in collectionNames)
                    {
                        collectionData.Collections.Add(value.Value as string);
                    }
                }

                if (timeoutsEnabled)
                {
                    MapTypeToCollectionName(typeof(TimeoutPersisters.RavenDB.TimeoutData), collectionData);
                }
                if (sagasEnabled)
                {
                    foreach (var sagaType in types.Where(IsSagaEntity))
                    {
                        MapTypeToCollectionName(sagaType, collectionData);
                    }
                }

                if (collectionData.Changed)
                {
                    var newDoc = new RavenJObject();
                    var list = new RavenJArray();
                    foreach (var name in collectionData.Collections)
                    {
                        list.Add(new RavenJValue(name));
                    }
                    newDoc["EndpointName"] = this.endpointName;
                    newDoc["EndpointName"] = this.endpointName;
                    newDoc["Collections"] = list;
                    var metadata = new RavenJObject();
                    store.DatabaseCommands.Put(collectionNamesDocId, null, newDoc, metadata);
                }

                // Completes initialization
                this.mappedTypes = collectionData.Mappings;
            }
        }
Ejemplo n.º 10
0
		private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, bool allowStale, Func<string, TransactionInformation, object> batchOperation)
		{
			var array = new RavenJArray();
			var bulkIndexQuery = new IndexQuery
			{
				Query = indexQuery.Query,
				Start = indexQuery.Start,
				Cutoff = indexQuery.Cutoff,
                WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow,
				PageSize = int.MaxValue,
				FieldsToFetch = new[] { Constants.DocumentIdFieldName },
				SortedFields = indexQuery.SortedFields,
				HighlighterPreTags = indexQuery.HighlighterPreTags,
				HighlighterPostTags = indexQuery.HighlighterPostTags,
				HighlightedFields = indexQuery.HighlightedFields,
				SortHints = indexQuery.SortHints
			};

			bool stale;
            var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);

			if (stale && allowStale == false)
			{
				throw new InvalidOperationException(
						"Bulk operation cancelled because the index is stale and allowStale is false");
			}

		    var token = tokenSource.Token;

			const int batchSize = 1024;
			using (var enumerator = queryResults.GetEnumerator())
			{
				while (true)
				{
					if (timeout != null)
						timeout.Delay();
					var batchCount = 0;
                    token.ThrowIfCancellationRequested();
					database.TransactionalStorage.Batch(actions =>
					{
						while (batchCount < batchSize && enumerator.MoveNext())
						{
							batchCount++;
							var result = batchOperation(enumerator.Current, transactionInformation);
							array.Add(RavenJObject.FromObject(result));
						}
					});
					if (batchCount < batchSize) break;
				}
			}
			return array;
		}
		public override void OnPut(string key, Stream data, RavenJObject metadata)
		{
			using (Database.DisableAllTriggersForCurrentThread())
			{
				metadata.Remove(Constants.RavenReplicationConflict);// you can't put conflicts

				var oldVersion = Database.Attachments.GetStatic(key);
				if (oldVersion == null)
					return;
				if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null)
					return;

				var history = new RavenJArray(ReplicationData.GetHistory(metadata));
				metadata[Constants.RavenReplicationHistory] = history;

				var ravenJTokenEqualityComparer = new RavenJTokenEqualityComparer();
				// this is a conflict document, holding document keys in the 
				// values of the properties
				var conflictData = oldVersion.Data().ToJObject();
				var conflicts = conflictData.Value<RavenJArray>("Conflicts");
				if (conflicts == null)
					return;
				foreach (var prop in conflicts)
				{
					var id = prop.Value<string>();
					Attachment attachment = Database.Attachments.GetStatic(id);
					if(attachment == null)
						continue;
					Database.Attachments.DeleteStatic(id, null);

					// add the conflict history to the mix, so we make sure that we mark that we resolved the conflict
					var conflictHistory = new RavenJArray(ReplicationData.GetHistory(attachment.Metadata));
					conflictHistory.Add(new RavenJObject
					{
						{Constants.RavenReplicationVersion, attachment.Metadata[Constants.RavenReplicationVersion]},
						{Constants.RavenReplicationSource, attachment.Metadata[Constants.RavenReplicationSource]}
					});

					foreach (var item in conflictHistory)
					{
						if (history.Any(x => ravenJTokenEqualityComparer.Equals(x, item)))
							continue;
						history.Add(item);
					}
				}
			}
		}
Ejemplo n.º 12
0
		public override IEnumerable<Row> Execute(IEnumerable<Row> rows)
		{
			int count = 0;
			foreach (var commentsForPosts in rows.Partition(Constants.BatchSize))
			{
				var cmds = new List<ICommandData>();

				foreach (var commentsForPost in commentsForPosts.GroupBy(row => row["PostId"]))
				{
					var comments = new RavenJArray();
					foreach (var row in commentsForPost)
					{
						comments.Add(new RavenJObject
						{
							{"Score", new RavenJValue(row["Score"])},
							{"CreationDate", new RavenJValue(row["CreationDate"])},
							{"Text", new RavenJValue(row["Text"])},
							{"UserId", new RavenJValue(row["UserId"])}
						});
							
					}
					cmds.Add(new PatchCommandData
					{
						Key = "posts/" + commentsForPost.Key,
						Patches = new[]
						{
							new PatchRequest
							{
								Name = "Comments",
								Type = PatchCommandType.Set,
								Value = comments
							},
						}
					});
				}

				count++;

				WriteCommandsTo("Comments #" + count.ToString("00000") + ".json", cmds);
			}

			yield break;
		}
Ejemplo n.º 13
0
		private static void WriteJsonArray(RavenJArray array, StringWriter sw, int margin, int indent = 0)
		{
			sw.WriteLine('[');
			indent += 1;
			var isFirstItem = true;
			foreach (var token in array.Values())
			{
				if (isFirstItem)
					isFirstItem = false;
				else
					sw.WriteLine(',');
				Indent(sw, indent);
				WriteValue(token, sw, margin, indent);
			}
			sw.WriteLine();
			indent -= 1;
			Indent(sw, indent);
			sw.Write(']');
		}
Ejemplo n.º 14
0
		public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation)
		{
			using (Database.DisableAllTriggersForCurrentThread())
			{
				metadata.Remove(Constants.RavenReplicationConflict);// you can't put conflicts

				var oldVersion = Database.Get(key, transactionInformation);
				if (oldVersion == null)
					return;
				if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null)
					return;

				RavenJArray history = new RavenJArray(ReplicationData.GetHistory(metadata));
				metadata[Constants.RavenReplicationHistory] = history;

				var ravenJTokenEqualityComparer = new RavenJTokenEqualityComparer();
				// this is a conflict document, holding document keys in the 
				// values of the properties
				var conflicts = oldVersion.DataAsJson.Value<RavenJArray>("Conflicts");
				if(conflicts == null)
					return;
				foreach (var prop in conflicts)
				{
					RavenJObject deletedMetadata;
					Database.Delete(prop.Value<string>(), null, transactionInformation, out deletedMetadata);

					// add the conflict history to the mix, so we make sure that we mark that we resolved the conflict
					var conflictHistory = new RavenJArray(ReplicationData.GetHistory(deletedMetadata));
					conflictHistory.Add(new RavenJObject
					{
						{Constants.RavenReplicationVersion, deletedMetadata[Constants.RavenReplicationVersion]},
						{Constants.RavenReplicationSource, deletedMetadata[Constants.RavenReplicationSource]}
					});

					foreach (var item in conflictHistory)
					{
						if(history.Any(x=>ravenJTokenEqualityComparer.Equals(x, item)))
							continue;
						history.Add(item);
					}
				}
			}
		}
Ejemplo n.º 15
0
		private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, bool allowStale, Func<string, TransactionInformation, object> batchOperation)
		{
			var array = new RavenJArray();
			var bulkIndexQuery = new IndexQuery
			{
				Query = indexQuery.Query,
				Start = indexQuery.Start,
				Cutoff = indexQuery.Cutoff,
				PageSize = int.MaxValue,
				FieldsToFetch = new[] { Constants.DocumentIdFieldName },
				SortedFields = indexQuery.SortedFields
			};

			bool stale;
			var queryResults = database.QueryDocumentIds(index, bulkIndexQuery, out stale);

			if (stale && allowStale == false)
			{
				throw new InvalidOperationException(
						"Bulk operation cancelled because the index is stale and allowStale is false");
			}

			const int batchSize = 1024;
			using (var enumerator = queryResults.GetEnumerator())
			{
				while (true)
				{
					var batchCount = 0;
					database.TransactionalStorage.Batch(actions =>
					{
						while (batchCount < batchSize && enumerator.MoveNext())
						{
							batchCount++;
							var result = batchOperation(enumerator.Current, transactionInformation);
							array.Add(RavenJObject.FromObject(result, JsonExtensions.CreateDefaultJsonSerializer()));
						}
					});
					if (batchCount < batchSize) break;
				}
			}
			return array;
		}
Ejemplo n.º 16
0
        private static void WriteJsonArray(RavenJArray array, CountingWriter sw, int width, int numberOfLines)
		{
			sw.WriteLine("[");
			sw.PushIndent();

			var isFirstItem = true;
			foreach (var token in array.Values())
			{
                if (sw.LinesWritten >= numberOfLines)
                    break;

				if (isFirstItem)
					isFirstItem = false;
				else
					sw.WriteLine(",");
                WriteValue(token, sw, width, numberOfLines);
			}
			sw.WriteLine("");
			sw.PopIndent();
			sw.Write("]");
		}
Ejemplo n.º 17
0
			public void ForEach(RavenJToken result, RavenJToken item, Action<PathPart, RavenJToken, RavenJToken> action)
			{
				if (string.IsNullOrEmpty(FinalName) == false)
				{
					action(this, item, result);
					return;
				}
				RavenJToken newResult = GetTheNewResultOrWireTheDefault(result);
				if (item == null)
				{
					foreach (var pathPart in Items)
						pathPart.Value.ForEach(newResult, null, action);
					return;
				}
				if (item is RavenJArray == false)
				{
					foreach (var pathPart in Items)
						pathPart.Value.ForEach(newResult, item.SelectToken(pathPart.Key), action);
				}
				else
				{
					var jArray = newResult as RavenJArray;
					if (jArray == null)
					{
						jArray = new RavenJArray();
						((RavenJObject)result)[Name] = jArray;
					}
					foreach (var subItem in item.Values<RavenJToken>())
					{
						newResult = new RavenJObject();
						jArray.Add(newResult);
						foreach (var pathPart in Items)
						{
							pathPart.Value.ForEach(newResult, subItem.SelectToken(pathPart.Key), action);
						}
					}
				}
			}
        public async Task<HttpResponseMessage> ReplicationTopologyDiscover()
        {
            var ttlAsString = GetQueryStringValue("ttl");

            int ttl;
            RavenJArray from;

            if (string.IsNullOrEmpty(ttlAsString))
            {
                ttl = 10;
                from = new RavenJArray();
            }
            else
            {
                ttl = int.Parse(ttlAsString);
                from = await ReadJsonArrayAsync();
            }

            var replicationSchemaDiscoverer = new ReplicationTopologyDiscoverer(Database, from, ttl, Log);
            var node = replicationSchemaDiscoverer.Discover();

            return GetMessageWithObject(node);
        }
Ejemplo n.º 19
0
		public static RavenJToken MinimizeToken(RavenJToken obj, int depth = 0)
		{
			switch (obj.Type)
			{
				case JTokenType.Array:
					var array = new RavenJArray();
					foreach (var item in ((RavenJArray)obj))
					{
						array.Add(MinimizeToken(item, depth + 1));
					}
					return array;
				case JTokenType.Object:
					var ravenJObject = ((RavenJObject)obj);
					if (ravenJObject.ContainsKey(Constants.Metadata) == false)
					{
						// this might be a wrapper object, let check for first level arrays
						if (depth == 0)
						{
							var newRootObj = new RavenJObject();

							foreach (var prop in ravenJObject)
							{
								newRootObj[prop.Key] = prop.Value.Type == JTokenType.Array ?
									MinimizeToken(prop.Value, depth + 1) :
									prop.Value;
							}
							return newRootObj;
						}
						return obj;
					}
					var newObj = new RavenJObject();
					newObj[Constants.Metadata] = ravenJObject[Constants.Metadata];
					return newObj;
				default:
					return obj;
			}
		}
Ejemplo n.º 20
0
		/// <summary>
		/// Filters the headers from unwanted headers
		/// </summary>
		/// <param name="self">The self.</param>
		/// <param name="isServerDocument">if set to <c>true</c> [is server document].</param>
		/// <returns></returns>public static RavenJObject FilterHeaders(this System.Collections.Specialized.NameValueCollection self, bool isServerDocument)
		public static RavenJObject FilterHeaders(this NameValueCollection self, bool isServerDocument)
		{
			var metadata = new RavenJObject(StringComparer.InvariantCultureIgnoreCase);
			foreach (string header in self)
			{
				try
				{
					if(header.StartsWith("Temp"))
						continue;
					if (HeadersToIgnoreClient.Contains(header))
						continue;
					if (isServerDocument && HeadersToIgnoreServerDocument.Contains(header))
						continue;
					var values = self.GetValues(header);
					var headerName = CaptureHeaderName(header);
					// TODO: Can values be null?
					if (values.Length == 1)
						metadata[headerName] = GetValue(values[0]);
					else
						metadata[headerName] = new RavenJArray(values.Select(GetValue));
				}
				catch (Exception exc)
				{
					throw new JsonReaderException(string.Concat("Unable to Filter Header: ", header), exc);
				}
			}
			return metadata;
		}
Ejemplo n.º 21
0
 public static IEnumerable <dynamic> GetDocumentsFromString(string json)
 {
     return(RavenJArray.Parse(json).Cast <RavenJObject>().Select(JsonToExpando.Convert));
 }
        private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, BulkOperationOptions options, Func<string, TransactionInformation, object> batchOperation)
        {
	        options = options ?? new BulkOperationOptions();
			var array = new RavenJArray();
			var bulkIndexQuery = new IndexQuery
			{
				Query = indexQuery.Query,
				Start = indexQuery.Start,
				Cutoff = indexQuery.Cutoff ?? SystemTime.UtcNow,
                WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow,
				PageSize = int.MaxValue,
				FieldsToFetch = new[] { Constants.DocumentIdFieldName },
				SortedFields = indexQuery.SortedFields,
				HighlighterPreTags = indexQuery.HighlighterPreTags,
				HighlighterPostTags = indexQuery.HighlighterPostTags,
				HighlightedFields = indexQuery.HighlightedFields,
				SortHints = indexQuery.SortHints
			};

			bool stale;
            var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);

            if (stale && options.AllowStale == false)
			{
			    if (options.StaleTimeout != null)
			    {
			        var staleWaitTimeout = Stopwatch.StartNew();
			        while (stale && staleWaitTimeout.Elapsed < options.StaleTimeout)
			        {
                        queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);
                        if(stale)
                            SystemTime.Wait(100);
			        }
			    }
			    if (stale)
			    {
				    if (options.StaleTimeout != null)
					    throw new InvalidOperationException("Bulk operation cancelled because the index is stale and StaleTimout  of " + options.StaleTimeout + "passed");
			        
					throw new InvalidOperationException("Bulk operation cancelled because the index is stale and allowStale is false");
			    }
			}

		    var token = tokenSource.Token;		    
			const int batchSize = 1024;
            int maxOpsPerSec = options.MaxOpsPerSec ?? int.MaxValue;
			using (var enumerator = queryResults.GetEnumerator())
			{
			    var duration = Stopwatch.StartNew();
			    var operations = 0;
				while (true)
				{
					database.WorkContext.UpdateFoundWork();
					if (timeout != null)
						timeout.Delay();
					var batchCount = 0;
				    var shouldWaitNow = false;
                    token.ThrowIfCancellationRequested();
					using (database.DocumentLock.Lock())
					{
						database.TransactionalStorage.Batch(actions =>
						{
							while (batchCount < batchSize && enumerator.MoveNext())
							{
								batchCount++;
							    operations++;
								var result = batchOperation(enumerator.Current, transactionInformation);

								if(options.RetrieveDetails)
									array.Add(RavenJObject.FromObject(result));

							    if (operations >= maxOpsPerSec && duration.ElapsedMilliseconds < 1000)
							    {
							        shouldWaitNow = true;
                                    break;
							    }
							}
						});
                        if (shouldWaitNow)
					    {
                            SystemTime.Wait(500);
                            operations = 0;
                            duration.Restart();
						    continue;
					    }
					}
					if (batchCount < batchSize) break;
				}
			}
			return array;
		}
Ejemplo n.º 23
0
        private JsValue ToJsArray(Engine engine, RavenJArray array)
        {
            var elements = new JsValue[array.Length];
	        for (var i = 0; i < array.Length; i++)
	        {
				elements[i] = ToJsInstance(engine, array[i]);
	        }

	        return engine.Array.Construct(elements);
        }
Ejemplo n.º 24
0
        public override void Respond(IHttpContext context)
        {
            RavenJArray itemsToLoad;

            if (context.Request.HttpMethod == "POST")
            {
                itemsToLoad = context.ReadJsonArray();
            }
            else
            {
                itemsToLoad = new RavenJArray(context.Request.QueryString.GetValues("id"));
            }
            var result    = new MultiLoadResult();
            var loadedIds = new HashSet <string>();
            var includes  = context.Request.QueryString.GetValues("include") ?? new string[0];
            var transactionInformation = GetRequestTransaction(context);
            var includedEtags          = new List <byte>();

            Database.TransactionalStorage.Batch(actions =>
            {
                var addIncludesCommand = new AddIncludesCommand(Database, transactionInformation, (etag, includedDoc) =>
                {
                    includedEtags.AddRange(etag.ToByteArray());
                    result.Includes.Add(includedDoc);
                }, includes, loadedIds);
                foreach (RavenJToken item in itemsToLoad)
                {
                    var value = item.Value <string>();
                    if (loadedIds.Add(value) == false)
                    {
                        continue;
                    }
                    var documentByKey = Database.Get(value, transactionInformation);
                    if (documentByKey == null)
                    {
                        continue;
                    }
                    result.Results.Add(documentByKey.ToJson());

                    if (documentByKey.Etag != null)
                    {
                        includedEtags.AddRange(documentByKey.Etag.Value.ToByteArray());
                    }
                    includedEtags.Add((documentByKey.NonAuthoritativeInformation ?? false) ? (byte)0 : (byte)1);
                    addIncludesCommand.Execute(documentByKey.DataAsJson);
                }
            });

            Guid computedEtag;

            using (var md5 = MD5.Create())
            {
                var computeHash = md5.ComputeHash(includedEtags.ToArray());
                computedEtag = new Guid(computeHash);
            }

            if (context.MatchEtag(computedEtag))
            {
                context.SetStatusToNotModified();
                return;
            }

            context.Response.AddHeader("ETag", computedEtag.ToString());
            context.WriteJson(result);
        }
Ejemplo n.º 25
0
        private bool TryReplicationDocuments(ReplicationStrategy destination, RavenJArray jsonDocuments, out string lastError)
        {
            try
            {
                log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination);
                var url = destination.ConnectionStringOptions.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl()
                          + "&dbid=" + docDb.TransactionalStorage.Id +
                          "&count=" + jsonDocuments.Length;

                var sp = Stopwatch.StartNew();

                using (HttpRavenRequestFactory.Expect100Continue(destination.ConnectionStringOptions.Url))
                {
                    var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions, GetRequestBuffering(destination));
                    request.Write(jsonDocuments);
                    request.ExecuteRequest(docDb.WorkContext.CancellationToken);

                    log.Info("Replicated {0} documents to {1} in {2:#,#;;0} ms", jsonDocuments.Length, destination, sp.ElapsedMilliseconds);
                    lastError = "";
                    return true;
                }
            }
            catch (WebException e)
            {
                HandleRequestBufferingErrors(e, destination);

                var response = e.Response as HttpWebResponse;
                if (response != null)
                {
                    var responseStream = response.GetResponseStream();
                    if (responseStream != null)
                    {
                        using (var streamReader = new StreamReader(responseStream))
                        {
                            var error = streamReader.ReadToEnd();
                            log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
                        }
                    }
                    else
                    {
                        log.WarnException("Replication to " + destination + " had failed", e);
                    }
                }
                else
                {
                    log.WarnException("Replication to " + destination + " had failed", e);
                }
                lastError = e.Message;
                return false;
            }
            catch (Exception e)
            {
                log.WarnException("Replication to " + destination + " had failed", e);
                lastError = e.Message;
                return false;
            }
        }
Ejemplo n.º 26
0
 public ArrayWithWarning(RavenJArray mergedArray)
 {
     MergedArray = mergedArray;
 }
Ejemplo n.º 27
0
        protected override Guid FlushBatch(List <RavenJObject> batch)
        {
            var sw = Stopwatch.StartNew();

            var commands = new RavenJArray();

            foreach (var doc in batch)
            {
                var metadata = doc.Value <RavenJObject>("@metadata");

                doc.Remove("@metadata");
                commands.Add(new RavenJObject
                {
                    { "Method", "PUT" },
                    { "Document", doc },
                    { "Metadata", metadata },
                    { "Key", metadata.Value <string>("@id") }
                });
            }

            var retries = retriesCount;
            HttpRavenRequest request = null;

            BatchResult[] results;
            while (true)
            {
                try
                {
                    request = CreateRequest("/bulk_docs", "POST");
                    request.Write(commands);
                    results = request.ExecuteRequest <BatchResult[]>();
                    sw.Stop();
                    break;
                }
                catch (Exception e)
                {
                    if (--retries == 0 || request == null)
                    {
                        throw;
                    }
                    sw.Stop();
                    LastRequestErrored = true;
                    ShowProgress("Error flushing to database, remaining attempts {0} - time {2:#,#} ms, will retry [{3:#,#.##;;0} kb compressed to {4:#,#.##;;0} kb]. Error: {1}",
                                 retriesCount - retries, e, sw.ElapsedMilliseconds,
                                 (double)request.NumberOfBytesWrittenUncompressed / 1024,
                                 (double)request.NumberOfBytesWrittenCompressed / 1024);
                }
            }
            total += batch.Count;
            ShowProgress("{2,5:#,#}: Wrote {0:#,#;;0} in {1,6:#,#;;0} ms ({6:0.00} ms per doc) (total of {3:#,#;;0}) documents [{4:#,#.##;;0} kb compressed to {5:#,#.##;;0} kb]",
                         batch.Count, sw.ElapsedMilliseconds, ++count, total,
                         (double)request.NumberOfBytesWrittenUncompressed / 1024,
                         (double)request.NumberOfBytesWrittenCompressed / 1024,
                         Math.Round((double)sw.ElapsedMilliseconds / Math.Max(1, batch.Count), 2));

            batch.Clear();

            if (results.Length == 0)
            {
                return(Guid.Empty);
            }
            return(results.Last().Etag.Value);
        }
Ejemplo n.º 28
0
        private bool TryHandleArrayValue(int index, Dictionary <string, object> result, KeyValuePair <string, RavenJToken> prop)
        {
            var arrays = new List <RavenJArray>
            {
                (RavenJArray)prop.Value
            };

            for (var i = 0; i < docs.Length; i++)
            {
                if (i == index)
                {
                    continue;
                }

                RavenJToken token;
                if (docs[i].TryGetValue(prop.Key, out token) && token.Type != JTokenType.Array)
                {
                    return(false);
                }
                if (token == null)
                {
                    continue;
                }
                if (token.IsSnapshot)
                {
                    token = token.CreateSnapshot();
                }
                arrays.Add((RavenJArray)token);
            }

            var mergedArray = new RavenJArray();

            while (arrays.Count > 0)
            {
                var set = new HashSet <RavenJToken>(ravenJTokenEqualityComparer);
                for (var i = 0; i < arrays.Count; i++)
                {
                    if (arrays[i].Length == 0)
                    {
                        arrays.RemoveAt(i);
                        i -= 1;
                        continue;
                    }
                    set.Add(arrays[i][0]);
                    arrays[i].RemoveAt(0);
                }

                foreach (var ravenJToken in set)
                {
                    mergedArray.Add(ravenJToken);
                }
            }

            if (ravenJTokenEqualityComparer.Equals(mergedArray, prop.Value))
            {
                result.Add(prop.Key, mergedArray);
                return(true);
            }

            result.Add(prop.Key, new ArrayWithWarning(mergedArray));
            return(true);
        }
Ejemplo n.º 29
0
		private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource)
		{
			RavenJArray jsonDocuments = null;
			try
			{
				var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();

				docDb.TransactionalStorage.Batch(actions =>
				{
					jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag)
						.Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs
						.Where(x => x.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source
						.Where(x=> x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict
						.Select(x=>
						{
							DocumentRetriever.EnsureIdInMetadata(x);
							return x;
						})
						.Take(100)
						.Select(x => x.ToJson()));
				});
			}
			catch (Exception e)
			{
				log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
			}
			return jsonDocuments;
		}
Ejemplo n.º 30
0
        private async Task <HttpResponseMessage> GetQueriesResponse(bool isGet)
        {
            RavenJArray itemsToLoad;

            if (isGet == false)
            {
                try
                {
                    itemsToLoad = await ReadJsonArrayAsync().ConfigureAwait(false);
                }
                catch (InvalidOperationException e)
                {
                    if (Log.IsDebugEnabled)
                    {
                        Log.DebugException("Failed to deserialize query request.", e);
                    }
                    return(GetMessageWithObject(new
                    {
                        Message = "Could not understand json, please check its validity."
                    }, (HttpStatusCode)422)); //http code 422 - Unprocessable entity
                }
                catch (InvalidDataException e)
                {
                    if (Log.IsDebugEnabled)
                    {
                        Log.DebugException("Failed to deserialize query request.", e);
                    }
                    return(GetMessageWithObject(new
                    {
                        e.Message
                    }, (HttpStatusCode)422)); //http code 422 - Unprocessable entity
                }

                AddRequestTraceInfo(sb =>
                {
                    foreach (var item in itemsToLoad)
                    {
                        sb.Append("\t").Append(item).AppendLine();
                    }
                });
            }
            else
            {
                itemsToLoad = new RavenJArray(GetQueryStringValues("id").Cast <object>());
            }

            var result                = new MultiLoadResult();
            var loadedIds             = new HashSet <string>();
            var includedIds           = new HashSet <string>();
            var includes              = GetQueryStringValues("include") ?? new string[0];
            var transformer           = GetQueryStringValue("transformer") ?? GetQueryStringValue("resultTransformer");
            var transformerParameters = this.ExtractTransformerParameters();

            var includedEtags = new List <byte>();

            if (string.IsNullOrEmpty(transformer) == false)
            {
                var transformerDef = Database.IndexDefinitionStorage.GetTransformer(transformer);
                if (transformerDef == null)
                {
                    return(GetMessageWithObject(new { Error = "No such transformer: " + transformer }, HttpStatusCode.BadRequest));
                }
                includedEtags.AddRange(transformerDef.GetHashCodeBytes());
            }

            Database.TransactionalStorage.Batch(actions =>
            {
                foreach (RavenJToken item in itemsToLoad)
                {
                    var value = item.Value <string>();
                    if (loadedIds.Add(value) == false)
                    {
                        continue;
                    }
                    var documentByKey = string.IsNullOrEmpty(transformer)
                                        ? Database.Documents.Get(value)
                                        : Database.Documents.GetWithTransformer(value, transformer, transformerParameters, out includedIds);
                    if (documentByKey == null)
                    {
                        if (ClientIsV3OrHigher(Request))
                        {
                            result.Results.Add(null);
                        }
                        continue;
                    }
                    result.Results.Add(documentByKey.ToJson());

                    if (documentByKey.Etag != null)
                    {
                        includedEtags.AddRange(documentByKey.Etag.ToByteArray());
                    }

                    // TODO: Revise this.
                    includedEtags.Add((false) ? (byte)0 : (byte)1);
                }

                var addIncludesCommand = new AddIncludesCommand(Database, (etag, includedDoc) =>
                {
                    includedEtags.AddRange(etag.ToByteArray());
                    result.Includes.Add(includedDoc);
                }, includes, loadedIds);

                foreach (var item in result.Results.Where(item => item != null))
                {
                    addIncludesCommand.Execute(item);
                }
            });


            foreach (var includedId in includedIds)
            {
                var doc = Database.Documents.Get(includedId);
                if (doc == null)
                {
                    continue;
                }
                includedEtags.AddRange(doc.Etag.ToByteArray());
                result.Includes.Add(doc.ToJson());
            }

            var  computeHash  = Hashing.Metro128.Calculate(includedEtags.ToArray());
            Etag computedEtag = Etag.FromHash(computeHash);

            if (MatchEtag(computedEtag))
            {
                return(GetEmptyMessage(HttpStatusCode.NotModified));
            }

            var msg = GetMessageWithObject(result);

            WriteETag(computedEtag, msg);

            AddRequestTraceInfo(sb => sb.Append("Results count: {0}, includes count: {1}", result.Results.Count, result.Includes.Count).AppendLine());

            return(msg);
        }
Ejemplo n.º 31
0
        /* Checks for updates and returns release informations when there is newer one.
         * Returns null if there is no newer release.
         * An existing last checked release will be discarded.
         * Throws UpdaterException if error occurs.
         */
        public static Release CheckForUpdates()
        {
            if (Latest != null)
            {
                if (Latest.IsDownloading)
                {
                    throw new UpdaterException("Download already in progress");
                }
                Latest.Dispose();
            }
            Latest    = null;
            IsChecked = IsInstalled = false;

            var webClient = new UpdaterWebClient();

            webClient.Encoding = Encoding.UTF8;

            try
            {
                string json = webClient.DownloadString(GitAPILatestReleaseURL);

                RavenJArray releases = RavenJArray.Parse(json);
                if (releases.Length < 1)
                {
                    throw new UpdaterException("No release found");
                }

                RavenJObject latest = (RavenJObject)releases[0];
                RavenJArray  assets = (RavenJArray)latest["assets"];
                if (assets.Length < 1)
                {
                    throw new UpdaterException("Package for release is missing");
                }

                string current = GetCurrentVersion();
                string tag     = latest["tag_name"].Value <string>();
                if (tag == current)
                {
                    IsChecked = true;

                    return(null);
                }

                string url = ((RavenJObject)assets[0])["browser_download_url"].Value <string>();

                IsChecked = true;
                Latest    = new Release
                {
                    Name        = latest["name"].Value <string>(),
                    Description = latest["body"].Value <string>(),
                    Version     = tag,
                    URI         = new Uri(url)
                };
            }
            catch (WebException e)
            {
                if (e.Status == WebExceptionStatus.ProtocolError)
                {
                    throw new UpdaterException("HTTP " + ((int)((HttpWebResponse)e.Response).StatusCode) + " " + ((HttpWebResponse)e.Response).StatusDescription);
                }
                else
                {
                    throw new UpdaterException(e.Message, e);
                }
            }
            catch (Exception e)
            {
                throw new UpdaterException(e.Message, e);
            }

            return(Latest);
        }
Ejemplo n.º 32
0
 public static RavenJArray ReadJsonArray(this IHttpContext context)
 {
     using (var streamReader = new StreamReader(context.Request.InputStream, GetRequestEncoding(context)))
         using (var jsonReader = new JsonTextReader(streamReader))
             return(RavenJArray.Load(jsonReader));
 }
Ejemplo n.º 33
0
        private Tuple <RavenJArray, Guid> GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
        {
            RavenJArray attachments        = null;
            Guid        lastAttachmentEtag = Guid.Empty;

            try
            {
                var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();

                docDb.TransactionalStorage.Batch(actions =>
                {
                    int attachmentSinceLastEtag = 0;
                    List <AttachmentInformation> attachmentsToReplicate;
                    List <AttachmentInformation> filteredAttachmentsToReplicate;
                    lastAttachmentEtag = destinationsReplicationInformationForSource.LastAttachmentEtag;
                    while (true)
                    {
                        attachmentsToReplicate = actions.Attachments.GetAttachmentsAfter(lastAttachmentEtag, 100, 1024 * 1024 * 10)
                                                 .Concat(actions.Lists.Read(Constants.RavenReplicationAttachmentsTombstones, lastAttachmentEtag, 100)
                                                         .Select(x => new AttachmentInformation
                        {
                            Key      = x.Key,
                            Etag     = x.Etag,
                            Metadata = x.Data,
                            Size     = 0,
                        }))
                                                 .OrderBy(x => x.Etag)

                                                 .ToList();

                        filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList();

                        attachmentSinceLastEtag += attachmentsToReplicate.Count;

                        if (attachmentsToReplicate.Count == 0 ||
                            filteredAttachmentsToReplicate.Count != 0)
                        {
                            break;
                        }

                        AttachmentInformation jsonDocument = attachmentsToReplicate.Last();
                        Guid attachmentEtag = jsonDocument.Etag;
                        log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag);
                        lastAttachmentEtag = attachmentEtag;
                    }

                    log.Debug(() =>
                    {
                        if (attachmentSinceLastEtag == 0)
                        {
                            return(string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination,
                                                 destinationsReplicationInformationForSource.LastDocumentEtag));
                        }

                        if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count)
                        {
                            return(string.Format("Replicating {0} attachments [>{1}] to {2}.",
                                                 attachmentSinceLastEtag,
                                                 destinationsReplicationInformationForSource.LastDocumentEtag,
                                                 destination));
                        }

                        var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key);
                        return(string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
                                             attachmentSinceLastEtag,
                                             filteredAttachmentsToReplicate.Count,
                                             destination,
                                             string.Join(", ", diff),
                                             destinationsReplicationInformationForSource.LastDocumentEtag));
                    });

                    attachments = new RavenJArray(filteredAttachmentsToReplicate
                                                  .Select(x =>
                    {
                        var data = new byte[0];
                        if (x.Size > 0)
                        {
                            data = actions.Attachments.GetAttachment(x.Key).Data().ReadData();
                        }
                        return(new RavenJObject
                        {
                            { "@metadata", x.Metadata },
                            { "@id", x.Key },
                            { "@etag", x.Etag.ToByteArray() },
                            { "data", data }
                        });
                    }));
                });
            }
            catch (Exception e)
            {
                log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
            }
            return(Tuple.Create(attachments, lastAttachmentEtag));
        }
Ejemplo n.º 34
0
 public static RavenJArray ToJArray <T>(IEnumerable <T> result)
 {
     return((RavenJArray)RavenJArray.FromObject(result, CreateDefaultJsonSerializer()));
 }
Ejemplo n.º 35
0
        public async Task DataDumperExportHandlesMaxEtagCorrectly()
        {
            using (var store = NewDocumentStore())
            {
                using (var session = store.OpenSession())
                {
                    for (var i = 0; i < 10; i++)
                    {
                        session.Store(new User {
                            Name = "oren #" + (i + 1)
                        });
                    }
                    session.SaveChanges();
                }

                using (var textStream = new StringWriter())
                    using (var writer = new JsonTextWriter(textStream))
                    {
                        var dumper = new CustomDataDumper(store.SystemDatabase);

                        var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);
                        var endEtag   = startEtag.IncrementBy(2);

                        writer.WriteStartArray();
                        var lastEtag = await dumper.ExportDocuments(writer, startEtag, endEtag);

                        writer.WriteEndArray();
                        writer.Flush();

                        // read exported content
                        var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
                        Assert.Equal(2, exportedDocs.Count());

                        Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value <RavenJObject>("@metadata").Value <string>("@etag"));
                        Assert.Equal("01000000-0000-0001-0000-000000000008", exportedDocs.Last().Value <RavenJObject>("@metadata").Value <string>("@etag"));
                        Assert.Equal("01000000-0000-0001-0000-000000000008", lastEtag.ToString());
                    }

                using (var textStream = new StringWriter())
                    using (var writer = new JsonTextWriter(textStream))
                    {
                        var dumper = new CustomDataDumper(store.SystemDatabase);

                        var startEtag = store.SystemDatabase.Statistics.LastDocEtag.IncrementBy(-5);

                        writer.WriteStartArray();
                        var lastEtag = await dumper.ExportDocuments(writer, startEtag, null);

                        writer.WriteEndArray();
                        writer.Flush();

                        // read exported content
                        var exportedDocs = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
                        Assert.Equal(5, exportedDocs.Count());

                        Assert.Equal("01000000-0000-0001-0000-000000000007", exportedDocs.First().Value <RavenJObject>("@metadata").Value <string>("@etag"));
                        Assert.Equal("01000000-0000-0001-0000-00000000000B", exportedDocs.Last().Value <RavenJObject>("@metadata").Value <string>("@etag"));
                        Assert.Equal("01000000-0000-0001-0000-00000000000B", lastEtag.ToString());
                    }

                for (var i = 0; i < 10; i++)
                {
                    store.DatabaseCommands.PutAttachment("attach/" + (i + 1), null, new MemoryStream(new [] { (byte)i }), new RavenJObject());
                }

                using (var textStream = new StringWriter())
                    using (var writer = new JsonTextWriter(textStream))
                    {
                        var dumper = new CustomDataDumper(store.SystemDatabase);

                        var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);
                        var endEtag   = startEtag.IncrementBy(2);

                        writer.WriteStartArray();
                        var lastEtag = await dumper.ExportAttachments(writer, startEtag, endEtag);

                        writer.WriteEndArray();
                        writer.Flush();

                        // read exported content
                        var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
                        Assert.Equal(2, exportedAttachments.Count());

                        Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value <string>("Etag"));
                        Assert.Equal("02000000-0000-0001-0000-000000000007", exportedAttachments.Last().Value <string>("Etag"));
                        Assert.Equal("02000000-0000-0001-0000-000000000007", lastEtag.ToString());
                    }

                using (var textStream = new StringWriter())
                    using (var writer = new JsonTextWriter(textStream))
                    {
                        var dumper = new CustomDataDumper(store.SystemDatabase);

                        var startEtag = store.SystemDatabase.Statistics.LastAttachmentEtag.IncrementBy(-5);

                        writer.WriteStartArray();
                        var lastEtag = await dumper.ExportAttachments(writer, startEtag, null);

                        writer.WriteEndArray();
                        writer.Flush();

                        // read exported content
                        var exportedAttachments = RavenJArray.Parse(textStream.GetStringBuilder().ToString());
                        Assert.Equal(5, exportedAttachments.Count());

                        Assert.Equal("02000000-0000-0001-0000-000000000006", exportedAttachments.First().Value <string>("Etag"));
                        Assert.Equal("02000000-0000-0001-0000-00000000000A", exportedAttachments.Last().Value <string>("Etag"));
                        Assert.Equal("02000000-0000-0001-0000-00000000000A", lastEtag.ToString());
                    }

                WaitForIndexing(store);

                store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery()
                {
                    Query = "Tag:Users"
                }).WaitForCompletion();

                for (var i = 0; i < 10; i++)
                {
                    store.DatabaseCommands.DeleteAttachment("attach/" + (i + 1), null);
                }

                Etag user6DeletionEtag = null, user9DeletionEtag = null, attach5DeletionEtag = null, attach7DeletionEtag = null;

                WaitForUserToContinueTheTest(store);

                store.SystemDatabase.TransactionalStorage.Batch(accessor =>
                {
                    user6DeletionEtag =
                        accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/6").Etag;
                    user9DeletionEtag =
                        accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, "users/9").Etag;
                    attach5DeletionEtag =
                        accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/5").Etag;
                    attach7DeletionEtag =
                        accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, "attach/7").Etag;
                });

                using (var textStream = new StringWriter())
                    using (var writer = new JsonTextWriter(textStream))
                    {
                        var dumper = new CustomDataDumper(store.SystemDatabase);

                        writer.WriteStartObject();
                        var lastEtags    = new LastEtagsInfo();
                        var exportResult = new ExportDataResult
                        {
                            LastDocDeleteEtag         = user6DeletionEtag,
                            LastAttachmentsDeleteEtag = attach5DeletionEtag
                        };

                        lastEtags.LastDocDeleteEtag         = user9DeletionEtag;
                        lastEtags.LastAttachmentsDeleteEtag = attach7DeletionEtag;
                        dumper.ExportDeletions(writer, exportResult, lastEtags).Wait();
                        writer.WriteEndObject();
                        writer.Flush();

                        // read exported content
                        var exportJson = RavenJObject.Parse(textStream.GetStringBuilder().ToString());
                        var docsKeys   =
                            exportJson.Value <RavenJArray>("DocsDeletions").Select(x => x.Value <string>("Key")).ToArray();
                        var attachmentsKeys =
                            exportJson.Value <RavenJArray>("AttachmentsDeletions")
                            .Select(x => x.Value <string>("Key"))
                            .ToArray();
                        Assert.Equal(new [] { "users/7", "users/8", "users/9" }, docsKeys);
                        Assert.Equal(new [] { "attach/6", "attach/7" }, attachmentsKeys);
                    }
            }
        }
Ejemplo n.º 36
0
        /// <summary>
        /// Deserialize a <param name="self"/> to a list of instances of<typeparam name="T"/>
        /// </summary>
        public static T[] JsonDeserialization <T>(this RavenJArray self)
        {
            var serializer = CreateDefaultJsonSerializer();

            return(self.Select(x => (T)serializer.Deserialize(new RavenJTokenReader(x), typeof(T))).ToArray());
        }
        public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation)
        {
            using (Database.DisableAllTriggersForCurrentThread())
            {
                metadata.Remove(Constants.RavenReplicationConflict);                // you can't put conflicts

                var oldVersion = Database.Documents.Get(key, transactionInformation);
                if (oldVersion == null)
                {
                    return;
                }
                if (oldVersion.Metadata[Constants.RavenReplicationConflict] == null)
                {
                    return;
                }

                var history = new RavenJArray();
                metadata[Constants.RavenReplicationHistory] = history;

                // this is a conflict document, holding document keys in the
                // values of the properties
                var conflicts = oldVersion.DataAsJson.Value <RavenJArray>("Conflicts");
                if (conflicts == null)
                {
                    return;
                }

                var list = new List <RavenJArray>
                {
                    new RavenJArray(ReplicationData.GetHistory(metadata))             // first item to interleave
                };

                foreach (var prop in conflicts)
                {
                    RavenJObject deletedMetadata;
                    Database.Documents.Delete(prop.Value <string>(), null, transactionInformation, out deletedMetadata);

                    if (deletedMetadata != null)
                    {
                        var conflictHistory = new RavenJArray(ReplicationData.GetHistory(deletedMetadata));
                        conflictHistory.Add(new RavenJObject
                        {
                            { Constants.RavenReplicationVersion, deletedMetadata[Constants.RavenReplicationVersion] },
                            { Constants.RavenReplicationSource, deletedMetadata[Constants.RavenReplicationSource] }
                        });
                        list.Add(conflictHistory);
                    }
                }


                int  index = 0;
                bool added = true;
                while (added) // interleave the history from all conflicts
                {
                    added = false;
                    foreach (var deletedMetadata in list)
                    {
                        // add the conflict history to the mix, so we make sure that we mark that we resolved the conflict
                        if (index < deletedMetadata.Length)
                        {
                            history.Add(deletedMetadata[index]);
                            added = true;
                        }
                    }
                    index++;
                }

                while (history.Length > Constants.ChangeHistoryLength)
                {
                    history.RemoveAt(0);
                }
            }
        }
        private void ReplicateDelete(string id, RavenJObject metadata, TExternal incoming)
        {
            TInternal existingItem;
            Etag      existingEtag;
            bool      deleted;
            var       existingMetadata = TryGetExisting(id, out existingItem, out existingEtag, out deleted);

            if (existingMetadata == null)
            {
                log.Debug("Replicating deleted item {0} from {1} that does not exist, ignoring", id, Src);
                return;
            }
            if (existingMetadata.Value <bool>(Constants.RavenDeleteMarker))            //deleted locally as well
            {
                log.Debug("Replicating deleted item {0} from {1} that was deleted locally. Merging histories", id, Src);
                var existingHistory = new RavenJArray(ReplicationData.GetHistory(existingMetadata));
                var newHistory      = new RavenJArray(ReplicationData.GetHistory(metadata));

                foreach (var item in newHistory)
                {
                    existingHistory.Add(item);
                }


                if (metadata.ContainsKey(Constants.RavenReplicationVersion) &&
                    metadata.ContainsKey(Constants.RavenReplicationSource))
                {
                    existingHistory.Add(new RavenJObject
                    {
                        { Constants.RavenReplicationVersion, metadata[Constants.RavenReplicationVersion] },
                        { Constants.RavenReplicationSource, metadata[Constants.RavenReplicationSource] }
                    });
                }

                while (existingHistory.Length > Constants.ChangeHistoryLength)
                {
                    existingHistory.RemoveAt(0);
                }

                MarkAsDeleted(id, metadata);
                return;
            }
            if (Historian.IsDirectChildOfCurrent(metadata, existingMetadata))            // not modified
            {
                log.Debug("Delete of existing item {0} was replicated successfully from {1}", id, Src);
                DeleteItem(id, existingEtag);
                MarkAsDeleted(id, metadata);
                return;
            }

            CreatedConflict createdConflict;

            if (existingMetadata.Value <bool>(Constants.RavenReplicationConflict))            // locally conflicted
            {
                log.Debug("Replicating deleted item {0} from {1} that is already conflicted, adding to conflicts.", id, Src);
                var savedConflictedItemId = SaveConflictedItem(id, metadata, incoming, existingEtag);
                createdConflict = AppendToCurrentItemConflicts(id, savedConflictedItemId, existingMetadata, existingItem);
            }
            else
            {
                var newConflictId = SaveConflictedItem(id, metadata, incoming, existingEtag);
                log.Debug("Existing item {0} is in conflict with replicated delete from {1}, marking item as conflicted", id, Src);

                // we have a new conflict  move the existing doc to a conflict and create a conflict document
                var existingDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(existingEtag);
                createdConflict = CreateConflict(id, newConflictId, existingDocumentConflictId, existingItem, existingMetadata);
            }

            Database.TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() =>
                                                                                         Database.RaiseNotifications(new ReplicationConflictNotification()
            {
                Id            = id,
                Etag          = createdConflict.Etag,
                Conflicts     = createdConflict.ConflictedIds,
                ItemType      = ReplicationConflictTypes.DocumentReplicationConflict,
                OperationType = ReplicationOperationTypes.Delete
            }));
        }
Ejemplo n.º 39
0
        private bool TryReplicationAttachments(ReplicationStrategy destination, RavenJArray jsonAttachments, out string errorMessage)
        {
            try
            {
                var url = destination.ConnectionStringOptions.Url + "/replication/replicateAttachments?from=" +
                          UrlEncodedServerUrl() + "&dbid=" + docDb.TransactionalStorage.Id;

                var sp = Stopwatch.StartNew();
                using (HttpRavenRequestFactory.Expect100Continue(destination.ConnectionStringOptions.Url))
                {
                    var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions, GetRequestBuffering(destination));

                    request.WriteBson(jsonAttachments);
                    request.ExecuteRequest(docDb.WorkContext.CancellationToken);
                    log.Info("Replicated {0} attachments to {1} in {2:#,#;;0} ms", jsonAttachments.Length, destination, sp.ElapsedMilliseconds);
                    errorMessage = "";
                    return true;
                }
            }
            catch (WebException e)
            {
                HandleRequestBufferingErrors(e, destination);

                var response = e.Response as HttpWebResponse;
                if (response != null)
                {
                    using (var streamReader = new StreamReader(response.GetResponseStreamWithHttpDecompression()))
                    {
                        var error = streamReader.ReadToEnd();
                        try
                        {
                            var ravenJObject = RavenJObject.Parse(error);
                            log.WarnException("Replication to " + destination + " had failed\r\n" + ravenJObject.Value<string>("Error"), e);
                            errorMessage = error;
                            return false;
                        }
                        catch (Exception)
                        {
                        }

                        log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
                        errorMessage = error;
                    }
                }
                else
                {
                    log.WarnException("Replication to " + destination + " had failed", e);
                    errorMessage = e.Message;
                }
                return false;
            }
            catch (Exception e)
            {
                log.WarnException("Replication to " + destination + " had failed", e);
                errorMessage = e.Message;
                return false;
            }
        }
Ejemplo n.º 40
0
 public DestinationStats()
 {
     LastStats = new RavenJArray();
 }
Ejemplo n.º 41
0
        public static RavenJObject[] ReadAllEntriesFromIndex(IndexReader reader)
        {
            if (reader.MaxDoc > 512 * 1024)
            {
                throw new InvalidOperationException("Refusing to extract all index entires from an index with " + reader.MaxDoc +
                                                    " entries, because of the probable time / memory costs associated with that." +
                                                    Environment.NewLine +
                                                    "Viewing Index Entries are a debug tool, and should not be used on indexes of this size. You might want to try Luke, instead.");
            }
            var results = new RavenJObject[reader.MaxDoc];
            using (var termDocs = reader.TermDocs())
            using (var termEnum = reader.Terms())
            {
                while (termEnum.Next())
                {
                    var term = termEnum.Term;
                    if (term == null)
                        break;

                    var text = term.Text;

                    termDocs.Seek(termEnum);
                    for (int i = 0; i < termEnum.DocFreq() && termDocs.Next(); i++)
                    {
                        RavenJObject result = results[termDocs.Doc];
                        if (result == null)
                            results[termDocs.Doc] = result = new RavenJObject();
                        var propertyName = term.Field;
                        if (propertyName.EndsWith("_ConvertToJson") ||
                            propertyName.EndsWith("_IsArray"))
                            continue;
                        if (result.ContainsKey(propertyName))
                        {
                            switch (result[propertyName].Type)
                            {
                                case JTokenType.Array:
                                    ((RavenJArray)result[propertyName]).Add(text);
                                    break;
                                case JTokenType.String:
                                    result[propertyName] = new RavenJArray
									{
										result[propertyName],
										text
									};
                                    break;
                                default:
                                    throw new ArgumentException("No idea how to handle " + result[propertyName].Type);
                            }
                        }
                        else
                        {
                            result[propertyName] = text;
                        }
                    }
                }
            }
            return results;
        }
Ejemplo n.º 42
0
 private bool TryReplicationDocuments(RavenConnectionStringOptions destination, RavenJArray jsonDocuments)
 {
     try
     {
         log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination);
         var request = (HttpWebRequest)WebRequest.Create(destination.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl());
         request.UseDefaultCredentials = true;
         request.ContentType           = "application/json; charset=utf-8";
         request.Credentials           = destination.Credentials ?? CredentialCache.DefaultNetworkCredentials;
         request.Method = "POST";
         using (var stream = request.GetRequestStream())
             using (var streamWriter = new StreamWriter(stream, Encoding.UTF8))
             {
                 jsonDocuments.WriteTo(new JsonTextWriter(streamWriter));
                 streamWriter.Flush();
                 stream.Flush();
             }
         using (request.GetResponse())
         {
             log.Info("Replicated {0} documents to {1}", jsonDocuments.Length, destination);
         }
         return(true);
     }
     catch (WebException e)
     {
         var response = e.Response as HttpWebResponse;
         if (response != null)
         {
             using (var streamReader = new StreamReader(response.GetResponseStream()))
             {
                 var error = streamReader.ReadToEnd();
                 log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
             }
         }
         else
         {
             log.WarnException("Replication to " + destination + " had failed", e);
         }
         return(false);
     }
     catch (Exception e)
     {
         log.WarnException("Replication to " + destination + " had failed", e);
         return(false);
     }
 }
Ejemplo n.º 43
0
        public override void OnPut(string key, RavenJObject jsonReplicationDocument, RavenJObject metadata, TransactionInformation transactionInformation)
        {
            if (key.StartsWith("Raven/", StringComparison.OrdinalIgnoreCase) &&             // we don't deal with system documents
                key.StartsWith("Raven/Hilo/", StringComparison.OrdinalIgnoreCase) == false) // except for hilos
            {
                return;
            }

            using (Database.DisableAllTriggersForCurrentThread())
            {
                var documentMetadata = GetDocumentMetadata(key);
                if (documentMetadata != null)
                {
                    var history = new RavenJArray(ReplicationData.GetHistory(documentMetadata));

                    if (documentMetadata.ContainsKey(Constants.RavenReplicationMergedHistory) == false)
                    {
                        if (documentMetadata.ContainsKey(Constants.RavenReplicationVersion) &&
                            documentMetadata.ContainsKey(Constants.RavenReplicationSource))
                        {
                            history.Add(new RavenJObject
                            {
                                { Constants.RavenReplicationVersion, documentMetadata[Constants.RavenReplicationVersion] },
                                { Constants.RavenReplicationSource, documentMetadata[Constants.RavenReplicationSource] }
                            });
                        }
                        else
                        {
                            history.Add(new RavenJObject
                            {
                                { Constants.RavenReplicationVersion, 0 },
                                { Constants.RavenReplicationSource, RavenJToken.FromObject(Database.TransactionalStorage.Id) }
                            });
                        }

                        var sources = new HashSet <RavenJToken>(RavenJTokenEqualityComparer.Default);
                        int pos     = history.Length - 1;
                        for (; pos >= 0; pos--)
                        {
                            var source = ((RavenJObject)history[pos])[Constants.RavenReplicationSource];
                            if (sources.Contains(source))
                            {
                                history.RemoveAt(pos);
                                continue;
                            }
                            sources.Add(source);
                        }
                        metadata[Constants.RavenReplicationMergedHistory] = true;
                        metadata[Constants.RavenReplicationHistory]       = history;
                    }
                    //If we have the flag we must have Constants.RavenReplicationVersion and Constants.RavenReplicationSource too
                    //Here we assume that the replication history is in the form of a "sorted dictionary" so we just need to remove
                    //the entry with the current source id and insert the new version at the end of the history.
                    else
                    {
                        int i = history.Length - 1;
                        for (; i >= 0; i--)
                        {
                            var currentEntry = history[i];
                            if (RavenJTokenEqualityComparer.Default.Equals(((RavenJObject)currentEntry)
                                                                           [Constants.RavenReplicationSource], documentMetadata[Constants.RavenReplicationSource]))
                            {
                                break;
                            }
                        }
                        if (i != -1)
                        {
                            history.RemoveAt(i);
                        }
                        history.Add(new RavenJObject
                        {
                            { Constants.RavenReplicationVersion, documentMetadata[Constants.RavenReplicationVersion] },
                            { Constants.RavenReplicationSource, documentMetadata[Constants.RavenReplicationSource] }
                        });
                        metadata[Constants.RavenReplicationHistory] = history;
                    }
                }

                metadata[Constants.RavenReplicationVersion] = RavenJToken.FromObject(ReplicationHiLo.NextId(Database));
                metadata[Constants.RavenReplicationSource]  = RavenJToken.FromObject(Database.TransactionalStorage.Id);
            }
        }
Ejemplo n.º 44
0
        private RavenJToken ToRavenJToken(JsInstance v)
        {
            switch (v.Class)
            {
            case JsInstance.TYPE_OBJECT:
            case JsInstance.CLASS_OBJECT:
                return(ToRavenJObject((JsObject)v));

            case JsInstance.CLASS_DATE:
                var dt = (DateTime)v.Value;
                return(new RavenJValue(dt));

            case JsInstance.TYPE_NUMBER:
            case JsInstance.CLASS_NUMBER:
                var num     = (double)v.Value;
                var integer = Math.Truncate(num);
                if (Math.Abs(num - integer) < double.Epsilon)
                {
                    return(new RavenJValue((long)integer));
                }
                return(new RavenJValue(num));

            case JsInstance.TYPE_STRING:
            case JsInstance.TYPE_BOOLEAN:
            case JsInstance.CLASS_STRING:
            case JsInstance.CLASS_BOOLEAN:
                return(new RavenJValue(v.Value));

            case JsInstance.CLASS_NULL:
            case JsInstance.TYPE_NULL:
                return(RavenJValue.Null);

            case JsInstance.CLASS_UNDEFINED:
            case JsInstance.TYPE_UNDEFINED:
                return(RavenJValue.Null);

            case JsInstance.CLASS_ARRAY:
                var jsArray = ((JsArray)v);
                var rja     = new RavenJArray();

                for (int i = 0; i < jsArray.Length; i++)
                {
                    var jsInstance  = jsArray.get(i);
                    var ravenJToken = ToRavenJToken(jsInstance);
                    if (ravenJToken == null)
                    {
                        continue;
                    }
                    rja.Add(ravenJToken);
                }
                return(rja);

            case JsInstance.CLASS_REGEXP:
            case JsInstance.CLASS_ERROR:
            case JsInstance.CLASS_ARGUMENTS:
            case JsInstance.CLASS_DESCRIPTOR:
            case JsInstance.CLASS_FUNCTION:
                return(null);

            default:
                throw new NotSupportedException(v.Class);
            }
        }
Ejemplo n.º 45
0
        private RavenJToken ToRavenJToken(JsValue v, string propertyKey)
        {
            if (v.IsBoolean())
                return new RavenJValue(v.AsBoolean());
            if (v.IsString())
            {
                const string RavenDataByteArrayToBase64 = "raven-data:byte[];base64,";
                var value = v.AsString();
                if (value != null && value.StartsWith(RavenDataByteArrayToBase64))
                {
                    value = value.Remove(0, RavenDataByteArrayToBase64.Length);
                    var byteArray = Convert.FromBase64String(value);
                    return new RavenJValue(byteArray);
                }
                return new RavenJValue(value);
            }
            if (v.IsNumber())
            {
                var num = v.AsNumber();

				KeyValuePair<RavenJValue, JsValue> property;
				if (propertiesByValue.TryGetValue(propertyKey, out property))
                {
                    var originalValue = property.Key;
                    if (originalValue.Type == JTokenType.Float)
                        return new RavenJValue(num);
                    if (originalValue.Type == JTokenType.Integer)
                    {
                        // If the current value is exactly as the original value, we can return the original value before we made the JS conversion, 
                        // which will convert a Int64 to jsFloat.
                        var originalJsValue = property.Value;
                        if (originalJsValue.IsNumber() && Math.Abs(num - originalJsValue.AsNumber()) < double.Epsilon)
                            return originalValue;

                        return new RavenJValue((long)num);
                    }
                }

                // If we don't have the type, assume that if the number ending with ".0" it actually an integer.
                var integer = Math.Truncate(num);
                if (Math.Abs(num - integer) < double.Epsilon)
                    return new RavenJValue((long)integer);
                return new RavenJValue(num);
            }
            if (v.IsNull())
                return RavenJValue.Null;
            if (v.IsUndefined())
                return RavenJValue.Null;
            if (v.IsArray())
            {
                var jsArray = v.AsArray();
                var rja = new RavenJArray();

                foreach (var property in jsArray.Properties)
                {
                    if (property.Key == "length")
                        continue;

                    var jsInstance = property.Value.Value;
                    if (!jsInstance.HasValue)
                        continue;

                    var ravenJToken = ToRavenJToken(jsInstance.Value, CreatePropertyKey(property.Key, propertyKey));
                    if (ravenJToken == null)
                        continue;

                    rja.Add(ravenJToken);
                }

                return rja;
            }
            if (v.IsObject())
                return ToRavenJObject(v, propertyKey);
            if (v.IsRegExp())
                return null;

            throw new NotSupportedException(v.Type.ToString());
        }
            private static RavenJObject CreateJsonDocumentFromLuceneDocument(Document document)
            {
                var field = document.GetField(Constants.ReduceValueFieldName);

                if (field != null)
                {
                    return(RavenJObject.Parse(field.StringValue));
                }

                var ravenJObject = new RavenJObject();

                var fields       = document.GetFields();
                var arrayMarkers = fields
                                   .Where(x => x.Name.EndsWith("_IsArray"))
                                   .Select(x => x.Name)
                                   .ToList();

                foreach (var fieldable in fields)
                {
                    var stringValue   = GetStringValue(fieldable);
                    var isArrayMarker = fieldable.Name.EndsWith("_IsArray");
                    var isArray       = !isArrayMarker && arrayMarkers.Contains(fieldable.Name + "_IsArray");

                    RavenJToken token;
                    var         isJson = RavenJToken.TryParse(stringValue, out token);

                    RavenJToken value;
                    if (ravenJObject.TryGetValue(fieldable.Name, out value) == false)
                    {
                        if (isArray)
                        {
                            ravenJObject[fieldable.Name] = new RavenJArray {
                                isJson?token : stringValue
                            }
                        }
                        ;
                        else if (isArrayMarker)
                        {
                            var fieldName = fieldable.Name.Substring(0, fieldable.Name.Length - 8);
                            ravenJObject[fieldable.Name] = isJson ? token : stringValue;
                            ravenJObject[fieldName]      = new RavenJArray();
                        }
                        else
                        {
                            ravenJObject[fieldable.Name] = isJson ? token : stringValue;
                        }
                    }
                    else
                    {
                        var ravenJArray = value as RavenJArray;
                        if (ravenJArray != null)
                        {
                            ravenJArray.Add(isJson ? token : stringValue);
                        }
                        else
                        {
                            ravenJArray = new RavenJArray {
                                value, isJson ? token : stringValue
                            };
                            ravenJObject[fieldable.Name] = ravenJArray;
                        }
                    }
                }
                return(ravenJObject);
            }
Ejemplo n.º 47
0
		protected override Guid FlushBatch(List<RavenJObject> batch)
		{
			var sw = Stopwatch.StartNew();

			var commands = new RavenJArray();
			foreach (var doc in batch)
			{
				var metadata = doc.Value<RavenJObject>("@metadata");
				doc.Remove("@metadata");
				commands.Add(new RavenJObject
								{
									{"Method", "PUT"},
									{"Document", doc},
									{"Metadata", metadata},
									{"Key", metadata.Value<string>("@id")}
								});
			}

			var retries = retriesCount;
			HttpRavenRequest request = null;
			BatchResult[] results;
			while (true)
			{
				try
				{
					request = CreateRequest("/bulk_docs", "POST");
					request.Write(commands);
					results = request.ExecuteRequest<BatchResult[]>();
					sw.Stop();
					break;
				}
				catch (Exception e)
				{
					if (--retries == 0 || request == null)
						throw;
					sw.Stop();
					LastRequestErrored = true;
					ShowProgress("Error flushing to database, remaining attempts {0} - time {2:#,#} ms, will retry [{3:#,#.##;;0} kb compressed to {4:#,#.##;;0} kb]. Error: {1}",
								 retriesCount - retries, e, sw.ElapsedMilliseconds,
								 (double)request.NumberOfBytesWrittenUncompressed / 1024,
								 (double)request.NumberOfBytesWrittenCompressed / 1024);
				}
			}
			total += batch.Count;
			ShowProgress("{2,5:#,#}: Wrote {0:#,#;;0} in {1,6:#,#;;0} ms ({6:0.00} ms per doc) (total of {3:#,#;;0}) documents [{4:#,#.##;;0} kb compressed to {5:#,#.##;;0} kb]",
				batch.Count, sw.ElapsedMilliseconds, ++count, total,
				(double)request.NumberOfBytesWrittenUncompressed / 1024,
				(double)request.NumberOfBytesWrittenCompressed / 1024,
				Math.Round((double)sw.ElapsedMilliseconds / Math.Max(1, batch.Count), 2));

			batch.Clear();

			if (results.Length == 0)
				return Guid.Empty;
			return results.Last().Etag.Value;
		}
Ejemplo n.º 48
0
        private RavenJToken ToRavenJToken(JsValue v, string propertyKey, bool recursiveCall)
        {
            if (v.IsBoolean())
            {
                return(new RavenJValue(v.AsBoolean()));
            }
            if (v.IsString())
            {
                const string RavenDataByteArrayToBase64 = "raven-data:byte[];base64,";
                var          valueAsObject = v.ToObject();
                var          value         = valueAsObject != null?valueAsObject.ToString() : null;

                if (value != null && value.StartsWith(RavenDataByteArrayToBase64))
                {
                    value = value.Remove(0, RavenDataByteArrayToBase64.Length);
                    var byteArray = Convert.FromBase64String(value);
                    return(new RavenJValue(byteArray));
                }
                return(new RavenJValue(value));
            }
            if (v.IsNumber())
            {
                var num = v.AsNumber();

                KeyValuePair <RavenJValue, JsValue> property;
                if (propertiesByValue.TryGetValue(propertyKey, out property))
                {
                    var originalValue = property.Key;
                    if (originalValue.Type == JTokenType.Float)
                    {
                        return(new RavenJValue(num));
                    }
                    if (originalValue.Type == JTokenType.Integer)
                    {
                        // If the current value is exactly as the original value, we can return the original value before we made the JS conversion,
                        // which will convert a Int64 to jsFloat.
                        var originalJsValue = property.Value;
                        if (originalJsValue.IsNumber() && Math.Abs(num - originalJsValue.AsNumber()) < double.Epsilon)
                        {
                            return(originalValue);
                        }

                        return(new RavenJValue((long)num));
                    }
                }

                // If we don't have the type, assume that if the number ending with ".0" it actually an integer.
                var integer = Math.Truncate(num);
                if (Math.Abs(num - integer) < double.Epsilon)
                {
                    return(new RavenJValue((long)integer));
                }
                return(new RavenJValue(num));
            }
            if (v.IsNull())
            {
                return(RavenJValue.Null);
            }
            if (v.IsUndefined())
            {
                return(RavenJValue.Null);
            }
            if (v.IsArray())
            {
                var jsArray = v.AsArray();
                var rja     = new RavenJArray();

                foreach (var property in jsArray.GetOwnProperties())
                {
                    if (InheritedProperties.Contains(property.Key))
                    {
                        continue;
                    }

                    var jsInstance = property.Value.Value;
                    if (!jsInstance.HasValue)
                    {
                        continue;
                    }

                    var ravenJToken = ToRavenJToken(jsInstance.Value, CreatePropertyKey(property.Key, propertyKey), recursiveCall);
                    if (ravenJToken == null)
                    {
                        continue;
                    }

                    rja.Add(ravenJToken);
                }

                return(rja);
            }
            if (v.IsObject())
            {
                return(ToRavenJObject(v, propertyKey, recursiveCall));
            }
            if (v.IsRegExp())
            {
                return(null);
            }

            throw new NotSupportedException(v.Type.ToString());
        }
Ejemplo n.º 49
0
		private bool TryReplicationDocuments(RavenConnectionStringOptions destination, RavenJArray jsonDocuments)
		{
			try
			{
				log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination);
				var request = (HttpWebRequest)WebRequest.Create(destination.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl());
				request.UseDefaultCredentials = true;
				request.ContentType = "application/json; charset=utf-8";
				request.Credentials = destination.Credentials ?? CredentialCache.DefaultNetworkCredentials;
				request.Method = "POST";
				using (var stream = request.GetRequestStream())
				using (var streamWriter = new StreamWriter(stream, Encoding.UTF8))
				{
					jsonDocuments.WriteTo(new JsonTextWriter(streamWriter));
					streamWriter.Flush();
					stream.Flush();
				}
				using (request.GetResponse())
				{
					log.Info("Replicated {0} documents to {1}", jsonDocuments.Length, destination);
				}
				return true;
			}
			catch (WebException e)
			{
				var response = e.Response as HttpWebResponse;
				if (response != null)
				{
					using (var streamReader = new StreamReader(response.GetResponseStream()))
					{
						var error = streamReader.ReadToEnd();
						log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
					}
				}
				else
				{
					log.WarnException("Replication to " + destination + " had failed", e);
				}
				return false;
			}
			catch (Exception e)
			{
				log.WarnException("Replication to " + destination + " had failed", e);
				return false;
			}
		}
Ejemplo n.º 50
0
        public static RavenJObject[] ReadAllEntriesFromIndex(IndexReader reader)
        {
            if (reader.MaxDoc > 512 * 1024)
            {
                throw new InvalidOperationException("Refusing to extract all index entires from an index with " + reader.MaxDoc +
                                                    " entries, because of the probable time / memory costs associated with that." +
                                                    Environment.NewLine +
                                                    "Viewing Index Entries are a debug tool, and should not be used on indexes of this size. You might want to try Luke, instead.");
            }
            var results = new RavenJObject[reader.MaxDoc];

            using (var termDocs = reader.TermDocs())
                using (var termEnum = reader.Terms())
                {
                    while (termEnum.Next())
                    {
                        var term = termEnum.Term;
                        if (term == null)
                        {
                            break;
                        }

                        var text = term.Text;

                        termDocs.Seek(termEnum);
                        for (int i = 0; i < termEnum.DocFreq() && termDocs.Next(); i++)
                        {
                            RavenJObject result = results[termDocs.Doc];
                            if (result == null)
                            {
                                results[termDocs.Doc] = result = new RavenJObject();
                            }
                            var propertyName = term.Field;
                            if (propertyName.EndsWith("_ConvertToJson") ||
                                propertyName.EndsWith("_IsArray"))
                            {
                                continue;
                            }
                            if (result.ContainsKey(propertyName))
                            {
                                switch (result[propertyName].Type)
                                {
                                case JTokenType.Array:
                                    ((RavenJArray)result[propertyName]).Add(text);
                                    break;

                                case JTokenType.String:
                                    result[propertyName] = new RavenJArray
                                    {
                                        result[propertyName],
                                        text
                                    };
                                    break;

                                default:
                                    throw new ArgumentException("No idea how to handle " + result[propertyName].Type);
                                }
                            }
                            else
                            {
                                result[propertyName] = text;
                            }
                        }
                    }
                }
            return(results);
        }
Ejemplo n.º 51
0
		private RavenJArray GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource)
		{
			RavenJArray jsonAttachments = null;
			try
			{
				string destinationInstanceId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();

				docDb.TransactionalStorage.Batch(actions =>
				{
					jsonAttachments = new RavenJArray(actions.Attachments.GetAttachmentsAfter(destinationsReplicationInformationForSource.LastAttachmentEtag)
						.Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs
						.Where(x => x.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) != destinationInstanceId) // prevent replicating back to source
						.Where(x => x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict
						.Take(100)
						.Select(x => new RavenJObject
						{
							{"@metadata", x.Metadata},
							{"@id", x.Key},
							{"@etag", x.Etag.ToByteArray()},
							{"data", actions.Attachments.GetAttachment(x.Key).Data}
						}));
				});
			}
			catch (Exception e)
			{
				log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
			}
			return jsonAttachments;
		}
Ejemplo n.º 52
0
        protected override async Task <Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag)
        {
            var totalCount = 0;

            while (true)
            {
                try
                {
                    if (SmugglerOptions.Limit - totalCount <= 0)
                    {
                        ShowProgress("Done with reading attachments, total: {0}", totalCount);
                        return(lastEtag);
                    }

                    var         maxRecords     = Math.Min(SmugglerOptions.Limit - totalCount, SmugglerOptions.BatchSize);
                    RavenJArray attachmentInfo = null;
                    var         request        = CreateRequest("/static/?pageSize=" + maxRecords + "&etag=" + lastEtag);
                    request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader)));

                    if (attachmentInfo.Length == 0)
                    {
                        var databaseStatistics = await GetStats();

                        var lastEtagComparable = new ComparableByteArray(lastEtag);
                        if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
                        {
                            lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
                            ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
                            continue;
                        }
                        ShowProgress("Done with reading attachments, total: {0}", totalCount);
                        return(lastEtag);
                    }

                    ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length,
                                 totalCount);
                    foreach (var item in attachmentInfo)
                    {
                        ShowProgress("Downloading attachment: {0}", item.Value <string>("Key"));

                        byte[] attachmentData = null;
                        var    requestData    = CreateRequest("/static/" + item.Value <string>("Key"));
                        requestData.ExecuteRequest(reader => attachmentData = reader.ReadData());

                        new RavenJObject
                        {
                            { "Data", attachmentData },
                            { "Metadata", item.Value <RavenJObject>("Metadata") },
                            { "Key", item.Value <string>("Key") }
                        }
                        .WriteTo(jsonWriter);
                        totalCount++;
                        lastEtag = Etag.Parse(item.Value <string>("Etag"));
                    }
                }
                catch (Exception e)
                {
                    ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
                    ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag);
                    throw new SmugglerExportException(e.Message, e)
                          {
                              LastEtag = lastEtag,
                          };
                }
            }
        }