Example #1
0
        public void Execute(DocumentDatabase database)
        {
            if (!database.IsBundleActive("IndexedAttachments"))
                return;

            var index = new IndexDefinition
                        {
                            Map = @"from doc in docs
            where doc[""@metadata""][""Raven-Attachment-Key""] != null
            select new
            {
            AttachmentKey = doc[""@metadata""][""Raven-Attachment-Key""],
            Filename = doc[""@metadata""][""Raven-Attachment-Filename""],
            Text = doc.Text
            }",
                            TransformResults = @"from result in results
            select new
            {
            AttachmentKey = result[""@metadata""][""Raven-Attachment-Key""],
            Filename = result[""@metadata""][""Raven-Attachment-Filename""]
            }"
                        };

            // NOTE: The transform above is specifically there to keep the Text property
            //       from being returned.  The results could get very large otherwise.

            index.Indexes.Add("Text", FieldIndexing.Analyzed);
            index.Stores.Add("Text", FieldStorage.Yes);
            index.TermVectors.Add("Text", FieldTermVector.WithPositionsAndOffsets);

            database.PutIndex("Raven/Attachments", index);
        }
		public void LinqQueryWithStaticCallOnEnumerableIsCanBeCompiledAndRun()
		{
			var indexDefinition = new IndexDefinition<Page>
			{
				Map = pages => from p in pages
							   from coAuthor in p.CoAuthors.DefaultIfEmpty()
							   select new
							   {
								   p.Id,
								   CoAuthorUserID = coAuthor != null ? coAuthor.UserId : -1
							   }
			}.ToIndexDefinition(new DocumentConvention());

			var mapInstance = new DynamicViewCompiler("testView",
													  indexDefinition, new AbstractDynamicCompilationExtension[] { }).
				GenerateInstance();

			var conventions = new DocumentConvention();
			var o = JObject.FromObject(page,conventions.CreateSerializer());
			o["@metadata"] = new JObject(
				new JProperty("Raven-Entity-Name", "Pages")
				);
			dynamic dynamicObject = new DynamicJsonObject(o);

			var result = mapInstance.MapDefinition(new[] { dynamicObject }).ToList();
			Assert.Equal("{ Id = 0, CoAuthorUserID = 1, __document_id =  }", result[0].ToString());
			Assert.Equal("{ Id = 0, CoAuthorUserID = 2, __document_id =  }", result[1].ToString());
		}
Example #3
0
        public void LinqQueryWithIndexIsCaseInsensitive()
        {
            using (var store = this.NewDocumentStore())
            {
                var definition = new IndexDefinition<Company>
                {
                    Map = docs => from doc in docs
                                  select new
                                  {
                                      doc.Name
                                  }
                }.ToIndexDefinition(store.Conventions);
                store.DatabaseCommands.PutIndex("CompanyByName",
                                                definition);

                using (var session = store.OpenSession())
                {
                    session.Store(new Company { Name = "Google" });
                    session.Store(new Company
                    {
                        Name =
                            "HibernatingRhinos"
                    });
                    session.SaveChanges();

                    var company =
                        session.Query<Company>("CompanyByName")
                            .Customize(x=>x.WaitForNonStaleResults())
                            .Where(x=>x.Name == "Google")
                            .FirstOrDefault();

                    Assert.NotNull(company);
                }
            }
        }
 public TableDefinition AddIndex(params string[] columns)
 {
     var def = new IndexDefinition();
     def.AddRange(columns);
     Indexes.Add(def);
     return this;
 }
Example #5
0
		public void CanUseNullCoalescingOperator()
		{
			using (var store = NewDocumentStore())
			{
				var indexDefinition = new IndexDefinition
				{
					Map = "from e in docs.Events select new { Tag = \"Event\", _ = SpatialGenerate(e.Latitude ?? 38.9103000, e.Longitude ?? -77.3942) }",
					Indexes = {
					{ "Tag", FieldIndexing.NotAnalyzed }
				}
				};

				store.DocumentDatabase.PutIndex("eventsByLatLng", indexDefinition);

				store.DocumentDatabase.Put("Events/1", null,
					RavenJObject.Parse(@"{""Venue"": ""Jimmy's Old Town Tavern"", ""Latitude"": null, ""Longitude"": null }"),
					RavenJObject.Parse("{'Raven-Entity-Name': 'Events'}"), null);

				using (var session = store.OpenSession())
				{
					var objects = session.Query<object>("eventsByLatLng")
						.Customize(x => x.WithinRadiusOf(6, 38.9103000, -77.3942))
						.Customize(x => x.WaitForNonStaleResults())
						.ToArray();

					Assert.Empty(store.DocumentDatabase.Statistics.Errors);

					Assert.Equal(1, objects.Length);
				}
			}

		}
Example #6
0
		public void CanPerformSpatialSearch()
		{
			var indexDefinition = new IndexDefinition
			{
				Map = "from e in docs.Events select new { Tag = \"Event\", _ = SpatialGenerate(e.Latitude, e.Longitude) }",
				Indexes = {
					{ "Tag", FieldIndexing.NotAnalyzed }
				}
			};

			db.PutIndex("eventsByLatLng", indexDefinition);

			var events = SpatialIndexTestHelper.GetEvents();

			for (int i = 0; i < events.Length; i++)
			{
				db.Put("Events/" + (i + 1), null,
					RavenJObject.FromObject(events[i]),
					RavenJObject.Parse("{'Raven-Entity-Name': 'Events'}"), null);
			}

			const double lat = 38.96939, lng = -77.386398;
			const double radiusInKm = 6.0*1.609344;
			QueryResult queryResult;
			do
			{
				queryResult = db.Query("eventsByLatLng", new SpatialIndexQuery()
				{
					Query = "Tag:[[Event]]",
					QueryShape = SpatialIndexQuery.GetQueryShapeFromLatLon(lat, lng, radiusInKm),
					SpatialRelation = SpatialRelation.Within,
					SpatialFieldName = Constants.DefaultSpatialFieldName,
					SortedFields = new[] { new SortedField("__distance"), }
				});
				if (queryResult.IsStale)
					Thread.Sleep(100);
			} while (queryResult.IsStale);

			var expected = events.Count(e => Raven.Database.Indexing.SpatialIndex.GetDistance(lat, lng, e.Latitude, e.Longitude) <= radiusInKm);

			Assert.Equal(expected, queryResult.Results.Count);
			Assert.Equal(7, queryResult.Results.Count);

			double previous = 0;
			foreach (var r in queryResult.Results)
			{
				Event e = r.JsonDeserialization<Event>();

				double distance = Raven.Database.Indexing.SpatialIndex.GetDistance(lat, lng, e.Latitude, e.Longitude);
				Console.WriteLine("Venue: " + e.Venue + ", Distance " + distance);

				Assert.True(distance < radiusInKm);
				Assert.True(distance >= previous);
				previous = distance;
			}
		}
Example #7
0
        public IndexHasChanged()
		{
			using (var store = new DocumentStore())
			{
                IndexDefinition indexDefinition = new IndexDefinition();
                #region index_has_changed_2
                store.DatabaseCommands.IndexHasChanged("Orders/Totals", indexDefinition);
				#endregion
			}
		}
Example #8
0
		//Failing test from http://groups.google.com/group/ravendb/browse_thread/thread/7a93f37036297d48/
		public void CanSuccessfullyDoSpatialQueryOfNearbyLocations()
		{
			// These items is in a radius of 4 miles (approx 6,5 km)
			var areaOneDocOne = new DummyGeoDoc(55.6880508001, 13.5717346673);
			var areaOneDocTwo = new DummyGeoDoc(55.6821978456, 13.6076183965);
			var areaOneDocThree = new DummyGeoDoc(55.673251569, 13.5946697607);

			// This item is 12 miles (approx 19 km) from the closest in areaOne 
			var closeButOutsideAreaOne = new DummyGeoDoc(55.8634157297, 13.5497731987);

			// This item is about 3900 miles from areaOne
			var newYork = new DummyGeoDoc(40.7137578228, -74.0126901936);

			using (var documentStore = new EmbeddableDocumentStore { RunInMemory = true }.Initialize())
			using (var session = documentStore.OpenSession())
			{

				session.Store(areaOneDocOne);
				session.Store(areaOneDocTwo);
				session.Store(areaOneDocThree);
				session.Store(closeButOutsideAreaOne);
				session.Store(newYork);
				session.SaveChanges();

				var indexDefinition = new IndexDefinition
				                      	{
				                      		Map = "from doc in docs select new { _ = SpatialIndex.Generate(doc.Latitude, doc.Longitude) }"
				                      	};

				documentStore.DatabaseCommands.PutIndex("FindByLatLng", indexDefinition);

				// Wait until the index is built
				session.Advanced.LuceneQuery<DummyGeoDoc>("FindByLatLng")
					.WaitForNonStaleResults()
					.ToArray();

				const double lat = 55.6836422426, lng = 13.5871808352; // in the middle of AreaOne
				const double radius = 5.0;

				// Expected is that 5.0 will return 3 results
				var nearbyDocs = session.Advanced.LuceneQuery<DummyGeoDoc>("FindByLatLng")
					.WithinRadiusOf(radius, lat, lng)
					.WaitForNonStaleResults()
					.ToArray();

				Assert.NotEqual(null, nearbyDocs);
				Assert.Equal(3, nearbyDocs.Length);

				//TODO
				//var dist = DistanceUtils.GetInstance();
				//Assert.Equal(true, nearbyDocs.All(x => dist.GetDistanceMi(x.Latitude, x.Longitude, lat, lng) < radius));

				session.Dispose();
			}
		}
        public void CanDefineHierarchicalIndexOnTheClient_WithLinq()
        {
            var indexDefinition = new IndexDefinition<Person>
            {
                Map = people => from p in people
                                from c in p.Hierarchy(x=>x.Children)
                                select c.Name
            }.ToIndexDefinition(new DocumentConvention());

            Assert.Equal("docs.People\r\n\t.SelectMany(p => Hierarchy(p, \"Children\"), (p, c) => c.Name)", indexDefinition.Map);
        }
Example #10
0
		public DynamicViewCompiler(string name, IndexDefinition indexDefinition, OrderedPartCollection<AbstractDynamicCompilationExtension> extensions, string basePath, InMemoryRavenConfiguration configuration)
		{
			this.indexDefinition = indexDefinition;
			this.extensions = extensions;
			if (configuration.RunInMemory == false)
			{
				this.basePath = Path.Combine(basePath, "TemporaryIndexDefinitionsAsSource");
				if (Directory.Exists(this.basePath) == false)
					Directory.CreateDirectory(this.basePath);
			}
			this.name = MonoHttpUtility.UrlEncode(name);
		    RequiresSelectNewAnonymousType = true;
		}
Example #11
0
		public void CreateDeleteCreateIndex()
		{
			using (var store = NewDocumentStore(requestedStorage:"esent"))
			{
				var indexDefinition = new IndexDefinition
				{
					Map = "from d in docs select new {}"
				};
				store.DatabaseCommands.PutIndex("test", indexDefinition);
				store.DatabaseCommands.DeleteIndex("test");
				store.DatabaseCommands.PutIndex("test", indexDefinition);
			}
		}
        public override IndexDefinition CreateIndexDefinition()
        {
            var index = new IndexDefinition();
            index.Name = this.IndexName;
            index.Map = @"from doc in docs
            let DocumentType = ((dynamic)doc)[""@metadata""][""Raven-Entity-Name""]
            let Id = doc[""@metadata""][""Id""]
            let LastModified = doc[""@metadata""][""Last-Modified""]
            select new {DocumentType = ((dynamic)doc)[""@metadata""][""Raven-Entity-Name""], Id, LastModified}";

            index.TransformResults = @"from result in results
            select new {Id = result.Id, DocumentType = result.DocumentType, LastModified = result.LastModified}";

            return index;
        }
Example #13
0
        public async Task Index_replication_with_side_by_side_indexes_should_not_propagate_replaced_index_tombstones()
        {
            using (var source = CreateStore())
            using (var destination = CreateStore())
            {
                var oldIndexDef = new IndexDefinition
                {
                    Map = "from person in docs.People\nselect new {\n\tFirstName = person.FirstName\n}"
                };
                var testIndex = new RavenDB_3232.TestIndex();

                var sourceDatabase = await servers[0].Server.GetDatabaseInternal(source.DefaultDatabase);
                sourceDatabase.StopBackgroundWorkers();

                source.DatabaseCommands.PutIndex(testIndex.IndexName, oldIndexDef);

                using (var session = source.OpenSession())
                {
                    session.Store(new RavenDB_3232.Person { FirstName = "John", LastName = "Doe" });
                    session.SaveChanges();
                }
                var sourceReplicationTask = sourceDatabase.StartupTasks.OfType<ReplicationTask>().First();
                sourceReplicationTask.IndexReplication.TimeToWaitBeforeSendingDeletesOfIndexesToSiblings = TimeSpan.FromSeconds(0);

                sourceReplicationTask.Pause(); //pause replciation task _before_ setting up replication

                SetupReplication(source.DatabaseCommands, destination);

                var mre = new ManualResetEventSlim();

                sourceDatabase.Notifications.OnIndexChange += (database, notification) =>
                {
                    if (notification.Type == IndexChangeTypes.SideBySideReplace)
                        mre.Set();
                };

                shouldRecordRequests = true;
                testIndex.SideBySideExecute(source);

                sourceDatabase.SpinBackgroundWorkers();
                WaitForIndexing(source); //now old index should be a tombstone and side-by-side replaced it.
                mre.Wait();
                sourceReplicationTask.IndexReplication.Execute();

                Assert.Equal(0, requestLog.Count(x => x.Method.Method == "DELETE"));
            }
        }
		public void LinqQueryWithStaticCallOnEnumerableIsTranslatedToExtensionMethod()
		{
			var indexDefinition = new IndexDefinition<Page>
			{
				Map = pages => from p in pages
							   from coAuthor in Enumerable.DefaultIfEmpty(p.CoAuthors)
							   select new
							   {
								   p.Id,
								   CoAuthorUserID = coAuthor != null ? coAuthor.UserId : -1
							   }
			}.ToIndexDefinition(new DocumentConvention());
			var expectedMapTranslation =
				@"docs.Pages
	.SelectMany(p => p.CoAuthors.DefaultIfEmpty(), (p, coAuthor) => new {Id = p.Id, CoAuthorUserID = coAuthor != null ? coAuthor.UserId : -1})";
			Assert.Equal(expectedMapTranslation, indexDefinition.Map);
		}
Example #15
0
 public void CanRunSpatialQueriesInMemory()
 {
     var documentStore = new EmbeddableDocumentStore { RunInMemory = true };
     documentStore.Initialize();
     var def = new IndexDefinition<Listing>
     {
         Map = listings => from listingItem in listings
                           select new
                           {
                               listingItem.ClassCodes,
                               listingItem.Latitude,
                               listingItem.Longitude,
                               _ = SpatialIndex.Generate(listingItem.Latitude, listingItem.Longitude)
                           }
     };
     documentStore.DatabaseCommands.PutIndex("RadiusClassifiedSearch", def);
 }
Example #16
0
        private static string TransformGeoIndexes(string value, IndexDefinition definition, DocumentConvention conventions)
        {
            if (value == null)
                return null;

            return Regex.Replace(value, @"GeoIndex\((?<pre>[^.]+)[.](?<prop>[^),]+)(?<remainder>[^)]*)[)]", match =>
            {
                var fieldPrefix = match.Groups["prop"].Value.Replace(".", "_");
                return string.Format("SpatialGenerate(\"{0}_{1}\", {2}.{3}.{1}{4})",
                        match.Groups["prop"].Value.Replace(".", "_"),
                        SpatialField.Name,
                        match.Groups["pre"].Value,
                        match.Groups["prop"].Value,
                        match.Groups["remainder"].Value
                    );
            });
        }
        public void CanProjectIdFromTransformResults()
        {
            using (var store = NewDocumentStore())
            {
            	var indexDefinition = new IndexDefinition<Shipment, Shipment>()
            	                      	{
            	                      		Map = docs => from doc in docs
            	                      		              select new
            	                      		                     	{
            	                      		                     		doc.Id
            	                      		                     	},
            	                      		TransformResults = (database, results)  => from doc in results
            	                      		                                           select new 
            	                      		                                                  	{
            	                      		                                                  		Id = doc.Id,
            	                      		                                                  		Name = doc.Name
            	                      		                                                  	}
                                                   
            	                      	}.ToIndexDefinition(store.Conventions);
            	store.DatabaseCommands.PutIndex(
                    "AmazingIndex",
                    indexDefinition);


                using (var session = store.OpenSession())
                {
                    session.Store(new Shipment()
                    {
                        Id = "shipment1",
                        Name = "Some shipment"
                    });
                    session.SaveChanges();

                    var shipment = session.Query<Shipment>("AmazingIndex")
                        .Customize(x=>x.WaitForNonStaleResults())
                        .Select(x => new Shipment
                        {
                            Id = x.Id,
                            Name = x.Name
                        }).Take(1).SingleOrDefault();
                    
                    Assert.NotNull(shipment.Id);
                }
            }
        }
Example #18
0
        public void DefaultIndexingBehaviourAllowStartsWith()
        {
            using (var store = this.NewDocumentStore())
            {
                var index = new IndexDefinition<Blog, BlogTagItem>()
                {
                    Map = docs => from doc in docs
                                  from tag in doc.Tags
                                  select new
                                  {
                                      tag.Name
                                  },
                    Reduce = results => from result in results
                                        group result by result.Name into g
                                        select new
                                        {
                                            Name = g.Key,
                                            Count = g.Count()
                                        }

                }.ToIndexDefinition(store.Conventions);

                store.DatabaseCommands.PutIndex("TagInfo", index);


                using (var session = store.OpenSession())
                {
                    var newBlog = new Blog()
                    {
                        Tags = new[]{
                             new BlogTag() { Name = "SuperCallaFragalisticExpealadocious" }
                        }
                    };
                    session.Store(newBlog);
                    session.SaveChanges();

                    var result = session.Query<BlogTagItem>("TagInfo")
                        .Customize(x => x.WaitForNonStaleResults())
                        .Where(x => x.Name.StartsWith("Su"))
                        .FirstOrDefault();

                    Assert.NotNull(result);
                }
            }
        }
        public static void CreateIndex(DocumentDatabase database)
        {
            var index = new IndexDefinition {
                                                Map = string.Format(
                                                    @"from doc in docs
            where doc[""{0}""][""{1}""] == ""{2}""
               && doc[""{0}""][""{3}""] == true
            select new
            {{
            {4} = doc[""{0}""][""{5}""],
            }}",
                                                    Constants.Metadata,
                                                    TemporalMetadata.RavenDocumentTemporalStatus, TemporalStatus.Revision,
                                                    TemporalMetadata.RavenDocumentTemporalPending,
                                                    Activation, TemporalMetadata.RavenDocumentTemporalEffectiveStart)
                                            };

            if (database.GetIndexDefinition(TemporalConstants.PendingRevisionsIndex) == null)
                database.PutIndex(TemporalConstants.PendingRevisionsIndex, index);
        }
Example #20
0
        public SpatialSorting()
        {
            store = NewDocumentStore(databaseName:"SpatialSorting");

            var indexDefinition = new IndexDefinition
                {
                    Map = "from e in docs.Shops select new { e.Venue, _ = SpatialGenerate(e.Latitude, e.Longitude) }",
                    Indexes =
                    {
                        {"Tag", FieldIndexing.NotAnalyzed}
                    }
                };

           store.DatabaseCommands.PutIndex("eventsByLatLng", indexDefinition);

           var indexDefinition2 = new IndexDefinition
           {
               Map = "from e in docs.Shops select new { e.Venue, MySpacialField = SpatialGenerate(e.Latitude, e.Longitude) }",
               Indexes =
                    {
                        {"Tag", FieldIndexing.NotAnalyzed}
                    }
           };

           store.DatabaseCommands.PutIndex("eventsByLatLngWSpecialField", indexDefinition2);

           for (int i = 0; i < shops.Length; i++)
           {
               store.DatabaseCommands.Put("Shops/" + (i + 1), null,
                   RavenJObject.FromObject(shops[i]),
                   RavenJObject.Parse("{'Raven-Entity-Name': 'Shops'}"));
           }

           WaitForIndexing(store);

        }
		public async Task CanUpdate()
		{
			using (var agg = new AggregationEngine())
			{
				await agg.InitAsync();

				await agg.CreateAggregationAsync(new IndexDefinition
				{
					Name = "Test",
					Map = "from doc in docs select new { Count = 1}",
					Reduce = "from result in results group result by 1 into g select new { Count = g.Sum(x=>x.Count) }"
				});

				var indexDefinition = new IndexDefinition
					{
						Name = "Test",
						Map = "from doc in docs select new { Sum = 1}",
						Reduce = "from result in results group result by 1 into g select new { Sum = g.Sum(x=>x.Sum) }"
					};
				await agg.CreateAggregationAsync(indexDefinition);

				Assert.Contains(indexDefinition.Reduce, agg.GetAggregation("Test").Generator.ViewText);
			}
		}
Example #22
0
        private Action TryCreateTaskForApplyingPrecomputedBatchForNewIndex(Index index, IndexDefinition definition)
        {
            if (Database.Configuration.MaxPrecomputedBatchSizeForNewIndex <= 0) //precaution -> should never be lower than 0
            {
                index.IsMapIndexingInProgress = false;
                return(null);
            }

            var generator = IndexDefinitionStorage.GetViewGenerator(definition.IndexId);

            if (generator.ForEntityNames.Count == 0 && index.IsTestIndex == false)
            {
                // we don't optimize if we don't have what to optimize _on_, we know this is going to return all docs.
                // no need to try to optimize that, then
                index.IsMapIndexingInProgress = false;
                return(null);
            }

            //only one precomputed batch can run at a time except for test indexes
            if (index.IsTestIndex == false)
            {
                lock (precomputedLock)
                {
                    if (isPrecomputedBatchForNewIndexIsRunning)
                    {
                        index.IsMapIndexingInProgress = false;
                        return(null);
                    }

                    isPrecomputedBatchForNewIndexIsRunning = true;
                }
            }

            try
            {
                var cts  = new CancellationTokenSource();
                var task = new Task(() =>
                {
                    try
                    {
                        ApplyPrecomputedBatchForNewIndex(index, generator,
                                                         index.IsTestIndex == false ?
                                                         Database.Configuration.MaxPrecomputedBatchSizeForNewIndex :
                                                         Database.Configuration.Indexing.MaxNumberOfItemsToProcessInTestIndexes, cts);
                    }
                    catch (TotalDataSizeExceededException e)
                    {
                        //expected error
                        Log.Info(e.Message);
                    }
                    catch (Exception e)
                    {
                        Log.Warn("Could not apply precomputed batch for index " + index, e);
                    }
                    finally
                    {
                        if (index.IsTestIndex == false)
                        {
                            isPrecomputedBatchForNewIndexIsRunning = false;
                        }
                        index.IsMapIndexingInProgress = false;
                        WorkContext.ShouldNotifyAboutWork(() => "Precomputed indexing batch for " + index.PublicName + " is completed");
                        WorkContext.NotifyAboutWork();
                    }
                }, TaskCreationOptions.LongRunning);

                return(() =>
                {
                    try
                    {
                        task.Start();

                        long id;
                        Database
                        .Tasks
                        .AddTask(
                            task,
                            new TaskBasedOperationState(task),
                            new TaskActions.PendingTaskDescription
                        {
                            StartTime = DateTime.UtcNow,
                            Description = index.PublicName,
                            TaskType = TaskActions.PendingTaskType.NewIndexPrecomputedBatch
                        },
                            out id,
                            cts);
                    }
                    catch (Exception)
                    {
                        index.IsMapIndexingInProgress = false;
                        if (index.IsTestIndex == false)
                        {
                            isPrecomputedBatchForNewIndexIsRunning = false;
                        }
                        throw;
                    }
                });
            }
            catch (Exception)
            {
                index.IsMapIndexingInProgress = false;
                if (index.IsTestIndex == false)
                {
                    isPrecomputedBatchForNewIndexIsRunning = false;
                }
                throw;
            }
        }
		private DynamicViewCompiler AddAndCompileIndex(IndexDefinition indexDefinition)
		{
			var name = FixupIndexName(indexDefinition.Name, path);
			var transformer = new DynamicViewCompiler(name, indexDefinition, extensions, path, configuration);
			var generator = transformer.GenerateInstance();
			indexCache.AddOrUpdate(name, generator, (s, viewGenerator) => generator);
			
			logger.Info("New index {0}:\r\n{1}\r\nCompiled to:\r\n{2}", transformer.Name, transformer.CompiledQueryText,
							  transformer.CompiledQueryText);
			return transformer;
		}
Example #24
0
        private JsonDocument RetrieveDocumentInternal(
            IndexQueryResult queryResult,
            HashSet <string> loadedIds,
            FieldsToFetch fieldsToFetch,
            IndexDefinition indexDefinition)
        {
            var queryScore = queryResult.Score;

            if (float.IsNaN(queryScore))
            {
                queryScore = 0f;
            }

            if (queryResult.Projection == null)
            {
                // duplicate document, filter it out
                if (loadedIds.Add(queryResult.Key) == false)
                {
                    return(null);
                }
                var document = GetDocumentWithCaching(queryResult.Key);
                if (document != null)
                {
                    document.Metadata[Constants.TemporaryScoreValue] = queryScore;
                }
                return(document);
            }

            if (fieldsToFetch.IsProjection)
            {
                if (indexDefinition.IsMapReduce == false)
                {
                    bool hasStoredFields = false;
                    foreach (var fieldToFetch in fieldsToFetch)
                    {
                        FieldStorage value;
                        if (indexDefinition.Stores.TryGetValue(fieldToFetch, out value) == false &&
                            value != FieldStorage.No)
                        {
                            continue;
                        }
                        hasStoredFields = true;
                    }
                    if (hasStoredFields == false)
                    {
                        // duplicate document, filter it out
                        if (loadedIds.Add(queryResult.Key) == false)
                        {
                            return(null);
                        }
                    }
                }
                var fieldsToFetchFromDocument = fieldsToFetch.Where(fieldToFetch => queryResult.Projection[fieldToFetch] == null);
                var doc = GetDocumentWithCaching(queryResult.Key);
                if (doc != null)
                {
                    var result = doc.DataAsJson.SelectTokenWithRavenSyntax(fieldsToFetchFromDocument.ToArray());
                    foreach (var property in result)
                    {
                        if (property.Value == null || property.Value.Type == JTokenType.Null)
                        {
                            continue;
                        }
                        queryResult.Projection[property.Key] = property.Value;
                    }
                }
            }

            return(new JsonDocument
            {
                Key = queryResult.Key,
                DataAsJson = queryResult.Projection,
                Metadata = new RavenJObject {
                    { Constants.TemporaryScoreValue, queryScore }
                }
            });
        }
        protected virtual IList <IndexDefinition> ReadIndexes(string schemaName, string tableName)
        {
            string    query = @"SELECT OBJECT_SCHEMA_NAME(T.[object_id],DB_ID()) AS [Schema],  
              T.[name] AS [table_name], I.[name] AS [index_name], AC.[name] AS [column_name],  
              I.[type_desc], I.[is_unique], I.[data_space_id], I.[ignore_dup_key], I.[is_primary_key], 
              I.[is_unique_constraint], I.[fill_factor],    I.[is_padded], I.[is_disabled], I.[is_hypothetical], 
              I.[allow_row_locks], I.[allow_page_locks], IC.[is_descending_key], IC.[is_included_column] 
            FROM sys.[tables] AS T  
              INNER JOIN sys.[indexes] I ON T.[object_id] = I.[object_id]  
              INNER JOIN sys.[index_columns] IC ON I.[object_id] = IC.[object_id] 
              INNER JOIN sys.[all_columns] AC ON T.[object_id] = AC.[object_id] AND IC.[column_id] = AC.[column_id] 
            WHERE T.[is_ms_shipped] = 0 AND I.[type_desc] <> 'HEAP' 
            AND T.object_id = OBJECT_ID('[{0}].[{1}]')
            ORDER BY T.[name], I.[index_id], IC.[key_ordinal]";
            DataSet   ds    = Read(query, schemaName, tableName);
            DataTable dt    = ds.Tables[0];
            IList <IndexDefinition> indexes = new List <IndexDefinition>();

            foreach (DataRow dr in dt.Rows)
            {
                List <IndexDefinition> matches = (from i in indexes
                                                  where i.Name == dr["index_name"].ToString() &&
                                                  i.SchemaName == dr["Schema"].ToString()
                                                  select i).ToList();

                IndexDefinition iDef = null;
                if (matches.Count > 0)
                {
                    iDef = matches[0];
                }

                // create the table if not found
                if (iDef == null)
                {
                    iDef = new IndexDefinition()
                    {
                        Name        = dr["index_name"].ToString(),
                        SchemaName  = dr["Schema"].ToString(),
                        IsClustered = dr["type_desc"].ToString() == "CLUSTERED",
                        IsUnique    = dr["is_unique"].ToString() == "1",
                        TableName   = dr["table_name"].ToString()
                    };
                    indexes.Add(iDef);
                }

                ICollection <IndexColumnDefinition> ms;
                // columns
                ms = (from m in iDef.Columns
                      where m.Name == dr["column_name"].ToString()
                      select m).ToList();
                if (ms.Count == 0)
                {
                    iDef.Columns.Add(new IndexColumnDefinition()
                    {
                        Name      = dr["column_name"].ToString(),
                        Direction = dr["is_descending_key"].ToString() == "1" ? Direction.Descending : Direction.Ascending
                    });
                }
            }

            return(indexes);
        }
 public FaultyIndexDefinition(string name, IEnumerable <string> collections, IndexLockMode lockMode, IndexPriority priority, IndexState state, IndexField[] mapFields, IndexDefinition definition)
     : base(name, collections, lockMode, priority, state, mapFields, IndexVersion.CurrentVersion)
 {
     _definition = definition;
 }
Example #27
0
 public abstract IndexDefinitionCompareDifferences Compare(IndexDefinition indexDefinition);
Example #28
0
 public string DirectPutIndex(string name, OperationMetadata operationMetadata, bool overwrite,
                              IndexDefinition definition)
 {
     return(asyncServerClient.DirectPutIndexAsync(name, definition, overwrite, operationMetadata).Result);
 }
Example #29
0
 public bool IndexHasChanged(string name, IndexDefinition indexDef)
 {
     return(AsyncHelpers.RunSync(() => asyncServerClient.IndexHasChangedAsync(name, indexDef)));
 }
Example #30
0
        private bool CanUpdateIndex(string name, IndexDefinition definition, bool isUpdateBySideSide, IndexDefinition existingIndex)
        {
            switch (existingIndex.LockMode)
            {
            case IndexLockMode.SideBySide:
                if (isUpdateBySideSide == false)
                {
                    Log.Info("Index {0} not saved because it might be only updated by side-by-side index", name);
                    throw new InvalidOperationException("Can not overwrite locked index: " + name + ". This index can be only updated by side-by-side index.");
                }

                //keep the SideBySide lock mode from the replaced index
                definition.LockMode = IndexLockMode.SideBySide;
                break;

            case IndexLockMode.LockedIgnore:
                Log.Info("Index {0} not saved because it was lock (with ignore)", name);
                return(false);

            case IndexLockMode.LockedError:
                throw new InvalidOperationException("Can not overwrite locked index: " + name);
            }

            return(true);
        }
 public void WriteIndex(IndexDefinition indexDefinition)
 {
     AsyncHelpers.RunSync(() => _database.IndexStore.CreateIndex(indexDefinition));
 }
Example #32
0
        private WIndexDefinition ParseIndexDefinition(IndexDefinition idxDef)
        {
            if (idxDef == null)
                return null;
            var wIdxDef = new WIndexDefinition
            {
                FirstTokenIndex = idxDef.FirstTokenIndex,
                LastTokenIndex = idxDef.LastTokenIndex,
                IndexType = idxDef.IndexType,
                Name = idxDef.Name,
            };
            if (idxDef.Columns == null)
                return wIdxDef;

            wIdxDef.Columns = new List<Tuple<WColumnReferenceExpression, SortOrder>>();
            foreach (var col in idxDef.Columns)
            {
                wIdxDef.Columns.Add(new Tuple<WColumnReferenceExpression, SortOrder>(
                    new WColumnReferenceExpression
                    {
                        ColumnType = col.Column.ColumnType,
                        MultiPartIdentifier = ParseMultiPartIdentifier(col.Column.MultiPartIdentifier),
                        FirstTokenIndex = col.Column.FirstTokenIndex,
                        LastTokenIndex = col.Column.LastTokenIndex,
                    },
                    col.SortOrder));
            }
            return wIdxDef;
        }
Example #33
0
        public async Task If_deleted_original_index_on_destination_but_not_side_by_side_index()
        {
            using (var source = CreateStore())
                using (var destination = CreateStore())
                {
                    var sourceDatabase      = await servers[0].Server.GetDatabaseInternal(source.DefaultDatabase);
                    var destinationDatabase = await servers[1].Server.GetDatabaseInternal(destination.DefaultDatabase);
                    sourceDatabase.StopBackgroundWorkers();
                    destinationDatabase.StopBackgroundWorkers();

                    var testIndex = new UserIndex();

                    var oldIndexDef = new IndexDefinition
                    {
                        Map = "from user in docs.Users\n select new {\n\tName = user.Name\n}"
                    };

                    source.DatabaseCommands.PutIndex(testIndex.IndexName, oldIndexDef);

                    using (var session = source.OpenSession())
                    {
                        session.Store(new User
                        {
                            Name = "John Doe"
                        });
                        session.SaveChanges();
                    }

                    var sourceReplicationTask = sourceDatabase.StartupTasks.OfType <ReplicationTask>().First();
                    sourceReplicationTask.Pause();

                    SetupReplication(source.DatabaseCommands, destination);

                    sourceReplicationTask.IndexReplication.Execute(); //replicate the usual index

                    source.SideBySideExecuteIndex(testIndex);

                    //the side by side index will be automatically replicated
                    SpinWait.SpinUntil(() =>
                    {
                        var index = destinationDatabase.Indexes.GetIndexDefinition(Constants.SideBySideIndexNamePrefix + testIndex.IndexName);
                        return(index != null);
                    }, 5000);

                    Assert.NotNull(destinationDatabase.Indexes.GetIndexDefinition(Constants.SideBySideIndexNamePrefix + testIndex.IndexName));

                    destinationDatabase.Indexes.DeleteIndex(testIndex.IndexName); //delete the original index

                    var sideBySideIndex = destinationDatabase.Indexes.GetIndexDefinition(Constants.SideBySideIndexNamePrefix + testIndex.IndexName);
                    Assert.NotNull(sideBySideIndex);

                    VerifyReplacementDocumentIsThere(Constants.SideBySideIndexNamePrefix + testIndex.IndexName, destinationDatabase);

                    sourceReplicationTask.IndexReplication.Execute();

                    var indexDefinition = destinationDatabase.Indexes.Definitions.First(x => x.Name == Constants.SideBySideIndexNamePrefix + testIndex.IndexName);
                    destinationDatabase.IndexReplacer.ForceReplacement(indexDefinition);

                    //wait until the index will be replaced
                    SpinWait.SpinUntil(() =>
                    {
                        var index = destinationDatabase.Indexes.GetIndexDefinition(testIndex.IndexName);
                        return(index != null);
                    }, 5000);

                    var replacingIndex = destinationDatabase.Indexes.GetIndexDefinition(testIndex.IndexName);
                    Assert.True(replacingIndex != null);

                    var oldIndex = destinationDatabase.Indexes.GetIndexDefinition(testIndex.IndexName);
                    Assert.True(oldIndex.Equals(testIndex.CreateIndexDefinition(), false));

                    sideBySideIndex = destinationDatabase.Indexes.GetIndexDefinition(Constants.SideBySideIndexNamePrefix + testIndex.IndexName);
                    Assert.Null(sideBySideIndex);

                    SpinWait.SpinUntil(() =>
                    {
                        var doc = destinationDatabase.Documents.Get(Constants.IndexReplacePrefix + Constants.SideBySideIndexNamePrefix + testIndex.IndexName, null);
                        return(doc == null);
                    }, 5000);

                    Assert.Null(destinationDatabase.Documents.Get(Constants.IndexReplacePrefix + Constants.SideBySideIndexNamePrefix + testIndex.IndexName, null));
                }
        }
Example #34
0
        public async Task Side_by_side_index_after_replication_should_have_appropriate_minimum_etag_for_the_destination_if_applicable()
        {
            using (var source = CreateStore())
                using (var destination = CreateStore())
                {
                    var testIndex   = new UserIndex();
                    var oldIndexDef = new IndexDefinition
                    {
                        Map = "from user in docs.Users\n select new {\n\tName = user.Name\n}"
                    };
                    source.DatabaseCommands.PutIndex(testIndex.IndexName, oldIndexDef);

                    var sourceDatabase      = await servers[0].Server.GetDatabaseInternal(source.DefaultDatabase);
                    var destinationDatabase = await servers[1].Server.GetDatabaseInternal(destination.DefaultDatabase);

                    using (var session = source.OpenSession())
                    {
                        for (int i = 0; i < 10; i++)
                        {
                            session.Store(new User
                            {
                                Name = "User - " + i
                            });
                        }

                        session.SaveChanges();
                    }

                    WaitForIndexing(source);
                    sourceDatabase.StopBackgroundWorkers();
                    destinationDatabase.StopBackgroundWorkers();

                    var sourceReplicationTask = sourceDatabase.StartupTasks.OfType <ReplicationTask>().First();
                    sourceReplicationTask.Pause();

                    SetupReplication(source.DatabaseCommands, destination);

                    //replicate the original index
                    await sourceReplicationTask.ExecuteReplicationOnce(true);

                    sourceReplicationTask.IndexReplication.Execute();

                    destinationDatabase.SpinBackgroundWorkers();

                    WaitForIndexing(destination);

                    destinationDatabase.StopBackgroundWorkers();

                    testIndex.SideBySideExecute(source, sourceDatabase.Statistics.LastDocEtag);

                    var replacementIndexName = Constants.SideBySideIndexNamePrefix + testIndex.IndexName;

                    //do side-by-side index replication -> since in the destination there is original index,
                    //simply create the side-by-side index as so it will replace the original when it catches up
                    sourceReplicationTask.IndexReplication.Execute();

                    var originalDefinition = destination.DatabaseCommands.GetIndex(testIndex.IndexName);
                    Assert.NotNull(originalDefinition);
                    Assert.True(originalDefinition.Equals(oldIndexDef, false));

                    var sideBySideDefinition = destination.DatabaseCommands.GetIndex(replacementIndexName);
                    Assert.NotNull(sideBySideDefinition);
                    Assert.True(sideBySideDefinition.Equals(testIndex.CreateIndexDefinition(), false));

                    VerifyReplacementDocumentIsThere(replacementIndexName, destinationDatabase, true);
                }
        }
Example #35
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: public static int[] getOrCreatePropertyKeyIds(org.neo4j.internal.kernel.api.TokenWrite tokenWrite, org.neo4j.graphdb.schema.IndexDefinition indexDefinition) throws org.neo4j.internal.kernel.api.exceptions.schema.IllegalTokenNameException
        public static int[] GetOrCreatePropertyKeyIds(TokenWrite tokenWrite, IndexDefinition indexDefinition)
        {
            return(GetOrCreatePropertyKeyIds(tokenWrite, GetPropertyKeysArrayOf(indexDefinition)));
        }
        private CreateCoveringIndexResult TryCreateCoveringIndex(DAL.Contracts.StatementQuery query, StatementQueryExtractedData queryExtractedData, IndexDefinition baseIndex, out IndexDefinition coveringIndex)
        {
            var allWhereAttributes      = queryExtractedData.WhereAttributes.All.Where(x => x.Relation.ID == baseIndex.Relation.ID);
            var bTreeWhereAttributes    = queryExtractedData.WhereAttributes.BTreeApplicable.Where(x => x.Relation.ID == baseIndex.Relation.ID);
            var allJoinAttributes       = queryExtractedData.JoinAttributes.All.Where(x => x.Relation.ID == baseIndex.Relation.ID);
            var bTreeJoinAttributes     = queryExtractedData.JoinAttributes.BTreeApplicable.Where(x => x.Relation.ID == baseIndex.Relation.ID);
            var allGroupByAttributes    = queryExtractedData.GroupByAttributes.All.Where(x => x.Relation.ID == baseIndex.Relation.ID);
            var bTreeGroupByAttributes  = queryExtractedData.GroupByAttributes.BTreeApplicable.Where(x => x.Relation.ID == baseIndex.Relation.ID);
            var allOrderByAttributes    = queryExtractedData.OrderByAttributes.All.Where(x => x.Relation.ID == baseIndex.Relation.ID);
            var bTreeOrderByAttributes  = queryExtractedData.OrderByAttributes.BTreeApplicable.Where(x => x.Relation.ID == baseIndex.Relation.ID);
            var allProjectionAttributes = queryExtractedData.ProjectionAttributes.All.Where(x => x.Relation.ID == baseIndex.Relation.ID);

            if (allWhereAttributes.Count() == bTreeWhereAttributes.Count() &&
                allJoinAttributes.Count() == bTreeJoinAttributes.Count() &&
                allGroupByAttributes.Count() == bTreeGroupByAttributes.Count() &&
                allOrderByAttributes.Count() == bTreeOrderByAttributes.Count() &&
                query.CommandType != DAL.Contracts.StatementQueryCommandType.Insert)
            {
                var includeAttributes = new HashSet <AttributeData>(bTreeWhereAttributes);
                includeAttributes.UnionWith(bTreeJoinAttributes);
                includeAttributes.UnionWith(bTreeGroupByAttributes);
                includeAttributes.UnionWith(bTreeOrderByAttributes);
                includeAttributes.UnionWith(allProjectionAttributes);
                includeAttributes.ExceptWith(baseIndex.Attributes);
                List <AttributeData> includeSortedAttributes = new List <AttributeData>(includeAttributes.OrderBy(x => x.CardinalityIndicator));
                coveringIndex = new IndexDefinition(baseIndex.StructureType, baseIndex.Relation, baseIndex.Attributes, includeSortedAttributes);
                return(baseIndex.IncludeAttributes.Count != coveringIndex.IncludeAttributes.Count ? CreateCoveringIndexResult.CreatedNew : CreateCoveringIndexResult.CreatedSameAsBaseIndex);
            }
            else
            {
                coveringIndex = null;
                return(CreateCoveringIndexResult.NotPossible);
            }
        }
Example #37
0
 public FaultyInMemoryIndex(Exception e, string name, IndexingConfiguration configuration, IndexDefinition definition)
     : this(e, configuration, new FaultyIndexDefinition(name, new HashSet <string> { "@FaultyIndexes" }, IndexLockMode.Unlock, IndexPriority.Normal, IndexState.Normal, new IndexField[0], definition))
 {
 }
		public IndexCreationOptions FindIndexCreationOptions(IndexDefinition indexDef)
		{
			var indexDefinition = GetIndexDefinition(indexDef.Name);
			if (indexDefinition != null)
			{
				return indexDefinition.Equals(indexDef)
					? IndexCreationOptions.Noop
					: IndexCreationOptions.Update;
			}
			return IndexCreationOptions.Create;
		}
Example #39
0
 public string PutIndex(string name, IndexDefinition definition, bool overwrite)
 {
     return(AsyncHelpers.RunSync(() => asyncServerClient.PutIndexAsync(name, definition, overwrite)));
 }
		private void ReadIndexesFromCatalog(IEnumerable<AbstractViewGenerator> compiledGenerators, ITransactionalStorage transactionalStorage)
		{
			foreach (var generator in compiledGenerators)
			{
				var copy = generator;
				var displayNameAtt = TypeDescriptor.GetAttributes(copy)
					.OfType<DisplayNameAttribute>()
					.FirstOrDefault();

				var name = displayNameAtt != null ? displayNameAtt.DisplayName : copy.GetType().Name;

				transactionalStorage.Batch(actions =>
				{
					if (actions.Indexing.GetIndexesStats().Any(x => x.Name == name))
						return;

					actions.Indexing.AddIndex(name, copy.ReduceDefinition != null);
				});

				var indexDefinition = new IndexDefinition
				{
					Name = name,
					Map = "Compiled map function: " + generator.GetType().AssemblyQualifiedName,
					// need to supply this so the index storage will create map/reduce index
					Reduce = generator.ReduceDefinition == null ? null : "Compiled reduce function: " + generator.GetType().AssemblyQualifiedName,
					Indexes = generator.Indexes,
					Stores = generator.Stores,
					IsCompiled = true
				};
				indexCache.AddOrUpdate(name, copy, (s, viewGenerator) => copy);
				indexDefinitions.AddOrUpdate(name, indexDefinition, (s1, definition) => indexDefinition);
			}
		}
 public override IndexDefinitionCompareDifferences Compare(IndexDefinition indexDefinition)
 {
     return(_definition.Compare(indexDefinition));
 }
Example #42
0
 public DynamicViewCompiler(string name, IndexDefinition indexDefinition, AbstractDynamicCompilationExtension[] extensions)
 {
     this.indexDefinition = indexDefinition;
     this.extensions      = extensions;
     this.name            = MonoHttpUtility.UrlEncode(name);
 }
Example #43
0
        public static void ValidateReduceResultsCollectionName(IndexDefinition definition, StaticIndexBase index, DocumentDatabase database)
        {
            var outputReduceToCollection = definition.OutputReduceToCollection;

            if (string.IsNullOrWhiteSpace(outputReduceToCollection))
            {
                return;
            }

            var collections = index.Maps.Keys.ToHashSet(StringComparer.OrdinalIgnoreCase);

            if (collections.Contains(Constants.Documents.Collections.AllDocumentsCollection))
            {
                throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " +
                                                $"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " +
                                                $"as this index is mapping all documents " +
                                                $"and this will result in an infinite loop.");
            }

            foreach (var referencedCollection in index.ReferencedCollections)
            {
                foreach (var collectionName in referencedCollection.Value)
                {
                    collections.Add(collectionName.Name);
                }
            }
            if (collections.Contains(outputReduceToCollection))
            {
                throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " +
                                                $"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " +
                                                $"as this index is mapping or referencing the '{outputReduceToCollection}' collection " +
                                                $"and this will result in an infinite loop.");
            }

            var indexes = database.IndexStore.GetIndexes()
                          .Where(x => x.Type == IndexType.MapReduce)
                          .Cast <MapReduceIndex>()
                          .Where(mapReduceIndex => string.IsNullOrWhiteSpace(mapReduceIndex.Definition.OutputReduceToCollection) == false &&
                                 mapReduceIndex.Name != definition.Name)
                          .ToList();

            foreach (var otherIndex in indexes)
            {
                if (otherIndex.Definition.OutputReduceToCollection.Equals(outputReduceToCollection, StringComparison.OrdinalIgnoreCase))
                {
                    var sideBySideIndex = definition.Name.StartsWith(Constants.Documents.Indexing.SideBySideIndexNamePrefix, StringComparison.OrdinalIgnoreCase);
                    if (sideBySideIndex)
                    {
                        throw new IndexInvalidException($"In order to create the '{definition.Name}' side by side index " +
                                                        $"you firstly need to set {nameof(IndexDefinition.OutputReduceToCollection)} to be null " +
                                                        $"on the '{otherIndex.Name}' index " +
                                                        $"and than delete all of the documents in the '{otherIndex.Definition.OutputReduceToCollection}' collection.");
                    }

                    throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " +
                                                    $"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " +
                                                    $"as there is another index named '{otherIndex.Name}' " +
                                                    $"which also output reduce results to documents in the same '{outputReduceToCollection}' collection. " +
                                                    $"{nameof(IndexDefinition.OutputReduceToCollection)} must by set to unique value for each index or be null.");
                }

                var otherIndexCollections = otherIndex.Collections;
                foreach (var referencedCollection in otherIndex.GetReferencedCollections())
                {
                    foreach (var collectionName in referencedCollection.Value)
                    {
                        otherIndexCollections.Add(collectionName.Name);
                    }
                }
                if (otherIndexCollections.Contains(outputReduceToCollection) &&
                    CheckIfThereIsAnIndexWhichWillOutputReduceDocumentsWhichWillBeUsedAsMapOnTheSpecifiedIndex(otherIndex, collections, indexes, out string description))
                {
                    description += Environment.NewLine + $"--> {definition.Name}: {string.Join(",", collections)} => *{outputReduceToCollection}*";
                    throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " +
                                                    $"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " +
                                                    $"as '{outputReduceToCollection}' collection is consumed by other index in a way that would " +
                                                    $"lead to an infinite loop." +
                                                    Environment.NewLine + description);
                }
            }

            using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
                using (context.OpenReadTransaction())
                {
                    var stats = database.DocumentsStorage.GetCollection(outputReduceToCollection, context);
                    if (stats.Count > 0)
                    {
                        throw new IndexInvalidException($"In order to create the '{definition.Name}' index " +
                                                        $"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " +
                                                        $"you firstly need to delete all of the documents in the '{stats.Name}' collection " +
                                                        $"(currently have {stats.Count} document{(stats.Count == 1 ? "" : "s")}).");
                    }
                }
        }
Example #44
0
 public string PutIndex <TDocument, TReduceResult>(string name, IndexDefinition <TDocument, TReduceResult> indexDef)
 {
     return(PutIndex(name, indexDef.ToIndexDefinition(convention)));
 }
Example #45
0
        private string PutIndexInternal(string name, IndexDefinition definition, bool disableIndexBeforePut = false,
                                        bool isUpdateBySideSide = false, IndexCreationOptions?creationOptions = null, bool isReplication = false)
        {
            if (name == null)
            {
                throw new ArgumentNullException(nameof(name));
            }

            name = name.Trim();
            IsIndexNameValid(name);

            var existingIndex = IndexDefinitionStorage.GetIndexDefinition(name);

            if (existingIndex != null)
            {
                var newIndexVersion     = definition.IndexVersion;
                var currentIndexVersion = existingIndex.IndexVersion;

                // whether we update the index definition or not,
                // we need to update the index version
                existingIndex.IndexVersion = definition.IndexVersion =
                    Math.Max(currentIndexVersion ?? 0, newIndexVersion ?? 0);

                switch (isReplication)
                {
                case true:
                    if (newIndexVersion != null && currentIndexVersion != null &&
                        newIndexVersion <= currentIndexVersion)
                    {
                        //this new index is an older version of the current one
                        return(null);
                    }

                    // we need to update the lock mode only if it was updated by another server
                    existingIndex.LockMode = definition.LockMode;
                    break;

                default:
                    if (CanUpdateIndex(name, definition, isUpdateBySideSide, existingIndex) == false)
                    {
                        return(null);
                    }
                    break;
                }
            }

            AssertAnalyzersValid(definition);

            switch (creationOptions ?? FindIndexCreationOptions(definition, ref name))
            {
            case IndexCreationOptions.Noop:
                return(null);

            case IndexCreationOptions.UpdateWithoutUpdatingCompiledIndex:
                // ensure that the code can compile
                new DynamicViewCompiler(definition.Name, definition, Database.Extensions, IndexDefinitionStorage.IndexDefinitionsPath, Database.Configuration).GenerateInstance();
                IndexDefinitionStorage.UpdateIndexDefinitionWithoutUpdatingCompiledIndex(definition);
                if (isReplication == false)
                {
                    definition.IndexVersion = (definition.IndexVersion ?? 0) + 1;
                }
                return(null);

            case IndexCreationOptions.Update:
                // ensure that the code can compile
                new DynamicViewCompiler(definition.Name, definition, Database.Extensions, IndexDefinitionStorage.IndexDefinitionsPath, Database.Configuration).GenerateInstance();
                DeleteIndex(name);
                if (isReplication == false)
                {
                    definition.IndexVersion = (definition.IndexVersion ?? 0) + 1;
                }
                break;

            case IndexCreationOptions.Create:
                if (isReplication == false)
                {
                    // we create a new index,
                    // we need to restore its previous IndexVersion (if it was deleted before)
                    var deletedIndexVersion   = IndexDefinitionStorage.GetDeletedIndexVersion(definition);
                    var replacingIndexVersion = GetOriginalIndexVersion(definition.Name);
                    definition.IndexVersion = Math.Max(deletedIndexVersion, replacingIndexVersion);
                    definition.IndexVersion++;
                }

                break;
            }

            PutNewIndexIntoStorage(name, definition, disableIndexBeforePut);

            WorkContext.ClearErrorsFor(name);

            TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() => Database.Notifications.RaiseNotifications(new IndexChangeNotification
            {
                Name    = name,
                Type    = IndexChangeTypes.IndexAdded,
                Version = definition.IndexVersion
            }));

            return(name);
        }
Example #46
0
 public string PutIndex(string name, IndexDefinition definition, bool isReplication = false)
 {
     return(PutIndexInternal(name, definition, isReplication: isReplication));
 }
Example #47
0
 public JsonDocument RetrieveDocumentForQuery(IndexQueryResult queryResult, IndexDefinition indexDefinition, FieldsToFetch fieldsToFetch)
 {
     return(ExecuteReadTriggers(ProcessReadVetoes(
                                    RetrieveDocumentInternal(queryResult, loadedIdsForRetrieval, fieldsToFetch, indexDefinition),
                                    null, ReadOperation.Query), null, ReadOperation.Query));
 }
Example #48
0
        public void ReplaceOfNonStaleIndex()
        {
            var path = NewDataPath();

            using (var store = GetDocumentStore(new Options
            {
                Path = path
            }))
            {
                var oldIndexDef = new IndexDefinition
                {
                    Name = "TestIndex",
                    Maps = { "from person in docs.People\nselect new {\n\tFirstName = person.FirstName\n}" }
                };

                store.Maintenance.Send(new PutIndexesOperation(oldIndexDef));

                using (var session = store.OpenSession())
                {
                    session.Store(new Person {
                        FirstName = "John", LastName = "Doe"
                    });
                    session.SaveChanges();
                }

                WaitForIndexing(store);

                store.Maintenance.Send(new StopIndexingOperation());

                new TestIndex().Execute(store);

                var e = Assert.Throws <RavenException>(() =>
                {
                    using (var session = store.OpenSession())
                    {
                        var count = session.Query <Person, TestIndex>()
                                    .Count(x => x.LastName == "Doe");
                    }
                });

                Assert.Contains("The field 'LastName' is not indexed, cannot query/sort on fields that are not indexed", e.InnerException.Message);

                var mre = new ManualResetEventSlim();

                var changes    = AsyncHelpers.RunSync(() => store.Changes().EnsureConnectedNow());
                var observable = changes.ForAllIndexes();
                AsyncHelpers.RunSync(() => observable.EnsureSubscribedNow());
                observable.Subscribe(change =>
                {
                    if (change.Type == IndexChangeTypes.SideBySideReplace)
                    {
                        mre.Set();
                    }
                });

                store.Maintenance.Send(new StartIndexingOperation());

                WaitForIndexing(store);

                Assert.True(mre.Wait(TimeSpan.FromSeconds(15)));

                using (var session = store.OpenSession())
                {
                    var count = session.Query <Person, TestIndex>()
                                .Count(x => x.LastName == "Doe");

                    Assert.Equal(1, count);
                }
            }
        }
Example #49
0
        private static CompilationResult CompileInternal(string originalName, string cSharpSafeName, MemberDeclarationSyntax @class, IndexDefinition definition)
        {
            var name = cSharpSafeName + "." + Guid.NewGuid() + IndexExtension;

            var @namespace = RoslynHelper.CreateNamespace(IndexNamespace)
                             .WithMembers(SyntaxFactory.SingletonList(@class));

            var res = GetUsingDirectiveAndSyntaxTreesAndReferences(definition);

            var compilationUnit = SyntaxFactory.CompilationUnit()
                                  .WithUsings(RoslynHelper.CreateUsings(res.UsingDirectiveSyntaxes))
                                  .WithMembers(SyntaxFactory.SingletonList <MemberDeclarationSyntax>(@namespace))
                                  .NormalizeWhitespace();

            SyntaxNode formattedCompilationUnit;

            using (var workspace = new AdhocWorkspace())
            {
                formattedCompilationUnit = Formatter.Format(compilationUnit, workspace);
            }

            string sourceFile = null;

            if (EnableDebugging)
            {
                sourceFile = Path.Combine(Path.GetTempPath(), name + ".cs");
                File.WriteAllText(sourceFile, formattedCompilationUnit.ToFullString(), Encoding.UTF8);
            }

            var st = EnableDebugging
                ? SyntaxFactory.ParseSyntaxTree(File.ReadAllText(sourceFile), path: sourceFile, encoding: Encoding.UTF8)
                : SyntaxFactory.ParseSyntaxTree(formattedCompilationUnit.ToFullString());

            res.SyntaxTrees.Add(st);

            var compilation = CSharpCompilation.Create(
                assemblyName: name,
                syntaxTrees: res.SyntaxTrees,
                references: res.References,
                options: new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary)
                .WithOptimizationLevel(EnableDebugging ? OptimizationLevel.Debug : OptimizationLevel.Release)
                );

            var code = formattedCompilationUnit.SyntaxTree.ToString();

            var asm = new MemoryStream();
            var pdb = EnableDebugging ? new MemoryStream() : null;

            var result = compilation.Emit(asm, pdb, options: new EmitOptions(debugInformationFormat: DebugInformationFormat.PortablePdb));

            if (result.Success == false)
            {
                IEnumerable <Diagnostic> failures = result.Diagnostics
                                                    .Where(diagnostic => diagnostic.IsWarningAsError || diagnostic.Severity == DiagnosticSeverity.Error);

                var sb = new StringBuilder();
                sb.AppendLine($"Failed to compile index {originalName}");
                sb.AppendLine();
                sb.AppendLine(code);
                sb.AppendLine();

                foreach (var diagnostic in failures)
                {
                    sb.AppendLine(diagnostic.ToString());
                }

                throw new IndexCompilationException(sb.ToString());
            }

            asm.Position = 0;

            Assembly assembly;

            if (EnableDebugging)
            {
                pdb.Position = 0;
                assembly     = AssemblyLoadContext.Default.LoadFromStream(asm, pdb);
            }
            else
            {
                assembly = AssemblyLoadContext.Default.LoadFromStream(asm);
            }

            return(new CompilationResult
            {
                Code = code,
                Type = assembly.GetType($"{IndexNamespace}.{cSharpSafeName}")
            });
        }
Example #50
0
        internal void PutNewIndexIntoStorage(string name, IndexDefinition definition, bool disableIndex = false)
        {
            Debug.Assert(Database.IndexStorage != null);
            Debug.Assert(TransactionalStorage != null);
            Debug.Assert(WorkContext != null);

            Index index = null;

            TransactionalStorage.Batch(actions =>
            {
                var maxId = 0;
                if (Database.IndexStorage.Indexes.Length > 0)
                {
                    maxId = Database.IndexStorage.Indexes.Max();
                }
                definition.IndexId = (int)Database.Documents.GetNextIdentityValueWithoutOverwritingOnExistingDocuments("IndexId", actions);
                if (definition.IndexId <= maxId)
                {
                    actions.General.SetIdentityValue("IndexId", maxId + 1);
                    definition.IndexId = (int)Database.Documents.GetNextIdentityValueWithoutOverwritingOnExistingDocuments("IndexId", actions);
                }

                IndexDefinitionStorage.RegisterNewIndexInThisSession(name, definition);

                // this has to happen in this fashion so we will expose the in memory status after the commit, but
                // before the rest of the world is notified about this.

                IndexDefinitionStorage.CreateAndPersistIndex(definition);
                Database.IndexStorage.CreateIndexImplementation(definition);
                index = Database.IndexStorage.GetIndexInstance(definition.IndexId);

                // If we execute multiple indexes at once and want to activate them all at once we will disable the index from the endpoint
                if (disableIndex)
                {
                    index.Priority = IndexingPriority.Disabled;
                }

                //ensure that we don't start indexing it right away, let the precomputation run first, if applicable
                index.IsMapIndexingInProgress = true;
                if (definition.IsTestIndex)
                {
                    index.MarkQueried(); // test indexes should be mark queried, so the cleanup task would not delete them immediately
                }
                InvokeSuggestionIndexing(name, definition, index);

                actions.Indexing.AddIndex(definition.IndexId, definition.IsMapReduce);
            });

            Debug.Assert(index != null);

            Action precomputedTask = null;

            if (WorkContext.RunIndexing &&
                name.Equals(Constants.DocumentsByEntityNameIndex, StringComparison.InvariantCultureIgnoreCase) == false &&
                Database.IndexStorage.HasIndex(Constants.DocumentsByEntityNameIndex) && isPrecomputedBatchForNewIndexIsRunning == false)
            {
                // optimization of handling new index creation when the number of document in a database is significantly greater than
                // number of documents that this index applies to - let us use built-in RavenDocumentsByEntityName to get just appropriate documents

                precomputedTask = TryCreateTaskForApplyingPrecomputedBatchForNewIndex(index, definition);
            }
            else
            {
                index.IsMapIndexingInProgress = false;// we can't apply optimization, so we'll make it eligible for running normally
            }

            // The act of adding it here make it visible to other threads
            // we have to do it in this way so first we prepare all the elements of the
            // index, then we add it to the storage in a way that make it public
            IndexDefinitionStorage.AddIndex(definition.IndexId, definition);

            // we start the precomputedTask _after_ we finished adding the index
            precomputedTask?.Invoke();

            WorkContext.ShouldNotifyAboutWork(() => "PUT INDEX " + name);
            WorkContext.NotifyAboutWork();
        }
Example #51
0
        void InsertOrModifyEventSourceIdAndVersionIndex()
        {
            var alreadyExists = true;
            var updated = false;
            var indexName = "Temp/Events/ByEventSourceIdAndVersionSortByVersion";
            var index = _documentStore.DatabaseCommands.GetIndex(indexName);
            if (index == null)
            {
                index = new IndexDefinition
                {
                    Map = "from doc in docs.Events select new { EventSourceId = doc.EventSourceId, Version = doc.Version }",
                    Fields = new List<string> { "EventSourceId", "Version", "__document_id" }
                };
                alreadyExists = false;
            }

            if (alreadyExists && index.SortOptions.First().Value != SortOptions.Double)
            {
                _documentStore.DatabaseCommands.DeleteIndex(indexName);
                updated = true;
            }

            index.SortOptions = new Dictionary<string, SortOptions> { { "Version", SortOptions.Double } };

            if (!alreadyExists || updated)
                _documentStore.DatabaseCommands.PutIndex(indexName, index);
        }
Example #52
0
        private static (UsingDirectiveSyntax[] UsingDirectiveSyntaxes, List <SyntaxTree> SyntaxTrees, MetadataReference[] References) GetUsingDirectiveAndSyntaxTreesAndReferences(IndexDefinition definition)
        {
            if (definition.AdditionalSources == null && definition.AdditionalSources == null)
            {
                return(Usings, new List <SyntaxTree>(), References);
            }

            (UsingDirectiveSyntax[] UsingDirectiveSyntaxes, List <SyntaxTree> SyntaxTrees, MetadataReference[] References)result;
            var syntaxTrees = new List <SyntaxTree>();
            var usings      = new HashSet <string>();

            if (definition.AdditionalSources != null)
            {
                foreach (var ext in definition.AdditionalSources)
                {
                    var tree = SyntaxFactory.ParseSyntaxTree(AddUsingIndexStatic(ext.Value));
                    syntaxTrees.Add(tree);

                    var ns = tree.GetRoot().DescendantNodes()
                             .OfType <NamespaceDeclarationSyntax>()
                             .FirstOrDefault();

                    if (ns != null)
                    {
                        usings.Add(ns.Name.ToString());
                    }
                }
            }

            if (definition.AdditionalAssemblies != null)
            {
                foreach (var additionalAssembly in definition.AdditionalAssemblies)
                {
                    if (additionalAssembly.Usings != null)
                    {
                        foreach (var @using in additionalAssembly.Usings)
                        {
                            usings.Add(@using);
                        }
                    }
                }
            }

            if (usings.Count > 0)
            {
                //Adding using directive with duplicates to avoid O(n*m) operation and confusing code
                var newUsing = usings.Select(x => SyntaxFactory.UsingDirective(SyntaxFactory.ParseName(x))).ToList();
                newUsing.AddRange(Usings);
                result.UsingDirectiveSyntaxes = newUsing.ToArray();
            }
            else
            {
                result.UsingDirectiveSyntaxes = Usings;
            }

            result.References = GetReferences(definition);

            var tempCompilation = CSharpCompilation.Create(
                assemblyName: string.Empty,
                syntaxTrees: syntaxTrees,
                references: result.References,
                options: new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary)
                .WithOptimizationLevel(EnableDebugging ? OptimizationLevel.Debug : OptimizationLevel.Release)
                );

            var rewriter = new MethodDynamicParametersRewriter();

            result.SyntaxTrees = new List <SyntaxTree>();

            foreach (var tree in syntaxTrees) //now do the rewrites
            {
                rewriter.SemanticModel = tempCompilation.GetSemanticModel(tree);

                var rewritten = rewriter.Visit(tree.GetRoot()).NormalizeWhitespace();
                result.SyntaxTrees.Add(SyntaxFactory.SyntaxTree(rewritten, new CSharpParseOptions(documentationMode: DocumentationMode.None)));
            }

            return(result);
        }
		public string CreateAndPersistIndex(IndexDefinition indexDefinition)
		{
			var transformer = AddAndCompileIndex(indexDefinition);
			if (configuration.RunInMemory == false)
			{
				var encodeIndexNameIfNeeded = FixupIndexName(indexDefinition.Name, path);
				var indexName = Path.Combine(path, MonoHttpUtility.UrlEncode(encodeIndexNameIfNeeded) + ".index");
				// Hash the name if it's too long (as a path)
				File.WriteAllText(indexName, JsonConvert.SerializeObject(indexDefinition, Formatting.Indented, Default.Converters));
			}
			return transformer.Name;
		}
Example #54
0
 public override IndexDefinitionCompareDifferences Compare(IndexDefinition indexDefinition)
 {
     throw new NotImplementedException();
 }
		public void AddIndex(string name, IndexDefinition definition)
		{
			indexDefinitions.AddOrUpdate(name, definition, (s1, def) =>
			{
				if (def.IsCompiled)
					throw new InvalidOperationException("Index " + name + " is a compiled index, and cannot be replaced");
				return definition;
			});
		}
Example #56
0
 public void Init(IndexDefinition definition)
 {
     indexDefinition = definition;
 }
		public static void ResolveAnalyzers(IndexDefinition indexDefinition)
		{
			// Stick Lucene.Net's namespace to all analyzer aliases that are missing a namespace
			var analyzerNames = (from analyzer in indexDefinition.Analyzers
								 where analyzer.Value.IndexOf('.') == -1
								 select analyzer).ToArray();

			// Only do this for analyzer that actually exist; we do this here to be able to throw a correct error later on
			foreach (var a in analyzerNames.Where(a => typeof(StandardAnalyzer).Assembly.GetType("Lucene.Net.Analysis." + a.Value) != null))
			{
				indexDefinition.Analyzers[a.Key] = "Lucene.Net.Analysis." + a.Value;
			}
		}
        public static Sort GetSort(this IndexQuery self, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator)
        {
            var spatialQuery = self as SpatialIndexQuery;
            var sortedFields = self.SortedFields;

            if (sortedFields == null || sortedFields.Length <= 0)
            {
                if (spatialQuery == null || string.IsNullOrEmpty(self.Query) == false)
                {
                    return(null);
                }
                sortedFields = new[] { new SortedField(Constants.DistanceFieldName), };
            }

            return(new Sort(sortedFields
                            .Select(sortedField =>
            {
                if (sortedField.Field == Constants.TemporaryScoreValue)
                {
                    return SortField.FIELD_SCORE;
                }
                if (sortedField.Field.StartsWith(Constants.RandomFieldName))
                {
                    var parts = sortedField.Field.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
                    if (parts.Length < 2)                                                     // truly random
                    {
                        return new RandomSortField(Guid.NewGuid().ToString());
                    }
                    return new RandomSortField(parts[1]);
                }
                if (sortedField.Field.StartsWith(Constants.CustomSortFieldName))
                {
                    var parts = sortedField.Field.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
                    if (parts.Length < 2)                                                     // truly random
                    {
                        throw new InvalidOperationException("Cannot figure out type for custom sort");
                    }
                    return new CustomSortField(parts[1], self, parts[0][parts[0].Length - 1] == '-');
                }
                if (sortedField.Field.StartsWith(Constants.DistanceFieldName))
                {
                    SpatialField spatialField = null;
                    Shape shape;

                    if (sortedField.Field.Length == Constants.DistanceFieldName.Length)
                    {
                        if (spatialQuery == null)
                        {
                            throw new InvalidOperationException("Illegal Spatial Sort Parameter: Blank Spatial sort cannot be used without a spatial query");
                        }

                        spatialField = viewGenerator.GetSpatialField(spatialQuery.SpatialFieldName);

                        shape = spatialField.ReadShape(spatialQuery.QueryShape);
                    }
                    else
                    {
                        var sortParams = sortedField.Field.Split(';');
                        double lat, lng;

                        if (sortParams.Length < 3 || !double.TryParse(sortParams[1], out lat) || !double.TryParse(sortParams[2], out lng))
                        {
                            throw new InvalidOperationException("Illegal Spatial Sort Parameter");
                        }


                        string spatialFieldName;

                        if (sortParams.Length >= 4)
                        {
                            spatialFieldName = sortParams[3];
                        }
                        else
                        {
                            if (spatialQuery == null)
                            {
                                spatialFieldName = Constants.DefaultSpatialFieldName;
                            }
                            else
                            {
                                spatialFieldName = spatialQuery.SpatialFieldName;
                            }
                        }

                        spatialField = viewGenerator.GetSpatialField(spatialFieldName);

                        shape = new PointImpl(lng, lat, spatialField.GetContext());
                    }
                    var dsort = new SpatialDistanceFieldComparatorSource(spatialField, shape.GetCenter());
                    return new SortField(sortedField.Field, dsort, sortedField.Descending);
                }
                var sortOptions = GetSortOption(indexDefinition, sortedField.Field, self);

                if (sortOptions == null || sortOptions == SortOptions.None)
                {
                    return new SortField(sortedField.Field, CultureInfo.InvariantCulture, sortedField.Descending);
                }

                if (sortOptions.Value == SortOptions.Short)
                {
                    sortOptions = SortOptions.Int;
                }
                return new SortField(sortedField.Field, (int)sortOptions.Value, sortedField.Descending);
            })
                            .ToArray()));
        }
 public static async Task CreateDefaultModelByMetaTypeIndexAsync(this MetaType @this) {
     var indexName = Storage.DefaultByMetaTypeNameIndexNameConvention.FormatString(@this.Name);
     var map = @"from model in docs.{0} select new {{ MetaType = model._shurikenMeta.MetaTypeName }}".FormatString(@this.Name);
     var indexDefinition = new IndexDefinition { Map = map };
     await Storage.DocumentStore.AsyncDatabaseCommands.PutIndexAsync(indexName, indexDefinition);
 }
Example #60
0
 public string PutIndex(string name, IndexDefinition definition)
 {
     return(PutIndex(name, definition, false));
 }