public void Execute(DocumentDatabase database) { if (!database.IsBundleActive("IndexedAttachments")) return; var index = new IndexDefinition { Map = @"from doc in docs where doc[""@metadata""][""Raven-Attachment-Key""] != null select new { AttachmentKey = doc[""@metadata""][""Raven-Attachment-Key""], Filename = doc[""@metadata""][""Raven-Attachment-Filename""], Text = doc.Text }", TransformResults = @"from result in results select new { AttachmentKey = result[""@metadata""][""Raven-Attachment-Key""], Filename = result[""@metadata""][""Raven-Attachment-Filename""] }" }; // NOTE: The transform above is specifically there to keep the Text property // from being returned. The results could get very large otherwise. index.Indexes.Add("Text", FieldIndexing.Analyzed); index.Stores.Add("Text", FieldStorage.Yes); index.TermVectors.Add("Text", FieldTermVector.WithPositionsAndOffsets); database.PutIndex("Raven/Attachments", index); }
public void LinqQueryWithStaticCallOnEnumerableIsCanBeCompiledAndRun() { var indexDefinition = new IndexDefinition<Page> { Map = pages => from p in pages from coAuthor in p.CoAuthors.DefaultIfEmpty() select new { p.Id, CoAuthorUserID = coAuthor != null ? coAuthor.UserId : -1 } }.ToIndexDefinition(new DocumentConvention()); var mapInstance = new DynamicViewCompiler("testView", indexDefinition, new AbstractDynamicCompilationExtension[] { }). GenerateInstance(); var conventions = new DocumentConvention(); var o = JObject.FromObject(page,conventions.CreateSerializer()); o["@metadata"] = new JObject( new JProperty("Raven-Entity-Name", "Pages") ); dynamic dynamicObject = new DynamicJsonObject(o); var result = mapInstance.MapDefinition(new[] { dynamicObject }).ToList(); Assert.Equal("{ Id = 0, CoAuthorUserID = 1, __document_id = }", result[0].ToString()); Assert.Equal("{ Id = 0, CoAuthorUserID = 2, __document_id = }", result[1].ToString()); }
public void LinqQueryWithIndexIsCaseInsensitive() { using (var store = this.NewDocumentStore()) { var definition = new IndexDefinition<Company> { Map = docs => from doc in docs select new { doc.Name } }.ToIndexDefinition(store.Conventions); store.DatabaseCommands.PutIndex("CompanyByName", definition); using (var session = store.OpenSession()) { session.Store(new Company { Name = "Google" }); session.Store(new Company { Name = "HibernatingRhinos" }); session.SaveChanges(); var company = session.Query<Company>("CompanyByName") .Customize(x=>x.WaitForNonStaleResults()) .Where(x=>x.Name == "Google") .FirstOrDefault(); Assert.NotNull(company); } } }
public TableDefinition AddIndex(params string[] columns) { var def = new IndexDefinition(); def.AddRange(columns); Indexes.Add(def); return this; }
public void CanUseNullCoalescingOperator() { using (var store = NewDocumentStore()) { var indexDefinition = new IndexDefinition { Map = "from e in docs.Events select new { Tag = \"Event\", _ = SpatialGenerate(e.Latitude ?? 38.9103000, e.Longitude ?? -77.3942) }", Indexes = { { "Tag", FieldIndexing.NotAnalyzed } } }; store.DocumentDatabase.PutIndex("eventsByLatLng", indexDefinition); store.DocumentDatabase.Put("Events/1", null, RavenJObject.Parse(@"{""Venue"": ""Jimmy's Old Town Tavern"", ""Latitude"": null, ""Longitude"": null }"), RavenJObject.Parse("{'Raven-Entity-Name': 'Events'}"), null); using (var session = store.OpenSession()) { var objects = session.Query<object>("eventsByLatLng") .Customize(x => x.WithinRadiusOf(6, 38.9103000, -77.3942)) .Customize(x => x.WaitForNonStaleResults()) .ToArray(); Assert.Empty(store.DocumentDatabase.Statistics.Errors); Assert.Equal(1, objects.Length); } } }
public void CanPerformSpatialSearch() { var indexDefinition = new IndexDefinition { Map = "from e in docs.Events select new { Tag = \"Event\", _ = SpatialGenerate(e.Latitude, e.Longitude) }", Indexes = { { "Tag", FieldIndexing.NotAnalyzed } } }; db.PutIndex("eventsByLatLng", indexDefinition); var events = SpatialIndexTestHelper.GetEvents(); for (int i = 0; i < events.Length; i++) { db.Put("Events/" + (i + 1), null, RavenJObject.FromObject(events[i]), RavenJObject.Parse("{'Raven-Entity-Name': 'Events'}"), null); } const double lat = 38.96939, lng = -77.386398; const double radiusInKm = 6.0*1.609344; QueryResult queryResult; do { queryResult = db.Query("eventsByLatLng", new SpatialIndexQuery() { Query = "Tag:[[Event]]", QueryShape = SpatialIndexQuery.GetQueryShapeFromLatLon(lat, lng, radiusInKm), SpatialRelation = SpatialRelation.Within, SpatialFieldName = Constants.DefaultSpatialFieldName, SortedFields = new[] { new SortedField("__distance"), } }); if (queryResult.IsStale) Thread.Sleep(100); } while (queryResult.IsStale); var expected = events.Count(e => Raven.Database.Indexing.SpatialIndex.GetDistance(lat, lng, e.Latitude, e.Longitude) <= radiusInKm); Assert.Equal(expected, queryResult.Results.Count); Assert.Equal(7, queryResult.Results.Count); double previous = 0; foreach (var r in queryResult.Results) { Event e = r.JsonDeserialization<Event>(); double distance = Raven.Database.Indexing.SpatialIndex.GetDistance(lat, lng, e.Latitude, e.Longitude); Console.WriteLine("Venue: " + e.Venue + ", Distance " + distance); Assert.True(distance < radiusInKm); Assert.True(distance >= previous); previous = distance; } }
public IndexHasChanged() { using (var store = new DocumentStore()) { IndexDefinition indexDefinition = new IndexDefinition(); #region index_has_changed_2 store.DatabaseCommands.IndexHasChanged("Orders/Totals", indexDefinition); #endregion } }
//Failing test from http://groups.google.com/group/ravendb/browse_thread/thread/7a93f37036297d48/ public void CanSuccessfullyDoSpatialQueryOfNearbyLocations() { // These items is in a radius of 4 miles (approx 6,5 km) var areaOneDocOne = new DummyGeoDoc(55.6880508001, 13.5717346673); var areaOneDocTwo = new DummyGeoDoc(55.6821978456, 13.6076183965); var areaOneDocThree = new DummyGeoDoc(55.673251569, 13.5946697607); // This item is 12 miles (approx 19 km) from the closest in areaOne var closeButOutsideAreaOne = new DummyGeoDoc(55.8634157297, 13.5497731987); // This item is about 3900 miles from areaOne var newYork = new DummyGeoDoc(40.7137578228, -74.0126901936); using (var documentStore = new EmbeddableDocumentStore { RunInMemory = true }.Initialize()) using (var session = documentStore.OpenSession()) { session.Store(areaOneDocOne); session.Store(areaOneDocTwo); session.Store(areaOneDocThree); session.Store(closeButOutsideAreaOne); session.Store(newYork); session.SaveChanges(); var indexDefinition = new IndexDefinition { Map = "from doc in docs select new { _ = SpatialIndex.Generate(doc.Latitude, doc.Longitude) }" }; documentStore.DatabaseCommands.PutIndex("FindByLatLng", indexDefinition); // Wait until the index is built session.Advanced.LuceneQuery<DummyGeoDoc>("FindByLatLng") .WaitForNonStaleResults() .ToArray(); const double lat = 55.6836422426, lng = 13.5871808352; // in the middle of AreaOne const double radius = 5.0; // Expected is that 5.0 will return 3 results var nearbyDocs = session.Advanced.LuceneQuery<DummyGeoDoc>("FindByLatLng") .WithinRadiusOf(radius, lat, lng) .WaitForNonStaleResults() .ToArray(); Assert.NotEqual(null, nearbyDocs); Assert.Equal(3, nearbyDocs.Length); //TODO //var dist = DistanceUtils.GetInstance(); //Assert.Equal(true, nearbyDocs.All(x => dist.GetDistanceMi(x.Latitude, x.Longitude, lat, lng) < radius)); session.Dispose(); } }
public void CanDefineHierarchicalIndexOnTheClient_WithLinq() { var indexDefinition = new IndexDefinition<Person> { Map = people => from p in people from c in p.Hierarchy(x=>x.Children) select c.Name }.ToIndexDefinition(new DocumentConvention()); Assert.Equal("docs.People\r\n\t.SelectMany(p => Hierarchy(p, \"Children\"), (p, c) => c.Name)", indexDefinition.Map); }
public DynamicViewCompiler(string name, IndexDefinition indexDefinition, OrderedPartCollection<AbstractDynamicCompilationExtension> extensions, string basePath, InMemoryRavenConfiguration configuration) { this.indexDefinition = indexDefinition; this.extensions = extensions; if (configuration.RunInMemory == false) { this.basePath = Path.Combine(basePath, "TemporaryIndexDefinitionsAsSource"); if (Directory.Exists(this.basePath) == false) Directory.CreateDirectory(this.basePath); } this.name = MonoHttpUtility.UrlEncode(name); RequiresSelectNewAnonymousType = true; }
public void CreateDeleteCreateIndex() { using (var store = NewDocumentStore(requestedStorage:"esent")) { var indexDefinition = new IndexDefinition { Map = "from d in docs select new {}" }; store.DatabaseCommands.PutIndex("test", indexDefinition); store.DatabaseCommands.DeleteIndex("test"); store.DatabaseCommands.PutIndex("test", indexDefinition); } }
public override IndexDefinition CreateIndexDefinition() { var index = new IndexDefinition(); index.Name = this.IndexName; index.Map = @"from doc in docs let DocumentType = ((dynamic)doc)[""@metadata""][""Raven-Entity-Name""] let Id = doc[""@metadata""][""Id""] let LastModified = doc[""@metadata""][""Last-Modified""] select new {DocumentType = ((dynamic)doc)[""@metadata""][""Raven-Entity-Name""], Id, LastModified}"; index.TransformResults = @"from result in results select new {Id = result.Id, DocumentType = result.DocumentType, LastModified = result.LastModified}"; return index; }
public async Task Index_replication_with_side_by_side_indexes_should_not_propagate_replaced_index_tombstones() { using (var source = CreateStore()) using (var destination = CreateStore()) { var oldIndexDef = new IndexDefinition { Map = "from person in docs.People\nselect new {\n\tFirstName = person.FirstName\n}" }; var testIndex = new RavenDB_3232.TestIndex(); var sourceDatabase = await servers[0].Server.GetDatabaseInternal(source.DefaultDatabase); sourceDatabase.StopBackgroundWorkers(); source.DatabaseCommands.PutIndex(testIndex.IndexName, oldIndexDef); using (var session = source.OpenSession()) { session.Store(new RavenDB_3232.Person { FirstName = "John", LastName = "Doe" }); session.SaveChanges(); } var sourceReplicationTask = sourceDatabase.StartupTasks.OfType<ReplicationTask>().First(); sourceReplicationTask.IndexReplication.TimeToWaitBeforeSendingDeletesOfIndexesToSiblings = TimeSpan.FromSeconds(0); sourceReplicationTask.Pause(); //pause replciation task _before_ setting up replication SetupReplication(source.DatabaseCommands, destination); var mre = new ManualResetEventSlim(); sourceDatabase.Notifications.OnIndexChange += (database, notification) => { if (notification.Type == IndexChangeTypes.SideBySideReplace) mre.Set(); }; shouldRecordRequests = true; testIndex.SideBySideExecute(source); sourceDatabase.SpinBackgroundWorkers(); WaitForIndexing(source); //now old index should be a tombstone and side-by-side replaced it. mre.Wait(); sourceReplicationTask.IndexReplication.Execute(); Assert.Equal(0, requestLog.Count(x => x.Method.Method == "DELETE")); } }
public void LinqQueryWithStaticCallOnEnumerableIsTranslatedToExtensionMethod() { var indexDefinition = new IndexDefinition<Page> { Map = pages => from p in pages from coAuthor in Enumerable.DefaultIfEmpty(p.CoAuthors) select new { p.Id, CoAuthorUserID = coAuthor != null ? coAuthor.UserId : -1 } }.ToIndexDefinition(new DocumentConvention()); var expectedMapTranslation = @"docs.Pages .SelectMany(p => p.CoAuthors.DefaultIfEmpty(), (p, coAuthor) => new {Id = p.Id, CoAuthorUserID = coAuthor != null ? coAuthor.UserId : -1})"; Assert.Equal(expectedMapTranslation, indexDefinition.Map); }
public void CanRunSpatialQueriesInMemory() { var documentStore = new EmbeddableDocumentStore { RunInMemory = true }; documentStore.Initialize(); var def = new IndexDefinition<Listing> { Map = listings => from listingItem in listings select new { listingItem.ClassCodes, listingItem.Latitude, listingItem.Longitude, _ = SpatialIndex.Generate(listingItem.Latitude, listingItem.Longitude) } }; documentStore.DatabaseCommands.PutIndex("RadiusClassifiedSearch", def); }
private static string TransformGeoIndexes(string value, IndexDefinition definition, DocumentConvention conventions) { if (value == null) return null; return Regex.Replace(value, @"GeoIndex\((?<pre>[^.]+)[.](?<prop>[^),]+)(?<remainder>[^)]*)[)]", match => { var fieldPrefix = match.Groups["prop"].Value.Replace(".", "_"); return string.Format("SpatialGenerate(\"{0}_{1}\", {2}.{3}.{1}{4})", match.Groups["prop"].Value.Replace(".", "_"), SpatialField.Name, match.Groups["pre"].Value, match.Groups["prop"].Value, match.Groups["remainder"].Value ); }); }
public void CanProjectIdFromTransformResults() { using (var store = NewDocumentStore()) { var indexDefinition = new IndexDefinition<Shipment, Shipment>() { Map = docs => from doc in docs select new { doc.Id }, TransformResults = (database, results) => from doc in results select new { Id = doc.Id, Name = doc.Name } }.ToIndexDefinition(store.Conventions); store.DatabaseCommands.PutIndex( "AmazingIndex", indexDefinition); using (var session = store.OpenSession()) { session.Store(new Shipment() { Id = "shipment1", Name = "Some shipment" }); session.SaveChanges(); var shipment = session.Query<Shipment>("AmazingIndex") .Customize(x=>x.WaitForNonStaleResults()) .Select(x => new Shipment { Id = x.Id, Name = x.Name }).Take(1).SingleOrDefault(); Assert.NotNull(shipment.Id); } } }
public void DefaultIndexingBehaviourAllowStartsWith() { using (var store = this.NewDocumentStore()) { var index = new IndexDefinition<Blog, BlogTagItem>() { Map = docs => from doc in docs from tag in doc.Tags select new { tag.Name }, Reduce = results => from result in results group result by result.Name into g select new { Name = g.Key, Count = g.Count() } }.ToIndexDefinition(store.Conventions); store.DatabaseCommands.PutIndex("TagInfo", index); using (var session = store.OpenSession()) { var newBlog = new Blog() { Tags = new[]{ new BlogTag() { Name = "SuperCallaFragalisticExpealadocious" } } }; session.Store(newBlog); session.SaveChanges(); var result = session.Query<BlogTagItem>("TagInfo") .Customize(x => x.WaitForNonStaleResults()) .Where(x => x.Name.StartsWith("Su")) .FirstOrDefault(); Assert.NotNull(result); } } }
public static void CreateIndex(DocumentDatabase database) { var index = new IndexDefinition { Map = string.Format( @"from doc in docs where doc[""{0}""][""{1}""] == ""{2}"" && doc[""{0}""][""{3}""] == true select new {{ {4} = doc[""{0}""][""{5}""], }}", Constants.Metadata, TemporalMetadata.RavenDocumentTemporalStatus, TemporalStatus.Revision, TemporalMetadata.RavenDocumentTemporalPending, Activation, TemporalMetadata.RavenDocumentTemporalEffectiveStart) }; if (database.GetIndexDefinition(TemporalConstants.PendingRevisionsIndex) == null) database.PutIndex(TemporalConstants.PendingRevisionsIndex, index); }
public SpatialSorting() { store = NewDocumentStore(databaseName:"SpatialSorting"); var indexDefinition = new IndexDefinition { Map = "from e in docs.Shops select new { e.Venue, _ = SpatialGenerate(e.Latitude, e.Longitude) }", Indexes = { {"Tag", FieldIndexing.NotAnalyzed} } }; store.DatabaseCommands.PutIndex("eventsByLatLng", indexDefinition); var indexDefinition2 = new IndexDefinition { Map = "from e in docs.Shops select new { e.Venue, MySpacialField = SpatialGenerate(e.Latitude, e.Longitude) }", Indexes = { {"Tag", FieldIndexing.NotAnalyzed} } }; store.DatabaseCommands.PutIndex("eventsByLatLngWSpecialField", indexDefinition2); for (int i = 0; i < shops.Length; i++) { store.DatabaseCommands.Put("Shops/" + (i + 1), null, RavenJObject.FromObject(shops[i]), RavenJObject.Parse("{'Raven-Entity-Name': 'Shops'}")); } WaitForIndexing(store); }
public async Task CanUpdate() { using (var agg = new AggregationEngine()) { await agg.InitAsync(); await agg.CreateAggregationAsync(new IndexDefinition { Name = "Test", Map = "from doc in docs select new { Count = 1}", Reduce = "from result in results group result by 1 into g select new { Count = g.Sum(x=>x.Count) }" }); var indexDefinition = new IndexDefinition { Name = "Test", Map = "from doc in docs select new { Sum = 1}", Reduce = "from result in results group result by 1 into g select new { Sum = g.Sum(x=>x.Sum) }" }; await agg.CreateAggregationAsync(indexDefinition); Assert.Contains(indexDefinition.Reduce, agg.GetAggregation("Test").Generator.ViewText); } }
private Action TryCreateTaskForApplyingPrecomputedBatchForNewIndex(Index index, IndexDefinition definition) { if (Database.Configuration.MaxPrecomputedBatchSizeForNewIndex <= 0) //precaution -> should never be lower than 0 { index.IsMapIndexingInProgress = false; return(null); } var generator = IndexDefinitionStorage.GetViewGenerator(definition.IndexId); if (generator.ForEntityNames.Count == 0 && index.IsTestIndex == false) { // we don't optimize if we don't have what to optimize _on_, we know this is going to return all docs. // no need to try to optimize that, then index.IsMapIndexingInProgress = false; return(null); } //only one precomputed batch can run at a time except for test indexes if (index.IsTestIndex == false) { lock (precomputedLock) { if (isPrecomputedBatchForNewIndexIsRunning) { index.IsMapIndexingInProgress = false; return(null); } isPrecomputedBatchForNewIndexIsRunning = true; } } try { var cts = new CancellationTokenSource(); var task = new Task(() => { try { ApplyPrecomputedBatchForNewIndex(index, generator, index.IsTestIndex == false ? Database.Configuration.MaxPrecomputedBatchSizeForNewIndex : Database.Configuration.Indexing.MaxNumberOfItemsToProcessInTestIndexes, cts); } catch (TotalDataSizeExceededException e) { //expected error Log.Info(e.Message); } catch (Exception e) { Log.Warn("Could not apply precomputed batch for index " + index, e); } finally { if (index.IsTestIndex == false) { isPrecomputedBatchForNewIndexIsRunning = false; } index.IsMapIndexingInProgress = false; WorkContext.ShouldNotifyAboutWork(() => "Precomputed indexing batch for " + index.PublicName + " is completed"); WorkContext.NotifyAboutWork(); } }, TaskCreationOptions.LongRunning); return(() => { try { task.Start(); long id; Database .Tasks .AddTask( task, new TaskBasedOperationState(task), new TaskActions.PendingTaskDescription { StartTime = DateTime.UtcNow, Description = index.PublicName, TaskType = TaskActions.PendingTaskType.NewIndexPrecomputedBatch }, out id, cts); } catch (Exception) { index.IsMapIndexingInProgress = false; if (index.IsTestIndex == false) { isPrecomputedBatchForNewIndexIsRunning = false; } throw; } }); } catch (Exception) { index.IsMapIndexingInProgress = false; if (index.IsTestIndex == false) { isPrecomputedBatchForNewIndexIsRunning = false; } throw; } }
private DynamicViewCompiler AddAndCompileIndex(IndexDefinition indexDefinition) { var name = FixupIndexName(indexDefinition.Name, path); var transformer = new DynamicViewCompiler(name, indexDefinition, extensions, path, configuration); var generator = transformer.GenerateInstance(); indexCache.AddOrUpdate(name, generator, (s, viewGenerator) => generator); logger.Info("New index {0}:\r\n{1}\r\nCompiled to:\r\n{2}", transformer.Name, transformer.CompiledQueryText, transformer.CompiledQueryText); return transformer; }
private JsonDocument RetrieveDocumentInternal( IndexQueryResult queryResult, HashSet <string> loadedIds, FieldsToFetch fieldsToFetch, IndexDefinition indexDefinition) { var queryScore = queryResult.Score; if (float.IsNaN(queryScore)) { queryScore = 0f; } if (queryResult.Projection == null) { // duplicate document, filter it out if (loadedIds.Add(queryResult.Key) == false) { return(null); } var document = GetDocumentWithCaching(queryResult.Key); if (document != null) { document.Metadata[Constants.TemporaryScoreValue] = queryScore; } return(document); } if (fieldsToFetch.IsProjection) { if (indexDefinition.IsMapReduce == false) { bool hasStoredFields = false; foreach (var fieldToFetch in fieldsToFetch) { FieldStorage value; if (indexDefinition.Stores.TryGetValue(fieldToFetch, out value) == false && value != FieldStorage.No) { continue; } hasStoredFields = true; } if (hasStoredFields == false) { // duplicate document, filter it out if (loadedIds.Add(queryResult.Key) == false) { return(null); } } } var fieldsToFetchFromDocument = fieldsToFetch.Where(fieldToFetch => queryResult.Projection[fieldToFetch] == null); var doc = GetDocumentWithCaching(queryResult.Key); if (doc != null) { var result = doc.DataAsJson.SelectTokenWithRavenSyntax(fieldsToFetchFromDocument.ToArray()); foreach (var property in result) { if (property.Value == null || property.Value.Type == JTokenType.Null) { continue; } queryResult.Projection[property.Key] = property.Value; } } } return(new JsonDocument { Key = queryResult.Key, DataAsJson = queryResult.Projection, Metadata = new RavenJObject { { Constants.TemporaryScoreValue, queryScore } } }); }
protected virtual IList <IndexDefinition> ReadIndexes(string schemaName, string tableName) { string query = @"SELECT OBJECT_SCHEMA_NAME(T.[object_id],DB_ID()) AS [Schema], T.[name] AS [table_name], I.[name] AS [index_name], AC.[name] AS [column_name], I.[type_desc], I.[is_unique], I.[data_space_id], I.[ignore_dup_key], I.[is_primary_key], I.[is_unique_constraint], I.[fill_factor], I.[is_padded], I.[is_disabled], I.[is_hypothetical], I.[allow_row_locks], I.[allow_page_locks], IC.[is_descending_key], IC.[is_included_column] FROM sys.[tables] AS T INNER JOIN sys.[indexes] I ON T.[object_id] = I.[object_id] INNER JOIN sys.[index_columns] IC ON I.[object_id] = IC.[object_id] INNER JOIN sys.[all_columns] AC ON T.[object_id] = AC.[object_id] AND IC.[column_id] = AC.[column_id] WHERE T.[is_ms_shipped] = 0 AND I.[type_desc] <> 'HEAP' AND T.object_id = OBJECT_ID('[{0}].[{1}]') ORDER BY T.[name], I.[index_id], IC.[key_ordinal]"; DataSet ds = Read(query, schemaName, tableName); DataTable dt = ds.Tables[0]; IList <IndexDefinition> indexes = new List <IndexDefinition>(); foreach (DataRow dr in dt.Rows) { List <IndexDefinition> matches = (from i in indexes where i.Name == dr["index_name"].ToString() && i.SchemaName == dr["Schema"].ToString() select i).ToList(); IndexDefinition iDef = null; if (matches.Count > 0) { iDef = matches[0]; } // create the table if not found if (iDef == null) { iDef = new IndexDefinition() { Name = dr["index_name"].ToString(), SchemaName = dr["Schema"].ToString(), IsClustered = dr["type_desc"].ToString() == "CLUSTERED", IsUnique = dr["is_unique"].ToString() == "1", TableName = dr["table_name"].ToString() }; indexes.Add(iDef); } ICollection <IndexColumnDefinition> ms; // columns ms = (from m in iDef.Columns where m.Name == dr["column_name"].ToString() select m).ToList(); if (ms.Count == 0) { iDef.Columns.Add(new IndexColumnDefinition() { Name = dr["column_name"].ToString(), Direction = dr["is_descending_key"].ToString() == "1" ? Direction.Descending : Direction.Ascending }); } } return(indexes); }
public FaultyIndexDefinition(string name, IEnumerable <string> collections, IndexLockMode lockMode, IndexPriority priority, IndexState state, IndexField[] mapFields, IndexDefinition definition) : base(name, collections, lockMode, priority, state, mapFields, IndexVersion.CurrentVersion) { _definition = definition; }
public abstract IndexDefinitionCompareDifferences Compare(IndexDefinition indexDefinition);
public string DirectPutIndex(string name, OperationMetadata operationMetadata, bool overwrite, IndexDefinition definition) { return(asyncServerClient.DirectPutIndexAsync(name, definition, overwrite, operationMetadata).Result); }
public bool IndexHasChanged(string name, IndexDefinition indexDef) { return(AsyncHelpers.RunSync(() => asyncServerClient.IndexHasChangedAsync(name, indexDef))); }
private bool CanUpdateIndex(string name, IndexDefinition definition, bool isUpdateBySideSide, IndexDefinition existingIndex) { switch (existingIndex.LockMode) { case IndexLockMode.SideBySide: if (isUpdateBySideSide == false) { Log.Info("Index {0} not saved because it might be only updated by side-by-side index", name); throw new InvalidOperationException("Can not overwrite locked index: " + name + ". This index can be only updated by side-by-side index."); } //keep the SideBySide lock mode from the replaced index definition.LockMode = IndexLockMode.SideBySide; break; case IndexLockMode.LockedIgnore: Log.Info("Index {0} not saved because it was lock (with ignore)", name); return(false); case IndexLockMode.LockedError: throw new InvalidOperationException("Can not overwrite locked index: " + name); } return(true); }
public void WriteIndex(IndexDefinition indexDefinition) { AsyncHelpers.RunSync(() => _database.IndexStore.CreateIndex(indexDefinition)); }
private WIndexDefinition ParseIndexDefinition(IndexDefinition idxDef) { if (idxDef == null) return null; var wIdxDef = new WIndexDefinition { FirstTokenIndex = idxDef.FirstTokenIndex, LastTokenIndex = idxDef.LastTokenIndex, IndexType = idxDef.IndexType, Name = idxDef.Name, }; if (idxDef.Columns == null) return wIdxDef; wIdxDef.Columns = new List<Tuple<WColumnReferenceExpression, SortOrder>>(); foreach (var col in idxDef.Columns) { wIdxDef.Columns.Add(new Tuple<WColumnReferenceExpression, SortOrder>( new WColumnReferenceExpression { ColumnType = col.Column.ColumnType, MultiPartIdentifier = ParseMultiPartIdentifier(col.Column.MultiPartIdentifier), FirstTokenIndex = col.Column.FirstTokenIndex, LastTokenIndex = col.Column.LastTokenIndex, }, col.SortOrder)); } return wIdxDef; }
public async Task If_deleted_original_index_on_destination_but_not_side_by_side_index() { using (var source = CreateStore()) using (var destination = CreateStore()) { var sourceDatabase = await servers[0].Server.GetDatabaseInternal(source.DefaultDatabase); var destinationDatabase = await servers[1].Server.GetDatabaseInternal(destination.DefaultDatabase); sourceDatabase.StopBackgroundWorkers(); destinationDatabase.StopBackgroundWorkers(); var testIndex = new UserIndex(); var oldIndexDef = new IndexDefinition { Map = "from user in docs.Users\n select new {\n\tName = user.Name\n}" }; source.DatabaseCommands.PutIndex(testIndex.IndexName, oldIndexDef); using (var session = source.OpenSession()) { session.Store(new User { Name = "John Doe" }); session.SaveChanges(); } var sourceReplicationTask = sourceDatabase.StartupTasks.OfType <ReplicationTask>().First(); sourceReplicationTask.Pause(); SetupReplication(source.DatabaseCommands, destination); sourceReplicationTask.IndexReplication.Execute(); //replicate the usual index source.SideBySideExecuteIndex(testIndex); //the side by side index will be automatically replicated SpinWait.SpinUntil(() => { var index = destinationDatabase.Indexes.GetIndexDefinition(Constants.SideBySideIndexNamePrefix + testIndex.IndexName); return(index != null); }, 5000); Assert.NotNull(destinationDatabase.Indexes.GetIndexDefinition(Constants.SideBySideIndexNamePrefix + testIndex.IndexName)); destinationDatabase.Indexes.DeleteIndex(testIndex.IndexName); //delete the original index var sideBySideIndex = destinationDatabase.Indexes.GetIndexDefinition(Constants.SideBySideIndexNamePrefix + testIndex.IndexName); Assert.NotNull(sideBySideIndex); VerifyReplacementDocumentIsThere(Constants.SideBySideIndexNamePrefix + testIndex.IndexName, destinationDatabase); sourceReplicationTask.IndexReplication.Execute(); var indexDefinition = destinationDatabase.Indexes.Definitions.First(x => x.Name == Constants.SideBySideIndexNamePrefix + testIndex.IndexName); destinationDatabase.IndexReplacer.ForceReplacement(indexDefinition); //wait until the index will be replaced SpinWait.SpinUntil(() => { var index = destinationDatabase.Indexes.GetIndexDefinition(testIndex.IndexName); return(index != null); }, 5000); var replacingIndex = destinationDatabase.Indexes.GetIndexDefinition(testIndex.IndexName); Assert.True(replacingIndex != null); var oldIndex = destinationDatabase.Indexes.GetIndexDefinition(testIndex.IndexName); Assert.True(oldIndex.Equals(testIndex.CreateIndexDefinition(), false)); sideBySideIndex = destinationDatabase.Indexes.GetIndexDefinition(Constants.SideBySideIndexNamePrefix + testIndex.IndexName); Assert.Null(sideBySideIndex); SpinWait.SpinUntil(() => { var doc = destinationDatabase.Documents.Get(Constants.IndexReplacePrefix + Constants.SideBySideIndexNamePrefix + testIndex.IndexName, null); return(doc == null); }, 5000); Assert.Null(destinationDatabase.Documents.Get(Constants.IndexReplacePrefix + Constants.SideBySideIndexNamePrefix + testIndex.IndexName, null)); } }
public async Task Side_by_side_index_after_replication_should_have_appropriate_minimum_etag_for_the_destination_if_applicable() { using (var source = CreateStore()) using (var destination = CreateStore()) { var testIndex = new UserIndex(); var oldIndexDef = new IndexDefinition { Map = "from user in docs.Users\n select new {\n\tName = user.Name\n}" }; source.DatabaseCommands.PutIndex(testIndex.IndexName, oldIndexDef); var sourceDatabase = await servers[0].Server.GetDatabaseInternal(source.DefaultDatabase); var destinationDatabase = await servers[1].Server.GetDatabaseInternal(destination.DefaultDatabase); using (var session = source.OpenSession()) { for (int i = 0; i < 10; i++) { session.Store(new User { Name = "User - " + i }); } session.SaveChanges(); } WaitForIndexing(source); sourceDatabase.StopBackgroundWorkers(); destinationDatabase.StopBackgroundWorkers(); var sourceReplicationTask = sourceDatabase.StartupTasks.OfType <ReplicationTask>().First(); sourceReplicationTask.Pause(); SetupReplication(source.DatabaseCommands, destination); //replicate the original index await sourceReplicationTask.ExecuteReplicationOnce(true); sourceReplicationTask.IndexReplication.Execute(); destinationDatabase.SpinBackgroundWorkers(); WaitForIndexing(destination); destinationDatabase.StopBackgroundWorkers(); testIndex.SideBySideExecute(source, sourceDatabase.Statistics.LastDocEtag); var replacementIndexName = Constants.SideBySideIndexNamePrefix + testIndex.IndexName; //do side-by-side index replication -> since in the destination there is original index, //simply create the side-by-side index as so it will replace the original when it catches up sourceReplicationTask.IndexReplication.Execute(); var originalDefinition = destination.DatabaseCommands.GetIndex(testIndex.IndexName); Assert.NotNull(originalDefinition); Assert.True(originalDefinition.Equals(oldIndexDef, false)); var sideBySideDefinition = destination.DatabaseCommands.GetIndex(replacementIndexName); Assert.NotNull(sideBySideDefinition); Assert.True(sideBySideDefinition.Equals(testIndex.CreateIndexDefinition(), false)); VerifyReplacementDocumentIsThere(replacementIndexName, destinationDatabase, true); } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: public static int[] getOrCreatePropertyKeyIds(org.neo4j.internal.kernel.api.TokenWrite tokenWrite, org.neo4j.graphdb.schema.IndexDefinition indexDefinition) throws org.neo4j.internal.kernel.api.exceptions.schema.IllegalTokenNameException public static int[] GetOrCreatePropertyKeyIds(TokenWrite tokenWrite, IndexDefinition indexDefinition) { return(GetOrCreatePropertyKeyIds(tokenWrite, GetPropertyKeysArrayOf(indexDefinition))); }
private CreateCoveringIndexResult TryCreateCoveringIndex(DAL.Contracts.StatementQuery query, StatementQueryExtractedData queryExtractedData, IndexDefinition baseIndex, out IndexDefinition coveringIndex) { var allWhereAttributes = queryExtractedData.WhereAttributes.All.Where(x => x.Relation.ID == baseIndex.Relation.ID); var bTreeWhereAttributes = queryExtractedData.WhereAttributes.BTreeApplicable.Where(x => x.Relation.ID == baseIndex.Relation.ID); var allJoinAttributes = queryExtractedData.JoinAttributes.All.Where(x => x.Relation.ID == baseIndex.Relation.ID); var bTreeJoinAttributes = queryExtractedData.JoinAttributes.BTreeApplicable.Where(x => x.Relation.ID == baseIndex.Relation.ID); var allGroupByAttributes = queryExtractedData.GroupByAttributes.All.Where(x => x.Relation.ID == baseIndex.Relation.ID); var bTreeGroupByAttributes = queryExtractedData.GroupByAttributes.BTreeApplicable.Where(x => x.Relation.ID == baseIndex.Relation.ID); var allOrderByAttributes = queryExtractedData.OrderByAttributes.All.Where(x => x.Relation.ID == baseIndex.Relation.ID); var bTreeOrderByAttributes = queryExtractedData.OrderByAttributes.BTreeApplicable.Where(x => x.Relation.ID == baseIndex.Relation.ID); var allProjectionAttributes = queryExtractedData.ProjectionAttributes.All.Where(x => x.Relation.ID == baseIndex.Relation.ID); if (allWhereAttributes.Count() == bTreeWhereAttributes.Count() && allJoinAttributes.Count() == bTreeJoinAttributes.Count() && allGroupByAttributes.Count() == bTreeGroupByAttributes.Count() && allOrderByAttributes.Count() == bTreeOrderByAttributes.Count() && query.CommandType != DAL.Contracts.StatementQueryCommandType.Insert) { var includeAttributes = new HashSet <AttributeData>(bTreeWhereAttributes); includeAttributes.UnionWith(bTreeJoinAttributes); includeAttributes.UnionWith(bTreeGroupByAttributes); includeAttributes.UnionWith(bTreeOrderByAttributes); includeAttributes.UnionWith(allProjectionAttributes); includeAttributes.ExceptWith(baseIndex.Attributes); List <AttributeData> includeSortedAttributes = new List <AttributeData>(includeAttributes.OrderBy(x => x.CardinalityIndicator)); coveringIndex = new IndexDefinition(baseIndex.StructureType, baseIndex.Relation, baseIndex.Attributes, includeSortedAttributes); return(baseIndex.IncludeAttributes.Count != coveringIndex.IncludeAttributes.Count ? CreateCoveringIndexResult.CreatedNew : CreateCoveringIndexResult.CreatedSameAsBaseIndex); } else { coveringIndex = null; return(CreateCoveringIndexResult.NotPossible); } }
public FaultyInMemoryIndex(Exception e, string name, IndexingConfiguration configuration, IndexDefinition definition) : this(e, configuration, new FaultyIndexDefinition(name, new HashSet <string> { "@FaultyIndexes" }, IndexLockMode.Unlock, IndexPriority.Normal, IndexState.Normal, new IndexField[0], definition)) { }
public IndexCreationOptions FindIndexCreationOptions(IndexDefinition indexDef) { var indexDefinition = GetIndexDefinition(indexDef.Name); if (indexDefinition != null) { return indexDefinition.Equals(indexDef) ? IndexCreationOptions.Noop : IndexCreationOptions.Update; } return IndexCreationOptions.Create; }
public string PutIndex(string name, IndexDefinition definition, bool overwrite) { return(AsyncHelpers.RunSync(() => asyncServerClient.PutIndexAsync(name, definition, overwrite))); }
private void ReadIndexesFromCatalog(IEnumerable<AbstractViewGenerator> compiledGenerators, ITransactionalStorage transactionalStorage) { foreach (var generator in compiledGenerators) { var copy = generator; var displayNameAtt = TypeDescriptor.GetAttributes(copy) .OfType<DisplayNameAttribute>() .FirstOrDefault(); var name = displayNameAtt != null ? displayNameAtt.DisplayName : copy.GetType().Name; transactionalStorage.Batch(actions => { if (actions.Indexing.GetIndexesStats().Any(x => x.Name == name)) return; actions.Indexing.AddIndex(name, copy.ReduceDefinition != null); }); var indexDefinition = new IndexDefinition { Name = name, Map = "Compiled map function: " + generator.GetType().AssemblyQualifiedName, // need to supply this so the index storage will create map/reduce index Reduce = generator.ReduceDefinition == null ? null : "Compiled reduce function: " + generator.GetType().AssemblyQualifiedName, Indexes = generator.Indexes, Stores = generator.Stores, IsCompiled = true }; indexCache.AddOrUpdate(name, copy, (s, viewGenerator) => copy); indexDefinitions.AddOrUpdate(name, indexDefinition, (s1, definition) => indexDefinition); } }
public override IndexDefinitionCompareDifferences Compare(IndexDefinition indexDefinition) { return(_definition.Compare(indexDefinition)); }
public DynamicViewCompiler(string name, IndexDefinition indexDefinition, AbstractDynamicCompilationExtension[] extensions) { this.indexDefinition = indexDefinition; this.extensions = extensions; this.name = MonoHttpUtility.UrlEncode(name); }
public static void ValidateReduceResultsCollectionName(IndexDefinition definition, StaticIndexBase index, DocumentDatabase database) { var outputReduceToCollection = definition.OutputReduceToCollection; if (string.IsNullOrWhiteSpace(outputReduceToCollection)) { return; } var collections = index.Maps.Keys.ToHashSet(StringComparer.OrdinalIgnoreCase); if (collections.Contains(Constants.Documents.Collections.AllDocumentsCollection)) { throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " + $"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " + $"as this index is mapping all documents " + $"and this will result in an infinite loop."); } foreach (var referencedCollection in index.ReferencedCollections) { foreach (var collectionName in referencedCollection.Value) { collections.Add(collectionName.Name); } } if (collections.Contains(outputReduceToCollection)) { throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " + $"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " + $"as this index is mapping or referencing the '{outputReduceToCollection}' collection " + $"and this will result in an infinite loop."); } var indexes = database.IndexStore.GetIndexes() .Where(x => x.Type == IndexType.MapReduce) .Cast <MapReduceIndex>() .Where(mapReduceIndex => string.IsNullOrWhiteSpace(mapReduceIndex.Definition.OutputReduceToCollection) == false && mapReduceIndex.Name != definition.Name) .ToList(); foreach (var otherIndex in indexes) { if (otherIndex.Definition.OutputReduceToCollection.Equals(outputReduceToCollection, StringComparison.OrdinalIgnoreCase)) { var sideBySideIndex = definition.Name.StartsWith(Constants.Documents.Indexing.SideBySideIndexNamePrefix, StringComparison.OrdinalIgnoreCase); if (sideBySideIndex) { throw new IndexInvalidException($"In order to create the '{definition.Name}' side by side index " + $"you firstly need to set {nameof(IndexDefinition.OutputReduceToCollection)} to be null " + $"on the '{otherIndex.Name}' index " + $"and than delete all of the documents in the '{otherIndex.Definition.OutputReduceToCollection}' collection."); } throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " + $"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " + $"as there is another index named '{otherIndex.Name}' " + $"which also output reduce results to documents in the same '{outputReduceToCollection}' collection. " + $"{nameof(IndexDefinition.OutputReduceToCollection)} must by set to unique value for each index or be null."); } var otherIndexCollections = otherIndex.Collections; foreach (var referencedCollection in otherIndex.GetReferencedCollections()) { foreach (var collectionName in referencedCollection.Value) { otherIndexCollections.Add(collectionName.Name); } } if (otherIndexCollections.Contains(outputReduceToCollection) && CheckIfThereIsAnIndexWhichWillOutputReduceDocumentsWhichWillBeUsedAsMapOnTheSpecifiedIndex(otherIndex, collections, indexes, out string description)) { description += Environment.NewLine + $"--> {definition.Name}: {string.Join(",", collections)} => *{outputReduceToCollection}*"; throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " + $"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " + $"as '{outputReduceToCollection}' collection is consumed by other index in a way that would " + $"lead to an infinite loop." + Environment.NewLine + description); } } using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.OpenReadTransaction()) { var stats = database.DocumentsStorage.GetCollection(outputReduceToCollection, context); if (stats.Count > 0) { throw new IndexInvalidException($"In order to create the '{definition.Name}' index " + $"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " + $"you firstly need to delete all of the documents in the '{stats.Name}' collection " + $"(currently have {stats.Count} document{(stats.Count == 1 ? "" : "s")})."); } } }
public string PutIndex <TDocument, TReduceResult>(string name, IndexDefinition <TDocument, TReduceResult> indexDef) { return(PutIndex(name, indexDef.ToIndexDefinition(convention))); }
private string PutIndexInternal(string name, IndexDefinition definition, bool disableIndexBeforePut = false, bool isUpdateBySideSide = false, IndexCreationOptions?creationOptions = null, bool isReplication = false) { if (name == null) { throw new ArgumentNullException(nameof(name)); } name = name.Trim(); IsIndexNameValid(name); var existingIndex = IndexDefinitionStorage.GetIndexDefinition(name); if (existingIndex != null) { var newIndexVersion = definition.IndexVersion; var currentIndexVersion = existingIndex.IndexVersion; // whether we update the index definition or not, // we need to update the index version existingIndex.IndexVersion = definition.IndexVersion = Math.Max(currentIndexVersion ?? 0, newIndexVersion ?? 0); switch (isReplication) { case true: if (newIndexVersion != null && currentIndexVersion != null && newIndexVersion <= currentIndexVersion) { //this new index is an older version of the current one return(null); } // we need to update the lock mode only if it was updated by another server existingIndex.LockMode = definition.LockMode; break; default: if (CanUpdateIndex(name, definition, isUpdateBySideSide, existingIndex) == false) { return(null); } break; } } AssertAnalyzersValid(definition); switch (creationOptions ?? FindIndexCreationOptions(definition, ref name)) { case IndexCreationOptions.Noop: return(null); case IndexCreationOptions.UpdateWithoutUpdatingCompiledIndex: // ensure that the code can compile new DynamicViewCompiler(definition.Name, definition, Database.Extensions, IndexDefinitionStorage.IndexDefinitionsPath, Database.Configuration).GenerateInstance(); IndexDefinitionStorage.UpdateIndexDefinitionWithoutUpdatingCompiledIndex(definition); if (isReplication == false) { definition.IndexVersion = (definition.IndexVersion ?? 0) + 1; } return(null); case IndexCreationOptions.Update: // ensure that the code can compile new DynamicViewCompiler(definition.Name, definition, Database.Extensions, IndexDefinitionStorage.IndexDefinitionsPath, Database.Configuration).GenerateInstance(); DeleteIndex(name); if (isReplication == false) { definition.IndexVersion = (definition.IndexVersion ?? 0) + 1; } break; case IndexCreationOptions.Create: if (isReplication == false) { // we create a new index, // we need to restore its previous IndexVersion (if it was deleted before) var deletedIndexVersion = IndexDefinitionStorage.GetDeletedIndexVersion(definition); var replacingIndexVersion = GetOriginalIndexVersion(definition.Name); definition.IndexVersion = Math.Max(deletedIndexVersion, replacingIndexVersion); definition.IndexVersion++; } break; } PutNewIndexIntoStorage(name, definition, disableIndexBeforePut); WorkContext.ClearErrorsFor(name); TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() => Database.Notifications.RaiseNotifications(new IndexChangeNotification { Name = name, Type = IndexChangeTypes.IndexAdded, Version = definition.IndexVersion })); return(name); }
public string PutIndex(string name, IndexDefinition definition, bool isReplication = false) { return(PutIndexInternal(name, definition, isReplication: isReplication)); }
public JsonDocument RetrieveDocumentForQuery(IndexQueryResult queryResult, IndexDefinition indexDefinition, FieldsToFetch fieldsToFetch) { return(ExecuteReadTriggers(ProcessReadVetoes( RetrieveDocumentInternal(queryResult, loadedIdsForRetrieval, fieldsToFetch, indexDefinition), null, ReadOperation.Query), null, ReadOperation.Query)); }
public void ReplaceOfNonStaleIndex() { var path = NewDataPath(); using (var store = GetDocumentStore(new Options { Path = path })) { var oldIndexDef = new IndexDefinition { Name = "TestIndex", Maps = { "from person in docs.People\nselect new {\n\tFirstName = person.FirstName\n}" } }; store.Maintenance.Send(new PutIndexesOperation(oldIndexDef)); using (var session = store.OpenSession()) { session.Store(new Person { FirstName = "John", LastName = "Doe" }); session.SaveChanges(); } WaitForIndexing(store); store.Maintenance.Send(new StopIndexingOperation()); new TestIndex().Execute(store); var e = Assert.Throws <RavenException>(() => { using (var session = store.OpenSession()) { var count = session.Query <Person, TestIndex>() .Count(x => x.LastName == "Doe"); } }); Assert.Contains("The field 'LastName' is not indexed, cannot query/sort on fields that are not indexed", e.InnerException.Message); var mre = new ManualResetEventSlim(); var changes = AsyncHelpers.RunSync(() => store.Changes().EnsureConnectedNow()); var observable = changes.ForAllIndexes(); AsyncHelpers.RunSync(() => observable.EnsureSubscribedNow()); observable.Subscribe(change => { if (change.Type == IndexChangeTypes.SideBySideReplace) { mre.Set(); } }); store.Maintenance.Send(new StartIndexingOperation()); WaitForIndexing(store); Assert.True(mre.Wait(TimeSpan.FromSeconds(15))); using (var session = store.OpenSession()) { var count = session.Query <Person, TestIndex>() .Count(x => x.LastName == "Doe"); Assert.Equal(1, count); } } }
private static CompilationResult CompileInternal(string originalName, string cSharpSafeName, MemberDeclarationSyntax @class, IndexDefinition definition) { var name = cSharpSafeName + "." + Guid.NewGuid() + IndexExtension; var @namespace = RoslynHelper.CreateNamespace(IndexNamespace) .WithMembers(SyntaxFactory.SingletonList(@class)); var res = GetUsingDirectiveAndSyntaxTreesAndReferences(definition); var compilationUnit = SyntaxFactory.CompilationUnit() .WithUsings(RoslynHelper.CreateUsings(res.UsingDirectiveSyntaxes)) .WithMembers(SyntaxFactory.SingletonList <MemberDeclarationSyntax>(@namespace)) .NormalizeWhitespace(); SyntaxNode formattedCompilationUnit; using (var workspace = new AdhocWorkspace()) { formattedCompilationUnit = Formatter.Format(compilationUnit, workspace); } string sourceFile = null; if (EnableDebugging) { sourceFile = Path.Combine(Path.GetTempPath(), name + ".cs"); File.WriteAllText(sourceFile, formattedCompilationUnit.ToFullString(), Encoding.UTF8); } var st = EnableDebugging ? SyntaxFactory.ParseSyntaxTree(File.ReadAllText(sourceFile), path: sourceFile, encoding: Encoding.UTF8) : SyntaxFactory.ParseSyntaxTree(formattedCompilationUnit.ToFullString()); res.SyntaxTrees.Add(st); var compilation = CSharpCompilation.Create( assemblyName: name, syntaxTrees: res.SyntaxTrees, references: res.References, options: new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary) .WithOptimizationLevel(EnableDebugging ? OptimizationLevel.Debug : OptimizationLevel.Release) ); var code = formattedCompilationUnit.SyntaxTree.ToString(); var asm = new MemoryStream(); var pdb = EnableDebugging ? new MemoryStream() : null; var result = compilation.Emit(asm, pdb, options: new EmitOptions(debugInformationFormat: DebugInformationFormat.PortablePdb)); if (result.Success == false) { IEnumerable <Diagnostic> failures = result.Diagnostics .Where(diagnostic => diagnostic.IsWarningAsError || diagnostic.Severity == DiagnosticSeverity.Error); var sb = new StringBuilder(); sb.AppendLine($"Failed to compile index {originalName}"); sb.AppendLine(); sb.AppendLine(code); sb.AppendLine(); foreach (var diagnostic in failures) { sb.AppendLine(diagnostic.ToString()); } throw new IndexCompilationException(sb.ToString()); } asm.Position = 0; Assembly assembly; if (EnableDebugging) { pdb.Position = 0; assembly = AssemblyLoadContext.Default.LoadFromStream(asm, pdb); } else { assembly = AssemblyLoadContext.Default.LoadFromStream(asm); } return(new CompilationResult { Code = code, Type = assembly.GetType($"{IndexNamespace}.{cSharpSafeName}") }); }
internal void PutNewIndexIntoStorage(string name, IndexDefinition definition, bool disableIndex = false) { Debug.Assert(Database.IndexStorage != null); Debug.Assert(TransactionalStorage != null); Debug.Assert(WorkContext != null); Index index = null; TransactionalStorage.Batch(actions => { var maxId = 0; if (Database.IndexStorage.Indexes.Length > 0) { maxId = Database.IndexStorage.Indexes.Max(); } definition.IndexId = (int)Database.Documents.GetNextIdentityValueWithoutOverwritingOnExistingDocuments("IndexId", actions); if (definition.IndexId <= maxId) { actions.General.SetIdentityValue("IndexId", maxId + 1); definition.IndexId = (int)Database.Documents.GetNextIdentityValueWithoutOverwritingOnExistingDocuments("IndexId", actions); } IndexDefinitionStorage.RegisterNewIndexInThisSession(name, definition); // this has to happen in this fashion so we will expose the in memory status after the commit, but // before the rest of the world is notified about this. IndexDefinitionStorage.CreateAndPersistIndex(definition); Database.IndexStorage.CreateIndexImplementation(definition); index = Database.IndexStorage.GetIndexInstance(definition.IndexId); // If we execute multiple indexes at once and want to activate them all at once we will disable the index from the endpoint if (disableIndex) { index.Priority = IndexingPriority.Disabled; } //ensure that we don't start indexing it right away, let the precomputation run first, if applicable index.IsMapIndexingInProgress = true; if (definition.IsTestIndex) { index.MarkQueried(); // test indexes should be mark queried, so the cleanup task would not delete them immediately } InvokeSuggestionIndexing(name, definition, index); actions.Indexing.AddIndex(definition.IndexId, definition.IsMapReduce); }); Debug.Assert(index != null); Action precomputedTask = null; if (WorkContext.RunIndexing && name.Equals(Constants.DocumentsByEntityNameIndex, StringComparison.InvariantCultureIgnoreCase) == false && Database.IndexStorage.HasIndex(Constants.DocumentsByEntityNameIndex) && isPrecomputedBatchForNewIndexIsRunning == false) { // optimization of handling new index creation when the number of document in a database is significantly greater than // number of documents that this index applies to - let us use built-in RavenDocumentsByEntityName to get just appropriate documents precomputedTask = TryCreateTaskForApplyingPrecomputedBatchForNewIndex(index, definition); } else { index.IsMapIndexingInProgress = false;// we can't apply optimization, so we'll make it eligible for running normally } // The act of adding it here make it visible to other threads // we have to do it in this way so first we prepare all the elements of the // index, then we add it to the storage in a way that make it public IndexDefinitionStorage.AddIndex(definition.IndexId, definition); // we start the precomputedTask _after_ we finished adding the index precomputedTask?.Invoke(); WorkContext.ShouldNotifyAboutWork(() => "PUT INDEX " + name); WorkContext.NotifyAboutWork(); }
void InsertOrModifyEventSourceIdAndVersionIndex() { var alreadyExists = true; var updated = false; var indexName = "Temp/Events/ByEventSourceIdAndVersionSortByVersion"; var index = _documentStore.DatabaseCommands.GetIndex(indexName); if (index == null) { index = new IndexDefinition { Map = "from doc in docs.Events select new { EventSourceId = doc.EventSourceId, Version = doc.Version }", Fields = new List<string> { "EventSourceId", "Version", "__document_id" } }; alreadyExists = false; } if (alreadyExists && index.SortOptions.First().Value != SortOptions.Double) { _documentStore.DatabaseCommands.DeleteIndex(indexName); updated = true; } index.SortOptions = new Dictionary<string, SortOptions> { { "Version", SortOptions.Double } }; if (!alreadyExists || updated) _documentStore.DatabaseCommands.PutIndex(indexName, index); }
private static (UsingDirectiveSyntax[] UsingDirectiveSyntaxes, List <SyntaxTree> SyntaxTrees, MetadataReference[] References) GetUsingDirectiveAndSyntaxTreesAndReferences(IndexDefinition definition) { if (definition.AdditionalSources == null && definition.AdditionalSources == null) { return(Usings, new List <SyntaxTree>(), References); } (UsingDirectiveSyntax[] UsingDirectiveSyntaxes, List <SyntaxTree> SyntaxTrees, MetadataReference[] References)result; var syntaxTrees = new List <SyntaxTree>(); var usings = new HashSet <string>(); if (definition.AdditionalSources != null) { foreach (var ext in definition.AdditionalSources) { var tree = SyntaxFactory.ParseSyntaxTree(AddUsingIndexStatic(ext.Value)); syntaxTrees.Add(tree); var ns = tree.GetRoot().DescendantNodes() .OfType <NamespaceDeclarationSyntax>() .FirstOrDefault(); if (ns != null) { usings.Add(ns.Name.ToString()); } } } if (definition.AdditionalAssemblies != null) { foreach (var additionalAssembly in definition.AdditionalAssemblies) { if (additionalAssembly.Usings != null) { foreach (var @using in additionalAssembly.Usings) { usings.Add(@using); } } } } if (usings.Count > 0) { //Adding using directive with duplicates to avoid O(n*m) operation and confusing code var newUsing = usings.Select(x => SyntaxFactory.UsingDirective(SyntaxFactory.ParseName(x))).ToList(); newUsing.AddRange(Usings); result.UsingDirectiveSyntaxes = newUsing.ToArray(); } else { result.UsingDirectiveSyntaxes = Usings; } result.References = GetReferences(definition); var tempCompilation = CSharpCompilation.Create( assemblyName: string.Empty, syntaxTrees: syntaxTrees, references: result.References, options: new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary) .WithOptimizationLevel(EnableDebugging ? OptimizationLevel.Debug : OptimizationLevel.Release) ); var rewriter = new MethodDynamicParametersRewriter(); result.SyntaxTrees = new List <SyntaxTree>(); foreach (var tree in syntaxTrees) //now do the rewrites { rewriter.SemanticModel = tempCompilation.GetSemanticModel(tree); var rewritten = rewriter.Visit(tree.GetRoot()).NormalizeWhitespace(); result.SyntaxTrees.Add(SyntaxFactory.SyntaxTree(rewritten, new CSharpParseOptions(documentationMode: DocumentationMode.None))); } return(result); }
public string CreateAndPersistIndex(IndexDefinition indexDefinition) { var transformer = AddAndCompileIndex(indexDefinition); if (configuration.RunInMemory == false) { var encodeIndexNameIfNeeded = FixupIndexName(indexDefinition.Name, path); var indexName = Path.Combine(path, MonoHttpUtility.UrlEncode(encodeIndexNameIfNeeded) + ".index"); // Hash the name if it's too long (as a path) File.WriteAllText(indexName, JsonConvert.SerializeObject(indexDefinition, Formatting.Indented, Default.Converters)); } return transformer.Name; }
public override IndexDefinitionCompareDifferences Compare(IndexDefinition indexDefinition) { throw new NotImplementedException(); }
public void AddIndex(string name, IndexDefinition definition) { indexDefinitions.AddOrUpdate(name, definition, (s1, def) => { if (def.IsCompiled) throw new InvalidOperationException("Index " + name + " is a compiled index, and cannot be replaced"); return definition; }); }
public void Init(IndexDefinition definition) { indexDefinition = definition; }
public static void ResolveAnalyzers(IndexDefinition indexDefinition) { // Stick Lucene.Net's namespace to all analyzer aliases that are missing a namespace var analyzerNames = (from analyzer in indexDefinition.Analyzers where analyzer.Value.IndexOf('.') == -1 select analyzer).ToArray(); // Only do this for analyzer that actually exist; we do this here to be able to throw a correct error later on foreach (var a in analyzerNames.Where(a => typeof(StandardAnalyzer).Assembly.GetType("Lucene.Net.Analysis." + a.Value) != null)) { indexDefinition.Analyzers[a.Key] = "Lucene.Net.Analysis." + a.Value; } }
public static Sort GetSort(this IndexQuery self, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator) { var spatialQuery = self as SpatialIndexQuery; var sortedFields = self.SortedFields; if (sortedFields == null || sortedFields.Length <= 0) { if (spatialQuery == null || string.IsNullOrEmpty(self.Query) == false) { return(null); } sortedFields = new[] { new SortedField(Constants.DistanceFieldName), }; } return(new Sort(sortedFields .Select(sortedField => { if (sortedField.Field == Constants.TemporaryScoreValue) { return SortField.FIELD_SCORE; } if (sortedField.Field.StartsWith(Constants.RandomFieldName)) { var parts = sortedField.Field.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries); if (parts.Length < 2) // truly random { return new RandomSortField(Guid.NewGuid().ToString()); } return new RandomSortField(parts[1]); } if (sortedField.Field.StartsWith(Constants.CustomSortFieldName)) { var parts = sortedField.Field.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries); if (parts.Length < 2) // truly random { throw new InvalidOperationException("Cannot figure out type for custom sort"); } return new CustomSortField(parts[1], self, parts[0][parts[0].Length - 1] == '-'); } if (sortedField.Field.StartsWith(Constants.DistanceFieldName)) { SpatialField spatialField = null; Shape shape; if (sortedField.Field.Length == Constants.DistanceFieldName.Length) { if (spatialQuery == null) { throw new InvalidOperationException("Illegal Spatial Sort Parameter: Blank Spatial sort cannot be used without a spatial query"); } spatialField = viewGenerator.GetSpatialField(spatialQuery.SpatialFieldName); shape = spatialField.ReadShape(spatialQuery.QueryShape); } else { var sortParams = sortedField.Field.Split(';'); double lat, lng; if (sortParams.Length < 3 || !double.TryParse(sortParams[1], out lat) || !double.TryParse(sortParams[2], out lng)) { throw new InvalidOperationException("Illegal Spatial Sort Parameter"); } string spatialFieldName; if (sortParams.Length >= 4) { spatialFieldName = sortParams[3]; } else { if (spatialQuery == null) { spatialFieldName = Constants.DefaultSpatialFieldName; } else { spatialFieldName = spatialQuery.SpatialFieldName; } } spatialField = viewGenerator.GetSpatialField(spatialFieldName); shape = new PointImpl(lng, lat, spatialField.GetContext()); } var dsort = new SpatialDistanceFieldComparatorSource(spatialField, shape.GetCenter()); return new SortField(sortedField.Field, dsort, sortedField.Descending); } var sortOptions = GetSortOption(indexDefinition, sortedField.Field, self); if (sortOptions == null || sortOptions == SortOptions.None) { return new SortField(sortedField.Field, CultureInfo.InvariantCulture, sortedField.Descending); } if (sortOptions.Value == SortOptions.Short) { sortOptions = SortOptions.Int; } return new SortField(sortedField.Field, (int)sortOptions.Value, sortedField.Descending); }) .ToArray())); }
public static async Task CreateDefaultModelByMetaTypeIndexAsync(this MetaType @this) { var indexName = Storage.DefaultByMetaTypeNameIndexNameConvention.FormatString(@this.Name); var map = @"from model in docs.{0} select new {{ MetaType = model._shurikenMeta.MetaTypeName }}".FormatString(@this.Name); var indexDefinition = new IndexDefinition { Map = map }; await Storage.DocumentStore.AsyncDatabaseCommands.PutIndexAsync(indexName, indexDefinition); }
public string PutIndex(string name, IndexDefinition definition) { return(PutIndex(name, definition, false)); }