/// <summary>
/// Adds an INCLUDE clause to the index definition with the specified property names.
/// This clause specifies a list of columns which will be included as a non-key part in the index.
/// </summary>
/// <remarks>
/// https://www.postgresql.org/docs/current/sql-createindex.html
/// </remarks>
/// <param name="indexBuilder"> The builder for the index being configured. </param>
/// <param name="propertyNames"> An array of property names to be used in INCLUDE clause. </param>
/// <returns> A builder to further configure the index. </returns>
public static IndexBuilder ForNpgsqlInclude(
[NotNull] this IndexBuilder indexBuilder,
[CanBeNull, ItemNotNull] params string[] propertyNames)
{
Check.NotNull(indexBuilder, nameof(indexBuilder));
Check.NullButNotEmpty(propertyNames, nameof(propertyNames));
indexBuilder.Metadata.Npgsql().IncludeProperties = propertyNames;
return(indexBuilder);
}
/// <summary>
/// The PostgreSQL index collation to be used.
/// </summary>
/// <remarks>
/// https://www.postgresql.org/docs/current/static/indexes-collations.html
/// </remarks>
/// <param name="indexBuilder">The builder for the index being configured.</param>
/// <param name="values">The sort options to use for each column.</param>
/// <returns>A builder to further configure the index.</returns>
public static IndexBuilder UseCollation(
this IndexBuilder indexBuilder,
params string[]?values)
{
Check.NotNull(indexBuilder, nameof(indexBuilder));
Check.NullButNotEmpty(values, nameof(values));
indexBuilder.Metadata.SetCollation(values);
return(indexBuilder);
}
/// <summary>
/// The PostgreSQL index method to be used. Null selects the default (currently btree).
/// </summary>
/// <remarks>
/// http://www.postgresql.org/docs/current/static/sql-createindex.html
/// </remarks>
/// <param name="indexBuilder"> The builder for the index being configured. </param>
/// <param name="method"> The name of the index. </param>
/// <returns> A builder to further configure the index. </returns>
public static IndexBuilder ForNpgsqlHasMethod(
[NotNull] this IndexBuilder indexBuilder,
[CanBeNull] string method)
{
Check.NotNull(indexBuilder, nameof(indexBuilder));
Check.NullButNotEmpty(method, nameof(method));
indexBuilder.Metadata.Npgsql().Method = method;
return(indexBuilder);
}
/// <summary>
/// The PostgreSQL index operators to be used.
/// </summary>
/// <remarks>
/// https://www.postgresql.org/docs/current/static/indexes-opclass.html
/// </remarks>
/// <param name="indexBuilder"> The builder for the index being configured. </param>
/// <param name="operators"> The operators to use for each column. </param>
/// <returns> A builder to further configure the index. </returns>
public static IndexBuilder ForNpgsqlHasOperators(
[NotNull] this IndexBuilder indexBuilder,
[CanBeNull, ItemNotNull] params string[] operators)
{
Check.NotNull(indexBuilder, nameof(indexBuilder));
Check.NullButNotEmpty(operators, nameof(operators));
indexBuilder.Metadata.Npgsql().Operators = operators;
return(indexBuilder);
}
/// <summary>
/// The PostgreSQL index method to be used. Null selects the default (currently btree).
/// </summary>
/// <remarks>
/// http://www.postgresql.org/docs/current/static/sql-createindex.html
/// </remarks>
/// <param name="indexBuilder">The builder for the index being configured.</param>
/// <param name="method">The name of the index.</param>
/// <returns>A builder to further configure the index.</returns>
public static IndexBuilder HasMethod(
this IndexBuilder indexBuilder,
string?method)
{
Check.NotNull(indexBuilder, nameof(indexBuilder));
Check.NullButNotEmpty(method, nameof(method));
indexBuilder.Metadata.SetMethod(method);
return(indexBuilder);
}
/// <summary>
/// Adds an INCLUDE clause to the index definition with property names from the specified expression.
/// This clause specifies a list of columns which will be included as a non-key part in the index.
/// </summary>
/// <remarks>
/// https://www.postgresql.org/docs/current/sql-createindex.html
/// </remarks>
/// <param name="indexBuilder"> The builder for the index being configured. </param>
/// <param name="includeExpression">
/// <para>
/// A lambda expression representing the property(s) to be included in the INCLUDE clause
/// (<c>blog => blog.Url</c>).
/// </para>
/// <para>
/// If multiple properties are to be included then specify an anonymous type including the
/// properties (<c>post => new { post.Title, post.BlogId }</c>).
/// </para>
/// </param>
/// <returns> A builder to further configure the index. </returns>
public static IndexBuilder <TEntity> ForNpgsqlInclude <TEntity>(
[NotNull] this IndexBuilder <TEntity> indexBuilder,
[NotNull] Expression <Func <TEntity, object> > includeExpression)
{
Check.NotNull(indexBuilder, nameof(indexBuilder));
Check.NotNull(includeExpression, nameof(includeExpression));
indexBuilder.ForNpgsqlInclude(includeExpression.GetPropertyAccessList().Select(x => x.Name).ToArray());
return(indexBuilder);
}
/// <summary>
/// Adds an INCLUDE clause to the index definition with the specified property names.
/// This clause specifies a list of columns which will be included as a non-key part in the index.
/// </summary>
/// <remarks>
/// https://www.postgresql.org/docs/current/sql-createindex.html
/// </remarks>
/// <param name="indexBuilder">The builder for the index being configured.</param>
/// <param name="propertyNames">An array of property names to be used in INCLUDE clause.</param>
/// <returns>A builder to further configure the index.</returns>
public static IndexBuilder <TEntity> IncludeProperties <TEntity>(
this IndexBuilder <TEntity> indexBuilder,
params string[] propertyNames)
{
Check.NotNull(indexBuilder, nameof(indexBuilder));
Check.NullButNotEmpty(propertyNames, nameof(propertyNames));
indexBuilder.Metadata.SetIncludeProperties(propertyNames);
return(indexBuilder);
}
/// <summary>
/// Adds an INCLUDE clause to the index definition with the specified property names.
/// This clause specifies a list of columns which will be included as a non-key part in the index.
/// </summary>
/// <remarks>
/// https://www.postgresql.org/docs/current/sql-createindex.html
/// </remarks>
/// <param name="indexBuilder">The builder for the index being configured.</param>
/// <param name="propertyNames">An array of property names to be used in INCLUDE clause.</param>
/// <returns>A builder to further configure the index.</returns>
public static IndexBuilder IncludeProperties(
[NotNull] this IndexBuilder indexBuilder,
[NotNull, ItemNotNull] params string[] propertyNames)
{
Check.NotNull(indexBuilder, nameof(indexBuilder));
Check.NullButNotEmpty(propertyNames, nameof(propertyNames));
indexBuilder.Metadata.SetIncludeProperties(propertyNames);
return(indexBuilder);
}
/// <summary>
/// The PostgreSQL index operators to be used.
/// </summary>
/// <remarks>
/// https://www.postgresql.org/docs/current/static/indexes-opclass.html
/// </remarks>
/// <param name="indexBuilder">The builder for the index being configured.</param>
/// <param name="operators">The operators to use for each column.</param>
/// <returns>A builder to further configure the index.</returns>
public static IndexBuilder HasOperators(
this IndexBuilder indexBuilder,
params string[]?operators)
{
Check.NotNull(indexBuilder, nameof(indexBuilder));
Check.NullButNotEmpty(operators, nameof(operators));
indexBuilder.Metadata.SetOperators(operators);
return(indexBuilder);
}
public static IndexBuilder ForSqlServer(
[NotNull] this IndexBuilder indexBuilder,
[NotNull] Action <SqlServerIndexBuilder> builderAction)
{
Check.NotNull(indexBuilder, nameof(indexBuilder));
Check.NotNull(builderAction, nameof(builderAction));
builderAction(ForSqlServer(indexBuilder));
return(indexBuilder);
}
/// <summary>
/// Configures index include properties when targeting SQL Server.
/// </summary>
/// <param name="indexBuilder"> The builder for the index being configured. </param>
/// <param name="includeExpression">
/// <para>
/// A lambda expression representing the property(s) to be included in the 'include' clause
/// (<c>blog => blog.Url</c>).
/// </para>
/// <para>
/// If multiple properties are to be included then specify an anonymous type including the
/// properties (<c>post => new { post.Title, post.BlogId }</c>).
/// </para>
/// </param>
/// <returns> A builder to further configure the index. </returns>
public static IndexBuilder <TEntity> IncludeProperties <TEntity>(
[NotNull] this IndexBuilder <TEntity> indexBuilder, [NotNull] Expression <Func <TEntity, object> > includeExpression)
{
Check.NotNull(indexBuilder, nameof(indexBuilder));
Check.NotNull(includeExpression, nameof(includeExpression));
IncludeProperties(
indexBuilder,
includeExpression.GetPropertyAccessList().Select(MemberInfoExtensions.GetSimpleMemberName).ToArray());
return(indexBuilder);
}
private static void CreateIndex()
{
var db = _Database;
// # tag::query-index[]
// For value types, this is optional but provides performance enhancements
var index = IndexBuilder.ValueIndex(
ValueIndexItem.Expression(Expression.Property("type")),
ValueIndexItem.Expression(Expression.Property("name")));
db.CreateIndex("TypeNameIndex", index);
// # end::query-index[]
}
public void Dispose()
{
if (!IsDisposing)
{
IsDisposing = true;
Status = IndexStatus.Disposing;
TokenSource.Cancel();
TokenSource.Dispose();
FilesWatcher?.Dispose();
IndexBuilder?.Dispose();
Status = IndexStatus.Disposed;
}
}
static void BuildDawgIndex()
{
var reader = new CorpusZipReader <IEnumerable <string> >(tokenizedPath, tokenizedDataSerializer);
var buildableIndex = new DawgBuildableIndex();
var indexBuilder = new IndexBuilder <string, IEnumerable <string> >(buildableIndex);
indexBuilder.IndexCorpus(reader.Read());
var index = buildableIndex.Build();
using var file = File.Create(dawgIndexPath);
index.Serialize(file);
}
private void CreateDatabaseIndexes(Database db)
{
// For searches on type property
db.CreateIndex("type", IndexBuilder.ValueIndex(ValueIndexItem.Expression(Expression.Property("type"))));
// For full text searches on airports and hotels
db.CreateIndex("airportName",
IndexBuilder.FullTextIndex(FullTextIndexItem.Property("airportname")));
db.CreateIndex("description",
IndexBuilder.FullTextIndex(FullTextIndexItem.Property("description")));
db.CreateIndex("name",
IndexBuilder.FullTextIndex(FullTextIndexItem.Property("name")));
}
static void ProcessAndIndexWikipedia()
{
var reader = new CorpusZipReader <IList <char> >(wikiPath, charDataSerializer);
var index = new DictionaryIndex <int>(rareWordThreshold: 5);
var indexBuilder = new IndexBuilder <int, IEnumerable <int> >(index);
var processor = new WikitextProcessor();
indexBuilder.IndexCorpus(processor.Transform(reader.Read()));
Console.WriteLine("Serializing index...");
using var file = File.Create(indexPath);
index.Serialize(file);
}
public void TestConcurrentCreateNCreateIndexDB()
{
const int nDocs = 1000;
var exp1 = new WaitAssert();
var ignore = exp1.RunAssertAsync(() =>
{
CreateDocs(nDocs, "Create").ToList();
});
Db.CreateIndex("sentence", IndexBuilder.FullTextIndex(FullTextIndexItem.Property("sentence")));
exp1.WaitForResult(TimeSpan.FromSeconds(60));
}
async Task MaintainIndexesCore()
{
FetchIntervalSeconds.RequireRange(nameof(FetchIntervalSeconds), 3, 1);
while (!TokenSource.Token.IsCancellationRequested)
{
TokenSource.Token.ThrowIfCancellationRequested();
var fetchEndDate = DateTime.UtcNow.AddSeconds(-FetchIntervalSeconds * 2);
var notChangedDuring = fetchEndDate.AddSeconds(FetchIntervalSeconds);
if (ChangedSources.Count(u => u.ChangedUTCDate > fetchEndDate && u.ChangedUTCDate <= notChangedDuring) == 0)
{
var orderedNeedProcessingChanges = ChangedSources.Where(u => u.ChangedUTCDate <= fetchEndDate).ToList();
if (orderedNeedProcessingChanges.Count > 0)
{
foreach (var _ in orderedNeedProcessingChanges)
{
ChangedSources.TryDequeue(out var _);
}
ProcessingChanges(orderedNeedProcessingChanges);
IndexBuilder.Commit();
TriggerCommitFinished();
}
}
TokenSource.Token.ThrowIfCancellationRequested();
var fetchRetryEndDate = DateTime.UtcNow.AddSeconds(-3);
var needRetry = PendingRetryCodeSources.Count(u => u.LastRetryUTCDate <= fetchRetryEndDate);
if (needRetry > 0)
{
var needRetrySources = new List <ChangedSource>();
for (var index = 0; index < needRetry; index++)
{
if (PendingRetryCodeSources.TryDequeue(out var pendingRetrySource))
{
needRetrySources.Add(pendingRetrySource);
}
}
ProcessingChanges(needRetrySources.OrderBy(u => u.ChangedUTCDate).ToList(), true);
IndexBuilder.Commit();
}
await Task.Delay(FetchIntervalSeconds * 1000, TokenSource.Token);
}
}
/// <summary>
/// The PostgreSQL index sort ordering to be used.
/// </summary>
/// <remarks>
/// https://www.postgresql.org/docs/current/static/indexes-ordering.html
/// </remarks>
/// <param name="indexBuilder">The builder for the index being configured.</param>
/// <param name="values">The sort order to use for each column.</param>
/// <returns>A builder to further configure the index.</returns>
public static IndexBuilder HasSortOrder(
this IndexBuilder indexBuilder,
params SortOrder[]?values)
{
Check.NotNull(indexBuilder, nameof(indexBuilder));
Check.NullButNotEmpty(values, nameof(values));
if (!SortOrderHelper.IsDefaultSortOrder(values))
{
indexBuilder.Metadata.SetSortOrder(values);
}
return(indexBuilder);
}
private static void UsePredictiveIndex()
{
using (var db = new Database("mydb")) {
// tag::predictive-query-predictive-index[]
var input = Expression.Dictionary(new Dictionary <string, object>
{
["photo"] = Expression.Property("photo")
});
var index = IndexBuilder.PredictiveIndex("ImageClassifier", input);
db.CreateIndex("predictive-index-image-classifier", index);
// end::predictive-query-predictive-index[]
}
}
private void btnLuceneIndex_Click(object sender, EventArgs e)
{
btnLuceneIndex.Enabled = false;
Task.Run(() =>
{
IndexBuilder.Build();
}).ContinueWith(t =>
{
base.Invoke(new Action(() =>
{
btnLuceneIndex.Enabled = true;
}));
});
}
public void add_single_word_and_check_node_count()
{
using (var headerStream = new MemoryStream())
{
using (var indexStream = new MemoryStream())
{
var builder = new IndexBuilder(headerStream, indexStream);
builder.Add("John");
var processedNodeCount = builder.Build();
Assert.AreEqual(5, processedNodeCount);
}
}
}
public void add_range_and_check_processes_node_count_are_equal()
{
using (var headerStream = new MemoryStream())
{
using (var indexStream = new MemoryStream())
{
var builder = new IndexBuilder(headerStream, indexStream);
builder.AddRange(new string[] { "John", "Smith", "Jane" });
var processedNodeCount = builder.Build();
Assert.AreEqual(13, processedNodeCount);
}
}
}
public void IndexBuilderTest()
{
var docs1 = new List <Document <string> >
{
new Document <string> (
new DocumentMetadata(new DocumentId(0), "Title 1"),
"abc"
),
new Document <string> (
new DocumentMetadata(new DocumentId(1), "Title 2"),
"defa"
),
};
var docs2 = new List <Document <string> >
{
new Document <string> (
new DocumentMetadata(new DocumentId(10), "Title 3"),
"bd"
),
};
var corpus = new List <Block <string> >
{
Block <string> .Make(0, docs1),
Block <string> .Make(1, docs2),
};
var indexedWords = new List <(DocumentId, char, int)>();
var index = new Mock <IBuildableIndex <char> >();
index.Setup(x => x.IndexTerm(It.IsAny <DocumentId>(), It.IsAny <char>(), It.IsAny <int>()))
.Callback <DocumentId, char, int>((id, term, pos) => indexedWords.Add((id, term, pos)));
var indexer = new IndexBuilder <char, string>(index.Object);
indexer.IndexCorpus(corpus);
var expectedIndexedWords = new List <(DocumentId, char, int)>
{
(new DocumentId(0), 'a', 0), (new DocumentId(0), 'b', 1), (new DocumentId(0), 'c', 2),
(new DocumentId(1), 'd', 0), (new DocumentId(1), 'e', 1), (new DocumentId(1), 'f', 2), (new DocumentId(1), 'a', 3),
(new DocumentId(10), 'b', 0), (new DocumentId(10), 'd', 1),
};
Assert.Equal(expectedIndexedWords, indexedWords);
}
public string StartIndexing()
{
SetConnectionString();
var indexuilder = IndexBuilder.CreateIndexer(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "indexer"));
var tobeIndexedTable = TableStore.Instance.GetTable(TableStore.ToBeIndexedTableName) as ToBeIndexedTable;
var movies = tobeIndexedTable.GetMoviesTobeIndexed();
var reviews = tobeIndexedTable.GetReviewsToBeIndexed();
indexuilder.IndexSelectedMovies(movies);
indexuilder.IndexSelectedReviews(reviews);
return("done in " + Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "indexer"));
}
private void button4_Click(object sender, EventArgs e)
{
try
{
LogHelper.WriteLog("开始构建索引");
IndexBuilder.Build();
}
catch (Exception ex)
{
LogHelper.WriteLog("构建索引出错,出错原因:" + ex.Message, ex);
}
finally
{
LogHelper.WriteLog("结束构建索引");
}
}
public static IndexBuilder Include <TEntity>(this IndexBuilder indexBuilder, Expression <Func <TEntity, object> > indexExpression)
{
var includeStatement = new StringBuilder();
foreach (var column in indexExpression.GetPropertyAccessList())
{
if (includeStatement.Length > 0)
{
includeStatement.Append(", ");
}
includeStatement.AppendFormat("[{0}]", column.Name);
}
indexBuilder.HasAnnotation("SqlServer:IncludeIndex", includeStatement.ToString());
return(indexBuilder);
}
public static IndexFieldBuilder Create(IndexBuilder builder, string name)
{
if (builder is null)
{
throw new ArgumentNullException(nameof(builder));
}
if (string.IsNullOrWhiteSpace(name))
{
throw new ArgumentException(nameof(name));
}
var field = new Domain.IndexField(name);
return(new IndexFieldBuilder(builder, field));
}
/// <summary>
/// The PostgreSQL index NULL sort ordering to be used.
/// </summary>
/// <remarks>
/// https://www.postgresql.org/docs/current/static/indexes-ordering.html
/// </remarks>
/// <param name="indexBuilder">The builder for the index being configured.</param>
/// <param name="values">The sort order to use for each column.</param>
/// <returns>A builder to further configure the index.</returns>
public static IndexBuilder HasNullSortOrder(
[NotNull] this IndexBuilder indexBuilder,
[CanBeNull] params NullSortOrder[] values)
{
Check.NotNull(indexBuilder, nameof(indexBuilder));
Check.NullButNotEmpty(values, nameof(values));
var sortOrders = indexBuilder.Metadata.GetSortOrder();
if (!SortOrderHelper.IsDefaultNullSortOrder(values, sortOrders))
{
indexBuilder.Metadata.SetNullSortOrder(values);
}
return(indexBuilder);
}
public async Task BuildNuGetIndex()
{
const string outputFolder = "BuildNuGetIndexTest";
// Arrange
var builder = new IndexBuilder();
builder.Sources.Add(new IndexSourceEntry(TestNuGetFeedUrl));
// Act
var index = await builder.BuildAsync(outputFolder);
// Assert
Assert.True(index.TryGetNuGetSourcePath(TestNuGetFeedUrl, out _));
Assert.True(Directory.Exists(index.BaseUrl.LocalPath));
}
/// <summary>
/// Rebuild the full-text index.
/// </summary>
/// <param name="contentItemsToIndex"></param>
public void RebuildIndex(Site site, IEnumerable<IContentItem> contentItemsToIndex)
{
//Site currentSite = this._cuyahogaContextProvider.GetContext().CurrentSite;
string indexDirectory = GetIndexDirectoryBySite(site);
log.DebugFormat("SearchService.RebuildIndex: Site: {0}, indexDirectory = {1}", site.Name, indexDirectory);
using (IndexBuilder indexBuilder = new IndexBuilder(indexDirectory, true, this._textExtractor))
{
// Add all content items
//var contentItemsToIndex = this._contentItemService.FindAllBySite(currentSite);
foreach (IContentItem contentItem in contentItemsToIndex)
{
// Index ONLY published items
if (contentItem.PublishedDate.HasValue && contentItem is ISearchableContent)
{
try
{
indexBuilder.AddContent(contentItem);
}
catch (Exception ex)
{
log.Error(string.Format("SearchService.FindContent: Error while indexing ContentItem with id {0}", contentItem.Id), ex);
throw;
}
}
}
log.Debug("SearchService.FindContent: Optimizing index...");
indexBuilder.Optimize();
}
}
private static int CreateIndexBuilder(IRuntime runtime, int manager, HttpListenerResponse response, string path, int reader)
{
// item 883
var builder = new IndexBuilder();
builder.Manager = manager;
builder.FolderReader = reader;
builder.Folder = path;
builder.Response = response;
// item 884
int actorId = runtime.AddActor(builder);
// item 886
runtime.SendMessage(
actorId,
Codes.Start,
null,
0
);
// item 885
return actorId;
}
/// <summary>
/// Rebuild the full-text index.
/// </summary>
/// <param name="searchableModules">A list of (legacy) searchable modules in the installation.</param>
public void RebuildIndex(IEnumerable<ISearchable> searchableModules)
{
Site currentSite = this._cuyahogaContextProvider.GetContext().CurrentSite;
string indexDirectory = GetIndexDirectory();
using (IndexBuilder indexBuilder = new IndexBuilder(indexDirectory, true, this._textExtractor))
{
// Add all content items
var contentItemsToIndex = this._contentItemService.FindContentItemsBySite(currentSite);
foreach (IContentItem contentItem in contentItemsToIndex)
{
if (contentItem is ISearchableContent)
{
try
{
indexBuilder.AddContent(contentItem);
}
catch (Exception ex)
{
Logger.Error(string.Format("Error while indexing ContentItem with id {0}", contentItem.Id), ex);
throw;
}
}
}
// Add legacy searchable content
if (searchableModules != null)
{
foreach (ISearchable searchableModule in searchableModules)
{
foreach (SearchContent searchContent in searchableModule.GetAllSearchableContent())
{
try
{
indexBuilder.AddContent(searchContent);
}
catch (Exception ex)
{
Logger.Error(string.Format("Indexing of legacy searchContent item with path {0} failed.", searchContent.Path),
ex);
throw;
}
}
}
}
}
}