public async Task RemoveOutdatedContent(DateTimeOffset olderThan, IList <ContentReference> contentReferencesToPreserve)
{
const int maxContentReferencesToPreserve = 1000;
if (contentReferencesToPreserve.Count > maxContentReferencesToPreserve)
{
throw new ArgumentException($"Number of content references to preserve cannot be greater than {maxContentReferencesToPreserve}", nameof(contentReferencesToPreserve));
}
const int topResults = 1000;
var query = new AzureSearchQueryBuilder()
.Top(topResults)
.Filter(new NotFilter(new AzureSearchFilterSearchIn(nameof(ContentDocument.ContentComplexReference), contentReferencesToPreserve.Select(reference => reference.ToString()))))
.Filter(AzureSearchQueryFilter.LessThan(nameof(ContentDocument.IndexedAt), new DateTimeOffsetPropertyValue(olderThan)))
.Build();
var deletedDocumentsCount = await DeleteDocuments(query);
if (deletedDocumentsCount == topResults)
{
do
{
deletedDocumentsCount = await DeleteDocuments(query);
} while (deletedDocumentsCount > 0);
}
}
public void WithSearchInFilter_WithValueNullOrWhiteSpaceValue_ThrowsException(string value)
{
var builder = new AzureSearchQueryBuilder("TestSearchText");
Func <AzureSearchQueryBuilder> action = () => builder.WithSearchInFilter("TestVariable1", new[] { "TestValue1", value, "TestValue3" });
action.Should().ThrowExactly <ArgumentException>();
}
public void WithSearchInFilter_WithValuesContainsDelimiter_ThrowsException()
{
var builder = new AzureSearchQueryBuilder("TestSearchText");
Func <AzureSearchQueryBuilder> action = () => builder.WithSearchInFilter("TestVariable1", new[] { "TestValue1", "Test|Value2", "TestValue3" });
action.Should().ThrowExactly <ArgumentException>();
}
public void WithSize_WithInvalidSkip_ThrowsException(int?skip)
{
var builder = new AzureSearchQueryBuilder("TestSearchText");
Func <AzureSearchQueryBuilder> action = () => builder.WithSkip(skip);
action.Should().ThrowExactly <ArgumentOutOfRangeException>();
}
public void Build_WithSearchText_ReturnsQueryWithExpectedSearchText(string searchText, string expectedResult)
{
var builder = new AzureSearchQueryBuilder(searchText);
var result = builder.Build();
result.SearchText.Should().Be(expectedResult);
}
public void Build_WithSingleSearchInFilterAndMultipleValues_ReturnsQueryWithExpectedFilter()
{
var builder = new AzureSearchQueryBuilder("TestSearchText")
.WithSearchInFilter("TestVariable1", new[] { "TestValue1", "TestValue2", "TestValue3" });
var result = builder.Build();
result.Options.Filter.Should().Be("search.in(TestVariable1, 'TestValue1|TestValue2|TestValue3', '|')");
}
public void Build_WithDelimiter_ReturnsQueryWithExpectedFilter()
{
var builder = new AzureSearchQueryBuilder("TestSearchText")
.WithSearchInFilter("TestVariable1", new[] { "TestValue1", "TestValue2", "TestValue3" }, ',');
var result = builder.Build();
result.Options.Filter.Should().Be("search.in(TestVariable1, 'TestValue1,TestValue2,TestValue3', ',')");
}
public void Build_WithInvludeTotalCount_ReturnsQueryWithExpectedTotalCount(bool?includeTotalCount)
{
var builder = new AzureSearchQueryBuilder("TestSearchText")
.WithIncludeTotalCount(includeTotalCount);
var result = builder.Build();
result.Options.IncludeTotalCount.Should().Be(includeTotalCount);
}
public void Build_WithSearchMode_ReturnsQueryWithExpectedSearchMode(SearchMode?searchMode)
{
var builder = new AzureSearchQueryBuilder("TestSearchText")
.WithSearchMode(searchMode);
var result = builder.Build();
result.Options.SearchMode.Should().Be(searchMode);
}
public void Build_WithSkip_ReturnsQueryWithExpectedSkip(int?skip)
{
var builder = new AzureSearchQueryBuilder("TestSearchText")
.WithSkip(skip);
var result = builder.Build();
result.Options.Skip.Should().Be(skip);
}
public void Build_WithScoringProfile_ReturnsQueryWithExpectedScoringProfile(string scoringProfile)
{
var builder = new AzureSearchQueryBuilder("TestSearchText")
.WithScoringProfile(scoringProfile);
var result = builder.Build();
result.Options.ScoringProfile.Should().Be(scoringProfile);
}
public void Build_WithSize_ReturnsQueryWithExpectedSize(int?size)
{
var builder = new AzureSearchQueryBuilder("TestSearchText")
.WithSize(size);
var result = builder.Build();
result.Options.Size.Should().Be(size);
}
public void Can_create_azuresearch_index()
{
var scope = "test";
var queryBuilder = new AzureSearchQueryBuilder();
var conn = new SearchConnection(Datasource, scope, accessKey: AccessKey);
var provider = new AzureSearchProvider(queryBuilder, conn);
SearchHelper.CreateSampleIndex(provider, scope);
provider.RemoveAll(scope, String.Empty);
}
/// <summary>
/// Method filters search items by allowed roles and users
/// </summary>
/// <param name="queryBuilder"></param>
/// <returns></returns>
public static AzureSearchQueryBuilder FilterOnRoleAccess(this AzureSearchQueryBuilder queryBuilder)
{
var currentPrincipalRoles = PrincipalInfo.Current?.RoleList ?? Enumerable.Empty <string>();
var rolesAccessFilters = currentPrincipalRoles
.Select(roleName => new AzureSearchQueryFilter(nameof(ContentDocument.AccessRoles),
ComparisonExpression.Eq, roleName)
{
GroupingExpression = GroupingExpression.Any
});
return(queryBuilder.Filter(new FilterComposite(Operator.Or, rolesAccessFilters)));
}
public (string SearchText, SearchOptions Options) GenerateSearchQuery()
{
var builder = new AzureSearchQueryBuilder(LearnAimRef)
.WithSearchMode(SearchMode.All)
.WithSearchFields(nameof(Lars.LearnAimRef))
.WithSize(1);
if (CertificationEndDateFilter.HasValue)
{
builder.WithFilters($"({nameof(Lars.CertificationEndDate)} ge {CertificationEndDateFilter.Value:O} or {nameof(Lars.CertificationEndDate)} eq null)");
}
return(builder.Build());
}
private static AzureSearchQuery CreateSearchQuery(EPiServer.Shell.Search.Query query, CultureInfo currentCulture)
{
var queryBuilder = new AzureSearchQueryBuilder()
.Top(query.MaxResults)
.SearchTerm(query.SearchQuery);
if (query.FilterOnCulture)
{
queryBuilder.Filter(AzureSearchQueryFilter.Equals(nameof(ContentDocument.ContentLanguage),
currentCulture.Name));
}
return(queryBuilder.Build());
}
public void Build_ReturnsExpectedQuery()
{
var searchFields = new[]
{
"TestSearchField1",
"TestSearchField2",
"TestSearchField3"
};
var facets = new[]
{
"TestFacet1",
"TestFacet2",
"TestFacet3"
};
var orderBy = new[]
{
"TestOrderBy1",
"TestOrderBy2",
"TestOrderBy3"
};
var builder = new AzureSearchQueryBuilder("TestSearchText")
.WithSearchMode(SearchMode.All)
.WithSearchFields(searchFields)
.WithFacets(facets)
.WithSearchInFilter("TestVariable1", new[] { "TestValue1" })
.WithSearchInFilter("TestVariable2", new[] { "TestValue2", "TestValue3" })
.WithSearchInFilter("TestVariable3", new[] { "TestValue4", "TestValue5", "TestValue6" }, ',')
.WithOrderBy(orderBy)
.WithScoringProfile("TestScoringProfile")
.WithSize(12)
.WithSkip(34)
.WithIncludeTotalCount();
var result = builder.Build();
result.SearchText.Should().Be("TestSearchText");
result.Options.SearchMode.Should().Be(SearchMode.All);
result.Options.SearchFields.Should().Equal(searchFields);
result.Options.Facets.Should().Equal(facets);
result.Options.Filter.Should().Be("search.in(TestVariable1, 'TestValue1', '|') and search.in(TestVariable2, 'TestValue2|TestValue3', '|') and search.in(TestVariable3, 'TestValue4,TestValue5,TestValue6', ',')");
result.Options.OrderBy.Should().Equal(orderBy);
result.Options.ScoringProfile.Should().Be("TestScoringProfile");
result.Options.Size.Should().Be(12);
result.Options.Skip.Should().Be(34);
result.Options.IncludeTotalCount.Should().BeTrue();
}
public void Build_WithSearchFields_ReturnsQueryWithExpectedSearchFields()
{
var searchFields = new[]
{
"TestSearchField1",
"TestSearchField2",
"TestSearchField3"
};
var builder = new AzureSearchQueryBuilder("TestSearchText")
.WithSearchFields(searchFields);
var result = builder.Build();
result.Options.SearchFields.Should().Equal(searchFields);
}
public void Build_WithFacets_ReturnsQueryWithExpectedFacets()
{
var facets = new[]
{
"TestFacet1",
"TestFacet2",
"TestFacet3"
};
var builder = new AzureSearchQueryBuilder("TestSearchText")
.WithFacets(facets);
var result = builder.Build();
result.Options.Facets.Should().Equal(facets);
}
public void Build_WithOrderBy_ReturnsQueryWithExpectedOrderBy()
{
var orderBy = new[]
{
"TestOrderBy1",
"TestOrderBy2",
"TestOrderBy3"
};
var builder = new AzureSearchQueryBuilder("TestSearchText")
.WithOrderBy(orderBy);
var result = builder.Build();
result.Options.OrderBy.Should().Equal(orderBy);
}
protected ISearchProvider GetSearchProvider(string searchProvider, string scope, string dataSource = null)
{
ISearchProvider provider = null;
var phraseSearchCriteriaPreprocessor = new PhraseSearchCriteriaPreprocessor(new SearchPhraseParser()) as ISearchCriteriaPreprocessor;
var catalogSearchCriteriaPreprocessor = new CatalogSearchCriteriaPreprocessor();
var searchCriteriaPreprocessors = new[] { phraseSearchCriteriaPreprocessor, catalogSearchCriteriaPreprocessor };
if (searchProvider == "Lucene")
{
var connection = new SearchConnection(_luceneStorageDir, scope);
var queryBuilder = new LuceneSearchQueryBuilder() as ISearchQueryBuilder;
provider = new LuceneSearchProvider(new[] { queryBuilder }, connection, searchCriteriaPreprocessors);
}
if (searchProvider == "Elastic")
{
var elasticsearchHost = dataSource ?? Environment.GetEnvironmentVariable("TestElasticsearchHost") ?? "localhost:9200";
var connection = new SearchConnection(elasticsearchHost, scope);
var queryBuilder = new ElasticSearchQueryBuilder() as ISearchQueryBuilder;
var elasticSearchProvider = new ElasticSearchProvider(connection, searchCriteriaPreprocessors, new[] { queryBuilder }, GetSettingsManager())
{
EnableTrace = true
};
provider = elasticSearchProvider;
}
if (searchProvider == "Azure")
{
var azureSearchServiceName = Environment.GetEnvironmentVariable("TestAzureSearchServiceName");
var azureSearchAccessKey = Environment.GetEnvironmentVariable("TestAzureSearchAccessKey");
var connection = new SearchConnection(azureSearchServiceName, scope, accessKey: azureSearchAccessKey);
var queryBuilder = new AzureSearchQueryBuilder() as ISearchQueryBuilder;
provider = new AzureSearchProvider(connection, searchCriteriaPreprocessors, new[] { queryBuilder });
}
if (provider == null)
{
throw new ArgumentException($"Search provider '{searchProvider}' is not supported", nameof(searchProvider));
}
return(provider);
}
public void Can_find_item_azuresearch()
{
var scope = "test";
var queryBuilder = new AzureSearchQueryBuilder();
var conn = new SearchConnection(Datasource, scope, accessKey: AccessKey);
var provider = new AzureSearchProvider(queryBuilder, conn);
provider.RemoveAll(scope, String.Empty);
SearchHelper.CreateSampleIndex(provider, scope);
var criteria = new CatalogItemSearchCriteria
{
SearchPhrase = "product",
IsFuzzySearch = true,
Catalog = "goods",
RecordsToRetrieve = 10,
StartingRecord = 0,
Pricelists = new string[] { }
};
// force delay, otherwise records are not available
Thread.Sleep(1000);
var results = provider.Search(scope, criteria);
Assert.True(results.DocCount == 1, String.Format("Returns {0} instead of 1", results.DocCount));
criteria = new CatalogItemSearchCriteria
{
SearchPhrase = "sample product ",
IsFuzzySearch = true,
Catalog = "goods",
RecordsToRetrieve = 10,
StartingRecord = 0,
Pricelists = new string[] { }
};
results = provider.Search(scope, criteria);
Assert.True(results.DocCount == 1, String.Format("\"Sample Product\" search returns {0} instead of 1", results.DocCount));
provider.RemoveAll(scope, String.Empty);
}
public (string SearchText, SearchOptions Options) GenerateSearchQuery()
{
var searchTerms = SearchText.TransformSegments(s => s
.TransformWhen(s => s.EndsWith("'s"), s => s.Substring(0, s.Length - 2))
.TransformWhen(s => s.EndsWith("s'"), s => s.Substring(0, s.Length - 2))
.TransformWhen(s => s.EndsWith("s"), s => s.Substring(0, s.Length - 1))
.RemoveNonAlphanumericChars()
.AppendWildcardWhenLastCharIsAlphanumeric());
var searchText = searchTerms.Any()
? $"\"{SearchText.Trim().EscapeSimpleQuerySearchOperators()}\" | ({searchTerms})"
: !string.IsNullOrWhiteSpace(SearchText)
? $"{SearchText.Trim().EscapeSimpleQuerySearchOperators()}*"
: "*";
var builder = new AzureSearchQueryBuilder(searchText)
.WithSearchMode(SearchMode.All)
.WithSearchFields(SearchFields?.ToArray())
.WithSearchInFilter(nameof(Lars.NotionalNVQLevelv2), NotionalNVQLevelv2Filters)
.WithSearchInFilter(nameof(Lars.AwardOrgCode), AwardOrgCodeFilters)
.WithSearchInFilter(nameof(Lars.AwardOrgAimRef), AwardOrgAimRefFilters)
.WithSearchInFilter(nameof(Lars.SectorSubjectAreaTier1), SectorSubjectAreaTier1Filters)
.WithSearchInFilter(nameof(Lars.SectorSubjectAreaTier2), SectorSubjectAreaTier2Filters)
.WithFacets(Facets?.ToArray())
.WithSize(PageSize)
.WithIncludeTotalCount();
if (CertificationEndDateFilter.HasValue)
{
builder.WithFilters($"({nameof(Lars.CertificationEndDate)} ge {CertificationEndDateFilter.Value:O} or {nameof(Lars.CertificationEndDate)} eq null)");
}
if (PageNumber.HasValue)
{
builder.WithSkip(PageSize * (PageNumber - 1));
}
return(builder.Build());
}
public void Can_find_items_azuresearch()
{
var scope = "default";
var queryBuilder = new AzureSearchQueryBuilder();
var conn = new SearchConnection(Datasource, scope, accessKey: AccessKey);
var provider = new AzureSearchProvider(queryBuilder, conn);
var criteria = new CatalogItemSearchCriteria
{
SearchPhrase = "sony",
IsFuzzySearch = true,
Catalog = "vendorvirtual",
RecordsToRetrieve = 10,
StartingRecord = 0,
Pricelists = new string[] { },
StartDate = DateTime.UtcNow,
Sort = new SearchSort("price_usd_saleusd")
};
criteria.Outlines.Add("vendorvirtual*");
//"(startdate lt 2014-09-26T22:05:11Z) and sys__hidden eq 'false' and sys__outline/any(t:t eq 'VendorVirtual/e1b56012-d877-4bdd-92d8-3fc186899d0f*') and catalog/any(t: t eq 'VendorVirtual')"
var results = provider.Search(scope, criteria);
}
public void GivenQueryBuilder_ThenFilterValueIsProper(AzureSearchQueryBuilder builder, string expectedFilterValue)
{
var query = builder.Build();
Assert.That(query.Filter, Is.EqualTo(expectedFilterValue));
}
