private static IndexingParameters Configure(IndexingParameters parameters, string key, object value)
{
Argument.AssertNotNull(parameters, nameof(parameters));
parameters.Configuration[key] = value;
return(parameters);
}
/// <summary>
/// Modifie la valeur d'un élément d'une collection.
/// L'élément est désigné par les paramètres d'indexation.
/// </summary>
/// <param name="var"></param>
/// <param name="indexingParameters"></param>
void SetElementValue(object val, object newValue, Context context)
{
dynamic newValueDynamic = newValue;
dynamic index;
// TODO : support des arrays multi-dimensionels.
if (IndexingParameters.Count > 1)
{
for (int i = 0; i <= IndexingParameters.Count - 1; i++)
{
List <IGettable> parameters = IndexingParameters[i];
if (parameters.Count == 0)
{
throw new Exception("Expected indexing parameter");
}
// On laisse faire le runtime pour la résolution dynamique =D
dynamic tmp = val;
index = IndexingParameters[0][0].GetValue(context);
val = tmp[index];
}
dynamic enumValFinal = val;
index = IndexingParameters.Last().First().GetValue(context);
enumValFinal[index] = newValueDynamic;
}
else if (IndexingParameters.Count == 1)
{
dynamic enumValFinal = val;
index = IndexingParameters.First().First().GetValue(context);
enumValFinal[index] = newValueDynamic;
}
}
public AzurSearchIndexerService(
IOptions <AzureBlobOptions> azureBlobOptions,
IOptions <SearchIndexerClientOptions> searchIndexerClientOptions
)
{
_azureBlobOptions = azureBlobOptions;
_indexingParameters = new();
_indexingParameters.Configuration.Add("parsingMode", "json");
_searchIndexerClient = new SearchIndexerClient(null, null);
_azureBlobIndexer = new SearchIndexer(
name: searchIndexerClientOptions.Value.IndexerName,
dataSourceName: searchIndexerClientOptions.Value.DataSourceName,
targetIndexName: searchIndexerClientOptions.Value.IndexName)
{
Parameters = _indexingParameters,
Schedule = new IndexingSchedule(TimeSpan.FromDays(1))
};
_azureBlobIndexer.FieldMappings.Add(
new FieldMapping("Id")
{
TargetFieldName = "HotelId"
});
_searchIndexerDataContainer = new SearchIndexerDataContainer("hotel-rooms");
}
/// <summary>
/// Tells the indexer to assume that blobs should be parsed as text files in the desired encoding.
/// See <see href="https://docs.microsoft.com/azure/search/search-howto-indexing-azure-blob-storage#indexing-plain-text"/> for details.
/// </summary>
/// <param name="parameters">The <see cref="IndexingParameters"/> to configure.</param>
/// <param name="encoding">Encoding used to read the text stored in blobs.</param>
/// <returns>The <see cref="IndexingParameters"/> instance.</returns>
public static IndexingParameters SetParseText(this IndexingParameters parameters, Encoding encoding)
{
Argument.AssertNotNull(encoding, nameof(encoding));
Configure(parameters, ParsingModeKey, "text");
Configure(parameters, "encoding", encoding.WebName);
return(parameters);
}
public void ParseDelimitedTextFilesWithInlineHeadersSetCorrectly()
{
const int ExpectedCount = 2;
var parameters = new IndexingParameters().ParseDelimitedTextFiles();
AssertHasConfigItem(parameters, ExpectedParsingModeKey, "delimitedText", ExpectedCount);
AssertHasConfigItem(parameters, "firstLineContainsHeaders", true, ExpectedCount);
}
public void ParseDelimitedTextFilesWithGivenHeadersSetCorrectly()
{
const int ExpectedCount = 2;
var parameters = new IndexingParameters().ParseDelimitedTextFiles("id", "name", "address");
AssertHasConfigItem(parameters, ExpectedParsingModeKey, "delimitedText", ExpectedCount);
AssertHasConfigItem(parameters, "delimitedTextHeaders", "id,name,address", ExpectedCount);
}
public void ParseTextSetCorrectly()
{
const int ExpectedCount = 2;
var parameters = new IndexingParameters().ParseText();
AssertHasConfigItem(parameters, ExpectedParsingModeKey, "text", ExpectedCount);
AssertHasConfigItem(parameters, "encoding", "utf-8", ExpectedCount);
}
public void ParseTextWithEncodingSetCorrectly()
{
const int ExpectedCount = 2;
var parameters = new IndexingParameters().ParseText(Encoding.ASCII);
AssertHasConfigItem(parameters, ExpectedParsingModeKey, "text", ExpectedCount);
AssertHasConfigItem(parameters, "encoding", "us-ascii", ExpectedCount);
}
public void ParseJsonArraysWithDocumentRootSetCorrectly()
{
const int ExpectedCount = 2;
var parameters = new IndexingParameters().ParseJsonArrays("/my/path");
AssertHasConfigItem(parameters, ExpectedParsingModeKey, "jsonArray", ExpectedCount);
AssertHasConfigItem(parameters, "documentRoot", "/my/path", ExpectedCount);
}
private static IndexingParameters ConfigureFileNameExtensions(this IndexingParameters parameters, string key, string[] extensions)
{
if (extensions != null && extensions.Length > 0)
{
string delimitedExtensions = ProcessFileNameExtensions(extensions).CommaJoin();
Configure(parameters, key, delimitedExtensions);
}
return(parameters);
}
public void ConfigurationKeysAreReadOnly()
{
IndexingParameters parameters = new IndexingParameters();
ICollection <string> keys = parameters.Configuration.Keys;
Assert.IsTrue(keys.IsReadOnly);
Assert.Throws <NotSupportedException>(() => keys.Add("customTestProperty"));
Assert.Throws <NotSupportedException>(() => keys.Remove("customTestProperty"));
Assert.Throws <NotSupportedException>(() => keys.Clear());
}
public void ParseTabDelimitedTextFilesWithGivenHeadersSetCorrectly()
{
const int ExpectedCount = 3;
IndexingParameters parameters = new IndexingParameters().SetParseDelimitedTextFiles('\t', new[] { "id", "name", "address" });
AssertHasConfigItem(parameters, ExpectedParsingModeKey, "delimitedText", ExpectedCount);
AssertHasConfigItem(parameters, "delimitedTextDelimiter", "\t", ExpectedCount);
AssertHasConfigItem(parameters, "delimitedTextHeaders", "id,name,address", ExpectedCount);
}
public void ConfigurationValuesAreReadOnly()
{
IndexingParameters parameters = new IndexingParameters();
ICollection <object> values = parameters.Configuration.Values;
Assert.IsTrue(values.IsReadOnly);
Assert.Throws <NotSupportedException>(() => values.Add("custom"));
Assert.Throws <NotSupportedException>(() => values.Remove("custom"));
Assert.Throws <NotSupportedException>(() => values.Clear());
}
/// <summary>
/// Tells the indexer to assume that all blobs contain JSON arrays, which it will then parse such that each JSON object in each array will
/// map to a single document in the search index.
/// See <see href="https://docs.microsoft.com/azure/search/search-howto-index-json-blobs" /> for details.
/// </summary>
/// <param name="parameters">The <see cref="IndexingParameters"/> to configure.</param>
/// <param name="documentRoot">
/// An optional <see href="https://tools.ietf.org/html/rfc6901">JSON Pointer</see> that tells the indexer how to find the JSON array if it's not the top-level JSON property of each blob.
/// If this parameter is null or empty, the indexer will assume that the JSON array can be found in the top-level JSON property of each blob.
/// Default is null.
/// </param>
/// <remarks>
/// This option only applies to indexers that index Azure Blob Storage.
/// </remarks>
/// <returns>The <see cref="IndexingParameters"/> instance.</returns>
/// <exception cref="ArgumentNullException"><paramref name="parameters"/> is null.</exception>
public static IndexingParameters SetParseJsonArrays(this IndexingParameters parameters, string documentRoot = null)
{
Configure(parameters, ParsingModeKey, "jsonArray");
if (!string.IsNullOrEmpty(documentRoot))
{
Configure(parameters, "documentRoot", documentRoot);
}
return(parameters);
}
private static void AssertHasConfigItem(
IndexingParameters parameters,
string expectedKey,
object expectedValue,
int expectedCount = 1)
{
IDictionary <string, object> config = parameters.Configuration;
Assert.NotNull(config);
Assert.True(config.ContainsKey(expectedKey));
Assert.Equal(expectedValue, config[expectedKey]);
Assert.Equal(expectedCount, config.Count);
}
public void AddExistingConfigurationThrows()
{
IndexingParameters parameters = new IndexingParameters
{
IndexingParametersConfiguration = new IndexingParametersConfiguration
{
ParsingMode = BlobIndexerParsingMode.Json,
},
};
//Assert.Throws<ArgumentException>(() => parameters.Configuration.Add("parsingMode", "json"));
Assert.Throws <ArgumentException>(() => parameters.Configuration.Add(new KeyValuePair <string, object>("parsingMode", "json")));
}
private void AssertParametersEqual(IndexingParameters expected, IndexingParameters actual)
{
if (expected == null)
{
Assert.Null(actual);
}
else
{
Assert.NotNull(actual);
Assert.Equal(expected.Base64EncodeKeys, actual.Base64EncodeKeys);
Assert.Equal(expected.MaxFailedItems, actual.MaxFailedItems);
Assert.Equal(expected.MaxFailedItemsPerBatch, actual.MaxFailedItemsPerBatch);
}
}
public void ConfigurationInitializesIndexingParametersConfiguration()
{
IndexingParameters parameters = new IndexingParameters();
Assert.IsNull(parameters.IndexingParametersConfiguration);
// Setting Configuration should initialize IndexingParametersConfiguration
Assert.IsNotNull(parameters.Configuration);
Assert.AreEqual(0, parameters.Configuration.Count);
parameters.Configuration["customTestProperty"] = "custom";
Assert.IsNotNull(parameters.IndexingParametersConfiguration);
Assert.AreEqual(1, parameters.Configuration.Count);
}
public void CopiesConfigurationKeys()
{
IndexingParameters parameters = new IndexingParameters
{
IndexingParametersConfiguration = new IndexingParametersConfiguration
{
ParsingMode = BlobIndexerParsingMode.Json,
["customTestProperty"] = "custom",
},
};
string[] keys = new string[parameters.Configuration.Count];
parameters.Configuration.Keys.CopyTo(keys, 0);
Assert.AreEqual(new[] { "parsingMode", "customTestProperty" }, keys);
}
public void CopiesConfigurationValues()
{
IndexingParameters parameters = new IndexingParameters
{
IndexingParametersConfiguration = new IndexingParametersConfiguration
{
ParsingMode = BlobIndexerParsingMode.Json,
["customTestProperty"] = "custom",
},
};
object[] values = new object[parameters.Configuration.Count];
parameters.Configuration.Values.CopyTo(values, 0);
Assert.AreEqual(new object[] { BlobIndexerParsingMode.Json, "custom" }, values);
}
public void RemoveWellKnownConfigurationNullsValue()
{
IndexingParameters parameters = new IndexingParameters
{
IndexingParametersConfiguration = new IndexingParametersConfiguration
{
ParsingMode = BlobIndexerParsingMode.Json,
},
};
parameters.Configuration.Remove("parsingMode");
Assert.AreEqual(0, parameters.Configuration.Count);
Assert.IsNull(parameters.IndexingParametersConfiguration.ParsingMode);
}
public void RemovesCustomConfiguration()
{
IndexingParameters parameters = new IndexingParameters
{
IndexingParametersConfiguration = new IndexingParametersConfiguration
{
["customTestProperty"] = "custom",
},
};
parameters.Configuration.Remove("customTestProperty");
Assert.AreEqual(0, parameters.Configuration.Count);
Assert.AreEqual(0, parameters.IndexingParametersConfiguration.Count());
}
public void RoundtripsFromConfiguration()
{
IndexingParameters parameters = new IndexingParameters
{
Configuration =
{
["parsingMode"] = "json",
["excludedFileNameExtensions"] = ".png",
["indexedFileNameExtensions"] = ".json,.jsonc",
["failOnUnsupportedContentType"] = false,
["failOnUnprocessableDocument"] = false,
["indexStorageMetadataOnlyForOversizedDocuments"] = true,
["delimitedTextHeaders"] = "A,B",
["delimitedTextDelimiter"] = "|",
["firstLineContainsHeaders"] = true,
["documentRoot"] = "$.values",
["dataToExtract"] = "allMetadata",
["imageAction"] = "generateNormalizedImages",
["allowSkillsetToReadFileData"] = true,
["pdfTextRotationAlgorithm"] = "detectAngles",
["executionEnvironment"] = "standard",
["queryTimeout"] = "12:34:56",
["customTestProperty"] = "custom",
},
};
IndexingParametersConfiguration configuration = parameters.IndexingParametersConfiguration;
Assert.AreEqual(BlobIndexerParsingMode.Json, configuration.ParsingMode);
Assert.AreEqual(".png", configuration.ExcludedFileNameExtensions);
Assert.AreEqual(".json,.jsonc", configuration.IndexedFileNameExtensions);
Assert.IsFalse(configuration.FailOnUnsupportedContentType);
Assert.IsFalse(configuration.FailOnUnprocessableDocument);
Assert.IsTrue(configuration.IndexStorageMetadataOnlyForOversizedDocuments);
Assert.AreEqual("A,B", configuration.DelimitedTextHeaders);
Assert.AreEqual("|", configuration.DelimitedTextDelimiter);
Assert.IsTrue(configuration.FirstLineContainsHeaders);
Assert.AreEqual("$.values", configuration.DocumentRoot);
Assert.AreEqual(BlobIndexerDataToExtract.AllMetadata, configuration.DataToExtract);
Assert.AreEqual(BlobIndexerImageAction.GenerateNormalizedImages, configuration.ImageAction);
Assert.IsTrue(configuration.AllowSkillsetToReadFileData);
Assert.AreEqual(BlobIndexerPdfTextRotationAlgorithm.DetectAngles, configuration.PdfTextRotationAlgorithm);
Assert.AreEqual(IndexerExecutionEnvironment.Standard, configuration.ExecutionEnvironment);
Assert.AreEqual(TimeSpan.Parse("12:34:56"), configuration.QueryTimeout);
Assert.AreEqual(1, configuration.Count());
Assert.AreEqual("custom", configuration["customTestProperty"]);
}
public void RoundtripsFromConfigurationWithExplicitNulls()
{
IndexingParameters parameters = new IndexingParameters
{
Configuration =
{
["parsingMode"] = null,
["excludedFileNameExtensions"] = null,
["indexedFileNameExtensions"] = null,
["failOnUnsupportedContentType"] = null,
["failOnUnprocessableDocument"] = null,
["indexStorageMetadataOnlyForOversizedDocuments"] = null,
["delimitedTextHeaders"] = null,
["delimitedTextDelimiter"] = null,
["firstLineContainsHeaders"] = null,
["documentRoot"] = null,
["dataToExtract"] = null,
["imageAction"] = null,
["allowSkillsetToReadFileData"] = null,
["pdfTextRotationAlgorithm"] = null,
["executionEnvironment"] = null,
["queryTimeout"] = null,
["customTestProperty"] = null,
},
};
IndexingParametersConfiguration configuration = parameters.IndexingParametersConfiguration;
Assert.IsNull(configuration.ParsingMode);
Assert.IsNull(configuration.ExcludedFileNameExtensions);
Assert.IsNull(configuration.IndexedFileNameExtensions);
Assert.IsNull(configuration.FailOnUnsupportedContentType);
Assert.IsNull(configuration.FailOnUnprocessableDocument);
Assert.IsNull(configuration.IndexStorageMetadataOnlyForOversizedDocuments);
Assert.IsNull(configuration.DelimitedTextHeaders);
Assert.IsNull(configuration.DelimitedTextDelimiter);
Assert.IsNull(configuration.FirstLineContainsHeaders);
Assert.IsNull(configuration.DocumentRoot);
Assert.IsNull(configuration.DataToExtract);
Assert.IsNull(configuration.ImageAction);
Assert.IsNull(configuration.AllowSkillsetToReadFileData);
Assert.IsNull(configuration.PdfTextRotationAlgorithm);
Assert.IsNull(configuration.ExecutionEnvironment);
Assert.IsNull(configuration.QueryTimeout);
Assert.AreEqual(1, configuration.Count());
Assert.IsNull(configuration["customTestProperty"]);
}
public void EmptyConfigurationSerializesWithoutConfiguration()
{
IUtf8JsonSerializable parameters = new IndexingParameters()
.SetIndexedFileNameExtensions()
.SetExcludeFileNameExtensions();
using MemoryStream ms = new MemoryStream();
using (Utf8JsonWriter writer = new Utf8JsonWriter(ms))
{
parameters.Write(writer);
}
string jsonContent = Encoding.UTF8.GetString(ms.ToArray());
StringAssert.DoesNotContain(@"""configuration""", jsonContent);
}
public void RemovePairChecksCustomPropertyValue()
{
IndexingParameters parameters = new IndexingParameters
{
IndexingParametersConfiguration = new IndexingParametersConfiguration
{
["customTestProperty"] = "custom",
},
};
Assert.IsFalse(parameters.Configuration.Remove(new KeyValuePair <string, object>("customTestProperty", "other")));
Assert.AreEqual(1, parameters.Configuration.Count);
Assert.IsTrue(parameters.Configuration.Remove(new KeyValuePair <string, object>("customTestProperty", "custom")));
Assert.AreEqual(0, parameters.Configuration.Count);
}
/// <summary>
/// Tells the indexer to assume that all blobs are delimited text files.
/// See <see href="https://docs.microsoft.com/azure/search/search-howto-index-csv-blobs" /> for details.
/// </summary>
/// <param name="parameters">The <see cref="IndexingParameters"/> to configure.</param>
/// <param name="delimiter">
/// Specifies the end-of-line or field delimiter.
/// </param>
/// <param name="headers">
/// Specifies column headers that the indexer will use to map values to specific fields in the search index. If you don't specify any
/// headers, the indexer assumes that the first non-blank line of each blob contains comma-separated headers.
/// </param>
/// <remarks>
/// This option only applies to indexers that index Azure Blob Storage.
/// </remarks>
/// <returns>The <see cref="IndexingParameters"/> instance.</returns>
/// <exception cref="ArgumentNullException"><paramref name="parameters"/> is null.</exception>
public static IndexingParameters SetParseDelimitedTextFiles(this IndexingParameters parameters, char delimiter, string[] headers)
{
Configure(parameters, ParsingModeKey, "delimitedText");
Configure(parameters, "delimitedTextDelimiter", delimiter.ToString());
if (headers != null && headers.Length > 0)
{
Configure(parameters, "delimitedTextHeaders", headers.CommaJoin());
}
else
{
Configure(parameters, "firstLineContainsHeaders", true);
}
return(parameters);
}
public void AddsToCorrectConfiguration()
{
IndexingParameters parameters = new IndexingParameters();
parameters.Configuration.Add("parsingMode", "json");
Assert.AreEqual(1, parameters.Configuration.Count);
Assert.IsTrue(parameters.Configuration.ContainsKey("parsingMode"));
Assert.IsFalse(parameters.Configuration.Contains(new KeyValuePair <string, object>("parsingMode", "json")));
Assert.IsTrue(parameters.Configuration.Contains(new KeyValuePair <string, object>("parsingMode", BlobIndexerParsingMode.Json)));
Assert.AreEqual(0, parameters.IndexingParametersConfiguration.Count());
Assert.IsFalse(parameters.IndexingParametersConfiguration.ContainsKey("parsingMode"));
parameters.Configuration.Add("customTestProperty", "custom");
Assert.AreEqual(2, parameters.Configuration.Count);
Assert.IsTrue(parameters.Configuration.ContainsKey("customTestProperty"));
Assert.AreEqual(1, parameters.IndexingParametersConfiguration.Count());
Assert.IsTrue(parameters.IndexingParametersConfiguration.ContainsKey("customTestProperty"));
Assert.IsTrue(parameters.IndexingParametersConfiguration.Contains(new KeyValuePair <string, object>("customTestProperty", "custom")));
}
public void RemovePairChecksWellKnownPropertyValue()
{
IndexingParameters parameters = new IndexingParameters
{
IndexingParametersConfiguration = new IndexingParametersConfiguration
{
ParsingMode = BlobIndexerParsingMode.Json,
},
};
Assert.IsFalse(parameters.Configuration.Remove(new KeyValuePair <string, object>("parsingMode", "text")));
Assert.AreEqual(1, parameters.Configuration.Count);
Assert.IsFalse(parameters.Configuration.Remove(new KeyValuePair <string, object>("parsingMode", "json")));
Assert.AreEqual(1, parameters.Configuration.Count);
Assert.IsTrue(parameters.Configuration.Remove(new KeyValuePair <string, object>("parsingMode", BlobIndexerParsingMode.Json)));
Assert.AreEqual(0, parameters.Configuration.Count);
}
public void CopiesConfiguration()
{
IndexingParameters parameters = new IndexingParameters
{
IndexingParametersConfiguration = new IndexingParametersConfiguration
{
ParsingMode = BlobIndexerParsingMode.Json,
IndexedFileNameExtensions = ".json",
["customTestProperty"] = "custom",
},
};
KeyValuePair <string, object>[] pairs = new KeyValuePair <string, object> [parameters.Configuration.Count + 1];
parameters.Configuration.CopyTo(pairs, 1);
Assert.IsNull(pairs[0].Key);
// Dictionary order is guaranteed, so check the last one which should be from AdditionalProperties.
Assert.AreEqual("customTestProperty", pairs[3].Key);
}