Example #1
0
        private static IndexingParameters Configure(IndexingParameters parameters, string key, object value)
        {
            Argument.AssertNotNull(parameters, nameof(parameters));

            parameters.Configuration[key] = value;
            return(parameters);
        }
Example #2
0
        /// <summary>
        /// Modifie la valeur d'un élément d'une collection.
        /// L'élément est désigné par les paramètres d'indexation.
        /// </summary>
        /// <param name="var"></param>
        /// <param name="indexingParameters"></param>
        void SetElementValue(object val, object newValue, Context context)
        {
            dynamic newValueDynamic = newValue;
            dynamic index;

            // TODO : support des arrays multi-dimensionels.
            if (IndexingParameters.Count > 1)
            {
                for (int i = 0; i <= IndexingParameters.Count - 1; i++)
                {
                    List <IGettable> parameters = IndexingParameters[i];
                    if (parameters.Count == 0)
                    {
                        throw new Exception("Expected indexing parameter");
                    }

                    // On laisse faire le runtime pour la résolution dynamique =D
                    dynamic tmp = val;
                    index = IndexingParameters[0][0].GetValue(context);
                    val   = tmp[index];
                }
                dynamic enumValFinal = val;
                index = IndexingParameters.Last().First().GetValue(context);
                enumValFinal[index] = newValueDynamic;
            }
            else if (IndexingParameters.Count == 1)
            {
                dynamic enumValFinal = val;
                index = IndexingParameters.First().First().GetValue(context);
                enumValFinal[index] = newValueDynamic;
            }
        }
        public AzurSearchIndexerService(
            IOptions <AzureBlobOptions> azureBlobOptions,
            IOptions <SearchIndexerClientOptions> searchIndexerClientOptions
            )
        {
            _azureBlobOptions = azureBlobOptions;

            _indexingParameters = new();
            _indexingParameters.Configuration.Add("parsingMode", "json");

            _searchIndexerClient = new SearchIndexerClient(null, null);

            _azureBlobIndexer = new SearchIndexer(
                name: searchIndexerClientOptions.Value.IndexerName,
                dataSourceName: searchIndexerClientOptions.Value.DataSourceName,
                targetIndexName: searchIndexerClientOptions.Value.IndexName)
            {
                Parameters = _indexingParameters,
                Schedule   = new IndexingSchedule(TimeSpan.FromDays(1))
            };

            _azureBlobIndexer.FieldMappings.Add(
                new FieldMapping("Id")
            {
                TargetFieldName = "HotelId"
            });
            _searchIndexerDataContainer = new SearchIndexerDataContainer("hotel-rooms");
        }
Example #4
0
        /// <summary>
        /// Tells the indexer to assume that blobs should be parsed as text files in the desired encoding.
        /// See <see href="https://docs.microsoft.com/azure/search/search-howto-indexing-azure-blob-storage#indexing-plain-text"/> for details.
        /// </summary>
        /// <param name="parameters">The <see cref="IndexingParameters"/> to configure.</param>
        /// <param name="encoding">Encoding used to read the text stored in blobs.</param>
        /// <returns>The <see cref="IndexingParameters"/> instance.</returns>
        public static IndexingParameters SetParseText(this IndexingParameters parameters, Encoding encoding)
        {
            Argument.AssertNotNull(encoding, nameof(encoding));

            Configure(parameters, ParsingModeKey, "text");
            Configure(parameters, "encoding", encoding.WebName);
            return(parameters);
        }
Example #5
0
        public void ParseDelimitedTextFilesWithInlineHeadersSetCorrectly()
        {
            const int ExpectedCount = 2;

            var parameters = new IndexingParameters().ParseDelimitedTextFiles();

            AssertHasConfigItem(parameters, ExpectedParsingModeKey, "delimitedText", ExpectedCount);
            AssertHasConfigItem(parameters, "firstLineContainsHeaders", true, ExpectedCount);
        }
Example #6
0
        public void ParseDelimitedTextFilesWithGivenHeadersSetCorrectly()
        {
            const int ExpectedCount = 2;

            var parameters = new IndexingParameters().ParseDelimitedTextFiles("id", "name", "address");

            AssertHasConfigItem(parameters, ExpectedParsingModeKey, "delimitedText", ExpectedCount);
            AssertHasConfigItem(parameters, "delimitedTextHeaders", "id,name,address", ExpectedCount);
        }
Example #7
0
        public void ParseTextSetCorrectly()
        {
            const int ExpectedCount = 2;

            var parameters = new IndexingParameters().ParseText();

            AssertHasConfigItem(parameters, ExpectedParsingModeKey, "text", ExpectedCount);
            AssertHasConfigItem(parameters, "encoding", "utf-8", ExpectedCount);
        }
Example #8
0
        public void ParseTextWithEncodingSetCorrectly()
        {
            const int ExpectedCount = 2;

            var parameters = new IndexingParameters().ParseText(Encoding.ASCII);

            AssertHasConfigItem(parameters, ExpectedParsingModeKey, "text", ExpectedCount);
            AssertHasConfigItem(parameters, "encoding", "us-ascii", ExpectedCount);
        }
Example #9
0
        public void ParseJsonArraysWithDocumentRootSetCorrectly()
        {
            const int ExpectedCount = 2;

            var parameters = new IndexingParameters().ParseJsonArrays("/my/path");

            AssertHasConfigItem(parameters, ExpectedParsingModeKey, "jsonArray", ExpectedCount);
            AssertHasConfigItem(parameters, "documentRoot", "/my/path", ExpectedCount);
        }
Example #10
0
        private static IndexingParameters ConfigureFileNameExtensions(this IndexingParameters parameters, string key, string[] extensions)
        {
            if (extensions != null && extensions.Length > 0)
            {
                string delimitedExtensions = ProcessFileNameExtensions(extensions).CommaJoin();
                Configure(parameters, key, delimitedExtensions);
            }

            return(parameters);
        }
        public void ConfigurationKeysAreReadOnly()
        {
            IndexingParameters   parameters = new IndexingParameters();
            ICollection <string> keys       = parameters.Configuration.Keys;

            Assert.IsTrue(keys.IsReadOnly);
            Assert.Throws <NotSupportedException>(() => keys.Add("customTestProperty"));
            Assert.Throws <NotSupportedException>(() => keys.Remove("customTestProperty"));
            Assert.Throws <NotSupportedException>(() => keys.Clear());
        }
Example #12
0
        public void ParseTabDelimitedTextFilesWithGivenHeadersSetCorrectly()
        {
            const int ExpectedCount = 3;

            IndexingParameters parameters = new IndexingParameters().SetParseDelimitedTextFiles('\t', new[] { "id", "name", "address" });

            AssertHasConfigItem(parameters, ExpectedParsingModeKey, "delimitedText", ExpectedCount);
            AssertHasConfigItem(parameters, "delimitedTextDelimiter", "\t", ExpectedCount);
            AssertHasConfigItem(parameters, "delimitedTextHeaders", "id,name,address", ExpectedCount);
        }
        public void ConfigurationValuesAreReadOnly()
        {
            IndexingParameters   parameters = new IndexingParameters();
            ICollection <object> values     = parameters.Configuration.Values;

            Assert.IsTrue(values.IsReadOnly);
            Assert.Throws <NotSupportedException>(() => values.Add("custom"));
            Assert.Throws <NotSupportedException>(() => values.Remove("custom"));
            Assert.Throws <NotSupportedException>(() => values.Clear());
        }
Example #14
0
        /// <summary>
        /// Tells the indexer to assume that all blobs contain JSON arrays, which it will then parse such that each JSON object in each array will
        /// map to a single document in the search index.
        /// See <see href="https://docs.microsoft.com/azure/search/search-howto-index-json-blobs" /> for details.
        /// </summary>
        /// <param name="parameters">The <see cref="IndexingParameters"/> to configure.</param>
        /// <param name="documentRoot">
        /// An optional <see href="https://tools.ietf.org/html/rfc6901">JSON Pointer</see> that tells the indexer how to find the JSON array if it's not the top-level JSON property of each blob.
        /// If this parameter is null or empty, the indexer will assume that the JSON array can be found in the top-level JSON property of each blob.
        /// Default is null.
        /// </param>
        /// <remarks>
        /// This option only applies to indexers that index Azure Blob Storage.
        /// </remarks>
        /// <returns>The <see cref="IndexingParameters"/> instance.</returns>
        /// <exception cref="ArgumentNullException"><paramref name="parameters"/> is null.</exception>
        public static IndexingParameters SetParseJsonArrays(this IndexingParameters parameters, string documentRoot = null)
        {
            Configure(parameters, ParsingModeKey, "jsonArray");

            if (!string.IsNullOrEmpty(documentRoot))
            {
                Configure(parameters, "documentRoot", documentRoot);
            }

            return(parameters);
        }
Example #15
0
        private static void AssertHasConfigItem(
            IndexingParameters parameters,
            string expectedKey,
            object expectedValue,
            int expectedCount = 1)
        {
            IDictionary <string, object> config = parameters.Configuration;

            Assert.NotNull(config);
            Assert.True(config.ContainsKey(expectedKey));
            Assert.Equal(expectedValue, config[expectedKey]);
            Assert.Equal(expectedCount, config.Count);
        }
        public void AddExistingConfigurationThrows()
        {
            IndexingParameters parameters = new IndexingParameters
            {
                IndexingParametersConfiguration = new IndexingParametersConfiguration
                {
                    ParsingMode = BlobIndexerParsingMode.Json,
                },
            };

            //Assert.Throws<ArgumentException>(() => parameters.Configuration.Add("parsingMode", "json"));
            Assert.Throws <ArgumentException>(() => parameters.Configuration.Add(new KeyValuePair <string, object>("parsingMode", "json")));
        }
 private void AssertParametersEqual(IndexingParameters expected, IndexingParameters actual)
 {
     if (expected == null)
     {
         Assert.Null(actual);
     }
     else
     {
         Assert.NotNull(actual);
         Assert.Equal(expected.Base64EncodeKeys, actual.Base64EncodeKeys);
         Assert.Equal(expected.MaxFailedItems, actual.MaxFailedItems);
         Assert.Equal(expected.MaxFailedItemsPerBatch, actual.MaxFailedItemsPerBatch);
     }
 }
        public void ConfigurationInitializesIndexingParametersConfiguration()
        {
            IndexingParameters parameters = new IndexingParameters();

            Assert.IsNull(parameters.IndexingParametersConfiguration);

            // Setting Configuration should initialize IndexingParametersConfiguration
            Assert.IsNotNull(parameters.Configuration);
            Assert.AreEqual(0, parameters.Configuration.Count);

            parameters.Configuration["customTestProperty"] = "custom";
            Assert.IsNotNull(parameters.IndexingParametersConfiguration);
            Assert.AreEqual(1, parameters.Configuration.Count);
        }
        public void CopiesConfigurationKeys()
        {
            IndexingParameters parameters = new IndexingParameters
            {
                IndexingParametersConfiguration = new IndexingParametersConfiguration
                {
                    ParsingMode            = BlobIndexerParsingMode.Json,
                    ["customTestProperty"] = "custom",
                },
            };

            string[] keys = new string[parameters.Configuration.Count];
            parameters.Configuration.Keys.CopyTo(keys, 0);
            Assert.AreEqual(new[] { "parsingMode", "customTestProperty" }, keys);
        }
        public void CopiesConfigurationValues()
        {
            IndexingParameters parameters = new IndexingParameters
            {
                IndexingParametersConfiguration = new IndexingParametersConfiguration
                {
                    ParsingMode            = BlobIndexerParsingMode.Json,
                    ["customTestProperty"] = "custom",
                },
            };

            object[] values = new object[parameters.Configuration.Count];
            parameters.Configuration.Values.CopyTo(values, 0);
            Assert.AreEqual(new object[] { BlobIndexerParsingMode.Json, "custom" }, values);
        }
        public void RemoveWellKnownConfigurationNullsValue()
        {
            IndexingParameters parameters = new IndexingParameters
            {
                IndexingParametersConfiguration = new IndexingParametersConfiguration
                {
                    ParsingMode = BlobIndexerParsingMode.Json,
                },
            };

            parameters.Configuration.Remove("parsingMode");

            Assert.AreEqual(0, parameters.Configuration.Count);
            Assert.IsNull(parameters.IndexingParametersConfiguration.ParsingMode);
        }
        public void RemovesCustomConfiguration()
        {
            IndexingParameters parameters = new IndexingParameters
            {
                IndexingParametersConfiguration = new IndexingParametersConfiguration
                {
                    ["customTestProperty"] = "custom",
                },
            };

            parameters.Configuration.Remove("customTestProperty");

            Assert.AreEqual(0, parameters.Configuration.Count);
            Assert.AreEqual(0, parameters.IndexingParametersConfiguration.Count());
        }
        public void RoundtripsFromConfiguration()
        {
            IndexingParameters parameters = new IndexingParameters
            {
                Configuration =
                {
                    ["parsingMode"] = "json",
                    ["excludedFileNameExtensions"]   = ".png",
                    ["indexedFileNameExtensions"]    = ".json,.jsonc",
                    ["failOnUnsupportedContentType"] = false,
                    ["failOnUnprocessableDocument"]  = false,
                    ["indexStorageMetadataOnlyForOversizedDocuments"] = true,
                    ["delimitedTextHeaders"]     = "A,B",
                    ["delimitedTextDelimiter"]   = "|",
                    ["firstLineContainsHeaders"] = true,
                    ["documentRoot"]             = "$.values",
                    ["dataToExtract"]            = "allMetadata",
                    ["imageAction"] = "generateNormalizedImages",
                    ["allowSkillsetToReadFileData"] = true,
                    ["pdfTextRotationAlgorithm"]    = "detectAngles",
                    ["executionEnvironment"]        = "standard",
                    ["queryTimeout"]       = "12:34:56",
                    ["customTestProperty"] = "custom",
                },
            };

            IndexingParametersConfiguration configuration = parameters.IndexingParametersConfiguration;

            Assert.AreEqual(BlobIndexerParsingMode.Json, configuration.ParsingMode);
            Assert.AreEqual(".png", configuration.ExcludedFileNameExtensions);
            Assert.AreEqual(".json,.jsonc", configuration.IndexedFileNameExtensions);
            Assert.IsFalse(configuration.FailOnUnsupportedContentType);
            Assert.IsFalse(configuration.FailOnUnprocessableDocument);
            Assert.IsTrue(configuration.IndexStorageMetadataOnlyForOversizedDocuments);
            Assert.AreEqual("A,B", configuration.DelimitedTextHeaders);
            Assert.AreEqual("|", configuration.DelimitedTextDelimiter);
            Assert.IsTrue(configuration.FirstLineContainsHeaders);
            Assert.AreEqual("$.values", configuration.DocumentRoot);
            Assert.AreEqual(BlobIndexerDataToExtract.AllMetadata, configuration.DataToExtract);
            Assert.AreEqual(BlobIndexerImageAction.GenerateNormalizedImages, configuration.ImageAction);
            Assert.IsTrue(configuration.AllowSkillsetToReadFileData);
            Assert.AreEqual(BlobIndexerPdfTextRotationAlgorithm.DetectAngles, configuration.PdfTextRotationAlgorithm);
            Assert.AreEqual(IndexerExecutionEnvironment.Standard, configuration.ExecutionEnvironment);
            Assert.AreEqual(TimeSpan.Parse("12:34:56"), configuration.QueryTimeout);

            Assert.AreEqual(1, configuration.Count());
            Assert.AreEqual("custom", configuration["customTestProperty"]);
        }
        public void RoundtripsFromConfigurationWithExplicitNulls()
        {
            IndexingParameters parameters = new IndexingParameters
            {
                Configuration =
                {
                    ["parsingMode"] = null,
                    ["excludedFileNameExtensions"]   = null,
                    ["indexedFileNameExtensions"]    = null,
                    ["failOnUnsupportedContentType"] = null,
                    ["failOnUnprocessableDocument"]  = null,
                    ["indexStorageMetadataOnlyForOversizedDocuments"] = null,
                    ["delimitedTextHeaders"]     = null,
                    ["delimitedTextDelimiter"]   = null,
                    ["firstLineContainsHeaders"] = null,
                    ["documentRoot"]             = null,
                    ["dataToExtract"]            = null,
                    ["imageAction"] = null,
                    ["allowSkillsetToReadFileData"] = null,
                    ["pdfTextRotationAlgorithm"]    = null,
                    ["executionEnvironment"]        = null,
                    ["queryTimeout"]       = null,
                    ["customTestProperty"] = null,
                },
            };

            IndexingParametersConfiguration configuration = parameters.IndexingParametersConfiguration;

            Assert.IsNull(configuration.ParsingMode);
            Assert.IsNull(configuration.ExcludedFileNameExtensions);
            Assert.IsNull(configuration.IndexedFileNameExtensions);
            Assert.IsNull(configuration.FailOnUnsupportedContentType);
            Assert.IsNull(configuration.FailOnUnprocessableDocument);
            Assert.IsNull(configuration.IndexStorageMetadataOnlyForOversizedDocuments);
            Assert.IsNull(configuration.DelimitedTextHeaders);
            Assert.IsNull(configuration.DelimitedTextDelimiter);
            Assert.IsNull(configuration.FirstLineContainsHeaders);
            Assert.IsNull(configuration.DocumentRoot);
            Assert.IsNull(configuration.DataToExtract);
            Assert.IsNull(configuration.ImageAction);
            Assert.IsNull(configuration.AllowSkillsetToReadFileData);
            Assert.IsNull(configuration.PdfTextRotationAlgorithm);
            Assert.IsNull(configuration.ExecutionEnvironment);
            Assert.IsNull(configuration.QueryTimeout);

            Assert.AreEqual(1, configuration.Count());
            Assert.IsNull(configuration["customTestProperty"]);
        }
Example #25
0
        public void EmptyConfigurationSerializesWithoutConfiguration()
        {
            IUtf8JsonSerializable parameters = new IndexingParameters()
                                               .SetIndexedFileNameExtensions()
                                               .SetExcludeFileNameExtensions();

            using MemoryStream ms = new MemoryStream();
            using (Utf8JsonWriter writer = new Utf8JsonWriter(ms))
            {
                parameters.Write(writer);
            }

            string jsonContent = Encoding.UTF8.GetString(ms.ToArray());

            StringAssert.DoesNotContain(@"""configuration""", jsonContent);
        }
        public void RemovePairChecksCustomPropertyValue()
        {
            IndexingParameters parameters = new IndexingParameters
            {
                IndexingParametersConfiguration = new IndexingParametersConfiguration
                {
                    ["customTestProperty"] = "custom",
                },
            };

            Assert.IsFalse(parameters.Configuration.Remove(new KeyValuePair <string, object>("customTestProperty", "other")));
            Assert.AreEqual(1, parameters.Configuration.Count);

            Assert.IsTrue(parameters.Configuration.Remove(new KeyValuePair <string, object>("customTestProperty", "custom")));
            Assert.AreEqual(0, parameters.Configuration.Count);
        }
Example #27
0
        /// <summary>
        /// Tells the indexer to assume that all blobs are delimited text files.
        /// See <see href="https://docs.microsoft.com/azure/search/search-howto-index-csv-blobs" /> for details.
        /// </summary>
        /// <param name="parameters">The <see cref="IndexingParameters"/> to configure.</param>
        /// <param name="delimiter">
        /// Specifies the end-of-line or field delimiter.
        /// </param>
        /// <param name="headers">
        /// Specifies column headers that the indexer will use to map values to specific fields in the search index. If you don't specify any
        /// headers, the indexer assumes that the first non-blank line of each blob contains comma-separated headers.
        /// </param>
        /// <remarks>
        /// This option only applies to indexers that index Azure Blob Storage.
        /// </remarks>
        /// <returns>The <see cref="IndexingParameters"/> instance.</returns>
        /// <exception cref="ArgumentNullException"><paramref name="parameters"/> is null.</exception>
        public static IndexingParameters SetParseDelimitedTextFiles(this IndexingParameters parameters, char delimiter, string[] headers)
        {
            Configure(parameters, ParsingModeKey, "delimitedText");
            Configure(parameters, "delimitedTextDelimiter", delimiter.ToString());

            if (headers != null && headers.Length > 0)
            {
                Configure(parameters, "delimitedTextHeaders", headers.CommaJoin());
            }
            else
            {
                Configure(parameters, "firstLineContainsHeaders", true);
            }

            return(parameters);
        }
        public void AddsToCorrectConfiguration()
        {
            IndexingParameters parameters = new IndexingParameters();

            parameters.Configuration.Add("parsingMode", "json");
            Assert.AreEqual(1, parameters.Configuration.Count);
            Assert.IsTrue(parameters.Configuration.ContainsKey("parsingMode"));
            Assert.IsFalse(parameters.Configuration.Contains(new KeyValuePair <string, object>("parsingMode", "json")));
            Assert.IsTrue(parameters.Configuration.Contains(new KeyValuePair <string, object>("parsingMode", BlobIndexerParsingMode.Json)));
            Assert.AreEqual(0, parameters.IndexingParametersConfiguration.Count());
            Assert.IsFalse(parameters.IndexingParametersConfiguration.ContainsKey("parsingMode"));

            parameters.Configuration.Add("customTestProperty", "custom");
            Assert.AreEqual(2, parameters.Configuration.Count);
            Assert.IsTrue(parameters.Configuration.ContainsKey("customTestProperty"));
            Assert.AreEqual(1, parameters.IndexingParametersConfiguration.Count());
            Assert.IsTrue(parameters.IndexingParametersConfiguration.ContainsKey("customTestProperty"));
            Assert.IsTrue(parameters.IndexingParametersConfiguration.Contains(new KeyValuePair <string, object>("customTestProperty", "custom")));
        }
        public void RemovePairChecksWellKnownPropertyValue()
        {
            IndexingParameters parameters = new IndexingParameters
            {
                IndexingParametersConfiguration = new IndexingParametersConfiguration
                {
                    ParsingMode = BlobIndexerParsingMode.Json,
                },
            };

            Assert.IsFalse(parameters.Configuration.Remove(new KeyValuePair <string, object>("parsingMode", "text")));
            Assert.AreEqual(1, parameters.Configuration.Count);

            Assert.IsFalse(parameters.Configuration.Remove(new KeyValuePair <string, object>("parsingMode", "json")));
            Assert.AreEqual(1, parameters.Configuration.Count);

            Assert.IsTrue(parameters.Configuration.Remove(new KeyValuePair <string, object>("parsingMode", BlobIndexerParsingMode.Json)));
            Assert.AreEqual(0, parameters.Configuration.Count);
        }
        public void CopiesConfiguration()
        {
            IndexingParameters parameters = new IndexingParameters
            {
                IndexingParametersConfiguration = new IndexingParametersConfiguration
                {
                    ParsingMode = BlobIndexerParsingMode.Json,
                    IndexedFileNameExtensions = ".json",
                    ["customTestProperty"]    = "custom",
                },
            };

            KeyValuePair <string, object>[] pairs = new KeyValuePair <string, object> [parameters.Configuration.Count + 1];
            parameters.Configuration.CopyTo(pairs, 1);

            Assert.IsNull(pairs[0].Key);

            // Dictionary order is guaranteed, so check the last one which should be from AdditionalProperties.
            Assert.AreEqual("customTestProperty", pairs[3].Key);
        }