public async Task TestInitializeIndex() { var indexConfig = new IndexConfig { ExcludedExtensions = ".dll", SaveIntervalSeconds = 1, IndexName = "ABC", MonitorFolder = MonitorFolder }; var fileName1 = Path.Combine(MonitorFolder, "A.txt"); var fileName2 = Path.Combine(MonitorFolder, "B.txt"); File.AppendAllText(fileName1, "ABCD ABCD" + Environment.NewLine + "ABCD"); File.AppendAllText(fileName2, "ABCD EFGH"); using var maintainer = new IndexMaintainer(indexConfig, Config, Log); Assert.AreEqual(IndexStatus.Idle, maintainer.Status); await maintainer.InitializeIndex(false); Assert.AreEqual(IndexStatus.Initialized, maintainer.Status); var codeDocuments = maintainer.IndexBuilder.CodeIndexPool.Search(new MatchAllDocsQuery(), int.MaxValue); Assert.AreEqual(2, codeDocuments.Length); CollectionAssert.AreEquivalent(new[] { "ABCD ABCD" + Environment.NewLine + "ABCD", "ABCD EFGH" }, codeDocuments.Select(u => u.Get(nameof(CodeSource.Content)))); var hintDocuments = maintainer.IndexBuilder.HintIndexPool.Search(new MatchAllDocsQuery(), int.MaxValue); Assert.AreEqual(2, hintDocuments.Length); CollectionAssert.AreEquivalent(new[] { "ABCD", "EFGH" }, hintDocuments.Select(u => u.Get(nameof(CodeWord.Word)))); }
public FetchResult <bool> EditIndex(IndexConfig indexConfig) { if (IsDisposing) { return(ManagementIsDisposing <bool>()); } lock (syncLock) { indexConfig.TrimValues(); if (ValidToEdit(indexConfig, out var message, out var needDisposeAndRemovedWrapper) && needDisposeAndRemovedWrapper != null) { ConfigMaintainer.EditIndexConfig(indexConfig); needDisposeAndRemovedWrapper.Dispose(); MaintainerPool.TryUpdate(indexConfig.Pk, new IndexMaintainerWrapper(indexConfig, CodeIndexConfiguration, Log), needDisposeAndRemovedWrapper); Log.LogInformation($"Edit Index Config {indexConfig} Successful"); return(new FetchResult <bool> { Result = true, Status = new Status { Success = true } }); }
public void TestAddIndex() { var indexConfig = new IndexConfig { IndexName = "ABC", MonitorFolder = MonitorFolder }; using var management = new IndexManagement(Config, Log); CollectionAssert.IsEmpty(management.GetIndexList().Result); Assert.IsTrue(management.AddIndex(indexConfig).Status.Success); CollectionAssert.AreEquivalent(new[] { indexConfig.ToString() }, management.GetIndexList().Result.Select(u => u.IndexConfig.ToString())); Assert.IsFalse(management.AddIndex(indexConfig).Status.Success); Assert.IsFalse(management.AddIndex(indexConfig with { IndexName = null }).Status.Success); Assert.IsFalse(management.AddIndex(indexConfig with { IndexName = "BBB" }).Status.Success); Assert.IsFalse(management.AddIndex(indexConfig with { IndexName = "A".PadLeft(101, 'C') }).Status.Success); Assert.IsFalse(management.AddIndex(indexConfig with { Pk = Guid.NewGuid(), IndexName = "BBB", MonitorFolder = string.Empty }).Status.Success); Assert.IsFalse(management.AddIndex(indexConfig with { Pk = Guid.NewGuid(), IndexName = "BBB" }).Status.Success); Assert.IsFalse(management.AddIndex(indexConfig with { Pk = Guid.NewGuid(), IndexName = "BBB", MonitorFolder = "Dummy" }).Status.Success); CollectionAssert.AreEquivalent(new[] { indexConfig.ToString() }, management.GetIndexList().Result.Select(u => u.IndexConfig.ToString())); }
/// <summary> /// 索引处理 /// </summary> /// <param name="type">索引类型</param> /// <param name="log"></param> public DataInit(string type, ILog log) { _log = log; _indexType = type; _indexConfig = IndexConfigHelper.IndexConfig(type); if (_indexConfig == null) { throw new Exception(string.Format("{0}:未找到对应索引类型的配置信息!", _indexType)); } _log.Info(string.Format("{0}:开始操作索引...", type)); tempIndexName = _indexConfig.IndexName + DateTime.Now.ToString("yyyyMMddHHmmss"); //es操作对象 _es = new EsHandle(tempIndexName, _log); //创建一个索引 _es.CreateIndex(_indexConfig.Replicas, _indexConfig.Shards); try { _log.Info(string.Format("{0}:索引操作完成...", type)); //索引器容器 _indexerContainer = new IndexerContainer(_log); _log.Info(string.Format("{0}:开始索引数据...", type)); } catch { //有异常删除已创建的索引 _es.DeleteIndex(tempIndexName); throw; } }
public void TestEditIndexConfig() { using var configBuilder = new ConfigIndexBuilder(TempConfigDir); var config1 = new IndexConfig { IndexName = "ABC" }; var config2 = new IndexConfig { IndexName = "EFG" }; configBuilder.AddIndexConfig(config1); configBuilder.AddIndexConfig(config2); Assert.AreEqual(2, configBuilder.GetConfigs().Count()); config1.IndexName = "NEW"; configBuilder.EditIndexConfig(config1); CollectionAssert.AreEquivalent(new[] { "NEW", "EFG" }, configBuilder.GetConfigs().Select(u => u.IndexName)); config2.IndexName = "NEW NEW"; configBuilder.EditIndexConfig(config2); CollectionAssert.AreEquivalent(new[] { "NEW", "NEW NEW" }, configBuilder.GetConfigs().Select(u => u.IndexName)); }
public void TestConstructor() { var config = new IndexConfig { ExcludedExtensions = "A|B|C", ExcludedPaths = "B|C|D", IncludedExtensions = "E|F", IndexName = "ABC", MaxContentHighlightLength = 100, MonitorFolder = "BCA", MonitorFolderRealPath = "AAA", OpenIDEUriFormat = "BBB", SaveIntervalSeconds = 10 }; Assert.AreNotEqual(Guid.Empty, config.Pk); Assert.AreEqual("A|B|C", config.ExcludedExtensions); Assert.AreEqual("B|C|D", config.ExcludedPaths); Assert.AreEqual("E|F", config.IncludedExtensions); CollectionAssert.AreEquivalent(config.ExcludedExtensionsArray, new[] { "A", "B", "C" }); CollectionAssert.AreEquivalent(config.ExcludedPathsArray, new[] { "B", "C", "D" }); CollectionAssert.AreEquivalent(config.IncludedExtensionsArray, new[] { "E", "F" }); Assert.AreEqual("ABC", config.IndexName); Assert.AreEqual(100, config.MaxContentHighlightLength); Assert.AreEqual("BCA", config.MonitorFolder); Assert.AreEqual("AAA", config.MonitorFolderRealPath); Assert.AreEqual("BBB", config.OpenIDEUriFormat); Assert.AreEqual(10, config.SaveIntervalSeconds); config.IncludedExtensions = null; Assert.AreEqual(0, config.IncludedExtensionsArray.Count()); }
public void IndexConfigTest() { var x = new IndexConfig(); Console.WriteLine(x.ToString()); x.Name = "name"; Assert.That(x.Name, Is.EqualTo("name")); Assert.That(x.Type, Is.EqualTo(IndexConfig.DefaultType)); x.Type = IndexType.Hashed; Assert.That(x.Type, Is.EqualTo(IndexType.Hashed)); x = new IndexConfig(new [] { "aaa", "bbb" }); Assert.That(x.Attributes.Count, Is.EqualTo(2)); x.AddAttribute("ccc"); Assert.That(x.Attributes.Count, Is.EqualTo(3)); Assert.That(x.Attributes, Does.Contain("aaa")); Assert.That(x.Attributes, Does.Contain("bbb")); Assert.That(x.Attributes, Does.Contain("ccc")); Assert.That(x.BitmapIndexOptions, Is.Not.Null); var y = new BitmapIndexOptions(); x.BitmapIndexOptions = y; Assert.That(x.BitmapIndexOptions, Is.SameAs(y)); IndexConfig.ValidateAttribute(x, "flub"); Assert.Throws <ArgumentNullException>(() => IndexConfig.ValidateAttribute(x, null)); Assert.Throws <ArgumentException>(() => IndexConfig.ValidateAttribute(x, "")); Assert.Throws <ArgumentException>(() => IndexConfig.ValidateAttribute(x, "duh.")); }
public InitManagement(string monitorFolder, CodeIndexConfiguration codeIndexConfiguration, bool initFiles = false, int maxContentHighlightLength = Constants.DefaultMaxContentHighlightLength) { indexConfig = new IndexConfig { IndexName = "ABC", MonitorFolder = monitorFolder, MaxContentHighlightLength = maxContentHighlightLength }; if (initFiles) { var fileName1 = Path.Combine(monitorFolder, "A.txt"); var fileName2 = Path.Combine(monitorFolder, "B.txt"); var fileName3 = Path.Combine(monitorFolder, "C.txt"); File.AppendAllText(fileName1, "ABCD"); File.AppendAllText(fileName2, "ABCD EFGH"); File.AppendAllText(fileName3, "ABCD EFGH IJKL" + Environment.NewLine + "ABCD ABCE"); } management = new IndexManagement(codeIndexConfiguration, log1); management.AddIndex(indexConfig); var maintainer = management.GetIndexMaintainerWrapperAndInitializeIfNeeded(indexConfig.Pk); // Wait initialized finished while (maintainer.Result.Status == IndexStatus.Initializing_ComponentInitializeFinished || maintainer.Result.Status == IndexStatus.Initialized) { Thread.Sleep(100); } }
public MergeIndexTimer(ILogger <IMergeIndexTimer> logger, IOptions <JobTrackerConfig> config, IndexStorageAccountWrapper wrapper) { _logger = logger; _account = wrapper.TableAccount; _indexConfig = config.Value.JobIndexConfig; _tableName = _indexConfig.TableName; }
private static void ValidateAttribute(IndexConfig config, string attributeName) { if (string.IsNullOrEmpty(attributeName) || attributeName.Trim().EndsWith(".", StringComparison.Ordinal)) { throw new ArgumentException($"Invalid attribute '{attributeName}' in index configuration '{config.Name}'. " + "Attributes must not be null nor empty, nor end with a dot."); } }
public void AddIndex(IndexConfig indexConfig) { CheckNotNull(indexConfig, "Index config cannot be null."); var indexConfig0 = ValidateAndNormalize(Name, indexConfig); var request = MapAddIndexCodec.EncodeRequest(Name, indexConfig0); Invoke(request); }
public IndexMaintainerWrapper(IndexConfig indexConfig, CodeIndexConfiguration codeIndexConfiguration, ILogger log) { indexConfig.RequireNotNull(nameof(indexConfig)); codeIndexConfiguration.RequireNotNull(nameof(codeIndexConfiguration)); log.RequireNotNull(nameof(log)); Maintainer = new IndexMaintainer(indexConfig, codeIndexConfiguration, log); IndexConfig = indexConfig; }
public MergeJobIndexWorker(IClusterClient client, ILogger <MergeJobIndexWorker> logger, IOptions <JobTrackerConfig> config, IndexStorageAccountWrapper wrapper) { _client = client; _account = wrapper.Account; _logger = logger; _indexConfig = config.Value.JobIndexConfig; _tableName = _indexConfig.TableName; }
public void AddIndex(IndexType indexType, params string[] attributes) { CheckNotNull(attributes, "Index attributes cannot be null."); var indexConfig = new IndexConfig { Type = indexType, Attributes = attributes }; AddIndex(indexConfig); }
public ActionResult Index() { var result = new IndexConfig(); var assembly = Assembly.GetExecutingAssembly().GetReferencedAssemblies().Where(n => n.Name.Equals("Microsoft.PowerBI.Api")).FirstOrDefault(); if (assembly != null) { result.DotNETSDK = assembly.Version.ToString(3); } return(View(result)); }
public ActionResult Index() { // Assembly info is not needed in production apps and the following 6 lines can be removed var result = new IndexConfig(); var assembly = Assembly.GetExecutingAssembly().GetReferencedAssemblies().Where(n => n.Name.Equals("Microsoft.PowerBI.Api")).FirstOrDefault(); if (assembly != null) { result.DotNetSDK = assembly.Version.ToString(3); } return(View(result)); }
public void TestIndexConfigConstructor() { string indexName = "indexName"; string[] indexedColumns = { "idx1" }; string[] includedColumns = { "inc1", "inc2", "inc3" }; var config = new IndexConfig(indexName, indexedColumns, includedColumns); // Validate that the config was built correctly. Assert.Equal(indexName, config.IndexName); Assert.Equal(indexedColumns, config.IndexedColumns); Assert.Equal(includedColumns, config.IncludedColumns); }
public void CreateIndex <T>(string indexName, IndexConfig settings) { var client = _provider.CreateSearchServiceClient(); var buIndexDefinition = new Index() { Name = indexName, Fields = FieldBuilder.BuildForType <T>() }; DeleteBUIndexIfExists(client, indexName); client.Indexes.Create(buIndexDefinition); }
public static IndexConfigForView GetIndexConfigForView(IndexConfig indexConfig) { indexConfig.RequireNotNull(nameof(indexConfig)); return(new IndexConfigForView { MaxContentHighlightLength = indexConfig.MaxContentHighlightLength, SaveIntervalSeconds = indexConfig.SaveIntervalSeconds, OpenIDEUriFormat = indexConfig.OpenIDEUriFormat, MonitorFolderRealPath = indexConfig.MonitorFolderRealPath, IndexName = indexConfig.IndexName, Pk = indexConfig.Pk }); }
private void LogDBFragLevel(IndexConfig config) { StringBuilder builder = new StringBuilder(); builder.Append("USE "); builder.Append(config.DatabaseName); builder.Append("\r\n"); builder.Append("SELECT DB_NAME(DPS.DATABASE_ID) AS "); builder.Append(DBHealthCheckExecAction.Utils.Constants.DATABASE_NAME); builder.Append(",OBJECT_NAME(DPS.OBJECT_ID) AS "); builder.Append(DBHealthCheckExecAction.Utils.Constants.TABLE_NAME); builder.Append(",SI.NAME AS "); builder.Append(DBHealthCheckExecAction.Utils.Constants.INDEX_NAME); builder.Append(",DPS.INDEX_TYPE_DESC AS "); builder.Append(DBHealthCheckExecAction.Utils.Constants.INDEX_TYPE); builder.Append(",DPS.AVG_FRAGMENTATION_IN_PERCENT AS "); builder.Append(DBHealthCheckExecAction.Utils.Constants.AVG_PAGE_FRAGMENTATION); builder.Append(",DPS.PAGE_COUNT AS "); builder.Append(DBHealthCheckExecAction.Utils.Constants.PAGE_COUNTS); builder.Append(" FROM sys.dm_db_index_physical_stats"); builder.Append(" (DB_ID(), NULL, NULL , NULL, NULL) DPS INNER JOIN sysindexes SI "); builder.Append("ON DPS.OBJECT_ID = SI.ID AND DPS.INDEX_ID = SI.INDID "); // WHERE DPS.AVG_FRAGMENTATION_IN_PERCENT > 1 "); builder.Append("ORDER BY DPS.avg_fragmentation_in_percent DESC "); DatabaseManager manager = new DatabaseManager(); List <object> items = manager.ExecuteQuery(builder.ToString(), QueryMode.Reader, null, new FragmentationMeasurer()) as List <object>; // logging string line = FormatedSpace("DatabaseName", 30) + "\t" + FormatedSpace("TableName", 30) + "\t" + FormatedSpace("IndexType", 30) + "\t" + FormatedSpace("IndexName", 80) + "\t" + FormatedSpace("AvgPageFragmentation", 30) + "\t" + FormatedSpace("PageCounts", 15); Log(line); line = FormatedSpace("*", 30, '*') + "\t" + FormatedSpace("*", 30, '*') + "\t" + FormatedSpace("*", 30, '*') + "\t" + FormatedSpace("*", 80, '*') + "\t" + FormatedSpace("*", 30, '*') + "\t" + FormatedSpace("*", 15, '*'); Log(line); foreach (object obj in items) { FragmentationMeasurer measurer = obj as FragmentationMeasurer; line = FormatedSpace(measurer.DatabaseName, 30) + "\t" + FormatedSpace(measurer.TableName, 30) + "\t" + FormatedSpace(measurer.IndexType, 30) + "\t" + FormatedSpace(measurer.IndexName, 80) + "\t" + FormatedSpace(measurer.AvgPageFragmentation.ToString(), 30) + "\t" + FormatedSpace(measurer.PageCounts.ToString(), 15); Log(line); } }
public void TestGetConfigs() { using var maintainer = new ConfigIndexMaintainer(Config, Log); CollectionAssert.IsEmpty(maintainer.GetConfigs()); var indexConfig = new IndexConfig { IndexName = "ABC" }; maintainer.AddIndexConfig(indexConfig); CollectionAssert.AreEquivalent(new[] { indexConfig }, maintainer.GetConfigs()); }
public HyperspaceTests(HyperspaceFixture fixture) { _spark = fixture.SparkFixture.Spark; _hyperspaceSystemDirectory = new TemporaryDirectory(); _spark.Conf().Set("spark.hyperspace.system.path", _hyperspaceSystemDirectory.Path); _hyperspace = new Hyperspace(_spark); _sampleDataFrame = _spark.Read() .Option("header", true) .Option("delimiter", ";") .Csv($"{TestEnvironment.ResourceDirectory}people.csv"); _sampleIndexName = "sample_dataframe"; _sampleIndexConfig = new IndexConfig(_sampleIndexName, new[] { "job" }, new[] { "name" }); _hyperspace.CreateIndex(_sampleDataFrame, _sampleIndexConfig); }
public void TestEditConfig() { using var maintainer = new ConfigIndexMaintainer(Config, Log); var indexConfig = new IndexConfig { IndexName = "ABC" }; maintainer.AddIndexConfig(indexConfig); CollectionAssert.AreEquivalent(new[] { indexConfig }, maintainer.GetConfigs()); maintainer.EditIndexConfig(indexConfig with { IndexName = "EFG" }); CollectionAssert.AreEquivalent(new[] { indexConfig with {
public static IndexConfig ValidateAndNormalize(this IndexConfig config, string mapName) { if (config == null) { throw new ArgumentNullException(nameof(config)); } // Validate attributes. var originalAttributeNames = config.Attributes; if (originalAttributeNames.Count == 0) { throw new ArgumentException($"Index must have at least one attribute: {config}"); } if (originalAttributeNames.Count > MaxAttributes) { throw new ArgumentException($"Index cannot have more than {MaxAttributes} attributes: {config}"); } var normalizedAttributeNames = new List <string>(originalAttributeNames.Count); foreach (var originalAttributeName in originalAttributeNames) { var attributeName = originalAttributeName.Trim(); ValidateAttribute(config, attributeName); var normalizedAttributeName = CanonicalizeAttribute(attributeName); var existingIdx = normalizedAttributeNames.IndexOf(normalizedAttributeName); if (existingIdx != -1) { var duplicateOriginalAttributeName = originalAttributeNames[existingIdx]; if (duplicateOriginalAttributeName == attributeName) { throw new ArgumentException($"Duplicate attribute name [attributeName={attributeName}, indexConfig={config} ]"); } throw new ArgumentException( $"Duplicate attribute names [attributeName1={duplicateOriginalAttributeName}, attributeName2={attributeName}, indexConfig={config}]"); } normalizedAttributeNames.Add(normalizedAttributeName); } // Construct final index. var name = config.Name; if (name != null && name.Trim().Length == 0) { name = null; } return(BuildNormalizedConfig(mapName, config.Type, name, normalizedAttributeNames)); }
Task AddIndexAsync(IndexConfig indexConfig, CancellationToken cancellationToken) { if (indexConfig == null) { throw new ArgumentNullException(nameof(indexConfig)); } var requestMessage = MapAddIndexCodec.EncodeRequest(Name, indexConfig.ValidateAndNormalize(Name)); var task = Cluster.Messaging.SendAsync(requestMessage, cancellationToken); #if HZ_OPTIMIZE_ASYNC return(task); #else await task.CAF(); #endif }
public IndexMaintainer(IndexConfig indexConfig, CodeIndexConfiguration codeIndexConfiguration, ILogger log) { indexConfig.RequireNotNull(nameof(indexConfig)); codeIndexConfiguration.RequireNotNull(nameof(codeIndexConfiguration)); log.RequireNotNull(nameof(log)); IndexConfig = indexConfig; CodeIndexConfiguration = codeIndexConfiguration; Log = log; Status = IndexStatus.Idle; ExcludedExtensions = indexConfig.ExcludedExtensionsArray.Select(u => u.ToUpperInvariant()).ToArray(); ExcludedPaths = FilePathHelper.GetPaths(indexConfig.ExcludedPathsArray, codeIndexConfiguration.IsInLinux); IncludedExtensions = indexConfig.IncludedExtensionsArray.Select(u => u.ToUpperInvariant()).ToArray() ?? Array.Empty <string>(); TokenSource = new CancellationTokenSource(); }
public static Document GetDocument(IndexConfig indexConfig) { return(new Document { new StringField(nameof(IndexConfig.Pk), indexConfig.Pk.ToString(), Field.Store.YES), new StringField(nameof(IndexConfig.IndexName), indexConfig.IndexName.ToStringSafe(), Field.Store.YES), new StringField(nameof(IndexConfig.MonitorFolder), indexConfig.MonitorFolder.ToStringSafe(), Field.Store.YES), new Int32Field(nameof(IndexConfig.MaxContentHighlightLength), indexConfig.MaxContentHighlightLength, Field.Store.YES), new Int32Field(nameof(IndexConfig.SaveIntervalSeconds), indexConfig.SaveIntervalSeconds, Field.Store.YES), new StringField(nameof(IndexConfig.OpenIDEUriFormat), indexConfig.OpenIDEUriFormat.ToStringSafe(), Field.Store.YES), new StringField(nameof(IndexConfig.MonitorFolderRealPath), indexConfig.MonitorFolderRealPath.ToStringSafe(), Field.Store.YES), new StringField(nameof(IndexConfig.ExcludedPaths), indexConfig.ExcludedPaths.ToStringSafe(), Field.Store.YES), new StringField(nameof(IndexConfig.IncludedExtensions), indexConfig.IncludedExtensions.ToStringSafe(), Field.Store.YES), new StringField(nameof(IndexConfig.ExcludedExtensions), indexConfig.ExcludedExtensions.ToStringSafe(), Field.Store.YES), }); }
public void TestConstructor_InitializeMaintainerPool() { var indexConfig = new IndexConfig { IndexName = "ABC", MonitorFolder = MonitorFolder }; using var management1 = new IndexManagement(Config, Log); CollectionAssert.IsEmpty(management1.GetIndexList().Result); management1.AddIndex(indexConfig); management1.Dispose(); using var management2 = new IndexManagement(Config, Log); CollectionAssert.AreEquivalent(new[] { indexConfig.ToString() }, management2.GetIndexList().Result.Select(u => u.IndexConfig.ToString())); }
public static void ValidateAttribute(IndexConfig config, string attributeName) { if (attributeName == null) { throw new NullReferenceException($"Attribute name cannot be null: {config}"); } var attributeName0 = attributeName.Trim(); if (attributeName0.Length == 0) { throw new ArgumentException($"Attribute name cannot be empty: {config}"); } if (attributeName0.EndsWith(".")) { throw new ArgumentException($"Attribute name cannot end with dot: {attributeName}"); } }
private ScheduleConfig GetConfigFromFileName(string[] args) { string _config = args[1]; ConfigType configType = (ConfigType)Enum.Parse(typeof(ConfigType), _config); switch (configType) { case ConfigType.DoBackupConfig: string type = args[3]; return(DoBackupConfig.GetInstance(type.ToUpper())); case ConfigType.IndexConfig: return(IndexConfig.GetInstance()); case ConfigType.IntegrityCheckConfig: return(IntegrityCheckConfig.GetInstance()); } return(null); }