private SubCategory GetSubCategory(MongodbDatabaseDescription description, ServerInfo serverInfo) { try { var subCategory = new SubCategory() { DatabasePrefix = description.DatabasePrefix, DisplayName = description.DisplayName, Name = description.Name, TableNames = new List <string>(), TypeFullName = description.TypeFullName, }; subCategory.TableNames = serverInfo.Databases .Where(database => database.DatabaseName.StartsWith(description.DatabasePrefix)) .SelectMany(database => database.Collections).Select(collection => collection.CollectionName).Distinct().ToList(); return(subCategory); } catch (Exception ex) { LocalLoggingService.Error("{0} {1} {2} {3}", MongodbServerConfiguration.ModuleName, "MongodbServer_Slave", "GetSubCategory", ex.Message); throw; } }
public void SubmitData(IList <MongodbData> dataList, MongodbDatabaseDescription databaseDescription) { if (dataList == null || dataList.Count == 0) { return; } var typeFullName = dataList.First().TypeFullName; try { var config = MongodbServerConfiguration.GetMongodbServerConfigurationItem(typeFullName); if (config == null) { LocalLoggingService.Warning("{0} {1} {2} {3}", MongodbServerConfiguration.ModuleName, "MongodbServer_Master", "SubmitData", string.Format("没取到服务器配置项,参数:{0}", typeFullName)); return; } if (!submitDataMemoryQueueServices.ContainsKey(typeFullName)) { lock (submitDataMemoryQueueServices) { if (!submitDataMemoryQueueServices.ContainsKey(typeFullName)) { var memoryQueueService = LocalServiceLocator.GetService <IMemoryQueueService>(); memoryQueueService.Init(new MemoryQueueServiceConfiguration(string.Format("{0}_{1}", ServiceName, typeFullName), InternalSubmitData) { ConsumeErrorAction = config.ConsumeErrorAction, ConsumeIntervalMilliseconds = config.ConsumeIntervalMilliseconds, ConsumeIntervalWhenErrorMilliseconds = config.ConsumeIntervalWhenErrorMilliseconds, ConsumeItemCountInOneBatch = config.ConsumeItemCountInOneBatch, ConsumeThreadCount = config.ConsumeThreadCount, MaxItemCount = config.MaxItemCount, NotReachBatchCountConsumeAction = config.NotReachBatchCountConsumeAction, ReachMaxItemCountAction = config.ReachMaxItemCountAction, }); submitDataMemoryQueueServices.Add(typeFullName, memoryQueueService); } } } if (databaseDescription != null) { InitMongodbDatabaseDescription(typeFullName, databaseDescription); } if (config.SubmitToDatabase) { submitDataMemoryQueueServices[typeFullName].EnqueueBatch(dataList); } } catch (Exception ex) { LocalLoggingService.Error("{0} {1} {2} {3}", MongodbServerConfiguration.ModuleName, "MongodbServer_Master", "SubmitData", ex.Message); throw; } }
private void InitMongodbDatabaseDescription(string typeFullName, MongodbDatabaseDescription databaseDescription) { try { var server = CreateMasterMongoServer(typeFullName); if (server != null) { var database = server.GetDatabase(MongodbServerConfiguration.MetaDataDbName); var collection = database.GetCollection(databaseDescription.DatabasePrefix); collection.Save(databaseDescription); } } catch (Exception ex) { LocalLoggingService.Error("{0} {1} {2} {3} {4}", MongodbServerConfiguration.ModuleName, "MongodbServer_Master", "InitMongodbDatabaseDescription", string.Format("更新元数据到数据库出现错误,参数:{0} DatabaseName:{1}", typeFullName), ex.Message); } }
private MongodbDatabaseDescription GetMongodbDatabaseDescription <T>(T item) { var type = item.GetType(); var typeFullName = type.FullName; try { var persistenceEntityAttribute = type.GetCustomAttributes(typeof(MongodbPersistenceEntityAttribute), true).Cast <MongodbPersistenceEntityAttribute>().SingleOrDefault(); if (persistenceEntityAttribute == null) { throw new Exception(string.Format("请在类型 '{0}' 上定义MongodbPersistenceEntityAttribute!", typeFullName)); } var mongodbDatabaseDescription = new MongodbDatabaseDescription(); var entityName = persistenceEntityAttribute.Name ?? type.Name; var databasePrefix = string.Format("{0}__{1}", persistenceEntityAttribute.CategoryName, entityName); mongodbDatabaseDescription.DatabasePrefix = databasePrefix; mongodbDatabaseDescription.TypeFullName = typeFullName; mongodbDatabaseDescription.CategoryName = persistenceEntityAttribute.CategoryName; mongodbDatabaseDescription.Name = entityName; mongodbDatabaseDescription.DisplayName = persistenceEntityAttribute.DisplayName ?? entityName; mongodbDatabaseDescription.ExpireDays = persistenceEntityAttribute.ExpireDays; var mongodbColumnDescriptionList = new List <MongodbColumnDescription>(); var mongodbEnumColumnDescriptionList = new List <MongodbEnumColumnDescription>(); foreach (var property in GetPropertyListFromCache(type)) { GetMongodbColumnDescription(typeFullName, string.Empty, mongodbColumnDescriptionList, mongodbEnumColumnDescriptionList, property); } mongodbDatabaseDescription.MongodbColumnDescriptionList = mongodbColumnDescriptionList; mongodbDatabaseDescription.MongodbEnumColumnDescriptionList = mongodbEnumColumnDescriptionList; return(mongodbDatabaseDescription); } catch (Exception ex) { LocalLoggingService.Error("{0} {1} {2} {3}", MongodbServiceConfiguration.ModuleName, "GetMongodbDatabaseDescription", typeFullName, ex.Message); throw; } }
private SubCategory GetSubCategory(MongodbDatabaseDescription description, ServerInfo serverInfo) { try { var subCategory = new SubCategory() { DatabasePrefix = description.DatabasePrefix, DisplayName = description.DisplayName, Name = description.Name, TableNames = new List<string>(), TypeFullName = description.TypeFullName, }; subCategory.TableNames = serverInfo.Databases .Where(database => database.DatabaseName.StartsWith(description.DatabasePrefix)) .SelectMany(database => database.Collections).Select(collection => collection.CollectionName).Distinct().ToList(); return subCategory; } catch (Exception ex) { LocalLoggingService.Error("{0} {1} {2} {3}", MongodbServerConfiguration.ModuleName, "MongodbServer_Slave", "GetSubCategory", ex.Message); throw; } }
public void SubmitData(IList<MongodbData> dataList, MongodbDatabaseDescription databaseDescription) { if (dataList == null || dataList.Count == 0) return; var typeFullName = dataList.First().TypeFullName; try { var config = MongodbServerConfiguration.GetMongodbServerConfigurationItem(typeFullName); if (config == null) { LocalLoggingService.Warning("{0} {1} {2} {3}", MongodbServerConfiguration.ModuleName, "MongodbServer_Master", "SubmitData", string.Format("没取到服务器配置项,参数:{0}", typeFullName)); return; } if (!submitDataMemoryQueueServices.ContainsKey(typeFullName)) { lock (submitDataMemoryQueueServices) { if (!submitDataMemoryQueueServices.ContainsKey(typeFullName)) { var memoryQueueService = LocalServiceLocator.GetService<IMemoryQueueService>(); memoryQueueService.Init(new MemoryQueueServiceConfiguration(string.Format("{0}_{1}", ServiceName, typeFullName), InternalSubmitData) { ConsumeErrorAction = config.ConsumeErrorAction, ConsumeIntervalMilliseconds = config.ConsumeIntervalMilliseconds, ConsumeIntervalWhenErrorMilliseconds = config.ConsumeIntervalWhenErrorMilliseconds, ConsumeItemCountInOneBatch = config.ConsumeItemCountInOneBatch, ConsumeThreadCount = config.ConsumeThreadCount, MaxItemCount = config.MaxItemCount, NotReachBatchCountConsumeAction = config.NotReachBatchCountConsumeAction, ReachMaxItemCountAction = config.ReachMaxItemCountAction, }); submitDataMemoryQueueServices.Add(typeFullName, memoryQueueService); } } } if (databaseDescription != null) { InitMongodbDatabaseDescription(typeFullName, databaseDescription); } if (config.SubmitToDatabase) { submitDataMemoryQueueServices[typeFullName].EnqueueBatch(dataList); } } catch (Exception ex) { LocalLoggingService.Error("{0} {1} {2} {3}", MongodbServerConfiguration.ModuleName, "MongodbServer_Master", "SubmitData", ex.Message); throw; } }
//internal static List<BaseInfo> GetMemoeyQueueState() //{ // lock (submitDataMemoryQueueServices) // { // var state = new List<BaseInfo>() // { // new MongodbServiceStateInfo // { // MemoryQueueServiceStates = submitDataMemoryQueueServices.ToDictionary(s=>s.Key, s=>s.Value.GetState()), // } // }; // return state; // } //} public void Insert(object item) { if (!Enabled) { return; } var type = item.GetType(); var typeFullName = type.FullName; try { var config = MongodbServiceConfiguration.GetMongodbInsertServiceConfigurationItem(typeFullName); if (config == null) { //AppInfoCenterService.LoggingService.Warning(MongodbServiceConfiguration.ModuleName, "Insert", typeFullName, // string.Format("没取到类型名为 {0} 的配置信息!", typeFullName)); return; } if (!submitDataMemoryQueueServices.ContainsKey(typeFullName)) { lock (submitDataMemoryQueueServices) { if (!submitDataMemoryQueueServices.ContainsKey(typeFullName)) { var memoryQueueService = LocalServiceLocator.GetService <IMemoryQueueService>(); memoryQueueService.Init(new MemoryQueueServiceConfiguration(string.Format("{0}_{1}", ServiceName, typeFullName), InternalSubmitData) { ConsumeErrorAction = config.ConsumeErrorAction, ConsumeIntervalMilliseconds = config.ConsumeIntervalMilliseconds, ConsumeIntervalWhenErrorMilliseconds = config.ConsumeIntervalWhenErrorMilliseconds, ConsumeItemCountInOneBatch = config.ConsumeItemCountInOneBatch, ConsumeThreadCount = config.ConsumeThreadCount, MaxItemCount = config.MaxItemCount, NotReachBatchCountConsumeAction = config.NotReachBatchCountConsumeAction, ReachMaxItemCountAction = config.ReachMaxItemCountAction, }); submitDataMemoryQueueServices.Add(typeFullName, memoryQueueService); } } } if (!mongodbDatabaseDescriptionCache.ContainsKey(typeFullName)) { lock (mongodbDatabaseDescriptionCache) { if (!mongodbDatabaseDescriptionCache.ContainsKey(typeFullName)) { MongodbDatabaseDescription mongodbDatabaseDescription = GetMongodbDatabaseDescription(item); CheckMongodbDatabaseDescription(mongodbDatabaseDescription); mongodbDatabaseDescriptionCache.Add(typeFullName, mongodbDatabaseDescription); } } } if (config.SubmitToServer && submitDataMemoryQueueServices.ContainsKey(typeFullName)) { submitDataMemoryQueueServices[typeFullName].Enqueue(item); } } catch (Exception ex) { LocalLoggingService.Error("{0} {1} {2} {3}", MongodbServiceConfiguration.ModuleName, "Insert", typeFullName, ex.Message); throw; } }
private void CheckMongodbDatabaseDescription(MongodbDatabaseDescription mongodbDatabaseDescription) { if (mongodbDatabaseDescription.MongodbColumnDescriptionList.Count(column => column.IsTableColumn) > 1) { throw new Exception("作为表名的列不能超过一个!"); } if (mongodbDatabaseDescription.MongodbColumnDescriptionList.Count(column => column.IsContextIdentityColumn) > 1) { throw new Exception("作为上下文标识的列不能超过一个!"); } var tableColumn = mongodbDatabaseDescription.MongodbColumnDescriptionList.SingleOrDefault(column => column.IsTableColumn); if (tableColumn != null && tableColumn.MongodbIndexOption != MongodbIndexOption.None) { throw new Exception(string.Format("作为表名的列 '{0}' 不能有索引!", tableColumn.Name)); } if (mongodbDatabaseDescription.MongodbColumnDescriptionList.Count(column => column.IsPrimaryKey) != 1) { throw new Exception("必须有并且只有一个主键列!"); } if (mongodbDatabaseDescription.MongodbColumnDescriptionList.Count(column => column.IsTimeColumn) != 1) { throw new Exception("必须有并且只有一个时间列!"); } if (mongodbDatabaseDescription.MongodbColumnDescriptionList.Count(column => column.MongodbSortOption != MongodbSortOption.None) > 1) { throw new Exception("排序列不能超过两个!"); } if (mongodbDatabaseDescription.MongodbColumnDescriptionList.Count(column => column.MongodbCascadeFilterOption == MongodbCascadeFilterOption.LevelOne) > 1) { throw new Exception("第一级的级联过滤列不能超过一个!"); } if (mongodbDatabaseDescription.MongodbColumnDescriptionList.Count(column => column.MongodbCascadeFilterOption == MongodbCascadeFilterOption.LevelTwo) > 1) { throw new Exception("第二级的级联过滤列不能超过一个!"); } if (mongodbDatabaseDescription.MongodbColumnDescriptionList.Count(column => column.MongodbCascadeFilterOption == MongodbCascadeFilterOption.LevelThree) > 1) { throw new Exception("第三级的级联过滤列不能超过一个!"); } var pkColumn = mongodbDatabaseDescription.MongodbColumnDescriptionList.Single(column => column.IsPrimaryKey); if (pkColumn.MongodbIndexOption != MongodbIndexOption.AscendingAndUnique && pkColumn.MongodbIndexOption != MongodbIndexOption.DescendingAndUnique) { throw new Exception(string.Format("主键列' {0}' 必须具有唯一索引!", pkColumn.Name)); } var timeColumn = mongodbDatabaseDescription.MongodbColumnDescriptionList.Single(column => column.IsTimeColumn); if (timeColumn.MongodbIndexOption != MongodbIndexOption.Ascending && timeColumn.MongodbIndexOption != MongodbIndexOption.Descending) { throw new Exception(string.Format("时间列 '{0}' 必须具有不唯一索引!", timeColumn.Name)); } if (timeColumn.TypeName != "DateTime") { throw new Exception(string.Format("时间列 '{0}' 的类型必须是DateTime!", timeColumn)); } var sortColumn = mongodbDatabaseDescription.MongodbColumnDescriptionList.SingleOrDefault(column => column.MongodbSortOption != MongodbSortOption.None); if (sortColumn != null && sortColumn.MongodbIndexOption != MongodbIndexOption.Ascending && sortColumn.MongodbIndexOption != MongodbIndexOption.Descending) { throw new Exception(string.Format("排序列 '{0}' 必须具有不唯一索引!", sortColumn.Name)); } var contextColumn = mongodbDatabaseDescription.MongodbColumnDescriptionList.SingleOrDefault(column => column.IsContextIdentityColumn); if (contextColumn != null && contextColumn.MongodbIndexOption != MongodbIndexOption.Ascending && contextColumn.MongodbIndexOption != MongodbIndexOption.Descending) { throw new Exception(string.Format("上下文标识列 '{0}' 必须具有不唯一索引!", contextColumn.Name)); } var filterColumns = mongodbDatabaseDescription.MongodbColumnDescriptionList.Where(column => column.MongodbFilterOption != MongodbFilterOption.None || column.MongodbCascadeFilterOption != MongodbCascadeFilterOption.None).ToList(); filterColumns.Each(filterColumn => { if (filterColumn.MongodbIndexOption == MongodbIndexOption.None) { throw new Exception(string.Format("作为搜索条件的列 '{0}' 必须具有索引!", filterColumn.Name)); } }); }