/// <summary> /// 基于Linq的查询 /// </summary> /// <param name="filter">过滤器</param> /// <returns></returns> public List <Dictionary <string, object> > Find2(string dbName, string collectionName, string query, string protection = "{}", int pageIndex = 0, int pageSize = int.MaxValue, Dictionary <string, object> updateData = null) { List <Dictionary <string, object> > res = new List <Dictionary <string, object> >(); //var filterDict = Convertor.FromJsonToDict2(jsonString); //var filterBuilder = Builders<BsonDocument>.Filter; //var filter = this.FilterConvertor(jsonString); FilterDefinition <BsonDocument> filter = query; ProjectionDefinition <BsonDocument> protectionD = protection; var db = this.client.GetDatabase(dbName); var collection = db.GetCollection <BsonDocument>(collectionName); var cursor = collection.Find(filter).Project(protectionD).Skip(pageIndex * pageSize).Limit(pageSize).ToCursor(); foreach (var document in cursor.ToEnumerable()) { if (null == this.EventTraverse) { res.Add(document.ToDictionary()); } else { EventProc.TriggerEvent(this.EventTraverse, this, EventProcEventArgs.Create(document.ToDictionary())); } } return(res); }
/// <summary> /// 根据新增,修改情况 热词分析 分词 发文/修改统计 /// </summary> public void DataAnalyse() { var startTime = DateTime.Now;///开始运行时间 Console.Title = "文档维护进程进程 启动时间:" + startTime; while (true) { ///遍历写入数据 var dbName = CONST.DB.DBName_DocService; var collectionName = CONST.DB.CollectionName_ModifyLogItem; var db = DataStorage.GetInstance(DBType.MongoDB); db.EventTraverse += (object sender, EventArgs e) => { EventProcEventArgs ee = e as EventProcEventArgs; var dict = ee.Default as Dictionary <string, object>; var targetDbName = dict["DatabaseName"].ToString(); var targetCollectionName = dict["CollectionName"].ToString(); var targetID = (ObjectId)dict["TargetID"]; var id = dict["_id"].ToString(); if (targetCollectionName == CONST.DB.CollectionName_DocItem) { ///重新分词,聚类 var doc = new DocItem(); var plainText = doc.PlainText; var res = FenCi.GetResult(plainText); var queryDel = "{'TargetID':ObjectId('" + targetID.ToString() + "')}"; db.Remove(dbName, collectionName, queryDel); foreach (var item in res) { var svItem = new { DbName = targetDbName, CollectionName = targetCollectionName, TargetID = targetID, Word = item.Key, Count = item.Value, CreateTime = DateTime.Now, TargetCreateTime = dict["CreateTime"] }; db.Save3(CONST.DB.DBName_DocService, CONST.DB.CollectionName_FenCi, svItem); } } else if (targetCollectionName == CONST.DB.CollectionName_CategoryItem) { ///重新统计,当前目录的子目录数量 , 每一级文档数量 var subCategoryList = CategoryManager.GetInstance().GetSubCategory(targetID.ToString()); var category = CategoryItem.Load(targetID.ToString()); category.SubCategoryCount = subCategoryList.Count; category.Save(); } //ModifyLogItem.Remove(id); }; db.Traverse(dbName, collectionName, "{}"); ThreadManager.Pause(minutes: 2); } }
/// <summary> /// 以GZip的格式进行下载 /// </summary> /// <param name="url"></param> /// <returns></returns> public string GetGzip2(string url, Encoding encoding, Dictionary <HttpRequestHeader, string> headers = null) { try { this.SetHeaders(headers); byte[] byteArray = this.http.DownloadData(url); // 处理 gzip string sContentEncoding = this.http.ResponseHeaders["Content-Encoding"]; if (sContentEncoding == "gzip") { var sourceStream = new MemoryStream(byteArray); var targetStream = new MemoryStream(); int count = 0; // 解压 GZipStream gzip = new GZipStream(sourceStream, CompressionMode.Decompress); byte[] buf = new byte[512]; while ((count = gzip.Read(buf, 0, buf.Length)) > 0) { targetStream.Write(buf, 0, count); } var res = encoding.GetString(targetStream.GetBuffer()); sourceStream.Close(); targetStream.Close(); return(res); } else if (string.IsNullOrWhiteSpace(sContentEncoding)) { var res = encoding.GetString(byteArray); return(res); } else { var p = 0; } } catch (Exception e) { var dict = new Dictionary <string, object>(); dict["Url"] = url; dict["Exception"] = e.Message; dict["CreateTime"] = DateTime.Now; EventProc.TriggerEvent(this.EventException, this, EventProcEventArgs.Create(dict)); Console.WriteLine("异常:{0}", e.Message); } return(string.Empty); }
public string PostGZip(string url, string postData) { string res = string.Empty; try { this.http.Headers.Clear(); this.http.Headers.Add("Accept-Encoding", "gzip,deflate"); this.http.Headers.Add("Accept", "application/json, text/javascript, */*; q=0.01"); this.http.Headers.Add("X-Requested-With", "XMLHttpRequest"); this.http.Headers.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); this.http.Headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6"); this.http.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36"); byte[] byteArray = this.http.UploadData(url, Encoding.UTF8.GetBytes(postData)); // 处理 gzip string sContentEncoding = this.http.ResponseHeaders["Content-Encoding"]; if (sContentEncoding == "gzip") { var sourceStream = new MemoryStream(byteArray); var targetStream = new MemoryStream(); int count = 0; // 解压 GZipStream gzip = new GZipStream(sourceStream, CompressionMode.Decompress); byte[] buf = new byte[512]; while ((count = gzip.Read(buf, 0, buf.Length)) > 0) { targetStream.Write(buf, 0, count); } res = Encoding.UTF8.GetString(targetStream.GetBuffer()); sourceStream.Close(); targetStream.Close(); return(res); } } catch (Exception e) { var dict = new Dictionary <string, object>(); dict["Url"] = url; dict["Exception"] = e.Message; dict["CreateTime"] = DateTime.Now; EventProc.TriggerEvent(this.EventException, this, EventProcEventArgs.Create(dict)); } return(string.Empty); }
/// <summary> /// 遍历处理 /// </summary> /// <param name="dbName"></param> /// <param name="tableName"></param> /// <param name="jsonString"></param> public void Traverse(string dbName, string tableName, string query) { var pageSize = 1000; var index = 0; var startTime = DateTime.Now; var prevTime = startTime; var list = mongo.Find3(dbName, tableName, query, "{}", "{}", index++, pageSize); var hasData = (0 < list.Count) ? true : false; var queue = new Queue <List <Dictionary <string, object> > >(); queue.Enqueue(list); while (0 < queue.Count) { var cost = DateTime.Now - prevTime; list = queue.Dequeue(); foreach (var item in list) { EventProc.TriggerEvent(this.EventTraverse, this, EventProcEventArgs.Create(item)); } var task = Task.Factory.StartNew <object>(() => { prevTime = DateTime.Now; Console.WriteLine(" 准备查找第{0}页数据 页面大小:{1} 上次耗时:{2}", index, pageSize, cost); var resList = mongo.Find3(dbName, tableName, query, "{}", "{}", index++, pageSize); return(resList); }); var nextList = task.Result as List <Dictionary <string, object> >; if (0 < nextList.Count) { queue.Enqueue(nextList); Console.WriteLine(" 队列深度 {0} ", queue.Count); } else { Console.WriteLine(" 数据已经全部遍历完毕 "); } } }
/// <summary> /// 添加聚合计算 /// </summary> /// <param name="dbName"></param> /// <param name="collectionName"></param> /// <param name="pipeline"></param> /// <returns></returns> public List <Dictionary <string, object> > Aggregate(string dbName, string collectionName, string[] pipeline) { List <Dictionary <string, object> > res = new List <Dictionary <string, object> >(); var pipelineDefinition = PipelineDefinition <BsonDocument, BsonDocument> .Create(pipeline);//"{$match:{}}", "{$group:{_id:'$CategoryName',total:{$sum:1}}}" var db = this.client.GetDatabase(dbName); var collection = db.GetCollection <BsonDocument>(collectionName); var cursor = collection.Aggregate(pipelineDefinition); foreach (var document in cursor.ToEnumerable()) { if (null == this.EventTraverse) { res.Add(document.ToDictionary()); } else { EventProc.TriggerEvent(this.EventTraverse, this, EventProcEventArgs.Create(document.ToDictionary())); } } return(res); }