/// <summary> /// 基于Linq的查询 /// </summary> /// <param name="filter">过滤器</param> /// <returns></returns> public List <Dictionary <string, object> > Find2(string dbName, string collectionName, string query, string protection = "{}", int pageIndex = 0, int pageSize = int.MaxValue, Dictionary <string, object> updateData = null) { List <Dictionary <string, object> > res = new List <Dictionary <string, object> >(); //var filterDict = Convertor.FromJsonToDict2(jsonString); //var filterBuilder = Builders<BsonDocument>.Filter; //var filter = this.FilterConvertor(jsonString); FilterDefinition <BsonDocument> filter = query; ProjectionDefinition <BsonDocument> protectionD = protection; var db = this.client.GetDatabase(dbName); var collection = db.GetCollection <BsonDocument>(collectionName); var cursor = collection.Find(filter).Project(protectionD).Skip(pageIndex * pageSize).Limit(pageSize).ToCursor(); foreach (var document in cursor.ToEnumerable()) { if (null == this.EventTraverse) { res.Add(document.ToDictionary()); } else { EventProc.TriggerEvent(this.EventTraverse, this, EventProcEventArgs.Create(document.ToDictionary())); } } return(res); }
/// <summary> /// 以GZip的格式进行下载 /// </summary> /// <param name="url"></param> /// <returns></returns> public string GetGzip2(string url, Encoding encoding, Dictionary <HttpRequestHeader, string> headers = null) { try { this.SetHeaders(headers); byte[] byteArray = this.http.DownloadData(url); // 处理 gzip string sContentEncoding = this.http.ResponseHeaders["Content-Encoding"]; if (sContentEncoding == "gzip") { var sourceStream = new MemoryStream(byteArray); var targetStream = new MemoryStream(); int count = 0; // 解压 GZipStream gzip = new GZipStream(sourceStream, CompressionMode.Decompress); byte[] buf = new byte[512]; while ((count = gzip.Read(buf, 0, buf.Length)) > 0) { targetStream.Write(buf, 0, count); } var res = encoding.GetString(targetStream.GetBuffer()); sourceStream.Close(); targetStream.Close(); return(res); } else if (string.IsNullOrWhiteSpace(sContentEncoding)) { var res = encoding.GetString(byteArray); return(res); } else { var p = 0; } } catch (Exception e) { var dict = new Dictionary <string, object>(); dict["Url"] = url; dict["Exception"] = e.Message; dict["CreateTime"] = DateTime.Now; EventProc.TriggerEvent(this.EventException, this, EventProcEventArgs.Create(dict)); Console.WriteLine("异常:{0}", e.Message); } return(string.Empty); }
public string PostGZip(string url, string postData) { string res = string.Empty; try { this.http.Headers.Clear(); this.http.Headers.Add("Accept-Encoding", "gzip,deflate"); this.http.Headers.Add("Accept", "application/json, text/javascript, */*; q=0.01"); this.http.Headers.Add("X-Requested-With", "XMLHttpRequest"); this.http.Headers.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); this.http.Headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6"); this.http.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36"); byte[] byteArray = this.http.UploadData(url, Encoding.UTF8.GetBytes(postData)); // 处理 gzip string sContentEncoding = this.http.ResponseHeaders["Content-Encoding"]; if (sContentEncoding == "gzip") { var sourceStream = new MemoryStream(byteArray); var targetStream = new MemoryStream(); int count = 0; // 解压 GZipStream gzip = new GZipStream(sourceStream, CompressionMode.Decompress); byte[] buf = new byte[512]; while ((count = gzip.Read(buf, 0, buf.Length)) > 0) { targetStream.Write(buf, 0, count); } res = Encoding.UTF8.GetString(targetStream.GetBuffer()); sourceStream.Close(); targetStream.Close(); return(res); } } catch (Exception e) { var dict = new Dictionary <string, object>(); dict["Url"] = url; dict["Exception"] = e.Message; dict["CreateTime"] = DateTime.Now; EventProc.TriggerEvent(this.EventException, this, EventProcEventArgs.Create(dict)); } return(string.Empty); }
/// <summary> /// 遍历处理 /// </summary> /// <param name="dbName"></param> /// <param name="tableName"></param> /// <param name="jsonString"></param> public void Traverse(string dbName, string tableName, string query) { var pageSize = 1000; var index = 0; var startTime = DateTime.Now; var prevTime = startTime; var list = mongo.Find3(dbName, tableName, query, "{}", "{}", index++, pageSize); var hasData = (0 < list.Count) ? true : false; var queue = new Queue <List <Dictionary <string, object> > >(); queue.Enqueue(list); while (0 < queue.Count) { var cost = DateTime.Now - prevTime; list = queue.Dequeue(); foreach (var item in list) { EventProc.TriggerEvent(this.EventTraverse, this, EventProcEventArgs.Create(item)); } var task = Task.Factory.StartNew <object>(() => { prevTime = DateTime.Now; Console.WriteLine(" 准备查找第{0}页数据 页面大小:{1} 上次耗时:{2}", index, pageSize, cost); var resList = mongo.Find3(dbName, tableName, query, "{}", "{}", index++, pageSize); return(resList); }); var nextList = task.Result as List <Dictionary <string, object> >; if (0 < nextList.Count) { queue.Enqueue(nextList); Console.WriteLine(" 队列深度 {0} ", queue.Count); } else { Console.WriteLine(" 数据已经全部遍历完毕 "); } } }
/// <summary> /// 添加聚合计算 /// </summary> /// <param name="dbName"></param> /// <param name="collectionName"></param> /// <param name="pipeline"></param> /// <returns></returns> public List <Dictionary <string, object> > Aggregate(string dbName, string collectionName, string[] pipeline) { List <Dictionary <string, object> > res = new List <Dictionary <string, object> >(); var pipelineDefinition = PipelineDefinition <BsonDocument, BsonDocument> .Create(pipeline);//"{$match:{}}", "{$group:{_id:'$CategoryName',total:{$sum:1}}}" var db = this.client.GetDatabase(dbName); var collection = db.GetCollection <BsonDocument>(collectionName); var cursor = collection.Aggregate(pipelineDefinition); foreach (var document in cursor.ToEnumerable()) { if (null == this.EventTraverse) { res.Add(document.ToDictionary()); } else { EventProc.TriggerEvent(this.EventTraverse, this, EventProcEventArgs.Create(document.ToDictionary())); } } return(res); }