/// <summary> /// 拼接含参数Url /// 重写 /// </summary> /// <param name="param">参数列表</param> /// <returns>完整URL</returns> public override string MakeUrl(AqiParam param) { return(string.Format(Url, param["area"], param["time"], token)); }
/// <summary> /// 加载参数 /// 读取JSON中的depthsParams字段 /// </summary> /// <returns></returns> public override bool LoadParams() { this.listParamCache = this.loadDepthsParams(); this.dtParamCacheTime = AqiParam.ReadWriteTimeFormJson(this); return(true); }
/// <summary> /// 重试处理 /// </summary> /// <param name="arn"></param> private bool retryProcess(AqiRetryNode arn) { ISrcUrl isu = arn.SRCURL; AqiParam ap = arn.PARAM; byte[] data = null; try { if (ap != null) { data = isu.getDate(ap); } else { data = isu.getDate(); } } catch (Exception ex) { //再 入重试队列 PutAgain(arn, ex); return(false); } am.AqiRun.saveProcess(isu, ap, data); //重置 arn.Reset(); return(true); }
/// <summary> /// 加载depthsParams用于计算 /// </summary> /// <returns></returns> private List <AqiParam> loadDepthsParams() { List <AqiParam> apList = new List <AqiParam>(); //扩展参数 List <AqiParam> apListTemp = AqiParam.CreateListFormJson(this, this.Tag, "depthsParams"); //根据 扩展参数 生成最终参数 foreach (AqiParam apTemp in apListTemp) { int idS = Int32.Parse(apTemp["yearidStart"]); int idE = Int32.Parse(apTemp["yearidEnd"]); int idD = Int32.Parse(apTemp["yearidDepth"]); for (int yearid = idS; yearid <= idE; yearid++) { int id = yearid % 100; if (id > idD) { yearid = (yearid / 100) * 100 + 101; continue; } AqiParam ap = new AqiParam(apTemp.Name + "\\" + yearid); ap.Add("", yearid.ToString()); apList.Add(ap); } } return(apList); }
/// <summary> /// 拼接请求体 /// .可以重写 /// </summary> /// <remarks> /// TODO 应该是SOAP消息 /// </remarks> /// <param name="param">参数列表</param> /// <returns></returns> public override byte[] MakeRequestBody(AqiParam param) { XDocument doc = new XDocument(); XNamespace nsS = XNamespace.Get("http://schemas.xmlsoap.org/soap/envelope/"); XElement eEnvelope = new XElement(XName.Get("Envelope", nsS.NamespaceName), new XAttribute(XNamespace.Xmlns + "s", nsS)); XElement eBody = new XElement(XName.Get("Body", nsS.NamespaceName)); XNamespace ns = XNamespace.Get("http://tempuri.org/"); XElement eName = new XElement(XName.Get(this.Tag, ns.NamespaceName)); eBody.Add(eName); eEnvelope.Add(eBody); doc.Add(eEnvelope); MemoryStream ms = new MemoryStream(); //StreamWriter w = new StreamWriter(ms, Encoding.UTF8); XmlWriterSettings xws = new XmlWriterSettings { OmitXmlDeclaration = true, Encoding = Encoding.UTF8 }; XmlWriter writer2 = XmlWriter.Create(ms, xws); using (writer2) { doc.WriteTo(writer2); writer2.Flush(); } byte[] requestBody = ms.ToArray(); return(requestBody); }
/// <summary> /// 加载boundsParams用于计算 /// </summary> /// <returns></returns> private List <AqiParam> PermutationParams() { List <AqiParam> apList = new List <AqiParam>(); //扩展参数 List <AqiParam> apListTemp = AqiParam.CreateListFormJson(this, this.Tag, "Params"); //根据 扩展参数 生成最终参数 foreach (AqiParam apTemp in apListTemp) { Dictionary <string, string[]> data = new Dictionary <string, string[]>(); foreach (var kv in apTemp) { string[] v = kv.Value.Split('.'); data.Add(kv.Key, v); } List <Dictionary <string, string> > result = new List <Dictionary <string, string> >(); ForPermutation(data, ref result); foreach (Dictionary <string, string> map in result) { AqiParam ap = new AqiParam(apTemp, map); apList.Add(ap); } } return(apList); }
/// <summary> /// 合并 /// 仅定时线程调用 /// </summary> /// <param name="node"></param> public void Concat(AqiRetryNode node) { thisLock.EnterWriteLock(); try { this.isu = node.isu; this.ap = node.ap; this.starttime = DateTime.Now; this.endtime = DateTime.Now; this.count = 1; //将NAME_DATA计数保留,其他清空 int i = 0; if (this.counts.ContainsKey(NAME_DATA)) { i = this.counts[NAME_DATA]; } this.counts.Clear(); this.counts.Add(NAME_DATA, i); this.counts = this.counts.Concat(node.counts).ToDictionary(x => x.Key, y => y.Value); AddNameCount(NAME_DATA); } finally { thisLock.ExitWriteLock(); } }
/// <summary> /// 新数据 /// </summary> /// <param name="isu"></param> /// <param name="data"></param> public NoteNode AddNew(ISrcUrl isu, AqiParam ap, byte[] data) { int size = 0; if (data != null) { size = data.Length; } string name = null; if (ap == null) { name = isu.Tag; } else { name = isu.Tag + ap.Name; } NoteNode item = new NoteNode(name, size); lock (this.history) { this.history.Add(item); this.endtime = DateTime.Now; this.size += item.dateSize; this.count += 1L; } return(item); }
/// <summary> /// 加载boundsParams用于计算 /// </summary> /// <returns></returns> private List <AqiParam> loadBoundsParams() { List <AqiParam> apList = new List <AqiParam>(); //扩展参数 List <AqiParam> apListTemp = AqiParam.CreateListFormJson(this, this.Tag, "boundsParams"); //根据 扩展参数 生成最终参数 foreach (AqiParam apTemp in apListTemp) { double lonS = double.Parse(apTemp["lonStart"]); double latS = double.Parse(apTemp["latStart"]); double lonEnd = double.Parse(apTemp["lonEnd"]); double latEnd = double.Parse(apTemp["latEnd"]); double increase = double.Parse(apTemp["increase"]); for (double lon = lonS; lon < lonEnd; lon = lon + increase) { for (double lat = latS; lat < latEnd; lat = lat + increase) { AqiParam ap = new AqiParam("[枚举图块]\\" + apTemp.Name + "\\" + lon.ToString() + "-" + lat.ToString()); //ap.Add("", createurlByLonLat(lon, lat, increase)); ap.Add("lurlv2", apTemp["lurlv2"]); ap.Add("z", apTemp["z"]); ap.Add("bounds", createUrlByLonLat(lon, lat, increase)); apList.Add(ap); } } } return(apList); }
public List <AqiParam> ParseParam(byte[] data) { List <AqiParam> listParam = new List <AqiParam>(); string json = Encoding.UTF8.GetString(data); JArray ja = JArray.Parse(json); JEnumerable <JToken> je = ja.Children(); foreach (JToken j in je) { if (!(j is JObject)) { throw new DataDifferentException("与预期的数据不一致(JSON数组子元素应该是对象),可能数据源已经发生变化"); } JObject joOne = j as JObject; JToken jttf = joOne.GetValue("tfid"); if (jttf == null) { throw new DataDifferentException("与预期的数据不一致(JSON数组对象应该包含tfid属性),可能数据源已经发生变化"); } AqiParam ap = new AqiParam(jttf.ToString() + "号台风"); ap.Add("", jttf.ToString()); listParam.Add(ap); } return(listParam); }
public List <AqiParam> ParseParam(byte[] data) { List <AqiParam> listParam = new List <AqiParam>(); //XNamespace b = "http://schemas.datacontract.org/2004/07/Env.Publish.Province.DAL"; //XDocument xd = XDocument.Load(new MemoryStream(data)); //IEnumerable<XElement> elements = xd.Descendants(XName.Get("AQICityStation", b.ToString())); //foreach (XElement node in elements) //{ // string name = node.Element(XName.Get("PositionName", b.ToString())).Value; // string id = node.Element(XName.Get("StationID", b.ToString())).Value; // string cityName = node.Element(XName.Get("CityName", b.ToString())).Value; // AqiParam ap = new AqiParam(id + "_" + name); // ap.Add("stationCode", id); // listParam.Add(ap); //} XNamespace b = "http://schemas.datacontract.org/2004/07/Env.Publish.Province.DAL"; XDocument xd = XDocument.Load(new MemoryStream(data)); IEnumerable <XElement> elements = xd.Descendants(XName.Get("AQIDataPublishLive", b.ToString())); foreach (XElement node in elements) { string name = node.Element(XName.Get("PositionName", b.ToString())).Value; string id = node.Element(XName.Get("StationCode", b.ToString())).Value; string cityName = node.Element(XName.Get("Area", b.ToString())).Value; AqiParam ap = new AqiParam(id + "_" + name); ap.Add("stationCode", id); listParam.Add(ap); } return(listParam); }
/// <summary> /// 新数据 /// </summary> /// <param name="isu"></param> /// <param name="data"></param> public AqiNoteNode AddNew(ISrcUrl isu, AqiParam ap, byte[] data) { int nSize = data.Length; string nName = null; if (ap == null) { nName = isu.TAG; } else { nName = isu.TAG + ap.Name; } AqiNoteNode n = new AqiNoteNode(nName, nSize); lock (history) { history.Add(n.Name, n); endtime = DateTime.Now; size = size + n.DateSize; count++; } return(n); }
/// <summary> /// 新数据 /// </summary> /// <param name="isu"></param> /// <param name="data"></param> public AqiNoteNode AddNew(ISrcUrl isu, AqiParam ap, byte[] data) { int nSize = data.Length; string nName = null; if(ap == null) { nName = isu.TAG; } else { nName = isu.TAG + ap.Name; } AqiNoteNode n = new AqiNoteNode(nName, nSize); lock(history) { history.Add(n.Name, n); endtime = DateTime.Now; size = size + n.DateSize; count++; } return n; }
public List <AqiParam> ParseParam(byte[] data) { List <AqiParam> listParam = new List <AqiParam>(); string json = Encoding.UTF8.GetString(data); JObject joOne = JObject.Parse(json); JToken jtimg = joOne.GetValue("VisibleImage"); if (jtimg == null) { throw new DataDifferentException("与预期的数据不一致(JSON数组对象应该包含VisibleImage属性),可能数据源已经发生变化"); } if (String.IsNullOrEmpty(jtimg.ToString())) { return(listParam); } string name = jtimg.ToString(); AqiParam ap = new AqiParam(name); ap.Unique = true; ap.Add("", jtimg.ToString()); listParam.Add(ap); return(listParam); }
/// <summary> /// 新数据 /// </summary> /// <param name="isu"></param> /// <param name="data"></param> public NoteNode AddNew(ISrcUrl isu, AqiParam ap, byte[] data) { int size = 0; if (data != null) { size = data.Length; } string name = null; if (ap == null) { name = isu.Tag; } else { name = isu.Tag + ap.Name; } NoteNode item = new NoteNode(name, size); lock (this.history) { this.history.Add(item); this.endtime = DateTime.Now; this.size += item.dateSize; this.count += 1L; } return item; }
/// <summary> /// 加载参数 /// 可以重写 /// </summary> /// <returns></returns> public virtual bool LoadParams() { switch (ParamSourceType) { case AqiConstant.ParamSourceType.JSON: this.listParamCache = AqiParam.CreateListFormJson(this, this.Tag, AqiParam.PARAMS); this.dtParamCacheTime = AqiParam.ReadWriteTimeFormJson(this); break; case AqiConstant.ParamSourceType.ISrcUrl: if (this is IParseSrcUrlParam) { IParseSrcUrlParam ipp = this as IParseSrcUrlParam; this.listParamCache = AqiParam.CreateListFormSrcUrl(ipp, ipp.ParamSrcUrl); this.dtParamCacheTime = DateTime.Now; } else { throw new ParamException("参数来源类型(ISRCUrl)与接口(IParseParam)不匹配"); } break; default: throw new ParamException("参数来源类型(ParamSourceType)未知"); } //TODO //验证参数是否匹配ParamName return(true); }
/// <summary> /// 保存处理 /// </summary> /// <param name="data">数据</param> /// <param name="isu">数据接口</param> /// <param name="ap">参数</param> public void SaveProcess(byte[] data, ISrcUrl isu, AqiParam ap) { try { //统计信息 NoteNode node = this.an.AddNew(isu, ap, data); //保存 if (ap == null) { node.Saved = this.ias.Save(isu, data); } else { node.Saved = this.ias.Save(isu, ap, data); } if (node.Saved) { AqiManage.Remind.Log_Info("数据保存成功", new string[] { this.name, isu.Name }); } else { node.Data = data; AqiManage.Remind.Log_Error("数据保存失败", new string[] { this.name, isu.Name }); } } catch (Exception exception) { AqiManage.Remind.Log_Error("数据保存失败", exception, new string[] { this.name, isu.Name }); } }
public bool Save(ISrcUrl isu, AqiParam param, byte[] data) { try { string str; if (data == null) { data = new byte[0]; if (!this.saveEmpty) { return false; } } //目录 string path = this.basePath + isu.IAW.Tag + @"\" + isu.Tag + @"\"; //分组 string grouptag = ""; //文件名 string nameFile = this.getDate() + "_" + this.getTime(); if (param != null) { grouptag = param.Name; if (param.Unique) { nameFile = grouptag; } else { path = path + grouptag + @"\"; } } //扩展名 string nameExtension = "." + isu.IAW.DAT.ToString().ToLower(); if (isu is IDataType) { IDataType idt = (isu as IDataType); if (idt.DAT == AqiConstant.DataType.NONE) { nameExtension = ""; }else{ nameExtension = "." + idt.DAT.ToString().ToLower(); } } str = nameFile + nameExtension; if (!Directory.Exists(path)) { Directory.CreateDirectory(path); } AqiManage.Remind.Log_Debug("保存路径为" + path + str, new string[] { this.Name }); return FileReadSaveUtil.Save(data, path + str); } catch (Exception exception) { AqiManage.Remind.Log_Error("保存失败", exception, new string[] { this.Name }); return false; } }
/// <summary> /// 重试处理 /// 应该由Retryer线程调用 /// </summary> /// <param name="arn">重试节点</param> /// <returns></returns> public bool RetryProcess(RetryNode arn) { ISrcUrl isu = arn.SrcUrl; AqiParam ap = arn.Param; byte[] data = null; try { if (isu is ISrcUrlParam) { ISrcUrlParam isup = isu as ISrcUrlParam; data = isup.GetData(ap); } else { data = isu.GetData(); } } catch (Exception ex) { AqiManage.Remind.Log_Error("数据重试失败,再入重试队列", new string[] { this.name, arn.Name, isu.Name }); this.ar.PutAgain(arn, ex); return(false); } this.SaveProcess(data, isu, ap); arn.Reset(); return(true); }
/// <summary> /// 拼接请求头 /// .可以重写 /// </summary> /// <remarks> /// 读取参数 Header ,将 Header 作为 Http Header /// TODO 考虑其他格式 /// </remarks> /// <param name="param"></param> /// <returns>RequestHeader键值集合</returns> public virtual Dictionary <string, string> MakeRequestHeader(AqiParam param) { //TODO 允许null if (param.Header == null) { return(new Dictionary <string, string>()); } return(param.Header); }
/// <summary> /// 拼接请求头 /// .可以重写 /// </summary> /// <remarks> /// 读取参数,若不存在‘Content-Type’则添加默认‘application/x-www-form-urlencoded; charset=UTF-8’ /// </remarks> /// <param name="param"></param> /// <returns></returns> public override Dictionary<string, string> MakeRequestHeader(AqiParam param) { Dictionary<string, string> header = base.MakeRequestHeader(param); if (!header.ContainsKey("Content-Type")) { header.Add("Content-Type", @"application/x-www-form-urlencoded; charset=UTF-8"); } return header; }
/// <summary> /// 拼接请求体 /// .可以重写 /// </summary> /// <remarks> /// 将XML形式的WCFMessage转为Bin形式的WCFMessage /// </remarks> /// <param name="param">参数列表</param> /// <returns></returns> public override byte[] MakeRequestBody(AqiParam param) { //TODO 创建MakeWCFMsg接口 string wcfmessage = MakeWCFMsg(param); byte[] requestBody = _mh.GetWcfBinaryMessageAsBinary(wcfmessage); return(requestBody); }
private ReaderWriterLockSlim thisLock; //读写锁 #endregion Fields #region Constructors public AqiRetryNode(ISrcUrl isrcurl, AqiParam aqiparam) { this.ap = aqiparam; this.isu = isrcurl; this.starttime = DateTime.Now; this.endtime = DateTime.Now; this.count = 1; this.counts = new Dictionary<string, int>(); this.thisLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); }
/// <summary> /// 拼接请求头 /// .可以重写 /// </summary> /// <remarks> /// 读取参数,若不存在‘Content-Type’则添加默认‘application/x-www-form-urlencoded; charset=UTF-8’ /// </remarks> /// <param name="param"></param> /// <returns></returns> public override Dictionary <string, string> MakeRequestHeader(AqiParam param) { Dictionary <string, string> header = base.MakeRequestHeader(param); if (!header.ContainsKey("Content-Type")) { header.Add("Content-Type", @"application/x-www-form-urlencoded; charset=UTF-8"); } return(header); }
public AqiRetryNode(ISrcUrl isrcurl, AqiParam aqiparam) { this.ap = aqiparam; this.isu = isrcurl; this.starttime = DateTime.Now; this.endtime = DateTime.Now; this.count = 1; this.counts = new Dictionary <string, int>(); this.thisLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); }
/// <summary> /// 拼接请求头 /// .可以重写 /// </summary> /// <remarks> /// 读取参数,若不存在‘Content-Type’则添加默认‘application/x-amf’ /// </remarks> /// <param name="param"></param> /// <returns></returns> public override Dictionary <string, string> MakeRequestHeader(AqiParam param) { Dictionary <string, string> header = base.MakeRequestHeader(param); if (!header.ContainsKey("Content-Type")) { header.Add("Content-Type", @"application/x-amf"); } return(header); }
/// <summary> /// 拼接含参数Url /// 重写 /// </summary> /// <param name="param">参数列表</param> /// <returns>完整URL</returns> public override string MakeUrl(AqiParam param) { return(string.Format(Url, param["pollutant"], param["timestart"], param["timeend"], param["citycode"], param["stationcode"], token)); }
public List<AqiParam> ParseParam(byte[] data) { List<AqiParam> listParam = new List<AqiParam>(); string json = Encoding.UTF8.GetString(data); JObject jo = JObject.Parse(json); JToken jt = jo.SelectToken("items"); if (!(jt is JArray)) { throw new DataDifferentException("与预期的数据不一致(items属性应该是JSON数组),可能数据源已经发生变化"); } JArray ja = jt as JArray; JEnumerable<JToken> je = ja.Children(); foreach (JToken j in je) { if (!(j is JObject)) { throw new DataDifferentException("与预期的数据不一致(JSON数组子元素应该是对象),可能数据源已经发生变化"); } JObject joOne = j as JObject; JToken jttype = joOne.GetValue("type"); if (jttype.ToString().Equals("1")) { //跳过无效参数 continue; } JToken jtcode = joOne.GetValue("subid"); if (jtcode == null) { throw new DataDifferentException("与预期的数据不一致(JSON数组对象应该包含subid属性),可能数据源已经发生变化"); } string name = jtcode.ToString() + "号站点"; //名称 JToken jtname = joOne.GetValue("pname"); if (jtname != null) { name = name + "-" + jtname; } JToken jtcity = joOne.GetValue("cityName"); if (jtcity != null) { name = name + "-" + jtcity; } AqiParam ap = new AqiParam(name); ap.Add("Method", "GetNjdValue"); ap.Add("StationID", jtcode.ToString()); listParam.Add(ap); } return listParam; }
public List <AqiParam> ParseParam(byte[] data) { List <AqiParam> list = AqiParam.CreateListFormJson(this, this.Tag, AqiParam.PARAMS); List <AqiParam> listParam = new List <AqiParam>(); string json = Encoding.UTF8.GetString(data); JObject jo = JObject.Parse(json); JToken jdata = jo.GetValue("data"); if (!(jdata is JArray)) { throw new DataDifferentException("与预期的数据不一致(JSON对象应该包含data属性),可能数据源已经发生变化"); } JEnumerable <JToken> je = jdata.Children(); foreach (JToken j in je) { if (!(j is JObject)) { throw new DataDifferentException("与预期的数据不一致(JSON数组子元素应该是对象),可能数据源已经发生变化"); } JObject joOne = j as JObject; JToken jcity = joOne.GetValue("xzqdm"); if (jcity == null) { throw new DataDifferentException("与预期的数据不一致(JSON数组对象应该包含xzqdm属性),可能数据源已经发生变化"); } JToken jstationname = joOne.GetValue("zdmc"); if (jstationname == null) { throw new DataDifferentException("与预期的数据不一致(JSON数组对象应该包含zdmc属性),可能数据源已经发生变化"); } JToken jstation = joOne.GetValue("zddm"); if (jstation == null) { throw new DataDifferentException("与预期的数据不一致(JSON数组对象应该包含zddm属性),可能数据源已经发生变化"); } foreach (AqiParam p in list) { AqiParam ap = new AqiParam(p["pollutant"] + @"\" + jstation.ToString() + "-" + jstationname.ToString()); ap.Add("pollutant", p["pollutant"]); ap.Add("timestart", DateTime.Now.AddDays(-1).ToString("yyyyMMddHH0000")); ap.Add("timeend", DateTime.Now.ToString("yyyyMMddHH0000")); ap.Add("citycode", jcity.ToString()); ap.Add("stationcode", jstation.ToString()); listParam.Add(ap); } } return(listParam); }
/// <summary> /// 拼接请求体 /// .可以重写 /// </summary> /// <remarks> /// DSId必须要与Cookie对应,需要通过 <see cref="_5"/> 动态获取,DSId将以字节方式替换旧的DSId /// </remarks> /// <param name="param">参数列表</param> /// <returns></returns> public override byte[] MakeRequestBody(AqiParam param) { byte[] requestbody = base.MakeRequestBody(param); byte[] newDSId = Encoding.ASCII.GetBytes(param["DSId"]); byte[] oldDSId = Encoding.ASCII.GetBytes(param["replaceDSId"]); while (requestbody.FindBytes(oldDSId) >= 0) { requestbody = requestbody.ReplaceBytes(oldDSId, newDSId); } return(requestbody); }
/// <summary> /// 入重试队列 /// 定时器线程 /// </summary> /// <param name="arName"></param> /// <param name="isu"></param> /// <param name="ap"></param> /// <param name="ex"></param> public void PutNew(string arName, ISrcUrl isu, AqiParam ap, Exception ex) { //封装为重试节点 RetryNode arn = new RetryNode(arName, isu, ap); arn.NodeEvent += new RetryNode.NodeEventHandler(this.node_RunEvent); //更新计数 arn.Update(ex); //添加历史记录 arn = this.AddHistory(arn); //入队列 this.Push(arn.Name); AqiManage.Remind.Log_Info("已添加到重试队列", new string[] { this.name, arn.RunnerName, arn.Name }); }
/// <summary> /// 拼接请求头 /// .可以重写 /// </summary> /// <remarks> /// 读取参数,添加2个Header // SOAPAction: "http://tempuri.org/IEnvCriteriaAqiService/GetAllAQIPublish" // Content-Type: text/xml; charset=utf-8 /// </remarks> /// <param name="param"></param> /// <returns></returns> public override Dictionary <string, string> MakeRequestHeader(AqiParam param) { Dictionary <string, string> header = base.MakeRequestHeader(param); if (!header.ContainsKey("Content-Type")) { header.Add("Content-Type", @"text/xml; charset=utf-8"); } if (!header.ContainsKey("SOAPAction")) { header.Add("SOAPAction", @"http://tempuri.org/IEnvCriteriaAqiService/" + this.Tag); } return(header); }
/// <summary> /// 入重试队列 /// 定时器线程 /// </summary> /// <param name="isu"></param> /// <param name="ap"></param> /// <param name="ex"></param> public void PutNew(ISrcUrl isu, AqiParam ap, Exception ex) { //封装为重试节点 AqiRetryNode arn = new AqiRetryNode(isu, ap); //更新计数 if (!updateNode(arn, ex)) { return; } //添加历史记录 arn = AddHistory(arn); //入队列 Push(arn.NAME); }
/// <summary> /// 检查过期 /// 可以重写 /// </summary> /// <remarks> /// 判断是否需要重新设置Cookie /// </remarks> /// <returns>true过期;false未过期</returns> public override bool IsParamsExpired() { if (ParamCache == null || ParamCache.Count == 0) { return(true); } IParseSrcUrlParam ipp = this as IParseSrcUrlParam; AqiParam dsidAqiParam = AqiParam.CreateListFormSrcUrl(ipp, ipp.ParamSrcUrl)[0]; HttpData data = ((ICacheData)ipp.ParamSrcUrl).Data; if (data.Header.ContainsKey("Set-Cookie")) { return(true); } return(false); }
/// <summary> /// 拼接请求体 /// .可以重写 /// </summary> /// <remarks> /// DSId必须要与Cookie对应,需要通过 <see cref="_5"/> 动态获取,DSId将以字节方式替换旧的DSId;替换cityCode /// </remarks> /// <param name="param">参数列表</param> /// <returns></returns> public override byte[] MakeRequestBody(AqiParam param) { byte[] requestbody = base.MakeRequestBody(param); byte[] newDSId = Encoding.ASCII.GetBytes(param["DSId"]); byte[] oldDSId = Encoding.ASCII.GetBytes(param["replaceDSId"]); while (requestbody.FindBytes(oldDSId) >= 0) { requestbody = requestbody.ReplaceBytes(oldDSId, newDSId); } if (param.ContainsKey("stationCode")) { byte[] newCode1 = Encoding.ASCII.GetBytes(param["cityCode"]); byte[] newCode2 = Encoding.ASCII.GetBytes(param["stationCode"]); byte[] newCode = new byte[newCode1.Length + newCode2.Length + 2]; Array.Copy(newCode1, newCode, newCode1.Length); Array.Copy(newCode2, 0, newCode, newCode1.Length + 2, newCode2.Length); newCode[newCode1.Length] = 0x06; newCode[newCode1.Length + 1] = 0x05; byte[] oldCode1 = Encoding.ASCII.GetBytes(param["replaceCityCode"]); byte[] oldCode2 = Encoding.ASCII.GetBytes("01"); byte[] oldCode = new byte[oldCode1.Length + oldCode2.Length + 2]; Array.Copy(oldCode1, oldCode, oldCode1.Length); Array.Copy(newCode2, 0, oldCode, oldCode1.Length + 2, oldCode2.Length); newCode[oldCode1.Length] = 0x06; newCode[oldCode1.Length + 1] = 0x05; requestbody = requestbody.ReplaceBytes(oldCode, newCode); } else if (param.ContainsKey("cityCode")) { byte[] newCode = Encoding.ASCII.GetBytes(param["cityCode"]); byte[] oldCode = Encoding.ASCII.GetBytes(param["replaceCityCode"]); requestbody = requestbody.ReplaceBytes(oldCode, newCode); } else if (param.ContainsKey("pollutantCode")) { byte[] arrOutput = { 0x09, 0x03, 0x01, 0x06, 0x03, 0x38, 0x01 }; byte[] arrOutput2 = { 0x09, 0x03, 0x01, 0x06, 0x03, 0x38, 0x01 }; arrOutput2[5] = (byte)param["pollutantCode"].ElementAt(0); requestbody = requestbody.ReplaceBytes(arrOutput, arrOutput2); } return(requestbody); }
/// <summary> /// 拼接含参数Url /// 重写 /// </summary> /// <param name="param">参数列表</param> /// <returns>完整URL</returns> public override string MakeUrl(AqiParam param) { return string.Format(Url, param["pollutant"], param["timestart"], param["timeend"], param["citycode"], param["stationcode"], token); }
/// <summary> /// 获取处理 /// </summary> /// <param name="isu">数据接口</param> /// <param name="ap">参数,无null</param> public void getProcess(ISrcUrl isu, AqiParam ap) { byte[] data = null; try { if (ap != null) { data = isu.getDate(ap); } else { data = isu.getDate(); } } catch (Exception ex) { ThrowEvent(RunMessage.RunType.ERR, isu.NAME + ":数据获取失败,进入重试队列"); //入重试队列 ar.PutNew(isu, ap, ex); return; } saveProcess(isu, ap, data); }
public List<AqiParam> ParseParam(byte[] data) { List<AqiParam> list = AqiParam.CreateListFormJson(this, this.Tag, AqiParam.PARAMS); List<AqiParam> listParam = new List<AqiParam>(); string json = Encoding.UTF8.GetString(data); JObject jo = JObject.Parse(json); JToken jdata = jo.GetValue("data"); if (!(jdata is JArray)) { throw new DataDifferentException("与预期的数据不一致(JSON对象应该包含data属性),可能数据源已经发生变化"); } JEnumerable<JToken> je = jdata.Children(); foreach (JToken j in je) { if (!(j is JObject)) { throw new DataDifferentException("与预期的数据不一致(JSON数组子元素应该是对象),可能数据源已经发生变化"); } JObject joOne = j as JObject; JToken jcity = joOne.GetValue("xzqdm"); if (jcity == null) { throw new DataDifferentException("与预期的数据不一致(JSON数组对象应该包含xzqdm属性),可能数据源已经发生变化"); } JToken jstationname = joOne.GetValue("zdmc"); if (jstationname == null) { throw new DataDifferentException("与预期的数据不一致(JSON数组对象应该包含zdmc属性),可能数据源已经发生变化"); } JToken jstation = joOne.GetValue("zddm"); if (jstation == null) { throw new DataDifferentException("与预期的数据不一致(JSON数组对象应该包含zddm属性),可能数据源已经发生变化"); } foreach(AqiParam p in list) { AqiParam ap = new AqiParam(p["pollutant"] + @"\" + jstation.ToString() + "-" + jstationname.ToString()); ap.Add("pollutant", p["pollutant"]); ap.Add("timestart", DateTime.Now.AddDays(-1).ToString("yyyyMMddHH0000")); ap.Add("timeend", DateTime.Now.ToString("yyyyMMddHH0000")); ap.Add("citycode", jcity.ToString()); ap.Add("stationcode", jstation.ToString()); listParam.Add(ap); } } return listParam; }
/// <summary> /// 加载boundsParams用于计算 /// </summary> /// <returns></returns> private List<AqiParam> loadBoundsParams() { List<AqiParam> apList = new List<AqiParam>(); //扩展参数 List<AqiParam> apListTemp = AqiParam.CreateListFormJson(this, this.Tag, "boundsParams"); //根据 扩展参数 生成最终参数 foreach (AqiParam apTemp in apListTemp) { int yearS = DateTime.Now.Year; int yearE = DateTime.Now.Year; if (apTemp.ContainsKey("yearStart")) { yearS = Int32.Parse(apTemp["yearStart"]); } if (apTemp.ContainsKey("yearEnd")) { yearE = Int32.Parse(apTemp["yearEnd"]); } if (yearS > yearE) { //错误 throw new ParamException("开始年份大于结束年份"); } for (int year = yearS; year <= yearE; year++) { AqiParam ap = new AqiParam(apTemp.Name + "\\" + year.ToString()); ap.Group = apTemp.Group; ap.Add("", year.ToString()); apList.Add(ap); } } return apList; }
/// <summary> /// 拼接请求体 /// .可以重写 /// </summary> /// <remarks> /// DSId必须要与Cookie对应,需要通过 <see cref="_5"/> 动态获取,DSId将以字节方式替换旧的DSId;替换cityCode /// </remarks> /// <param name="param">参数列表</param> /// <returns></returns> public override byte[] MakeRequestBody(AqiParam param) { byte[] requestbody = base.MakeRequestBody(param); byte[] newDSId = Encoding.ASCII.GetBytes(param["DSId"]); byte[] oldDSId = Encoding.ASCII.GetBytes(param["replaceDSId"]); while (requestbody.FindBytes(oldDSId) >= 0) { requestbody = requestbody.ReplaceBytes(oldDSId, newDSId); } if (param.ContainsKey("stationCode")) { byte[] newCode1 = Encoding.ASCII.GetBytes(param["cityCode"]); byte[] newCode2 = Encoding.ASCII.GetBytes(param["stationCode"]); byte[] newCode = new byte[newCode1.Length + newCode2.Length + 2]; Array.Copy(newCode1,newCode,newCode1.Length); Array.Copy(newCode2, 0, newCode, newCode1.Length+2, newCode2.Length); newCode[newCode1.Length] = 0x06; newCode[newCode1.Length+1] = 0x05; byte[] oldCode1 = Encoding.ASCII.GetBytes(param["replaceCityCode"]); byte[] oldCode2 = Encoding.ASCII.GetBytes("01"); byte[] oldCode = new byte[oldCode1.Length + oldCode2.Length + 2]; Array.Copy(oldCode1, oldCode, oldCode1.Length); Array.Copy(oldCode2, 0, oldCode, oldCode1.Length + 2, oldCode2.Length); oldCode[oldCode1.Length] = 0x06; oldCode[oldCode1.Length + 1] = 0x05; requestbody = requestbody.ReplaceBytes(oldCode, newCode); } else if (param.ContainsKey("cityCode")) { byte[] newCode = Encoding.ASCII.GetBytes(param["cityCode"]); byte[] oldCode = Encoding.ASCII.GetBytes(param["replaceCityCode"]); requestbody = requestbody.ReplaceBytes(oldCode, newCode); } return requestbody; }
/// <summary> /// 拼接请求体 /// .可以重写 /// </summary> /// <remarks> /// DSId必须要与Cookie对应,需要通过 <see cref="_5"/> 动态获取,DSId将以字节方式替换旧的DSId /// </remarks> /// <param name="param">参数列表</param> /// <returns></returns> public override byte[] MakeRequestBody(AqiParam param) { byte[] requestbody = base.MakeRequestBody(param); byte[] newDSId = Encoding.ASCII.GetBytes(param["DSId"]); byte[] oldDSId = Encoding.ASCII.GetBytes(param["replaceDSId"]); while (requestbody.FindBytes(oldDSId) >= 0) { requestbody = requestbody.ReplaceBytes(oldDSId, newDSId); } return requestbody; }
/// <summary> /// 获取内容 /// 可以重写 /// </summary> /// <param name="param">参数列表</param> /// <returns></returns> public virtual byte[] GetData(AqiParam param) { //得到responsebody byte[] responsebody = null; switch (ParamSendType) { case AqiConstant.ParamSendType.GET: string urlparam = MakeUrl(param); responsebody = HttpUtilV2.doGetRequest(urlparam); break; case AqiConstant.ParamSendType.POST: byte[] requestbody = MakeRequestBody(param); responsebody = HttpUtilV2.doPostRequest(Url, requestbody); break; default: responsebody = HttpUtilV2.doGetRequest(Url); break; } return ExtractData(responsebody); }
public List<AqiParam> ParseParam(byte[] data) { List<AqiParam> listParam = new List<AqiParam>(); string json = Encoding.UTF8.GetString(data); JObject jo = JObject.Parse(json); JToken jt = jo.SelectToken("items"); if (!(jt is JArray)) { throw new DataDifferentException("与预期的数据不一致(items属性应该是JSON数组),可能数据源已经发生变化"); } JArray ja = jt as JArray; JEnumerable<JToken> je = ja.Children(); foreach (JToken j in je) { if (!(j is JObject)) { throw new DataDifferentException("与预期的数据不一致(JSON数组子元素应该是对象),可能数据源已经发生变化"); } JObject joOne = j as JObject; JToken jttype = joOne.GetValue("type"); if (jttype.ToString().Equals("1")) { //跳过无效参数 continue; } JToken jtcode = joOne.GetValue("subid"); if (jtcode == null) { throw new DataDifferentException("与预期的数据不一致(JSON数组对象应该包含subid属性),可能数据源已经发生变化"); } string name = jtcode.ToString() + "号站点"; //名称 JToken jtname = joOne.GetValue("pname"); if (jtname != null) { name = name + "-" + jtname; } JToken jtcity = joOne.GetValue("cityName"); if (jtcity != null) { name = name + "-" + jtcity; } AqiParam ap = new AqiParam(name); ap.Add("Method", "GetNjdValue"); ap.Add("StationID", jtcode.ToString()); listParam.Add(ap); } return listParam; }
/// <summary> /// 拼接请求头 /// .可以重写 /// </summary> /// <remarks> /// 读取参数 Header ,将 Header 作为 Http Header /// TODO 考虑其他格式 /// </remarks> /// <param name="param"></param> /// <returns></returns> public virtual Dictionary<string, string> MakeRequestHeader(AqiParam param) { //允许null return param.Header; }
/// <summary> /// 合并 /// 仅定时线程调用 /// </summary> /// <param name="node"></param> public void Concat(RetryNode node) { thisLock.EnterWriteLock(); try { this.isu = node.isu; this.ap = node.ap; this.starttime = DateTime.Now; this.endtime = DateTime.Now; this.count = 1; //将NAME_DATA计数保留,其他清空 int i = 0; if (this.counts.ContainsKey(NAME_DATA)) { i = this.counts[NAME_DATA]; } this.counts.Clear(); this.counts.Add(NAME_DATA, i); this.counts = this.counts.Concat(node.counts).ToDictionary(x => x.Key, y => y.Value); AddNameCount(NAME_DATA); } finally { thisLock.ExitWriteLock(); } }
public List<AqiParam> ParseParam(byte[] data) { List<AqiParam> listParam = new List<AqiParam>(); string json = Encoding.UTF8.GetString(data); JObject joOne = JObject.Parse(json); JToken jtimg = joOne.GetValue("VisibleImage"); if (jtimg == null) { throw new DataDifferentException("与预期的数据不一致(JSON数组对象应该包含VisibleImage属性),可能数据源已经发生变化"); } if(String.IsNullOrEmpty(jtimg.ToString())) { return listParam; } string name = jtimg.ToString(); AqiParam ap = new AqiParam(name); ap.Unique = true; ap.Add("", jtimg.ToString()); listParam.Add(ap); return listParam; }
public bool Save(ISrcUrl isu, AqiParam param, byte[] data) { throw new NotImplementedException(); }
/// <summary> /// 入重试队列 /// 定时器线程 /// </summary> /// <param name="isu"></param> /// <param name="ap"></param> /// <param name="ex"></param> public void PutNew(ISrcUrl isu, AqiParam ap, Exception ex) { //封装为重试节点 AqiRetryNode arn = new AqiRetryNode(isu, ap); //更新计数 if (!updateNode(arn, ex)) { return; } //添加历史记录 arn = AddHistory(arn); //入队列 Push(arn.NAME); }
/// <summary> /// 保存处理 /// </summary> /// <param name="data">数据</param> /// <param name="isu">数据接口</param> /// <param name="ap">参数</param> public void SaveProcess(byte[] data, ISrcUrl isu, AqiParam ap) { try { //统计信息 NoteNode node = this.an.AddNew(isu, ap, data); //保存 if (ap == null) { node.Saved = this.ias.Save(isu, data); } else { node.Saved = this.ias.Save(isu, ap, data); } if (node.Saved) { AqiManage.Remind.Log_Info("数据保存成功", new string[] { this.name, isu.Name }); } else { node.Data = data; AqiManage.Remind.Log_Error("数据保存失败", new string[] { this.name, isu.Name }); } } catch (Exception exception) { AqiManage.Remind.Log_Error("数据保存失败", exception, new string[] { this.name, isu.Name }); } }
/// <summary> /// 拼接请求头 /// .可以重写 /// </summary> /// <remarks> /// 读取参数,若不存在‘Content-Type’则添加默认‘application/soap+msbin1’ /// </remarks> /// <param name="param"></param> /// <returns></returns> public override Dictionary<string, string> MakeRequestHeader(AqiParam param) { Dictionary<string, string> header = base.MakeRequestHeader(param); if (!header.ContainsKey("Content-Type")) { header.Add("Content-Type", @"application/soap+msbin1"); } return header; }
/// <summary> /// 入重试队列 /// 定时器线程 /// </summary> /// <param name="arName"></param> /// <param name="isu"></param> /// <param name="ap"></param> /// <param name="ex"></param> public void PutNew(string arName, ISrcUrl isu, AqiParam ap, Exception ex) { //封装为重试节点 RetryNode arn = new RetryNode(arName, isu, ap); arn.NodeEvent += new RetryNode.NodeEventHandler(this.node_RunEvent); //更新计数 arn.Update(ex); //添加历史记录 arn = this.AddHistory(arn); //入队列 this.Push(arn.Name); AqiManage.Remind.Log_Info("已添加到重试队列", new string[] { this.name, arn.RunnerName, arn.Name }); }
public List<AqiParam> ParseParam(byte[] data) { List<AqiParam> listParam = new List<AqiParam>(); string json = Encoding.UTF8.GetString(data); JArray ja = JArray.Parse(json); JEnumerable<JToken> je = ja.Children(); foreach (JToken j in je) { if (!(j is JObject)) { throw new DataDifferentException("与预期的数据不一致(JSON数组子元素应该是对象),可能数据源已经发生变化"); } JObject joOne = j as JObject; JToken jttf = joOne.GetValue("tfid"); if (jttf == null) { throw new DataDifferentException("与预期的数据不一致(JSON数组对象应该包含tfid属性),可能数据源已经发生变化"); } AqiParam ap = new AqiParam(jttf.ToString() + "号台风"); ap.Add("", jttf.ToString()); listParam.Add(ap); } return listParam; }
/// <summary> /// 获取内容 /// 可以重写 /// </summary> /// <param name="param">参数列表</param> /// <returns></returns> public virtual byte[] GetData(AqiParam param) { HttpWebResponse response = null; try { Dictionary<string, string> header = MakeRequestHeader(param); switch (ParamSendType) { case AqiConstant.ParamSendType.GET: string urlparam = MakeUrl(param); response = HttpUtilV2.createGetResponse(urlparam, -1, header); break; case AqiConstant.ParamSendType.POST: byte[] requestbody = MakeRequestBody(param); response = HttpUtilV2.createPostResponse(Url, -1, header, requestbody); break; default: throw new NotSupportedException("ParamSendType only for GET/POST"); } } catch (System.Exception ex) { throw new DataUnHoldException("无法获取数据", ex); } HttpData data = HttpUtilV2.GetHttpData(response); //缓存数据 if (this is ICacheData) { ((ICacheData)this).Data = data; } //提取数据 if (this is IExtractData) { return ((IExtractData)this).ExtractData(data.Body); } return data.Body; }
/// <summary> /// 保存处理 /// </summary> /// <param name="isu">数据接口</param> /// <param name="ap">参数</param> /// <param name="data">数据</param> public void saveProcess(ISrcUrl isu, AqiParam ap, byte[] data) { //统计信息 AqiNoteNode n = an.AddNew(isu, ap, data); //保存 if(ap == null) { n.Saved = ias.Save(isu, data); } else { n.Saved = ias.Save(isu, ap.Name, data); } if (n.Saved) { ThrowEvent(RunMessage.RunType.OK, isu.IAW.NAME + ":" + isu.NAME + ":数据获取成功"); } else { ThrowEvent(RunMessage.RunType.ERR, isu.NAME + ":数据获取失败"); } }
public List<AqiParam> ParseParam(byte[] data) { List<AqiParam> listParam = new List<AqiParam>(); XNamespace b = "http://schemas.datacontract.org/2004/07/Env.Publish.Province.DAL"; XDocument xd = XDocument.Load(new MemoryStream(data)); IEnumerable<XElement> elements = xd.Descendants(XName.Get("AQICityStation", b.ToString())); foreach (XElement node in elements) { string name = node.Element(XName.Get("PositionName", b.ToString())).Value; string id = node.Element(XName.Get("StationID", b.ToString())).Value; string cityName = node.Element(XName.Get("CityName", b.ToString())).Value; AqiParam ap = new AqiParam(id + "_" + name); ap.Add("stationCode", id); listParam.Add(ap); } return listParam; }
/// <summary> /// 拼接请求头 /// .可以重写 /// </summary> /// <remarks> /// 读取参数 Header ,将 Header 作为 Http Header /// TODO 考虑其他格式 /// </remarks> /// <param name="param"></param> /// <returns>RequestHeader键值集合</returns> public virtual Dictionary<string, string> MakeRequestHeader(AqiParam param) { //TODO 允许null if (param.Header == null) { return new Dictionary<string, string>(); } return param.Header; }
/// <summary> /// 拼接请求体 /// .可以重写 /// </summary> /// <remarks> /// 读取参数 所有 ,转为 html form 表单字符串,转为字节数组 /// </remarks> /// <param name="param">参数列表</param> /// <returns>RequestBody字节数组</returns> public virtual byte[] MakeRequestBody(AqiParam param) { if (param.Body == null) { switch (ParamBodyType) { case AqiConstant.ParamBodyType.NONE: param.Body = new byte[0]; break; case AqiConstant.ParamBodyType.HTTP_FORM: param.Body = MakeFromBody(param); break; case AqiConstant.ParamBodyType.TEXT: param.Body = MakeTextBody(param); break; default: throw new NotSupportedException("不被支持的参数类型,ParamBodyType=" + (int)ParamBodyType); } } return param.Body; }
/// <summary> /// 拼接请求体 /// .可以重写 /// </summary> /// <remarks> /// DSId必须要与Cookie对应,需要通过 <see cref="_5"/> 动态获取,DSId将以字节方式替换旧的DSId;替换cityCode /// </remarks> /// <param name="param">参数列表</param> /// <returns></returns> public override byte[] MakeRequestBody(AqiParam param) { byte[] requestbody = base.MakeRequestBody(param); byte[] newDSId = Encoding.ASCII.GetBytes(param["DSId"]); byte[] oldDSId = Encoding.ASCII.GetBytes(param["replaceDSId"]); while (requestbody.FindBytes(oldDSId) >= 0) { requestbody = requestbody.ReplaceBytes(oldDSId, newDSId); } if (param.ContainsKey("stationCode")) { byte[] newCode1 = Encoding.ASCII.GetBytes(param["cityCode"]); byte[] newCode2 = Encoding.ASCII.GetBytes(param["stationCode"]); byte[] newCode = new byte[newCode1.Length + newCode2.Length + 2]; Array.Copy(newCode1,newCode,newCode1.Length); Array.Copy(newCode2, 0, newCode, newCode1.Length+2, newCode2.Length); newCode[newCode1.Length] = 0x06; newCode[newCode1.Length+1] = 0x05; byte[] oldCode1 = Encoding.ASCII.GetBytes(param["replaceCityCode"]); byte[] oldCode2 = Encoding.ASCII.GetBytes("01"); byte[] oldCode = new byte[oldCode1.Length + oldCode2.Length + 2]; Array.Copy(oldCode1, oldCode, oldCode1.Length); Array.Copy(newCode2, 0, oldCode, oldCode1.Length + 2, oldCode2.Length); newCode[oldCode1.Length] = 0x06; newCode[oldCode1.Length + 1] = 0x05; requestbody = requestbody.ReplaceBytes(oldCode, newCode); } else if (param.ContainsKey("cityCode")) { byte[] newCode = Encoding.ASCII.GetBytes(param["cityCode"]); byte[] oldCode = Encoding.ASCII.GetBytes(param["replaceCityCode"]); requestbody = requestbody.ReplaceBytes(oldCode, newCode); } else if (param.ContainsKey("pollutantCode")) { byte[] arrOutput = { 0x09, 0x03, 0x01, 0x06, 0x03, 0x38, 0x01 }; byte[] arrOutput2 = { 0x09, 0x03, 0x01, 0x06, 0x03, 0x38, 0x01 }; arrOutput2[5] = (byte)param["pollutantCode"].ElementAt(0); requestbody = requestbody.ReplaceBytes(arrOutput, arrOutput2); } return requestbody; }
/// <summary> /// 拼接含参数Url /// .可以重写 /// </summary> /// <remarks> /// 读取参数 所有 ,根据实例属性ParamUrlType处理 /// 1、 ParamUrlType 为 PATH ,将 所有 交由 MakePathUrl 方法处理,与 Url 属性合并为字符串; /// 2、 ParamUrlType 为 KEY_VALUE ,将 所有 交由 MakeKeyValueUrl 方法处理,与 Url 属性合并为字符串; /// </remarks> /// <param name="param">参数列表</param> /// <returns>完整URL</returns> public virtual string MakeUrl(AqiParam param) { StringBuilder sb = new StringBuilder(); sb.Append(Url); if (ParamUrlType == AqiConstant.ParamUrlType.PATH) { sb.Append(MakePathUrl(param.Values.ToList<string>())); } else if (ParamUrlType == AqiConstant.ParamUrlType.KEY_VALUE) { sb.Append(MakeKeyValueUrl(param)); } else { throw new ParamException("使用了未知的ParamUrlType"); } return sb.ToString(); }
public List<AqiParam> ParseParam(byte[] data) { List<AqiParam> listParam = new List<AqiParam>(); string json = Encoding.UTF8.GetString(data); JObject ja = JObject.Parse(json); JToken jt = ja.SelectToken("Bodies.[0].Content.headers.DSId"); if (jt == null) { throw new DataDifferentException("与预期的数据不一致(Bodies.[0].Content.headers.DSId应有数据),可能数据源已经发生变化"); } AqiParam ap = new AqiParam("DSId"); ap.Add("DSId", jt.ToString()); listParam.Add(ap); return listParam; }