private void ReadChannelAttributes(string url, string xml, DateTime timeStart, Dictionary <string, string> channelAttr) { XmlTextReader reader = new XmlTextReader(new StringReader(xml)); mLogger.Info("ProduceData", "Reading channel attributes ..."); while (reader.Read()) { if (reader.NodeType == XmlNodeType.Element && reader.Name == "channel" && !reader.IsEmptyElement) { // handle channel while (reader.Read() && !(reader.NodeType == XmlNodeType.EndElement && reader.Name == "channel")) { if (reader.NodeType == XmlNodeType.Element) { // handle channel attributes if (mChannelElements.Contains(reader.Name)) { string attrName = reader.Name; string value = Utils.XmlReadValue(reader, attrName); string oldValue; if (attrName == "pubDate") { string tmp = Utils.NormalizeDateTimeStr(value); if (tmp != null) { value = tmp; } } if (channelAttr.TryGetValue(attrName, out oldValue)) { channelAttr[attrName] = oldValue + " ;; " + value; } else { channelAttr.Add(attrName, value); } } else { Utils.XmlSkip(reader, reader.Name); } } } } } reader.Close(); channelAttr.Add("siteId", mSiteId); channelAttr.Add("provider", GetType().ToString()); channelAttr.Add("sourceUrl", url); if (mIncludeRssXml) { channelAttr.Add("source", xml); } channelAttr.Add("timeBetweenPolls", TimeBetweenPolls.ToString()); channelAttr.Add("timeStart", timeStart.ToString(Utils.DATE_TIME_SIMPLE)); }
protected override object ProduceData() { for (int i = 0; i < mSources.Count; i++) { string url = mSources[i]; try { DateTime timeStart = DateTime.Now; // get RSS XML string xml; try { mLogger.Info("ProduceData", "Getting RSS XML from {0} ...", url); xml = WebUtils.GetWebPageDetectEncoding(url); xml = FixXml(xml); } catch (Exception e) { mLogger.Error("ProduceData", e); return(null); } Dictionary <string, string> channelAttr = new Dictionary <string, string>(); ArrayList <DocumentCorpus> corpora = new ArrayList <DocumentCorpus>(new DocumentCorpus[] { new DocumentCorpus() }); corpora.Last.Features.SetFeatureValue("guid", Guid.NewGuid().ToString()); XmlTextReader reader = new XmlTextReader(new StringReader(xml)); // first pass: items mLogger.Info("ProduceData", "Reading items ..."); while (reader.Read()) { if (reader.NodeType == XmlNodeType.Element && reader.Name == "item" && !reader.IsEmptyElement) { Dictionary <string, string> itemAttr = new Dictionary <string, string>(); while (reader.Read() && !(reader.NodeType == XmlNodeType.EndElement && reader.Name == "item")) { if (reader.NodeType == XmlNodeType.Element) { // handle item attributes if (mItemElements.Contains(reader.Name)) { string attrName = reader.Name; string value = Utils.XmlReadValue(reader, attrName); if (value.Trim() != "") { string oldValue; if (attrName == "pubDate") { string tmp = Utils.NormalizeDateTimeStr(value); if (tmp != null) { value = tmp; } } if (itemAttr.TryGetValue(attrName, out oldValue)) { itemAttr[attrName] = oldValue + " ;; " + value; } else { itemAttr.Add(attrName, value); } } } else { Utils.XmlSkip(reader, reader.Name); } } } // stopped? if (mStopped) { if (corpora[0].Documents.Count == 0) { return(null); } break; } ProcessItem(itemAttr, corpora, url); } } reader.Close(); reader = new XmlTextReader(new StringReader(xml)); if (corpora[0].Documents.Count > 0) { // second pass: channel attributes mLogger.Info("ProduceData", "Reading channel attributes ..."); while (reader.Read()) { if (reader.NodeType == XmlNodeType.Element && reader.Name == "channel" && !reader.IsEmptyElement) { // handle channel while (reader.Read() && !(reader.NodeType == XmlNodeType.EndElement && reader.Name == "channel")) { if (reader.NodeType == XmlNodeType.Element) { // handle channel attributes if (mChannelElements.Contains(reader.Name)) { string attrName = reader.Name; string value = Utils.XmlReadValue(reader, attrName); string oldValue; if (attrName == "pubDate") { string tmp = Utils.NormalizeDateTimeStr(value); if (tmp != null) { value = tmp; } } if (channelAttr.TryGetValue(attrName, out oldValue)) { channelAttr[attrName] = oldValue + " ;; " + value; } else { channelAttr.Add(attrName, value); } } else { Utils.XmlSkip(reader, reader.Name); } } } } } reader.Close(); channelAttr.Add("siteId", mSiteId); channelAttr.Add("provider", GetType().ToString()); channelAttr.Add("sourceUrl", url); if (mIncludeRssXml) { channelAttr.Add("source", xml); } channelAttr.Add("timeBetweenPolls", TimeBetweenPolls.ToString()); channelAttr.Add("timeStart", timeStart.ToString(Utils.DATE_TIME_SIMPLE)); channelAttr.Add("timeEnd", DateTime.Now.ToString(Utils.DATE_TIME_SIMPLE)); int newItems = 0; foreach (DocumentCorpus corpus in corpora) { newItems += corpus.Documents.Count; foreach (KeyValuePair <string, string> attr in channelAttr) { corpus.Features.SetFeatureValue(attr.Key, attr.Value); } } mLogger.Info("ProduceData", "{0} new items.", newItems); // dispatch data foreach (DocumentCorpus corpus in corpora) { DispatchData(corpus); } } else { mLogger.Info("ProduceData", "No new items."); } // stopped? if (mStopped) { return(null); } } catch (Exception e) { mLogger.Error("ProduceData", e); } } return(null); }