static void Main(string[] blogs) { foreach (var blog in blogs) { var feeds = RssReader.GetRssFeed(blog); var writeCount = RssReader.Save(feeds).GetAwaiter().GetResult(); Console.WriteLine(blog); Console.WriteLine("Read count {0}", feeds.Count()); Console.WriteLine("Saved count {0}", writeCount); } Console.ReadKey(); }
private static RssFeed read(string url, HttpWebRequest request, RssFeed oldFeed) { // ***** Marked for substantial improvement RssFeed feed = new RssFeed(); RssElement element = null; Stream stream = null; Uri uri = new Uri(url); feed.url = url; switch (uri.Scheme) { case "file": feed.lastModified = File.GetLastWriteTime(url); if ((oldFeed != null) && (feed.LastModified == oldFeed.LastModified)) { oldFeed.cached = true; return(oldFeed); } stream = new FileStream(url, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); break; case "https": goto case "http"; case "http": if (request == null) { request = (HttpWebRequest)WebRequest.Create(uri); } if (oldFeed != null) { request.IfModifiedSince = oldFeed.LastModified; request.Headers.Add("If-None-Match", oldFeed.ETag); } try { HttpWebResponse response = (HttpWebResponse)request.GetResponse(); feed.lastModified = response.LastModified; feed.etag = response.Headers["ETag"]; try { if (response.ContentEncoding != "") { feed.encoding = Encoding.GetEncoding(response.ContentEncoding); } } catch {} stream = response.GetResponseStream(); } catch (WebException we) { if (oldFeed != null) { oldFeed.cached = true; return(oldFeed); } else { throw we; // bad } } break; } if (stream != null) { RssReader reader = null; try { reader = new RssReader(stream); do { element = reader.Read(); if (element is RssChannel) { feed.Channels.Add((RssChannel)element); } }while (element != null); feed.rssVersion = reader.Version; } finally { feed.exceptions = reader.Exceptions; reader.Close(); } } else { throw new ApplicationException("Not a valid Url"); } return(feed); }
private static RssFeed read(Stream stream) { RssFeed feed = new RssFeed(); RssElement element = null; if (stream != null) { RssReader reader = null; try { reader = new RssReader(stream); do { element = reader.Read(); if (element is RssChannel) feed.Channels.Add((RssChannel)element); } while (element != null); feed.rssVersion = reader.Version; } finally { feed.exceptions = reader.Exceptions; reader.Close(); } } else throw new ApplicationException("Not a valid Url"); return feed; }
private static RssFeed read(string url, HttpWebRequest request, RssFeed oldFeed) { // ***** Marked for substantial improvement RssFeed feed = new RssFeed(); RssElement element = null; Stream stream = null; Uri uri = new Uri(url); feed.url = url; switch (uri.Scheme) { case "file": feed.lastModified = File.GetLastWriteTime(url); if ((oldFeed != null) && (feed.LastModified == oldFeed.LastModified)) { oldFeed.cached = true; return oldFeed; } stream = new FileStream(url, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); break; case "https": goto case "http"; case "http": if (request == null) request = (HttpWebRequest)WebRequest.Create(uri); if (oldFeed != null) { request.IfModifiedSince = oldFeed.LastModified; request.Headers.Add("If-None-Match", oldFeed.ETag); } try { HttpWebResponse response = (HttpWebResponse)request.GetResponse(); feed.lastModified = response.LastModified; feed.etag = response.Headers["ETag"]; try { if (response.ContentEncoding != "") feed.encoding = Encoding.GetEncoding(response.ContentEncoding); } catch {} stream = response.GetResponseStream(); } catch (WebException we) { if (oldFeed != null) { oldFeed.cached = true; return oldFeed; } else throw we; // bad } break; } if (stream != null) { RssReader reader = null; try { reader = new RssReader(stream); do { element = reader.Read(); if (element is RssChannel) feed.Channels.Add((RssChannel)element); } while (element != null); feed.rssVersion = reader.Version; } finally { feed.exceptions = reader.Exceptions; reader.Close(); } } else throw new ApplicationException("Not a valid Url"); return feed; }
// Token: 0x06000634 RID: 1588 RVA: 0x0002301C File Offset: 0x0002121C private static RssFeed read(string url, HttpWebRequest request, RssFeed oldFeed) { RssFeed rssFeed = new RssFeed(); Stream stream = null; Uri uri = new Uri(url); rssFeed.url = url; string scheme; if ((scheme = uri.Scheme) != null) { if (!(scheme == "file")) { if (scheme == "https" || scheme == "http") { if (request == null) { request = (HttpWebRequest)WebRequest.Create(uri); } request.Credentials = Util.GetNetworkCredential(); string value = "Basic " + Convert.ToBase64String(Encoding.Default.GetBytes(Util.GetHtaccessUser() + ":" + Util.GetHtaccessPwd())); request.AuthenticationLevel = AuthenticationLevel.MutualAuthRequested; request.Headers.Add("Authorization", value); request.UserAgent = Util.GetUserAgent(); request.PreAuthenticate = true; if (oldFeed != null) { request.IfModifiedSince = oldFeed.LastModified; request.Headers.Add("If-None-Match", oldFeed.ETag); } try { HttpWebResponse httpWebResponse = (HttpWebResponse)request.GetResponse(); rssFeed.lastModified = httpWebResponse.LastModified; rssFeed.etag = httpWebResponse.Headers["ETag"]; try { if (httpWebResponse.ContentEncoding != "") { rssFeed.encoding = Encoding.GetEncoding(httpWebResponse.ContentEncoding); } } catch { } stream = httpWebResponse.GetResponseStream(); } catch (WebException ex) { if (oldFeed != null) { oldFeed.cached = true; return(oldFeed); } throw ex; } } } else { rssFeed.lastModified = File.GetLastWriteTime(url); if (oldFeed != null && rssFeed.LastModified == oldFeed.LastModified) { oldFeed.cached = true; return(oldFeed); } stream = new FileStream(url, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); } } if (stream != null) { RssReader rssReader = null; try { rssReader = new RssReader(stream); RssElement rssElement; do { rssElement = rssReader.Read(); if (rssElement is RssChannel) { rssFeed.Channels.Add((RssChannel)rssElement); } }while (rssElement != null); rssFeed.rssVersion = rssReader.Version; return(rssFeed); } finally { rssFeed.exceptions = rssReader.Exceptions; rssReader.Close(); } } throw new ApplicationException("Not a valid Url"); }
private static RssFeed read(string url, HttpWebRequest request, RssFeed oldFeed, string username, string password) { // ***** Marked for substantial improvement RssFeed feed = new RssFeed(); RssElement element = null; Stream stream = null; Uri uri = new Uri(url); feed.url = url; string ErrorMessage = null; switch (uri.Scheme) { case "file": feed.lastModified = File.GetLastWriteTime(url); if ((oldFeed != null) && (feed.LastModified == oldFeed.LastModified)) { oldFeed.cached = true; return(oldFeed); } stream = new FileStream(url, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); break; case "https": goto case "http"; case "http": if (request == null) { request = (HttpWebRequest)WebRequest.Create(uri); } if (oldFeed != null) { request.IfModifiedSince = oldFeed.LastModified; request.Headers.Add("If-None-Match", oldFeed.ETag); request.Headers.Add("Accept-Encoding", "gzip, deflate"); } try { if (username != null && password != null) { request.Credentials = new System.Net.NetworkCredential(username, password); } HttpWebResponse response = (HttpWebResponse)request.GetResponse(); feed.lastModified = response.LastModified; feed.etag = response.Headers["ETag"]; try { if (response.ContentEncoding != "") { feed.encoding = Encoding.GetEncoding(response.ContentEncoding); } } catch {} //stream = response.GetResponseStream(); stream = GetResponseStream(response); } catch (WebException we) { if (oldFeed != null) { oldFeed.cached = true; return(oldFeed); } else { ErrorMessage = we.Message; } //else throw we; // bad } break; } if (ErrorMessage == null) { if (stream != null) { RssReader reader = null; try { reader = new RssReader(stream); //do //{ element = reader.ReadXPath(); //element = reader.Read(); if (element is RssChannel) { feed.Channels.Add((RssChannel)element); } //} //while (element != null); feed.rssVersion = reader.Version; } finally { feed.exceptions = reader.Exceptions; reader.Close(); } } else if (ErrorMessage != null) { throw new ApplicationException(ErrorMessage); } else { throw new ApplicationException("Invalid Url"); } } else { throw new ApplicationException(ErrorMessage); } return(feed); }