private static bool parseContentsAndSave(string url, ContentSource contentSource) { bool contentFetched = false; WebRequest req = WebRequest.Create(url); if (req.Proxy != null) req.Proxy.Credentials = CredentialCache.DefaultCredentials; HttpWebResponse webResponse = (HttpWebResponse)req.GetResponse(); string response = null; if (!String.IsNullOrEmpty(contentSource.Encoding)) { string orjResponse = new System.IO.StreamReader(webResponse.GetResponseStream(), Encoding.GetEncoding(contentSource.Encoding)).ReadToEnd(); response = orjResponse.ConvertEncoding(contentSource.Encoding, "iso-8859-9"); } else response = new System.IO.StreamReader(webResponse.GetResponseStream(), Encoding.GetEncoding("iso-8859-9")).ReadToEnd(); response = response.Replace(Environment.NewLine, "\n"); Regex regexObj = new Regex(contentSource.ListRegExp, RegexOptions.Singleline); Match match = regexObj.Match(response); while (match.Success) { Content content = new Content(); if (match.Groups["author"].Value != "") { string authorName = CMSUtility.ClearTextFromWeb(match.Groups["author"].Value); Author author = (Author)Provider.Database.Read(typeof(Author), "Name={0}", authorName); //object authorId = Provider.Database.GetValue("select Id from Author where Name={0}", authorName); if (author == null) { author = new Author(); author.Description = url + " adresinden otomatik olarak kaydedildi."; author.Name = authorName; if (match.Groups["authorpic"].Value != "") { string authorImgUrl = match.Groups["authorpic"].Value.Trim().ConvertToAbsoluteURL(url); WebClient wc = new WebClient(); wc.Proxy.Credentials = CredentialCache.DefaultCredentials; string imgFileName = Provider.AppSettings["authorDir"] + "/" + authorName.MakeFileName() + authorImgUrl.Substring(authorImgUrl.LastIndexOf('.')); wc.DownloadFile(authorImgUrl, MapPath(imgFileName)); author.Picture = imgFileName; } author.Save(); content.AuthorId = author.Id; } else { content.AuthorId = author.Id; if (author.DisableAutoContent) { match = match.NextMatch(); continue; //*** } } } try { content.PublishDate = DateTime.Parse(match.Groups["date"].Value.Trim()); } catch { } content.Title = CMSUtility.ClearTextFromWeb(match.Groups["title"].Value); content.Description = "<p>" + CMSUtility.ClearTextFromWeb(match.Groups["desc"].Value) + "</p>"; content.Metin = ""; // burada metin kaydedilemez (liste çünkü) (metin ve picture daha sonra kaydediliyor - detaya bakıldığında) content.CategoryId = contentSource.CategoryId; content.ClassName = contentSource.ClassName; content.SourceId = contentSource.SourceId; string sourceLink = match.Groups["link"].Value.Trim(); content.SourceLink = (new Uri(new Uri(url), sourceLink)).ToString(); content.ContentSourceId = contentSource.Id; if (content.AuthorId == 0) content.AuthorId = contentSource.AuthorId; try { content.Save(); } catch { } contentFetched = true; match = match.NextMatch(); } return contentFetched; }
public static void FetchAutoContent(ContentSource contentSource) { // bugün fetch edildiyse bir daha fetch etmeye gerek yok if (contentSource.LastFetched.Date == DateTime.Now.Date) return; // en son bir saatten daha önce fetch edildiyse bir daha fetch etmeyi dene if (DateTime.Now - contentSource.LastFetchTrial > TimeSpan.FromMinutes((double)contentSource.FetchFrequency)) { string url = parseLinkPattern(contentSource.ListPageAddress); bool contentFetched = Provider.parseContentsAndSave(url, contentSource); if (contentFetched) contentSource.LastFetched = DateTime.Now; contentSource.LastFetchTrial = DateTime.Now; contentSource.Save(); } }