protected override void InternalProcessRecord() { bool copyWorkflows = !NoWorkflows.IsPresent; bool copyColumns = !NoColumns.IsPresent; bool copyDocConversions = !NoDocConversions.IsPresent; bool copyDocInfoPanel = !NoDocInfoPanel.IsPresent; bool copyPolicies = !NoPolicies.IsPresent; bool copyDocTemplate = !NoDocTemplate.IsPresent; SPWeb sourceWeb = null; SPContentType sourceCT = null; if (SourceWeb != null) { sourceWeb = SourceWeb.Read(); } try { sourceCT = SourceContentType.Read(sourceWeb); } finally { if (sourceWeb != null) { sourceWeb.Dispose(); } } using (SPWeb targetWeb = TargetWeb.Read()) { Logger.Write("Start Time: {0}", DateTime.Now.ToString()); Common.ContentTypes.CopyContentTypes ctCopier = new Common.ContentTypes.CopyContentTypes( copyWorkflows, copyColumns, copyDocConversions, copyDocInfoPanel, copyPolicies, copyDocTemplate); ctCopier.Copy(sourceCT, targetWeb); } Logger.Write("Finish Time: {0}", DateTime.Now.ToString()); }
private static void Collect(Models.Source source, IList<Models.UserSourceLink> userSourceLinks, DateTime collectTime, SourceContentType sourceContentType, Models.ICContext context, string response, List<SourceContentRegexGroup> groupRegexes) { var domain = source.Domain; var hrefhead = GetUrlHead(source.Url); var datetime = DateTime.Now.Date.AddMonths(-1); foreach (var regexGroup in groupRegexes) { if (regexGroup.linkRegex == null) { continue; } var sourceDate = collectTime; var match = regexGroup.groupRegex.Match(response, 0); while (match.Success) { var matchValue = regexGroup.decode ? System.Net.WebUtility.HtmlDecode(match.Value) : match.Value; if (!regexGroup.IsNotMatchSourceRegexes(matchValue, regexGroup.containerRegexes)) { if (regexGroup.dateRegex != null) { var dateMatch = regexGroup.dateRegex.Match(matchValue); if (!dateMatch.Success) { match = match.NextMatch(); continue; } var year = Convert.ToInt32(dateMatch.Groups["year"].Value); var month = Convert.ToInt32(dateMatch.Groups["month"].Value); var day = Convert.ToInt32(dateMatch.Groups["day"].Value); sourceDate = new DateTime(year, month, day); } var linkMatch = regexGroup.linkRegex.Match(matchValue); if (linkMatch.Success) { if (linkMatch.Groups["url"].Success && linkMatch.Groups["content"].Success)// && !regexGroup.IsNotMatchSourceRegexes(linkValue, regexGroup.hrefRegexes)) { var url = linkMatch.Groups["url"].Value; url = url.Substring(1, url.Length - 2); if (!url.StartsWith("http")) { if (url.StartsWith("./")) { url = hrefhead.TrimEnd('/') + url.TrimStart('.'); } else if (url.StartsWith("/")) { url = source.Domain + url.TrimStart('/'); } else { url = hrefhead + url; } } var content = crlfRegex.Replace(linkMatch.Groups["content"].Value, ""); if (!regexGroup.IsNotMatchSourceRegexes(content, regexGroup.contentRegexes)) { content = System.Net.WebUtility.HtmlDecode(htmlRegex.Replace(dateRegex.Replace(content, ""), "")).Trim('·', ' ', '\t'); // remove date string var sourceContent = context.SourceContents.Where(p => p.SourceId == source.SourceId && p.Url == url).FirstOrDefault(); if (sourceContent == null) { sourceContent = new Models.SourceContent { AddTime = collectTime, AddDate = collectTime.Date, AddHour = collectTime.Hour, SourceDate = sourceDate, Content = content, ContentType = (int)sourceContentType, SourceId = source.SourceId, Url = url, }; if (sourceDate >= datetime) { foreach (var userSouceLink in userSourceLinks) { sourceContent.UserSourceContentLinks.Add(new Models.UserSourceContentLink { UserId = userSouceLink.UserId }); } } context.SourceContents.Add(sourceContent); context.SaveChanges(); } } } } } match = match.NextMatch(); } } }
/// <summary> /// Get source content /// </summary> /// <param name="source">watched source</param> /// <param name="collectTime">collect time</param> public static void Collect(Models.Source source, DateTime collectTime, SourceContentType sourceContentType) { using (Models.ICContext context = new Models.ICContext()) { var userSourceLinks = context.UserSourceLinks.Where(p => p.SourceId == source.SourceId).ToList(); if (userSourceLinks.Count == 0) { return; } var sourceUrls = context.SourceUrls.Where(p => p.SourceId == source.SourceId && p.Enabled).ToList(); foreach (var sourceUrl in sourceUrls) { string response = Utility.RequestHelper.GetResponse(sourceUrl.Url); var regexGroups = context.SourceRegexGroups.Where(p => p.Domain == source.Domain).ToList(); if (regexGroups != null && regexGroups.Count > 0) { List<SourceContentRegexGroup> groupRegexes = new List<SourceContentRegexGroup>(); foreach (var regexGroup in regexGroups) { var regexes = regexGroup.SourceRegexes.ToList(); if (regexes.Count > 0) { groupRegexes.Add(new SourceContentRegexGroup(regexGroup.Name, regexGroup.Regex, regexGroup.Decode, regexes)); } } Collect(source, userSourceLinks, collectTime, sourceContentType, context, response, groupRegexes); } else { Collect(source, userSourceLinks, collectTime, sourceContentType, context, response, allGroupRegexes); } } } }