protected override void InternalProcessRecord()
{
bool copyWorkflows = !NoWorkflows.IsPresent;
bool copyColumns = !NoColumns.IsPresent;
bool copyDocConversions = !NoDocConversions.IsPresent;
bool copyDocInfoPanel = !NoDocInfoPanel.IsPresent;
bool copyPolicies = !NoPolicies.IsPresent;
bool copyDocTemplate = !NoDocTemplate.IsPresent;
SPWeb sourceWeb = null;
SPContentType sourceCT = null;
if (SourceWeb != null)
{
sourceWeb = SourceWeb.Read();
}
try
{
sourceCT = SourceContentType.Read(sourceWeb);
}
finally
{
if (sourceWeb != null)
{
sourceWeb.Dispose();
}
}
using (SPWeb targetWeb = TargetWeb.Read())
{
Logger.Write("Start Time: {0}", DateTime.Now.ToString());
Common.ContentTypes.CopyContentTypes ctCopier = new Common.ContentTypes.CopyContentTypes(
copyWorkflows, copyColumns, copyDocConversions, copyDocInfoPanel, copyPolicies, copyDocTemplate);
ctCopier.Copy(sourceCT, targetWeb);
}
Logger.Write("Finish Time: {0}", DateTime.Now.ToString());
}
private static void Collect(Models.Source source, IList<Models.UserSourceLink> userSourceLinks, DateTime collectTime, SourceContentType sourceContentType, Models.ICContext context, string response, List<SourceContentRegexGroup> groupRegexes)
{
var domain = source.Domain;
var hrefhead = GetUrlHead(source.Url);
var datetime = DateTime.Now.Date.AddMonths(-1);
foreach (var regexGroup in groupRegexes)
{
if (regexGroup.linkRegex == null)
{
continue;
}
var sourceDate = collectTime;
var match = regexGroup.groupRegex.Match(response, 0);
while (match.Success)
{
var matchValue = regexGroup.decode ? System.Net.WebUtility.HtmlDecode(match.Value) : match.Value;
if (!regexGroup.IsNotMatchSourceRegexes(matchValue, regexGroup.containerRegexes))
{
if (regexGroup.dateRegex != null)
{
var dateMatch = regexGroup.dateRegex.Match(matchValue);
if (!dateMatch.Success)
{
match = match.NextMatch();
continue;
}
var year = Convert.ToInt32(dateMatch.Groups["year"].Value);
var month = Convert.ToInt32(dateMatch.Groups["month"].Value);
var day = Convert.ToInt32(dateMatch.Groups["day"].Value);
sourceDate = new DateTime(year, month, day);
}
var linkMatch = regexGroup.linkRegex.Match(matchValue);
if (linkMatch.Success)
{
if (linkMatch.Groups["url"].Success && linkMatch.Groups["content"].Success)// && !regexGroup.IsNotMatchSourceRegexes(linkValue, regexGroup.hrefRegexes))
{
var url = linkMatch.Groups["url"].Value;
url = url.Substring(1, url.Length - 2);
if (!url.StartsWith("http"))
{
if (url.StartsWith("./"))
{
url = hrefhead.TrimEnd('/') + url.TrimStart('.');
}
else if (url.StartsWith("/"))
{
url = source.Domain + url.TrimStart('/');
}
else
{
url = hrefhead + url;
}
}
var content = crlfRegex.Replace(linkMatch.Groups["content"].Value, "");
if (!regexGroup.IsNotMatchSourceRegexes(content, regexGroup.contentRegexes))
{
content = System.Net.WebUtility.HtmlDecode(htmlRegex.Replace(dateRegex.Replace(content, ""), "")).Trim('·', ' ', '\t'); // remove date string
var sourceContent = context.SourceContents.Where(p => p.SourceId == source.SourceId && p.Url == url).FirstOrDefault();
if (sourceContent == null)
{
sourceContent = new Models.SourceContent
{
AddTime = collectTime,
AddDate = collectTime.Date,
AddHour = collectTime.Hour,
SourceDate = sourceDate,
Content = content,
ContentType = (int)sourceContentType,
SourceId = source.SourceId,
Url = url,
};
if (sourceDate >= datetime)
{
foreach (var userSouceLink in userSourceLinks)
{
sourceContent.UserSourceContentLinks.Add(new Models.UserSourceContentLink
{
UserId = userSouceLink.UserId
});
}
}
context.SourceContents.Add(sourceContent);
context.SaveChanges();
}
}
}
}
}
match = match.NextMatch();
}
}
}
/// <summary>
/// Get source content
/// </summary>
/// <param name="source">watched source</param>
/// <param name="collectTime">collect time</param>
public static void Collect(Models.Source source, DateTime collectTime, SourceContentType sourceContentType)
{
using (Models.ICContext context = new Models.ICContext())
{
var userSourceLinks = context.UserSourceLinks.Where(p => p.SourceId == source.SourceId).ToList();
if (userSourceLinks.Count == 0)
{
return;
}
var sourceUrls = context.SourceUrls.Where(p => p.SourceId == source.SourceId && p.Enabled).ToList();
foreach (var sourceUrl in sourceUrls)
{
string response = Utility.RequestHelper.GetResponse(sourceUrl.Url);
var regexGroups = context.SourceRegexGroups.Where(p => p.Domain == source.Domain).ToList();
if (regexGroups != null && regexGroups.Count > 0)
{
List<SourceContentRegexGroup> groupRegexes = new List<SourceContentRegexGroup>();
foreach (var regexGroup in regexGroups)
{
var regexes = regexGroup.SourceRegexes.ToList();
if (regexes.Count > 0)
{
groupRegexes.Add(new SourceContentRegexGroup(regexGroup.Name, regexGroup.Regex, regexGroup.Decode, regexes));
}
}
Collect(source, userSourceLinks, collectTime, sourceContentType, context, response, groupRegexes);
}
else
{
Collect(source, userSourceLinks, collectTime, sourceContentType, context, response, allGroupRegexes);
}
}
}
}
