public static string RemoveInvalidChars(ImportConfig Config, string data, bool root, bool report = true)
{
string originalName = data;
if (data.Contains(".") && !root)
{
int index = data.IndexOf('.');
data = data.Remove(index);
}
data = ItemUtil.ProposeValidItemName(data);
foreach (var cleanup in Config.ItemNameCleanups)
{
if (!data.Contains(cleanup.Find))
{
continue;
}
data = data.Replace(cleanup.Find, cleanup.Replace);
if (!report)
{
continue;
}
ImportReporter.Write(cleanup.CleanupItem, Level.Info, " To: " + data + "", "Name > From: " + originalName, "Name Change", "");
}
return(data);
}
public void Process(IDataMap map)
{
RunPostProcessors(map);
try {
ImportReporter.Print();
}
catch {
}
}
public void Run(Item processor, Item itemToProcess, Item fieldMapping)
{
BaseMapping baseMap = new BaseMapping(fieldMapping);
string findPattern = processor.Fields["Find"].Value;
string replacePattern = processor.Fields["Replace"].Value;
string replaceReportingText = string.IsNullOrEmpty(replacePattern) ? "*remove*" : replacePattern;
try
{
HtmlDocument document = new HtmlDocument();
string content = itemToProcess.Fields[baseMap.NewItemField].Value;
document.LoadHtml(content);
HtmlNode node = Helper.HandleNodesLookup(findPattern, document);
using (new SecurityModel.SecurityDisabler())
{
itemToProcess.Editing.BeginEdit();
if (node != null)
{
if (findPattern.Contains("/*") && node.ChildNodes != null)
{
foreach (var child in node.ChildNodes)
{
content = content.Replace(child.OuterHtml, replacePattern);
itemToProcess.Fields[baseMap.NewItemField].Value = content;
}
}
else
{
itemToProcess.Fields[baseMap.NewItemField].Value = content.Replace(node.OuterHtml, replacePattern);
}
ImportReporter.Write(itemToProcess, Level.Info, string.Format("Replaced '{0}' with '{1}'", findPattern, replaceReportingText), baseMap.NewItemField, "Cleanup");
}
else
{
bool occuruncesFound = itemToProcess.Fields[baseMap.NewItemField].Value.Contains(findPattern);
itemToProcess.Fields[baseMap.NewItemField].Value = content.Replace(findPattern, replacePattern);
if (occuruncesFound)
{
ImportReporter.Write(itemToProcess, Level.Info, string.Format("Replaced '{0}' with '{1}'", findPattern, replaceReportingText), baseMap.NewItemField, "Cleanup");
}
}
itemToProcess.Editing.EndEdit();
}
}
catch (Exception ex)
{
ImportReporter.Write(itemToProcess, Level.Info, string.Format("Replace failed on '{0}' with '{1}'", findPattern, replaceReportingText), baseMap.NewItemField, "Cleanup");
}
}
/// <summary>
/// There is no custom data for this type
/// </summary>
/// <param name="newItem"></param>
/// <param name="importRow"></param>
public override void ProcessCustomData(ref Item newItem, object importRow)
{
DataRow dataRow = importRow as DataRow;
string requestedURL = dataRow[RequestedURL].ToString();
ImportReporter.Write(newItem, Level.Info, "", "", "Item Added/Updated", requestedURL);
foreach (var field in FieldDefinitions)
{
RunProcessor(field, newItem);
}
}
public static void Run(Item itemToProcess, Item fieldMapping)
{
MultilistField WarningTags = fieldMapping.Fields["Warning Trigger Tags"];
ItemReport itemReport;
if (!ImportReporter.ItemReports.ContainsKey(itemToProcess.Paths.Path))
{
ImportReporter.ItemReports.Add(itemToProcess.Paths.Path, new ItemReport());
}
ImportReporter.ItemReports.TryGetValue(itemToProcess.Paths.Path, out itemReport);
if (string.IsNullOrEmpty(itemReport.ItemName))
{
itemReport.ItemName = itemToProcess.Name;
}
if (string.IsNullOrEmpty(itemReport.NewItemPath))
{
itemReport.NewItemPath = itemToProcess.Paths.Path;
}
BaseMapping baseMap = new BaseMapping(fieldMapping);
List <string> tagsToCheck = new List <string>();
HtmlDocument document = new HtmlDocument();
string content = itemToProcess.Fields[baseMap.NewItemField].Value;
document.LoadHtml(content);
string findPattern;
HtmlNode node;
foreach (var id in WarningTags.TargetIDs)
{
findPattern = itemToProcess.Database.GetItem(id).Fields["Identifier"].Value;
node = Helper.HandleNodesLookup(findPattern, document);
if (node == null || node.ChildNodes == null || !node.ChildNodes.Any())
{
continue;
}
tagsToCheck.Add(findPattern);
}
if (!tagsToCheck.Any())
{
return;
}
string fieldName = fieldMapping.Fields["To What Field"].Value;
string messageText = string.Join("/", tagsToCheck.ToArray()) + " warning tags were found and requires additional review";
ImportReporter.Write(itemToProcess, Level.Warning, messageText, fieldName, "Warning Trigger Tags");
}
public string Run(Item processor, HtmlDocument doc, string currentDirURL, string defaultTemplateID)
{
string identifier = processor.Fields["Identifier"].Value;
string action = string.Empty;
HtmlNode node = Helper.HandleNodesLookup(identifier, doc);
if (node != null)
{
action = processor.Fields["Action"].Value;
}
ImportReporter.Write(processor, Level.Info, "Template change From: " + defaultTemplateID + " To: " + action, "N/A", "Template Change", itemFetchURL: currentDirURL);
return(action);
}
public void Run(Item processor, Item itemToProcess, Item fieldMapping)
{
BaseMapping baseMap = new BaseMapping(fieldMapping);
try
{
using (new SecurityModel.SecurityDisabler())
{
itemToProcess.Editing.BeginEdit();
itemToProcess.Fields[baseMap.NewItemField].Value = itemToProcess.Fields[baseMap.NewItemField].Value.Trim();
itemToProcess.Editing.EndEdit();
}
ImportReporter.Write(itemToProcess, Level.Info, "", baseMap.NewItemField, "Trim Value");
}
catch (Exception ex)
{
ImportReporter.Write(itemToProcess, Level.Error, ex.ToString(), baseMap.NewItemField, "Trim Value Field");
}
}
public string Run(Item processor, HtmlDocument doc, string currentDirURL, string defaultTemplateID)
{
List <string> checkStrings = new List <string>();
string messages = string.Empty;
var warnings = GetDomCheckStrings(processor);
foreach (ComplexWarningTag tag in warnings)
{
foreach (string searchText in tag.Identifiers)
{
if (doc.DocumentNode.OuterHtml.Contains(searchText))
{
ImportReporter.Write(processor, Level.Warning, tag.Action, "HTML Source", "DOM Parsing", currentDirURL);
}
}
}
return("");
}
public static MediaItem UploadMedia(string mediaURl, string destination, Item item, ImageField imageField = null)
{
string uploadedMediaPath = string.Empty;
MediaItem mediaItem = null;
string cleanMediaURl = mediaURl;
if (mediaURl.Contains("?"))
{
cleanMediaURl = cleanMediaURl.Remove(cleanMediaURl.IndexOf('?'));
}
string extension = Path.GetExtension(cleanMediaURl);
string mediaName = Path.GetFileName(cleanMediaURl);
mediaName = mediaName.Replace(extension, "").Replace("-", " ").Replace("_", " ").Trim();
//int nameStartIndex = mediaURl.LastIndexOf('/');
//mediaURl.Substring(nameStartIndex).Replace(extension, "");
try
{
var webRequest = WebRequest.Create(mediaURl);
using (var webResponse = webRequest.GetResponse())
{
using (var stream = webResponse.GetResponseStream())
{
if (stream != null)
{
using (var memoryStream = new MemoryStream())
{
stream.CopyTo(memoryStream);
var mediaCreator = new MediaCreator();
var options = new MediaCreatorOptions
{
Versioned = false,
IncludeExtensionInItemName = true,
Database = item.Database,
Destination = destination + mediaName
};
using (new SecurityDisabler())
{
mediaItem = item.Database.GetItem(options.Destination);
if (mediaItem == null)
{
var creator = new MediaCreator();
mediaItem = MediaManager.Creator.CreateFromStream(memoryStream, mediaName + extension, options);
ImportReporter.Write(item, Level.Info, string.Format("New media item has been imported from {0}", mediaURl), extension, "Media Import");
}
mediaItem.BeginEdit();
if (imageField != null)
{
imageField.MediaID = mediaItem.ID;
}
uploadedMediaPath = mediaItem.MediaPath;
mediaItem.Alt = mediaName;
mediaItem.EndEdit();
}
}
}
}
}
}
catch (Exception ex)
{
ImportReporter.Write(item, Level.Error, string.Format("Error during media import from {0}. Error: {1}", mediaURl, ex.Message), extension, "Media Import");
}
return(mediaItem);
}
public void Run(Item processor, Item itemToProcess, Item fieldMapping)
{
ImportConfig config = new ImportConfig(fieldMapping.Parent.Parent, fieldMapping.Database, "");
Uri baseUri = new Uri(config.BaseUrl);
BaseMapping baseMap = new BaseMapping(fieldMapping);
MediaProcessor mediaProcessor = new MediaProcessor(processor);
HtmlDocument document = new HtmlDocument();
string content = itemToProcess.Fields[baseMap.NewItemField].Value;
document.LoadHtml(content);
using (new SecurityModel.SecurityDisabler())
{
foreach (var mediaType in mediaProcessor.MediaTypes)
{
var nodes = document.DocumentNode.SelectNodes(string.Format("//{0}/@{1}", mediaType.Identifier, mediaType.Attribute));
if (nodes == null)
{
continue;
}
//select nodes in html where path ends with extension listed in config
List <HtmlNode> targetedNodes = nodes.Where(n => n.Attributes[mediaType.Attribute].Value.Trim().ToLower()
.EndsWith(mediaType.Extension.Trim().ToLower())).ToList();
if (targetedNodes == null)
{
targetedNodes = nodes.Where(n => n.Attributes[mediaType.Attribute].Value.Trim().ToLower()
.Contains(mediaType.Extension.Trim().ToLower())).ToList();
}
else
{
targetedNodes.AddRange(nodes.Where(n => n.Attributes[mediaType.Attribute].Value.Trim().ToLower()
.Contains(mediaType.Extension.Trim().ToLower())).ToList());
}
foreach (var child in targetedNodes)
{
try
{
//Make sure the selected tags have media items to import, with existing parameters to follow
string source = child.Attributes[mediaType.Attribute].Value;
string mediaExtension = Path.GetExtension(source);
if (string.IsNullOrEmpty(source))
{
continue;
}
Uri mediaSource = new Uri(source, UriKind.RelativeOrAbsolute);
if (mediaSource.IsAbsoluteUri)
{
if ((mediaSource.Host != baseUri.Host))
{
continue;
}
}
else
{
//This is for internal/relative path images
mediaSource = new Uri(baseUri, source);
}
string destination = mediaProcessor.RetrieveDestination(processor, mediaProcessor.RootDestination, source, mediaSource, baseUri);
Item importedMediaItem = MediaUpload.UploadMedia(mediaSource.ToString(), destination, itemToProcess);
if (importedMediaItem != null)
{
var mediaUrl = "-/media/" + importedMediaItem.ID.ToShortID().ToString() + ".ashx";
//Swap the old link with the new link to our media library
HtmlNode newChild = child.Clone();
newChild.Attributes[mediaType.Attribute].Value = mediaUrl;
content = content.Replace(child.OuterHtml, newChild.OuterHtml);
itemToProcess.Editing.BeginEdit();
itemToProcess.Fields[baseMap.NewItemField].Value = content;
itemToProcess.Editing.EndEdit();
ImportReporter.Write(itemToProcess, Level.Info, string.Format("Link updated for: {0}", importedMediaItem.Name), baseMap.NewItemField, "Media Importer");
}
}
catch (Exception ex)
{
ImportReporter.Write(itemToProcess, Level.Error, string.Format("There was an error importing media from {0}. Error: {1}.", string.Concat(baseUri, child.Attributes[mediaType.Attribute].Value), ex.Message), baseMap.NewItemField, "Media Import");
//Error importing media and/or updating links to media
}
}
}
}
}
private void UpdateFields(Item importLocation, ImportConfig config, string fieldName, List <string> internalDomains, Item processor)
{
using (new SecurityModel.SecurityDisabler())
{
Item[] items = config.MaintainHierarchy ? importLocation.Axes.GetDescendants() : importLocation.Children.ToArray();
Item linkItem = null;
foreach (var item in items)
{
HtmlDocument doc = new HtmlDocument();
string fieldContent = "";
if (item.Fields[fieldName] != null)
{
fieldContent = item.Fields[fieldName].Value;
doc.LoadHtml(fieldContent);
}
if (string.IsNullOrEmpty(fieldContent))
{
continue;
}
foreach (HtmlNode node in doc.DocumentNode.Descendants("a"))
{
string url = node.Attributes["href"].Value.ToLower();
string olderValue = url;
string newValue = string.Empty;
string internalPath = string.Empty;
string urls = string.Empty;
string rootPath = string.Empty;
if (url.StartsWith("#"))
{
continue;
}
if (url.Contains("."))
{
url = url.Remove(url.IndexOf('.'));
}
if (config.ExcludeDirectories != null && config.ExcludeDirectories.Any())
{
foreach (var dir in config.ExcludeDirectories)
{
string excludeDir = "/" + dir.ToLower();
if (url.Contains(excludeDir))
{
url = url.Replace(excludeDir, "");
}
}
foreach (var domain in internalDomains)
{
if (url.ToLower().Contains(domain.ToLower()))
{
url = url.Replace(domain, "");
}
}
}
if (url.StartsWith("http://"))
{
continue;
}
urls = url;
List <string> directories = urls.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries).ToList();
if (config.MaintainHierarchy)
{
try
{
Uri baseURi = new Uri(config.BaseUrl);
rootPath = config.ImportLocation.Paths.FullPath + "/" + Helper.RemoveInvalidChars(config, baseURi.Host, true);
Item rootItem = config.ImportLocation.Database.GetItem(rootPath);
if (rootItem == null)
{
foreach (var domain in internalDomains)
{
rootPath = config.ImportLocation.Paths.FullPath + "/" + Helper.RemoveInvalidChars(config, domain, true);
rootItem = config.ImportLocation.Database.GetItem(rootPath);
if (rootItem != null)
{
break;
}
}
}
rootPath = rootItem != null ? rootItem.Paths.FullPath : config.ImportLocation.Children.FirstOrDefault().Paths.FullPath;
internalPath = config.IgnoreRootDirectories ? config.ImportLocation.Paths.FullPath + "/"
: rootPath + "/";
}
catch {
internalPath = config.IgnoreRootDirectories ? config.ImportLocation.Paths.FullPath + "/"
: config.ImportLocation.Children.FirstOrDefault().Paths.FullPath + "/";
}
foreach (var dir in directories)
{
internalPath += Helper.RemoveInvalidChars(config, dir, false);
if (dir != directories.LastOrDefault())
{
internalPath += "/";
}
}
linkItem = config.ImportLocation.Database.GetItem(internalPath);
}
else
{
string itemName = Helper.RemoveInvalidChars(config, directories.LastOrDefault(), false);
internalPath = config.ImportLocation.Paths.FullPath + "/" + itemName;
linkItem = config.ImportLocation.Database.GetItem(internalPath);
}
if (linkItem != null)
{
newValue = "~/link.aspx?_id=" + linkItem.ID.ToShortID() + "&_z=z";
node.Attributes["href"].Value = newValue;
ImportReporter.Write(item, Level.Info, "From: " + olderValue + " To: " + newValue, fieldName, "Link Update");
}
}
item.Editing.BeginEdit();
item.Fields[fieldName].Value = doc.DocumentNode.InnerHtml;
item.Editing.EndEdit();
}
}
}
