public virtual string BuildNewItemName(object importRow)
{
if (!ItemNameFields.Any())
{
throw new NullReferenceException("there are no 'Name' fields specified");
}
StringBuilder strItemName = new StringBuilder();
foreach (string nameField in ItemNameFields)
{
try
{
strItemName.Append(GetFieldValue(importRow, nameField));
}
catch (ArgumentException)
{
throw new NullReferenceException(string.Format("the field name: '{0}' does not exist in the import row", nameField));
}
}
string nameValue = strItemName.ToString();
if (string.IsNullOrEmpty(nameValue))
{
throw new NullReferenceException(string.Format("the name fields: '{0}' are empty in the import row", string.Join(",", ItemNameFields)));
}
return(StringService.GetValidItemName(nameValue, this.ItemNameMaxLength).Trim());
}
/// <summary>
/// uses the query field to retrieve file data
/// </summary>
/// <returns></returns>
public override IEnumerable <object> GetImportData()
{
mappings = GetMappings();
DataTable dt = new DataTable();
dt.Columns.Add(ItemNameColumn);
dt.Columns.Add(PathColumn);
dt.Columns.Add(RequestedURL);
dt.Columns.Add(ActionColumn);
ImportConfig config = new ImportConfig(ImportItem, ToDB, this.Query);
config.ImportLocation = ImportToWhere;
Config = config;
if (ItemNameFields.FirstOrDefault() == "[URL]")
{
ItemNameFields[0] = ItemNameColumn;
}
//Adding columns to the table from field mapping
foreach (var key in mappings)
{
string toField = mappings[key.ToString()];
dt.Columns.Add(toField);
}
List <string> urls = config.StoredURLs;
foreach (var url in urls)
{
string relativeURL = url.Replace("http://", "").Replace("https://", "");
Char[] splitChars = { '/' };
//parts are the directory list in array format, ie. second array is the child of first etc..
List <string> levels = config.IgnoreRootDirectories ? relativeURL.Split(splitChars, StringSplitOptions.RemoveEmptyEntries).Skip(1).ToList() :
relativeURL.Split(splitChars, StringSplitOptions.RemoveEmptyEntries).ToList();
if (config.ExcludeDirectories != null && config.ExcludeDirectories.Any())
{
levels.RemoveAll(x => config.ExcludeDirectories.Contains(x.ToLower()));
}
BuildData(config, levels, url, dt);
}
return((dt.Rows).Cast <object>());
}
/// <summary>
/// uses the query field to retrieve file data
/// </summary>
/// <returns></returns>
public override IEnumerable <object> GetImportData()
{
mappings = GetMappings();
DataTable dt = new DataTable();
dt.Columns.Add(ItemNameColumn);
dt.Columns.Add(PathColumn);
dt.Columns.Add(RequestedURL);
dt.Columns.Add(ActionColumn);
AllowedExtensions = new List <string>();
ExcludeDirectories = new List <string>();
Name = ImportItem.Name;
int count = 0;
StoredURLs = new List <string>();
IgnoreRootDirectories = ImportItem.Fields["Ignore Root Directories"].Value == "1" ? true : false;
MaintainHierarchy = ImportItem.Fields["Maintain Hierarchy"].Value == "1" ? true : false;
ImportTextOnly = ImportItem.Fields["Import Text Only"].Value == "1" ? true : false;
AllowedExtensions = ImportItem.Fields["Allowed URL Extensions"].Value.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).ToList();
BaseUrl = ImportItem.Fields["Base URL"].Value;
URLCount = ImportItem.Fields["Top x URLs"].Value;
ExcludeDirectories = ImportItem.Fields["Exclude Directories"].Value.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).ToList();
SelectedMapping = ImportItem;
string storedValues = !string.IsNullOrEmpty(Query) ? Query : ImportItem.Fields["Query"].Value;
storedValues.Split(new[] { '\r' }, StringSplitOptions.RemoveEmptyEntries).ToList();
List <string> urls = storedValues.Split(new[] { '\n' }).ToList();
urls = urls.Where(u => !u.StartsWith("//")).ToList();
List <string> sitemapURLs = urls.Where(u => u.Trim().EndsWith(".xml")).ToList();
List <string> pageURLs = urls.Where(u => !u.Trim().EndsWith(".xml")).ToList();
if (sitemapURLs.Any())
{
ImportFromSitemap(sitemapURLs, StoredURLs);
}
if (pageURLs.Any())
{
StoredURLs.AddRange(pageURLs);
}
StoredURLs = StoredURLs.Select(u => u.TrimEnd('\r', '\n').ToLower()).ToList();
if (AllowedExtensions != null && AllowedExtensions.Any())
{
StoredURLs = StoredURLs.Where(x => AllowedExtensions.Any(e => x.EndsWith(("." + e.Trim())))).ToList();
if (pageURLs.Any(p => p.Contains("?")))
{
var queryPages = pageURLs;
queryPages = queryPages.Select(u => u.TrimEnd('\r', '\n').ToLower()).ToList();
queryPages = queryPages.Where(x => AllowedExtensions.Any(e => x.Contains(("." + e.Trim() + "?")))).ToList();
StoredURLs.AddRange(queryPages);
}
}
if (int.TryParse(URLCount, out count))
{
StoredURLs = StoredURLs.Take(count).ToList();
}
ImportLocation = ImportToWhere;
if (ItemNameFields.FirstOrDefault() == "[URL]")
{
ItemNameFields[0] = ItemNameColumn;
}
//Adding columns to the table from field mapping
foreach (var key in mappings)
{
string toField = mappings[key.ToString()];
dt.Columns.Add(toField);
}
foreach (var url in StoredURLs)
{
string relativeURL = url.Replace("http://", "").Replace("https://", "");
Char[] splitChars = { '/' };
//parts are the directory list in array format, ie. second array is the child of first etc..
List <string> levels = IgnoreRootDirectories ? relativeURL.Split(splitChars, StringSplitOptions.RemoveEmptyEntries).Skip(1).ToList() :
relativeURL.Split(splitChars, StringSplitOptions.RemoveEmptyEntries).ToList();
if (ExcludeDirectories != null && ExcludeDirectories.Any())
{
levels.RemoveAll(x => ExcludeDirectories.Contains(x.ToLower()));
}
BuildData(levels, url, dt);
}
return((dt.Rows).Cast <object>());
}
