public virtual string BuildNewItemName(object importRow) { if (!ItemNameFields.Any()) { throw new NullReferenceException("there are no 'Name' fields specified"); } StringBuilder strItemName = new StringBuilder(); foreach (string nameField in ItemNameFields) { try { strItemName.Append(GetFieldValue(importRow, nameField)); } catch (ArgumentException) { throw new NullReferenceException(string.Format("the field name: '{0}' does not exist in the import row", nameField)); } } string nameValue = strItemName.ToString(); if (string.IsNullOrEmpty(nameValue)) { throw new NullReferenceException(string.Format("the name fields: '{0}' are empty in the import row", string.Join(",", ItemNameFields))); } return(StringService.GetValidItemName(nameValue, this.ItemNameMaxLength).Trim()); }
/// <summary> /// uses the query field to retrieve file data /// </summary> /// <returns></returns> public override IEnumerable <object> GetImportData() { mappings = GetMappings(); DataTable dt = new DataTable(); dt.Columns.Add(ItemNameColumn); dt.Columns.Add(PathColumn); dt.Columns.Add(RequestedURL); dt.Columns.Add(ActionColumn); ImportConfig config = new ImportConfig(ImportItem, ToDB, this.Query); config.ImportLocation = ImportToWhere; Config = config; if (ItemNameFields.FirstOrDefault() == "[URL]") { ItemNameFields[0] = ItemNameColumn; } //Adding columns to the table from field mapping foreach (var key in mappings) { string toField = mappings[key.ToString()]; dt.Columns.Add(toField); } List <string> urls = config.StoredURLs; foreach (var url in urls) { string relativeURL = url.Replace("http://", "").Replace("https://", ""); Char[] splitChars = { '/' }; //parts are the directory list in array format, ie. second array is the child of first etc.. List <string> levels = config.IgnoreRootDirectories ? relativeURL.Split(splitChars, StringSplitOptions.RemoveEmptyEntries).Skip(1).ToList() : relativeURL.Split(splitChars, StringSplitOptions.RemoveEmptyEntries).ToList(); if (config.ExcludeDirectories != null && config.ExcludeDirectories.Any()) { levels.RemoveAll(x => config.ExcludeDirectories.Contains(x.ToLower())); } BuildData(config, levels, url, dt); } return((dt.Rows).Cast <object>()); }
/// <summary> /// uses the query field to retrieve file data /// </summary> /// <returns></returns> public override IEnumerable <object> GetImportData() { mappings = GetMappings(); DataTable dt = new DataTable(); dt.Columns.Add(ItemNameColumn); dt.Columns.Add(PathColumn); dt.Columns.Add(RequestedURL); dt.Columns.Add(ActionColumn); AllowedExtensions = new List <string>(); ExcludeDirectories = new List <string>(); Name = ImportItem.Name; int count = 0; StoredURLs = new List <string>(); IgnoreRootDirectories = ImportItem.Fields["Ignore Root Directories"].Value == "1" ? true : false; MaintainHierarchy = ImportItem.Fields["Maintain Hierarchy"].Value == "1" ? true : false; ImportTextOnly = ImportItem.Fields["Import Text Only"].Value == "1" ? true : false; AllowedExtensions = ImportItem.Fields["Allowed URL Extensions"].Value.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).ToList(); BaseUrl = ImportItem.Fields["Base URL"].Value; URLCount = ImportItem.Fields["Top x URLs"].Value; ExcludeDirectories = ImportItem.Fields["Exclude Directories"].Value.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).ToList(); SelectedMapping = ImportItem; string storedValues = !string.IsNullOrEmpty(Query) ? Query : ImportItem.Fields["Query"].Value; storedValues.Split(new[] { '\r' }, StringSplitOptions.RemoveEmptyEntries).ToList(); List <string> urls = storedValues.Split(new[] { '\n' }).ToList(); urls = urls.Where(u => !u.StartsWith("//")).ToList(); List <string> sitemapURLs = urls.Where(u => u.Trim().EndsWith(".xml")).ToList(); List <string> pageURLs = urls.Where(u => !u.Trim().EndsWith(".xml")).ToList(); if (sitemapURLs.Any()) { ImportFromSitemap(sitemapURLs, StoredURLs); } if (pageURLs.Any()) { StoredURLs.AddRange(pageURLs); } StoredURLs = StoredURLs.Select(u => u.TrimEnd('\r', '\n').ToLower()).ToList(); if (AllowedExtensions != null && AllowedExtensions.Any()) { StoredURLs = StoredURLs.Where(x => AllowedExtensions.Any(e => x.EndsWith(("." + e.Trim())))).ToList(); if (pageURLs.Any(p => p.Contains("?"))) { var queryPages = pageURLs; queryPages = queryPages.Select(u => u.TrimEnd('\r', '\n').ToLower()).ToList(); queryPages = queryPages.Where(x => AllowedExtensions.Any(e => x.Contains(("." + e.Trim() + "?")))).ToList(); StoredURLs.AddRange(queryPages); } } if (int.TryParse(URLCount, out count)) { StoredURLs = StoredURLs.Take(count).ToList(); } ImportLocation = ImportToWhere; if (ItemNameFields.FirstOrDefault() == "[URL]") { ItemNameFields[0] = ItemNameColumn; } //Adding columns to the table from field mapping foreach (var key in mappings) { string toField = mappings[key.ToString()]; dt.Columns.Add(toField); } foreach (var url in StoredURLs) { string relativeURL = url.Replace("http://", "").Replace("https://", ""); Char[] splitChars = { '/' }; //parts are the directory list in array format, ie. second array is the child of first etc.. List <string> levels = IgnoreRootDirectories ? relativeURL.Split(splitChars, StringSplitOptions.RemoveEmptyEntries).Skip(1).ToList() : relativeURL.Split(splitChars, StringSplitOptions.RemoveEmptyEntries).ToList(); if (ExcludeDirectories != null && ExcludeDirectories.Any()) { levels.RemoveAll(x => ExcludeDirectories.Contains(x.ToLower())); } BuildData(levels, url, dt); } return((dt.Rows).Cast <object>()); }