public void ReadDataExtend(ImportItemsArgs args) { Log.Info("Sitecore.Foundation.Import:Reading XSLX input data", this); try { IExcelDataReader excelReader; if (args.FileExtension.Equals(Constants.ExcelFileExtention, StringComparison.OrdinalIgnoreCase)) { excelReader = ExcelReaderFactory.CreateBinaryReader(args.FileStream, ReadOption.Loose); } else { excelReader = ExcelReaderFactory.CreateOpenXmlReader(args.FileStream); } excelReader.IsFirstRowAsColumnNames = args.ImportOptions.FirstRowAsColumnNames; if (!excelReader.IsValid) { Log.Error("Sitecore.Foundation.Import:Invalid Excel file '" + excelReader.ExceptionMessage + "'", this); return; } ConvertDataToDatatable(excelReader, args); } catch (Exception ex) { Log.Error("Sitecore.Foundation.Import:" + ex.ToString(), this); } }
public void ReadData(ImportItemsArgs args) { Log.Info("EzImporter:Reading Ziped folder input data", this); try { //IExcelDataReader excelReader; //if (args.FileExtension == "xls") //{ // excelReader = ExcelReaderFactory.CreateBinaryReader(args.FileStream, ReadOption.Loose); //} //else //{ // excelReader = ExcelReaderFactory.CreateOpenXmlReader(args.FileStream); //} FileInfo fi = new FileInfo(@"C:\darcy-out\www.shoosmiths.co.uk.zip"); string folder = Decompress(fi); PreProcessFilesRecursive(@"C:\darcy-out\www.shoosmiths.co.uk"); getFilesRecursive(@"C:\darcy-out\www.shoosmiths.co.uk", args); //excelReader.IsFirstRowAsColumnNames = args.ImportOptions.FirstRowAsColumnNames; //if (!excelReader.IsValid) //{ // Log.Error("EzImporter:Invalid Excel file '" + excelReader.ExceptionMessage + "'", this); // return; //} //DataSet result = excelReader.AsDataSet(); //if (result == null) //{ // Log.Error("EzImporter:No data could be retrieved from Excel file.", this); //} //if (result.Tables == null || result.Tables.Count == 0) //{ // Log.Error("EzImporter:No worksheets found in Excel file", this); // return; //} //var readDataTable = result.Tables[0]; //foreach (var readDataRow in readDataTable.AsEnumerable()) //{ // var row = args.ImportData.NewRow(); // for (int i = 0; i < args.Map.InputFields.Count; i++) // { // if (i < readDataTable.Columns.Count && readDataRow[i] != null) // { // row[i] = Convert.ToString(readDataRow[i]); // } // else // { // row[i] = ""; // } // } // args.ImportData.Rows.Add(row); //} //Log.Info(string.Format("EzImporter:{0} records read from input data.", readDataTable.Rows.Count), this); } catch (Exception ex) { Log.Error("EzImporter:" + ex.ToString(), this); } }
public void Process_Aborted() { var validateArgs = new ValidateArgs(); var args = new ImportItemsArgs { FileStream = null }; validateArgs.Process(args); Assert.True(args.Aborted); }
public void ReadData(ImportItemsArgs args) { Log.Info("EzImporter:Reading XSLX input data", this); try { IExcelDataReader excelReader; if (args.FileExtension == "xls") { excelReader = ExcelReaderFactory.CreateBinaryReader(args.FileStream, ReadOption.Loose); } else { excelReader = ExcelReaderFactory.CreateOpenXmlReader(args.FileStream); } excelReader.IsFirstRowAsColumnNames = args.ImportOptions.FirstRowAsColumnNames; if (!excelReader.IsValid) { Log.Error("EzImporter:Invalid Excel file '" + excelReader.ExceptionMessage + "'", this); return; } DataSet result = excelReader.AsDataSet(); if (result == null) { Log.Error("EzImporter:No data could be retrieved from Excel file.", this); } if (result.Tables == null || result.Tables.Count == 0) { Log.Error("EzImporter:No worksheets found in Excel file", this); return; } var readDataTable = result.Tables[0]; foreach (var readDataRow in readDataTable.AsEnumerable()) { var row = args.ImportData.NewRow(); for (int i = 0; i < args.Map.InputFields.Count; i++) { if (i < readDataTable.Columns.Count && readDataRow[i] != null) { row[i] = Convert.ToString(readDataRow[i]); } else { row[i] = ""; } } args.ImportData.Rows.Add(row); } Log.Info(string.Format("EzImporter:{0} records read from input data.", readDataTable.Rows.Count), this); } catch (Exception ex) { Log.Error("EzImporter:" + ex.ToString(), this); } }
public void ReadData(ImportItemsArgs args) { Log.Info("EzImporter:Reading CSV input data...", this); try { var reader = new StreamReader(args.FileStream); var insertLineCount = 0; var readLineCount = 0; do { var line = reader.ReadLine(); readLineCount++; if (line == null || (readLineCount == 1 && args.ImportOptions.FirstRowAsColumnNames)) { continue; } var row = args.ImportData.NewRow(); var csvDelimiters = new HashSet <char>(); foreach (var delimiter in args.ImportOptions.CsvDelimiter) { foreach (var delimiterChar in delimiter) { csvDelimiters.Add(delimiterChar); } } var quotationMark = (!string.IsNullOrEmpty(args.ImportOptions.QuotationMark) ? args.ImportOptions.QuotationMark[0]:'"'); var values = ParseLine(line, csvDelimiters, quotationMark); for (int j = 0; j < args.Map.InputFields.Count; j++) { if (j < values.Length) { row[j] = values[j]; } else { row[j] = ""; } } args.ImportData.Rows.Add(row); insertLineCount++; } while (!reader.EndOfStream); Log.Info(string.Format("EzImporter:{0} records read from input data.", insertLineCount), this); } catch (Exception ex) { Log.Error("EzImporter:" + ex.ToString(), this); } }
private void ConvertDataToDatatable(IExcelDataReader excelReader, ImportItemsArgs args) { DataSet result = excelReader.AsDataSet(); if (result == null) { Log.Error("Sitecore.Foundation.Import:No data could be retrieved from Excel file.", this); return; } if (result.Tables == null || result.Tables.Count == 0) { Log.Error("Sitecore.Foundation.Import:No worksheets found in Excel file", this); return; } DataTable readDataTable = null; DataTable readDataTable1 = null; if (args.ContentType == ContentType.Article.ToString()) { readDataTable = ConvertArticleTable(result, out readDataTable1); if (readDataTable1 == null) { readDataTable1 = new DataTable(); } } else { readDataTable = result.Tables[0]; } if (readDataTable == null) { readDataTable = new DataTable(); } args.ImportDatas.Add(readDataTable); if (readDataTable1 != null) { args.ImportDatas.Add(readDataTable1); } var countRow = readDataTable.Rows.Count; countRow = readDataTable1 != null ? countRow + readDataTable1.Rows.Count : countRow; Log.Info(string.Format("Sitecore.Foundation.Import:{0} records read from input data.", countRow), this); }
public string[] GetColumnNames(ImportItemsArgs args) { Log.Info("EzImporter:Reading column names from input CSV file...", this); try { using (var reader = new StreamReader(args.FileStream)) { var line = reader.ReadLine(); if (line != null) { return(line.Split(args.ImportOptions.CsvDelimiter, StringSplitOptions.None)); } } } catch (Exception ex) { Log.Error("EzImporter:" + ex.ToString(), this); } return(new string[] {}); }
public void ReadData(ImportItemsArgs args) { Log.Info("EzImporter:Reading CSV input data...", this); try { var reader = new StreamReader(args.FileStream); var insertLineCount = 0; var readLineCount = 0; do { var line = reader.ReadLine(); readLineCount++; if (line == null || (readLineCount == 1 && args.ImportOptions.FirstRowAsColumnNames)) { continue; } var row = args.ImportData.NewRow(); var values = line.Split(args.ImportOptions.CsvDelimiter, StringSplitOptions.None); for (int j = 0; j < args.Map.InputFields.Count; j++) { if (j < values.Length) { row[j] = values[j]; } else { row[j] = ""; } } args.ImportData.Rows.Add(row); insertLineCount++; } while (!reader.EndOfStream); Log.Info(string.Format("EzImporter:{0} records read from input data.", insertLineCount), this); } catch (Exception ex) { Log.Error("EzImporter:" + ex.ToString(), this); } }
public static Item SearchChildItem(ImportItemsArgs args, Item parent, string itemName, TemplateItem templateItem, ImportItemsProcessor processor, Action <Item, string> updateDisplayName = null, string displayName = null) { // search for the child by name Item newItem = parent.GetChildren()[itemName]; if (newItem != null) { if (args.ImportOptions.ExistingItemHandling == ExistingItemHandling.AddVersion) { args.Statistics.UpdatedItems++; newItem = newItem.Versions.AddVersion(); Log.Info(string.Format("Sitecore.Foundation.Import:Creating new version of item {0}", newItem.Paths.ContentPath), processor); } else if (args.ImportOptions.ExistingItemHandling == ExistingItemHandling.Skip) { Log.Info(string.Format("Sitecore.Foundation.Import:Skipping update of item {0}", newItem.Paths.ContentPath), processor); return(newItem); } else if (args.ImportOptions.ExistingItemHandling == ExistingItemHandling.Update) { //continue to update current item/version args.Statistics.UpdatedItems++; } } else { //if not found then create one args.Statistics.CreatedItems++; newItem = parent.Add(itemName, templateItem); Log.Info(string.Format("Sitecore.Foundation.Import:Creating item {0}", newItem.Paths.ContentPath), processor); //update displayname if (updateDisplayName != null && displayName != null) { updateDisplayName(newItem, displayName); } } return(newItem); }
public string[] GetColumnNames(ImportItemsArgs args) { Log.Info("Sitecore.Foundation.Import:Reading column names from input XSLX file...", this); try { //1. Reading from a binary Excel file ('97-2003 format; *.xls) //2. Reading from a OpenXml Excel file (2007 format; *.xlsx) IExcelDataReader excelReader = ExcelReaderFactory.CreateOpenXmlReader(args.FileStream); excelReader.IsFirstRowAsColumnNames = true; //assume first line is data, so we can read it if (!excelReader.IsValid) { Log.Info("Sitecore.Foundation.Import:Invalid Excel file '" + excelReader.ExceptionMessage + "'", this); return(new string[] {}); } DataSet result = excelReader.AsDataSet(); if (result == null) { Log.Info("Sitecore.Foundation.Import:No data could be retrieved from Excel file.", this); return(new string[] { }); } if (result.Tables == null || result.Tables.Count == 0) { Log.Info("Sitecore.Foundation.Import:No worksheets found in Excel file", this); return(new string[] {}); } var readDataTable = result.Tables[0]; return(readDataTable.Columns .Cast <DataColumn>() .Select(c => c.ColumnName).ToArray()); } catch (Exception ex) { Log.Error("Sitecore.Foundation.Import:" + ex.ToString(), this); } return(new string[] { }); }
public IHttpActionResult Import(ImportModel importModel) { var database = Sitecore.Configuration.Factory.GetDatabase("master"); var languageItem = database.GetItem(importModel.Language); var uploadedFile = (MediaItem)database.GetItem(importModel.MediaItemId); if (uploadedFile == null) { return(new JsonResult <ImportResultModel>(null, new JsonSerializerSettings(), Encoding.UTF8, this)); } ImportResultModel result; try { var args = new ImportItemsArgs { Database = database, FileExtension = uploadedFile.Extension.ToLower(), FileStream = uploadedFile.GetMediaStream(), RootItemId = new ID(importModel.ImportLocationId), TargetLanguage = Sitecore.Globalization.Language.Parse(languageItem.Name), Map = Map.Factory.BuildMapInfo(new ID(importModel.MappingId)), ImportOptions = new ImportOptions { CsvDelimiter = new[] { importModel.CsvDelimiter }, QuotationMark = importModel.QuotationMark, MultipleValuesImportSeparator = importModel.MultipleValuesSeparator, TreePathValuesImportSeparator = @"\", FirstRowAsColumnNames = importModel.FirstRowAsColumnNames } }; args.ImportOptions.ExistingItemHandling = (ExistingItemHandling) Enum.Parse(typeof(ExistingItemHandling), importModel.ExistingItemHandling); args.ImportOptions.InvalidLinkHandling = (InvalidLinkHandling) Enum.Parse(typeof(InvalidLinkHandling), importModel.InvalidLinkHandling); args.ImportOptions.DataStructureType = (DataStructureType) Enum.Parse(typeof(DataStructureType), importModel.DataStructureType); Sitecore.Diagnostics.Log.Info( string.Format("EzImporter: mappingId:{0} mediaItemId:{1} firstRowAsColumnNames:{2}", importModel.MappingId, importModel.MediaItemId, args.ImportOptions.FirstRowAsColumnNames), this); args.Timer.Start(); CorePipeline.Run("importItems", args); args.Timer.Stop(); if (args.Aborted) { result = new ImportResultModel { HasError = true, Log = args.Statistics.ToString(), ErrorMessage = args.Message, ErrorDetail = args.ErrorDetail }; } else { result = new ImportResultModel { Log = args.Statistics.ToString() + " Duration: " + args.Timer.Elapsed.ToString("c") }; } } catch (Exception ex) { result = new ImportResultModel { HasError = true, ErrorMessage = ex.Message, ErrorDetail = ex.ToString() }; } return(new JsonResult <ImportResultModel>(result, new JsonSerializerSettings(), Encoding.UTF8, this)); }
public string[] GetColumnNames(ImportItemsArgs args) { return(new string[0]); }
private void getFilesRecursive(string sDir, ImportItemsArgs args, string parent = "") { try { DirectoryInfo sDirinfo = new DirectoryInfo(sDir); IList <string> skipfolders = new List <string>(); skipfolders.Add(@"files"); skipfolders.Add(@"images"); skipfolders.Add(@"webvirtuals"); var mainroots = sDirinfo.GetFiles("Index.aspx"); if (mainroots != null && mainroots.Count() == 1) { var row = args.ImportData.NewRow(); row["Name"] = sDirinfo.Name.Replace("-", " ").Trim(); row["father"] = parent; row["ID"] = Guid.NewGuid(); parent = row["ID"].ToString(); row["Path"] = mainroots[0].FullName;//.Replace(sDir, ""); PopulateContent(mainroots[0].FullName, row, args); } foreach (string d in Directory.GetDirectories(sDir)) { Log.Info(d, this); var row = args.ImportData.NewRow(); DirectoryInfo di = new DirectoryInfo(d); if (!skipfolders.Contains(di.Name)) { row["Name"] = di.Name.Replace("-", " ").Trim(); row["father"] = parent; row["ID"] = Guid.NewGuid(); row["Path"] = di.FullName;//.Replace(sDir, ""); var roots = di.GetFiles("Index.aspx"); if (roots != null && roots.Count() == 1) { PopulateContent(roots[0].FullName, row, args); } else { args.ImportData.Rows.Add(row); } getFilesRecursive(d, args, row["ID"].ToString()); } } int filecounter = 0; foreach (var file in Directory.GetFiles(sDir, "*.aspx*")) { Log.Info(file, this); var row = args.ImportData.NewRow(); row["father"] = parent; row["ID"] = Guid.NewGuid(); row["Path"] = file;//.Replace(sDir, ""); PopulateContent(file, row, args); //if (++filecounter >= 10) // break; } } catch (System.Exception e) { Log.Error(e.Message, this); } }
public abstract void Process(ImportItemsArgs args);
public void Run(Item[] items, CommandItem command, ScheduleItem schedule) { var importCommand = new ImportCommandItem(command.InnerItem); Log.Info( "EzImporter.Tasks.Import.Run() Starting Import: " + " fileName=" + importCommand.FileName + " lang=" + importCommand.TargetLanguage + " location=" + importCommand.ImportLocationId + " importMap=" + importCommand.ImportMapId + " database=" + importCommand.Database.Name + " csvDelimiter=" + importCommand.CsvDelimiter + " ExistingItemHandling=" + importCommand.ExistingItemHandling + " InvalidLinkHandling=" + importCommand.InvalidLinkHandling + " MultipleValuesImportSeparator=" + importCommand.MultipleValuesImportSeparator + " TreePathValuesImportSeparator=" + importCommand.TreePathValuesImportSeparator, this); var options = Factory.GetDefaultImportOptions(); if (importCommand.CsvDelimiter != null) { options.CsvDelimiter = new[] { importCommand.CsvDelimiter }; } if (importCommand.ExistingItemHandling != null) { options.ExistingItemHandling = (ExistingItemHandling) Enum.Parse(typeof(ExistingItemHandling), importCommand.ExistingItemHandling); } if (importCommand.InvalidLinkHandling != null) { options.InvalidLinkHandling = (InvalidLinkHandling) Enum.Parse(typeof(InvalidLinkHandling), importCommand.InvalidLinkHandling); } if (importCommand.MultipleValuesImportSeparator != null) { options.MultipleValuesImportSeparator = importCommand.MultipleValuesImportSeparator; } if (importCommand.TreePathValuesImportSeparator != null) { options.TreePathValuesImportSeparator = importCommand.TreePathValuesImportSeparator; } options.FirstRowAsColumnNames = importCommand.FirstRowAsColumnNames; if (string.IsNullOrWhiteSpace(importCommand.FileName)) { Log.Error( "EzImporter.Tasks.Import.Run() - Import Error: File not specified", this); return; } string fileName; if (File.Exists(importCommand.FileName)) { fileName = importCommand.FileName; } else { fileName = HostingEnvironment.MapPath(importCommand.FileName); if (!File.Exists(fileName)) { Log.Error( "EzImporter.Tasks.Import.Run() - Import Error: File not found (" + importCommand.FileName + ")", this); return; } } var extension = GetFileExtension(fileName); if (extension == null) { Log.Error( "EzImporter.Tasks.Import.Run() - Import Error: Unknown file extension (" + importCommand.FileName + ")", this); return; } var stream = new StreamReader(fileName); var args = new ImportItemsArgs { Database = importCommand.ImportDatabase, FileExtension = extension, FileStream = stream.BaseStream, RootItemId = importCommand.ImportLocationId, TargetLanguage = importCommand.TargetLanguage, Map = Map.Factory.BuildMapInfo(importCommand.ImportMapId), ImportOptions = options }; try { CorePipeline.Run("importItems", args); Log.Info("EzImporter.Tasks.Import.Run() Import Finished " + args.Statistics, this); } catch (Exception ex) { Log.Error("EzImporter.Tasks.Import.Run() - Import Error: " + ex, this); throw; } }
private void PopulateContent(string file, DataRow row, ImportItemsArgs args) { FileInfo fi = new FileInfo(file); var doc = new HtmlDocument(); doc.Load(file, true); if (doc.DocumentNode != null) { foreach (var field in args.Map.InputFields) { if (field.Name.Equals("name", StringComparison.InvariantCultureIgnoreCase)) { row[field.Name] = GetCleanName(fi.Name, field); } else if (field.Name.Equals("originalvalue", StringComparison.InvariantCultureIgnoreCase)) { row[field.Name] = doc.DocumentNode.InnerHtml; } else { if (!string.IsNullOrWhiteSpace(field.XsltSelector)) { var nodes = doc.DocumentNode.SelectNodes(field.XsltSelector); row[field.Name] = string.Empty; if (nodes != null && nodes.Any()) { foreach (var node in nodes) { if (node != null) { if (!string.IsNullOrWhiteSpace(field.Property)) { var content = node.Attributes[field.Property]; if (content != null) { row[field.Name] += GetCleanContent(content, field); } } else { var content = node.Attributes["content"]; if (content != null) { row[field.Name] += GetCleanContent(content, field); } else { row[field.Name] += GetCleanContent(node, field); } } } } } } } // } args.ImportData.Rows.Add(row); doc = null; } }