public override void Process(ImportItemsArgs args) { DataReaders.IDataReader reader; if (args.FileExtension == "csv") { reader = new DataReaders.CsvDataReader(); } else if (args.FileExtension == "xlsx" || args.FileExtension == "xls") { reader = new DataReaders.XlsxDataReader(); } else if (args.FileExtension == "zip" || args.FileExtension == "gzip") { reader = new DataReaders.ZipedFolderDataReader(); } else { Log.Info("EzImporter:Unsupported file format supplied. DataImporter accepts *.CSV and *.XLSX files", this); return; } reader.ReadData(args); args.Statistics.InputDataRows = args.ImportData.Rows.Count; }
public void Process_Aborted() { var validateArgs = new ValidateArgs(); var args = new ImportItemsArgs { FileStream = null }; validateArgs.Process(args); Assert.True(args.Aborted); }
private void ImportMapItems(ImportItemsArgs args, DataTable dataTable, OutputMap outputMap, ItemDto parentItem, bool rootLevel) { var groupedTable = dataTable.GroupBy(outputMap.Fields.Select(f => f.SourceColumn).ToArray()); for (int i = 0; i < groupedTable.Rows.Count; i++) { var row = groupedTable.Rows[i]; if (rootLevel || Convert.ToString(row[outputMap.ParentMap.NameInputField]) == parentItem.Name) { var createdItem = CreateItem(row, outputMap); createdItem.Parent = parentItem; parentItem.Children.Add(createdItem); if (outputMap.ChildMaps != null && outputMap.ChildMaps.Any()) { foreach (var childMap in outputMap.ChildMaps) { ImportMapItems(args, dataTable, childMap, createdItem, false); } } } } }
public override void Process(ImportItemsArgs args) { var rootItem = new ItemDto("<root>"); //ick foreach (var outputMap in args.Map.OutputMaps) { ImportMapItems(args, args.ImportData, outputMap, rootItem, true); //ick } args.ImportItems.AddRange(rootItem.Children); //ick }
public override void Process(ImportItemsArgs args) { Log.Info("EzImporter:Processing import map...", this); args.ImportData.Columns.Clear(); foreach (var column in args.Map.InputFields) { args.ImportData.Columns.Add(column.Name, typeof(string)); } Log.Info(string.Format("EzImporter:{0} Columns defined in map.", args.Map.InputFields.Count), this); }
public void ReadData(ImportItemsArgs args) { Log.Info("EzImporter:Reading XSLX input data", this); try { IExcelDataReader excelReader; if (args.FileExtension == "xls") { excelReader = ExcelReaderFactory.CreateBinaryReader(args.FileStream, ReadOption.Loose); } else { excelReader = ExcelReaderFactory.CreateOpenXmlReader(args.FileStream); } excelReader.IsFirstRowAsColumnNames = args.ImportOptions.FirstRowAsColumnNames; if (!excelReader.IsValid) { Log.Error("EzImporter:Invalid Excel file '" + excelReader.ExceptionMessage + "'", this); return; } DataSet result = excelReader.AsDataSet(); if (result == null) { Log.Error("EzImporter:No data could be retrieved from Excel file.", this); } if (result.Tables == null || result.Tables.Count == 0) { Log.Error("EzImporter:No worksheets found in Excel file", this); return; } var readDataTable = result.Tables[0]; foreach (var readDataRow in readDataTable.AsEnumerable()) { var row = args.ImportData.NewRow(); for (int i = 0; i < args.Map.InputFields.Count; i++) { if (i < readDataTable.Columns.Count && readDataRow[i] != null) { row[i] = Convert.ToString(readDataRow[i]); } else { row[i] = ""; } } args.ImportData.Rows.Add(row); } Log.Info(string.Format("EzImporter:{0} records read from input data.", readDataTable.Rows.Count), this); } catch (Exception ex) { Log.Error("EzImporter:" + ex.ToString(), this); } }
public override void Process(ImportItemsArgs args) { var rootItem = new ItemDto("<root>"); OutputMapRegexs = new List <Regex>(); foreach (var map in args.Map.OutputMaps) { OutputMapRegexs.Add(new Regex(map.PathPattern, RegexOptions.Compiled)); } ImportMapItems(args, args.ImportData, args.Map.OutputMaps.First(), rootItem, true); args.ImportItems.AddRange(rootItem.Children); //ick }
public IHttpActionResult Import(ImportModel importModel) { var database = Sitecore.Configuration.Factory.GetDatabase("master"); var languageItem = database.GetItem(importModel.Language); var uploadedFile = (MediaItem)database.GetItem(importModel.MediaItemId); if (uploadedFile == null) { return new JsonResult<ImportResultModel>(null, new JsonSerializerSettings(), Encoding.UTF8, this); } var args = new ImportItemsArgs { Database = database, FileExtension = uploadedFile.Extension.ToLower(), FileStream = uploadedFile.GetMediaStream(), RootItemId = new ID(importModel.ImportLocationId), TargetLanguage = Sitecore.Globalization.Language.Parse(languageItem.Name), Map = Map.Factory.BuildMapInfo(new ID(importModel.MappingId)), ImportOptions = new ImportOptions { CsvDelimiter = new[] { importModel.CsvDelimiter }, ExistingItemHandling = (ExistingItemHandling) Enum.Parse(typeof(ExistingItemHandling), importModel.ExistingItemHandling), InvalidLinkHandling = (InvalidLinkHandling) Enum.Parse(typeof(InvalidLinkHandling), importModel.InvalidLinkHandling), MultipleValuesImportSeparator = importModel.MultipleValuesSeparator, TreePathValuesImportSeparator = @"\", FirstRowAsColumnNames = importModel.FirstRowAsColumnNames } }; ImportResultModel result; try { Sitecore.Diagnostics.Log.Info(string.Format("EzImporter: mappingId:{0} mediaItemId:{1} firstRowAsColumnNames:{2}", importModel.MappingId, importModel.MediaItemId, args.ImportOptions.FirstRowAsColumnNames), this); args.Timer.Start(); CorePipeline.Run("importItems", args); args.Timer.Stop(); if (args.Aborted) { result = new ImportResultModel { HasError = true, Log = args.Statistics.ToString(), ErrorMessage = args.Message, ErrorDetail = args.ErrorDetail }; } else { result = new ImportResultModel { Log = args.Statistics.ToString() + " Duration: " + args.Timer.Elapsed.ToString("c") }; } } catch (Exception ex) { result = new ImportResultModel { HasError = true, ErrorMessage = ex.Message, ErrorDetail = ex.ToString() }; } return new JsonResult<ImportResultModel>(result, new JsonSerializerSettings(), Encoding.UTF8, this); }
private void ImportMapItems(ImportItemsArgs args, DataTable dataTable, OutputMap outputMap, ItemDto parentItem, bool rootLevel, string parent = "") { switch (args.ImportOptions.DataStructureType) { case DataStructureType.Tabular: var groupedTable = dataTable.GroupBy(outputMap.Fields.Select(f => f.SourceColumn).ToArray()); for (int i = 0; i < groupedTable.Rows.Count; i++) { var row = groupedTable.Rows[i]; if (rootLevel || Convert.ToString(row[outputMap.ParentMap.NameInputField]) == parentItem.Name) { var createdItem = CreateItem(row, outputMap); createdItem.Parent = parentItem; parentItem.Children.Add(createdItem); if (outputMap.ChildMaps != null && outputMap.ChildMaps.Any()) { foreach (var childMap in outputMap.ChildMaps) { ImportMapItems(args, dataTable, childMap, createdItem, false); } } } } break; case DataStructureType.Hierarchichal: var currentleveltable = dataTable.Select("father='" + parent + "'"); foreach (var row in currentleveltable) { for (int i = 0; i < args.Map.OutputMaps.Count; i++) { var map = args.Map.OutputMaps[i]; var reg = OutputMapRegexs[i]; if (reg.IsMatch(row["Path"].ToString())) { var createdItem = CreateItem(row, map); createdItem.Parent = parentItem; parentItem.Children.Add(createdItem); ImportMapItems(args, dataTable, map, createdItem, false, row["ID"].ToString()); break; } } } break; default: break; } }
public override void Process(ImportItemsArgs args) { Errors = new List <string>(); foreach (var item in args.ImportItems) { ValidateName(item); } if (Errors.Any()) { args.AddMessage("Invalid item name(s) in import data."); args.ErrorDetail = string.Join("\n\n", Errors); args.AbortPipeline(); } }
public override void Process(ImportItemsArgs args) { Errors = new List<string>(); foreach (var item in args.ImportItems) { ValidateName(item); } if (Errors.Any()) { args.AddMessage("Invalid item name(s) in import data."); args.ErrorDetail = string.Join("\n\n", Errors); args.AbortPipeline(); } }
public override void Process(ImportItemsArgs args) { Log.Info("EzImporter:Validating input...", this); var argsValid = true; if (args.FileStream == null) { Log.Error("EzImporter:Input file not found.", this); argsValid = false; } if (!argsValid) { args.AddMessage("Error: Input file not found."); args.ErrorDetail = "FileStream = null"; args.AbortPipeline(); } }
public override void Process(ImportItemsArgs args) { var originalIndexingSetting = Sitecore.Configuration.Settings.Indexing.Enabled; Sitecore.Configuration.Settings.Indexing.Enabled = false; using (new BulkUpdateContext()) { using (new LanguageSwitcher(args.TargetLanguage)) { var parentItem = args.Database.GetItem(args.RootItemId); foreach (var importItem in args.ImportItems) { ImportItems(args, importItem, parentItem, true); } } } Sitecore.Configuration.Settings.Indexing.Enabled = originalIndexingSetting; }
private void ImportItems(ImportItemsArgs args, ItemDto importItem, Item parentItem, bool rootLevel) { if (rootLevel || importItem.Parent.Name == parentItem.Name) { var createdItem = CreateItem(args, importItem, parentItem); if (createdItem != null && importItem.Children != null && importItem.Children.Any()) { foreach (var childImportItem in importItem.Children) { ImportItems(args, childImportItem, createdItem, false); } } } }
private void ImportItems(ImportItemsArgs args, ItemDto importItem, Item parentItem, bool rootLevel) { if (rootLevel || string.Equals(importItem.Parent.Name, parentItem.Name, StringComparison.OrdinalIgnoreCase)) { var createdItem = CreateItem(args, importItem, parentItem); if (createdItem != null && importItem.Children != null && importItem.Children.Any()) { foreach (var childImportItem in importItem.Children) { ImportItems(args, childImportItem, createdItem, false); } } } }
public string[] GetColumnNames(ImportItemsArgs args) { Log.Info("EzImporter:Reading column names from input CSV file...", this); try { using (var reader = new StreamReader(args.FileStream)) { var line = reader.ReadLine(); if (line != null) { return line.Split(args.ImportOptions.CsvDelimiter, StringSplitOptions.None); } } } catch (Exception ex) { Log.Error("EzImporter:" + ex.ToString(), this); } return new string[] {}; }
public override void Process(ImportItemsArgs args) { DataReaders.IDataReader reader; if (args.FileExtension == "csv") { reader = new DataReaders.CsvDataReader(); } else if (args.FileExtension == "xlsx" || args.FileExtension == "xls") { reader = new DataReaders.XlsxDataReader(); } else { Log.Info("EzImporter:Unsupported file format supplied. DataImporter accepts *.CSV and *.XLSX files", this); return; } reader.ReadData(args); args.Statistics.InputDataRows = args.ImportData.Rows.Count; }
public string[] GetColumnNames(ImportItemsArgs args) { Log.Info("EzImporter:Reading column names from input XSLX file...", this); try { //1. Reading from a binary Excel file ('97-2003 format; *.xls) //IExcelDataReader excelReader = ExcelReaderFactory.CreateBinaryReader(stream); //2. Reading from a OpenXml Excel file (2007 format; *.xlsx) IExcelDataReader excelReader = ExcelReaderFactory.CreateOpenXmlReader(args.FileStream); excelReader.IsFirstRowAsColumnNames = true; //assume first line is data, so we can read it if (!excelReader.IsValid) { Log.Info("EzImporter:Invalid Excel file '" + excelReader.ExceptionMessage + "'", this); return new string[] {}; } DataSet result = excelReader.AsDataSet(); if (result == null) { Log.Info("EzImporter:No data could be retrieved from Excel file.", this); } if (result.Tables == null || result.Tables.Count == 0) { Log.Info("EzImporter:No worksheets found in Excel file", this); return new string[] {}; } var readDataTable = result.Tables[0]; return readDataTable.Columns .Cast<DataColumn>() .Select(c => c.ColumnName).ToArray(); } catch (Exception ex) { Log.Error("EzImporter:" + ex.ToString(), this); } return new string[] { }; }
public void ReadData(ImportItemsArgs args) { Log.Info("EzImporter:Reading CSV input data...", this); try { var reader = new StreamReader(args.FileStream); var lineCount = 0; do { var line = reader.ReadLine(); if (line != null) { var row = args.ImportData.NewRow(); var values = line.Split(args.ImportOptions.CsvDelimiter, StringSplitOptions.None); for (int j = 0; j < args.Map.InputFields.Count; j++) { if (j < values.Length) { row[j] = values[j]; } else { row[j] = ""; } } args.ImportData.Rows.Add(row); lineCount++; } } while (!reader.EndOfStream); Log.Info(string.Format("EzImporter:{0} records read from input data.", lineCount), this); } catch (Exception ex) { Log.Error("EzImporter:" + ex.ToString(), this); } }
private Item CreateItem(ImportItemsArgs args, ItemDto importItem, Item parentItem) { //CustomItemBase nItemTemplate = GetNewItemTemplate(dataRow); var templateItem = args.Database.GetTemplate(importItem.TemplateId); //get the parent in the specific language Item parent = args.Database.GetItem(parentItem.ID); Item item; //search for the child by name item = parent.GetChildren()[importItem.Name]; if (item != null) { if (args.ImportOptions.ExistingItemHandling == ExistingItemHandling.AddVersion) { args.Statistics.UpdatedItems++; item = item.Versions.AddVersion(); Log.Info(string.Format("EzImporter:Creating new version of item {0}", item.Paths.ContentPath), this); } else if (args.ImportOptions.ExistingItemHandling == ExistingItemHandling.Skip) { Log.Info(string.Format("EzImporter:Skipping update of item {0}", item.Paths.ContentPath), this); return(item); } else if (args.ImportOptions.ExistingItemHandling == ExistingItemHandling.Update) { //continue to update current item/version args.Statistics.UpdatedItems++; } } else { //if not found then create one args.Statistics.CreatedItems++; item = parent.Add(importItem.Name, templateItem); Log.Info(string.Format("EzImporter:Creating item {0}", item.Paths.ContentPath), this); } if (item == null) { throw new NullReferenceException("the new item created was null"); } using (new EditContext(item, true, false)) { //add in the field mappings foreach (var key in importItem.Fields.Keys) { var fieldValue = importItem.Fields[key]; var field = item.Fields[key]; if (field != null) { FieldUpdateManager.UpdateField(field, fieldValue, args.ImportOptions); Log.Info(string.Format("'{0}' field set to '{1}'", key, fieldValue), this); } else { Log.Info( string.Format( "EzImporter:Field '{0}' not found on item, skipping update for this field", key), this); } } return(item); } }
private Item CreateItem(ImportItemsArgs args, ItemDto importItem, Item parentItem) { //CustomItemBase nItemTemplate = GetNewItemTemplate(dataRow); var templateItem = args.Database.GetTemplate(importItem.TemplateId); //get the parent in the specific language Item parent = args.Database.GetItem(parentItem.ID); Item item; //search for the child by name item = parent.GetChildren()[importItem.Name]; if (item != null) { if (args.ImportOptions.ExistingItemHandling == ExistingItemHandling.AddVersion) { args.Statistics.UpdatedItems++; item = item.Versions.AddVersion(); Log.Info(string.Format("EzImporter:Creating new version of item {0}", item.Paths.ContentPath), this); } else if (args.ImportOptions.ExistingItemHandling == ExistingItemHandling.Skip) { Log.Info(string.Format("EzImporter:Skipping update of item {0}", item.Paths.ContentPath), this); return item; } else if (args.ImportOptions.ExistingItemHandling == ExistingItemHandling.Update) { //continue to update current item/version args.Statistics.UpdatedItems++; } } else { //if not found then create one args.Statistics.CreatedItems++; item = parent.Add(importItem.Name, templateItem); Log.Info(string.Format("EzImporter:Creating item {0}", item.Paths.ContentPath), this); } if (item == null) { throw new NullReferenceException("the new item created was null"); } using (new EditContext(item, true, false)) { //add in the field mappings foreach (var key in importItem.Fields.Keys) { var fieldValue = importItem.Fields[key]; var field = item.Fields[key]; if (field != null) { FieldUpdateManager.UpdateField(field, fieldValue, args.ImportOptions); Log.Info(string.Format("'{0}' field set to '{1}'", key, fieldValue), this); } else { Log.Info( string.Format( "EzImporter:Field '{0}' not found on item, skipping update for this field", key), this); } } return item; } }
public void Run(Item[] items, CommandItem command, ScheduleItem schedule) { var importCommand = new ImportCommandItem(command.InnerItem); Log.Info( "EzImporter.Tasks.Import.Run() Starting Import: " + " fileName=" + importCommand.FileName + " lang=" + importCommand.TargetLanguage + " location=" + importCommand.ImportLocationId + " importMap=" + importCommand.ImportMapId + " database=" + importCommand.Database.Name + " csvDelimiter=" + importCommand.CsvDelimiter + " ExistingItemHandling=" + importCommand.ExistingItemHandling + " InvalidLinkHandling=" + importCommand.InvalidLinkHandling + " MultipleValuesImportSeparator=" + importCommand.MultipleValuesImportSeparator + " TreePathValuesImportSeparator=" + importCommand.TreePathValuesImportSeparator, this); var options = Factory.GetDefaultImportOptions(); if (importCommand.CsvDelimiter != null) { options.CsvDelimiter = new[] {importCommand.CsvDelimiter}; } if (importCommand.ExistingItemHandling != null) { options.ExistingItemHandling = (ExistingItemHandling) Enum.Parse(typeof (ExistingItemHandling), importCommand.ExistingItemHandling); } if (importCommand.InvalidLinkHandling != null) { options.InvalidLinkHandling = (InvalidLinkHandling) Enum.Parse(typeof (InvalidLinkHandling), importCommand.InvalidLinkHandling); } if (importCommand.MultipleValuesImportSeparator != null) { options.MultipleValuesImportSeparator = importCommand.MultipleValuesImportSeparator; } if (importCommand.TreePathValuesImportSeparator != null) { options.TreePathValuesImportSeparator = importCommand.TreePathValuesImportSeparator; } options.FirstRowAsColumnNames = importCommand.FirstRowAsColumnNames; if (string.IsNullOrWhiteSpace(importCommand.FileName)) { Log.Error( "EzImporter.Tasks.Import.Run() - Import Error: File not specified", this); return; } string fileName; if (File.Exists(importCommand.FileName)) { fileName = importCommand.FileName; } else { fileName = HostingEnvironment.MapPath(importCommand.FileName); if (!File.Exists(fileName)) { Log.Error( "EzImporter.Tasks.Import.Run() - Import Error: File not found (" + importCommand.FileName + ")", this); return; } } var extension = GetFileExtension(fileName); if (extension == null) { Log.Error( "EzImporter.Tasks.Import.Run() - Import Error: Unknown file extension (" + importCommand.FileName + ")", this); return; } var stream = new StreamReader(fileName); var args = new ImportItemsArgs { Database = importCommand.ImportDatabase, FileExtension = extension, FileStream = stream.BaseStream, RootItemId = importCommand.ImportLocationId, TargetLanguage = importCommand.TargetLanguage, Map = Map.Factory.BuildMapInfo(importCommand.ImportMapId), ImportOptions = options }; try { CorePipeline.Run("importItems", args); Log.Info("EzImporter.Tasks.Import.Run() Import Finished " + args.Statistics, this); } catch (Exception ex) { Log.Error("EzImporter.Tasks.Import.Run() - Import Error: " + ex, this); throw; } }