public void ReadDataExtend(ImportItemsArgs args)
{
Log.Info("Sitecore.Foundation.Import:Reading XSLX input data", this);
try
{
IExcelDataReader excelReader;
if (args.FileExtension.Equals(Constants.ExcelFileExtention, StringComparison.OrdinalIgnoreCase))
{
excelReader = ExcelReaderFactory.CreateBinaryReader(args.FileStream, ReadOption.Loose);
}
else
{
excelReader = ExcelReaderFactory.CreateOpenXmlReader(args.FileStream);
}
excelReader.IsFirstRowAsColumnNames = args.ImportOptions.FirstRowAsColumnNames;
if (!excelReader.IsValid)
{
Log.Error("Sitecore.Foundation.Import:Invalid Excel file '" + excelReader.ExceptionMessage + "'", this);
return;
}
ConvertDataToDatatable(excelReader, args);
}
catch (Exception ex)
{
Log.Error("Sitecore.Foundation.Import:" + ex.ToString(), this);
}
}
public void ReadData(ImportItemsArgs args)
{
Log.Info("EzImporter:Reading Ziped folder input data", this);
try
{
//IExcelDataReader excelReader;
//if (args.FileExtension == "xls")
//{
// excelReader = ExcelReaderFactory.CreateBinaryReader(args.FileStream, ReadOption.Loose);
//}
//else
//{
// excelReader = ExcelReaderFactory.CreateOpenXmlReader(args.FileStream);
//}
FileInfo fi = new FileInfo(@"C:\darcy-out\www.shoosmiths.co.uk.zip");
string folder = Decompress(fi);
PreProcessFilesRecursive(@"C:\darcy-out\www.shoosmiths.co.uk");
getFilesRecursive(@"C:\darcy-out\www.shoosmiths.co.uk", args);
//excelReader.IsFirstRowAsColumnNames = args.ImportOptions.FirstRowAsColumnNames;
//if (!excelReader.IsValid)
//{
// Log.Error("EzImporter:Invalid Excel file '" + excelReader.ExceptionMessage + "'", this);
// return;
//}
//DataSet result = excelReader.AsDataSet();
//if (result == null)
//{
// Log.Error("EzImporter:No data could be retrieved from Excel file.", this);
//}
//if (result.Tables == null || result.Tables.Count == 0)
//{
// Log.Error("EzImporter:No worksheets found in Excel file", this);
// return;
//}
//var readDataTable = result.Tables[0];
//foreach (var readDataRow in readDataTable.AsEnumerable())
//{
// var row = args.ImportData.NewRow();
// for (int i = 0; i < args.Map.InputFields.Count; i++)
// {
// if (i < readDataTable.Columns.Count && readDataRow[i] != null)
// {
// row[i] = Convert.ToString(readDataRow[i]);
// }
// else
// {
// row[i] = "";
// }
// }
// args.ImportData.Rows.Add(row);
//}
//Log.Info(string.Format("EzImporter:{0} records read from input data.", readDataTable.Rows.Count), this);
}
catch (Exception ex)
{
Log.Error("EzImporter:" + ex.ToString(), this);
}
}
public void Process_Aborted()
{
var validateArgs = new ValidateArgs();
var args = new ImportItemsArgs {
FileStream = null
};
validateArgs.Process(args);
Assert.True(args.Aborted);
}
public void ReadData(ImportItemsArgs args)
{
Log.Info("EzImporter:Reading XSLX input data", this);
try
{
IExcelDataReader excelReader;
if (args.FileExtension == "xls")
{
excelReader = ExcelReaderFactory.CreateBinaryReader(args.FileStream, ReadOption.Loose);
}
else
{
excelReader = ExcelReaderFactory.CreateOpenXmlReader(args.FileStream);
}
excelReader.IsFirstRowAsColumnNames = args.ImportOptions.FirstRowAsColumnNames;
if (!excelReader.IsValid)
{
Log.Error("EzImporter:Invalid Excel file '" + excelReader.ExceptionMessage + "'", this);
return;
}
DataSet result = excelReader.AsDataSet();
if (result == null)
{
Log.Error("EzImporter:No data could be retrieved from Excel file.", this);
}
if (result.Tables == null || result.Tables.Count == 0)
{
Log.Error("EzImporter:No worksheets found in Excel file", this);
return;
}
var readDataTable = result.Tables[0];
foreach (var readDataRow in readDataTable.AsEnumerable())
{
var row = args.ImportData.NewRow();
for (int i = 0; i < args.Map.InputFields.Count; i++)
{
if (i < readDataTable.Columns.Count && readDataRow[i] != null)
{
row[i] = Convert.ToString(readDataRow[i]);
}
else
{
row[i] = "";
}
}
args.ImportData.Rows.Add(row);
}
Log.Info(string.Format("EzImporter:{0} records read from input data.", readDataTable.Rows.Count), this);
}
catch (Exception ex)
{
Log.Error("EzImporter:" + ex.ToString(), this);
}
}
public void ReadData(ImportItemsArgs args)
{
Log.Info("EzImporter:Reading CSV input data...", this);
try
{
var reader = new StreamReader(args.FileStream);
var insertLineCount = 0;
var readLineCount = 0;
do
{
var line = reader.ReadLine();
readLineCount++;
if (line == null ||
(readLineCount == 1 && args.ImportOptions.FirstRowAsColumnNames))
{
continue;
}
var row = args.ImportData.NewRow();
var csvDelimiters = new HashSet <char>();
foreach (var delimiter in args.ImportOptions.CsvDelimiter)
{
foreach (var delimiterChar in delimiter)
{
csvDelimiters.Add(delimiterChar);
}
}
var quotationMark = (!string.IsNullOrEmpty(args.ImportOptions.QuotationMark) ? args.ImportOptions.QuotationMark[0]:'"');
var values = ParseLine(line, csvDelimiters, quotationMark);
for (int j = 0; j < args.Map.InputFields.Count; j++)
{
if (j < values.Length)
{
row[j] = values[j];
}
else
{
row[j] = "";
}
}
args.ImportData.Rows.Add(row);
insertLineCount++;
} while (!reader.EndOfStream);
Log.Info(string.Format("EzImporter:{0} records read from input data.", insertLineCount), this);
}
catch (Exception ex)
{
Log.Error("EzImporter:" + ex.ToString(), this);
}
}
private void ConvertDataToDatatable(IExcelDataReader excelReader, ImportItemsArgs args)
{
DataSet result = excelReader.AsDataSet();
if (result == null)
{
Log.Error("Sitecore.Foundation.Import:No data could be retrieved from Excel file.", this);
return;
}
if (result.Tables == null || result.Tables.Count == 0)
{
Log.Error("Sitecore.Foundation.Import:No worksheets found in Excel file", this);
return;
}
DataTable readDataTable = null;
DataTable readDataTable1 = null;
if (args.ContentType == ContentType.Article.ToString())
{
readDataTable = ConvertArticleTable(result, out readDataTable1);
if (readDataTable1 == null)
{
readDataTable1 = new DataTable();
}
}
else
{
readDataTable = result.Tables[0];
}
if (readDataTable == null)
{
readDataTable = new DataTable();
}
args.ImportDatas.Add(readDataTable);
if (readDataTable1 != null)
{
args.ImportDatas.Add(readDataTable1);
}
var countRow = readDataTable.Rows.Count;
countRow = readDataTable1 != null ? countRow + readDataTable1.Rows.Count : countRow;
Log.Info(string.Format("Sitecore.Foundation.Import:{0} records read from input data.", countRow), this);
}
public string[] GetColumnNames(ImportItemsArgs args)
{
Log.Info("EzImporter:Reading column names from input CSV file...", this);
try
{
using (var reader = new StreamReader(args.FileStream))
{
var line = reader.ReadLine();
if (line != null)
{
return(line.Split(args.ImportOptions.CsvDelimiter, StringSplitOptions.None));
}
}
}
catch (Exception ex)
{
Log.Error("EzImporter:" + ex.ToString(), this);
}
return(new string[] {});
}
public void ReadData(ImportItemsArgs args)
{
Log.Info("EzImporter:Reading CSV input data...", this);
try
{
var reader = new StreamReader(args.FileStream);
var insertLineCount = 0;
var readLineCount = 0;
do
{
var line = reader.ReadLine();
readLineCount++;
if (line == null ||
(readLineCount == 1 && args.ImportOptions.FirstRowAsColumnNames))
{
continue;
}
var row = args.ImportData.NewRow();
var values = line.Split(args.ImportOptions.CsvDelimiter, StringSplitOptions.None);
for (int j = 0; j < args.Map.InputFields.Count; j++)
{
if (j < values.Length)
{
row[j] = values[j];
}
else
{
row[j] = "";
}
}
args.ImportData.Rows.Add(row);
insertLineCount++;
} while (!reader.EndOfStream);
Log.Info(string.Format("EzImporter:{0} records read from input data.", insertLineCount), this);
}
catch (Exception ex)
{
Log.Error("EzImporter:" + ex.ToString(), this);
}
}
public static Item SearchChildItem(ImportItemsArgs args, Item parent, string itemName, TemplateItem templateItem, ImportItemsProcessor processor, Action <Item, string> updateDisplayName = null, string displayName = null)
{
// search for the child by name
Item newItem = parent.GetChildren()[itemName];
if (newItem != null)
{
if (args.ImportOptions.ExistingItemHandling == ExistingItemHandling.AddVersion)
{
args.Statistics.UpdatedItems++;
newItem = newItem.Versions.AddVersion();
Log.Info(string.Format("Sitecore.Foundation.Import:Creating new version of item {0}", newItem.Paths.ContentPath), processor);
}
else if (args.ImportOptions.ExistingItemHandling == ExistingItemHandling.Skip)
{
Log.Info(string.Format("Sitecore.Foundation.Import:Skipping update of item {0}", newItem.Paths.ContentPath), processor);
return(newItem);
}
else if (args.ImportOptions.ExistingItemHandling == ExistingItemHandling.Update)
{
//continue to update current item/version
args.Statistics.UpdatedItems++;
}
}
else
{
//if not found then create one
args.Statistics.CreatedItems++;
newItem = parent.Add(itemName, templateItem);
Log.Info(string.Format("Sitecore.Foundation.Import:Creating item {0}", newItem.Paths.ContentPath), processor);
//update displayname
if (updateDisplayName != null && displayName != null)
{
updateDisplayName(newItem, displayName);
}
}
return(newItem);
}
public string[] GetColumnNames(ImportItemsArgs args)
{
Log.Info("Sitecore.Foundation.Import:Reading column names from input XSLX file...", this);
try
{
//1. Reading from a binary Excel file ('97-2003 format; *.xls)
//2. Reading from a OpenXml Excel file (2007 format; *.xlsx)
IExcelDataReader excelReader = ExcelReaderFactory.CreateOpenXmlReader(args.FileStream);
excelReader.IsFirstRowAsColumnNames = true; //assume first line is data, so we can read it
if (!excelReader.IsValid)
{
Log.Info("Sitecore.Foundation.Import:Invalid Excel file '" + excelReader.ExceptionMessage + "'", this);
return(new string[] {});
}
DataSet result = excelReader.AsDataSet();
if (result == null)
{
Log.Info("Sitecore.Foundation.Import:No data could be retrieved from Excel file.", this);
return(new string[] { });
}
if (result.Tables == null || result.Tables.Count == 0)
{
Log.Info("Sitecore.Foundation.Import:No worksheets found in Excel file", this);
return(new string[] {});
}
var readDataTable = result.Tables[0];
return(readDataTable.Columns
.Cast <DataColumn>()
.Select(c => c.ColumnName).ToArray());
}
catch (Exception ex)
{
Log.Error("Sitecore.Foundation.Import:" + ex.ToString(), this);
}
return(new string[] { });
}
public IHttpActionResult Import(ImportModel importModel)
{
var database = Sitecore.Configuration.Factory.GetDatabase("master");
var languageItem = database.GetItem(importModel.Language);
var uploadedFile = (MediaItem)database.GetItem(importModel.MediaItemId);
if (uploadedFile == null)
{
return(new JsonResult <ImportResultModel>(null, new JsonSerializerSettings(), Encoding.UTF8, this));
}
ImportResultModel result;
try
{
var args = new ImportItemsArgs
{
Database = database,
FileExtension = uploadedFile.Extension.ToLower(),
FileStream = uploadedFile.GetMediaStream(),
RootItemId = new ID(importModel.ImportLocationId),
TargetLanguage = Sitecore.Globalization.Language.Parse(languageItem.Name),
Map = Map.Factory.BuildMapInfo(new ID(importModel.MappingId)),
ImportOptions = new ImportOptions
{
CsvDelimiter = new[] { importModel.CsvDelimiter },
QuotationMark = importModel.QuotationMark,
MultipleValuesImportSeparator = importModel.MultipleValuesSeparator,
TreePathValuesImportSeparator = @"\",
FirstRowAsColumnNames = importModel.FirstRowAsColumnNames
}
};
args.ImportOptions.ExistingItemHandling = (ExistingItemHandling)
Enum.Parse(typeof(ExistingItemHandling), importModel.ExistingItemHandling);
args.ImportOptions.InvalidLinkHandling = (InvalidLinkHandling)
Enum.Parse(typeof(InvalidLinkHandling), importModel.InvalidLinkHandling);
args.ImportOptions.DataStructureType = (DataStructureType)
Enum.Parse(typeof(DataStructureType), importModel.DataStructureType);
Sitecore.Diagnostics.Log.Info(
string.Format("EzImporter: mappingId:{0} mediaItemId:{1} firstRowAsColumnNames:{2}",
importModel.MappingId, importModel.MediaItemId, args.ImportOptions.FirstRowAsColumnNames),
this);
args.Timer.Start();
CorePipeline.Run("importItems", args);
args.Timer.Stop();
if (args.Aborted)
{
result = new ImportResultModel
{
HasError = true,
Log = args.Statistics.ToString(),
ErrorMessage = args.Message,
ErrorDetail = args.ErrorDetail
};
}
else
{
result = new ImportResultModel
{
Log = args.Statistics.ToString() + " Duration: " + args.Timer.Elapsed.ToString("c")
};
}
}
catch (Exception ex)
{
result = new ImportResultModel
{
HasError = true,
ErrorMessage = ex.Message,
ErrorDetail = ex.ToString()
};
}
return(new JsonResult <ImportResultModel>(result, new JsonSerializerSettings(), Encoding.UTF8, this));
}
public string[] GetColumnNames(ImportItemsArgs args)
{
return(new string[0]);
}
private void getFilesRecursive(string sDir, ImportItemsArgs args, string parent = "")
{
try
{
DirectoryInfo sDirinfo = new DirectoryInfo(sDir);
IList <string> skipfolders = new List <string>();
skipfolders.Add(@"files");
skipfolders.Add(@"images");
skipfolders.Add(@"webvirtuals");
var mainroots = sDirinfo.GetFiles("Index.aspx");
if (mainroots != null && mainroots.Count() == 1)
{
var row = args.ImportData.NewRow();
row["Name"] = sDirinfo.Name.Replace("-", " ").Trim();
row["father"] = parent;
row["ID"] = Guid.NewGuid();
parent = row["ID"].ToString();
row["Path"] = mainroots[0].FullName;//.Replace(sDir, "");
PopulateContent(mainroots[0].FullName, row, args);
}
foreach (string d in Directory.GetDirectories(sDir))
{
Log.Info(d, this);
var row = args.ImportData.NewRow();
DirectoryInfo di = new DirectoryInfo(d);
if (!skipfolders.Contains(di.Name))
{
row["Name"] = di.Name.Replace("-", " ").Trim();
row["father"] = parent;
row["ID"] = Guid.NewGuid();
row["Path"] = di.FullName;//.Replace(sDir, "");
var roots = di.GetFiles("Index.aspx");
if (roots != null && roots.Count() == 1)
{
PopulateContent(roots[0].FullName, row, args);
}
else
{
args.ImportData.Rows.Add(row);
}
getFilesRecursive(d, args, row["ID"].ToString());
}
}
int filecounter = 0;
foreach (var file in Directory.GetFiles(sDir, "*.aspx*"))
{
Log.Info(file, this);
var row = args.ImportData.NewRow();
row["father"] = parent;
row["ID"] = Guid.NewGuid();
row["Path"] = file;//.Replace(sDir, "");
PopulateContent(file, row, args);
//if (++filecounter >= 10)
// break;
}
}
catch (System.Exception e)
{
Log.Error(e.Message, this);
}
}
public abstract void Process(ImportItemsArgs args);
public void Run(Item[] items, CommandItem command, ScheduleItem schedule)
{
var importCommand = new ImportCommandItem(command.InnerItem);
Log.Info(
"EzImporter.Tasks.Import.Run() Starting Import: "
+ " fileName=" + importCommand.FileName
+ " lang=" + importCommand.TargetLanguage
+ " location=" + importCommand.ImportLocationId
+ " importMap=" + importCommand.ImportMapId
+ " database=" + importCommand.Database.Name
+ " csvDelimiter=" + importCommand.CsvDelimiter
+ " ExistingItemHandling=" + importCommand.ExistingItemHandling
+ " InvalidLinkHandling=" + importCommand.InvalidLinkHandling
+ " MultipleValuesImportSeparator=" + importCommand.MultipleValuesImportSeparator
+ " TreePathValuesImportSeparator=" + importCommand.TreePathValuesImportSeparator, this);
var options = Factory.GetDefaultImportOptions();
if (importCommand.CsvDelimiter != null)
{
options.CsvDelimiter = new[] { importCommand.CsvDelimiter };
}
if (importCommand.ExistingItemHandling != null)
{
options.ExistingItemHandling = (ExistingItemHandling)
Enum.Parse(typeof(ExistingItemHandling), importCommand.ExistingItemHandling);
}
if (importCommand.InvalidLinkHandling != null)
{
options.InvalidLinkHandling = (InvalidLinkHandling)
Enum.Parse(typeof(InvalidLinkHandling), importCommand.InvalidLinkHandling);
}
if (importCommand.MultipleValuesImportSeparator != null)
{
options.MultipleValuesImportSeparator = importCommand.MultipleValuesImportSeparator;
}
if (importCommand.TreePathValuesImportSeparator != null)
{
options.TreePathValuesImportSeparator = importCommand.TreePathValuesImportSeparator;
}
options.FirstRowAsColumnNames = importCommand.FirstRowAsColumnNames;
if (string.IsNullOrWhiteSpace(importCommand.FileName))
{
Log.Error(
"EzImporter.Tasks.Import.Run() - Import Error: File not specified",
this);
return;
}
string fileName;
if (File.Exists(importCommand.FileName))
{
fileName = importCommand.FileName;
}
else
{
fileName = HostingEnvironment.MapPath(importCommand.FileName);
if (!File.Exists(fileName))
{
Log.Error(
"EzImporter.Tasks.Import.Run() - Import Error: File not found (" + importCommand.FileName + ")",
this);
return;
}
}
var extension = GetFileExtension(fileName);
if (extension == null)
{
Log.Error(
"EzImporter.Tasks.Import.Run() - Import Error: Unknown file extension (" + importCommand.FileName +
")", this);
return;
}
var stream = new StreamReader(fileName);
var args = new ImportItemsArgs
{
Database = importCommand.ImportDatabase,
FileExtension = extension,
FileStream = stream.BaseStream,
RootItemId = importCommand.ImportLocationId,
TargetLanguage = importCommand.TargetLanguage,
Map = Map.Factory.BuildMapInfo(importCommand.ImportMapId),
ImportOptions = options
};
try
{
CorePipeline.Run("importItems", args);
Log.Info("EzImporter.Tasks.Import.Run() Import Finished " + args.Statistics, this);
}
catch (Exception ex)
{
Log.Error("EzImporter.Tasks.Import.Run() - Import Error: " + ex, this);
throw;
}
}
public abstract void Process(ImportItemsArgs args);
private void PopulateContent(string file, DataRow row, ImportItemsArgs args)
{
FileInfo fi = new FileInfo(file);
var doc = new HtmlDocument();
doc.Load(file, true);
if (doc.DocumentNode != null)
{
foreach (var field in args.Map.InputFields)
{
if (field.Name.Equals("name", StringComparison.InvariantCultureIgnoreCase))
{
row[field.Name] = GetCleanName(fi.Name, field);
}
else if (field.Name.Equals("originalvalue", StringComparison.InvariantCultureIgnoreCase))
{
row[field.Name] = doc.DocumentNode.InnerHtml;
}
else
{
if (!string.IsNullOrWhiteSpace(field.XsltSelector))
{
var nodes = doc.DocumentNode.SelectNodes(field.XsltSelector);
row[field.Name] = string.Empty;
if (nodes != null && nodes.Any())
{
foreach (var node in nodes)
{
if (node != null)
{
if (!string.IsNullOrWhiteSpace(field.Property))
{
var content = node.Attributes[field.Property];
if (content != null)
{
row[field.Name] += GetCleanContent(content, field);
}
}
else
{
var content = node.Attributes["content"];
if (content != null)
{
row[field.Name] += GetCleanContent(content, field);
}
else
{
row[field.Name] += GetCleanContent(node, field);
}
}
}
}
}
}
}
//
}
args.ImportData.Rows.Add(row);
doc = null;
}
}
