public directAnalyticReporter(string reportName, folderNode reportRootDir, aceAuthorNotation __notation) : base(reportName, reportRootDir, __notation)
{
if (imbWEMManager.settings.directReportEngine.doDomainReport)
{
siteRecords = new folderNodeForInstances <modelSpiderSiteRecord>(folder[DRFolderEnum.sites], getSiteName);
}
if (imbWEMManager.settings.directReportEngine.doIterationReport)
{
iterationRecords = new folderNodeForInstances <int>(folder[DRFolderEnum.it], getIterationName);
}
REPORT_DOMAIN_PAGES = imbWEMManager.settings.directReportEngine.DR_ReportDomainPages;
REPORT_DOMAIN_TERMS = imbWEMManager.settings.directReportEngine.DR_ReportDomainTerms;
REPORT_ITERATION_TERMS = imbWEMManager.settings.directReportEngine.DR_ReportIterationTerms;
REPORT_ITERATION_URLS = imbWEMManager.settings.directReportEngine.DR_ReportIterationUrls;
REPORT_WRECORD_LOG = imbWEMManager.settings.directReportEngine.DR_ReportWRecordLog;
REPORT_TIMELINE = imbWEMManager.settings.directReportEngine.DR_ReportTimeline;
REPORT_MODULES = imbWEMManager.settings.directReportEngine.DR_ReportModules;
aceLog.consoleControl.setLogFileWriter(folder[DRFolderEnum.logs].pathFor("log.txt"));
/*
* TextWriter logOut = File.CreateText();
* aceLog.consoleControl.logWritter = logOut;
* aceLog.consoleControl.logFileWriteOn = true;
*/
}
/// <summary>
/// Loads the BibTex file and converts it to Excel
/// </summary>
/// <param name="filePath">The file path.</param>
/// <param name="author">The author.</param>
/// <param name="log">The log.</param>
/// <param name="customTemplate">Custom data annotation template, to be used for Excel file generation. If not specified, default <see cref="template"/> is used</param>
/// <returns>Extended DataTable that is written in Excel file</returns>
public static DataTableForStatistics ExportToExcel(String filePath, aceAuthorNotation author, ILogBuilder log = null, propertyAnnotationPreset customTemplate = null, folderNode outputFolder = null)
{
BibTexDataFile bibFile = new BibTexDataFile(filePath);
FileInfo fi = new FileInfo(filePath);
if (outputFolder == null)
{
outputFolder = fi.Directory;
}
if (customTemplate == null)
{
customTemplate = template;
}
DataTable reportTable = bibFile.ConvertToDataTable(null, customTemplate, log); //.GetReportAndSave(fi.Directory, author, bibFile.name);
DataTableForStatistics report = reportTable.GetReportAndSave(outputFolder, author);
if (log != null)
{
log.log("BibTex [" + filePath + "] -> Excel [" + report.lastFilePath + "]");
}
return(report);
}
//public List<TaskPropertyValidation> ValidPropertyList { get; set; } = new List<TaskPropertyValidation>();
//public List<TaskPropertyValidation> InValidPropertyList { get; set; } = new List<TaskPropertyValidation>();
// public TaskPropertyDictionary PropertyDictionary { get; set; } = new TaskPropertyDictionary();
public void Publish(folderNode folder, aceAuthorNotation notation)
{
objectSerialization.saveObjectToXML(task, folder.pathFor(task.name + "_declaration.xml", imbSCI.Data.enums.getWritableFileMode.overwrite, "Declaration of task [" + task.name + "]"));
ReportTable_PropertyValudation = PropertyValidation.GetReportTable(this);
var pvr = PropertyValidation.GetResults();
foreach (var pv in pvr[ValidationOutcome.Invalid])
{
reporter.AppendLine(pv.item.PropertyName + " : " + pv.item.DisplayName);
reporter.AppendLine(pv.Outcome + " : " + pv.Message);
reporter.nextTabLevel();
reporter.AppendParagraph(pv.reporter.GetContent());
reporter.prevTabLevel();
}
reporter.ReportSave(folder, task.name + "_validation_log.txt");
task.ExtractorCustomizationSettings.ReportSave(folder, task.name + "_settings", "Custom settings for extractor [" + task.ExtractorName + "] stored in task [" + task.name + "]");
//foreach (var t in ReportTables)
//{
// t.GetReportAndSave(folder, notation, task.name);
//}
}
/// <summary>
/// Generates the readme files for complete folder tree
/// </summary>
/// <param name="notation">The notation data object</param>
/// <param name="readmeFileName">Overrides the default readme file name, defined by <see cref="directory_readme_filename" />.</param>
/// <param name="directoryStructureDepthLimit">The directory structure depth limit.</param>
public void generateReadmeFiles(aceAuthorNotation notation, String readmeFileName = "", Int32 directoryStructureDepthLimit = 3)
{
if (readmeFileName.isNullOrEmpty())
{
readmeFileName = directory_readme_filename;
}
lock (GenerateReadmeLock)
{
String readme_filename = readmeFileName;
String mpath = this.pathFor(readme_filename); //imbSciStringExtensions.add(path, readme_filename, "\\");
generateFolderReadme(notation).saveStringToFile(mpath);
var folderNodes = this.getAllChildrenInType <folderNode>();
foreach (folderNode ni in folderNodes)
{
String path = ni.pathFor(readme_filename);
FileInfo fi = path.getWritableFile(getWritableFileMode.overwrite);
String content = ni.generateFolderReadme(notation, null, directoryStructureDepthLimit);
content.saveStringToFile(fi.FullName, getWritableFileMode.overwrite);
saveBase.saveToFile(fi.FullName, content);
}
}
}
public static DataSet Save(this DataSet source, folderNode folder, aceAuthorNotation notation = null, string filenamePrefix = "")
{
string path = source.serializeDataSet(filenamePrefix, folder, dataTableExportEnum.excel, notation); //.serializeDataTable(dataTableExportEnum.excel, filenamePrefix, folder, notation);
//source.serializeDataTable(enums.dataTableExportEnum.excel, filenamePrefix + "_source", folder, notation);
return(source);
}
public DataTableForStatisticsExportJob(DataTable _source, folderNode _folder, aceAuthorNotation _notation = null, string _filenamePrefix = "", bool _disablePrimaryKey = true)
{
source = _source;
folder = _folder;
notation = _notation;
filenamePrefix = _filenamePrefix;
disablePrimaryKey = _disablePrimaryKey;
}
/// <summary>
/// Initializes a new instance of the <see cref="directReporterBase"/> class.
/// </summary>
/// <param name="reportName">Name of the report.</param>
/// <param name="reportRootDir">The report root dir.</param>
/// <param name="__parent">The parent.</param>
/// <param name="__notation">The notation.</param>
public directReporterBase(string reportName, string reportRootDir, aceAuthorNotation __notation)
{
name = reportName;
rootPath = reportRootDir;
reportPath = rootPath.add(reportName, "\\");
// parent = __parent;
notation = __notation;
deploy();
}
public directReporterBase(string reportName, folderNode _folder, aceAuthorNotation __notation)
{
name = reportName;
rootPath = _folder.path.getPathVersion(1, "\\"); // ; //reportRootDir;
reportPath = _folder.path; // rootPath.add(reportName, "\\");
//parent = __parent;
notation = __notation;
deploy();
}
public static DataTable Save(this DataTable source, folderNode folder, aceAuthorNotation notation = null, string filenamePrefix = "")
{
if (source is DataTable)
{
// source.SetDefaults();
}
string path = source.serializeDataTable(dataTableExportEnum.excel, filenamePrefix, folder, notation);
//source.serializeDataTable(enums.dataTableExportEnum.excel, filenamePrefix + "_source", folder, notation);
return(source);
}
/// <summary>
/// Deploys the specified report options.
/// </summary>
/// <param name="_reportOptions">The report options.</param>
/// <param name="_signature">The signature.</param>
/// <param name="_averagingMethod">The averaging method.</param>
/// <param name="_rootFolder">The root folder.</param>
/// <param name="_experimentDescription">The experiment description.</param>
/// <param name="logger">The logger.</param>
public void Deploy(aceAuthorNotation _signature, classificationMetricComputation _averagingMethod, folderNode _rootFolder, String _experimentDescription, ILogBuilder logger)
{
// reportOptions = _reportOptions;
signature = _signature;
averagingMethod = _averagingMethod;
description = _experimentDescription;
experimentRootFolder = _rootFolder;
notes = new ToolkitExperimentNotes(experimentRootFolder, _experimentDescription);
notes.log("Starting [" + runName + "]");
notes.log("[" + description + "]:[" + signature + "]");
//ToolkitExperimentNotes.[" + _experimentDescription + "]:[" + signature + "]");
imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(notes, "Note");
}
#pragma warning disable CS1574 // XML comment has cref attribute 'generateReadmeFiles(aceAuthorNotation, string)' that could not be resolved
/// <summary>
/// Sets the folder description
/// </summary>
/// <param name="generateReadme">If true it will call <see cref="folderNode.generateReadmeFiles(aceAuthorNotation, string)"/> after description set.</param>
/// <param name="notation">Information on author or application</param>
public void SetFolderDescription(Boolean generateReadme = false, aceAuthorNotation notation = null)
#pragma warning restore CS1574 // XML comment has cref attribute 'generateReadmeFiles(aceAuthorNotation, string)' that could not be resolved
{
if (folder == null)
{
return;
}
List <String> output = new List <string>();
output.Add("Data structure: " + descriptor.name + " [" + descriptor.type.Name + "]");
output.Add("Description: " + descriptor.description);
output.Add("File: " + descriptor.filename);
output.Add("----");
Boolean hasFiles = descriptor.fileDataProperties.Any();
if (hasFiles)
{
output.Add("In this data structure:");
}
foreach (var pair in descriptor.fileDataProperties)
{
output.Add("[" + pair.Key.ToString("D3") + "] " + pair.Value.name + " [" + pair.Value.type.Name + "]");
output.Add(" > " + pair.Value.description);
output.Add(" > Path: " + pair.Value.filename + " [" + pair.Value.filenameMode + "]");
}
if (hasFiles)
{
output.Add("---");
}
output.AddRange(CustomFolderDescriptionLines());
folder.AdditionalDescriptionLines.AddRange(output, true);
if (generateReadme)
{
folder.generateReadmeFiles(notation);
}
}
/// <summary>
/// Publishes the specified notation.
/// </summary>
/// <param name="notation">The notation.</param>
/// <param name="__folder">The folder.</param>
public void Publish(aceAuthorNotation notation = null, folderNode __folder = null)
{
if (notation == null)
{
notation = new aceAuthorNotation();
}
if (__folder == null)
{
__folder = folder;
}
Save();
pageIndexTable.GetDataTable().GetReportAndSave(__folder, notation, "pageIndex");
domainIndexTable.GetDataTable().GetReportAndSave(__folder, notation, "domainIndex");
// indexSessionRecords.GetDataTable().GetReportAndSave(__folder, notation);
// domainIndexTable.GetDomainUrls(indexDomainContentEnum.onlyRelevant).saveContentOnFilePath(__folder.pathFor("indexDomains_onlyRelevant.txt"));
// domainIndexTable.GetDomainUrls(indexDomainContentEnum.onlyNonRelevant).saveContentOnFilePath(__folder.pathFor("indexDomains_onlyNonRelevant.txt"));
// domainIndexTable.GetDomainUrls(indexDomainContentEnum.bothRelevantAndNonRelevant).saveContentOnFilePath(__folder.pathFor("indexDomains_BothRelevantAndNon.txt"));
}
public static void MakeReport(classificationReportCollection reportCollection, classificationReportCollectionSettings settings, aceAuthorNotation appInfo, ILogBuilder log, classificationReportStyleDefinition style, classificationReportDataComplexContext context)
{
DataTableTypeExtended <classificationReportExpanded> table = reportCollection.MakeOverviewTable(context, reportCollection.name, reportCollection.description);
table.SetTitle(reportCollection.name);
var statDataTable = table.GetReportAndSave(reportCollection.rootFolder, appInfo);
log.log("Report [" + table.TableName + "] created at " + statDataTable.lastFilePath);
// context.cumulative_tables.Add(table);
var layers = reportCollection.GetSpaceLayers(style);
foreach (var pair in layers)
{
var reportSpace = classificationReportSpace.BuildReportSpace(pair.Value, reportCollection.datasetName, settings.SELECT_REPORT_NAME_PARTS, style, pair.Key);
if (!context.report_spaces.ContainsKey(reportSpace.name))
{
context.report_spaces.Add(reportSpace.name, new List <classificationReportSpace>());
context.comparative_tables.Add(reportSpace.name, new List <DataTable>());
context.comparative_narrow_tables.Add(reportSpace.name, new List <DataTable>());
}
context.report_spaces[reportSpace.name].Add(reportSpace);
System.Data.DataTable comparative_table = reportSpace.ConstructTable("comparative_" + reportCollection.name + "_" + reportSpace.name, reportCollection.description);
context.comparative_tables[reportSpace.name].Add(comparative_table);
comparative_table.AddExtra("Group path: " + reportCollection.rootFolder.path);
comparative_table.GetReportAndSave(reportCollection.rootFolder, appInfo);
System.Data.DataTable comparative_table_small = reportSpace.ConstructTable("comparative_" + reportCollection.name + "_" + reportSpace.name + "_small", reportCollection.description, classificationReportTableMode.onlyBasic);
context.comparative_narrow_tables[reportSpace.name].Add(comparative_table_small);
var styleFS = style.CloneViaXML();
styleFS.valueToUse = classificationReportStyleDefinition.GetFS(); //new reportExpandedDataPair(classificationReportStyleDefinition.VALUE_FS, "Selected Features", "Number of features actually selected");
reportSpace = classificationReportSpace.BuildReportSpace(pair.Value, reportCollection.datasetName, settings.SELECT_REPORT_NAME_PARTS, styleFS, pair.Key);
reportSpace.ConstructTable("featureSelected_" + reportCollection.name + "_" + reportSpace.name, reportCollection.description).GetReportAndSave(reportCollection.rootFolder, appInfo);
}
// return comparative_table;
}
/// <summary>
/// Gets the report and save.
/// </summary>
/// <param name="input">The input.</param>
/// <param name="folder">The folder.</param>
/// <param name="notation">The notation.</param>
/// <param name="fileprefix">The fileprefix.</param>
/// <returns></returns>
public static List <DataTableForStatistics> GetReportAndSave(this Dictionary <String, DataTable> input, folderNode folder, aceAuthorNotation notation, String fileprefix = "", DataTableConverterASCII converter = null)
{
List <DataTableForStatistics> output = new List <DataTableForStatistics>();
foreach (var pair in input)
{
String p_m = folder.pathFor(fileprefix + pair.Key);
output.Add(pair.Value.GetReportAndSave(folder, notation, fileprefix + pair.Key));
if (converter != null)
{
converter.ConvertToFile(pair.Value, folder, pair.Value.TableName, DataTableConverterASCIISettings.GetCommaSeparatedValues());
}
}
return(output);
}
public directAnalyticReporter makeReporter(string crawlerName, aceAuthorNotation notation)
{
directAnalyticReporter output = new directAnalyticReporter(crawlerName, folder[ACFolders.reports], notation);
return(output);
}
public static void MakeOverviewReports(this classificationReportCollection reportCollection, classificationReportCollectionSettings settings, aceAuthorNotation appInfo, ILogBuilder log, String name = "", String description = "")
{
classificationReportDataComplexContext context = new classificationReportDataComplexContext();
DataTableConverterASCII dataTableConverterASCII = new DataTableConverterASCII();
classificationReportStyleDefinition style = classificationReportStyleDefinition.GetDefault(settings.style.groups);
if (reportCollection.name.isNullOrEmpty())
{
reportCollection.name = name;
}
if (description.isNullOrEmpty())
{
reportCollection.description += description;
}
MakeReport(reportCollection, settings, appInfo, log, style, context); //reportSpace.ConstructTable("cumulative_" + reportCollection.name, reportCollection.description, style);
foreach (var pair in reportCollection.Children)
{
MakeReport(pair.Value, settings, appInfo, log, style, context);
}
foreach (var pair in context.comparative_tables)
{
var comparative_table = pair.Value.GetRowCollectionTable("complete_summary_" + pair.Key); //context.comparative_tables.GetRowCollectionTable("complete_comparative");
comparative_table.SetTitle("complete_summary_" + pair.Key);
comparative_table.GetReportAndSave(reportCollection.rootFolder, appInfo);
}
foreach (var pair in context.comparative_narrow_tables)
{
var comparative_table = pair.Value.GetRowCollectionTable("complete_summary_ascii" + pair.Key); //context.comparative_tables.GetRowCollectionTable("complete_comparative");
comparative_table.SetTitle("complete_summary_ascii" + pair.Key);
dataTableConverterASCII.ConvertToFile(comparative_table, reportCollection.rootFolder, comparative_table.TableName + "_ascii.txt", DataTableConverterASCIISettings.GetCommaSeparatedValues());
//builderForText bt = new builderForText();
//bt.AppendTable(comparative_table);
//bt.GetContent().saveStringToFile(reportCollection.rootFolder.pathFor(comparative_table.TableName + "_ascii.txt", Data.enums.getWritableFileMode.overwrite, "ASCII export of comparative table"));
comparative_table.GetReportAndSave(reportCollection.rootFolder, appInfo);
}
var overview_table = context.overview_tables.GetRowCollectionTable("complete_overview");
overview_table.SetTitle("complete_overview");
overview_table.GetReportAndSave(reportCollection.rootFolder, appInfo);
//.GetAggregatedTable("summary", Core.math.aggregation.dataPointAggregationAspect.subSetOfRows, log);
}
public static void Publish(this SourceTable source, folderNode outputFolder, ExtractionResultPublishFlags flags, String filename_prefix = "", aceAuthorNotation notation = null)
{
if (source == null)
{
return;
}
var sourcep = outputFolder.pathFor(filename_prefix + "_source.xml", imbSCI.Data.enums.getWritableFileMode.autoRenameThis, "Exported source table");
if (flags.HasFlag(ExtractionResultPublishFlags.sourceTableSerialization))
{
source.Save(sourcep);
}
String fl = Path.GetFileNameWithoutExtension(sourcep);
if (flags.HasFlag(ExtractionResultPublishFlags.sourceTableExcel))
{
source.GetDataTable().GetReportAndSave(outputFolder, notation, fl);
}
source.ExpandedData.SaveObjectPairs(filename_prefix + "_data", outputFolder);
}
/// <summary>
/// Generates the folder readme.
/// </summary>
/// <param name="notation">The notation.</param>
/// <param name="builder">The builder.</param>
/// <param name="directoryStructureDepthLimit">The directory structure depth limit - i.e. until what subdirectory depth is described in the readme file.</param>
/// <returns></returns>
internal string generateFolderReadme(aceAuthorNotation notation, ITextRender builder = null, Int32 directoryStructureDepthLimit = 3)
{
if (builder == null)
{
builder = new builderForMarkdown();
}
String prefix = "";
String mypath = path;
if (parent != null)
{
mypath = mypath.removeStartsWith(parent.path);
prefix = path;
}
builder.AppendHeading("Directory information", 2);
builder.AppendHeading(caption, 3);
builder.AppendLine(" > " + mypath);
builder.AppendLine(" > " + description);
builder.AppendHorizontalLine();
foreach (String st in AdditionalDescriptionLines)
{
builder.AppendLine(st);
}
if (AdditionalDescriptionLines.Any())
{
builder.AppendHorizontalLine();
}
if (AdditionalFileEntries.Any())
{
builder.AppendHeading("Files in this directory:", 2);
String format = "D" + AdditionalFileEntries.Count().ToString().Length.ToString();
Int32 flc = 1;
List <String> sortedKeys = AdditionalFileEntries.Keys.ToList();
sortedKeys.Sort(String.CompareOrdinal);
foreach (String key in sortedKeys)
{
builder.AppendLine(flc.ToString(format) + " : " + AdditionalFileEntries[key].description);
flc++;
}
}
//builder.AppendHeading("Folder treeview", 2);
//builder.Append(this.tree)
builder.AppendHorizontalLine();
builder.AppendHeading("Subdirectories of: " + prefix, 2);
var folderNodes = this.getAllChildrenInType <folderNode>(null, false, true, 0, directoryStructureDepthLimit);
foreach (var fold in folderNodes)
{
Int32 levelDistance = fold.level - level;
String insert = " -- ".Repeat(levelDistance);
if (levelDistance > directoryStructureDepthLimit)
{
if (fold.count() > 0)
{
builder.AppendCite(insert + "> directory " + fold.caption + " with [" + fold.count() + "] sub directories ...");
}
}
else
{
builder.AppendLine(String.Format("{0,-60} : {1,-100}", insert + "> " + fold.caption, fold.path.removeStartsWith(prefix)));
if (!fold.description.isNullOrEmpty())
{
builder.AppendLine(insert + "| " + fold.description);
}
}
//builder.AppendLine();
// builder.prevTabLevel();
}
// AdditionalFileEntries.Sort(String.CompareOrdinal);
if (notation != null)
{
builder.AppendHorizontalLine();
notation.GetDescription(builder);
}
builder.AppendLine();
builder.AppendHorizontalLine();
builder.AppendLine(imbSciStringExtensions.add(imbSciStringExtensions.add("File generated: ", DateTime.Now.ToLongDateString(), " "), DateTime.Now.ToLongTimeString()));
return(builder.ContentToString(true));
}
public static void Publish(this StructureGraphInformationSet info, folderNode folder, String name, aceAuthorNotation notation = null)
{
DataTable dt = info.items.ReportToDataTable <StructureGraphInformation>(true);
dt.SetTitle(name + " records");
dt.GetReportAndSave(folder, notation);
dt = info.changes.ReportToDataTable <StructureGraphInformation>(true);
dt.SetTitle(name + " changes");
dt.GetReportAndSave(folder, notation);
builderForText output = new builderForText();
foreach (StructureGraphInformation item in info.items)
{
item.Report(null, output);
}
output.ReportSave(folder, name + "_records", "Structure graph entries");
output = new builderForText();
foreach (var item in info.changes)
{
item.Report(null, output);
}
output.ReportSave(folder, name + "_changes", "Structure graph changes log");
}
public void Publish(folderNode folder, TableExtractionTaskReporting TaskReporting, aceAuthorNotation notation)
{
if (TaskReporting == null)
{
TaskReporting = new TableExtractionTaskReporting()
{
baseOutputFolder = folder,
currentOutputFolder = folder,
};
}
List <DataTable> allReportTables = new List <DataTable>();
List <DataTable> trainingReportTables = new List <DataTable>();
List <DataTable> validationReportTables = new List <DataTable>();
//List<DataTable> trainingReportTables = new List<DataTable>();
// DataSet complete_report = new DataSet();
TaskValidationResult firstResult = null;
foreach (KeyValuePair <TableExtractionTask, TaskValidationResult> pair in Results)
{
if (firstResult == null)
{
firstResult = pair.Value;
}
pair.Value.Publish(folder, notation);
propertyReport_invalid = pair.Value.PropertyValidation.GetReportTable(pair.Value, propertyReport_invalid, new List <ValidationOutcome>()
{
ValidationOutcome.Invalid
});
propertyReport_valid = pair.Value.PropertyValidation.GetReportTable(pair.Value, propertyReport_valid, new List <ValidationOutcome>()
{
ValidationOutcome.Validated
});
propertyReport_other = pair.Value.PropertyValidation.GetReportTable(pair.Value, propertyReport_other, new List <ValidationOutcome>()
{
ValidationOutcome.Modified, ValidationOutcome.undefined
});
trainingReportTables.Add(pair.Value.ReportTable_Training);
validationReportTables.Add(pair.Value.ReportTable_PropertyValudation);
// allReportTables.AddRange(pair.Value.ReportTables);
}
allReportTables.Add(propertyReport_invalid);
allReportTables.Add(propertyReport_valid);
allReportTables.Add(propertyReport_other);
DataTable training_report = TaskReporting.GetTaskScore(Results.Keys).GetReportAndSave(TaskReporting.currentOutputFolder, notation);
training_report.SetTitle("Training run results");
training_report.GetReportAndSave(folder, notation);
//allReportTables.Add(training_report);
var overview_table = GetReportDataTable();
//allReportTables.Add(overview_table);
propertyReport_invalid.SetTitle("Invalid");
propertyReport_valid.SetTitle("Valid");
propertyReport_other.SetTitle("Other");
allReportTables.GetReportAndSave(folder, notation, "property_reports");
trainingReportTables.GetReportAndSave(folder, notation, "training_reports");
validationReportTables.GetReportAndSave(folder, notation, "validation_reports");
overview_table.GetReportAndSave(folder, notation);
//complete_report.GetReportAndSave(folder, notation, "complete_evaluation");
}
/// <summary>
/// Exports all acceptable file formats, to excel tables with the same name
/// </summary>
/// <param name="filePaths">The file paths.</param>
/// <param name="author">The author.</param>
/// <param name="log">The log.</param>
/// <returns></returns>
public static List <DataTableForStatistics> ExportToExcel(IEnumerable <String> filePaths, aceAuthorNotation author, ILogBuilder log = null, propertyAnnotationPreset customTemplate = null, folderNode outputFolder = null)
{
List <DataTableForStatistics> output = new List <DataTableForStatistics>();
Int32 c = 0;
if (ExportInParallel)
{
Stack <DataTableForStatistics> outputStack = new Stack <DataTableForStatistics>();
Parallel.ForEach(filePaths, x =>
{
if (CheckExtension(x))
{
var excel = ExportToExcel(x, author, log, customTemplate, outputFolder);
outputStack.Push(excel);
c = 0;
}
else
{
c++;
}
});
output.AddRange(outputStack.ToArray());
}
else
{
foreach (String f in filePaths)
{
//if (c > FileExtensionAbortCount)
//{
// if (log != null) log.log("File list aborted after [" + c.ToString() + "] incompatibile files");
// break;
//}
if (CheckExtension(f))
{
output.Add(ExportToExcel(f, author, log, customTemplate, outputFolder));
c = 0;
}
else
{
c++;
}
}
}
return(output);
}
//public static String GetFileName(this DataTa)
/// <summary>
/// Saves the specified folder.
/// </summary>
/// <param name="folder">The folder.</param>
/// <param name="notation">The notation.</param>
/// <param name="filenamePrefix">The filename prefix.</param>
/// <returns></returns>
/// <exception cref="dataException">Excell: " + ex.Message - Export to excell</exception>
public string Save(folderNode folder, aceAuthorNotation notation = null, string filenamePrefix = "")
{
string msg = "tried to save a data table [" + this.GetTitle() + "][" + TableName + "] -> " + folder.path + " [" + DateTime.Now.ToLongTimeString() + "/" + DateTime.Now.ToLongDateString() + "]";
string fl = "note_savefailed_" + GetHashCode() + ".txt";
//FileInfo fi = msg.saveStringToFile(folder.pathFor(fl, getWritableFileMode.none), getWritableFileMode.autoRenameExistingToBack);
FileInfo fileInfo = null;
try
{
string output = "";
string filename = "";
if (DataTableForStatisticsExtension.tableReportCreation_useShortNames)
{
if (filenamePrefix.isNullOrEmpty())
{
filename = "dt_".add(TableName, "_").getFilename();
}
else
{
filename = "dt_".add(filenamePrefix, "_").getFilename();
}
}
else
{
filename = "dt_" + filenamePrefix.add(TableName).getFilename();
}
filename = filename.ensureEndsWith(".xlsx");
filename = folder.pathFor(filename);
fileInfo = filename.getWritableFile(getWritableFileMode.overwrite);
if (DataTableForStatisticsExtension.tableReportCreation_insertFilePathToTableExtra)
{
this.SetAdditionalInfoEntry("Folder", fileInfo.DirectoryName);
this.SetAdditionalInfoEntry("File", fileInfo.Name);
this.SetAdditionalInfoEntry("ID", filenamePrefix);
}
if (Rows.Count == 0)
{
return(fileInfo.FullName);
}
//if (File.Exists(fileInfo.FullName)) File.Delete(fileInfo.FullName);
try
{
using (ExcelPackage pck = new ExcelPackage(fileInfo))
{
ExcelWorksheet ws = pck.Workbook.Worksheets.Add(this.GetTitle());
ExcelWorksheet wsL = pck.Workbook.Worksheets.Add("LEGEND");
pck.Workbook.Properties.Title = this.GetTitle();
pck.Workbook.Properties.Category = "DataTable export";
if (notation != null)
{
pck.Workbook.Properties.Comments = notation.comment;
pck.Workbook.Properties.Author = notation.author;
pck.Workbook.Properties.Company = notation.organization;
pck.Workbook.Properties.Application = notation.software;
}
//pck.Workbook.Properties.Keywords = meta.keywords.content.toCsvInLine();
pck.Workbook.Properties.Created = DateTime.Now;
pck.Workbook.Properties.Subject = this.GetDescription();
RenderToWorksheet(ws);
RenderToLegendWorksheet(wsL);
pck.Save();
}
}
catch (Exception ex)
{
throw new dataException("Excell: " + ex.Message, ex, this, "Export to excell");
msg = msg + ex.LogException("Saving Excel file [" + fileInfo.FullName + "]", "DataTable");
msg.saveStringToFile(folder.pathFor(fl, getWritableFileMode.none), getWritableFileMode.autoRenameExistingToBack);
}
output = fileInfo.FullName;
return(output);
}
catch (Exception ex2)
{
msg = msg + ex2.LogException("Saving Excel file [" + fileInfo.FullName + "]", "DataTable");
msg.saveStringToFile(folder.pathFor(fl, getWritableFileMode.none), getWritableFileMode.autoRenameExistingToBack);
}
//fi.Delete();
if (fileInfo != null)
{
return(fileInfo.FullName);
}
else
{
return("error");
}
}
/// <summary>
/// Gets the report and save.
/// </summary>
/// <param name="source">The source.</param>
/// <param name="folder">The folder.</param>
/// <param name="notation">The notation.</param>
/// <param name="filenamePrefix">The filename prefix.</param>
/// <param name="disablePrimaryKey">if set to <c>true</c> [disable primary key].</param>
/// <param name="allowAsyncCall">if set to <c>true</c> [allow asynchronous call].</param>
/// <returns></returns>
public static DataSetForStatistics GetReportAndSave(this IEnumerable <DataTable> source, folderNode folder, aceAuthorNotation notation = null, string filenamePrefix = "", bool disablePrimaryKey = true, Boolean allowAsyncCall = false)
{
DataSet ds_source = new DataSet(filenamePrefix);
String table_name_proposal = "";
foreach (var dt in source)
{
if (dt != null)
{
table_name_proposal = dt.TableName;
Int32 c = 0;
while (ds_source.Tables.Contains(table_name_proposal))
{
c++;
table_name_proposal = dt.TableName + c.ToString();
}
dt.TableName = table_name_proposal;
ds_source.Tables.Add(dt);
}
}
if (ds_source.Tables.Count == 0)
{
return(null);
}
DataSetForStatistics output = GetReportVersion(ds_source, disablePrimaryKey);
output.Save(folder, notation, filenamePrefix);
return(output);
}
/// <summary>
/// Serializes the data table into choosed format and returns file path
/// </summary>
/// <param name="source">The source.</param>
/// <param name="format">The format.</param>
/// <param name="filename">The filename, without extension.</param>
/// <param name="directory">The directory to save into.</param>
/// <returns></returns>
public static string serializeDataTable(this DataTable source, dataTableExportEnum format, string filename, folderNode directory, params object[] resources)
{
dataTableIOFlags IOFlags = resources.GetIOFlags();
if (source == null)
{
return("");
}
if (source.Columns.Count == 0)
{
throw new dataException("Source table [0 columns]: ", null, source, "Export to excell : table is not applicable");
}
if (source.Rows.Count == 0)
{
return("");
}
format = checkFormatByFilename(format, filename);
string output = filename;
FileInfo fileInfo = null;
aceAuthorNotation authorNotation = resources.getFirstOfType <aceAuthorNotation>(false, false, true);
if (authorNotation == null)
{
authorNotation = new aceAuthorNotation();
}
if (directory == null)
{
directory = new DirectoryInfo(Directory.GetCurrentDirectory());
}
string cleanfilename = source.FilenameForTable(filename);
filename = directory.pathFor(cleanfilename, getWritableFileMode.overwrite, "Exported DataTable [" + source.GetTitle() + "][" + source.GetDescription() + "]. ");
switch (format)
{
case dataTableExportEnum.csv:
output = source.toCSV(true);
output = output.saveStringToFile(imbSciStringExtensions.ensureEndsWith(filename, ".csv")).FullName;
break;
case dataTableExportEnum.excel:
//fileInfo = new FileInfo(filename.ensureEndsWith(".xlsx"));
filename = imbSciStringExtensions.ensureEndsWith(filename, ".xlsx");
fileInfo = filename.getWritableFile(getWritableFileMode.overwrite);
//if (File.Exists(fileInfo.FullName)) File.Delete(fileInfo.FullName);
try
{
using (ExcelPackage pck = new ExcelPackage(fileInfo))
{
DataTableForStatistics dt_stat = null;
if (source is DataTableForStatistics)
{
dt_stat = source as DataTableForStatistics;
}
else
{
dt_stat = source.GetReportTableVersion(true);
}
ExcelWorksheet ws = pck.Workbook.Worksheets.Add(source.GetTitle());
dt_stat.RenderToWorksheet(ws);
/*
* pck.Workbook.Properties.Title = source.GetTitle();
* pck.Workbook.Properties.Comments = authorNotation.comment;
* pck.Workbook.Properties.Category = "DataTable export";
* pck.Workbook.Properties.Author = authorNotation.author;
* pck.Workbook.Properties.Company = authorNotation.organization;
* pck.Workbook.Properties.Application = authorNotation.software;
* //pck.Workbook.Properties.Keywords = meta.keywords.content.toCsvInLine();
* pck.Workbook.Properties.Created = DateTime.Now;
* pck.Workbook.Properties.Subject = source.GetDescription();
*
* ws.Cells["A1"].LoadFromDataTable(source, true);
* ExcelRow row = ws.Row(1); //.Height = 100;
* row.Height = 100;
* row.Style.WrapText = true;
*
* row.Style.Fill.PatternType = ExcelFillStyle.Solid;
* row.Style.Fill.BackgroundColor.SetColor(System.Drawing.Color.WhiteSmoke);
*
* Int32 rc = ws.Dimension.Rows - 1;
*
* for (int i = 0; i < rc; i++)
* {
* var ex_row = ws.Row(i+1);
* var in_row = source.Rows[i];
* if (dt_stat != null)
* {
* if (dt_stat.extraRows.Contains(in_row))
* {
* if (dt_stat.extraRows.IndexOf(in_row)%2 > 0)
* {
* ex_row.Style.Fill.PatternType = ExcelFillStyle.Solid;
* ex_row.Style.Fill.BackgroundColor.SetColor(System.Drawing.Color.LightGray);
* } else
* {
* ex_row.Style.Fill.PatternType = ExcelFillStyle.Solid;
* ex_row.Style.Fill.BackgroundColor.SetColor(System.Drawing.Color.WhiteSmoke);
* }
* }
* }
*
* ex_row.Style.WrapText = true;
* // row.Style.Fill.BackgroundColor.SetColor(System.Drawing.Color.WhiteSmoke);
* }
*/
pck.Save();
}
}
catch (Exception ex)
{
throw new dataException("Excell: " + ex.Message, ex, source, "Export to excell");
}
output = fileInfo.FullName;
break;
case dataTableExportEnum.json:
output = objectSerialization.SerializeJson(source);
// JsonConvert.SerializeObject(source, Newtonsoft.Json.Formatting.Indented);
fileInfo = output.saveStringToFile(imbSciStringExtensions.ensureEndsWith(filename, ".json")); //.FullName;
output = fileInfo.FullName;
break;
case dataTableExportEnum.markdown:
output = source.markdownTable();
fileInfo = output.saveStringToFile(imbSciStringExtensions.ensureEndsWith(filename, ".md"));
output = fileInfo.FullName;
break;
case dataTableExportEnum.xml:
filename = imbSciStringExtensions.ensureEndsWith(filename, ".xml");
source.WriteXml(filename, false);
output = openBase.openFileToString(filename, true, false);
break;
default:
break;
}
return(output);
}
public static string serializeDataSet(this DataSet source, string filename, folderNode directory, dataTableExportEnum format, params object[] resources)
{
dataTableIOFlags IOFlags = resources.GetIOFlags();
string output = filename;
FileInfo fileInfo = null;
format = checkFormatByFilename(format, filename);
aceAuthorNotation authorNotation = resources.getFirstOfType <aceAuthorNotation>(false, false, true);
if (authorNotation == null)
{
authorNotation = new aceAuthorNotation();
}
if (directory == null)
{
directory = new DirectoryInfo(Directory.GetCurrentDirectory());
}
string cleanfilename = source.FilenameForDataset(filename);
filename = directory.pathFor(cleanfilename, getWritableFileMode.overwrite, "Exported DataSet [" + source.DataSetName + "] with [" + source.Tables.Count + "] tables. " + source.GetDesc());
DirectoryInfo dix = null;
switch (format)
{
case dataTableExportEnum.csv:
dix = directory.Add(cleanfilename, source.DataSetName, "Folder with CSV exports of DataSet [" + source.DataSetName + "] with [" + source.Tables.Count + "] tables. " + source.GetDesc());
foreach (DataTable table in source.Tables)
{
table.serializeDataTable(format, table.FilenameForTable(), dix);
}
output = dix.FullName;
break;
case dataTableExportEnum.excel:
//fileInfo = new FileInfo(filename.ensureEndsWith(".xlsx"));
filename = imbSciStringExtensions.ensureEndsWith(filename, ".xlsx");
fileInfo = filename.getWritableFile(getWritableFileMode.overwrite);
//if (File.Exists(fileInfo.FullName)) File.Delete(fileInfo.FullName);
try
{
using (ExcelPackage pck = new ExcelPackage(fileInfo))
{
pck.Workbook.Properties.Title = source.GetTitle();
pck.Workbook.Properties.Comments = authorNotation.comment;
pck.Workbook.Properties.Category = "DataTable export";
pck.Workbook.Properties.Author = authorNotation.author;
pck.Workbook.Properties.Company = authorNotation.organization;
pck.Workbook.Properties.Application = authorNotation.software;
//pck.Workbook.Properties.Keywords = meta.keywords.content.toCsvInLine();
pck.Workbook.Properties.Created = DateTime.Now;
pck.Workbook.Properties.Subject = source.GetDesc();
int c = 0;
foreach (DataTable table in source.Tables)
{
string title = table.GetTitle();
if (title.Length > 20)
{
title = title.toWidthMaximum(15, "");
}
title = title + c.ToString("D3");
c++;
while (pck.Workbook.Worksheets.Any(x => x.Name == title))
{
title = title + c.ToString("D3");
c++;
}
if (title == dataTableRenderingSetup.TABLE_DEFAULTNAME)
{
title = "Table" + source.Tables.Count.ToString("D3");
}
ExcelWorksheet ws = pck.Workbook.Worksheets.Add(title);
DataTableForStatistics dt_stat = table as DataTableForStatistics;
if (table is DataTableForStatistics)
{
dt_stat = table as DataTableForStatistics;
}
else
{
dt_stat = table.GetReportTableVersion(true);
}
table.SetTitle(title);
table.TableName = title;
//DataTableForStatistics dt_stat = table as DataTableForStatistics;
dt_stat.RenderToWorksheet(ws);
/*
*
* ws.Cells["A1"].LoadFromDataTable(table, true);
*
* ExcelRow row = ws.Row(1); //.Height = 100;
* row.Height = 100;
* row.Style.WrapText = true;
*
* row.Style.Fill.PatternType = ExcelFillStyle.Solid;
* row.Style.Fill.BackgroundColor.SetColor(System.Drawing.Color.WhiteSmoke);
*
* Int32 rc = ws.Dimension.Rows - 1;
*
* for (int i = 0; i < rc; i++)
* {
* var ex_row = ws.Row(i + 1);
* var in_row = table.Rows[i];
* if (dt_stat != null)
* {
* if (dt_stat.extraRows.Contains(in_row))
* {
* if (dt_stat.extraRows.IndexOf(in_row) % 2 > 0)
* {
* ex_row.Style.Fill.PatternType = ExcelFillStyle.Solid;
* ex_row.Style.Fill.BackgroundColor.SetColor(System.Drawing.Color.LightGray);
* }
* else
* {
* ex_row.Style.Fill.PatternType = ExcelFillStyle.Solid;
* ex_row.Style.Fill.BackgroundColor.SetColor(System.Drawing.Color.WhiteSmoke);
* }
* }
* }
*
* ex_row.Style.WrapText = true;
* // row.Style.Fill.BackgroundColor.SetColor(System.Drawing.Color.WhiteSmoke);
* }
*/
}
pck.Save();
}
}
catch (Exception ex)
{
throw new dataException("Excell: " + ex.Message, ex, source, "Export to excell");
}
output = fileInfo.FullName;
break;
case dataTableExportEnum.json:
dix = directory.Add(cleanfilename, source.DataSetName, "Folder with JSON exports of DataSet [" + source.DataSetName + "] with [" + source.Tables.Count + "] tables. " + source.GetDesc());
output = objectSerialization.SerializeJson <DataSet>(source);
fileInfo = output.saveStringToFile(imbSciStringExtensions.ensureEndsWith(filename, ".json")); //.FullName;
output = fileInfo.FullName;
break;
case dataTableExportEnum.markdown:
dix = directory.Add(cleanfilename, source.DataSetName, "Folder with Markdown exports of DataSet [" + source.DataSetName + "] with [" + source.Tables.Count + "] tables. " + source.GetDesc());
foreach (DataTable table in source.Tables)
{
table.serializeDataTable(format, table.FilenameForTable(), dix);
}
output = dix.FullName;
break;
case dataTableExportEnum.xml:
dix = directory.Add(cleanfilename, source.DataSetName, "Folder with XML exports of DataSet [" + source.DataSetName + "] with [" + source.Tables.Count + "] tables. " + source.GetDesc());
foreach (DataTable table in source.Tables)
{
table.serializeDataTable(format, table.FilenameForTable(), dix);
}
output = dix.FullName;
break;
default:
break;
}
return(output);
}
/// <summary>
/// Gets the report and save.
/// </summary>
/// <param name="source">The source.</param>
/// <param name="folder">The folder.</param>
/// <param name="notation">The notation.</param>
/// <param name="filenamePrefix">The filename prefix.</param>
/// <param name="disablePrimaryKey">if set to <c>true</c> [disable primary key].</param>
/// <param name="allowAsyncCall">if set to <c>true</c> [allow asynchronous call].</param>
/// <returns></returns>
public static DataSetForStatistics GetReportAndSave(this DataSet source, folderNode folder, aceAuthorNotation notation = null, string filenamePrefix = "", bool disablePrimaryKey = true, Boolean allowAsyncCall = false)
{
DataSetForStatistics output = GetReportVersion(source, disablePrimaryKey);
output.Save(folder, notation, filenamePrefix);
return(output);
}
/// <summary>
/// Creates report table version for the <c>source</c> and saves the report on specified <c>folder</c>
/// </summary>
/// <param name="source">The source.</param>
/// <param name="folder">The folder.</param>
/// <param name="notation">The notation.</param>
/// <param name="filenamePrefix">The filename prefix.</param>
/// <param name="disablePrimaryKey">if set to <c>true</c> [disable primary key].</param>
/// <param name="allowAsyncCall">if set to <c>true</c> [allow asynchronous call].</param>
/// <returns></returns>
/// <exception cref="ArgumentNullException">Folder is null! at GetReportAndSave() for [" + source.TableName + "] at filename [" + filenamePrefix + "]</exception>
public static DataTableForStatistics GetReportAndSave(this DataTable source, folderNode folder, aceAuthorNotation notation = null, string filenamePrefix = "", bool disablePrimaryKey = true, Boolean allowAsyncCall = false)
{
if (notation == null)
{
notation = new aceAuthorNotation();
}
if (allowAsyncCall)
{
if (imbSCI.Core.config.imbSCICoreConfig.settings.DataTableReports_AsyncExportCalls)
{
DataTableForStatisticsExportJob job = new DataTableForStatisticsExportJob(source, folder, notation, filenamePrefix, disablePrimaryKey);
Thread t = new Thread(job.Do);
t.Start();
return(null);
// Task.Factory
}
}
// if (source == null) return new DataTableForStatistics();
if (folder == null)
{
throw new ArgumentNullException("Folder is null! at GetReportAndSave() for [" + source.TableName + "] at filename [" + filenamePrefix + "]");
}
if (source.Columns.Count > 0)
{
folderNode dataFolder = null;
if (DataTableForStatistics.AUTOSAVE_CleanDataTable || DataTableForStatistics.AUTOSAVE_FieldsText || imbSCI.Core.config.imbSCICoreConfig.settings.DataTableReports_DoExportXMLData)
{
dataFolder = folder.Add(EXTRAFOLDER, "Excel report meta data", "Folder containing clean data export (single header row, CSV format) for easier use by other software platforms and/or column meta descriptions - additional information - in separate txt file for each Excel report created.");
}
if (imbSCI.Core.config.imbSCICoreConfig.settings.DataTableReports_DoExportXMLData)
{
try
{
String xmlCode = objectSerialization.ObjectToXML(source);
xmlCode.saveStringToFile(dataFolder.pathFor(source.TableName.getFilename(".xml"), getWritableFileMode.overwrite, "XML Serialized DataTable [" + source.GetTitle() + "]", true));
}
catch (Exception ex)
{
source.SetAdditionalInfoEntry("XML data", "Serialization failed: " + ex.Message);
}
}
if (DataTableForStatistics.AUTOSAVE_CleanDataTable)
{
string cld = source.serializeDataTable(dataTableExportEnum.csv, PREFIX_CLEANDATATABLE + filenamePrefix.getFilename() + ".csv", dataFolder, notation);
source.SetAdditionalInfoEntry("Clean data", cld);
}
if (DataTableForStatistics.AUTOSAVE_FieldsText)
{
string cli = dataFolder.pathFor(PREFIX_COLUMNINFO + filenamePrefix.getFilename() + ".txt");
source.GetUserManualForTableSaved(cli);
source.SetAdditionalInfoEntry("Column info", cli);
}
if (tableReportCreation_insertFilePathToTableExtra)
{
}
}
DataTableForStatistics output = null;
if (source is DataTableForStatistics)
{
output = source as DataTableForStatistics;
}
else
{
output = source.GetReportTableVersion(disablePrimaryKey);
// output.SetDefaults();
//source.serializeDataTable(enums.dataTableExportEnum.excel, filenamePrefix + "_source", folder, notation);
}
output.Save(folder, notation, filenamePrefix);
return(output);
}
