/// <summary>
/// Get LINQPad Schema from CSV data model
/// </summary>
/// <param name="db"></param>
/// <returns></returns>
private static List <ExplorerItem> GetSchema(CsvDatabase db, ICsvDataContextDriverProperties props)
{
var schema = (
from table in db.Tables ?? Enumerable.Empty <CsvTable>()
select new ExplorerItem(table.DisplayName, ExplorerItemKind.QueryableObject, ExplorerIcon.Table)
{
DragText = table.CodeName,
IsEnumerable = true,
ToolTipText = table.FilePath,
Children = (
from c in table.Columns ?? Enumerable.Empty <CsvColumn>()
select new ExplorerItem(c.DisplayName, ExplorerItemKind.Property, ExplorerIcon.Column)
{
DragText = c.CodeName,
ToolTipText = (c.CsvColumnIndex + 1) + ":" + c.CsvColumnName,
}
).Concat(
from r in table.Relations
select new ExplorerItem(r.DisplayName, ExplorerItemKind.CollectionLink, ExplorerIcon.ManyToMany)
{
DragText = r.CodeName,
ToolTipText = string.Format("Relation to {0} where {1}.{2} == {3}.{4}", r.TargetTable.CodeName, r.SourceTable.CodeName, r.SourceColumn.CodeName, r.TargetTable.CodeName, r.TargetColumn.CodeName),
}
).ToList(),
}
).ToList();
return(schema);
}
public ConnectionDialog(ICsvDataContextDriverProperties csvDataContextDriverProperties)
{
Background = SystemColors.ControlBrush;
DataContext = csvDataContextDriverProperties;
InitializeComponent();
}
internal static List <ExplorerItem> GetSchemaAndBuildAssembly(ICsvDataContextDriverProperties csvDataContextDriverProperties, AssemblyName assemblyToBuild, ref string nameSpace, ref string typeName)
{
var csvDatabase = CsvDataModelGenerator.CreateModel(csvDataContextDriverProperties);
var(code, tableCodeGroups) = CsvCSharpCodeGenerator.GenerateCode(csvDatabase, ref nameSpace, ref typeName, csvDataContextDriverProperties);
var compileErrors = BuildAssembly(code, assemblyToBuild);
var schema = GetSchema(csvDatabase);
var hasCompilationErrors = compileErrors.Any();
var index = 0;
if (hasCompilationErrors || csvDataContextDriverProperties.DebugInfo)
{
schema.Insert(index++, new ExplorerItem("Data context source code", ExplorerItemKind.Schema, ExplorerIcon.Schema)
{
ToolTipText = "Drag&drop context source code to text window",
DragText = code
});
}
if (hasCompilationErrors)
{
schema.Insert(0, new ExplorerItem("Data context compilation failed", ExplorerItemKind.Schema, ExplorerIcon.Box)
{
ToolTipText = "Drag&drop data context compilation errors to text window",
DragText = string.Join(Environment.NewLine, compileErrors)
});
}
else
{
foreach (var tableCodeGroup in tableCodeGroups.Where(codeGroup => codeGroup.Count() > 1))
{
var codeNames = tableCodeGroup.Select(g => g.CodeName).ToImmutableList();
var total = $"({codeNames.Count} of {csvDatabase.Files.Count})";
schema.Insert(index++, new ExplorerItem($"{codeNames.First()} similar files {total} joined data", ExplorerItemKind.Schema, ExplorerIcon.View)
{
ToolTipText = $"Drag&drop similar files {total} joined data to text window{Environment.NewLine}{Environment.NewLine}" +
$"{string.Join(Environment.NewLine, codeNames.Count <= 3 ? codeNames : codeNames.Take(2).Concat(new []{ "..." }).Concat(codeNames.Skip(codeNames.Count-1)))}",
DragText = $@"new []
{{
{string.Join(Environment.NewLine, codeNames.Select(n => $"\t{n},"))}
}}.SelectMany(_ => _)"
});
}
}
if (!csvDatabase.Tables.Any())
{
schema.Insert(0, new ExplorerItem("No files found", ExplorerItemKind.Schema, ExplorerIcon.Box));
}
return(schema);
}
public void GetSchemaAndBuildAssemblyTest(ICsvDataContextDriverProperties properties, string id)
{
File.WriteAllText("TestA.csv",
@"a,b,c,TestAID
1,2,3,1
1,2,3,2
x");
File.WriteAllText("TestB.csv",
@"c,d,e,TestAID
1,2,3,1
1,2,3,2
x");
string nameSpace = "TestContextNamespace";
string contextTypeName = "TestContextClass";
var contextAssemblyName = new AssemblyName("TestContextAssembly" + id)
{
CodeBase = "TestContextAssembly" + id + ".dll"
};
if (File.Exists(contextAssemblyName.CodeBase))
File.Delete(contextAssemblyName.CodeBase);
var explorerItems = SchemaBuilder.GetSchemaAndBuildAssembly(
properties,
contextAssemblyName,
ref nameSpace,
ref contextTypeName
);
//debug info to console
Console.WriteLine(explorerItems[0].DragText);
Console.WriteLine(explorerItems[1].DragText);
//check returned explorer tree
Assert.AreEqual( 3, explorerItems.Count, "explorer items count");
explorerItems = explorerItems.Where(i => i.Kind == ExplorerItemKind.QueryableObject).ToList();
Assert.AreEqual( "TestA,TestB", string.Join(",", explorerItems.Select(i => i.DragText)));
Assert.AreEqual( "a,b,c,TestAID,TestB,c,d,e,TestAID,TestA", string.Join(",", explorerItems.SelectMany(i => i.Children.Select(c => c.DragText))));
//check compiled assembly
var contextAssembly = Assembly.Load(contextAssemblyName);
Assert.AreEqual("CsvDataContext,TTestA,TTestB", string.Join(",", contextAssembly.GetExportedTypes().Select(t => t.Name)));
var contextType = contextAssembly.GetType(nameSpace + "." + contextTypeName);
Assert.IsNotNull(contextType, "ContextType in assembly");
//check generated context runtime
dynamic contextInstance = contextType.GetConstructor(new Type[] {}).Invoke(new object[] {});
Assert.IsNotNull(contextInstance, "context created");
dynamic dataFirst = Enumerable.ToArray(contextInstance.TestA)[0];
Assert.AreEqual("3", dataFirst.c);
Assert.AreEqual(1, Enumerable.Count(dataFirst.TestB));
}
internal static List <ExplorerItem> GetSchemaAndBuildAssembly(ICsvDataContextDriverProperties props, AssemblyName assemblyName, ref string nameSpace, ref string typeName)
{
Logger.LogEnabled = props.DebugInfo;
Logger.Log("Build started: " + props.Files);
var sw = Stopwatch.StartNew();
var sw2 = Stopwatch.StartNew();
CsvDatabase db = CsvDataModelGenerator.CreateModel(props);
Logger.Log("Model created. ({0} ms)", sw2.ElapsedMilliseconds); sw2.Restart();
string code = CsvCSharpCodeGenerator.GenerateCode(db, ref nameSpace, ref typeName, props);
Logger.Log("Code generated. ({0} ms)", sw2.ElapsedMilliseconds); sw2.Restart();
string[] compileErrors = BuildAssembly(code, assemblyName);
Logger.Log("Assembly compiled. ({0} ms)", sw2.ElapsedMilliseconds); sw2.Restart();
List <ExplorerItem> schema = GetSchema(db, props);
Logger.Log("Schema tree created. ({0} ms)", sw2.ElapsedMilliseconds); sw2.Restart();
bool anyCompileError = compileErrors != null && compileErrors.Any();
if (anyCompileError || props.DebugInfo)
{
schema.Insert(0, new ExplorerItem("Context Source Code", ExplorerItemKind.Schema, ExplorerIcon.Schema)
{
ToolTipText = "Data Context source code. Drag&drop to text window.",
DragText = code,
});
}
if (anyCompileError)
{
Logger.Log("Errors: {0}", compileErrors.Length);
schema.Insert(0, new ExplorerItem("Context Compile ERROR", ExplorerItemKind.Schema, ExplorerIcon.Box)
{
ToolTipText = "Data context compile failed. Drag&Drop error messages to text window to see them.",
DragText = string.Join("\n", compileErrors),
});
}
if (db.Tables.Count == 0)
{
schema.Insert(0, new ExplorerItem("No files found.", ExplorerItemKind.Schema, ExplorerIcon.Box));
}
Logger.Log("Tables: {0} Columns: {1} Relations: {2}", db.Tables.Count(), db.Tables.Sum(t => t.Columns.Count()), db.Tables.Sum(t => t.Relations.Count()));
Logger.Log("Build finished. ({0} ms)", sw.ElapsedMilliseconds);
Logger.Log(string.Join("\n", db.Tables.SelectMany(t => t.Relations.Select(r => r.CodeName))));
return(schema);
}
internal static List<ExplorerItem> GetSchemaAndBuildAssembly(ICsvDataContextDriverProperties props, AssemblyName assemblyName, ref string nameSpace, ref string typeName)
{
Logger.LogEnabled = props.DebugInfo;
Logger.Log("Build started: " + props.Files);
var sw = Stopwatch.StartNew();
var sw2 = Stopwatch.StartNew();
CsvDatabase db = CsvDataModelGenerator.CreateModel(props);
Logger.Log("Model created. ({0} ms)", sw2.ElapsedMilliseconds); sw2.Restart();
string code = CsvCSharpCodeGenerator.GenerateCode(db, ref nameSpace, ref typeName, props);
Logger.Log("Code generated. ({0} ms)", sw2.ElapsedMilliseconds); sw2.Restart();
string[] compileErrors = BuildAssembly(code, assemblyName);
Logger.Log("Assembly compiled. ({0} ms)", sw2.ElapsedMilliseconds); sw2.Restart();
List<ExplorerItem> schema = GetSchema(db, props);
Logger.Log("Schema tree created. ({0} ms)", sw2.ElapsedMilliseconds); sw2.Restart();
bool anyCompileError = compileErrors != null && compileErrors.Any();
if (anyCompileError || props.DebugInfo)
{
schema.Insert(0, new ExplorerItem("Context Source Code", ExplorerItemKind.Schema, ExplorerIcon.Schema)
{
ToolTipText = "Data Context source code. Drag&drop to text window.",
DragText = code,
});
}
if (anyCompileError)
{
Logger.Log("Errors: {0}", compileErrors.Length);
schema.Insert(0, new ExplorerItem("Context Compile ERROR", ExplorerItemKind.Schema, ExplorerIcon.Box)
{
ToolTipText = "Data context compile failed. Drag&Drop error messages to text window to see them.",
DragText = string.Join("\n", compileErrors),
});
}
if (db.Tables.Count == 0)
{
schema.Insert(0, new ExplorerItem("No files found.", ExplorerItemKind.Schema, ExplorerIcon.Box));
}
Logger.Log("Tables: {0} Columns: {1} Relations: {2}", db.Tables.Count(), db.Tables.Sum(t => t.Columns.Count()), db.Tables.Sum(t => t.Relations.Count()) );
Logger.Log("Build finished. ({0} ms)", sw.ElapsedMilliseconds);
Logger.Log( string.Join("\n", db.Tables.SelectMany(t => t.Relations.Select( r => r.CodeName) )));
return schema;
}
public CsvDataModelGenerator(ICsvDataContextDriverProperties properties)
{
this.Properties = properties;
}
public ConnectionDialog(ICsvDataContextDriverProperties properties)
{
DataContext = properties;
Background = SystemColors.ControlBrush;
InitializeComponent ();
}
private CsvDataModelGenerator(ICsvDataContextDriverProperties csvDataContextDriverProperties) =>
/// <summary>
/// Create CSV DB model based on config - mainly CSV files dir.
/// </summary>
/// <param name="props"></param>
/// <returns></returns>
public static CsvDatabase CreateModel(ICsvDataContextDriverProperties props)
{
return(new CsvDataModelGenerator(props).CreateModel());
}
public CsvCSharpCodeGenerator(string contextNameSpace, string contextTypeName, ICsvDataContextDriverProperties properties)
{
this.contextNameSpace = contextNameSpace;
this.contextTypeName = contextTypeName;
this.properties = properties;
}
public void GetSchemaAndBuildAssemblyTest(ICsvDataContextDriverProperties properties, string id)
{
File.WriteAllText("TestA.csv",
@"a,b,c,TestAID
1,2,3,1
1,2,3,2
x");
File.WriteAllText("TestB.csv",
@"c,d,e,TestAID
1,2,3,1
1,2,3,2
x");
string nameSpace = "TestContextNamespace";
string contextTypeName = "TestContextClass";
var contextAssemblyName = new AssemblyName("TestContextAssembly" + id)
{
CodeBase = "TestContextAssembly" + id + ".dll"
};
if (File.Exists(contextAssemblyName.CodeBase))
{
File.Delete(contextAssemblyName.CodeBase);
}
var explorerItems = SchemaBuilder.GetSchemaAndBuildAssembly(
properties,
contextAssemblyName,
ref nameSpace,
ref contextTypeName
);
//debug info to console
Console.WriteLine(explorerItems[0].DragText);
Console.WriteLine(explorerItems[1].DragText);
//check returned explorer tree
Assert.AreEqual(3, explorerItems.Count, "explorer items count");
explorerItems = explorerItems.Where(i => i.Kind == ExplorerItemKind.QueryableObject).ToList();
Assert.AreEqual("TestA,TestB", string.Join(",", explorerItems.Select(i => i.DragText)));
Assert.AreEqual("a,b,c,TestAID,TestB,c,d,e,TestAID,TestA", string.Join(",", explorerItems.SelectMany(i => i.Children.Select(c => c.DragText))));
//check compiled assembly
var contextAssembly = Assembly.Load(contextAssemblyName);
Assert.AreEqual("CsvDataContext,TTestA,TTestB", string.Join(",", contextAssembly.GetExportedTypes().Select(t => t.Name)));
var contextType = contextAssembly.GetType(nameSpace + "." + contextTypeName);
Assert.IsNotNull(contextType, "ContextType in assembly");
//check generated context runtime
dynamic contextInstance = contextType.GetConstructor(new Type[] {}).Invoke(new object[] {});
Assert.IsNotNull(contextInstance, "context created");
dynamic dataFirst = Enumerable.ToArray(contextInstance.TestA)[0];
Assert.AreEqual("3", dataFirst.c);
Assert.AreEqual(1, Enumerable.Count(dataFirst.TestB));
}
public static string GenerateCode(CsvDatabase db, ref string nameSpace, ref string typeName, ICsvDataContextDriverProperties props)
{
typeName = DefaultContextTypeName;
return(new CsvCSharpCodeGenerator(nameSpace, DefaultContextTypeName, props).GenerateSrcFile(db));
}
internal static List <ExplorerItem> GetSchemaAndBuildAssembly(ICsvDataContextDriverProperties csvDataContextDriverProperties, AssemblyName assemblyToBuild, ref string nameSpace, ref string typeName)
{
const ExplorerItemKind errorExplorerItemKind = ExplorerItemKind.ReferenceLink;
const ExplorerIcon errorExplorerIcon = ExplorerIcon.Box;
var csvDatabase = CsvDataModelGenerator.CreateModel(csvDataContextDriverProperties);
var(code, tableCodeGroups) = CsvCSharpCodeGenerator.GenerateCode(csvDatabase, ref nameSpace, ref typeName, csvDataContextDriverProperties);
var compileErrors = BuildAssembly(code, assemblyToBuild);
var hasCompilationErrors = compileErrors.Any();
var schema = GetSchema(csvDatabase);
var index = 0;
if (hasCompilationErrors || csvDataContextDriverProperties.DebugInfo)
{
schema.Insert(index++, new ExplorerItem("Data context source code", ExplorerItemKind.Schema, ExplorerIcon.Schema)
{
ToolTipText = "Drag&drop context source code to text window",
DragText = code
});
}
var exceptions = csvDatabase.Exceptions;
if (exceptions.Any())
{
var fileOrFolder = $"{exceptions.Pluralize("file")} or {exceptions.Pluralize("folder")}";
schema.Insert(index++, new ExplorerItem($"{exceptions.Count} {fileOrFolder} {exceptions.Pluralize("was", "were")} not processed", errorExplorerItemKind, errorExplorerIcon)
{
ToolTipText = $"Drag&drop {fileOrFolder} processing {exceptions.Pluralize("error")} to text window",
DragText = exceptions.Select(exception => exception.Message).JoinNewLine()
});
}
if (hasCompilationErrors)
{
schema.Insert(0, new ExplorerItem("Data context compilation failed", errorExplorerItemKind, errorExplorerIcon)
{
ToolTipText = "Drag&drop data context compilation errors to text window",
DragText = compileErrors.JoinNewLine()
});
}
else
{
foreach (var tableCodeGroup in tableCodeGroups.Where(codeGroup => codeGroup.Count() > 1))
{
var codeNames = tableCodeGroup.Select(typeCodeResult => typeCodeResult.CodeName).ToImmutableList();
var similarFilesSize = tableCodeGroup.Select(typeCodeResult => typeCodeResult.FilePath).GetHumanizedFileSize();
var filePaths = new HashSet <string>(codeNames);
var similarFilesCount = codeNames.Count;
schema.Insert(index++, new ExplorerItem($"{codeNames.First()} similar files joined data ({similarFilesCount}/{csvDatabase.Files.Count} files {similarFilesSize})", ExplorerItemKind.QueryableObject, ExplorerIcon.View)
{
Children = schema.Where(IsSimilarFile).ToList(),
IsEnumerable = true,
ToolTipText =
$"Drag&drop {similarFilesCount} similar files joined data to text window".JoinNewLine(
string.Empty,
$"{string.Join(Environment.NewLine, similarFilesCount <= 3 ? codeNames : codeNames.Take(2).Concat(new []{ "..." }).Concat(codeNames.Skip(similarFilesCount - 1)))}"),
DragText = $@"new []
{{
{string.Join(Environment.NewLine, codeNames.Select(n => $"\t{n},"))}
}}.SelectMany(_ => _)
"
});
if (!csvDataContextDriverProperties.ShowSameFilesNonGrouped)
{
schema.RemoveAll(IsSimilarFile);
}
bool IsSimilarFile(ExplorerItem explorerItem) =>
filePaths !.Contains(explorerItem.Tag);
}
}
if (!csvDatabase.Tables.Any())
{
schema.Insert(0, new ExplorerItem("No files found", ExplorerItemKind.Schema, ExplorerIcon.Box));
}
return(schema);
}
private CsvCSharpCodeGenerator(string contextNameSpace, string contextTypeName, ICsvDataContextDriverProperties properties)
{
_contextNameSpace = contextNameSpace;
_contextTypeName = contextTypeName;
_properties = properties;
}
/// <summary>
/// Get LINQPad Schema from CSV data model
/// </summary>
/// <param name="db"></param>
/// <returns></returns>
private static List<ExplorerItem> GetSchema(CsvDatabase db, ICsvDataContextDriverProperties props)
{
var schema = (
from table in db.Tables ?? Enumerable.Empty<CsvTable>()
select new ExplorerItem(table.DisplayName, ExplorerItemKind.QueryableObject, ExplorerIcon.Table) {
DragText = table.CodeName,
IsEnumerable = true,
ToolTipText = table.FilePath,
Children = (
from c in table.Columns ?? Enumerable.Empty<CsvColumn>()
select new ExplorerItem(c.DisplayName, ExplorerItemKind.Property, ExplorerIcon.Column)
{
DragText = c.CodeName,
ToolTipText = (c.CsvColumnIndex + 1) + ":" + c.CsvColumnName,
}
).Concat(
from r in table.Relations
select new ExplorerItem(r.DisplayName, ExplorerItemKind.CollectionLink, ExplorerIcon.ManyToMany)
{
DragText = r.CodeName,
ToolTipText = string.Format("Relation to {0} where {1}.{2} == {3}.{4}", r.TargetTable.CodeName, r.SourceTable.CodeName, r.SourceColumn.CodeName, r.TargetTable.CodeName, r.TargetColumn.CodeName ),
}
).ToList(),
}
).ToList();
return schema;
}
GenerateCode(CsvDatabase db, ref string nameSpace, ref string typeName, ICsvDataContextDriverProperties props) =>
