public FileImportResult Import(FileInspectionRequest request, TflConnection output) {
var fileInformation = FileInformationFactory.Create(request, _logger);
var cfg = BuildProcess(fileInformation, request, output, _logger);
if (cfg.Connections.First(c => c.Name == "output").Provider == "internal") {
// nothing to init, so just run in default mode
return new FileImportResult {
Information = fileInformation,
Rows = ProcessFactory.CreateSingle(cfg, _logger).Execute()
};
}
// first run in init mode
cfg.Mode = "init";
var process = ProcessFactory.CreateSingle(cfg, _logger, new Options { Mode = "init" });
process.ExecuteScaler();
// now run in default mode
cfg.Mode = "default";
process = ProcessFactory.CreateSingle(cfg, _logger, new Options() { Mode = "default" });
return new FileImportResult {
Information = fileInformation,
Rows = process.Execute(),
RowCount = process.Entities[0].Inserts
};
}
public SqlServerConnection(TflConnection element, AbstractConnectionDependencies dependencies)
: base(element, dependencies) {
Type = ProviderType.SqlServer;
L = "[";
R = "]";
TextQualifier = "'";
IsDatabase = true;
InsertMultipleRows = true;
Top = true;
NoLock = true;
TableVariable = true;
NoCount = true;
IndexInclude = true;
Views = true;
Schemas = true;
MaxDop = true;
TableSample = true;
DefaultSchema = "dbo";
ConnectionStringProperties.UserProperty = "User Id";
ConnectionStringProperties.PasswordProperty = "Password";
ConnectionStringProperties.DatabaseProperty = "Database";
ConnectionStringProperties.ServerProperty = "Server";
ConnectionStringProperties.TrustedProperty = "Trusted_Connection";
ConnectionStringProperties.PersistSecurityInfoProperty = "Persist Security Info";
}
public SqlCeConnection(TflConnection element, AbstractConnectionDependencies dependencies)
: base(element, dependencies) {
Type = ProviderType.SqlCe4;
L = "[";
R = "]";
IsDatabase = true;
ConnectionStringProperties.ServerProperty = "Data Source";
ConnectionStringProperties.PersistSecurityInfoProperty = "Persist Security Info";
}
public SolrConnection(TflConnection element, AbstractConnectionDependencies dependencies)
: base(element, dependencies) {
Type = ProviderType.Solr;
IsDatabase = true;
TextQualifier = string.Empty;
_solrUrl = element.NormalizeUrl(DEFAULT_PORT);
if (!element.File.Equals(string.Empty)) {
_schemaFile = element.File;
}
}
public MySqlConnection(TflConnection element, AbstractConnectionDependencies dependencies)
: base(element, dependencies) {
Type = ProviderType.MySql;
L = "`";
R = "`";
TextQualifier = "'";
IsDatabase = true;
Views = true;
ConnectionStringProperties.UserProperty = "Uid";
ConnectionStringProperties.PasswordProperty = "Pwd";
ConnectionStringProperties.PortProperty = "Port";
ConnectionStringProperties.DatabaseProperty = "Database";
ConnectionStringProperties.ServerProperty = "Server";
}
/* Pooling=true;Min Pool Size=0;Max Pool Size=100;Connection Lifetime=0; */
public PostgreSqlConnection(TflConnection element, AbstractConnectionDependencies dependencies)
: base(element, dependencies) {
Type = ProviderType.PostgreSql;
L = "\"";
R = "\"";
TextQualifier = "'";
IsDatabase = true;
InsertMultipleRows = true;
Views = true;
Schemas = true;
DefaultSchema = "public";
ConnectionStringProperties.UserProperty = "User ID";
ConnectionStringProperties.PasswordProperty = "Password";
ConnectionStringProperties.PortProperty = "Port";
ConnectionStringProperties.DatabaseProperty = "Database";
ConnectionStringProperties.ServerProperty = "Host";
NoLock = false;
}
public MailConnection(TflConnection element, AbstractConnectionDependencies dependencies)
: base(element, dependencies) {
Type = ProviderType.Mail;
var port = Port > 0 ? Port : 25;
if (string.IsNullOrEmpty(User)) {
_smtpClient = new SmtpClient {
Port = port,
EnableSsl = EnableSsl,
Host = Server
};
} else {
_smtpClient = new SmtpClient {
Port = port,
EnableSsl = EnableSsl,
DeliveryMethod = SmtpDeliveryMethod.Network,
UseDefaultCredentials = false,
Credentials = new NetworkCredential(User, Password),
Host = Server
};
}
}
private static TflProcess BuildProcess(
FileInformation fileInformation,
FileInspectionRequest request,
TflConnection output,
ILogger logger) {
var process = new TflRoot().GetDefaultOf<TflProcess>(p => {
p.Name = request.EntityName;
p.Star = request.ProcessName;
p.StarEnabled = false;
p.ViewEnabled = false;
p.PipelineThreading = "MultiThreaded";
p.Connections = new List<TflConnection> {
p.GetDefaultOf<TflConnection>(c => {
c.Name = "input";
c.Provider = "file";
c.File = fileInformation.FileInfo.FullName;
c.Delimiter = fileInformation.Delimiter == default(char)
? "|"
: fileInformation.Delimiter.ToString(CultureInfo.InvariantCulture);
c.Start = fileInformation.FirstRowIsHeader ? 2 : 1;
}),
output
};
p.Entities = new List<TflEntity> {
p.GetDefaultOf<TflEntity>(e => {
e.Name = request.EntityName;
e.Connection = "input";
e.PrependProcessNameToOutputName = false;
e.DetectChanges = false;
e.Fields = GetFields(p, new FieldInspector(logger).Inspect(fileInformation, request), logger, request.EntityName);
})
};
});
return process;
}
private static TflProcess BuildProcess(
FileInformation fileInformation,
FileInspectionRequest request,
TflConnection output,
ILogger logger) {
var process = new TflProcess{
Name = request.EntityName,
Star = request.ProcessName,
StarEnabled = false,
ViewEnabled = false,
PipelineThreading = "MultiThreaded",
Connections = new List<TflConnection> {
new TflConnection{
Name = "input",
Provider = "file",
File = fileInformation.FileInfo.FullName,
Delimiter = fileInformation.Delimiter == default(char)
? "|"
: fileInformation.Delimiter.ToString(CultureInfo.InvariantCulture),
Start = fileInformation.FirstRowIsHeader ? 2 : 1
}.WithDefaults(),
output
},
Entities = new List<TflEntity> {
new TflEntity{
Name = request.EntityName,
Connection = "input",
PrependProcessNameToOutputName = false,
DetectChanges = false,
Fields = GetFields(new FieldInspector(logger).Inspect(fileInformation, request), logger, request.EntityName)
}.WithDefaults()
}
}.WithDefaults();
return process;
}
public FileImportResult Import(string fileName, TflConnection output) {
return Import(new FileInspectionRequest(fileName), output);
}
/// <summary>
/// Imports a file to an output with default file inspection settings
/// </summary>
/// <param name="fileName"></param>
/// <param name="output"></param>
/// <returns></returns>
public long ImportScaler(string fileName, TflConnection output) {
return ImportScaler(new FileInspectionRequest(fileName), output);
}
public void TestCopyFileToConnection() {
var file1 = Path.GetTempFileName();
File.WriteAllText(file1, "f1,f2,f3\nv1,v2,v3");
var element = new ProcessBuilder("CopyFile")
.Connection("output")
.Provider("SqlServer")
.Database("TestOutput")
.Action("Copy")
.From(file1)
.To("output")
.Process();
var process = ProcessFactory.CreateSingle(element, new TestLogger());
process.ExecuteScaler();
var expected = Common.CleanIdentifier(Path.GetFileNameWithoutExtension(file1));
var sqlServer = new TflConnection{
Name = "test",
Provider = "sqlserver",
Database = "TestOutput"
}.WithDefaults().Connection;
var rows = sqlServer.GetConnection().Query($"SELECT f1, f2, f3 FROM {expected}").ToArray();
Assert.AreEqual(1, rows.Length);
Assert.AreEqual("v1", rows[0].f1);
Assert.AreEqual("v2", rows[0].f2);
Assert.AreEqual("v3", rows[0].f3);
}
public FolderConnection(TflConnection element, AbstractConnectionDependencies dependencies)
: base(element, dependencies) {
Type = ProviderType.Folder;
}
public ElasticSearchConnection(TflConnection element, AbstractConnectionDependencies dependencies)
: base(element, dependencies) {
Type = ProviderType.ElasticSearch;
IsDatabase = true;
TextQualifier = string.Empty;
}
protected AbstractConnection(TflConnection element, AbstractConnectionDependencies dependencies) {
Source = element;
BatchSize = element.BatchSize;
Name = element.Name;
Start = element.Start;
End = element.End;
File = element.File;
Folder = element.Folder;
Delimiter = element.Delimiter == string.Empty ? default(char) : element.Delimiter[0];
DateFormat = element.DateFormat;
Header = element.Header;
Footer = element.Footer;
EnableSsl = element.EnableSsl;
Direct = element.Direct;
Encoding = element.Encoding;
Version = element.Version;
CheckMe = element.Check;
Path = element.Path;
ErrorMode = (ErrorMode)Enum.Parse(typeof(ErrorMode), element.ErrorMode, true);
SearchOption = (SearchOption)Enum.Parse(typeof(SearchOption), element.SearchOption, true);
SearchPattern = element.SearchPattern;
ProcessConnectionString(element);
TableQueryWriter = dependencies.TableQueryWriter;
ConnectionChecker = dependencies.ConnectionChecker;
EntityRecordsExist = dependencies.EntityRecordsExist;
EntityDropper = dependencies.EntityDropper;
EntityCreator = dependencies.EntityCreator;
ViewWriters = dependencies.ViewWriters;
TflWriter = dependencies.TflWriter;
ScriptRunner = dependencies.ScriptRunner;
DataTypeService = dependencies.DataTypeService;
Logger = dependencies.Logger;
Is = new ConnectionIs(this);
}
private void ProcessConnectionString(TflConnection element) {
if (element.ConnectionString != string.Empty) {
Database = ConnectionStringParser.GetDatabaseName(element.ConnectionString);
Server = ConnectionStringParser.GetServerName(element.ConnectionString);
User = ConnectionStringParser.GetUsername(element.ConnectionString);
Password = ConnectionStringParser.GetPassword(element.ConnectionString);
PersistSecurityInfo = ConnectionStringParser.GetPersistSecurityInfo(element.ConnectionString);
} else {
Server = element.Server;
Database = element.Database;
User = element.User;
Password = element.Password;
Port = element.Port;
}
}
public WebConnection(TflConnection element, AbstractConnectionDependencies dependencies)
: base(element, dependencies) {
_element = element;
Type = ProviderType.Web;
}
/// <summary>
/// Imports a file to an output with provided file inpections settings
/// </summary>
/// <param name="request"></param>
/// <param name="output"></param>
/// <returns></returns>
public long ImportScaler(FileInspectionRequest request, TflConnection output) {
return Import(request, output).RowCount;
}
public LuceneConnection(TflConnection element, AbstractConnectionDependencies dependencies)
: base(element, dependencies) {
Type = ProviderType.Lucene;
IsDatabase = true;
TextQualifier = string.Empty;
}
public InternalConnection(TflConnection element, AbstractConnectionDependencies dependencies)
: base(element, dependencies) {
Type = ProviderType.Internal;
}
public ConsoleConnection(TflConnection element, AbstractConnectionDependencies dependencies)
: base(element, dependencies) {
Type = ProviderType.Console;
}
public LogConnection(TflConnection element, AbstractConnectionDependencies dependencies)
: base(element, dependencies) {
Type = ProviderType.Log;
}
public AnalysisServicesConnection(TflConnection element, AbstractConnectionDependencies dependencies)
: base(element, dependencies) {
Type = ProviderType.AnalysisServices;
ConnectionStringProperties.DatabaseProperty = "Catalog";
ConnectionStringProperties.ServerProperty = "Data Source";
}