public void Start(TflProcess process) {
if (_started)
return;
_started = true;
Name = process.Name;
foreach (var log in process.Log) {
switch (log.Provider) {
case "file":
log.Folder = log.Folder.Replace('/', '\\');
log.File = log.File.Replace('/', '\\');
log.Folder = (log.Folder.Equals(Common.DefaultValue) ? "logs" : log.Folder).TrimEnd('\\') + "\\";
log.File = (log.File.Equals(Common.DefaultValue) ? "tfl-" + process.Name + ".log" : log.File).TrimStart('\\');
var fileListener = new ObservableEventListener();
fileListener.EnableEvents(TflEventSource.Log, (EventLevel)Enum.Parse(typeof(EventLevel), log.Level));
SinkSubscriptions.Add(fileListener.LogToRollingFlatFile(log.Folder + log.File, 5000, "yyyy-MM-dd", RollFileExistsBehavior.Increment, RollInterval.Day, new LegacyLogFormatter(), 0, log.Async));
EventListeners.Add(fileListener);
break;
case "mail":
if (log.Subject.Equals(Common.DefaultValue)) {
log.Subject = process.Name + " " + log.Level;
}
var mailListener = new ObservableEventListener();
mailListener.EnableEvents(TflEventSource.Log, EventLevel.Error);
SinkSubscriptions.Add(mailListener.LogToEmail(log));
EventListeners.Add(mailListener);
break;
}
}
}
public ShortHandFactory GetFactory(TflField field = null) {
var process = new TflProcess {
Name = "Test",
Entities = new List<TflEntity>{
new TflEntity { Name = "Test" }.WithDefaults()
}
}.WithDefaults();
if (field != null) {
process.Entities.First().Fields.Add(field);
}
return new ShortHandFactory(process);
}
// Custom constructor takeing an already created TflProcess
public TflRoot(TflProcess process)
{
var root = new TflRoot {
Processes = new List <TflProcess> {
process
}
};
var json = JsonConvert.SerializeObject(root, new JsonSerializerSettings()
{
NullValueHandling = NullValueHandling.Ignore
});
Load(json);
}
private static TflProcess BuildProcess(
FileInformation fileInformation,
FileInspectionRequest request,
TflConnection output,
ILogger logger) {
var process = new TflProcess{
Name = request.EntityName,
Star = request.ProcessName,
StarEnabled = false,
ViewEnabled = false,
PipelineThreading = "MultiThreaded",
Connections = new List<TflConnection> {
new TflConnection{
Name = "input",
Provider = "file",
File = fileInformation.FileInfo.FullName,
Delimiter = fileInformation.Delimiter == default(char)
? "|"
: fileInformation.Delimiter.ToString(CultureInfo.InvariantCulture),
Start = fileInformation.FirstRowIsHeader ? 2 : 1
}.WithDefaults(),
output
},
Entities = new List<TflEntity> {
new TflEntity{
Name = request.EntityName,
Connection = "input",
PrependProcessNameToOutputName = false,
DetectChanges = false,
Fields = GetFields(new FieldInspector(logger).Inspect(fileInformation, request), logger, request.EntityName)
}.WithDefaults()
}
}.WithDefaults();
return process;
}
// Custom constructor takeing an already created TflProcess
public TflRoot(TflProcess process) {
var root = new TflRoot { Processes = new List<TflProcess> { process } };
var json = JsonConvert.SerializeObject(root, new JsonSerializerSettings() {NullValueHandling = NullValueHandling.Ignore});
Load(json);
}
public ShortHandFactory(TflProcess process) {
_process = process;
_maps = process.Maps.ToDictionary(m => m.Name, m => default(byte));
_guard = new TflTransform { Method = "guard" }.WithDefaults();
_methods = new Dictionary<string, string> {
{"add", "add"},
{"append","append"},
{"collapse","collapse"},
{"compress","compress"},
{"concat","concat"},
{"convert","convert"},
{"copy","copy"},
{"cs","csharp"},
{"csharp","csharp"},
{"cyrtolat","cyrtolat"},
{"decompress","decompress"},
{"datepart","datepart"},
{"distinctwords","distinctwords"},
{"elipse","elipse"},
{"format","format"},
{"formatphone","formatphone"},
{"fromjson","fromjson"},
{"fromregex", "fromregex"},
{"fromsplit","fromsplit"},
{"fromxml","fromxml"},
{"gethashcode","hashcode"},
{"guid","guid"},
{"hashcode","hashcode"},
{"htmlencode","htmlencode"},
{"if","if"},
{"iif","if"},
{"insert","insert"},
{"insertinterval","insertinterval"},
{"isdaylightsavings","isdaylightsavings"},
{"javascript","javascript"},
{"join","join"},
{"js","javascript"},
{"left","left"},
{"lower","tolower"},
{"map","map"},
{"now","now"},
{"utcnow","utcnow"},
{"padleft","padleft"},
{"padright","padright"},
{"phone","formatphone"},
{"razor","template"},
{"regexreplace","regexreplace"},
{"remove","remove"},
{"replace","replace"},
{"right","right"},
{"slug","slug"},
{"slugify","slug"},
{"striphtml","striphtml"},
{"substr","substring"},
{"substring","substring"},
{"sum", "add"},
{"tag","tag"},
{"template","template"},
{"timezone","timezone"},
{"titlecase","totitlecase"},
{"tojson","tojson"},
{"toint","toint"},
{"toint32","toint"},
{"tolower","tolower"},
{"tostring","tostring"},
{"totitlecase","totitlecase"},
{"toupper","toupper"},
{"toyesno","toyesno"},
{"transliterate","transliterate"},
{"trim","trim"},
{"trimend","trimend"},
{"trimstart","trimstart"},
{"trimstartappend","trimstartappend"},
{"upper","toupper"},
{"urlencode","urlencode"},
{"velocity","velocity"},
{"web","web"},
{"timespan","timespan"},
{"isempty","isempty"},
{"isblank","isempty"},
{"equals","equals"},
{"isequalto", "equals"},
{"xmlencode","xmlencode"},
{"xmldecode","htmldecode"},
{"htmldecode", "htmldecode"},
{"timeago","timeago"},
{"timeahead","timeahead"},
{"weekofyear","weekofyear"}
};
_functions = new Dictionary<string, Func<string, TflField, TflTransform, TflTransform>> {
{"replace", Replace},
{"left", Left},
{"right", Right},
{"append", Append},
{"if", If},
{"convert", Convert},
{"copy", Copy},
{"concat", Concat},
{"hashcode", GetHashCode},
{"compress", Compress},
{"datepart", DatePart},
{"decompress", Decompress},
{"elipse", Elipse},
{"regexreplace", RegexReplace},
{"striphtml", StripHtml},
{"join", Join},
{"format", Format},
{"insert", Insert},
{"insertinterval", InsertInterval},
{"transliterate", Transliterate},
{"slug", Slug},
{"cyrtolat", CyrToLat},
{"distinctwords", DistinctWords},
{"guid", (arg, root, last) =>new TflTransform{Method = "guid", IsShortHand = true}.WithDefaults()},
{"now", (arg, root, last) =>new TflTransform{Method = "now", IsShortHand = true}.WithDefaults()},
{"utcnow", (arg, root, last) =>new TflTransform{Method = "utcnow", IsShortHand = true}.WithDefaults()},
{"remove", Remove},
{"trimstart", (arg, root, last) => new TflTransform{ Method = "trimstart", TrimChars = arg, IsShortHand = true}.WithDefaults()},
{"trimstartappend", TrimStartAppend},
{"trimend", (arg, root, last) => new TflTransform{ Method = "trimend", TrimChars = arg, IsShortHand = true}.WithDefaults()},
{"trim", (arg, root, last) => new TflTransform{ Method = "trim", TrimChars = arg, IsShortHand = true}.WithDefaults()},
{"substring", Substring},
{"map", Map},
{"urlencode", UrlEncode},
{"xmlencode", XmlEncode},
{"htmldecode", HtmlDecode},
{"web", Web},
{"add", Add},
{"fromjson", (arg,root, last) => NotImplemented("fromjson", root, last)},
{"padleft", PadLeft},
{"padright", PadRight},
{"tostring", ToString},
{"formatphone", FormatPhone},
{"tolower", ToLower},
{"toupper", ToUpper},
{"javascript", JavaScript},
{"csharp", CSharp},
{"template", Template},
{"totitlecase", ToTitleCase},
{"timezone", TimeZone},
{"toint", ToInt},
{"tojson", ToJson},
{"toyesno", ToYesNo},
{"fromxml", (arg,root, last) => NotImplemented("fromxml", root, last)},
{"fromregex", (arg,root, last) => NotImplemented("fromregex", root, last)},
{"fromsplit", (arg,root, last) => NotImplemented("fromsplit", root, last)},
{"velocity", Velocity},
{"tag", Tag},
{"htmlencode", HtmlEncode},
{"isdaylightsavings", IsDaylightSavings},
{"collapse", Collapse},
{"timespan", Timespan},
{"isempty", IsEmpty},
{"equals", Equals},
{"timeago", TimeAgo},
{"timeahead", TimeAhead},
{"weekofyear",WeekOfYear}
};
}
public Fields Inspect(FileInformation fileInformation, FileInspectionRequest request) {
var process = new TflProcess{
Name = request.ProcessName,
StarEnabled = false,
ViewEnabled = false,
PipelineThreading = "MultiThreaded"
}.WithDefaults();
process.Connections = new List<TflConnection> {
new TflConnection {
Name = "input",
Provider = "file",
File = fileInformation.FileInfo.FullName,
Delimiter = fileInformation.Delimiter == default(char)
? "|"
: fileInformation.Delimiter.ToString(CultureInfo.InvariantCulture),
Start = fileInformation.FirstRowIsHeader ? 2 : 1
}.WithDefaults(),
new TflConnection {
Name = "output",
Provider = "internal"
}.WithDefaults()
};
process.Entities.Add(new TflEntity {
Name = request.EntityName,
PrependProcessNameToOutputName = false,
DetectChanges = false,
Sample = System.Convert.ToInt32(request.Sample)
}.WithDefaults());
foreach (var fd in fileInformation.Fields) {
var field = fd;
process.Entities[0].Fields.Add(new TflField{
Name = field.Name,
Length = field.Length,
Type = field.Type,
QuotedWith = field.QuotedWith
}.WithDefaults());
}
for (var i = 0; i < request.DataTypes.Count; i++) {
var dataType = request.DataTypes[i];
foreach (var field in fileInformation.Fields) {
var result = IsDataTypeField(field.Name, dataType);
process.Entities[0].CalculatedFields.Add(
new TflField{
Name = result,
Type = "bool",
Input = false,
Transforms = new List<TflTransform> {
new TflTransform {
Method = "typeconversion",
Type = dataType,
Parameter = field.Name,
IgnoreEmpty = request.IgnoreEmpty
}.WithDefaults()
}
}.WithDefaults()
);
}
}
foreach (var field in fileInformation.Fields) {
var result = LengthField(field.Name);
process.Entities[0].CalculatedFields.Add(
new TflField {
Name = result,
Type = "int32",
Transforms = new List<TflTransform> {
new TflTransform {
Method = "length",
Parameter = field.Name
}.WithDefaults()
}
}.WithDefaults()
);
}
var runner = ProcessFactory.CreateSingle(new TflRoot(process).Processes[0], _logger);
var results = runner.Execute().ToList();
if (results.Count <= 0) {
_logger.Warn("Nothing imported from in {0}!", fileInformation.FileInfo.Name);
return fileInformation.Fields;
}
foreach (var field in fileInformation.Fields) {
if (!results.All(row => row[field.Name].Equals(string.Empty))) {
foreach (var dataType in request.DataTypes) {
var result = IsDataTypeField(field.Name, dataType);
if (!results.All(row => row[result].Equals(true)))
continue;
field.Type = dataType;
field.Length = request.MinLength.ToString(CultureInfo.InvariantCulture);
break;
}
}
if (!field.Type.Equals("string"))
continue;
var length = results.Max(row => (int)row[LengthField(field.Name)]) + 1;
if (request.MaxLength > 0 && length > request.MaxLength) {
length = request.MaxLength;
}
if (request.MinLength > 0 && length < request.MinLength) {
length = request.MinLength;
}
field.Length = length.ToString(CultureInfo.InvariantCulture);
}
return fileInformation.Fields;
}
public ProcessReader(TflProcess process, ILogger logger, Options options) {
_configuration = process;
_logger = logger;
_options = options;
_runner = GetRunner(DetermineMode());
}
private static string GetPipelineDescription(TflProcess process) {
var pipeline = process.PipelineThreading;
if (pipeline != "Default")
return pipeline;
if (process.Entities.All(e => e.PipelineThreading == "SingleThreaded")) {
pipeline = "SingleThreaded";
} else if (process.Entities.All(e => e.PipelineThreading == "MultiThreaded")) {
pipeline = "MultiThreaded";
} else {
pipeline = "Mixed";
}
return pipeline;
}
private static Dictionary<string, List<Row>> GetDataSets(TflProcess process) {
var dataSets = new Dictionary<string, List<Row>>();
if (!process.DataSets.Any())
return dataSets;
foreach (var dataSet in process.DataSets) {
dataSets[dataSet.Name] = new List<Row>();
var rows = dataSets[dataSet.Name];
foreach (var r in dataSet.Rows) {
var row = new Row();
foreach (var pair in r) {
row[pair.Key] = pair.Value;
}
rows.Add(row);
}
}
return dataSets;
}
public TransformFieldsToParametersAdapter(TflProcess process) {
_process = process;
}
public TransformFieldsMoveAdapter(TflProcess process) {
_process = process;
}
public void Start(TflProcess process) {
Name = process.Name;
Info("Start logging {0}!", process.Name);
}
public void Start(TflProcess process) {
}
public static Process CreateSingle(TflProcess process, ILogger logger, Options options = null) {
return Create(new List<TflProcess> { process }, logger, options)[0];
}
public NinjectBindings(TflProcess process, ILogger logger) {
_process = process;
_logger = logger;
}
public void Start(TflProcess process) {
if (!_reportedHost) {
EntityInfo(".", "Orchard version: {0}", _orchardVersion);
EntityInfo(".", "Transformalize.Orchard version: {0}", _moduleVersion);
EntityInfo(".", GetHost());
_reportedHost = true;
}
if (_level != "none") {
Info("{0} entit{1} in {2} mode.", process.Entities.Count, process.Entities.Count.Pluralize(), process.Mode == string.Empty ? "Default" : process.Mode);
Info("Running {0} with a {1} pipeline.", process.Parallel ? "Parallel" : "Serial", GetPipelineDescription(process));
}
if (!_orchardLogger.IsEnabled(LogLevel.Information))
return;
_orchardLogger.Information("TFL started logging process {0}.", Name);
}
