public async Task TestTransformerFileToLines()
{
#region arrange
var dir = AppContext.BaseDirectory;
var folderSql = Path.Combine(dir, "SqlToExecute");
var receiverFolder = new ReceiverFolderHierarchical(folderSql, "*.sql");
DirectoryInfo di = new DirectoryInfo(folderSql);
Console.WriteLine($"start files in {folderSql}");
foreach (var item in di.EnumerateFiles("*.sql"))
{
Console.WriteLine($"File {item.FullName}");
}
Console.WriteLine($"end files in {folderSql}");
#endregion
#region act
var transformer = new TransformerFileToLines();
var j = new SimpleJob();
j.Receivers.Add(0, receiverFolder);
j.FiltersAndTransformers.Add(0, transformer);
await j.Execute();
#endregion
#region assert
receiverFolder.valuesRead?.Length.ShouldBeGreaterThan(1);
transformer.valuesTransformed.ShouldNotBeNull();
var files = transformer.valuesTransformed.GroupBy(it => it.Values["FullName"]).ToArray();
files.Length.ShouldBeGreaterThan(1);
var file1Len = files.First().Count();
var file2Len = files.Last().Count();
file1Len.ShouldBeGreaterThan(0);
file2Len.ShouldBeGreaterThan(0);
(file1Len + file2Len).ShouldBeGreaterThan(2);
#endregion
}
static async Task MainAsync(string[] args)
{
var sj = new SimpleJob();
IReceive folder = new ReceiverFolderHierarchical(@"D:\test", "*.txt");
//await folder.LoadData();
var t = new TransformerFileToLines();
t.TrimEmptyLines = false;
//t.valuesRead = folder.valuesRead;
//await t.Run();
var addBak = new TransformerFieldAddString("FullName", "FullName", ".bak");
addBak.valuesRead = t.valuesTransformed;
//await addBak.Run();
//var regex = new TransformRowRegex(@"^(?x<ip>123).*?$","text");
var regex = new TransformRowRegexReplaceGuid(@"^.*x(?<ip>\w+)123.*?$", "text");
regex.ReplaceAllNextOccurences = true;
//regex.valuesRead = addBak.valuesTransformed;
//await regex.Run();
var file = new SenderByRowToFile("FullName", "lineNr", "text", ".txt");
file.FileMode = System.IO.FileMode.Append;
//file.valuesToBeSent = regex.valuesTransformed;
//await file.Send();
sj
.AddReceiver(folder)
.AddTransformer(t)
//.AddTransformer(addBak)
.AddTransformer(regex)
.AddSender(file);
File.WriteAllText("def.txt", sj.SerializeMe());
await sj.Execute();
}
static string PBXJob()
{
var dir = AppContext.BaseDirectory;
var dirPBX = Path.Combine(dir, "PBX");
var serialize = new SerializeDataOnFile("a.txt");
IReceive r = new ReceiverFolderHierarchical(dirPBX, "*.log");
IFilter filterFiles = new FilterForFilesHierarchical();
#region filter for remove dates serialize
var filterDateTime = new FilterComparableGreat(typeof(DateTime), DateTime.MinValue, "LastWriteTimeUtc");
IFilter filterDateTimeSerializable = new FilterComparableFromSerializable(filterDateTime, serialize);
#endregion
IFilter removeFilesMaxWritten = new FilterRemovePropertyMaxMinDateTime("LastWriteTimeUtc", GroupingFunctions.Max);
ITransform transformLines = new TransformerFileToLines()
{
TrimEmptyLines = true
};
var trDateRegex = new TransformRowRegex(@"^Date:\ (?<datePBX>.{23}).*?$", "text");
var trKeyValueRegex = new TransformRowRegex(@"(^|\r|\n|\r\n)(?<tkey>(\$?[a-zA-Z0-9_[\]/ ]+))='?(?<tvalue>([a-zA-Z0-9_ ]+))'?(\r|\r\n|$)", "text");
var trToDate = new TransformerFieldStringToDate("datePBX", "NewDatePBX", "yyyy/MM/dd HH:mm:ss.fff");
var trAddDate = new TransformAddFieldDown("NewDatePBX");
var trAddKey = new TransformAddNewField("tkey");
var trAddValue = new TransformAddNewField("tvalue");
var trSimpleFields = new TransformRowRemainsProperties("NewDatePBX", "lineNr", "text", "FullName", "LastWriteTimeUtc", "tkey", "tvalue");
var data = new DBTableDataConnection <SqlConnection>(new SerializeDataInMemory());
data.ConnectionString = "#file:SqlServerConnectionString#";
data.Fields = new string[] { "NewDatePBX", "lineNr", "text", "FullName", "tkey", "tvalue" };
data.TableName = "PBXData";
var bulk = new SenderSqlServerBulkCopy(data);
var md = new MediaTransformMaxMin <DateTime>();
md.GroupFunction = GroupingFunctions.Max;
md.FieldName = "LastWriteTimeUtc";
var serializeMaxDate = new SenderMediaSerialize <DateTime>(serialize, "LastWriteTimeUtc", md);
var si = new SimpleJob();
si.Receivers.Add(0, r);
int iFilterNr = 0;
si.FiltersAndTransformers.Add(iFilterNr++, filterFiles);
si.FiltersAndTransformers.Add(iFilterNr++, filterDateTimeSerializable);
si.FiltersAndTransformers.Add(iFilterNr++, removeFilesMaxWritten);
si.FiltersAndTransformers.Add(iFilterNr++, transformLines);
si.FiltersAndTransformers.Add(iFilterNr++, trDateRegex);
si.FiltersAndTransformers.Add(iFilterNr++, trKeyValueRegex);
si.FiltersAndTransformers.Add(iFilterNr++, trToDate);
si.FiltersAndTransformers.Add(iFilterNr++, trAddDate);
si.FiltersAndTransformers.Add(iFilterNr++, trAddKey);
si.FiltersAndTransformers.Add(iFilterNr++, trAddValue);
si.FiltersAndTransformers.Add(iFilterNr++, trSimpleFields);
//TODO: add transformer to add a field down for all fields
//TODO: add transformer regex for splitting Key=Value
//TODO: add field to separate Conn(1)Type(Any)User(InternalTask) CDL Request:RSVI(Get)
si.Senders.Add(0, bulk);
si.Senders.Add(1, serializeMaxDate);
return(si.SerializeMe());
}
public async Task TestPBXData()
{
#region arrange
string connectionString = GetSqlServerConnectionString();
using (var conn = new SqlConnection(connectionString))
{
await conn.OpenAsync();
using (var cmd = conn.CreateCommand())
{
cmd.CommandType = CommandType.Text;
cmd.CommandText = @"
IF OBJECT_ID('dbo.PBXData', 'U') IS NOT NULL
DROP TABLE dbo.PBXData;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = @"CREATE TABLE [PBXData](
[NewDatePBX] [datetime] NOT NULL,
[lineNr] [int] NOT NULL,
[text] [nvarchar](500) NOT NULL,
[FullName] [nvarchar](500) NOT NULL
)";
await cmd.ExecuteNonQueryAsync();
}
await Task.Delay(2000);
var dir = AppContext.BaseDirectory;
var dirPBX = Path.Combine(dir, "PBX");
File.AppendAllText(Path.Combine(dirPBX, "PBXRemove.log"), "aaaa");
#endregion
#region act
var serialize = new SerializeDataOnFile("a.txt");
IReceive r = new ReceiverFolderHierarchical(dirPBX, "*.log");
IFilter filterFiles = new FilterForFilesHierarchical();
#region filter for remove dates serialize
var filterDateTime = new FilterComparableGreat(typeof(DateTime), DateTime.MinValue, "LastWriteTimeUtc");
IFilter filterDateTimeSerializable = new FilterComparableFromSerializable(filterDateTime, serialize);
#endregion
IFilter removeFilesMaxWritten = new FilterRemovePropertyMaxMinDateTime("LastWriteTimeUtc", GroupingFunctions.Max);
ITransform transformLines = new TransformerFileToLines()
{
TrimEmptyLines = true
};
var trDateRegex = new TransformRowRegex(@"^Date:\ (?<datePBX>.{23}).*?$", "text");
var trToDate = new TransformerFieldStringToDate("datePBX", "NewDatePBX", "yyyy/MM/dd HH:mm:ss.fff");
var trAddDate = new TransformAddFieldDown("NewDatePBX");
var trSimpleFields = new TransformRowRemainsProperties("NewDatePBX", "lineNr", "text", "FullName", "LastWriteTimeUtc");
var data = new DBTableDataConnection <SqlConnection>(new SerializeDataInMemory());
data.ConnectionString = GetSqlServerConnectionString();
data.Fields = new string[] { "NewDatePBX", "lineNr", "text", "FullName" };
data.TableName = "PBXData";
var bulk = new SenderSqlServerBulkCopy(data);
var md = new MediaTransformMaxMin <DateTime>();
md.GroupFunction = GroupingFunctions.Max;
md.FieldName = "LastWriteTimeUtc";
var serializeMaxDate = new SenderMediaSerialize <DateTime>(serialize, "LastWriteTimeUtc", md);
var si = new SimpleJob();
si.Receivers.Add(0, r);
int iFilterNr = 0;
si.FiltersAndTransformers.Add(iFilterNr++, filterFiles);
si.FiltersAndTransformers.Add(iFilterNr++, filterDateTimeSerializable);
si.FiltersAndTransformers.Add(iFilterNr++, removeFilesMaxWritten);
si.FiltersAndTransformers.Add(iFilterNr++, transformLines);
si.FiltersAndTransformers.Add(iFilterNr++, trDateRegex);
si.FiltersAndTransformers.Add(iFilterNr++, trToDate);
si.FiltersAndTransformers.Add(iFilterNr++, trAddDate);
si.FiltersAndTransformers.Add(iFilterNr++, trSimpleFields);
//TODO: add transformer to add a field down for all fields
//TODO: add transformer regex for splitting Key=Value
//TODO: add field to separate Conn(1)Type(Any)User(InternalTask) CDL Request:RSVI(Get)
si.Senders.Add(0, bulk);
si.Senders.Add(1, serializeMaxDate);
await si.Execute();
#endregion
#region assert
filterFiles.valuesTransformed.Length.ShouldBe(3, "three files after first filter");
removeFilesMaxWritten.valuesTransformed.Length.ShouldBe(2, "last one file written dismissed");
transformLines.valuesTransformed.Length.ShouldBe(77251);
var d = transformLines.valuesTransformed.Select(it => it.Values["FullName"]).Distinct().ToArray();
d.Length.ShouldBe(2, "two files after reading contents");
//transformGroupingFiles.valuesTransformed.Length.ShouldBe(2);
//trDateRegex.valuesTransformed.Count(it => it.Values.ContainsKey("datePBX")).ShouldBeGreaterThan(0,"datePBX");
//trToDate.valuesTransformed.Count(it => it.Values.ContainsKey("NewDatePBX")).ShouldBeGreaterThan(0,"NewDatePBX");
foreach (var item in trAddDate.valuesTransformed)
{
item.Values.ShouldContainKey("NewDatePBX");
}
foreach (var item in trSimpleFields.valuesTransformed)
{
item.Values.Keys.Count.ShouldBe(5);
}
using (var con = new SqlConnection(connectionString))
{
await con.OpenAsync();
using (var cmd = con.CreateCommand())
{
cmd.CommandType = CommandType.Text;
cmd.CommandText = "select count(*) from PBXData";
var val = await cmd.ExecuteScalarAsync();
val.ShouldBe(77251);
}
}
md.Result.Year.ShouldBe(DateTime.Now.Year);
#endregion
#region arange to read again
r = new ReceiverFolderHierarchical(dirPBX, "*.log");
filterFiles = new FilterForFilesHierarchical();
#region filter for remove dates serialize
filterDateTime = new FilterComparableGreat(typeof(DateTime), DateTime.MinValue, "LastWriteTimeUtc");
filterDateTimeSerializable = new FilterComparableFromSerializable(filterDateTime, serialize);
#endregion
#endregion
#region act
si = new SimpleJob();
si.Receivers.Add(0, r);
iFilterNr = 0;
si.FiltersAndTransformers.Add(iFilterNr++, filterFiles);
si.FiltersAndTransformers.Add(iFilterNr++, filterDateTimeSerializable);
await si.Execute();
#endregion
#region assert
filterDateTime.valuesTransformed?.Length.ShouldBe(1, "next time 1 file read - the added one");
#endregion
}
}