public async Task TestGreater() { int value = 3; #region arrange var f = new FilterComparableGreat(typeof(int), value, "ID"); f.valuesRead = GenerateData(); #endregion #region act await f.Run(); #endregion #region assert //1 because it starts at 0 Assert.AreEqual(f.valuesRead.Length - value - 1, f.valuesTransformed.Length); #endregion }
public static SimpleJobConditionalTransformers GetJobCSV() { #region ARRANGE StringBuilder sb = new StringBuilder(); sb.AppendLine("model,Track_number,buyYear"); sb.AppendLine("Ford,B325ROS,1990"); sb.AppendLine("Audi,PL654CSM,2004"); sb.AppendLine("BMW,B325DFH,2005"); sb.AppendLine("Ford,B325IYS,2007"); string filename = "mycsv.csv"; File.WriteAllText(filename, sb.ToString()); //define a receiver var receiverCSV = new ReceiverCSVFileInt(filename, Encoding.ASCII); //receiverCSV.Name = "From model:" + filename; //define a sender to csv for all records var senderAllToCSV = new Sender_CSV(DeleteFileIfExists("myAll.csv")); //senderAllToCSV.Name = "to csv allRecords"; //define a filter for audi var filterAudi = new FilterComparableEqual(typeof(string), "Audi", "model"); //define a sender just for audi var senderCSVAudi = new Sender_CSV(DeleteFileIfExists("myAudi.csv")); //senderCSVAudi.Name= "sender CSV"; //define a filter to transform the buyYear to string var buyYearTOString = new TransformerFieldStringInt("buyYear", "NewBuyYear"); //buyYearTOString.Name = "transform buyYear to int"; //define a filter for year>2000 var filterYear2000 = new FilterComparableGreat(typeof(int), 2000, "NewBuyYear"); //define a sender the year > 2000 to csv var sender2000CSV = new Sender_CSV(DeleteFileIfExists("my2000.csv")); //sender2000CSV.Name = "sender CSV"; //define a sender the year > 2000 to json var sender2000JSon = new Sender_JSON(DeleteFileIfExists("my2000.js")); //sender2000JSon.Name = "sender json"; //define a filter for Ford var filterFord = new FilterComparableEqual(typeof(string), "Ford", "model"); //define a sender just for ford var senderCSVFord = new Sender_CSV(DeleteFileIfExists("myFord.csv")); //senderCSVFord.Name = "sender CSV"; var cond = new SimpleJobConditionalTransformers(); //add a receiver cond.Receivers.Add(0, receiverCSV); //add a sender to csv for all records cond.AddSender(senderAllToCSV); //add a filter to transform the buyYear to string //and then fiter for year>2000 var buy = cond.Add(buyYearTOString, filterYear2000); //send the year> 2000 to csv cond.Add(filterYear2000, sender2000CSV); //send the year >2000 to json cond.Add(filterYear2000, sender2000JSon); //add a filter for audi and a sender just for audi cond.Add(filterAudi, senderCSVAudi); //add a filter for ford and a sender just for ford cond.Add(filterFord, senderCSVFord); return(cond); #endregion }
static string PBXJob() { var dir = AppContext.BaseDirectory; var dirPBX = Path.Combine(dir, "PBX"); var serialize = new SerializeDataOnFile("a.txt"); IReceive r = new ReceiverFolderHierarchical(dirPBX, "*.log"); IFilter filterFiles = new FilterForFilesHierarchical(); #region filter for remove dates serialize var filterDateTime = new FilterComparableGreat(typeof(DateTime), DateTime.MinValue, "LastWriteTimeUtc"); IFilter filterDateTimeSerializable = new FilterComparableFromSerializable(filterDateTime, serialize); #endregion IFilter removeFilesMaxWritten = new FilterRemovePropertyMaxMinDateTime("LastWriteTimeUtc", GroupingFunctions.Max); ITransform transformLines = new TransformerFileToLines() { TrimEmptyLines = true }; var trDateRegex = new TransformRowRegex(@"^Date:\ (?<datePBX>.{23}).*?$", "text"); var trKeyValueRegex = new TransformRowRegex(@"(^|\r|\n|\r\n)(?<tkey>(\$?[a-zA-Z0-9_[\]/ ]+))='?(?<tvalue>([a-zA-Z0-9_ ]+))'?(\r|\r\n|$)", "text"); var trToDate = new TransformerFieldStringToDate("datePBX", "NewDatePBX", "yyyy/MM/dd HH:mm:ss.fff"); var trAddDate = new TransformAddFieldDown("NewDatePBX"); var trAddKey = new TransformAddNewField("tkey"); var trAddValue = new TransformAddNewField("tvalue"); var trSimpleFields = new TransformRowRemainsProperties("NewDatePBX", "lineNr", "text", "FullName", "LastWriteTimeUtc", "tkey", "tvalue"); var data = new DBTableDataConnection <SqlConnection>(new SerializeDataInMemory()); data.ConnectionString = "#file:SqlServerConnectionString#"; data.Fields = new string[] { "NewDatePBX", "lineNr", "text", "FullName", "tkey", "tvalue" }; data.TableName = "PBXData"; var bulk = new SenderSqlServerBulkCopy(data); var md = new MediaTransformMaxMin <DateTime>(); md.GroupFunction = GroupingFunctions.Max; md.FieldName = "LastWriteTimeUtc"; var serializeMaxDate = new SenderMediaSerialize <DateTime>(serialize, "LastWriteTimeUtc", md); var si = new SimpleJob(); si.Receivers.Add(0, r); int iFilterNr = 0; si.FiltersAndTransformers.Add(iFilterNr++, filterFiles); si.FiltersAndTransformers.Add(iFilterNr++, filterDateTimeSerializable); si.FiltersAndTransformers.Add(iFilterNr++, removeFilesMaxWritten); si.FiltersAndTransformers.Add(iFilterNr++, transformLines); si.FiltersAndTransformers.Add(iFilterNr++, trDateRegex); si.FiltersAndTransformers.Add(iFilterNr++, trKeyValueRegex); si.FiltersAndTransformers.Add(iFilterNr++, trToDate); si.FiltersAndTransformers.Add(iFilterNr++, trAddDate); si.FiltersAndTransformers.Add(iFilterNr++, trAddKey); si.FiltersAndTransformers.Add(iFilterNr++, trAddValue); si.FiltersAndTransformers.Add(iFilterNr++, trSimpleFields); //TODO: add transformer to add a field down for all fields //TODO: add transformer regex for splitting Key=Value //TODO: add field to separate Conn(1)Type(Any)User(InternalTask) CDL Request:RSVI(Get) si.Senders.Add(0, bulk); si.Senders.Add(1, serializeMaxDate); return(si.SerializeMe()); }
static string JobDllBlockly() { string dir = Environment.CurrentDirectory; IReceive folderWithDll = new ReceiverFolderHierarchical("#static:Directory.GetCurrentDirectory()#", "*.dll"); var filterFiles = new FilterForFilesHierarchical(); var recContentFile = new ReceiverFromDllRelational(); var loadDllFromFiles = new TransformerApplyReceiver(recContentFile, "DllFileName", "FullName"); //retain just types var filterTypes = new FilterRetainRelationalName(FilterType.Equal, "Types"); //load types that are not generic or abstract //var justTypes = new FilterComparableEqual(typeof(string), "Types", "NameRelation"); var notAbstract = new FilterComparableEqual(typeof(bool), false, "IsAbstract"); var notGeneric = new FilterComparableEqual(typeof(bool), false, "IsGeneric"); var notInterface = new FilterComparableEqual(typeof(bool), false, "IsInterface"); //var noInterface = new FilterExcludeRelation("Interfaces"); //var rel2plain = new TransformerRelationalToPlain(); var haveProps = new FilterComparableGreat(typeof(int), 0, "PropertiesNr"); //var justRelations = new FilterRetainItemsWithKey("NameRelation", FilterType.Equal); var fileRazor = Path.Combine(dir, "blockly.cshtml"); string filename = "#static:Directory.GetCurrentDirectory()#\\blockly.html"; ISend senderHTML = new SyncSenderMultiple( new Sender_Text(filename, "<html><body>") { FileMode = FileMode.Create }, new Sender_HTMLRazor("Views/" + Path.GetFileName(fileRazor), filename), //new Sender_HTMLRelationViz("Name", filename), new Sender_Text(filename, "</body></html>") ) ; var senderRow = new SenderByRowToFile("Name", "txt", "Block Definition", "txt"); var clear = new TransformClearValues(); var trReceiveHtml = new ReceiverHTMLTable(@"blockly.html", Encoding.UTF8); var decodeHtml = new TransformerHtmlDecode(); //var addJS = new TransformModifyField("Name", "{0}.js"); var senderBlockTag = new SenderByRowToFile("Name", "_def.txt", "Block definition", "txt"); var senderBlockDefinitionJs = new SenderByRowToFile("Name", "_block.js", "Block JS", "js"); var senderBlockCodeJs = new SenderByRowToFile("Name", "_code.js", "Block CodeGenerator", "js"); var job = new SimpleJobConditionalTransformers(); job .AddReceiver(folderWithDll) .AddFilter(filterFiles) .AddTransformer(loadDllFromFiles) .AddFilter(filterTypes) .AddFilter(notAbstract) .AddFilter(notGeneric) .AddFilter(notInterface) .AddFilter(haveProps) //.AddTransformer(noInterface) //.AddTransformer(rel2plain) .AddSender(senderHTML) .AddTransformer(clear) .Add(clear, trReceiveHtml) .Add(trReceiveHtml, decodeHtml) .Add(decodeHtml, senderBlockTag) .Add(decodeHtml, senderBlockDefinitionJs) .Add(decodeHtml, senderBlockCodeJs) ; return(job.SerializeMe()); }
public async Task TestPBXData() { #region arrange string connectionString = GetSqlServerConnectionString(); using (var conn = new SqlConnection(connectionString)) { await conn.OpenAsync(); using (var cmd = conn.CreateCommand()) { cmd.CommandType = CommandType.Text; cmd.CommandText = @" IF OBJECT_ID('dbo.PBXData', 'U') IS NOT NULL DROP TABLE dbo.PBXData;"; await cmd.ExecuteNonQueryAsync(); cmd.CommandText = @"CREATE TABLE [PBXData]( [NewDatePBX] [datetime] NOT NULL, [lineNr] [int] NOT NULL, [text] [nvarchar](500) NOT NULL, [FullName] [nvarchar](500) NOT NULL )"; await cmd.ExecuteNonQueryAsync(); } await Task.Delay(2000); var dir = AppContext.BaseDirectory; var dirPBX = Path.Combine(dir, "PBX"); File.AppendAllText(Path.Combine(dirPBX, "PBXRemove.log"), "aaaa"); #endregion #region act var serialize = new SerializeDataOnFile("a.txt"); IReceive r = new ReceiverFolderHierarchical(dirPBX, "*.log"); IFilter filterFiles = new FilterForFilesHierarchical(); #region filter for remove dates serialize var filterDateTime = new FilterComparableGreat(typeof(DateTime), DateTime.MinValue, "LastWriteTimeUtc"); IFilter filterDateTimeSerializable = new FilterComparableFromSerializable(filterDateTime, serialize); #endregion IFilter removeFilesMaxWritten = new FilterRemovePropertyMaxMinDateTime("LastWriteTimeUtc", GroupingFunctions.Max); ITransform transformLines = new TransformerFileToLines() { TrimEmptyLines = true }; var trDateRegex = new TransformRowRegex(@"^Date:\ (?<datePBX>.{23}).*?$", "text"); var trToDate = new TransformerFieldStringToDate("datePBX", "NewDatePBX", "yyyy/MM/dd HH:mm:ss.fff"); var trAddDate = new TransformAddFieldDown("NewDatePBX"); var trSimpleFields = new TransformRowRemainsProperties("NewDatePBX", "lineNr", "text", "FullName", "LastWriteTimeUtc"); var data = new DBTableDataConnection <SqlConnection>(new SerializeDataInMemory()); data.ConnectionString = GetSqlServerConnectionString(); data.Fields = new string[] { "NewDatePBX", "lineNr", "text", "FullName" }; data.TableName = "PBXData"; var bulk = new SenderSqlServerBulkCopy(data); var md = new MediaTransformMaxMin <DateTime>(); md.GroupFunction = GroupingFunctions.Max; md.FieldName = "LastWriteTimeUtc"; var serializeMaxDate = new SenderMediaSerialize <DateTime>(serialize, "LastWriteTimeUtc", md); var si = new SimpleJob(); si.Receivers.Add(0, r); int iFilterNr = 0; si.FiltersAndTransformers.Add(iFilterNr++, filterFiles); si.FiltersAndTransformers.Add(iFilterNr++, filterDateTimeSerializable); si.FiltersAndTransformers.Add(iFilterNr++, removeFilesMaxWritten); si.FiltersAndTransformers.Add(iFilterNr++, transformLines); si.FiltersAndTransformers.Add(iFilterNr++, trDateRegex); si.FiltersAndTransformers.Add(iFilterNr++, trToDate); si.FiltersAndTransformers.Add(iFilterNr++, trAddDate); si.FiltersAndTransformers.Add(iFilterNr++, trSimpleFields); //TODO: add transformer to add a field down for all fields //TODO: add transformer regex for splitting Key=Value //TODO: add field to separate Conn(1)Type(Any)User(InternalTask) CDL Request:RSVI(Get) si.Senders.Add(0, bulk); si.Senders.Add(1, serializeMaxDate); await si.Execute(); #endregion #region assert filterFiles.valuesTransformed.Length.ShouldBe(3, "three files after first filter"); removeFilesMaxWritten.valuesTransformed.Length.ShouldBe(2, "last one file written dismissed"); transformLines.valuesTransformed.Length.ShouldBe(77251); var d = transformLines.valuesTransformed.Select(it => it.Values["FullName"]).Distinct().ToArray(); d.Length.ShouldBe(2, "two files after reading contents"); //transformGroupingFiles.valuesTransformed.Length.ShouldBe(2); //trDateRegex.valuesTransformed.Count(it => it.Values.ContainsKey("datePBX")).ShouldBeGreaterThan(0,"datePBX"); //trToDate.valuesTransformed.Count(it => it.Values.ContainsKey("NewDatePBX")).ShouldBeGreaterThan(0,"NewDatePBX"); foreach (var item in trAddDate.valuesTransformed) { item.Values.ShouldContainKey("NewDatePBX"); } foreach (var item in trSimpleFields.valuesTransformed) { item.Values.Keys.Count.ShouldBe(5); } using (var con = new SqlConnection(connectionString)) { await con.OpenAsync(); using (var cmd = con.CreateCommand()) { cmd.CommandType = CommandType.Text; cmd.CommandText = "select count(*) from PBXData"; var val = await cmd.ExecuteScalarAsync(); val.ShouldBe(77251); } } md.Result.Year.ShouldBe(DateTime.Now.Year); #endregion #region arange to read again r = new ReceiverFolderHierarchical(dirPBX, "*.log"); filterFiles = new FilterForFilesHierarchical(); #region filter for remove dates serialize filterDateTime = new FilterComparableGreat(typeof(DateTime), DateTime.MinValue, "LastWriteTimeUtc"); filterDateTimeSerializable = new FilterComparableFromSerializable(filterDateTime, serialize); #endregion #endregion #region act si = new SimpleJob(); si.Receivers.Add(0, r); iFilterNr = 0; si.FiltersAndTransformers.Add(iFilterNr++, filterFiles); si.FiltersAndTransformers.Add(iFilterNr++, filterDateTimeSerializable); await si.Execute(); #endregion #region assert filterDateTime.valuesTransformed?.Length.ShouldBe(1, "next time 1 file read - the added one"); #endregion } }