示例#1
0
        public async Task TestTransformerFileToLines()
        {
            #region arrange
            var dir = AppContext.BaseDirectory;

            var           folderSql      = Path.Combine(dir, "SqlToExecute");
            var           receiverFolder = new ReceiverFolderHierarchical(folderSql, "*.sql");
            DirectoryInfo di             = new DirectoryInfo(folderSql);
            Console.WriteLine($"start files in {folderSql}");
            foreach (var item in di.EnumerateFiles("*.sql"))
            {
                Console.WriteLine($"File {item.FullName}");
            }
            Console.WriteLine($"end files in {folderSql}");
            #endregion
            #region act
            var transformer = new TransformerFileToLines();
            var j           = new SimpleJob();
            j.Receivers.Add(0, receiverFolder);
            j.FiltersAndTransformers.Add(0, transformer);
            await j.Execute();

            #endregion
            #region assert
            receiverFolder.valuesRead?.Length.ShouldBeGreaterThan(1);
            transformer.valuesTransformed.ShouldNotBeNull();
            var files = transformer.valuesTransformed.GroupBy(it => it.Values["FullName"]).ToArray();
            files.Length.ShouldBeGreaterThan(1);
            var file1Len = files.First().Count();
            var file2Len = files.Last().Count();
            file1Len.ShouldBeGreaterThan(0);
            file2Len.ShouldBeGreaterThan(0);
            (file1Len + file2Len).ShouldBeGreaterThan(2);
            #endregion
        }
示例#2
0
        static async Task MainAsync(string[] args)
        {
            var sj = new SimpleJob();

            IReceive folder = new ReceiverFolderHierarchical(@"D:\test", "*.txt");
            //await folder.LoadData();
            var t = new TransformerFileToLines();

            t.TrimEmptyLines = false;
            //t.valuesRead = folder.valuesRead;
            //await t.Run();

            var addBak = new TransformerFieldAddString("FullName", "FullName", ".bak");

            addBak.valuesRead = t.valuesTransformed;
            //await addBak.Run();


            //var regex = new TransformRowRegex(@"^(?x<ip>123).*?$","text");
            var regex = new TransformRowRegexReplaceGuid(@"^.*x(?<ip>\w+)123.*?$", "text");

            regex.ReplaceAllNextOccurences = true;
            //regex.valuesRead = addBak.valuesTransformed;
            //await regex.Run();
            var file = new SenderByRowToFile("FullName", "lineNr", "text", ".txt");

            file.FileMode = System.IO.FileMode.Append;
            //file.valuesToBeSent = regex.valuesTransformed;
            //await file.Send();

            sj
            .AddReceiver(folder)
            .AddTransformer(t)
            //.AddTransformer(addBak)
            .AddTransformer(regex)
            .AddSender(file);

            File.WriteAllText("def.txt", sj.SerializeMe());
            await sj.Execute();
        }
示例#3
0
        static string PBXJob()
        {
            var dir    = AppContext.BaseDirectory;
            var dirPBX = Path.Combine(dir, "PBX");


            var      serialize   = new SerializeDataOnFile("a.txt");
            IReceive r           = new ReceiverFolderHierarchical(dirPBX, "*.log");
            IFilter  filterFiles = new FilterForFilesHierarchical();

            #region filter for remove dates serialize

            var     filterDateTime             = new FilterComparableGreat(typeof(DateTime), DateTime.MinValue, "LastWriteTimeUtc");
            IFilter filterDateTimeSerializable = new FilterComparableFromSerializable(filterDateTime, serialize);
            #endregion

            IFilter    removeFilesMaxWritten = new FilterRemovePropertyMaxMinDateTime("LastWriteTimeUtc", GroupingFunctions.Max);
            ITransform transformLines        = new TransformerFileToLines()
            {
                TrimEmptyLines = true
            };
            var trDateRegex     = new TransformRowRegex(@"^Date:\ (?<datePBX>.{23}).*?$", "text");
            var trKeyValueRegex = new TransformRowRegex(@"(^|\r|\n|\r\n)(?<tkey>(\$?[a-zA-Z0-9_[\]/ ]+))='?(?<tvalue>([a-zA-Z0-9_ ]+))'?(\r|\r\n|$)", "text");
            var trToDate        = new TransformerFieldStringToDate("datePBX", "NewDatePBX", "yyyy/MM/dd HH:mm:ss.fff");

            var trAddDate  = new TransformAddFieldDown("NewDatePBX");
            var trAddKey   = new TransformAddNewField("tkey");
            var trAddValue = new TransformAddNewField("tvalue");

            var trSimpleFields = new TransformRowRemainsProperties("NewDatePBX", "lineNr", "text", "FullName", "LastWriteTimeUtc", "tkey", "tvalue");

            var data = new DBTableDataConnection <SqlConnection>(new SerializeDataInMemory());
            data.ConnectionString = "#file:SqlServerConnectionString#";
            data.Fields           = new string[] { "NewDatePBX", "lineNr", "text", "FullName", "tkey", "tvalue" };
            data.TableName        = "PBXData";
            var bulk = new SenderSqlServerBulkCopy(data);
            var md   = new MediaTransformMaxMin <DateTime>();
            md.GroupFunction = GroupingFunctions.Max;
            md.FieldName     = "LastWriteTimeUtc";
            var serializeMaxDate = new SenderMediaSerialize <DateTime>(serialize, "LastWriteTimeUtc", md);
            var si = new SimpleJob();
            si.Receivers.Add(0, r);
            int iFilterNr = 0;
            si.FiltersAndTransformers.Add(iFilterNr++, filterFiles);
            si.FiltersAndTransformers.Add(iFilterNr++, filterDateTimeSerializable);
            si.FiltersAndTransformers.Add(iFilterNr++, removeFilesMaxWritten);
            si.FiltersAndTransformers.Add(iFilterNr++, transformLines);
            si.FiltersAndTransformers.Add(iFilterNr++, trDateRegex);
            si.FiltersAndTransformers.Add(iFilterNr++, trKeyValueRegex);
            si.FiltersAndTransformers.Add(iFilterNr++, trToDate);
            si.FiltersAndTransformers.Add(iFilterNr++, trAddDate);
            si.FiltersAndTransformers.Add(iFilterNr++, trAddKey);
            si.FiltersAndTransformers.Add(iFilterNr++, trAddValue);
            si.FiltersAndTransformers.Add(iFilterNr++, trSimpleFields);
            //TODO: add transformer to add a field down for all fields
            //TODO: add transformer regex for splitting Key=Value
            //TODO: add field to separate Conn(1)Type(Any)User(InternalTask) CDL Request:RSVI(Get)
            si.Senders.Add(0, bulk);
            si.Senders.Add(1, serializeMaxDate);

            return(si.SerializeMe());
        }
示例#4
0
        public async Task TestPBXData()
        {
            #region arrange
            string connectionString = GetSqlServerConnectionString();
            using (var conn = new SqlConnection(connectionString))
            {
                await conn.OpenAsync();

                using (var cmd = conn.CreateCommand())
                {
                    cmd.CommandType = CommandType.Text;
                    cmd.CommandText = @"
IF OBJECT_ID('dbo.PBXData', 'U') IS NOT NULL
 DROP TABLE dbo.PBXData;";
                    await cmd.ExecuteNonQueryAsync();

                    cmd.CommandText = @"CREATE TABLE [PBXData](
	[NewDatePBX] [datetime] NOT NULL,
	[lineNr] [int] NOT NULL,
	[text] [nvarchar](500) NOT NULL,
	[FullName] [nvarchar](500) NOT NULL
)";
                    await cmd.ExecuteNonQueryAsync();
                }
                await Task.Delay(2000);

                var dir    = AppContext.BaseDirectory;
                var dirPBX = Path.Combine(dir, "PBX");
                File.AppendAllText(Path.Combine(dirPBX, "PBXRemove.log"), "aaaa");

                #endregion
                #region act
                var      serialize   = new SerializeDataOnFile("a.txt");
                IReceive r           = new ReceiverFolderHierarchical(dirPBX, "*.log");
                IFilter  filterFiles = new FilterForFilesHierarchical();
                #region filter for remove dates serialize

                var     filterDateTime             = new FilterComparableGreat(typeof(DateTime), DateTime.MinValue, "LastWriteTimeUtc");
                IFilter filterDateTimeSerializable = new FilterComparableFromSerializable(filterDateTime, serialize);
                #endregion

                IFilter    removeFilesMaxWritten = new FilterRemovePropertyMaxMinDateTime("LastWriteTimeUtc", GroupingFunctions.Max);
                ITransform transformLines        = new TransformerFileToLines()
                {
                    TrimEmptyLines = true
                };
                var trDateRegex = new TransformRowRegex(@"^Date:\ (?<datePBX>.{23}).*?$", "text");
                var trToDate    = new TransformerFieldStringToDate("datePBX", "NewDatePBX", "yyyy/MM/dd HH:mm:ss.fff");

                var trAddDate      = new TransformAddFieldDown("NewDatePBX");
                var trSimpleFields = new TransformRowRemainsProperties("NewDatePBX", "lineNr", "text", "FullName", "LastWriteTimeUtc");

                var data = new DBTableDataConnection <SqlConnection>(new SerializeDataInMemory());
                data.ConnectionString = GetSqlServerConnectionString();
                data.Fields           = new string[] { "NewDatePBX", "lineNr", "text", "FullName" };
                data.TableName        = "PBXData";
                var bulk = new SenderSqlServerBulkCopy(data);
                var md   = new MediaTransformMaxMin <DateTime>();
                md.GroupFunction = GroupingFunctions.Max;
                md.FieldName     = "LastWriteTimeUtc";
                var serializeMaxDate = new SenderMediaSerialize <DateTime>(serialize, "LastWriteTimeUtc", md);
                var si = new SimpleJob();
                si.Receivers.Add(0, r);
                int iFilterNr = 0;
                si.FiltersAndTransformers.Add(iFilterNr++, filterFiles);
                si.FiltersAndTransformers.Add(iFilterNr++, filterDateTimeSerializable);
                si.FiltersAndTransformers.Add(iFilterNr++, removeFilesMaxWritten);
                si.FiltersAndTransformers.Add(iFilterNr++, transformLines);
                si.FiltersAndTransformers.Add(iFilterNr++, trDateRegex);
                si.FiltersAndTransformers.Add(iFilterNr++, trToDate);
                si.FiltersAndTransformers.Add(iFilterNr++, trAddDate);
                si.FiltersAndTransformers.Add(iFilterNr++, trSimpleFields);
                //TODO: add transformer to add a field down for all fields
                //TODO: add transformer regex for splitting Key=Value
                //TODO: add field to separate Conn(1)Type(Any)User(InternalTask) CDL Request:RSVI(Get)
                si.Senders.Add(0, bulk);
                si.Senders.Add(1, serializeMaxDate);

                await si.Execute();

                #endregion
                #region assert
                filterFiles.valuesTransformed.Length.ShouldBe(3, "three files after first filter");
                removeFilesMaxWritten.valuesTransformed.Length.ShouldBe(2, "last one file written dismissed");
                transformLines.valuesTransformed.Length.ShouldBe(77251);
                var d = transformLines.valuesTransformed.Select(it => it.Values["FullName"]).Distinct().ToArray();
                d.Length.ShouldBe(2, "two files after reading contents");
                //transformGroupingFiles.valuesTransformed.Length.ShouldBe(2);
                //trDateRegex.valuesTransformed.Count(it => it.Values.ContainsKey("datePBX")).ShouldBeGreaterThan(0,"datePBX");
                //trToDate.valuesTransformed.Count(it => it.Values.ContainsKey("NewDatePBX")).ShouldBeGreaterThan(0,"NewDatePBX");
                foreach (var item in trAddDate.valuesTransformed)
                {
                    item.Values.ShouldContainKey("NewDatePBX");
                }
                foreach (var item in trSimpleFields.valuesTransformed)
                {
                    item.Values.Keys.Count.ShouldBe(5);
                }
                using (var con = new SqlConnection(connectionString))
                {
                    await con.OpenAsync();

                    using (var cmd = con.CreateCommand())
                    {
                        cmd.CommandType = CommandType.Text;
                        cmd.CommandText = "select count(*) from PBXData";
                        var val = await cmd.ExecuteScalarAsync();

                        val.ShouldBe(77251);
                    }
                }
                md.Result.Year.ShouldBe(DateTime.Now.Year);
                #endregion
                #region arange to read again
                r           = new ReceiverFolderHierarchical(dirPBX, "*.log");
                filterFiles = new FilterForFilesHierarchical();
                #region filter for remove dates serialize
                filterDateTime             = new FilterComparableGreat(typeof(DateTime), DateTime.MinValue, "LastWriteTimeUtc");
                filterDateTimeSerializable = new FilterComparableFromSerializable(filterDateTime, serialize);
                #endregion
                #endregion
                #region act
                si = new SimpleJob();
                si.Receivers.Add(0, r);
                iFilterNr = 0;
                si.FiltersAndTransformers.Add(iFilterNr++, filterFiles);
                si.FiltersAndTransformers.Add(iFilterNr++, filterDateTimeSerializable);
                await si.Execute();

                #endregion
                #region assert
                filterDateTime.valuesTransformed?.Length.ShouldBe(1, "next time 1 file read - the added one");
                #endregion
            }
        }