public void Extract_array_columns() { Schema schema = SchemaReflector.Reflect <ArrayColumns>(); Assert.Equal(2, schema.Length); var extractor = new ColumnExtractor(); ArrayColumns[] ac = { new ArrayColumns { Id = 1, Addresses = new[]{ "Fiddler", "On" } }, new ArrayColumns { Id = 2, Addresses = new[]{ "The", "Roof" } } }; List <DataColumn> columns = extractor.ExtractColumns(ac, schema).ToList(); Assert.Equal(new[] { 1, 2 }, columns[0].DefinedData); Assert.Equal(new[] { "Fiddler", "On", "The", "Roof" }, columns[1].DefinedData); Assert.Equal(new[] { 0, 1, 0, 1 }, columns[1].RepetitionLevels); }
public void ExtractDecimalDependingCultureInfo(string input, string culture, decimal expected) { var source = new TestResult(); var result = source as object; var extractor = new ColumnExtractor <TestResult, decimal?>(0, new CultureInfo(culture)); extractor.SetInto(a => a.Member4); extractor.Extract(result, input, out result); source.Member4.Should().Be(expected); }
public void ExtractDateTimeDependingCultureInfo(string input, string culture, int year, int month, int day) { var source = new TestResult(); var result = source as object; var extractor = new ColumnExtractor <TestResult, DateTime>(0, new CultureInfo(culture)); extractor.SetInto(a => a.Member3); extractor.Extract(result, input, out result); source.Member3.Should().Be(new DateTime(year, month, day)); }
public void ExtractDateAndPutInField() { var source = new TestResult(); var result = source as object; var extractor = new ColumnExtractor <TestResult, DateTime>(0, CultureInfo.CurrentCulture); extractor.SetInto(a => a.Member3); extractor.Extract(result, "01/07/1980", out result); source.Member3.Should().Be(new DateTime(1980, 7, 1)); }
public void ExtractIntAndPutInField() { var source = new TestResult(); var result = source as object; var extractor = new ColumnExtractor <TestResult, int>(0, CultureInfo.CurrentCulture); extractor.SetInto(a => a.Member2); extractor.Extract(result, "25", out result); source.Member2.Should().Be(25); }
public void ExtractStringAndPutInField() { var source = new TestResult(); var result = source as object; var extractor = new ColumnExtractor <TestResult, string>(0, CultureInfo.CurrentCulture); extractor.SetInto(a => a.Member1); extractor.Extract(source, "bonjour", out result); source.Member1.Should().Be("bonjour"); }
public void ExtractObjectInSpecialWay() { var source = new TestResult(); var result = source as object; var extractor = new ColumnExtractor <TestResult, string>(0, CultureInfo.CurrentCulture); extractor.SetInto(a => a.Member1); extractor.SetInThisWay(a => $"[TEST]{a}"); extractor.Extract(result, "ESSAI", out result); source.Member1.Should().Be("[TEST]ESSAI"); }
public static Schema Serialize <T>(IEnumerable <T> objectInstances, Stream destination, Schema schema = null, WriterOptions writerOptions = null, CompressionMethod compressionMethod = CompressionMethod.Snappy) where T : new() { if (objectInstances == null) { throw new ArgumentNullException(nameof(objectInstances)); } if (destination == null) { throw new ArgumentNullException(nameof(destination)); } if (!destination.CanWrite) { throw new ArgumentException("stream must be writeable", nameof(destination)); } //if schema is not passed reflect it if (schema == null) { schema = SchemaReflector.Reflect <T>(); } if (writerOptions == null) { writerOptions = new WriterOptions(); } var extractor = new ColumnExtractor(); using (var writer = new ParquetWriter3(schema, destination, writerOptions: writerOptions)) { writer.CompressionMethod = compressionMethod; foreach (IEnumerable <T> batch in objectInstances.Batch(writerOptions.RowGroupsSize)) { IReadOnlyCollection <DataColumn> columns = extractor.ExtractColumns(batch, schema); using (ParquetRowGroupWriter groupWriter = writer.CreateRowGroup(batch.Count())) { foreach (DataColumn dataColumn in columns) { groupWriter.Write(dataColumn); } } } } return(schema); }
public void ExtractStringWithMultiline() { const string input = "test1\r\ntest2\r\ntest3"; var source = new TestResult(); var result = source as object; var extractor = new ColumnExtractor <TestResult, string>(0, CultureInfo.CurrentCulture); extractor.SetInto(a => a.Member1); extractor.Extract(result, input, out result); source.Member1.Should().Be(input); }
public static ColumnExtractor ExtractColumns(TableValueExpression se, LambdaExpression expr) { var ce = new ColumnExtractor(se.Alias); ce.membershipPath = Expression.Parameter(se.Projection.ReturnType, "x"); ce.Visit(expr); if (se.Projection.Body is ColumnExpression) { var column = se.Projection.Body as ColumnExpression; ce.assignments.Add(new AssignmentInformation() { Declaration = column, MembershipPath = ce.membershipPath, }); } return ce; }
public void Extract_simple_columns() { Schema schema = new SchemaReflector(typeof(SimpleColumns)).Reflect(); var extractor = new ColumnExtractor(); SimpleColumns[] classes = new[] { new SimpleColumns { Id = 1, Name = "First" }, new SimpleColumns { Id = 2, Name = "Second" }, new SimpleColumns { Id = 3, Name = "Third" } }; List <DataColumn> columns = extractor.ExtractColumns(classes, schema); Assert.Equal(new[] { 1, 2, 3 }, columns[0].DefinedData); Assert.Equal(new[] { "First", "Second", "Third" }, columns[1].DefinedData); }