public void Extract_array_columns()
{
Schema schema = SchemaReflector.Reflect <ArrayColumns>();
Assert.Equal(2, schema.Length);
var extractor = new ColumnExtractor();
ArrayColumns[] ac =
{
new ArrayColumns
{
Id = 1,
Addresses = new[]{ "Fiddler", "On" }
},
new ArrayColumns
{
Id = 2,
Addresses = new[]{ "The", "Roof" }
}
};
List <DataColumn> columns = extractor.ExtractColumns(ac, schema).ToList();
Assert.Equal(new[] { 1, 2 }, columns[0].DefinedData);
Assert.Equal(new[] { "Fiddler", "On", "The", "Roof" }, columns[1].DefinedData);
Assert.Equal(new[] { 0, 1, 0, 1 }, columns[1].RepetitionLevels);
}
public void ExtractDecimalDependingCultureInfo(string input, string culture, decimal expected)
{
var source = new TestResult();
var result = source as object;
var extractor = new ColumnExtractor <TestResult, decimal?>(0, new CultureInfo(culture));
extractor.SetInto(a => a.Member4);
extractor.Extract(result, input, out result);
source.Member4.Should().Be(expected);
}
public void ExtractDateTimeDependingCultureInfo(string input, string culture, int year, int month, int day)
{
var source = new TestResult();
var result = source as object;
var extractor = new ColumnExtractor <TestResult, DateTime>(0, new CultureInfo(culture));
extractor.SetInto(a => a.Member3);
extractor.Extract(result, input, out result);
source.Member3.Should().Be(new DateTime(year, month, day));
}
public void ExtractDateAndPutInField()
{
var source = new TestResult();
var result = source as object;
var extractor = new ColumnExtractor <TestResult, DateTime>(0, CultureInfo.CurrentCulture);
extractor.SetInto(a => a.Member3);
extractor.Extract(result, "01/07/1980", out result);
source.Member3.Should().Be(new DateTime(1980, 7, 1));
}
public void ExtractIntAndPutInField()
{
var source = new TestResult();
var result = source as object;
var extractor = new ColumnExtractor <TestResult, int>(0, CultureInfo.CurrentCulture);
extractor.SetInto(a => a.Member2);
extractor.Extract(result, "25", out result);
source.Member2.Should().Be(25);
}
public void ExtractStringAndPutInField()
{
var source = new TestResult();
var result = source as object;
var extractor = new ColumnExtractor <TestResult, string>(0, CultureInfo.CurrentCulture);
extractor.SetInto(a => a.Member1);
extractor.Extract(source, "bonjour", out result);
source.Member1.Should().Be("bonjour");
}
public void ExtractObjectInSpecialWay()
{
var source = new TestResult();
var result = source as object;
var extractor = new ColumnExtractor <TestResult, string>(0, CultureInfo.CurrentCulture);
extractor.SetInto(a => a.Member1);
extractor.SetInThisWay(a => $"[TEST]{a}");
extractor.Extract(result, "ESSAI", out result);
source.Member1.Should().Be("[TEST]ESSAI");
}
public static Schema Serialize <T>(IEnumerable <T> objectInstances, Stream destination,
Schema schema = null,
WriterOptions writerOptions = null,
CompressionMethod compressionMethod = CompressionMethod.Snappy)
where T : new()
{
if (objectInstances == null)
{
throw new ArgumentNullException(nameof(objectInstances));
}
if (destination == null)
{
throw new ArgumentNullException(nameof(destination));
}
if (!destination.CanWrite)
{
throw new ArgumentException("stream must be writeable", nameof(destination));
}
//if schema is not passed reflect it
if (schema == null)
{
schema = SchemaReflector.Reflect <T>();
}
if (writerOptions == null)
{
writerOptions = new WriterOptions();
}
var extractor = new ColumnExtractor();
using (var writer = new ParquetWriter3(schema, destination, writerOptions: writerOptions))
{
writer.CompressionMethod = compressionMethod;
foreach (IEnumerable <T> batch in objectInstances.Batch(writerOptions.RowGroupsSize))
{
IReadOnlyCollection <DataColumn> columns = extractor.ExtractColumns(batch, schema);
using (ParquetRowGroupWriter groupWriter = writer.CreateRowGroup(batch.Count()))
{
foreach (DataColumn dataColumn in columns)
{
groupWriter.Write(dataColumn);
}
}
}
}
return(schema);
}
public void ExtractStringWithMultiline()
{
const string input = "test1\r\ntest2\r\ntest3";
var source = new TestResult();
var result = source as object;
var extractor = new ColumnExtractor <TestResult, string>(0, CultureInfo.CurrentCulture);
extractor.SetInto(a => a.Member1);
extractor.Extract(result, input, out result);
source.Member1.Should().Be(input);
}
public static ColumnExtractor ExtractColumns(TableValueExpression se, LambdaExpression expr)
{
var ce = new ColumnExtractor(se.Alias);
ce.membershipPath = Expression.Parameter(se.Projection.ReturnType, "x");
ce.Visit(expr);
if (se.Projection.Body is ColumnExpression)
{
var column = se.Projection.Body as ColumnExpression;
ce.assignments.Add(new AssignmentInformation()
{
Declaration = column,
MembershipPath = ce.membershipPath,
});
}
return ce;
}
public void Extract_simple_columns()
{
Schema schema = new SchemaReflector(typeof(SimpleColumns)).Reflect();
var extractor = new ColumnExtractor();
SimpleColumns[] classes = new[]
{
new SimpleColumns {
Id = 1, Name = "First"
}, new SimpleColumns {
Id = 2, Name = "Second"
}, new SimpleColumns {
Id = 3, Name = "Third"
}
};
List <DataColumn> columns = extractor.ExtractColumns(classes, schema);
Assert.Equal(new[] { 1, 2, 3 }, columns[0].DefinedData);
Assert.Equal(new[] { "First", "Second", "Third" }, columns[1].DefinedData);
}
