public void AddMapping(LogicalType logicalType, Type actualType)
{
Guard.NotNull(logicalType, "logicalType");
Guard.NotNull(actualType, "actualType");
if(logicalType.IsOpenType != actualType.IsGenericTypeDefinition)
{
throw logicalType.IsOpenType
? new ArgumentException("Actual type must be a generic type definition.", "actualType")
: new ArgumentException("Actual type must not be a generic type definition.", "actualType");
}
if (logicalType.IsOpenType && logicalType.Arity != actualType.GetGenericArguments().Length)
{
throw new ArgumentException("The number of generic type parameters does not match the logical type arity.", "actualType");
}
lock (_logicalToActualMappings)
{
if(_logicalToActualMappings.ContainsKey(logicalType) || _actualToLogicalMappings.ContainsKey(actualType))
throw new ArgumentException("Unable to add mapping; the mapping already exists for the provided types.", "logicalType");
_logicalToActualMappings.TryAdd(logicalType, actualType);
_actualToLogicalMappings.TryAdd(actualType, logicalType);
}
}
public static Type LookupActualType(this ILogicalTypeRegistry @this, LogicalType logicalType)
{
Guard.NotNull(@this, "this");
var result = @this.TryLookupActualType(logicalType);
if (!result.HasValue)
throw new InvalidOperationException(String.Format("Unable to find actual type for logical type: '{0}'.", logicalType));
return result.Value;
}
public void Equivalence_ImplementedCorrectly()
{
var type1 = new LogicalType("tst", "Foo");
var type2 = new LogicalType("tst", "foo");
var type3 = new LogicalType("sts", "Bar");
Object type4 = new LogicalType("sts", "Bar");
var type5 = new LogicalType("tst", "Foo", 0, 1.ToMaybe());
var type6 = new LogicalType("tst", "Foo", 0, 1.ToMaybe());
var type7 = new LogicalType("tst", "Foo", 0, 2.ToMaybe());
var type8 = new LogicalType("tst", "Foo", 1, Maybe.NoValue);
var type9 = new LogicalType("tst", "Foo", 1, Maybe.NoValue);
var type10 = new LogicalType("tst", "Foo", 1, 1.ToMaybe());
var type11 = new LogicalType("tst", "Foo", 1, 1.ToMaybe());
var type12 = new LogicalType("tst", "Foo", 1, 2.ToMaybe());
var type13 = new LogicalType("tst", "Foo", new[] { type1 }, Maybe.NoValue);
var type14 = new LogicalType("tst", "Foo", new[] { type1 }, Maybe.NoValue);
var type15 = new LogicalType("tst", "Foo", new[] { type1 }, 1.ToMaybe());
var type16 = new LogicalType("tst", "Foo", new[] { type1 }, 2.ToMaybe());
var type17 = new LogicalType("tst", "Foo", new[] { type2 }, Maybe.NoValue);
var type18 = new LogicalType("tst", "Foo", new[] { type1, type3 }, Maybe.NoValue);
LogicalType nullType = null;
(type1 == type2).Should().BeTrue();
(type1 != type3).Should().BeTrue();
(type3.Equals(type4)).Should().BeTrue();
(type5 == type6).Should().BeTrue();
(type1 != type5).Should().BeTrue();
(type6 != type7).Should().BeTrue();
(type7 != type8).Should().BeTrue();
(type8 == type9).Should().BeTrue();
(type9 != type10).Should().BeTrue();
(type10 == type11).Should().BeTrue();
(type11 != type12).Should().BeTrue();
(type8 != type13).Should().BeTrue();
(type13 == type14).Should().BeTrue();
(type14 != type15).Should().BeTrue();
(type15 != type16).Should().BeTrue();
(type14 == type17).Should().BeTrue();
(type13 != type18).Should().BeTrue();
(nullType == null).Should().BeTrue();
(type1 != nullType).Should().BeTrue();
}
///
/// <summary>
/// Parses logical expressions
/// </summary>
///
/// <param name="leftOp">Left operand</param>
/// <param name="rightOp">Right operand</param>
/// <param name="type">Logical expression type</param>
///
/// <returns>The result of applying the given logical expression operator</returns>
///
public static CseObject Parse(CseObject leftOp, CseObject rightOp, LogicalType type) {
CseObject obj = null;
if (type == LogicalType.NOT) {
obj = new CseObject(null) { IsLiteral = leftOp.IsLiteral };
}
else {
obj = new CseObject(null) { IsLiteral = leftOp.IsLiteral && rightOp.IsLiteral };
}
try {
switch (type) {
case LogicalType.AND:
// TODO: lazy eval if exception thrown
obj.Value = leftOp.Value && rightOp.Value;
break;
case LogicalType.OR:
// TODO: lazy eval if exception thrown
obj.Value = leftOp.Value || rightOp.Value;
break;
case LogicalType.NOT: {
//try {
obj.Value = !leftOp.Value;
/*}
catch {
MethodInfo mi = obj.Value.GetType().GetMethod(OpOverloadNames.FALSE);
if (mi != null)
obj.Value = obj.Value.GetType().InvokeMember(OpOverloadNames.FALSE, OpOverloadNames.Flags, null, CsEval.evalEnvironment, new object[] { obj.Value });
else
obj.Value = null;
}*/
}
break;
default:
throw new System.NotImplementedException("Not implemented.");
}
}
catch {
// TODO: Fill this out!
}
return obj;
}
public Maybe<Type> TryLookupActualType(LogicalType logicalType)
{
var result = _logicalToActualMappings.TryGetValue(logicalType);
if(!result.HasValue && !logicalType.IsOpenType)
{
var openActualType = _logicalToActualMappings.TryGetValue(logicalType.GetOpenType());
if (!openActualType.HasValue) return Maybe.NoValue;
var typeArguments = logicalType.TypeArguments
.Select(x => TryLookupActualType(x))
.Squash()
.ToArray();
var actualType = openActualType.Value;
if (typeArguments.Length == actualType.GetGenericArguments().Length)
return actualType.MakeGenericType(typeArguments).ToMaybe();
}
return result;
}
public ExampleSchemaBuilder WithDifferentGroupLogicalType()
{
_groupLogicalType = LogicalType.List();
return(this);
}
public Column(string name, LogicalType logicalTypeOverride = null)
: base(typeof(TLogicalType), name, logicalTypeOverride)
{
}
private static unsafe IntPtr Make(string name, Repetition repetition, IReadOnlyList <Node> fields, LogicalType logicalType)
{
var handles = fields.Select(f => (IntPtr)f.Handle).ToArray();
fixed(IntPtr *pHandles = handles)
{
ExceptionInfo.Check(GroupNode_Make(name, repetition, (IntPtr)pHandles, handles.Length, logicalType, out var groupNode));
GC.KeepAlive(fields);
return(groupNode);
}
}
public GroupNode(string name, Repetition repetition, IReadOnlyList <Node> fields, LogicalType logicalType = LogicalType.None)
: this(Make(name, repetition, fields, logicalType))
{
}
public Condition WithIs(LogicalType op, ExpressionKey key, ExpressionSyntax expression, bool hasNotPrefix)
{
var newType = CombineType(type, op);
return(new Condition(Kind, newType, parts.Add(new IsPart(key, expression, hasNotPrefix))));
}
public Condition With(LogicalType op, Condition condition)
{
var newType = CombineType(type, op);
return(new Condition(Kind, newType, parts.AddRange(condition.parts)));
}
internal XmlTypeMapping(LogicalType logicalType, XmlSerializationReflector reflector) : base(logicalType, reflector)
{
}
public ExampleSchemaBuilder WithDifferentPrimitiveLogicalType()
{
_primitiveLogicalType = LogicalType.Timestamp(false, TimeUnit.Millis);
_physicalType = PhysicalType.Int64;
return(this);
}
private static ExpectedPrimitive[] CreateExpectedPrimitives()
{
return(new[]
{
new ExpectedPrimitive(typeof(bool))
{
PhysicalType = PhysicalType.Boolean
},
new ExpectedPrimitive(typeof(sbyte))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(8, isSigned: true)
},
new ExpectedPrimitive(typeof(byte))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(8, isSigned: false)
},
new ExpectedPrimitive(typeof(short))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(16, isSigned: true)
},
new ExpectedPrimitive(typeof(ushort))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(16, isSigned: false)
},
new ExpectedPrimitive(typeof(int))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(32, isSigned: true)
},
new ExpectedPrimitive(typeof(uint))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(32, isSigned: false)
},
new ExpectedPrimitive(typeof(long))
{
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Int(64, isSigned: true)
},
new ExpectedPrimitive(typeof(ulong))
{
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Int(64, isSigned: false)
},
new ExpectedPrimitive(typeof(float))
{
PhysicalType = PhysicalType.Float
},
new ExpectedPrimitive(typeof(double))
{
PhysicalType = PhysicalType.Double
},
new ExpectedPrimitive(typeof(decimal))
{
PhysicalType = PhysicalType.FixedLenByteArray,
LogicalType = LogicalType.Decimal(29, 3),
LogicalTypeOverride = LogicalType.Decimal(29, 3),
Length = 16
},
new ExpectedPrimitive(typeof(Date))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Date()
},
new ExpectedPrimitive(typeof(DateTime))
{
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Timestamp(true, TimeUnit.Micros)
},
new ExpectedPrimitive(typeof(DateTime))
{
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Timestamp(true, TimeUnit.Millis),
LogicalTypeOverride = LogicalType.Timestamp(true, TimeUnit.Millis)
},
new ExpectedPrimitive(typeof(DateTimeNanos))
{
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Timestamp(true, TimeUnit.Nanos)
},
new ExpectedPrimitive(typeof(TimeSpan))
{
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Time(true, TimeUnit.Micros)
},
new ExpectedPrimitive(typeof(TimeSpan))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Time(true, TimeUnit.Millis),
LogicalTypeOverride = LogicalType.Time(true, TimeUnit.Millis)
},
new ExpectedPrimitive(typeof(TimeSpanNanos))
{
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Time(true, TimeUnit.Nanos)
},
new ExpectedPrimitive(typeof(string))
{
PhysicalType = PhysicalType.ByteArray,
LogicalType = LogicalType.String(),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(string))
{
PhysicalType = PhysicalType.ByteArray,
LogicalType = LogicalType.Json(),
LogicalTypeOverride = LogicalType.Json(),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(byte[]))
{
PhysicalType = PhysicalType.ByteArray,
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(byte[]))
{
PhysicalType = PhysicalType.ByteArray,
LogicalType = LogicalType.Bson(),
LogicalTypeOverride = LogicalType.Bson(),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(bool?))
{
PhysicalType = PhysicalType.Boolean,
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(sbyte?))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(8, isSigned: true),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(byte?))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(8, isSigned: false),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(short?))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(16, isSigned: true),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(ushort?))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(16, isSigned: false),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(int?))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(32, isSigned: true),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(uint?))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(32, isSigned: false),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(long?))
{
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Int(64, isSigned: true),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(ulong?))
{
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Int(64, isSigned: false),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(float?))
{
PhysicalType = PhysicalType.Float,
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(double?))
{
PhysicalType = PhysicalType.Double,
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(decimal?))
{
PhysicalType = PhysicalType.FixedLenByteArray,
LogicalType = LogicalType.Decimal(29, 2),
LogicalTypeOverride = LogicalType.Decimal(29, 2),
Repetition = Repetition.Optional,
Length = 16
},
new ExpectedPrimitive(typeof(Date?))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Date(),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(DateTime?))
{
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Timestamp(true, TimeUnit.Micros),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(DateTime?))
{
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Timestamp(true, TimeUnit.Millis),
LogicalTypeOverride = LogicalType.Timestamp(true, TimeUnit.Millis),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(DateTimeNanos?))
{
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Timestamp(true, TimeUnit.Nanos),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(TimeSpan?))
{
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Time(true, TimeUnit.Micros),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(TimeSpan?))
{
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Time(true, TimeUnit.Millis),
LogicalTypeOverride = LogicalType.Time(true, TimeUnit.Millis),
Repetition = Repetition.Optional
},
new ExpectedPrimitive(typeof(TimeSpanNanos?))
{
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Time(true, TimeUnit.Nanos),
Repetition = Repetition.Optional
},
});
}
public DecimalRead()
{
Console.WriteLine("Writing data...");
var timer = Stopwatch.StartNew();
var rand = new Random(123);
_values = Enumerable.Range(0, 1_000_000).Select(i =>
{
var n = rand.Next();
var sign = rand.NextDouble() < 0.5 ? -1M : +1M;
return(sign * ((decimal)n * n * n) / 1000M);
}).ToArray();
using (var fileWriter = new ParquetFileWriter(Filename, new Column[] { new Column <decimal>("Value", LogicalType.Decimal(precision: 29, scale: 3)) }))
{
using var rowGroupWriter = fileWriter.AppendRowGroup();
using var valueWriter = rowGroupWriter.NextColumn().LogicalWriter <decimal>();
valueWriter.WriteBatch(_values);
fileWriter.Close();
}
Console.WriteLine("Wrote {0:N0} rows in {1:N2} sec", _values.Length, timer.Elapsed.TotalSeconds);
Console.WriteLine();
}
public static void TestUnsupportedLogicalTypeOverride()
{
var exception = Assert.Throws <ParquetException>(() =>
new Column <DateTime>("DateTime", LogicalType.Json()).CreateSchemaNode());
Assert.That(
exception?.Message,
Contains.Substring("JSON can not be applied to primitive type INT64"));
}
private static extern IntPtr PrimitiveNode_Make(
string name, Repetition repetition, PhysicalType type, LogicalType logicalType,
int length, int precision, int scale, out IntPtr primitiveNode);
public PrimitiveNode(
string name, Repetition repetition, PhysicalType type, LogicalType logicalType = LogicalType.None,
int length = -1, int precision = -1, int scale = -1)
: this(Make(name, repetition, type, logicalType, length, precision, scale))
{
}
/// <summary>
/// Initializes a new instance of the <see cref="Logical" /> class
/// </summary>
/// <param name="type">Type of logical, Or, And, Xor or Not</param>
public Logical(LogicalType type)
{
TypeOfLogical = type;
}
public void TryParse_WithOneTypeArgument_ReturnsLogicalType()
{
String input = "tst:Foo<sys:Int32>";
var logicalType = LogicalType.TryParse(input);
logicalType.HasValue.Should().BeTrue();
var arg = new LogicalType("sys", "Int32");
(logicalType.Value == new LogicalType("tst", "Foo", new[] {arg}, Maybe.NoValue)).Should().BeTrue();
}
public ObjectFixup(LogicalType logicalType)
{
this.TargetType = logicalType;
}
public void TryParse_WithMultipleTypeArguments_ReturnsLogicalType()
{
String input = "tst:Foo<sys:Int32, sys:String, sys:DateTime>";
var logicalType = LogicalType.TryParse(input);
logicalType.HasValue.Should().BeTrue();
var arg1 = new LogicalType("sys", "Int32");
var arg2 = new LogicalType("sys", "String");
var arg3 = new LogicalType("sys", "DateTime");
(logicalType.Value == new LogicalType("tst", "Foo", new[] { arg1, arg2, arg3 }, Maybe.NoValue)).Should().BeTrue();
}
public ExpressionOperatorLogical(LogicalType type)
{
Type = type;
}
public void TryParse_SuperComplex_ReturnsLogicalType()
{
String input = "tst:Result<sys:Tuple<sys:Int32, tst:ComplexType:v47>, tst:Observation<tst:Error>>:v42";
var logicalType = LogicalType.TryParse(input);
logicalType.HasValue.Should().BeTrue();
var expected = new LogicalType("tst", "Result", new[]
{
new LogicalType("sys", "Tuple", new [] { new LogicalType("sys", "Int32"), new LogicalType("tst", "ComplexType", 0, 47.ToMaybe()) }, Maybe.NoValue),
new LogicalType("tst", "Observation", new [] { new LogicalType("tst", "Error")}, Maybe.NoValue)
}, 42.ToMaybe());
logicalType.Value.Equals(expected).Should().BeTrue();
}
public Condition With(LogicalType op, ExpressionKey key, ExpressionSyntax expression, ValueType value)
{
var newType = CombineType(type, op);
return(new Condition(Kind, newType, parts.Add(new Part(key, expression, value))));
}
public void NoTypeArgumentsAndZeroArity_IsNotOpenType()
{
var logicalType = new LogicalType("tst", "Foo");
logicalType.IsOpenType.Should().BeFalse();
}
private static ExpectedColumn[] CreateExpectedColumns()
{
return(new[]
{
new ExpectedColumn
{
Name = "boolean_field",
PhysicalType = PhysicalType.Boolean,
Values = Enumerable.Range(0, NumRows).Select(i => i % 3 == 0).ToArray(),
Min = false,
Max = true
},
new ExpectedColumn
{
Name = "boolean?_field",
PhysicalType = PhysicalType.Boolean,
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (bool?)null : i % 3 == 0).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = false,
Max = true
},
new ExpectedColumn
{
Name = "int8_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(8, isSigned: true),
Values = Enumerable.Range(0, NumRows).Select(i => (sbyte)i).ToArray(),
Min = 0,
Max = NumRows - 1
},
new ExpectedColumn
{
Name = "int8?_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(8, isSigned: true),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (sbyte?)null : (sbyte)i).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = 1,
Max = NumRows - 1
},
new ExpectedColumn
{
Name = "uint8_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(8, isSigned: false),
Values = Enumerable.Range(0, NumRows).Select(i => (byte)i).ToArray(),
Min = 0,
Max = NumRows - 1
},
new ExpectedColumn
{
Name = "uint8?_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(8, isSigned: false),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (byte?)null : (byte)i).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = 1,
Max = NumRows - 1
},
new ExpectedColumn
{
Name = "int16_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(16, isSigned: true),
Values = Enumerable.Range(0, NumRows).Select(i => (short)i).ToArray(),
Min = 0,
Max = NumRows - 1
},
new ExpectedColumn
{
Name = "int16?_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(16, isSigned: true),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (short?)null : (short)i).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = 1,
Max = NumRows - 1
},
new ExpectedColumn
{
Name = "uint16_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(16, isSigned: false),
Values = Enumerable.Range(0, NumRows).Select(i => (ushort)i).ToArray(),
Min = 0,
Max = NumRows - 1
},
new ExpectedColumn
{
Name = "uint16?_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(16, isSigned: false),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (ushort?)null : (ushort)i).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = 1,
Max = NumRows - 1
},
new ExpectedColumn
{
Name = "int32_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(32, isSigned: true),
Values = Enumerable.Range(0, NumRows).ToArray(),
Min = 0,
Max = NumRows - 1
},
new ExpectedColumn
{
Name = "int32?_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(32, isSigned: true),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (int?)null : i).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = 1,
Max = NumRows - 1
},
new ExpectedColumn
{
Name = "uint32_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(32, isSigned: false),
Values = Enumerable.Range(0, NumRows).Select(i => (uint)i).ToArray(),
Min = 0,
Max = NumRows - 1
},
new ExpectedColumn
{
Name = "uint32?_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Int(32, isSigned: false),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (uint?)null : (uint)i).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = 1,
Max = NumRows - 1
},
new ExpectedColumn
{
Name = "int64_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Int(64, isSigned: true),
Values = Enumerable.Range(0, NumRows).Select(i => (long)i * i).ToArray(),
Min = 0,
Max = (NumRows - 1) * (NumRows - 1)
},
new ExpectedColumn
{
Name = "int64?_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Int(64, isSigned: true),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (long?)null : (long)i * i).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = 1,
Max = (NumRows - 1) * (NumRows - 1)
},
new ExpectedColumn
{
Name = "uint64_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Int(64, isSigned: false),
Values = Enumerable.Range(0, NumRows).Select(i => (ulong)(i * i)).ToArray(),
Min = 0,
Max = (NumRows - 1) * (NumRows - 1)
},
new ExpectedColumn
{
Name = "uint64?_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Int(64, isSigned: false),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (ulong?)null : (ulong)(i * i)).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = 1,
Max = (NumRows - 1) * (NumRows - 1)
},
new ExpectedColumn
{
Name = "int96_field",
PhysicalType = PhysicalType.Int96,
LogicalType = LogicalType.None(),
Values = Enumerable.Range(0, NumRows).Select(i => new Int96(i, i * i, i * i * i)).ToArray(),
HasStatistics = false
},
new ExpectedColumn
{
Name = "int96?_field",
PhysicalType = PhysicalType.Int96,
LogicalType = LogicalType.None(),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (Int96?)null : new Int96(i, i * i, i * i * i)).ToArray(),
HasStatistics = false
},
new ExpectedColumn
{
Name = "float_field",
PhysicalType = PhysicalType.Float,
Values = Enumerable.Range(0, NumRows).Select(i => i % 5 == 0 ? float.NaN : (float)Math.Sqrt(i)).ToArray(),
Min = 1,
Max = (float)Math.Sqrt(NumRows - 1)
},
new ExpectedColumn
{
Name = "float?_field",
PhysicalType = PhysicalType.Float,
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (float?)null : i % 5 == 0 ? float.NaN : (float)Math.Sqrt(i)).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = 1,
Max = (float)Math.Sqrt(NumRows - 1)
},
new ExpectedColumn
{
Name = "double_field",
PhysicalType = PhysicalType.Double,
Values = Enumerable.Range(0, NumRows).Select(i => i % 7 == 0 ? double.NaN : i * Math.PI).ToArray(),
Min = Math.PI,
Max = (NumRows - 1) * Math.PI
},
new ExpectedColumn
{
Name = "double?_field",
PhysicalType = PhysicalType.Double,
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (double?)null : i % 7 == 0 ? double.NaN : i * Math.PI).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = Math.PI,
Max = (NumRows - 1) * Math.PI
},
new ExpectedColumn
{
Name = "decimal128_field",
PhysicalType = PhysicalType.FixedLenByteArray,
LogicalType = LogicalType.Decimal(29, 3),
LogicalTypeOverride = LogicalType.Decimal(29, 3),
Length = 16,
Values = Enumerable.Range(0, NumRows).Select(i => ((decimal)i * i * i) / 1000 - 10).ToArray(),
Min = -10m,
Max = ((NumRows - 1m) * (NumRows - 1m) * (NumRows - 1m)) / 1000 - 10,
Converter = v => LogicalRead.ToDecimal((FixedLenByteArray)v, 3)
},
new ExpectedColumn
{
Name = "decimal128?_field",
PhysicalType = PhysicalType.FixedLenByteArray,
LogicalType = LogicalType.Decimal(29, 3),
LogicalTypeOverride = LogicalType.Decimal(29, 3),
Length = 16,
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? null : ((decimal?)i * i * i) / 1000 - 10).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = -9.999m,
Max = ((NumRows - 1m) * (NumRows - 1m) * (NumRows - 1m)) / 1000 - 10,
Converter = v => LogicalRead.ToDecimal((FixedLenByteArray)v, 3)
},
new ExpectedColumn
{
Name = "uuid_field",
PhysicalType = PhysicalType.FixedLenByteArray,
LogicalType = LogicalType.Uuid(),
LogicalTypeOverride = LogicalType.Uuid(),
Length = 16,
Values = Enumerable.Range(0, NumRows).Select(i => new Guid(i, 0x1234, 0x5678, 0x9A, 0xBC, 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x7F)).ToArray(),
Min = new Guid(0, 0x1234, 0x5678, 0x9A, 0xBC, 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x7F),
Max = new Guid(NumRows - 1, 0x1234, 0x5678, 0x9A, 0xBC, 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x7F),
Converter = v => LogicalRead.ToUuid((FixedLenByteArray)v)
},
new ExpectedColumn
{
Name = "uuid?_field",
PhysicalType = PhysicalType.FixedLenByteArray,
LogicalType = LogicalType.Uuid(),
LogicalTypeOverride = LogicalType.Uuid(),
Length = 16,
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? null : (Guid?)new Guid(i, 0x1234, 0x5678, 0x9A, 0xBC, 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x7F)).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = new Guid(1, 0x1234, 0x5678, 0x9A, 0xBC, 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x7F),
Max = new Guid(NumRows - 1, 0x1234, 0x5678, 0x9A, 0xBC, 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x7F),
Converter = v => LogicalRead.ToUuid((FixedLenByteArray)v)
},
new ExpectedColumn
{
Name = "date_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Date(),
Values = Enumerable.Range(0, NumRows).Select(i => new Date(2018, 01, 01).AddDays(i)).ToArray(),
Min = new Date(2018, 01, 01).Days,
Max = new Date(2018, 01, 01).AddDays(NumRows - 1).Days
},
new ExpectedColumn
{
Name = "date?_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Date(),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (Date?)null : new Date(2018, 01, 01).AddDays(i)).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = new Date(2018, 01, 01).AddDays(1).Days,
Max = new Date(2018, 01, 01).AddDays(NumRows - 1).Days
},
new ExpectedColumn
{
Name = "datetime_micros_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Timestamp(true, TimeUnit.Micros),
Values = Enumerable.Range(0, NumRows).Select(i => new DateTime(2018, 01, 01) + TimeSpan.FromHours(i)).ToArray(),
Min = new DateTime(2018, 01, 01),
Max = new DateTime(2018, 01, 01) + TimeSpan.FromHours(NumRows - 1),
Converter = v => LogicalRead.ToDateTimeMicros((long)v)
},
new ExpectedColumn
{
Name = "datetime?_micros_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Timestamp(true, TimeUnit.Micros),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (DateTime?)null : new DateTime(2018, 01, 01) + TimeSpan.FromHours(i)).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = new DateTime(2018, 01, 01) + TimeSpan.FromHours(1),
Max = new DateTime(2018, 01, 01) + TimeSpan.FromHours(NumRows - 1),
Converter = v => LogicalRead.ToDateTimeMicros((long)v)
},
new ExpectedColumn
{
Name = "datetime_millis_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Timestamp(true, TimeUnit.Millis),
LogicalTypeOverride = LogicalType.Timestamp(true, TimeUnit.Millis),
Values = Enumerable.Range(0, NumRows).Select(i => new DateTime(2018, 01, 01) + TimeSpan.FromHours(i)).ToArray(),
Min = new DateTime(2018, 01, 01),
Max = new DateTime(2018, 01, 01) + TimeSpan.FromHours(NumRows - 1),
Converter = v => LogicalRead.ToDateTimeMillis((long)v)
},
new ExpectedColumn
{
Name = "datetime?_millis_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Timestamp(true, TimeUnit.Millis),
LogicalTypeOverride = LogicalType.Timestamp(true, TimeUnit.Millis),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (DateTime?)null : new DateTime(2018, 01, 01) + TimeSpan.FromHours(i)).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = new DateTime(2018, 01, 01) + TimeSpan.FromHours(1),
Max = new DateTime(2018, 01, 01) + TimeSpan.FromHours(NumRows - 1),
Converter = v => LogicalRead.ToDateTimeMillis((long)v)
},
new ExpectedColumn
{
Name = "datetime_nanos_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Timestamp(true, TimeUnit.Nanos),
Values = Enumerable.Range(0, NumRows).Select(i => new DateTimeNanos(new DateTime(2018, 01, 01) + TimeSpan.FromHours(i))).ToArray(),
Min = new DateTimeNanos(new DateTime(2018, 01, 01)),
Max = new DateTimeNanos(new DateTime(2018, 01, 01) + TimeSpan.FromHours(NumRows - 1)),
Converter = v => new DateTimeNanos((long)v)
},
new ExpectedColumn
{
Name = "datetime?_nanos_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Timestamp(true, TimeUnit.Nanos),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (DateTimeNanos?)null : new DateTimeNanos(new DateTime(2018, 01, 01) + TimeSpan.FromHours(i))).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = new DateTimeNanos(new DateTime(2018, 01, 01) + TimeSpan.FromHours(1)),
Max = new DateTimeNanos(new DateTime(2018, 01, 01) + TimeSpan.FromHours(NumRows - 1)),
Converter = v => new DateTimeNanos((long)v)
},
new ExpectedColumn
{
Name = "timespan_micros_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Time(true, TimeUnit.Micros),
Values = Enumerable.Range(0, NumRows).Select(i => TimeSpan.FromHours(-13) + TimeSpan.FromHours(i)).ToArray(),
Min = TimeSpan.FromHours(-13),
Max = TimeSpan.FromHours(-13 + NumRows - 1),
Converter = v => LogicalRead.ToTimeSpanMicros((long)v)
},
new ExpectedColumn
{
Name = "timespan?_micros_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Time(true, TimeUnit.Micros),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (TimeSpan?)null : TimeSpan.FromHours(-13) + TimeSpan.FromHours(i)).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = TimeSpan.FromHours(-13 + 1),
Max = TimeSpan.FromHours(-13 + NumRows - 1),
Converter = v => LogicalRead.ToTimeSpanMicros((long)v)
},
new ExpectedColumn
{
Name = "timespan_millis_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Time(true, TimeUnit.Millis),
LogicalTypeOverride = LogicalType.Time(true, TimeUnit.Millis),
Values = Enumerable.Range(0, NumRows).Select(i => TimeSpan.FromHours(-13) + TimeSpan.FromHours(i)).ToArray(),
Min = TimeSpan.FromHours(-13),
Max = TimeSpan.FromHours(-13 + NumRows - 1),
Converter = v => LogicalRead.ToTimeSpanMillis((int)v)
},
new ExpectedColumn
{
Name = "timespan?_millis_field",
PhysicalType = PhysicalType.Int32,
LogicalType = LogicalType.Time(true, TimeUnit.Millis),
LogicalTypeOverride = LogicalType.Time(true, TimeUnit.Millis),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (TimeSpan?)null : TimeSpan.FromHours(-13) + TimeSpan.FromHours(i)).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = TimeSpan.FromHours(-13 + 1),
Max = TimeSpan.FromHours(-13 + NumRows - 1),
Converter = v => LogicalRead.ToTimeSpanMillis((int)v)
},
new ExpectedColumn
{
Name = "timespan_nanos_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Time(true, TimeUnit.Nanos),
Values = Enumerable.Range(0, NumRows).Select(i => new TimeSpanNanos(TimeSpan.FromHours(-13) + TimeSpan.FromHours(i))).ToArray(),
Min = new TimeSpanNanos(TimeSpan.FromHours(-13)),
Max = new TimeSpanNanos(TimeSpan.FromHours(-13 + NumRows - 1)),
Converter = v => new TimeSpanNanos((long)v)
},
new ExpectedColumn
{
Name = "timespan?_nanos_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Time(true, TimeUnit.Nanos),
Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (TimeSpanNanos?)null : new TimeSpanNanos(TimeSpan.FromHours(-13) + TimeSpan.FromHours(i))).ToArray(),
NullCount = (NumRows + 10) / 11,
NumValues = NumRows - (NumRows + 10) / 11,
Min = new TimeSpanNanos(TimeSpan.FromHours(-13 + 1)),
Max = new TimeSpanNanos(TimeSpan.FromHours(-13 + NumRows - 1)),
Converter = v => new TimeSpanNanos((long)v)
},
new ExpectedColumn
{
Name = "string_field",
PhysicalType = PhysicalType.ByteArray,
LogicalType = LogicalType.String(),
Values = Enumerable.Range(0, NumRows).Select(i => i % 9 == 0 ? i % 18 == 0 ? null : "" : $"Hello, {i}!").ToArray(),
NullCount = (NumRows + 17) / 18,
NumValues = NumRows - (NumRows + 17) / 18,
Min = "",
Max = "Hello, 98!",
Converter = v => LogicalRead.ToString((ByteArray)v)
},
new ExpectedColumn
{
Name = "json_field",
PhysicalType = PhysicalType.ByteArray,
LogicalType = LogicalType.Json(),
LogicalTypeOverride = LogicalType.Json(),
Values = Enumerable.Range(0, NumRows).Select(i => i % 9 == 0 ? null : $"{{ \"id\", {i} }}").ToArray(),
NullCount = (NumRows + 8) / 9,
NumValues = NumRows - (NumRows + 8) / 9,
Min = "{ \"id\", 1 }",
Max = "{ \"id\", 98 }",
Converter = v => LogicalRead.ToString((ByteArray)v)
},
new ExpectedColumn
{
Name = "bytearray_field",
PhysicalType = PhysicalType.ByteArray,
Values = Enumerable.Range(0, NumRows).Select(i => i % 3 == 0 ? i % 6 == 0 ? null : new byte[0] : BitConverter.GetBytes(i)).ToArray(),
NullCount = (NumRows + 5) / 6,
NumValues = NumRows - (NumRows + 5) / 6,
Min = new byte[0],
Max = BitConverter.GetBytes(NumRows - 1),
Converter = v => LogicalRead.ToByteArray((ByteArray)v)
},
new ExpectedColumn
{
Name = "bson_field",
PhysicalType = PhysicalType.ByteArray,
LogicalType = LogicalType.Bson(),
LogicalTypeOverride = LogicalType.Bson(),
Values = Enumerable.Range(0, NumRows).Select(i => i % 3 == 0 ? null : BitConverter.GetBytes(i)).ToArray(),
NullCount = (NumRows + 2) / 3,
NumValues = NumRows - (NumRows + 2) / 3,
Min = BitConverter.GetBytes(1),
Max = BitConverter.GetBytes(NumRows - 1),
Converter = v => LogicalRead.ToByteArray((ByteArray)v)
},
new ExpectedColumn
{
Name = "nested_array_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Int(64, isSigned: true),
Values = Enumerable.Range(0, NumRows).Select(i =>
{
if (i % 3 == 0)
{
return new[]
{
new long[] { 1, 2 },
new long[] { 3, 4 }
};
}
if (i % 3 == 1)
{
return new[]
{
null,
null,
new long[] { 13, 14 },
null,
new long[] { 15, 16 }
};
}
return new long[][]
{
null
};
}).ToArray(),
NullCount = (NumRows / 3 + 1) * 4 - 1,
NumValues = (NumRows / 3 + 1) * 8,
Min = 1,
Max = 16
},
new ExpectedColumn
{
Name = "nullable_nested_array_field",
PhysicalType = PhysicalType.Int64,
LogicalType = LogicalType.Int(64, isSigned: true),
Values = Enumerable.Range(0, NumRows).Select(i =>
{
if (i % 3 == 0)
{
return new[]
{
new long?[] { 1, 2 },
new long?[] { 3, null }
};
}
if (i % 3 == 1)
{
return new[]
{
null,
null,
new long?[] { 13, 14 },
null,
new long?[] { null, 16 }
};
}
return new long?[][]
{
null
};
}).ToArray(),
NullCount = (NumRows / 3 + 1) * 6 - 1,
NumValues = (NumRows / 3 + 1) * 6,
Min = 1,
Max = 16
},
new ExpectedColumn
{
Name = "array_of_bytearrays",
PhysicalType = PhysicalType.ByteArray,
Values = Enumerable.Range(0, NumRows).Select(i =>
{
if (i % 3 == 0)
{
return new[]
{
BitConverter.GetBytes(3 * i),
BitConverter.GetBytes(2 * i)
};
}
if (i % 3 == 1)
{
return new[]
{
null,
null,
BitConverter.GetBytes(i),
null
};
}
return new byte[][]
{
null
};
}).ToArray(),
NullCount = (NumRows / 3 + 1) * 4 - 1,
NumValues = (NumRows / 3 + 1) * 3,
Min = BitConverter.GetBytes(0),
Max = BitConverter.GetBytes(252),
Converter = v => LogicalRead.ToByteArray((ByteArray)v)
}
});
}
public static void TestDecimalSeries()
{
var timer = Stopwatch.StartNew();
var rand = new Random(123);
Console.WriteLine("Generating data...");
var values = Enumerable.Range(0, 10_000_000).Select(i =>
{
var n = rand.Next();
var sign = rand.NextDouble() < 0.5 ? -1M : +1M;
return(sign * ((decimal)n * n * n) / 1000M);
}).ToArray();
Console.WriteLine("Generated {0:N0} rows in {1:N2} sec", values.Length, timer.Elapsed.TotalSeconds);
Console.WriteLine();
Console.WriteLine("Saving to Parquet");
timer.Restart();
using (var fileWriter = new ParquetFileWriter("decimal_timeseries.parquet", new Column[] { new Column <decimal>("Value", LogicalType.Decimal(precision: 29, scale: 3)) }))
using (var rowGroupWriter = fileWriter.AppendRowGroup())
using (var valueWriter = rowGroupWriter.NextColumn().LogicalWriter <decimal>())
{
valueWriter.WriteBatch(values);
}
Console.WriteLine("Saved to Parquet ({0:N0} bytes) in {1:N2} sec", new FileInfo("decimal_timeseries.parquet").Length, timer.Elapsed.TotalSeconds);
Console.WriteLine();
Console.WriteLine("Saving to Parquet.NET");
timer.Restart();
{
var valueField = new DecimalDataField("Value", precision: 29, scale: 3);
var schema = new Parquet.Data.Schema(valueField);
using (var stream = File.Create("decimal_timeseries.parquet.net"))
using (var parquetWriter = new ParquetWriter(schema, stream))
using (var groupWriter = parquetWriter.CreateRowGroup())
{
groupWriter.WriteColumn(new DataColumn(valueField, values));
}
}
Console.WriteLine("Saved to Parquet.NET ({0:N0} bytes) in {1:N2} sec", new FileInfo("decimal_timeseries.parquet.net").Length, timer.Elapsed.TotalSeconds);
}
private static extern IntPtr GroupNode_Make(
string name, Repetition repetition, IntPtr fields, int numFields, LogicalType logicalType, out IntPtr groupNode);
/// <summary>
/// Adds a LogicalType to the TypeContainer. The LogicalType is mapped by
/// Framework Type, i.e FX Type => LogicalType, as well as by XmlQualifiedName,
/// i.e (Name:Namespace) => LogicalType.
/// </summary>
public void AddType(LogicalType lType, bool encoded)
{
AddType(lType, lType.Name, lType.Namespace, encoded);
}
public void NoTypeArgumentsAndNonZeroArity_IsOpenType()
{
var logicalType = new LogicalType("tst", "Foo", 1, Maybe.NoValue);
logicalType.IsOpenType.Should().BeTrue();
}
public ExampleSchemaBuilder WithDifferentPhysicalType()
{
_primitiveLogicalType = LogicalType.Int(64, true);
_physicalType = PhysicalType.Int64;
return(this);
}
//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
// Static Constructor
//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
static TypeContainer()
{
//These types come the fx xml serialization file Types.cs
// I don't have a type for Object (i.e. any).
// Do I need ambiguous?
LogicalType[] iTypes = new LogicalType[] {
MakeIntrinsicType(typeof(string), "string", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, true),
MakeIntrinsicType(typeof(int), "int", Soap.XsdUrl, SerializerType.Primitive, TypeID.Int32, true),
MakeIntrinsicType(typeof(bool), "boolean", Soap.XsdUrl, SerializerType.Primitive, TypeID.Boolean, true),
MakeIntrinsicType(typeof(short), "short", Soap.XsdUrl, SerializerType.Primitive, TypeID.Int16, true),
MakeIntrinsicType(typeof(long), "long", Soap.XsdUrl, SerializerType.Primitive, TypeID.Int64, true),
MakeIntrinsicType(typeof(float), "float", Soap.XsdUrl, SerializerType.Primitive, TypeID.Single, true),
MakeIntrinsicType(typeof(double), "double", Soap.XsdUrl, SerializerType.Primitive, TypeID.Double, true),
MakeIntrinsicType(typeof(decimal), "decimal", Soap.XsdUrl, SerializerType.Primitive, TypeID.Decimal, true),
MakeIntrinsicType(typeof(DateTime), "dateTime", Soap.XsdUrl, SerializerType.Primitive, TypeID.DateTime, true),
MakeIntrinsicType(typeof(System.Xml.XmlQualifiedName), "QName", Soap.XsdUrl, SerializerType.Custom, TypeID.Compound, true, CustomSerializerType.QName),
MakeIntrinsicType(typeof(byte), "unsignedByte", Soap.XsdUrl, SerializerType.Primitive, TypeID.Byte, true),
MakeIntrinsicType(typeof(sbyte), "byte", Soap.XsdUrl, SerializerType.Primitive, TypeID.SByte, true),
MakeIntrinsicType(typeof(ushort), "unsignedShort", Soap.XsdUrl, SerializerType.Primitive, TypeID.UInt16, true),
MakeIntrinsicType(typeof(uint), "unsignedInt", Soap.XsdUrl, SerializerType.Primitive, TypeID.UInt32, true),
MakeIntrinsicType(typeof(ulong), "unsignedLong", Soap.XsdUrl, SerializerType.Primitive, TypeID.UInt64, true),
MakeIntrinsicType(typeof(object), "anyType", Soap.XsdUrl, SerializerType.Complex, TypeID.Compound, true, CustomSerializerType.Object),
//start ambiguous types
MakeIntrinsicType(typeof(DateTime), "date", Soap.XsdUrl, SerializerType.Primitive, TypeID.Date, false),
MakeIntrinsicType(typeof(DateTime), "time", Soap.XsdUrl, SerializerType.Primitive, TypeID.Time, false),
MakeIntrinsicType(typeof(string), "Name", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "NCName", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "NMTOKEN", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "NMTOKENS", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(byte[]), "base64Binary", Soap.XsdUrl, SerializerType.Custom, TypeID.Compound, true, CustomSerializerType.Base64),
MakeIntrinsicType(typeof(byte[]), "hexBinary", Soap.XsdUrl, SerializerType.Custom, TypeID.Compound, false, CustomSerializerType.Hex),
//start non-xsd types
MakeIntrinsicType(typeof(Guid), "guid", Soap.UrtTypesNS, SerializerType.Primitive, TypeID.Guid, true),
MakeIntrinsicType(typeof(char), "char", Soap.UrtTypesNS, SerializerType.Primitive, TypeID.Char, true),
MakeIntrinsicType(typeof(byte[]), "base64", Soap.Encoding, SerializerType.Custom, TypeID.Compound, false, CustomSerializerType.Base64),
//begin unsupported types.
MakeIntrinsicType(typeof(string), "anyURI", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "duration", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "ENTITY", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "ENTITIES", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "gDay", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "gMonth", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "gMonthDay", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "gYear", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "gYearMonth", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "ID", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "IDREF", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "IDREFS", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "integer", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "language", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "negativeInteger", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "nonNegativeInteger", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "nonPositiveInteger", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "normalizedString", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "NOTATION", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "positiveInteger", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false),
MakeIntrinsicType(typeof(string), "token", Soap.XsdUrl, SerializerType.Primitive, TypeID.String, false)
};
s_intrinsicsByType = new Dictionary <Type, LogicalType>();
for (int i = 0; i < iTypes.Length; ++i)
{
if ((iTypes[i].MappingFlags & TypeMappingFlags.AllowTypeMapping) != 0)
{
s_intrinsicsByType[iTypes[i].Type] = iTypes[i];
}
}
s_intrinsicsByName = new Dictionary <XmlQualifiedName, LogicalType>(iTypes.Length);
for (int i = 0; i < iTypes.Length; ++i)
{
XmlQualifiedName qname = new XmlQualifiedName(iTypes[i].Name, iTypes[i].Namespace);
Debug.Assert(!s_intrinsicsByName.ContainsKey(qname), "Adding the same intrinsic name twice");
s_intrinsicsByName[qname] = iTypes[i];
}
}
public void TypeArguments_IsNotOpenType()
{
var arguments = new[]
{
new LogicalType("sys", "Int32"),
new LogicalType("sys", "String")
};
var logicalType = new LogicalType("tst", "Foo", arguments, Maybe.NoValue);
logicalType.IsOpenType.Should().BeFalse();
}
/// <summary>
/// Adds a LogicalType to the TypeContainer. The LogicalType is mapped by
/// Framework Type, i.e FX Type => LogicalType, as well as by XmlQualifiedName,
/// i.e (Name:Namespace) => LogicalType.
/// </summary>
public void AddType(LogicalType lType, string name, string ns, bool encoded)
{
Dictionary <TypeAndNamespace, LogicalType> typesByTypeTable;
Dictionary <XmlQualifiedName, LogicalType> typesByNameTable;
bool allowTypeMapping, allowNameMapping;
XmlQualifiedName typeQName;
TypeAndNamespace typeAndNamespace;
// Most type should avoid the null namespace, but some types (like XmlNode)
// really don't have any need for a ns because they are inherently portable
// by carrying all their namespace info with them.
Debug.Assert(ns != null || lType.NamespaceIsPortable, "A non-portable type MUST specify a namespace.");
lock (_lookUpLock)
{
if (encoded)
{
typesByTypeTable = _userEncodedTypesByType;
typesByNameTable = _userEncodedTypesByQName;
}
else
{
typesByTypeTable = _userLiteralTypesByType;
typesByNameTable = _userLiteralTypesByQName;
}
allowTypeMapping = (lType.MappingFlags & TypeMappingFlags.AllowTypeMapping) != 0;
allowNameMapping = (lType.MappingFlags & TypeMappingFlags.AllowNameMapping) != 0;
typeQName = new XmlQualifiedName(name, ns);
typeAndNamespace = new TypeAndNamespace(lType.Type, lType.NamespaceIsPortable ? null : ns);
//if we're allowed to map the C# type to the LogicalType
if (allowTypeMapping)
{
// Don't add the type yet. Verify that I can also add it to the type table before committing it.
if (typesByTypeTable.ContainsKey(typeAndNamespace))
{
Debug.WriteLine("Adding " + ns + ":" + lType.Type + " twice.");
throw new InvalidOperationException(SR.Format(SR.XmlS_TwoMappings_1, lType.Type));
}
// We should never have both a null namespace for a type in our table and a non-null namespace.
// We should EITHER have the type in our table at most once, with the null namespace;
// OR we should have the type in our table any number of times, all with non-null namespaces.
Debug.Assert(!(typeAndNamespace.DefaultNamespace != null &&
typesByTypeTable.ContainsKey(new TypeAndNamespace(typeAndNamespace.Type, null))));
}
//If we're allowed to map the qname to this LogicalType
if (allowNameMapping)
{
if (typesByNameTable.ContainsKey(typeQName))
{
Debug.WriteLine("Two types in " + lType.Namespace + ":" + lType.Name);
throw new InvalidOperationException(SR.Format(SR.XmlS_TwoMappings_1, typeQName));
}
typesByNameTable[typeQName] = lType;
}
if (allowTypeMapping)
{
typesByTypeTable[typeAndNamespace] = lType;
}
if (encoded && !_userEncodedNamespaces.Contains(ns))
{
_userEncodedNamespaces.Add(ns);
}
}
}
public void GetOpenType()
{
var logicalType = new LogicalType("tst", "Result", new[] { new LogicalType("sys", "Int32"), new LogicalType("sys", "String") }, Maybe.NoValue);
var openLogicalType = logicalType.GetOpenType();
openLogicalType.Equals(LogicalType.Parse("tst:Result`2")).Should().BeTrue();
}
public long ParquetSharp()
{
using (var fileWriter = new ParquetFileWriter("decimal_timeseries.parquet", new Column[] { new Column <decimal>("Value", LogicalType.Decimal(precision: 29, scale: 3)) }))
{
using var rowGroupWriter = fileWriter.AppendRowGroup();
using var valueWriter = rowGroupWriter.NextColumn().LogicalWriter <decimal>();
valueWriter.WriteBatch(_values);
fileWriter.Close();
}
return(new FileInfo("decimal_timeseries.parquet").Length);
}
public void MakeClosedType()
{
var openLogicalType = new LogicalType("tst", "Result", 2, Maybe.NoValue);
var args = new[]
{
new LogicalType("sys", "Int32"),
new LogicalType("sys", "String")
};
var logicalType = openLogicalType.MakeClosedType(args);
logicalType.Equals(LogicalType.Parse("tst:Result<sys:Int32, sys:String>")).Should().BeTrue();
}
public ArrayLikeFixup(LogicalType collectionType, Fixup memberFixup, object memberFixupTarget) :
this(collectionType, null, memberFixup, memberFixupTarget, null)
{
}
private bool useDefaultNamespaces(LogicalType type)
{
return((!type.IsNullableType || isNullableWithStructValue(type)) && // Nullable<T> with a struct
(!typeof(IXmlSerializable).IsAssignableFrom(type.Type))); // Not an IXmlSerializable
}
/// <summary>
/// Returns true if the LogicalType represents a Nullable<T> type
/// and the T is a primitive or enum.
/// </summary>
private bool isNullableWithStructValue(LogicalType type)
{
return(type.IsNullableType &&
!SerializationHelper.IsSerializationPrimitive(type.NullableValueType.Type) &&
!type.NullableValueType.Type.GetTypeInfo().IsEnum);
}
private static extern IntPtr Node_Logical_Type(IntPtr node, out LogicalType logicalType);
public void QualifiedTypeName_IsPermitted()
{
var logicalType = new LogicalType("tst", "Outer.Inner");
logicalType.TypeName.Should().Be("Outer.Inner");
}
/// <summary>
/// Checks whether this XmlSerializer can begin deserialization at a given XmlElement.
/// </summary>
/// <remarks>
/// It may return false for types that the serializer can deserialize as part of another type.
/// </remarks>
public virtual bool CanDeserialize(XmlReader reader)
{
if (_logicalType == null)
{
return(false);
}
if (true /* AppDomain.CompatVersion >= AppDomain.Orcas */)
{
// Check the type passed to the constructor
if (_isEncoded)
{
if (startElementMatchesAccessor(reader, _logicalType.TypeAccessor, false) ||
startElementMatchesAccessor(reader, _logicalType.TypeAccessor, true))
{
return(true);
}
}
else
{
if (startElementMatchesAccessor(reader, _logicalType.RootAccessor, false))
{
return(true);
}
}
bool soap12;
XmlSerializationReader serialReader = initXmlSerializationReader(reader, null, out soap12);
LogicalType type = serialReader.deriveTypeFromTypeAttribute(reader, null);
if (type != null && _logicalType.Type.IsAssignableFrom(type.Type))
{
return(true);
}
}
else
{ /* Rogue or earlier */
// Check the type passed to the constructor
if (startElementMatchesAccessor(reader, _logicalType.TypeAccessor, false))
{
return(true);
}
// Checking extra types is only something we do for backward compatibility
if (_extraTypes != null)
{
for (int typeNdx = 0; typeNdx < _extraTypes.Length; ++typeNdx)
{
if (startElementMatchesAccessor(reader, _extraTypes[typeNdx].TypeAccessor, false))
{
return(true);
}
}
}
LogicalType type = _reflector.FindType(new XmlQualifiedName(reader.Name, reader.NamespaceURI), _isEncoded);
if (type != null)
{
return(true);
}
}
return(false);
}
