private static void ConvertDateTimeMillis(ReadOnlySpan <DateTime> source, Span <long> destination) { for (int i = 0; i < source.Length; ++i) { destination[i] = LogicalWrite.FromDateTimeMillis(source[i]); } }
private static void ConvertTimeSpanMillis(ReadOnlySpan <TimeSpan> source, Span <int> destination) { for (int i = 0; i < source.Length; ++i) { destination[i] = LogicalWrite.FromTimeSpanMillis(source[i]); } }
private static void ConvertUuid(ReadOnlySpan <Guid> source, Span <FixedLenByteArray> destination, ByteBuffer byteBuffer) { for (int i = 0; i < source.Length; ++i) { destination[i] = LogicalWrite.FromUuid(source[i], byteBuffer); } }
private static void ConvertDecimal128(ReadOnlySpan <decimal> source, Span <FixedLenByteArray> destination, decimal multiplier, ByteBuffer byteBuffer) { for (int i = 0; i < source.Length; ++i) { destination[i] = LogicalWrite.FromDecimal(source[i], multiplier, byteBuffer); } }
private static void ConvertTimeSpanMillis(ReadOnlySpan <TimeSpan?> source, Span <short> defLevels, Span <int> destination, short nullLevel) { for (int i = 0, dst = 0; i < source.Length; ++i) { var value = source[i]; if (value == null) { defLevels[i] = nullLevel; } else { destination[dst++] = LogicalWrite.FromTimeSpanMillis(value.Value); defLevels[i] = (short)(nullLevel + 1); } } }
private static void ConvertUuid(ReadOnlySpan <Guid?> source, Span <short> defLevels, Span <FixedLenByteArray> destination, short nullLevel, ByteBuffer byteBuffer) { for (int i = 0, dst = 0; i < source.Length; ++i) { var value = source[i]; if (value == null) { defLevels[i] = nullLevel; } else { destination[dst++] = LogicalWrite.FromUuid(value.Value, byteBuffer); defLevels[i] = (short)(nullLevel + 1); } } }
private static void ConvertDateTimeMicros(ReadOnlySpan <DateTime?> source, Span <short> defLevels, Span <long> destination, short nullLevel) { for (int i = 0, dst = 0; i != source.Length; ++i) { var value = source[i]; if (value == null) { defLevels[i] = nullLevel; } else { destination[dst++] = LogicalWrite.FromDateTimeMicros(value.Value); defLevels[i] = (short)(nullLevel + 1); } } }
private static void ConvertDecimal128(ReadOnlySpan <decimal?> source, Span <short> defLevels, Span <FixedLenByteArray> destination, decimal multiplier, short nullLevel, ByteBuffer byteBuffer) { for (int i = 0, dst = 0; i != source.Length; ++i) { var value = source[i]; if (value == null) { defLevels[i] = nullLevel; } else { destination[dst++] = LogicalWrite.FromDecimal(value.Value, multiplier, byteBuffer); defLevels[i] = (short)(nullLevel + 1); } } }
private static void ConvertByteArray(ReadOnlySpan <byte[]> source, Span <short> defLevels, Span <ByteArray> destination, short nullLevel, ByteBuffer byteBuffer) { for (int i = 0, dst = 0; i < source.Length; ++i) { var value = source[i]; if (value == null) { if (defLevels == null) { throw new ArgumentException("encountered null value despite column schema node repetition being marked as required"); } defLevels[i] = nullLevel; } else { destination[dst++] = LogicalWrite.FromByteArray(value, byteBuffer); defLevels[i] = (short)(nullLevel + 1); } } }
public static Delegate GetConverter(ColumnDescriptor columnDescriptor, ByteBuffer? byteBuffer) { if (typeof(TLogical) == typeof(bool) || typeof(TLogical) == typeof(int) || typeof(TLogical) == typeof(long) || typeof(TLogical) == typeof(Int96) || typeof(TLogical) == typeof(float) || typeof(TLogical) == typeof(double)) { return LogicalWrite.GetNativeConverter<TPhysical, TPhysical>(); } if (typeof(TLogical) == typeof(bool?) || typeof(TLogical) == typeof(int?) || typeof(TLogical) == typeof(long?) || typeof(TLogical) == typeof(Int96?) || typeof(TLogical) == typeof(float?) || typeof(TLogical) == typeof(double?)) { return LogicalWrite.GetNullableNativeConverter<TPhysical, TPhysical>(); } if (typeof(TLogical) == typeof(sbyte)) { return (LogicalWrite<sbyte, int>.Converter) ((s, _, d, _) => LogicalWrite.ConvertInt8(s, d)); } if (typeof(TLogical) == typeof(sbyte?)) { return (LogicalWrite<sbyte?, int>.Converter) LogicalWrite.ConvertInt8; } if (typeof(TLogical) == typeof(byte)) { return (LogicalWrite<byte, int>.Converter) ((s, _, d, _) => LogicalWrite.ConvertUInt8(s, d)); } if (typeof(TLogical) == typeof(byte?)) { return (LogicalWrite<byte?, int>.Converter) LogicalWrite.ConvertUInt8; } if (typeof(TLogical) == typeof(short)) { return (LogicalWrite<short, int>.Converter) ((s, _, d, _) => LogicalWrite.ConvertInt16(s, d)); } if (typeof(TLogical) == typeof(short?)) { return (LogicalWrite<short?, int>.Converter) LogicalWrite.ConvertInt16; } if (typeof(TLogical) == typeof(ushort)) { return (LogicalWrite<ushort, int>.Converter) ((s, _, d, _) => LogicalWrite.ConvertUInt16(s, d)); } if (typeof(TLogical) == typeof(ushort?)) { return (LogicalWrite<ushort?, int>.Converter) LogicalWrite.ConvertUInt16; } if (typeof(TLogical) == typeof(uint)) { return LogicalWrite.GetNativeConverter<uint, int>(); } if (typeof(TLogical) == typeof(uint?)) { return LogicalWrite.GetNullableNativeConverter<uint, int>(); } if (typeof(TLogical) == typeof(ulong)) { return LogicalWrite.GetNativeConverter<ulong, long>(); } if (typeof(TLogical) == typeof(ulong?)) { return LogicalWrite.GetNullableNativeConverter<ulong, long>(); } if (typeof(TLogical) == typeof(decimal)) { if (byteBuffer == null) throw new ArgumentNullException(nameof(byteBuffer)); var multiplier = Decimal128.GetScaleMultiplier(columnDescriptor.TypeScale); return (LogicalWrite<decimal, FixedLenByteArray>.Converter) ((s, _, d, _) => LogicalWrite.ConvertDecimal128(s, d, multiplier, byteBuffer)); } if (typeof(TLogical) == typeof(decimal?)) { if (byteBuffer == null) throw new ArgumentNullException(nameof(byteBuffer)); var multiplier = Decimal128.GetScaleMultiplier(columnDescriptor.TypeScale); return (LogicalWrite<decimal?, FixedLenByteArray>.Converter) ((s, dl, d, nl) => LogicalWrite.ConvertDecimal128(s, dl, d, multiplier, nl, byteBuffer)); } if (typeof(TLogical) == typeof(Guid)) { if (byteBuffer == null) throw new ArgumentNullException(nameof(byteBuffer)); return (LogicalWrite<Guid, FixedLenByteArray>.Converter) ((s, _, d, _) => LogicalWrite.ConvertUuid(s, d, byteBuffer)); } if (typeof(TLogical) == typeof(Guid?)) { if (byteBuffer == null) throw new ArgumentNullException(nameof(byteBuffer)); return (LogicalWrite<Guid?, FixedLenByteArray>.Converter) ((s, dl, d, nl) => LogicalWrite.ConvertUuid(s, dl, d, nl, byteBuffer)); } if (typeof(TLogical) == typeof(Date)) { return LogicalWrite.GetNativeConverter<Date, int>(); } if (typeof(TLogical) == typeof(Date?)) { return LogicalWrite.GetNullableNativeConverter<Date, int>(); } var logicalType = columnDescriptor.LogicalType; if (typeof(TLogical) == typeof(DateTime)) { switch (((TimestampLogicalType) logicalType).TimeUnit) { case TimeUnit.Millis: return (LogicalWrite<DateTime, long>.Converter) ((s, _, d, _) => LogicalWrite.ConvertDateTimeMillis(s, d)); case TimeUnit.Micros: return (LogicalWrite<DateTime, long>.Converter) ((s, _, d, _) => LogicalWrite.ConvertDateTimeMicros(s, d)); } } if (typeof(TLogical) == typeof(DateTimeNanos)) { return LogicalWrite.GetNativeConverter<DateTimeNanos, long>(); } if (typeof(TLogical) == typeof(DateTime?)) { switch (((TimestampLogicalType) logicalType).TimeUnit) { case TimeUnit.Millis: return (LogicalWrite<DateTime?, long>.Converter) LogicalWrite.ConvertDateTimeMillis; case TimeUnit.Micros: return (LogicalWrite<DateTime?, long>.Converter) LogicalWrite.ConvertDateTimeMicros; } } if (typeof(TLogical) == typeof(DateTimeNanos?)) { return LogicalWrite.GetNullableNativeConverter<DateTimeNanos, long>(); } if (typeof(TLogical) == typeof(TimeSpan)) { switch (((TimeLogicalType) logicalType).TimeUnit) { case TimeUnit.Millis: return (LogicalWrite<TimeSpan, int>.Converter) ((s, _, d, _) => LogicalWrite.ConvertTimeSpanMillis(s, d)); case TimeUnit.Micros: return (LogicalWrite<TimeSpan, long>.Converter) ((s, _, d, _) => LogicalWrite.ConvertTimeSpanMicros(s, d)); } } if (typeof(TLogical) == typeof(TimeSpanNanos)) { return LogicalWrite.GetNativeConverter<TimeSpanNanos, long>(); } if (typeof(TLogical) == typeof(TimeSpan?)) { switch (((TimeLogicalType) logicalType).TimeUnit) { case TimeUnit.Millis: return (LogicalWrite<TimeSpan?, int>.Converter) LogicalWrite.ConvertTimeSpanMillis; case TimeUnit.Micros: return (LogicalWrite<TimeSpan?, long>.Converter) LogicalWrite.ConvertTimeSpanMicros; } } if (typeof(TLogical) == typeof(TimeSpanNanos?)) { return LogicalWrite.GetNullableNativeConverter<TimeSpanNanos, long>(); } if (typeof(TLogical) == typeof(string)) { if (byteBuffer == null) throw new ArgumentNullException(nameof(byteBuffer)); return (LogicalWrite<string, ByteArray>.Converter) ((s, dl, d, nl) => LogicalWrite.ConvertString(s, dl, d, nl, byteBuffer)); } if (typeof(TLogical) == typeof(byte[])) { if (byteBuffer == null) throw new ArgumentNullException(nameof(byteBuffer)); return (LogicalWrite<byte[], ByteArray>.Converter) ((s, dl, d, nl) => LogicalWrite.ConvertByteArray(s, dl, d, nl, byteBuffer)); } throw new NotSupportedException($"unsupported logical system type {typeof(TLogical)} with logical type {logicalType}"); }
// tanguyf: 2020-04-15: there is no GetDirectWriter delegate like with LogicalReadConverterFactory. // While this would nicely mirror LogicalReadConverterFactory interface, it is actually not needed in practice // since Parquet column writing is much slower than reading. Hence there is limited value for such an optimisation in this case, // the overhead of needlessly copying the memory is dwarfed by everything else. /// <summary> /// Return a converter delegate that converts a TLogical readonly-span to a TPhysical span. /// </summary> /// <returns> /// A delegate of type LogicalWrite<TLogical, TPhysical>.Converter /// </returns> /// <param name="columnDescriptor">The descriptor of the column to be converted.</param> /// <param name="byteBuffer">The ByteBuffer allocation pool for efficiently handling byte arrays.</param> public virtual Delegate GetConverter <TLogical, TPhysical>(ColumnDescriptor columnDescriptor, ByteBuffer?byteBuffer) where TPhysical : unmanaged { return(LogicalWrite <TLogical, TPhysical> .GetConverter(columnDescriptor, byteBuffer)); }