예제 #1
0
        /// <summary>
        /// Writes next data column to parquet stream. Note that columns must be written in the order they are declared in the
        /// file schema.
        /// </summary>
        /// <param name="column"></param>
        public void WriteColumn(DataColumn column)
        {
            if (column == null)
            {
                throw new ArgumentNullException(nameof(column));
            }

            if (RowCount == null)
            {
                if (column.Data.Length > 0 || column.Field.MaxRepetitionLevel == 0)
                {
                    RowCount = column.CalculateRowCount();
                }
            }

            Thrift.SchemaElement tse = _thschema[_colIdx];
            if (!column.Field.Equals(tse))
            {
                throw new ArgumentException($"cannot write this column, expected '{tse.Name}', passed: '{column.Field.Name}'", nameof(column));
            }
            IDataTypeHandler dataTypeHandler = DataTypeFactory.Match(tse, _formatOptions);

            _colIdx += 1;

            List <string> path = _footer.GetPath(tse);

            var writer = new DataColumnWriter(_stream, _thriftStream, _footer, tse,
                                              _compressionMethod, _compressionLevel,
                                              (int)(RowCount ?? 0));

            Thrift.ColumnChunk chunk = writer.Write(path, column, dataTypeHandler);
            _thriftRowGroup.Columns.Add(chunk);
        }
예제 #2
0
        Field GetField(PropertyInfo property)
        {
            Type pt = property.PropertyType;

            if (pt.IsNullable())
            {
                pt = pt.GetNonNullable();
            }
            if (pt.IsArray)
            {
                pt = pt.GetElementType();
            }

            IDataTypeHandler handler = DataTypeFactory.Match(pt);

            if (handler == null)
            {
                return(null);
            }

            ParquetColumnAttribute columnAttr = property.GetCustomAttribute <ParquetColumnAttribute>();

            string   name = columnAttr?.Name ?? property.Name;
            DataType type = handler.DataType;

            var r = new DataField(name,
                                  property.PropertyType //use CLR type here as DF constructor will figure out nullability and other parameters
                                  );

            r.ClrPropName = property.Name;
            return(r);
        }
        private void CreateModelSchema(string path, IList <Field> container, int childCount, ref int si, ParquetOptions formatOptions)
        {
            for (int i = 0; i < childCount && si < _fileMeta.Schema.Count; i++)
            {
                Thrift.SchemaElement tse = _fileMeta.Schema[si];
                IDataTypeHandler     dth = DataTypeFactory.Match(tse, formatOptions);

                if (dth == null)
                {
                    throw new InvalidOperationException($"cannot find data type handler to create model schema for {tse.Describe()}");
                }

                Field se = dth.CreateSchemaElement(_fileMeta.Schema, ref si, out int ownedChildCount);

                se.Path = string.Join(Schema.PathSeparator, new[] { path, se.Path ?? se.Name }.Where(p => p != null));

                if (ownedChildCount > 0)
                {
                    var childContainer = new List <Field>();
                    CreateModelSchema(se.Path, childContainer, ownedChildCount, ref si, formatOptions);
                    foreach (Field cse in childContainer)
                    {
                        se.Assign(cse);
                    }
                }


                container.Add(se);
            }
        }
예제 #4
0
        public DataHelperTest()
        {
            TestConnectrionString = ConfigurationManager.ConnectionStrings["Test"].ConnectionString;
            var dataList = new List <IDataType>
            {
                new ColumnTypeBit(),
                new ColumnTypeDate(),
                new ColumnTypeSmallDateTime(),
                new ColumnTypeDateTime(),
                new ColumnTypeDecimal(),
                new ColumnTypeInt(),
                new ColumnTypeMoney(),
                new ColumnTypeNChar(),
                new ColumnTypeChar(),
                new ColumnTypeNVarchar(),
                new ColumnTypeReal(),
                new ColumnTypeSmallint(),
                new ColumnTypeTime(),
                new ColumnTypeTinyInt(),
                new ColumnTypeUniqueidentifier(),
                new ColumnTypeVarbinary(),
                new ColumnTypeBinary(),
                new ColumnTypeImage(),
                new ColumnTypeVarchar(),
            };

            DataTypeFactory.AddDataType(dataList.ToArray());
        }
예제 #5
0
        public override void CreateThrift(Field field, Thrift.SchemaElement parent, IList <Thrift.SchemaElement> container)
        {
            ListField listField = (ListField)field;

            parent.Num_children += 1;

            //add list container
            var root = new Thrift.SchemaElement(field.Name)
            {
                Converted_type  = Thrift.ConvertedType.LIST,
                Repetition_type = Thrift.FieldRepetitionType.OPTIONAL,
                Num_children    = 1 //field container below
            };

            container.Add(root);

            //add field container
            var list = new Thrift.SchemaElement("list")
            {
                Repetition_type = Thrift.FieldRepetitionType.REPEATED
            };

            container.Add(list);

            //add the list item as well
            IDataTypeHandler fieldHandler = DataTypeFactory.Match(listField.Item);

            fieldHandler.CreateThrift(listField.Item, list, container);
        }
예제 #6
0
        private void ParseSchemaExperimenal(SchemaElement parent, int childCount, ref int si, ParquetOptions formatOptions)
        {
            for (int i = 0; i < childCount && si < _fileMeta.Schema.Count; i++)
            {
                Thrift.SchemaElement tse = _fileMeta.Schema[si];
                IDataTypeHandler     dth = DataTypeFactory.Match(tse, formatOptions);

                if (dth == null)
                {
                    if (tse.Num_children > 0)
                    {
                        //it's an element
                        ParseSchemaExperimenal(parent, _fileMeta.Schema[si++].Num_children, ref si, formatOptions);
                        continue;
                    }
                    else
                    {
                        ThrowNoHandler(tse);
                    }
                }

                SchemaElement newRoot = dth.Create(parent, _fileMeta.Schema, ref si);

                if (newRoot != null)
                {
                    ParseSchemaExperimenal(newRoot, _fileMeta.Schema[si - 1].Num_children, ref si, formatOptions);
                }
            }
        }
예제 #7
0
        public void Read(long offset, long count)
        {
            Thrift.SchemaElement tse = _footer.GetSchemaElement(_thriftColumnChunk);

            IDataTypeHandler dataTypeHandler = DataTypeFactory.Match(tse, _parquetOptions);

            long fileOffset = GetFileOffset();
            long maxValues  = _thriftColumnChunk.Meta_data.Num_values;

            _inputStream.Seek(fileOffset, SeekOrigin.Begin);

            IList      dictionary  = null;
            List <int> indexes     = null;
            List <int> repetitions = null;
            List <int> definitions = null;
            IList      values      = null;

            //there can be only one dictionary page in column
            Thrift.PageHeader ph = _thriftStream.Read <Thrift.PageHeader>();
            if (TryReadDictionaryPage(ph, dataTypeHandler, out dictionary))
            {
                ph = _thriftStream.Read <Thrift.PageHeader>();
            }

            int pagesRead = 0;

            while (true)
            {
                int      valuesSoFar = Math.Max(indexes == null ? 0 : indexes.Count, values == null ? 0 : values.Count);
                PageData pd          = ReadDataPage(dataTypeHandler, ph, tse, maxValues - valuesSoFar);

                repetitions = AssignOrAdd(repetitions, pd.repetitions);
                definitions = AssignOrAdd(definitions, pd.definitions);
                indexes     = AssignOrAdd(indexes, pd.indexes);
                values      = AssignOrAdd(values, pd.values);

                pagesRead++;

                int totalCount = Math.Max(
                    (values == null ? 0 : values.Count) +
                    (indexes == null ? 0 : indexes.Count),
                    (definitions == null ? 0 : definitions.Count));
                if (totalCount >= maxValues)
                {
                    break;                      //limit reached
                }
                ph = _thriftStream.Read <Thrift.PageHeader>();
                if (ph.Type != Thrift.PageType.DATA_PAGE)
                {
                    break;
                }
            }

            //IList mergedValues = new ValueMerger(_schema, values)
            //   .Apply(dictionary, definitions, repetitions, indexes, (int)maxValues);
        }
예제 #8
0
        protected override IEnumerable <IDataType> PerformGetByQuery(IQuery <IDataType> query)
        {
            var sqlClause  = GetBaseQuery(false);
            var translator = new SqlTranslator <IDataType>(sqlClause, query);
            var sql        = translator.Translate();

            var dtos = Database.Fetch <DataTypeDto>(sql);

            return(dtos.Select(x => DataTypeFactory.BuildEntity(x, _editors.Value, Logger)).ToArray());
        }
예제 #9
0
    protected override IEnumerable <IDataType> PerformGetByQuery(IQuery <IDataType> query)
    {
        Sql <ISqlContext> sqlClause = GetBaseQuery(false);
        var translator        = new SqlTranslator <IDataType>(sqlClause, query);
        Sql <ISqlContext> sql = translator.Translate();

        List <DataTypeDto>?dtos = Database.Fetch <DataTypeDto>(sql);

        return(dtos.Select(x => DataTypeFactory.BuildEntity(x, _editors, _dataTypeLogger, _serializer)).ToArray());
    }
예제 #10
0
    protected override void PersistNewItem(IDataType entity)
    {
        entity.AddingEntity();

        // ensure a datatype has a unique name before creating it
        entity.Name = EnsureUniqueNodeName(entity.Name) !;

        // TODO: should the below be removed?
        // Cannot add a duplicate data type
        Sql <ISqlContext> existsSql = Sql()
                                      .SelectCount()
                                      .From <DataTypeDto>()
                                      .InnerJoin <NodeDto>().On <DataTypeDto, NodeDto>((left, right) => left.NodeId == right.NodeId)
                                      .Where <NodeDto>(x => x.Text == entity.Name);
        var exists = Database.ExecuteScalar <int>(existsSql) > 0;

        if (exists)
        {
            throw new DuplicateNameException("A data type with the name " + entity.Name + " already exists");
        }

        DataTypeDto dto = DataTypeFactory.BuildDto(entity, _serializer);

        // Logic for setting Path, Level and SortOrder
        NodeDto?parent    = Database.First <NodeDto>("WHERE id = @ParentId", new { entity.ParentId });
        var     level     = parent.Level + 1;
        var     sortOrder =
            Database.ExecuteScalar <int>(
                "SELECT COUNT(*) FROM umbracoNode WHERE parentID = @ParentId AND nodeObjectType = @NodeObjectType",
                new { entity.ParentId, NodeObjectType = NodeObjectTypeId });

        // Create the (base) node data - umbracoNode
        NodeDto nodeDto = dto.NodeDto;

        nodeDto.Path      = parent.Path;
        nodeDto.Level     = short.Parse(level.ToString(CultureInfo.InvariantCulture));
        nodeDto.SortOrder = sortOrder;
        var o = Database.IsNew(nodeDto) ? Convert.ToInt32(Database.Insert(nodeDto)) : Database.Update(nodeDto);

        // Update with new correct path
        nodeDto.Path = string.Concat(parent.Path, ",", nodeDto.NodeId);
        Database.Update(nodeDto);

        // Update entity with correct values
        entity.Id        = nodeDto.NodeId; // Set Id on entity to ensure an Id is set
        entity.Path      = nodeDto.Path;
        entity.SortOrder = sortOrder;
        entity.Level     = level;

        dto.NodeId = nodeDto.NodeId;
        Database.Insert(dto);

        entity.ResetDirtyProperties();
    }
예제 #11
0
        private void CreateThriftSchema(IEnumerable <Field> ses, Thrift.SchemaElement parent, IList <Thrift.SchemaElement> container)
        {
            foreach (Field se in ses)
            {
                IDataTypeHandler handler = DataTypeFactory.Match(se);

                //todo: check that handler is found indeed

                handler.CreateThrift(se, parent, container);
            }
        }
        private IDataType GetDataType()
        {
            var dataType = string.Empty;

            TypeSwitch.Do(
                CurrentColumnInfo.Column.FieldNameType,
                TypeSwitch.Case <string>(() => dataType    = "StringDataType"),
                TypeSwitch.Case <DateTime>(() => dataType  = "DatetimeDataType"),
                TypeSwitch.Case <DateTime?>(() => dataType = "DatetimeDataType"),
                TypeSwitch.Default(() => dataType          = "DefaultType")
                );
            return(DataTypeFactory.GetInstance().CreateInstance(dataType));
        }
예제 #13
0
        public void Level4_definitions_packed_when_none_are_null()
        {
            var values = new List <int?> {
                1, 2, 1, 2
            };
            var dh = DataTypeFactory.Match(typeof(int));
            var v  = dh.InsertDefinitions(values, 4, new List <int> {
                4, 4, 4, 4
            }, out var _);

            Assert.Equal(4, v.Count);
            Assert.Equal(Nullable <int>(1, 2, 1, 2), v);
        }
예제 #14
0
        public void First_and_lastis_null_packed()
        {
            var values = new List <int?> {
                1, 2
            };
            var dh = DataTypeFactory.Match(typeof(int));
            var v  = dh.InsertDefinitions(values, 1, new List <int> {
                0, 1, 1, 0
            }, out var _);

            Assert.Equal(4, v.Count);
            Assert.Equal(Nullable <int>(null, 1, 2, null), v);
        }
예제 #15
0
        public ColumnarReader(Stream inputStream, Thrift.ColumnChunk thriftColumnChunk, ThriftFooter footer, ParquetOptions parquetOptions)
        {
            _inputStream       = inputStream ?? throw new ArgumentNullException(nameof(inputStream));
            _thriftColumnChunk = thriftColumnChunk ?? throw new ArgumentNullException(nameof(thriftColumnChunk));
            _footer            = footer ?? throw new ArgumentNullException(nameof(footer));
            _parquetOptions    = parquetOptions ?? throw new ArgumentNullException(nameof(parquetOptions));

            _thriftStream = new ThriftStream(inputStream);
            _footer.GetLevels(_thriftColumnChunk, out int mrl, out int mdl);
            _maxRepetitionLevel  = mrl;
            _maxDefinitionLevel  = mdl;
            _thriftSchemaElement = _footer.GetSchemaElement(_thriftColumnChunk);
            _dataTypeHandler     = DataTypeFactory.Match(_thriftSchemaElement, _parquetOptions);
        }
예제 #16
0
        private Field GetField(PropertyInfo property)
        {
            Type pt = property.PropertyType;

            if (pt.IsNullable())
            {
                pt = pt.GetNonNullable();
            }
            if (pt.IsArray)
            {
                pt = pt.GetElementType();
            }

            IDataTypeHandler handler = DataTypeFactory.Match(pt);

            if (handler == null)
            {
                return(null);
            }

            ParquetColumnAttribute columnAttr = property.GetCustomAttribute <ParquetColumnAttribute>();

            string   name = columnAttr?.Name ?? property.Name;
            DataType type = handler.DataType;

            var r = new DataField(name,
                                  property.PropertyType //use CLR type here as DF constructor will figure out nullability and other parameters
                                  );

            if (columnAttr != null)
            {
                if (handler.ClrType == typeof(TimeSpan))
                {
                    r = new TimeSpanDataField(r.Name, columnAttr.TimeSpanFormat, r.HasNulls, r.IsArray);
                }
                if (handler.ClrType == typeof(DateTime) || handler.ClrType == typeof(DateTimeOffset))
                {
                    r = new DateTimeDataField(r.Name, columnAttr.DateTimeFormat, r.HasNulls, r.IsArray);
                }
                if (handler.ClrType == typeof(decimal))
                {
                    r = new DecimalDataField(r.Name, columnAttr.DecimalPrecision, columnAttr.DecimalScale, columnAttr.DecimalForceByteArrayEncoding, r.HasNulls, r.IsArray);
                }
            }

            r.ClrPropName = property.Name;

            return(r);
        }
예제 #17
0
        private void ParseParameterInfo(string paramsTypeStr)
        {
            parameterTypes = new List <IDataType>();

            string[] rawValues = paramsTypeStr.Split(';');
            foreach (string childTypeField in rawValues)
            {
                var correctedType = Helper.CorrectHeadItemString(childTypeField);
                if (correctedType == "")
                {
                    continue;
                }
                var childDataType = DataTypeFactory.GetBaseDataType(correctedType);
                parameterTypes.Add(childDataType);
            }
        }
예제 #18
0
        public void SetUp()
        {
            var dataTypeFactory = new DataTypeFactory();
            var seedAssembly    = GetType().Assembly;

            var typeFinderConduit = new TypeFinderConduit();

            var         typesCache                       = new CachedTypesFinder(dataTypeFactory);
            ITypeFinder genericTypeFindeer               = new GenericTypeFinder(typesCache, typeFinderConduit, dataTypeFactory);
            ITypeFinder typeInAssemblyFinder             = new TypeInAssemblyFinder(seedAssembly, typesCache, dataTypeFactory);
            ITypeFinder typeInReferencedAssembliesFinder = new TypeInReferencedAssemblyFinder(seedAssembly, typesCache, typeFinder, dataTypeFactory);

            typeFinder = new TypeFinder(typesCache, genericTypeFindeer, typeInAssemblyFinder,
                                        typeInReferencedAssembliesFinder);
            typeFinderConduit.SetTarget(typeFinder);
        }
예제 #19
0
        protected override IEnumerable <IDataType> PerformGetAll(params int[] ids)
        {
            var dataTypeSql = GetBaseQuery(false);

            if (ids.Any())
            {
                dataTypeSql.Where("umbracoNode.id in (@ids)", new { ids });
            }
            else
            {
                dataTypeSql.Where <NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
            }

            var dtos = Database.Fetch <DataTypeDto>(dataTypeSql);

            return(dtos.Select(x => DataTypeFactory.BuildEntity(x, _editors.Value, Logger)).ToArray());
        }
        public void CreateThrift(Field field, Thrift.SchemaElement parent, IList <Thrift.SchemaElement> container)
        {
            StructField structField = (StructField)field;

            Thrift.SchemaElement tseStruct = new Thrift.SchemaElement(field.Name)
            {
                Repetition_type = Thrift.FieldRepetitionType.OPTIONAL,
            };
            container.Add(tseStruct);
            parent.Num_children += 1;

            foreach (Field cf in structField.Fields)
            {
                IDataTypeHandler handler = DataTypeFactory.Match(cf);
                handler.CreateThrift(cf, tseStruct, container);
            }
        }
        public void Write(DataColumn column)
        {
            if (column == null)
            {
                throw new ArgumentNullException(nameof(column));
            }

            Thrift.SchemaElement tse             = _thschema[_colIdx++];
            IDataTypeHandler     dataTypeHandler = DataTypeFactory.Match(tse, _formatOptions);
            //todo: check if the column is in the right order


            List <string> path = _footer.GetPath(tse);

            Thrift.ColumnChunk chunk = WriteColumnChunk(tse, path, column, dataTypeHandler);
            _thriftRowGroup.Columns.Add(chunk);
        }
예제 #22
0
    protected override IEnumerable <IDataType> PerformGetAll(params int[]?ids)
    {
        Sql <ISqlContext> dataTypeSql = GetBaseQuery(false);

        if (ids?.Any() ?? false)
        {
            dataTypeSql.Where("umbracoNode.id in (@ids)", new { ids });
        }
        else
        {
            dataTypeSql.Where <NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
        }

        List <DataTypeDto>?dtos = Database.Fetch <DataTypeDto>(dataTypeSql);

        return(dtos.Select(x => DataTypeFactory.BuildEntity(x, _editors, _dataTypeLogger, _serializer)).ToArray());
    }
예제 #23
0
        static async Task Main(string[] args)
        {
            string fileName = @"C:\Users\thadh\Documents\GitHub\thadhouse\NiFpgaGen\roboRIO_FPGA_2020_20.1.2.lvbitx";

            using var file = new FileStream(fileName, FileMode.Open, FileAccess.Read);

            XElement bitfile = await XElement.LoadAsync(file, LoadOptions.None, CancellationToken.None);

            //var registerList = bitfile.Descendants("VI").Descendants("RegisterList").Descendants("Register").Select(x => new Register(x));

            var registerList2 = bitfile.Element("VI").Element("RegisterList").Elements("Register");

            DataTypeFactory factory = DataTypeFactory.Instance;

            List <Register> registers = new List <Register>();

            foreach (var register in registerList2)
            {
                registers.Add(new Register(register));
            }

            var frcRegisters = FRCClass.GetDefaultClassList();

            foreach (var register in registers)
            {
                FRCClass.AddRegisterToClassList(frcRegisters, register);
            }

            FRCClass.ValidateClasses(frcRegisters);

            var tempalteMapper = new CLanguageTemplates();

            foreach (var cls in frcRegisters)
            {
            }



            //var frcMapping = new FRCMapping(registerList);

            //frcMapping.GenerateC(@"C:\Users\thadh\Documents\GitHub\thadhouse\NiFpgaGen\Generated", "FRC");

            ;
        }
예제 #24
0
    protected override void PersistUpdatedItem(IDataType entity)
    {
        entity.Name = EnsureUniqueNodeName(entity.Name, entity.Id) !;

        // Cannot change to a duplicate alias
        Sql <ISqlContext> existsSql = Sql()
                                      .SelectCount()
                                      .From <DataTypeDto>()
                                      .InnerJoin <NodeDto>().On <DataTypeDto, NodeDto>((left, right) => left.NodeId == right.NodeId)
                                      .Where <NodeDto>(x => x.Text == entity.Name && x.NodeId != entity.Id);
        var exists = Database.ExecuteScalar <int>(existsSql) > 0;

        if (exists)
        {
            throw new DuplicateNameException("A data type with the name " + entity.Name + " already exists");
        }

        // Updates Modified date
        entity.UpdatingEntity();

        // Look up parent to get and set the correct Path if ParentId has changed
        if (entity.IsPropertyDirty("ParentId"))
        {
            NodeDto?parent = Database.First <NodeDto>("WHERE id = @ParentId", new { entity.ParentId });
            entity.Path  = string.Concat(parent.Path, ",", entity.Id);
            entity.Level = parent.Level + 1;
            var maxSortOrder =
                Database.ExecuteScalar <int>(
                    "SELECT coalesce(max(sortOrder),0) FROM umbracoNode WHERE parentid = @ParentId AND nodeObjectType = @NodeObjectType",
                    new { entity.ParentId, NodeObjectType = NodeObjectTypeId });
            entity.SortOrder = maxSortOrder + 1;
        }

        DataTypeDto dto = DataTypeFactory.BuildDto(entity, _serializer);

        // Updates the (base) node data - umbracoNode
        NodeDto nodeDto = dto.NodeDto;

        Database.Update(nodeDto);
        Database.Update(dto);

        entity.ResetDirtyProperties();
    }
예제 #25
0
        public override void CreateThrift(Field field, Thrift.SchemaElement parent, IList <Thrift.SchemaElement> container)
        {
            parent.Num_children += 1;

            //add the root container where map begins
            var root = new Thrift.SchemaElement(field.Name)
            {
                Converted_type  = Thrift.ConvertedType.MAP,
                Num_children    = 1,
                Repetition_type = Thrift.FieldRepetitionType.OPTIONAL
            };

            container.Add(root);

            //key-value is a container for column of keys and column of values
            var keyValue = new Thrift.SchemaElement(MapField.ContainerName)
            {
                Num_children    = 0, //is assigned by children
                Repetition_type = Thrift.FieldRepetitionType.REPEATED
            };

            container.Add(keyValue);

            //now add the key and value separately
            MapField         mapField     = field as MapField;
            IDataTypeHandler keyHandler   = DataTypeFactory.Match(mapField.Key);
            IDataTypeHandler valueHandler = DataTypeFactory.Match(mapField.Value);

            keyHandler.CreateThrift(mapField.Key, keyValue, container);
            Thrift.SchemaElement tseKey = container[container.Count - 1];
            valueHandler.CreateThrift(mapField.Value, keyValue, container);
            Thrift.SchemaElement tseValue = container[container.Count - 1];

            //fixups for weirdness in RLs
            if (tseKey.Repetition_type == Thrift.FieldRepetitionType.REPEATED)
            {
                tseKey.Repetition_type = Thrift.FieldRepetitionType.OPTIONAL;
            }
            if (tseValue.Repetition_type == Thrift.FieldRepetitionType.REPEATED)
            {
                tseValue.Repetition_type = Thrift.FieldRepetitionType.OPTIONAL;
            }
        }
        public ColumnarWriter(Stream output, ThriftStream thriftStream,
                              ThriftFooter footer,
                              Thrift.SchemaElement tse, List <string> path,
                              CompressionMethod compressionMethod,
                              ParquetOptions formatOptions,
                              WriterOptions writerOptions)
        {
            _output            = output;
            _thriftStream      = thriftStream;
            _footer            = footer;
            _tse               = tse;
            _compressionMethod = compressionMethod;
            _formatOptions     = formatOptions;
            _writerOptions     = writerOptions;
            _dataTypeHandler   = DataTypeFactory.Match(tse, _formatOptions);

            _chunk = _footer.CreateColumnChunk(_compressionMethod, _output, _tse.Type, path, 0);
            _ph    = _footer.CreateDataPage(0);
            _footer.GetLevels(_chunk, out int maxRepetitionLevel, out int maxDefinitionLevel);
            _maxRepetitionLevel = maxRepetitionLevel;
            _maxDefinitionLevel = maxDefinitionLevel;
        }
        private DataType CreateDataType(string name, string dataType)
        {
            var dataTypeEnum = new DataTypeFactory().NewWithDefaults(name);

            dataTypeEnum.TypeName             = "Colour";
            dataTypeEnum.Nullable             = false;
            dataTypeEnum.ValidationExpression = string.Empty;
            dataTypeEnum.BuiltIn = false;
            dataTypeEnum.Guid    = Guid.NewGuid();
            dataTypeEnum.Properties.Add(new EntityProperty
            {
                Guid            = Guid.NewGuid(),
                Version         = 1,
                Value           = dataType,
                EntityId        = dataTypeEnum.Guid,
                DefinitionField = dataTypeEnum.Definition.DefinitionFields.Single(f => f.Name == "Editor"),
            });

            dataTypeEnum.Save();

            return(dataTypeEnum);
        }
예제 #28
0
        /// <summary>
        /// Decodes raw bytes from <see cref="Thrift.Statistics"/> into a CLR value
        /// </summary>
        public static object DecodeSingleStatsValue(this Thrift.FileMetaData fileMeta, Thrift.ColumnChunk columnChunk, byte[] rawBytes)
        {
            if (rawBytes == null || rawBytes.Length == 0)
            {
                return(null);
            }

            var footer = new ThriftFooter(fileMeta);

            Thrift.SchemaElement schema = footer.GetSchemaElement(columnChunk);

            IDataTypeHandler handler = DataTypeFactory.Match(schema, new ParquetOptions {
                TreatByteArrayAsString = true
            });

            using (var ms = new MemoryStream(rawBytes))
                using (var reader = new BinaryReader(ms))
                {
                    object value = handler.Read(reader, schema, rawBytes.Length);
                    return(value);
                }
        }
예제 #29
0
 private BareANY WrapWithHl7DataType(string hl7DataType, BareDiff diff)
 {
     try
     {
         BareANY hl7Value = (BareANY)DataTypeFactory.CreateDataType(hl7DataType, false);
         if (diff != null)
         {
             if (diff.BareValue != null)
             {
                 ((BareANYImpl)hl7Value).BareValue = diff.BareValue;
             }
             else
             {
                 hl7Value.NullFlavor = diff.NullFlavor;
             }
         }
         return(hl7Value);
     }
     catch (Exception e)
     {
         throw new ModelToXmlTransformationException("Unable to instantiate HL7 data type: " + hl7DataType, e);
     }
 }
예제 #30
0
        private void Compare(Schema schema, Row pc, Row cc)
        {
            for (int i = 0; i < pc.Length; i++)
            {
                //todo: this comparison needs to be improved, probably doesn't handle nulls etc.

                Field se      = schema.Fields[i];
                Type  clrType = DataTypeFactory.Match(se).ClrType;

                object pv = pc[i];
                object cv = ChangeType(cc[i], clrType);

                if (pv == null)
                {
                    bool isCsvNull =
                        cv == null ||
                        (cv is string s && s == string.Empty);

                    Assert.True(isCsvNull,
                                $"expected null value in column {se.Name}, value #{i}");
                }
                else
                {
                    if (clrType == typeof(string))
                    {
                        Assert.True(((string)pv).Trim() == ((string)cv).Trim(),
                                    $"expected {cv} but was {pv} in column {se.Name}, value #{i}");
                    }
                    else
                    {
                        Assert.True(pv.Equals(cv),
                                    $"expected {cv} but was {pv} in column {se.Name}, value #{i}");
                    }
                }
            }
        }