Exemplo n.º 1
0
        static void AddStructure(Dictionary <string, IList> columns, StructField field, Row structRow)
        {
            if (structRow == null)
            {
                throw new ArgumentException($"expected {typeof(Row)} for field [{field}] value");
            }

            Append(columns, field.Fields, structRow);
        }
Exemplo n.º 2
0
        private static object CreateElement(Field field, int index, Dictionary <string, IList> columns)
        {
            if (field.SchemaType == SchemaType.Map)
            {
                return(((MapField)field).CreateCellValue(columns, index));
            }
            else if (field.SchemaType == SchemaType.Struct)
            {
                return(Extract(((StructField)field).Fields, index, columns));
            }
            else if (field.SchemaType == SchemaType.List)
            {
                ListField lf = (ListField)field;

                if (lf.Item.SchemaType == SchemaType.Struct)
                {
                    StructField structField = (StructField)lf.Item;
                    Dictionary <string, IList> elementColumns = CreateFieldColumns(structField.Fields, index, columns, out int count);

                    var rows = new List <Row>(count);
                    for (int i = 0; i < count; i++)
                    {
                        Row row = Extract(structField.Fields, i, elementColumns);
                        rows.Add(row);
                    }

                    return(rows);
                }
                else if (lf.Item.SchemaType == SchemaType.Data)
                {
                    DataField dataField = (DataField)lf.Item;
                    IList     values    = GetFieldPathValues(dataField, index, columns);
                    return(values);
                }
                else
                {
                    throw OtherExtensions.NotImplementedForPotentialAssholesAndMoaners($"reading {lf.Item.SchemaType} from lists");
                }
            }
            else
            {
                if (!columns.TryGetValue(field.Path, out IList values))
                {
                    throw new ParquetException($"something terrible happened, there is no column by name '{field.Name}' and path '{field.Path}'");
                }

                return(values[index]);
            }
        }
Exemplo n.º 3
0
        /// <summary>
        /// Gets a flat list of all data fields in this schema
        /// </summary>
        /// <returns></returns>
        public DataField[] GetDataFields()
        {
            var result = new List <DataField>();

            void analyse(Field f)
            {
                switch (f.SchemaType)
                {
                case SchemaType.Data:
                    result.Add((DataField)f);
                    break;

                case SchemaType.List:
                    analyse(((ListField)f).Item);
                    break;

                case SchemaType.Map:
                    MapField mf = (MapField)f;
                    analyse(mf.Key);
                    analyse(mf.Value);
                    break;

                case SchemaType.Struct:
                    StructField sf = (StructField)f;
                    traverse(sf.Fields);
                    break;
                }
            }

            void traverse(IEnumerable <Field> fields)
            {
                foreach (Field f in fields)
                {
                    analyse(f);
                }
            }

            traverse(Fields);

            return(result.ToArray());
        }
Exemplo n.º 4
0
        static void AddList(Dictionary <string, IList> columns, ListField listField, object value)
        {
            /*
             * Value slicing can happen only when entering a list and in no other cases.
             * Only list is changing hierarchy dramatically.
             */

            switch (listField.Item.SchemaType)
            {
            case SchemaType.Struct:
                StructField       structField = (StructField)listField.Item;
                IEnumerable <Row> rows        = value as IEnumerable <Row>;

                var deepColumns = new Dictionary <string, IList>();
                foreach (Row row in rows)
                {
                    Append(deepColumns, structField.Fields, row);
                }
                SliceIn(columns, deepColumns);
                break;

            case SchemaType.Data:
                DataField        dataField = (DataField)listField.Item;
                IDataTypeHandler handler   = DataTypeFactory.Match(dataField);
                IList            values    = handler.CreateEmptyList(dataField.HasNulls, dataField.IsArray, 0);

                foreach (object v in (IEnumerable)value)
                {
                    values.Add(v);
                }
                GetValues(columns, dataField, true, true).Add(values);
                break;

            default:
                throw OtherExtensions.NotImplementedForPotentialAssholesAndMoaners($"adding {listField.Item.SchemaType} to list");
            }
        }