/// <summary>
/// 创建字典缓存
/// </summary>
/// <typeparam name="keyType"></typeparam>
/// <param name="getKey">分组字典关键字获取器</param>
/// <param name="isReset">是否初始化</param>
/// <param name="isSave">是否保存缓存对象防止被垃圾回收</param>
/// <returns></returns>
public DictionaryArray<valueType, modelType, keyType> CreateDictionaryArray<keyType>(Func<valueType, keyType> getKey, bool isReset = true, bool isSave = false)
where keyType : IEquatable<keyType>
{
DictionaryArray<valueType, modelType, keyType> cache = new DictionaryArray<valueType, modelType, keyType>(this, getKey, isReset);
if (isSave) memberCaches.Add(cache);
return cache;
}
public void Visit(DictionaryArray array)
{
// Dictionary is serialized separately in Dictionary serialization.
// We are only interested in indices at this context.
_buffers.Add(CreateBuffer(array.NullBitmapBuffer));
_buffers.Add(CreateBuffer(array.IndicesBuffer));
}
private Tuple <StringArray, Int32Array, DictionaryArray> CreateSimpleTestData()
{
StringArray originalDictionary = new StringArray.Builder().AppendRange(new[] { "a", "b", "c" }).Build();
Int32Array originalIndicesArray = new Int32Array.Builder().AppendRange(new[] { 0, 0, 1, 1, 2, 2 }).Build();
var dictionaryArray = new DictionaryArray(new DictionaryType(Int32Type.Default, StringType.Default, false), originalIndicesArray, originalDictionary);
return(Tuple.Create(originalDictionary, originalIndicesArray, dictionaryArray));
}
public void Visit(DictionaryArray array)
{
Assert.IsAssignableFrom <DictionaryArray>(_expectedArray);
DictionaryArray expectedArray = (DictionaryArray)_expectedArray;
var indicesComparer = new ArrayComparer(expectedArray.Indices, _strictCompare);
var dictionaryComparer = new ArrayComparer(expectedArray.Dictionary, _strictCompare);
array.Indices.Accept(indicesComparer);
array.Dictionary.Accept(dictionaryComparer);
}
private List <RecordBatch> CreateMultipleDictionaryArraysTestData()
{
var dictionaryData = new List <string> {
"a", "b", "c"
};
int length = dictionaryData.Count;
var schemaForSimpleCase = new Schema(new List <Field> {
new Field("int8", Int8Type.Default, true),
new Field("uint8", UInt8Type.Default, true),
new Field("int16", Int16Type.Default, true),
new Field("uint16", UInt16Type.Default, true),
new Field("int32", Int32Type.Default, true),
new Field("uint32", UInt32Type.Default, true),
new Field("int64", Int64Type.Default, true),
new Field("uint64", UInt64Type.Default, true)
}, null);
StringArray dictionary = new StringArray.Builder().AppendRange(dictionaryData).Build();
IEnumerable <IArrowArray> indicesArraysForSimpleCase = TestData.CreateArrays(schemaForSimpleCase, length);
var fields = new List <Field>(capacity: length + 1);
var testTargetArrays = new List <IArrowArray>(capacity: length + 1);
foreach (IArrowArray indices in indicesArraysForSimpleCase)
{
var dictionaryArray = new DictionaryArray(
new DictionaryType(indices.Data.DataType, StringType.Default, false),
indices, dictionary);
testTargetArrays.Add(dictionaryArray);
fields.Add(new Field($"dictionaryField_{indices.Data.DataType.Name}", dictionaryArray.Data.DataType, false));
}
(Field dictionaryTypeListArrayField, ListArray dictionaryTypeListArray) = CreateDictionaryTypeListArrayTestData(dictionary);
fields.Add(dictionaryTypeListArrayField);
testTargetArrays.Add(dictionaryTypeListArray);
(Field listTypeDictionaryArrayField, DictionaryArray listTypeDictionaryArray) = CreateListTypeDictionaryArrayTestData(dictionaryData);
fields.Add(listTypeDictionaryArrayField);
testTargetArrays.Add(listTypeDictionaryArray);
var schema = new Schema(fields, null);
return(new List <RecordBatch> {
new RecordBatch(schema, testTargetArrays, length),
new RecordBatch(schema, testTargetArrays, length),
});
}
private Tuple <Field, DictionaryArray> CreateListTypeDictionaryArrayTestData(List <string> dictionaryDataBase)
{
var listBuilder = new ListArray.Builder(StringType.Default);
var valueBuilder = listBuilder.ValueBuilder as StringArray.Builder;
foreach (string data in dictionaryDataBase)
{
listBuilder.Append();
valueBuilder.Append(data);
}
ListArray dictionary = listBuilder.Build();
Int32Array indiceArray = new Int32Array.Builder().AppendRange(Enumerable.Range(0, dictionary.Length)).Build();
var dictionaryArrayType = new DictionaryType(Int32Type.Default, dictionary.Data.DataType, false);
var dictionaryArray = new DictionaryArray(dictionaryArrayType, indiceArray, dictionary);
return(Tuple.Create(new Field($"dictionaryField_{dictionaryArray.Data.DataType.Name}", dictionaryArrayType, false), dictionaryArray));
}
private Tuple <Field, ListArray> CreateDictionaryTypeListArrayTestData(StringArray dictionary)
{
Int32Array indiceArray = new Int32Array.Builder().AppendRange(Enumerable.Range(0, dictionary.Length)).Build();
//DictionaryArray has no Builder for now, so creating ListArray directly.
var dictionaryType = new DictionaryType(Int32Type.Default, StringType.Default, false);
var dictionaryArray = new DictionaryArray(dictionaryType, indiceArray, dictionary);
var valueOffsetsBufferBuilder = new ArrowBuffer.Builder <int>();
var validityBufferBuilder = new ArrowBuffer.BitmapBuilder();
foreach (int i in Enumerable.Range(0, dictionary.Length + 1))
{
valueOffsetsBufferBuilder.Append(i);
validityBufferBuilder.Append(true);
}
var dictionaryField = new Field("dictionaryField_list", dictionaryType, false);
var listType = new ListType(dictionaryField);
var listArray = new ListArray(listType, valueOffsetsBufferBuilder.Length - 1, valueOffsetsBufferBuilder.Build(), dictionaryArray, valueOffsetsBufferBuilder.Build());
return(Tuple.Create(new Field($"listField_{listType.ValueDataType.Name}", listType, false), listArray));
}
