public List <int> getPartitionKeys(IVector partitionCol)
{
if (partitionCol.getDataCategory() != cat)
{
throw new Exception("Data category incompatible.");
}
if (cat == DATA_CATEGORY.TEMPORAL && type != partitionCol.getDataType())
{
DATA_TYPE old = partitionCol.getDataType();
partitionCol = (IVector)Utils.castDateTime(partitionCol, type);
if (partitionCol == null)
{
throw new Exception("Can't convert type from " + old.ToString() + " to " + type.ToString());
}
}
if (type == DATA_TYPE.DT_LONG)
{
throw new Exception("Long type value can't be used as a partition column.");
}
int rows = partitionCol.rows();
List <int> keys = new List <int>(rows);
for (int i = 0; i < rows; ++i)
{
keys.Add(partitionCol.hashBucket(i, 1048576));
}
return(keys);
}
public List <int> getPartitionKeys(IVector partitionCol)
{
if (partitionCol.getDataCategory() != cat)
{
throw new Exception("Data category incompatible.");
}
if (cat == DATA_CATEGORY.TEMPORAL && type != partitionCol.getDataType())
{
DATA_TYPE old = partitionCol.getDataType();
partitionCol = (IVector)Utils.castDateTime(partitionCol, type);
if (partitionCol == null)
{
throw new Exception("Can't convert type from " + old.ToString() + " to " + type.ToString());
}
}
int partitions = range.rows() - 1;
int rows = partitionCol.rows();
List <int> keys = new List <int>(rows);
for (int i = 0; i < rows; ++i)
{
int index = range.asof(partitionCol.get(i));
if (index >= partitions)
{
keys.Add(-1);
}
else
{
keys.Add(index);
}
}
return(keys);
}
public int getPartitionKey(IScalar partitionCol)
{
if (partitionCol.getDataCategory() != cat)
{
throw new Exception("Data category incompatible.");
}
if (cat == DATA_CATEGORY.TEMPORAL && type != partitionCol.getDataType())
{
DATA_TYPE old = partitionCol.getDataType();
partitionCol = (IScalar)Utils.castDateTime(partitionCol, type);
if (partitionCol == null)
{
throw new Exception("Can't convert type from " + old.ToString() + " to " + type.ToString());
}
}
int index = 0;
if (dict.ContainsKey(partitionCol))
{
index = (int)dict[partitionCol];
}
else
{
index = -1;
}
return(index);
}
public BasicSet(DATA_TYPE keyType, int capacity)
{
if (keyType == DATA_TYPE.DT_VOID || keyType == DATA_TYPE.DT_SYMBOL || (int)keyType >= (int)DATA_TYPE.DT_FUNCTIONDEF)
{
throw new System.ArgumentException("Invalid keyType: " + keyType.ToString());
}
this.keyType = keyType;
set = new HashSet <IScalar>();
}
public BasicDictionary(DATA_TYPE keyType, DATA_TYPE valueType, int capacity)
{
if (keyType == DATA_TYPE.DT_VOID || keyType == DATA_TYPE.DT_ANY || keyType == DATA_TYPE.DT_DICTIONARY)
{
throw new System.ArgumentException("Invalid keyType: " + keyType.ToString());
}
this.keyType = keyType;
this.valueType = valueType;
dict = new Dictionary <IScalar, IEntity>();
}
public int getPartitionKey(IScalar partitionCol)
{
if (partitionCol.getDataCategory() != cat)
{
throw new Exception("Data category incompatible.");
}
if (cat == DATA_CATEGORY.TEMPORAL && type != partitionCol.getDataType())
{
//类型转化
DATA_TYPE old = partitionCol.getDataType();
partitionCol = (IScalar)Utils.castDateTime(partitionCol, type);
if (partitionCol == null)
{
throw new Exception("Can't convert type from " + old.ToString() + " to " + type.ToString());
}
}
int rows = partitionCol.rows();
int key = partitionCol.hashBucket(buckets);
return(key);
}
public int getPartitionKey(IScalar partitionCol)
{
if (partitionCol.getDataCategory() != cat)
{
throw new Exception("Data category incompatible.");
}
if (cat == DATA_CATEGORY.TEMPORAL && type != partitionCol.getDataType())
{
DATA_TYPE old = partitionCol.getDataType();
partitionCol = (IScalar)Utils.castDateTime(partitionCol, type);
if (partitionCol == null)
{
throw new Exception("Can't convert type from " + old.ToString() + " to " + type.ToString());
}
}
if (type == DATA_TYPE.DT_LONG)
{
throw new Exception("Long type value can't be used as a partition column.");
}
int key = partitionCol.hashBucket(1048576);
return(key);
}
public int getPartitionKey(IScalar partitionCol)
{
if (partitionCol.getDataCategory() != cat)
{
throw new Exception("Data category incompatible.");
}
if (cat == DATA_CATEGORY.TEMPORAL && type != partitionCol.getDataType())
{
DATA_TYPE old = partitionCol.getDataType();
partitionCol = (IScalar)Utils.castDateTime(partitionCol, type);
if (partitionCol == null)
{
throw new Exception("Can't convert type from " + old.ToString() + " to " + type.ToString());
}
}
int partitions = range.rows() - 1;
int key = range.asof(partitionCol);
if (key >= partitions)
{
key = -1;
}
return(key);
}
public virtual void write(ExtendedDataOutput @out)
{
if (valueType == DATA_TYPE.DT_DICTIONARY)
{
throw new IOException("Can't streamlize the dictionary with value type " + valueType.ToString());
}
BasicEntityFactory factory = new BasicEntityFactory();
IVector keys = (IVector)factory.createScalarWithDefaultValue(keyType);
IVector values = (IVector)factory.createVectorWithDefaultValue(valueType, dict.Count);
int index = 0;
try
{
IEnumerator <KeyValuePair <IScalar, IEntity> > itr = dict.GetEnumerator();
while (itr.MoveNext())
{
keys.set(index, itr.Current.Key);
values.set(index, (IScalar)itr.Current.Value);
++index;
}
}
catch (Exception ex)
{
throw new IOException(ex.Message);
}
int flag = ((int)DATA_FORM.DF_DICTIONARY << 8) + (int)getDataType();
@out.writeShort(flag);
keys.write(@out);
values.write(@out);
}
public IEntity createEntity(DATA_FORM form, DATA_TYPE type, ExtendedDataInput @in)
{
if (form == DATA_FORM.DF_TABLE)
{
return(new BasicTable(@in));
}
else if (form == DATA_FORM.DF_CHART)
{
//return new BasicChart(@in);
return(null);
}
else if (form == DATA_FORM.DF_DICTIONARY)
{
//return new BasicDictionary(type, @in);
return(null);
}
else if (form == DATA_FORM.DF_SET)
{
//return new BasicSet(type, @in);
return(null);
}
else if (form == DATA_FORM.DF_CHUNK)
{
//return new BasicChunkMeta(@in);
return(null);
}
else if (type == DATA_TYPE.DT_ANY && form == DATA_FORM.DF_VECTOR)
{
//return new BasicAnyVector(@in);
return(null);
}
else if (type == DATA_TYPE.DT_VOID && form == DATA_FORM.DF_SCALAR)
{
@in.readBoolean();
return(new Void());
}
else
{
int index = (int)type;
if (factories[index] == null)
{
throw new IOException("Data type " + type.ToString() + " is not supported yet.");
}
else if (form == DATA_FORM.DF_VECTOR)
{
return(factories[index].createVector(@in));
}
else if (form == DATA_FORM.DF_SCALAR)
{
return(factories[index].createScalar(@in));
}
else if (form == DATA_FORM.DF_MATRIX)
{
return(factories[index].createMatrix(@in));
}
else if (form == DATA_FORM.DF_PAIR)
{
return(factories[index].createPair(@in));
}
else
{
throw new IOException("Data form " + form.ToString() + " is not supported yet.");
}
}
}
public override void InitSlots()
{
base.InitSlots();
for (ushort i = 0; i < Capacity; i++)
{
ComputeBuffer buffer;
switch (DataType)
{
case DATA_TYPE.FLOAT:
buffer = new ComputeBuffer(TileSize, sizeof(float) * Channels, ComputeBufferType);
break;
case DATA_TYPE.INT:
buffer = new ComputeBuffer(TileSize, sizeof(int) * Channels, ComputeBufferType);
break;
case DATA_TYPE.BYTE:
buffer = new ComputeBuffer(TileSize, sizeof(byte) * Channels, ComputeBufferType);
break;
default:
{
buffer = new ComputeBuffer(TileSize, sizeof(float) * Channels, ComputeBufferType);
Debug.LogWarning(string.Format("TileStorage: {0} data type isn't supported by {1}! Float type will be used!", DataType.ToString(), GetType().Name));
break;
}
}
var slot = new CBSlot(this, buffer);
AddSlot(i, slot);
}
}
private void checkColumnType(int col, DATA_CATEGORY category, DATA_TYPE type)
{
DATA_CATEGORY expectCategory = this.columnCategories[col];
DATA_TYPE expectType = this.columnTypes[col];
if (category != expectCategory)
{
throw new Exception("column " + col + ", expect category " + expectCategory.ToString() + ", got category " + category.ToString());
}
else if (category == DATA_CATEGORY.TEMPORAL && type != expectType)
{
throw new Exception("column " + col + ", temporal column must have exactly the same type, expect " + expectType.ToString() + ", got " + type.ToString());
}
}
public override void InitSlots()
{
base.InitSlots();
// NOTE : Size is sqaured as the array is 2D (but stored as a 1D array)
var size = TileSize * TileSize * Channels;
for (ushort i = 0; i < Capacity; i++)
{
switch ((int)DataType)
{
case (int)DATA_TYPE.FLOAT:
AddSlot(i, new CPUSlot <float>(this, size));
break;
case (int)DATA_TYPE.INT:
AddSlot(i, new CPUSlot <int>(this, size));
break;
case (int)DATA_TYPE.SHORT:
AddSlot(i, new CPUSlot <short>(this, size));
break;
case (int)DATA_TYPE.BYTE:
AddSlot(i, new CPUSlot <byte>(this, size));
break;
default:
{
AddSlot(i, new CPUSlot <float>(this, size));
Debug.LogWarning(string.Format("TileStorage: {0} data type isn't supported by {1}! Float type will be used!", DataType.ToString(), GetType().Name));
break;
}
}
}
}
