public BatchMutation ToBatchMutation <T>(T obj) where T : class, IHbaseTable
{
var result = new BatchMutation
{
Row = obj.RowKey.ToBytes(),
Mutations = ToMutations(obj)
};
return(result);
}
public void AddColumn()
{
var bm = new BatchMutation();
var column = new Column("c_name", "c_val");
bm.AddColumn("key1", cfs, column);
Assert.Equal(1, bm["key1"].Count);
var column2 = new Column("c_name2", "c_val2");
bm.AddColumn("key1", cfs, column2);
Assert.Equal(2, bm["key1"]["Standard1"].Count);
}
public void AddColumn()
{
var bm = new BatchMutation();
var column = new Column("c_name", "c_val");
bm.AddColumn("key1", cfs, column);
Assert.Equal(1, bm["key1"].Count);
var column2 = new Column("c_name2", "c_val2");
bm.AddColumn("key1", cfs, column2);
Assert.Equal(2, bm["key1"]["Standard1"].Count);
}
public void AddDeletion()
{
var bm = new BatchMutation();
var d1 = new Deletion();
d1.SlicePredicate = new SlicePredicate();
d1.SlicePredicate.ColumnNames.Add("c_name");
bm.AddDeletion("key1", cfs, d1);
Assert.Equal(1, bm["key1"].Count);
var d2 = new Deletion();
d2.SlicePredicate = new SlicePredicate();
d2.SlicePredicate.ColumnNames.Add("c_name2");
bm.AddDeletion("key1", cfs, d2);
Assert.Equal(2, bm["key1"]["Standard1"].Count);
}
public void AddDeletion()
{
var bm = new BatchMutation();
var d1 = new Deletion();
d1.SlicePredicate = new SlicePredicate();
d1.SlicePredicate.ColumnNames.Add("c_name");
bm.AddDeletion("key1", cfs, d1);
Assert.Equal(1, bm["key1"].Count);
var d2 = new Deletion();
d2.SlicePredicate = new SlicePredicate();
d2.SlicePredicate.ColumnNames.Add("c_name2");
bm.AddDeletion("key1", cfs, d2);
Assert.Equal(2, bm["key1"]["Standard1"].Count);
}
public void AddSuperColumn()
{
var bm = new BatchMutation();
var sc = new SuperColumn("sc_name", new List<Column>
{
new Column("c_name1", "c_val1"),
new Column("c_name2", "c_val2"),
});
bm.AddSuperColumn("key1", cfs, sc);
Assert.Equal(1, bm["key1"].Count);
var sc2 = new SuperColumn("sc_name2", new List<Column>
{
new Column("c_name1", "c_val1")
});
bm.AddSuperColumn("key1", cfs, sc2);
Assert.Equal(2, bm["key1"]["Standard1"].Count);
}
public void AddSuperColumn()
{
var bm = new BatchMutation();
var sc = new SuperColumn("sc_name", new List <Column>
{
new Column("c_name1", "c_val1"),
new Column("c_name2", "c_val2"),
});
bm.AddSuperColumn("key1", cfs, sc);
Assert.Equal(1, bm["key1"].Count);
var sc2 = new SuperColumn("sc_name2", new List <Column>
{
new Column("c_name1", "c_val1")
});
bm.AddSuperColumn("key1", cfs, sc2);
Assert.Equal(2, bm["key1"]["Standard1"].Count);
}
static void InsertHugeDataTest(int rowNum, int bytesNum, int batchSize, string startKey, string endKey, int testCount)
{
//new MemoryFailPoint(10);
IHBaseClient client = new HBaseClient("zk=10.200.200.56:2181,10.200.200.57:2181,10.200.200.58:2181;zkTimeout=00:05:00;memSegSize=1048576;memSegMultiples=1000");
string tableName = string.Format("mediad_test_thrift_table_test_{0}", DateTime.Now.Second);
string hugeData = new string('a', bytesNum);
Stopwatch sw = new Stopwatch();
Stopwatch sw2 = new Stopwatch();
Console.WriteLine(((rowNum * Encoding.UTF8.GetBytes(hugeData).Length).ToString()));
try
{
IHTable table = client.CreateTable(tableName, "cf");
BatchMutation[] rows = new BatchMutation[rowNum];
for (int i = 1; i <= rowNum; i++)
{
rows[i - 1] = new BatchMutation
{
RowKey = TypeConversionHelper.StringToByteArray(i.ToString()),
Mutations = new[]
{
new Mutation {
ColumnName = "cf:col1", Value = TypeConversionHelper.StringToByteArray("value" + i)
},
new Mutation {
ColumnName = "cf:col2", Value = TypeConversionHelper.StringToByteArray(hugeData + i)
}
}
};
}
BatchMutation[] exceptionBatchMutations;
List <Thread> threads = new List <Thread>();
for (int i = 0; i < testCount; i++)
{
threads.Add(new Thread(() =>
{
Stopwatch swthread = new Stopwatch();
swthread.Restart();
Console.WriteLine(table.BatchInsert(out exceptionBatchMutations, rows));
swthread.Stop();
Console.WriteLine((swthread.ElapsedTicks / (decimal)Stopwatch.Frequency));
}));
}
foreach (Thread thread in threads)
{
thread.Start();
}
foreach (Thread thread in threads)
{
thread.Join();
}
Console.WriteLine("=End=");
Console.WriteLine("GetRows batchSize:{1} 1~{0}:", rowNum, batchSize);
Scanner scanner = table.NewScanner(new byte[] { 0 }, new byte[] { 0xff, 0xff, 0xff }, new List <string> {
"cf:col1", "cf:col2"
});
Dictionary <string, string> tmpDictionary = new Dictionary <string, string>();
sw.Restart();
RowInfo info;
while ((info = scanner.GetNext()) != null)
{
//Console.WriteLine("read row {0}", info.RowKey);
foreach (KeyValuePair <string, Cell> pair in info.Columns)
{
}
}
Console.WriteLine(sw.ElapsedTicks / (decimal)Stopwatch.Frequency);
if (rowNum != tmpDictionary.Keys.Count)
{
Console.WriteLine("rowNum != tmpDictionary.Keys.Count");
}
for (int i = 1; i <= rowNum; i++)
{
if (!(hugeData + i).Equals(tmpDictionary[i.ToString()]))
{
Console.WriteLine("content not eqal!");
return;
}
}
}
finally
{
client.DeleteTable(tableName);
}
}
public void Read (TProtocol iprot)
{
TField field;
iprot.ReadStructBegin();
while (true)
{
field = iprot.ReadFieldBegin();
if (field.Type == TType.Stop) {
break;
}
switch (field.ID)
{
case 1:
if (field.Type == TType.String) {
TableName = iprot.ReadBinary();
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 2:
if (field.Type == TType.List) {
{
RowBatches = new List<BatchMutation>();
TList _list200 = iprot.ReadListBegin();
for( int _i201 = 0; _i201 < _list200.Count; ++_i201)
{
BatchMutation _elem202 = new BatchMutation();
_elem202 = new BatchMutation();
_elem202.Read(iprot);
RowBatches.Add(_elem202);
}
iprot.ReadListEnd();
}
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 3:
if (field.Type == TType.I64) {
Timestamp = iprot.ReadI64();
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 4:
if (field.Type == TType.Map) {
{
Attributes = new Dictionary<byte[], byte[]>();
TMap _map203 = iprot.ReadMapBegin();
for( int _i204 = 0; _i204 < _map203.Count; ++_i204)
{
byte[] _key205;
byte[] _val206;
_key205 = iprot.ReadBinary();
_val206 = iprot.ReadBinary();
Attributes[_key205] = _val206;
}
iprot.ReadMapEnd();
}
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
default:
TProtocolUtil.Skip(iprot, field.Type);
break;
}
iprot.ReadFieldEnd();
}
iprot.ReadStructEnd();
}
public void Read (TProtocol iprot)
{
TField field;
iprot.ReadStructBegin();
while (true)
{
field = iprot.ReadFieldBegin();
if (field.Type == TType.Stop) {
break;
}
switch (field.ID)
{
case 1:
if (field.Type == TType.String) {
TableName = iprot.ReadBinary();
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 2:
if (field.Type == TType.List) {
{
RowBatches = new List<BatchMutation>();
TList _list191 = iprot.ReadListBegin();
for( int _i192 = 0; _i192 < _list191.Count; ++_i192)
{
BatchMutation _elem193 = new BatchMutation();
_elem193 = new BatchMutation();
_elem193.Read(iprot);
RowBatches.Add(_elem193);
}
iprot.ReadListEnd();
}
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 3:
if (field.Type == TType.Map) {
{
Attributes = new Dictionary<byte[], byte[]>();
TMap _map194 = iprot.ReadMapBegin();
for( int _i195 = 0; _i195 < _map194.Count; ++_i195)
{
byte[] _key196;
byte[] _val197;
_key196 = iprot.ReadBinary();
_val197 = iprot.ReadBinary();
Attributes[_key196] = _val197;
}
iprot.ReadMapEnd();
}
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
default:
TProtocolUtil.Skip(iprot, field.Type);
break;
}
iprot.ReadFieldEnd();
}
iprot.ReadStructEnd();
}
/// <summary>
/// 一次加入多行数据
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
protected void 一次加入多行数据_Click(object sender, EventArgs e)
{
//请在步骤2输入表名
if (string.IsNullOrEmpty(txt表名.Text))
{
lab添加数据状态.Text = "请在步骤2输入表名";
return;
}
string table = txt表名.Text;
byte[] tableName = table.ToUTF8Bytes();
TTransport transport = null;
try
{
//192.168.2.111:60010
//实例化Socket连接
transport = new TSocket("192.168.2.111", 9090);
//实例化一个协议对象
TProtocol tProtocol = new TBinaryProtocol(transport);
//实例化一个Hbase的Client对象
var client = new Hbase.Client(tProtocol);
//打开连接
transport.Open();
//获取所有列
#region 具体功能代码
//批量提交
var bat = new List <BatchMutation>();
//value：列数据集合 目前是单列
#region 生成批量提交的数据集合
var mu = new Mutation
{
/////////////////////////////////////////////////////////注意这里的列名表里必须有.
Column = System.Text.Encoding.UTF8.GetBytes("Data:Collections"),
Value = Guid.NewGuid().ToByteArray()
};
string row;
row = string.Format("{0:d15}{1}{2}", 99999, "ABCDEF", DateTime.Now.ToString("yyyyMMddHHmmssfff"));
Dictionary <byte[], byte[]> encodedAttributes = new Dictionary <byte[], byte[]>();
var batch = new BatchMutation
{
Row = System.Text.Encoding.UTF8.GetBytes(row),
Mutations = new List <Mutation>()
{
mu
}
};
#endregion
//添加到集合中
bat.Add(batch);
client.mutateRow(tableName, bat[0].Row, bat[0].Mutations, encodedAttributes);
client.mutateRows(tableName, bat, encodedAttributes);
#endregion
}
finally
{
if (transport != null)
{
transport.Close();
}
}
}
public void BatchMutate(BatchMutation batchMutation)
{
throw new NotImplementedException();
}
public void Read(TProtocol iprot)
{
TField field;
iprot.ReadStructBegin();
while (true)
{
field = iprot.ReadFieldBegin();
if (field.Type == TType.Stop) {
break;
}
switch (field.ID)
{
case 1:
if (field.Type == TType.String) {
TableName = iprot.ReadBinary();
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 2:
if (field.Type == TType.List) {
{
RowBatches = new List<BatchMutation>();
TList _list70 = iprot.ReadListBegin();
for( int _i71 = 0; _i71 < _list70.Count; ++_i71)
{
BatchMutation _elem72 = new BatchMutation();
_elem72 = new BatchMutation();
_elem72.Read(iprot);
RowBatches.Add(_elem72);
}
iprot.ReadListEnd();
}
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
default:
TProtocolUtil.Skip(iprot, field.Type);
break;
}
iprot.ReadFieldEnd();
}
iprot.ReadStructEnd();
}