/// <summary>
/// 插入多行数据
/// </summary>
/// <param name="tableName">表名</param>
/// <param name="iep">对应的Region的服务器地址</param>
/// <param name="batchMutation">列数据集合</param>
/// <exception cref="IOErrorException">IO错误</exception>
/// <exception cref="ArgumentNullException">参数不能为空</exception>
/// <exception cref="CommunicationTimeoutException">通信超时</exception>
/// <exception cref="CommunicationFailException">通信失败</exception>
/// <returns>是否成功插入</returns>
internal bool BatchInsert(string tableName, IPEndPoint iep, BatchMutation[] batchMutation)
{
if (batchMutation == null) throw new ArgumentNullException("batchMutation");
IThriftConnectionAgent agent = _connectionPool.GetChannel(iep, "RegionServer", _protocolStack, _transactionManager);
if (agent == null) throw new NoConnectionException();
Exception ex = null;
ThriftMessageTransaction transaction = agent.CreateTransaction();
AutoResetEvent autoResetEvent = new AutoResetEvent(false);
transaction.ResponseArrived += delegate(object sender, LightSingleArgEventArgs<ThriftMessage> e)
{
InsertNewRowsResponseMessage rspMsg = (InsertNewRowsResponseMessage)e.Target;
if (rspMsg.IOErrorMessage != null) ex = new IOErrorException(rspMsg.IOErrorMessage.Reason);
autoResetEvent.Set();
};
transaction.Timeout += delegate
{
ex = new CommunicationTimeoutException(transaction.SequenceId);
autoResetEvent.Set();
};
transaction.Failed += delegate
{
ex = new CommunicationFailException(transaction.SequenceId);
autoResetEvent.Set();
};
InsertNewRowsRequestMessage reqMsg = new InsertNewRowsRequestMessage { TableName = tableName, RowBatch = batchMutation, Attributes = new Dictionary<string, string>() };
transaction.SendRequest(reqMsg);
autoResetEvent.WaitOne();
if (ex != null) throw ex;
return true;
}
/// <summary>
/// 批量插入行操作
/// </summary>
/// <param name="exceptionMutations">插入异常的数据集合</param>
/// <param name="batchMutation">行集合信息</param>
/// <exception cref="IOErrorException">IO错误</exception>
/// <exception cref="ArgumentNullException">参数不能为空</exception>
/// <exception cref="CommunicationTimeoutException">通信超时</exception>
/// <exception cref="CommunicationFailException">通信失败</exception>
/// <returns>全部成功插入返回true，其它返回false</returns>
public bool BatchInsert(out BatchMutation[] exceptionMutations, params BatchMutation[] batchMutation)
{
exceptionMutations = null;
List<BatchMutation> exceptionMutationList = new List<BatchMutation>();
if (batchMutation == null || batchMutation.Length == 0) return true;
Dictionary<IPEndPoint, List<BatchMutation>> dic = new Dictionary<IPEndPoint, List<BatchMutation>>();
foreach (BatchMutation mutation in batchMutation)
{
IPEndPoint iep = _regionManager.GetRegionByRowKey(mutation.RowKey);
List<BatchMutation> list;
if (!dic.TryGetValue(iep, out list)) list = new List<BatchMutation>();
list.Add(mutation);
dic[iep] = list;
}
foreach (KeyValuePair<IPEndPoint, List<BatchMutation>> pair in dic)
{
try
{
_client.BatchInsert(TableName, pair.Key, pair.Value.ToArray());
}
catch(Exception ex)
{
exceptionMutationList.AddRange(pair.Value);
_tracing.Error(string.Format("[{0}]BatchInsert exception : {1}" , pair.Key.Address, ex.Message));
_tracing.Error(ex, null);
}
}
if (exceptionMutationList.Count > 0)
{
exceptionMutations = exceptionMutationList.ToArray();
return false;
}
return true;
}
static void InsertHugeDataTest(int rowNum, int bytesNum, int batchSize, string startKey, string endKey, int testCount)
{
//new MemoryFailPoint(10);
IHBaseClient client = new HBaseClient("zk=10.200.200.56:2181,10.200.200.57:2181,10.200.200.58:2181;zkTimeout=00:05:00;memSegSize=1048576;memSegMultiples=1000");
string tableName = string.Format("mediad_test_thrift_table_test_{0}", DateTime.Now.Second);
string hugeData = new string('a', bytesNum);
Stopwatch sw = new Stopwatch();
Stopwatch sw2 = new Stopwatch();
Console.WriteLine(((rowNum * Encoding.UTF8.GetBytes(hugeData).Length).ToString()));
try
{
IHTable table = client.CreateTable(tableName, "cf");
BatchMutation[] rows = new BatchMutation[rowNum];
for (int i = 1; i <= rowNum; i++)
{
rows[i-1] = new BatchMutation
{
RowKey = TypeConversionHelper.StringToByteArray(i.ToString()),
Mutations = new[]
{
new Mutation {ColumnName = "cf:col1", Value = TypeConversionHelper.StringToByteArray("value"+i)},
new Mutation {ColumnName = "cf:col2", Value = TypeConversionHelper.StringToByteArray(hugeData+i)}
}
};
}
BatchMutation[] exceptionBatchMutations;
List<Thread> threads = new List<Thread>();
for (int i = 0; i < testCount; i++)
{
threads.Add(new Thread(() =>
{
Stopwatch swthread = new Stopwatch();
swthread.Restart();
Console.WriteLine(table.BatchInsert(out exceptionBatchMutations, rows));
swthread.Stop();
Console.WriteLine((swthread.ElapsedTicks / (decimal)Stopwatch.Frequency));
}));
}
foreach (Thread thread in threads)
thread.Start();
foreach (Thread thread in threads)
thread.Join();
Console.WriteLine("=End=");
Console.WriteLine("GetRows batchSize:{1} 1~{0}:", rowNum, batchSize);
Scanner scanner = table.NewScanner(new byte[]{0}, new byte[]{0xff, 0xff, 0xff}, new List<string> {"cf:col1", "cf:col2"});
Dictionary<string, string> tmpDictionary = new Dictionary<string, string>();
sw.Restart();
RowInfo info;
while ((info = scanner.GetNext()) != null)
{
//Console.WriteLine("read row {0}", info.RowKey);
foreach (KeyValuePair<string, Cell> pair in info.Columns)
{
}
}
Console.WriteLine(sw.ElapsedTicks / (decimal)Stopwatch.Frequency);
if (rowNum != tmpDictionary.Keys.Count) Console.WriteLine("rowNum != tmpDictionary.Keys.Count");
for (int i = 1; i <= rowNum; i++)
{
if (!(hugeData + i).Equals(tmpDictionary[i.ToString()]))
{
Console.WriteLine("content not eqal!");
return;
}
}
}
finally
{
client.DeleteTable(tableName);
}
}
