public void TableStoreAddLocation(DataRowCollection drs)
{
int batchCount = 0;
int i;
List <string> list = new List <string>();//存储批量上传的主键字符串，用于判断批量上传数据中是否有重复
try
{
RowChanges rowChanges = new RowChanges();
for (i = 0; i < drs.Count; i++)
{
DataRow dr = drs[i];
i++;
var primaryKey = new PrimaryKey();
primaryKey.Add("d", new ColumnValue(Convert.ToInt64(dr["device_code"])));
primaryKey.Add("s", new ColumnValue(TimeHelper.ConvertDateTimeToInt(Convert.ToDateTime(dr["startTime"]))));
string primarykey = dr["device_code"].ToString() + ";" + dr["startTime"].ToString();
if (list.Contains(primarykey))
{
log.Info(i + "发现重复记录" + primarykey + "数据库记录：");
continue;
}
list.Add(primarykey);
var attribute = new AttributeColumns();
attribute.Add("e", new ColumnValue(TimeHelper.ConvertDateTimeToInt(Convert.ToDateTime(dr["endTime"]))));
//行驶数据
attribute.Add("r", new ColumnValue(ByteIntHelper.GetRouteByte(dr["startLongitude"], dr["startLatitude"], dr["endLongitude"], dr["endLatitude"], dr["drivingTime"], dr["mileage"], dr["topSpeed"])));
//行驶时长统计
attribute.Add("ds", new ColumnValue(ByteIntHelper.GetDurationstatsByte(dr["speedingTime"], dr["highSpeedTime"], dr["mediumSpeedTime"], dr["lowSpeedTime"], dr["idleTime"])));
//事件次数统计
attribute.Add("es", new ColumnValue(ByteIntHelper.GetEventStatsByte(dr["rapidAccelerationTimes"], dr["rapidDecelerationTimes"], dr["sharpTurnTimes"])));
rowChanges.AddPut(new Condition(RowExistenceExpectation.IGNORE), primaryKey, attribute);
batchCount++;
if (batchCount == 200)//200行数据进行批量提交一次
{
batchRequest.Add(TableName, rowChanges);
BatchWriteRowResponse batchWriteRowResponse = oTSClient.BatchWriteRow(batchRequest);
BatchWriteResult(batchWriteRowResponse, i, rowChanges);
rowChanges = new RowChanges();
batchRequest = new BatchWriteRowRequest();
batchCount = 0;
list = new List <string>();
}
}
if (rowChanges.PutOperations.Count > 0)
{
batchRequest.Add(TableName, rowChanges);
BatchWriteRowResponse batchWriteRowResponse = oTSClient.BatchWriteRow(batchRequest);
BatchWriteResult(batchWriteRowResponse, i, rowChanges);
}
batchRequest = new BatchWriteRowRequest();
}
catch (Exception ex)
{
log.Error(ex.Message);
}
}
public void TableStoreAddLocation(DataRowCollection drs)
{
if (drs.Count == 0)
{
return;
}
int batchCount = 0;
int i = 0;
List <string> list = new List <string>();//存储批量上传的主键字符串，用于判断批量上传数据中是否有重复
try
{
RowChanges rowChanges = new RowChanges();
foreach (DataRow dr in drs)
{
i++;
var primaryKey = new PrimaryKey();
primaryKey.Add("d", new ColumnValue(Convert.ToInt64(dr["device_code"])));
primaryKey.Add("t", new ColumnValue(TimeHelper.ConvertDateTimeToInt(Convert.ToDateTime(dr["gps_time"]))));
string primarykey = dr["device_code"].ToString() + ";" + dr["gps_time"].ToString();
if (list.Contains(primarykey))
{
log.Info(i + "发现重复记录" + primarykey + "数据库记录：id = " + dr["id"]);
continue;
}
list.Add(primarykey);
var attribute = new AttributeColumns();
//定位数据
attribute.Add("l", new ColumnValue(ByteIntHelper.GetLocationByte(dr["latitude"], dr["longitude"], Convert.ToInt32(dr["speed"]), Convert.ToInt32(dr["direct"]), 0)));
rowChanges.AddPut(new Condition(RowExistenceExpectation.IGNORE), primaryKey, attribute);
batchCount++;
if (batchCount == 200)//200行数据进行批量提交一次
{
batchRequest.Add(TableName, rowChanges);
BatchWriteRowResponse bwResponse = oTSClient.BatchWriteRow(batchRequest);
BatchWriteResult(bwResponse, i, rowChanges);
rowChanges = new RowChanges();
batchRequest = new BatchWriteRowRequest();
batchCount = 0;
list = new List <string>();
}
}
if (rowChanges.PutOperations.Count > 0)
{
batchRequest.Add(TableName, rowChanges);
BatchWriteRowResponse batchWriteRowResponse = oTSClient.BatchWriteRow(batchRequest);
BatchWriteResult(batchWriteRowResponse, i, rowChanges);
}
batchRequest = new BatchWriteRowRequest();
}
catch (Exception ex)
{
log.Error(ex.Message);
}
}
public void TestForError()
{
var schema = new PrimaryKeySchema();
schema.Add("pk1", ColumnValueType.String);
CreateTestTable(TestTableName, schema, new CapacityUnit(0, 0));
var attr1 = new AttributeColumns();
attr1.Add("attr", new ColumnValue("attr_value1"));
var attr2 = new AttributeColumns();
attr2.Add("attr", new ColumnValue("attr_value2"));
var attr3 = new AttributeColumns();
attr3.Add("attr", new ColumnValue("attr_value3"));
{
var pk1 = new PrimaryKey();
pk1.Add("pk1", new ColumnValue("1"));
pk1.Add("pk2", new ColumnValue("1"));
var pk2 = new PrimaryKey();
pk2.Add("pk1", new ColumnValue("2"));
pk2.Add("pk2", new ColumnValue("2"));
var pk3 = new PrimaryKey();
pk3.Add("pk1", new ColumnValue("3"));
pk3.Add("pk2", new ColumnValue("3"));
{
var request = new BatchWriteRowRequest();
var change = new RowChanges();
change.AddPut(new Condition(RowExistenceExpectation.IGNORE), pk1, attr1);
change.AddPut(new Condition(RowExistenceExpectation.IGNORE), pk2, attr2);
change.AddPut(new Condition(RowExistenceExpectation.IGNORE), pk3, attr3);
request.Add(TestTableName, change);
var response = OTSClient.BatchWriteRow(request);
var tables = response.TableRespones;
Assert.AreEqual(1, tables.Count);
var rows = tables[TestTableName];
Assert.AreEqual(3, rows.PutResponses.Count);
Assert.AreEqual("OTSInvalidPK", rows.PutResponses[0].ErrorCode);
Assert.AreEqual("OTSInvalidPK", rows.PutResponses[1].ErrorCode);
Assert.AreEqual("OTSInvalidPK", rows.PutResponses[2].ErrorCode);
Assert.AreEqual("Validate PK size fail. Input: 2, Meta: 1.", rows.PutResponses[0].ErrorMessage);
Assert.AreEqual("Validate PK size fail. Input: 2, Meta: 1.", rows.PutResponses[1].ErrorMessage);
Assert.AreEqual("Validate PK size fail. Input: 2, Meta: 1.", rows.PutResponses[2].ErrorMessage);
}
}
}
public static void BatchWriteRow()
{
Console.WriteLine("Start batch write row...");
PrepareTable();
PrepareData();
OTSClient otsClient = Config.GetClient();
// 一次批量导入100行数据
var request = new BatchWriteRowRequest();
var rowChanges = new RowChanges(TableName);
for (int i = 0; i < 100; i++)
{
PrimaryKey primaryKey = new PrimaryKey
{
{ "pk0", new ColumnValue(i) },
{ "pk1", new ColumnValue("abc") }
};
// 定义要写入改行的属性列
UpdateOfAttribute attribute = new UpdateOfAttribute();
attribute.AddAttributeColumnToPut("col0", new ColumnValue(0));
attribute.AddAttributeColumnToPut("col1", new ColumnValue("a"));
attribute.AddAttributeColumnToPut("col2", new ColumnValue(true));
rowChanges.AddUpdate(new Condition(RowExistenceExpectation.IGNORE), primaryKey, attribute);
}
request.Add(TableName, rowChanges);
var response = otsClient.BatchWriteRow(request);
var tableRows = response.TableRespones;
var rows = tableRows[TableName];
int succeedRows = 0;
int failedRows = 0;
foreach (var row in rows.Responses)
{
// 注意：batch操作可能部分成功部分失败，需要为每行检查状态
if (row.IsOK)
{
succeedRows++;
}
else
{
Console.WriteLine("Read row failed: " + row.ErrorMessage);
failedRows++;
}
}
Console.WriteLine("SucceedRows: " + succeedRows);
Console.WriteLine("FailedRows: " + failedRows);
}
public void PutPredefinedRows(int count)
{
int index = 0;
while (count > 0)
{
var rowChanges = new RowChanges();
for (int i = 0; i < (count > 100 ? 100 : count); i++)
{
rowChanges.AddPut(new Condition(RowExistenceExpectation.IGNORE),
GetPredefinedPrimaryKeyWith4PK(index + i),
GetPredefinedAttributeWith5PK(index + i)
);
}
var request = new BatchWriteRowRequest();
request.Add(TestTableName, rowChanges);
OTSClient.BatchWriteRow(request);
count -= 100;
index += 100;
}
}
public void TestBatchWriteRow()
{
String tableName = "condition_update_test_table";
CreateTable(tableName);
bool success = PutRow(tableName, 20, "Col1", new ColumnValue("Value20"), null);
Assert.IsTrue(success);
success = PutRow(tableName, 21, "Col1", new ColumnValue("Value21"), null);
Assert.IsTrue(success);
success = PutRow(tableName, 22, "Col1", new ColumnValue("Value22"), null);
Assert.IsTrue(success);
var primaryKey = new PrimaryKey
{
{ "PK0", new ColumnValue(20) }
};
var attribute = new AttributeColumns
{
{ "Col2", new ColumnValue("Value2") }
};
Condition cond1 = new Condition(RowExistenceExpectation.IGNORE)
{
ColumnCondition = new RelationalCondition(
"Col1", CompareOperator.NOT_EQUAL,
new ColumnValue("Value20"))
};
var updatePrimaryKey = new PrimaryKey
{
{ "PK0", new ColumnValue(21) }
};
UpdateOfAttribute update2 = new UpdateOfAttribute();
update2.AddAttributeColumnToPut("Col3", new ColumnValue("Value3"));
Condition cond2 = new Condition(RowExistenceExpectation.IGNORE)
{
ColumnCondition = new RelationalCondition(
"Col1", CompareOperator.EQUAL,
new ColumnValue("Value21"))
};
var deletePrimaryKey = new PrimaryKey
{
{ "PK0", new ColumnValue(22) }
};
Condition cond3 = new Condition(RowExistenceExpectation.IGNORE)
{
ColumnCondition = new RelationalCondition(
"Col1", CompareOperator.GREATER_THAN,
new ColumnValue("Value22"))
};
var batchWriteRequest = new BatchWriteRowRequest();
RowChanges changes = new RowChanges(tableName);
changes.AddPut(cond1, primaryKey, attribute);
changes.AddUpdate(cond2, updatePrimaryKey, update2);
changes.AddDelete(cond3, deletePrimaryKey);
batchWriteRequest.Add(tableName, changes);
var response = OTSClient.BatchWriteRow(batchWriteRequest);
var tables = response.TableRespones;
Assert.AreEqual(1, tables.Count);
var rows = tables[tableName];
Assert.IsFalse(rows.Responses[0].IsOK);
}
public void TestForNormal()
{
var schema = new PrimaryKeySchema();
schema.Add("pk1", ColumnValueType.String);
CreateTestTable(TestTableName, schema, new CapacityUnit(0, 0));
var attr1 = new AttributeColumns();
attr1.Add("attr", new ColumnValue("attr_value1"));
var attr2 = new AttributeColumns();
attr2.Add("attr", new ColumnValue("attr_value2"));
var attr3 = new AttributeColumns();
attr3.Add("attr", new ColumnValue("attr_value3"));
{
List <PrimaryKey> primaryKeys = new List <PrimaryKey>();
var pk1 = new PrimaryKey();
pk1.Add("pk1", new ColumnValue("1"));
var pk2 = new PrimaryKey();
pk2.Add("pk1", new ColumnValue("2"));
var pk3 = new PrimaryKey();
pk3.Add("pk1", new ColumnValue("3"));
primaryKeys.Add(pk1);
primaryKeys.Add(pk2);
primaryKeys.Add(pk3);
{
var request = new BatchWriteRowRequest();
var change = new RowChanges();
change.AddPut(new Condition(RowExistenceExpectation.IGNORE), pk1, attr1);
change.AddPut(new Condition(RowExistenceExpectation.IGNORE), pk2, attr2);
change.AddPut(new Condition(RowExistenceExpectation.IGNORE), pk3, attr3);
request.Add(TestTableName, change);
var response = OTSClient.BatchWriteRow(request);
}
{
var request = new BatchGetRowRequest();
request.Add(TestTableName, primaryKeys);
var response = OTSClient.BatchGetRow(request);
var tables = response.RowDataGroupByTable;
Assert.AreEqual(1, tables.Count);
var rows = tables[TestTableName];
Assert.AreEqual(3, rows.Count);
AssertPrimaryKey(pk1, rows[0].PrimaryKey);
AssertPrimaryKey(pk2, rows[1].PrimaryKey);
AssertPrimaryKey(pk3, rows[2].PrimaryKey);
AssertAttribute(attr1, rows[0].Attribute);
AssertAttribute(attr2, rows[1].Attribute);
AssertAttribute(attr3, rows[2].Attribute);
}
}
}
public void TableStoreAddLocation(DataRowCollection drs)
{
int batchCount = 0;
int i = 0;
List <string> list = new List <string>();//存储批量上传的主键字符串，用于判断批量上传数据中是否有重复
try
{
RowChanges rowChanges = new RowChanges();
foreach (DataRow dr in drs)
{
i++;
int alarm_no = dr["alarm_no"] == null ? -1 : Convert.ToInt32(dr["alarm_no"]);
if (alarm_no == 202 && dr["alarm_parameter"].ToString() == "0")
{
//【逻辑判断】当事件id=202且参数=0则设置ID=212
alarm_no = 212;
}
else if (alarm_no == 200)
{
if (dr["alarm_parameter"].ToString() == "3")
{
//【逻辑判断】当事件id=200且参数=3则设置ID=25
alarm_no = 25;
}
else
{
//【逻辑判断】当事件id=200且参数=4则设置ID=26
alarm_no = 26;
}
}
string primarykey = dr["device_code"].ToString() + ";" + dr["alarm_time"].ToString() + ";" + alarm_no;
if (list.Contains(primarykey))
{
log.Info(i + "发现重复记录" + primarykey + "数据库记录：id=" + dr["id"]);
continue;
}
list.Add(primarykey);
var primaryKey = new PrimaryKey();
primaryKey.Add("d", new ColumnValue(Convert.ToInt64(dr["device_code"])));
primaryKey.Add("et", new ColumnValue(TimeHelper.ConvertDateTimeToInt(Convert.ToDateTime(dr["alarm_time"])))); //事件事件（毫秒）
primaryKey.Add("ei", new ColumnValue(ByteIntHelper.intToBytes2(alarm_no, 1))); //事件ID
var attribute = new AttributeColumns();
#region 事件参数暂时不传递，默认空
//if (dr["alarm_parameter"] != null)//事件参数 不为空才添加
//{
// byte[] temp= ByteIntHelper.intToBytes(Convert.ToInt32(dr["alarm_parameter"]), 4);
// int startIndex = 0;
// foreach (var item in temp)
// {
// if (Convert.ToInt32(item) == 0)
// {
// startIndex++;
// continue;
// }
// else {
// break;
// }
// }
// byte[] byteAlarmNo = new byte[4-startIndex];
// for (int index = 0; index < byteAlarmNo.Length; index++)
// {
// byteAlarmNo[index] = temp[startIndex+index];
// }
// attribute.Add("ep", new ColumnValue(byteAlarmNo)); //事件参数
//}
#endregion
//定位时间
attribute.Add("t", new ColumnValue(TimeHelper.ConvertDateTimeToInt(Convert.ToDateTime(dr["create_time"]))));
//定位数据
attribute.Add("l", new ColumnValue(ByteIntHelper.GetLocationByte(dr["latitude"], dr["longitude"], Convert.ToInt32(dr["speed"]), Convert.ToInt32(dr["direct"]), 0)));
rowChanges.AddPut(new Condition(RowExistenceExpectation.IGNORE), primaryKey, attribute);
batchCount++;
if (batchCount == 200)//200行数据进行批量提交一次
{
batchRequest.Add(TableName, rowChanges);
BatchWriteRowResponse bwResponse = oTSClient.BatchWriteRow(batchRequest);
BatchWriteResult(bwResponse, i, rowChanges);
rowChanges = new RowChanges();
batchRequest = new BatchWriteRowRequest();
batchCount = 0;
list = new List <string>();
}
}
if (rowChanges.PutOperations.Count > 0)
{
batchRequest.Add(TableName, rowChanges);
BatchWriteRowResponse batchWriteRowResponse = oTSClient.BatchWriteRow(batchRequest);
BatchWriteResult(batchWriteRowResponse, i, rowChanges);
}
batchRequest = new BatchWriteRowRequest();
}
catch (Exception ex)
{
log.Error(ex.Message);
}
}
public static void ConditionBatchWriteRow()
{
Console.WriteLine("Start batch write row...");
PrepareTable();
OTSClient otsClient = Config.GetClient();
// 定义行的主键，必须与创建表时的TableMeta中定义的一致
PrimaryKey primaryKey = new PrimaryKey();
primaryKey.Add("pk0", new ColumnValue(3));
primaryKey.Add("pk1", new ColumnValue("abc"));
// 定义要写入改行的属性列
AttributeColumns attribute = new AttributeColumns();
attribute.Add("col0", new ColumnValue(0));
attribute.Add("col1", new ColumnValue("a"));
attribute.Add("col2", new ColumnValue(true));
PutRowRequest request = new PutRowRequest(tableName, new Condition(RowExistenceExpectation.IGNORE), primaryKey, attribute);
// 新创建一行数据
try
{
otsClient.PutRow(request);
Console.WriteLine("Put row succeeded.");
}
catch (Exception ex)
{
Console.WriteLine("Put row failed. error:{0}", ex.Message);
}
// 当col0列的值不等于5的时候，允许再次put row，覆盖掉原值，预期成功
try
{
// 构造条件语句：col0列的值不等于5
var condition = new Condition(RowExistenceExpectation.IGNORE);
condition.ColumnCondition = new RelationalCondition("col0",
CompareOperator.NOT_EQUAL,
new ColumnValue(5));
// 构造col2列的值
var attr1 = new AttributeColumns();
attr1.Add("col2", new ColumnValue(false));
// 构造批量写请求
var rowChange = new RowChanges(tableName);
rowChange.AddPut(condition, primaryKey, attr1);
var batchWriteRequest = new BatchWriteRowRequest();
batchWriteRequest.Add(tableName, rowChange);
// 批量写数据
otsClient.BatchWriteRow(batchWriteRequest);
Console.WriteLine("Batch write row succeeded.");
}
catch (Exception ex)
{
Console.WriteLine("Batch write row failed. error:{0}", ex.Message);
}
}
public void TableStoreAddLocation(DataRowCollection drs)
{
if (drs.Count == 0)
{
return;
}
int batchCount = 0;
int i = 0;
List <string> list = new List <string>();//存储批量上传的主键字符串，用于判断批量上传数据中是否有重复
try
{
RowChanges rowChanges = new RowChanges();
foreach (DataRow dr in drs)
{
var primaryKey = new PrimaryKey();
primaryKey.Add("d", new ColumnValue(Convert.ToInt64(dr["device_code"])));
primaryKey.Add("t", new ColumnValue(TimeHelper.ConvertDateTimeToInt(Convert.ToDateTime(dr["gps_time"]))));
string primarykey = dr["device_code"].ToString() + ";" + dr["gps_time"].ToString();
if (list.Contains(primarykey))
{
log.Info("发现重复记录" + primarykey + "数据库记录：");
continue;
}
list.Add(primarykey);
var attribute = new AttributeColumns();
//定位数据
attribute.Add("l", new ColumnValue(ByteIntHelper.GetLocationByte(dr["latitude"], dr["longitude"], Convert.ToInt32(dr["speed"]), Convert.ToInt32(dr["direct"]), 0)));
rowChanges.AddPut(new Condition(RowExistenceExpectation.IGNORE), primaryKey, attribute);
batchCount++;
i++;
if (batchCount == 200)//200行数据进行批量提交一次
{
batchRequest.Add(TableName, rowChanges);
BatchWriteRowResponse bwResponse = oTSClient.BatchWriteRow(batchRequest);
if (bwResponse.IsAllSucceed)
{
//所有数据插入成功
log.Info(i + "批量提交成功");
}
else
{
log.Error(i + "批量提交有出错记录");
//把批量提交的数据，赋值给新的list，进行异常插入处理方法
//分析具体是哪个index行数据出错
var tableRows = bwResponse.TableRespones;
var rows = tableRows[TableName];
for (int j = 0; j < rows.PutResponses.Count; j++)
{
if (rows.PutResponses[j].IsOK)
{
}
else
{
log.Error("批量提交 问题数据在第【" + j + "】行");
Task task = new Task(() => {
var errorRow = rowChanges.PutOperations[j];
//异步处理错误数据行
PutErrorHandle putError = new PutErrorLocation();
putError.HandlePutError(errorRow, TableName, log);
});
task.Start();
}
}
}
rowChanges = new RowChanges();
batchRequest = new BatchWriteRowRequest();
batchCount = 0;
list = new List <string>();
}
}
if (rowChanges.PutOperations.Count > 0)
{
batchRequest.Add("L_100000000", rowChanges);
oTSClient.BatchWriteRow(batchRequest);
log.Info(i + "批量提交成功");
}
batchRequest = new BatchWriteRowRequest();
}
catch (Exception ex)
{
log.Error(ex.Message);
}
}