Exemple #1
0
        public void TableStoreAddLocation(DataRowCollection drs)
        {
            if (drs.Count == 0)
            {
                return;
            }
            int           batchCount = 0;
            int           i          = 0;
            List <string> list       = new List <string>();//存储批量上传的主键字符串,用于判断批量上传数据中是否有重复

            try
            {
                RowChanges rowChanges = new RowChanges();
                foreach (DataRow dr in drs)
                {
                    var primaryKey = new PrimaryKey();
                    primaryKey.Add("d", new ColumnValue(Convert.ToInt64(dr["device_code"])));
                    primaryKey.Add("s", new ColumnValue(TimeHelper.ConvertDateTimeToInt(Convert.ToDateTime(dr["startTime"]))));

                    string primarykey = dr["device_code"].ToString() + ";" + dr["startTime"].ToString();
                    if (list.Contains(primarykey))
                    {
                        log.Info("发现重复记录" + primarykey + "数据库记录:");
                        continue;
                    }
                    list.Add(primarykey);
                    var attribute = new AttributeColumns();
                    attribute.Add("e", new ColumnValue(TimeHelper.ConvertDateTimeToInt(Convert.ToDateTime(dr["endTime"]))));
                    //行驶数据
                    attribute.Add("r", new ColumnValue(ByteIntHelper.GetRouteByte(dr["startLongitude"], dr["startLatitude"], dr["endLongitude"], dr["endLatitude"], dr["drivingTime"], dr["mileage"], dr["topSpeed"])));
                    //行驶时长统计
                    attribute.Add("ds", new ColumnValue(ByteIntHelper.GetDurationstatsByte(dr["speedingTime"], dr["highSpeedTime"], dr["mediumSpeedTime"], dr["lowSpeedTime"], dr["idleTime"])));
                    //事件次数统计
                    attribute.Add("es", new ColumnValue(ByteIntHelper.GetEventStatsByte(dr["rapidAccelerationTimes"], dr["rapidDecelerationTimes"], dr["sharpTurnTimes"])));

                    rowChanges.AddPut(new Condition(RowExistenceExpectation.IGNORE), primaryKey, attribute);
                    batchCount++;
                    i++;
                    if (batchCount == 200)//200行数据进行批量提交一次
                    {
                        batchRequest.Add("Route", rowChanges);
                        BatchWriteRowResponse batchWriteRowResponse = oTSClient.BatchWriteRow(batchRequest);
                        foreach (var item in batchWriteRowResponse.TableRespones)
                        {
                            BatchWriteRowResponseForOneTable bwrfo        = item.Value;
                            BatchWriteRowResponseItem        responseItem = bwrfo.PutResponses[0];
                            bool aa = responseItem.IsOK;
                        }
                        rowChanges   = new RowChanges();
                        batchRequest = new BatchWriteRowRequest();
                        batchCount   = 0;
                        list         = new List <string>();
                        log.Info(i + "批量提交成功");
                    }
                }
                if (rowChanges.PutOperations.Count > 0)
                {
                    batchRequest.Add("Route", rowChanges);
                    oTSClient.BatchWriteRow(batchRequest);
                    log.Info(i + "批量提交成功");
                }
                batchRequest = new BatchWriteRowRequest();
            }
            catch (Exception ex)
            {
                log.Error(ex.Message);
            }
        }
Exemple #2
0
        public void TableStoreAddLocation(DataRowCollection drs)
        {
            if (drs.Count == 0)
            {
                return;
            }
            int           batchCount = 0;
            int           i          = 0;
            List <string> list       = new List <string>();//存储批量上传的主键字符串,用于判断批量上传数据中是否有重复

            try
            {
                RowChanges rowChanges = new RowChanges();
                foreach (DataRow dr in drs)
                {
                    var primaryKey = new PrimaryKey();
                    primaryKey.Add("d", new ColumnValue(Convert.ToInt64(dr["device_code"])));
                    primaryKey.Add("t", new ColumnValue(TimeHelper.ConvertDateTimeToInt(Convert.ToDateTime(dr["gps_time"]))));

                    string primarykey = dr["device_code"].ToString() + ";" + dr["gps_time"].ToString();
                    if (list.Contains(primarykey))
                    {
                        log.Info("发现重复记录" + primarykey + "数据库记录:");
                        continue;
                    }
                    list.Add(primarykey);

                    var attribute = new AttributeColumns();
                    //定位数据
                    attribute.Add("l", new ColumnValue(ByteIntHelper.GetLocationByte(dr["latitude"], dr["longitude"], Convert.ToInt32(dr["speed"]), Convert.ToInt32(dr["direct"]), 0)));

                    rowChanges.AddPut(new Condition(RowExistenceExpectation.IGNORE), primaryKey, attribute);
                    batchCount++;
                    i++;
                    if (batchCount == 200)//200行数据进行批量提交一次
                    {
                        batchRequest.Add(TableName, rowChanges);
                        BatchWriteRowResponse bwResponse = oTSClient.BatchWriteRow(batchRequest);


                        if (bwResponse.IsAllSucceed)
                        {
                            //所有数据插入成功
                            log.Info(i + "批量提交成功");
                        }
                        else
                        {
                            log.Error(i + "批量提交有出错记录");
                            //把批量提交的数据,赋值给新的list,进行异常插入处理方法
                            //分析具体是哪个index行数据出错

                            var tableRows = bwResponse.TableRespones;
                            var rows      = tableRows[TableName];
                            for (int j = 0; j < rows.PutResponses.Count; j++)
                            {
                                if (rows.PutResponses[j].IsOK)
                                {
                                }
                                else
                                {
                                    log.Error("批量提交 问题数据在第【" + j + "】行");
                                    Task task = new Task(() => {
                                        var errorRow = rowChanges.PutOperations[j];
                                        //异步处理错误数据行
                                        PutErrorHandle putError = new PutErrorLocation();
                                        putError.HandlePutError(errorRow, TableName, log);
                                    });
                                    task.Start();
                                }
                            }
                        }
                        rowChanges   = new RowChanges();
                        batchRequest = new BatchWriteRowRequest();
                        batchCount   = 0;
                        list         = new List <string>();
                    }
                }
                if (rowChanges.PutOperations.Count > 0)
                {
                    batchRequest.Add("L_100000000", rowChanges);
                    oTSClient.BatchWriteRow(batchRequest);
                    log.Info(i + "批量提交成功");
                }
                batchRequest = new BatchWriteRowRequest();
            }
            catch (Exception ex)
            {
                log.Error(ex.Message);
            }
        }
Exemple #3
0
        public override void HandlePutError(Tuple <Condition, PrimaryKey, AttributeColumns> row, string tableName, ILog log)
        {
            try
            {
                OTSClient      otsClient   = new OTSClient(TableStoreModel.endPointPublic, TableStoreModel.accessKeyID, TableStoreModel.accessKeySecret, TableStoreModel.instanceName);
                var            request     = new PutRowRequest(tableName, row.Item1, row.Item2, row.Item3);
                PutRowResponse rowResponse = otsClient.PutRow(request);
                log.Info("数据,操作成功!");
            }
            catch (Exception ex)
            {
                //错误行重试失败,记录主键信息
                StringBuilder sbPrimaryKey = new StringBuilder();
                foreach (var item in row.Item2)
                {
                    if (item.Key == "et")//日期字段处理
                    {
                        sbPrimaryKey.Append(item.Key + ":" + item.Value.IntegerValue + "【" + TimeHelper.ConvertStringToDateTime(item.Value.IntegerValue.ToString()) + "】;");
                    }
                    else if (item.Key == "ei")//事件id二进制字段处理
                    {
                        sbPrimaryKey.Append(item.Key + ":" + ByteIntHelper.bytesToInt2(item.Value.BinaryValue, 0, 1) + ";");
                    }
                    else
                    {
                        sbPrimaryKey.Append(item.Key + ":" + item.Value.IntegerValue + ";");
                    }
                }

                //记录位置信息【属性列】
                StringBuilder sbAttributeColumns = new StringBuilder();
                foreach (var item in row.Item3)
                {
                    switch (item.Key)
                    {
                    case "ep":
                        //事件参数字段暂不做处理
                        //byte[] ep = item.Value.BinaryValue;
                        break;

                    case "t":
                        sbAttributeColumns.Append(item.Key + ":" + item.Value.IntegerValue + "【" + TimeHelper.ConvertStringToDateTime(item.Value.IntegerValue.ToString()) + "】;");
                        break;

                    case "l":
                        byte[] lbyte = item.Value.BinaryValue;
                        Dictionary <string, int> dictionary = ByteIntHelper.GetLocationByByte(lbyte);
                        foreach (var dic in dictionary)
                        {
                            sbAttributeColumns.Append(dic.Key + ":" + dic.Value + ";");
                        }
                        break;
                    }
                }

                log.Error(string.Format("错误行重试PUT失败:" + ex.Message + "--参数信息:PrimaryKey:{0},AttributeColumns{1}", sbPrimaryKey.ToString(), sbAttributeColumns.ToString()));
            }
        }
Exemple #4
0
        public void TableStoreAddLocation(DataRowCollection drs)
        {
            if (drs.Count == 0)
            {
                return;
            }
            int           batchCount = 0;
            int           i          = 0;
            List <string> list       = new List <string>();//存储批量上传的主键字符串,用于判断批量上传数据中是否有重复

            try
            {
                RowChanges rowChanges = new RowChanges();
                foreach (DataRow dr in drs)
                {
                    int alarm_no = dr["alarm_no"] == null ? -1 : Convert.ToInt32(dr["alarm_no"]);
                    if (alarm_no == 202 && dr["alarm_parameter"].ToString() == "0")
                    {
                        //【逻辑判断】当事件id=202且参数=0则设置ID=212
                        alarm_no = 212;
                    }
                    else if (alarm_no == 200)
                    {
                        if (dr["alarm_parameter"].ToString() == "3")
                        {
                            //【逻辑判断】当事件id=200且参数=3则设置ID=25
                            alarm_no = 25;
                        }
                        else
                        {
                            //【逻辑判断】当事件id=200且参数=4则设置ID=26
                            alarm_no = 26;
                        }
                    }
                    string primarykey = dr["device_code"].ToString() + ";" + dr["alarm_time"].ToString() + ";" + alarm_no;
                    if (list.Contains(primarykey))
                    {
                        log.Info("发现重复记录" + primarykey + "数据库记录:");
                        continue;
                    }
                    list.Add(primarykey);
                    var primaryKey = new PrimaryKey();
                    primaryKey.Add("d", new ColumnValue(Convert.ToInt64(dr["device_code"])));
                    primaryKey.Add("et", new ColumnValue(TimeHelper.ConvertDateTimeToInt(Convert.ToDateTime(dr["alarm_time"])))); //事件事件(毫秒)
                    primaryKey.Add("ei", new ColumnValue(ByteIntHelper.intToBytes2(alarm_no, 1)));                                //事件ID

                    var attribute = new AttributeColumns();
                    #region 事件参数暂时不传递,默认空

                    //if (dr["alarm_parameter"] != null)//事件参数 不为空才添加
                    //{
                    //    byte[] temp= ByteIntHelper.intToBytes(Convert.ToInt32(dr["alarm_parameter"]), 4);
                    //    int startIndex = 0;
                    //    foreach (var item in temp)
                    //    {
                    //        if (Convert.ToInt32(item) == 0)
                    //        {
                    //            startIndex++;
                    //            continue;
                    //        }
                    //        else {
                    //            break;
                    //        }
                    //    }
                    //    byte[] byteAlarmNo = new byte[4-startIndex];
                    //    for (int index = 0; index < byteAlarmNo.Length; index++)
                    //    {
                    //        byteAlarmNo[index] = temp[startIndex+index];
                    //    }
                    //    attribute.Add("ep", new ColumnValue(byteAlarmNo)); //事件参数
                    //}
                    #endregion
                    //定位时间
                    attribute.Add("t", new ColumnValue(TimeHelper.ConvertDateTimeToInt(Convert.ToDateTime(dr["create_time"]))));
                    //定位数据
                    attribute.Add("l", new ColumnValue(ByteIntHelper.GetLocationByte(dr["latitude"], dr["longitude"], Convert.ToInt32(dr["speed"]), Convert.ToInt32(dr["direct"]), 0)));

                    rowChanges.AddPut(new Condition(RowExistenceExpectation.IGNORE), primaryKey, attribute);
                    batchCount++;
                    i++;
                    if (batchCount == 200)//200行数据进行批量提交一次
                    {
                        batchRequest.Add("E_100000000", rowChanges);
                        oTSClient.BatchWriteRow(batchRequest);
                        rowChanges   = new RowChanges();
                        batchRequest = new BatchWriteRowRequest();
                        batchCount   = 0;
                        list         = new List <string>();
                        log.Info(i + "批量提交成功");
                    }
                }
                if (rowChanges.PutOperations.Count > 0)
                {
                    batchRequest.Add("E_100000000", rowChanges);
                    oTSClient.BatchWriteRow(batchRequest);
                    log.Info(i + "批量提交成功");
                }
                batchRequest = new BatchWriteRowRequest();
            }
            catch (Exception ex)
            {
                log.Error(ex.Message);
            }
        }