Ejemplo n.º 1
0
        protected PutRecordsResult InnerPutRecordsSupportingCache(PutRecordsRequest putRecordsParam, PutRecordMethod putRecordMethod)
        {
            if (_disConfig.IsDataCacheEnabled())
            {
                // 开启本地缓存
                PutRecordsResult putRecordsResult = null;
                try
                {
                    putRecordsResult = InnerPutRecordsWithRetry(putRecordsParam, putRecordMethod);
                }
                catch (Exception e)
                {
                    string errorMsg   = e.InnerException.Message;
                    int    statusCode = int.Parse(errorMsg.Split('\n')[0]);
                    // 如果不是可以重试的异常 或者 已达到重试次数,则直接抛出异常
                    if (Utils.Utils.IsCacheData(statusCode))
                    {
                        // 网络异常 全部记录上传失败
                        logger.Info("Local data cache is enabled, try to put failed records to local.");
                        CacheUtils.PutToCache(putRecordsParam, _disConfig); // 写入本地缓存
                    }
                    throw e;
                }

                try
                {
                    // 部分记录上传失败
                    if (putRecordsResult.FailedRecordCount > 0)
                    {
                        // 过滤出上传失败的记录
                        List <PutRecordsResultEntry>  putRecordsResultEntries        = putRecordsResult.Records;
                        List <PutRecordsRequestEntry> failedPutRecordsRequestEntries = new List <PutRecordsRequestEntry>();
                        int index = 0;
                        foreach (PutRecordsResultEntry putRecordsResultEntry in putRecordsResultEntries)
                        {
                            if (!String.IsNullOrEmpty(putRecordsResultEntry.ErrorCode))
                            {
                                failedPutRecordsRequestEntries.Add(putRecordsParam.Records[index]);
                            }
                            index++;
                        }
                        putRecordsParam.Records = failedPutRecordsRequestEntries;

                        logger.Info("Local data cache is enabled, try to put failed records to local.");

                        CacheUtils.PutToCache(putRecordsParam, _disConfig); // 写入本地缓存
                    }
                }
                catch (Exception e)
                {
                    throw e;
                }

                return(putRecordsResult);
            }
            else
            {
                return(InnerPutRecordsWithRetry(putRecordsParam, putRecordMethod));
            }
        }
Ejemplo n.º 2
0
        /// <summary>
        /// 上传数据
        /// </summary>
        /// <param name="streamName">通道名称</param>
        /// <param name="shardId">分区ID</param>
        public static PutRecordsResult RunProducerDemo(string streamName, string shardId)
        {
            var dic = new DISIngestionClient();
            PutRecordsRequest putRecordsRequest = new PutRecordsRequest();

            putRecordsRequest.StreamName = streamName;
            var putRecordsRequestEntries = new List <PutRecordsRequestEntry>();

            for (int i = 0; i < 3; i++)
            {
                string a = shardId + i;

                var putRecordsRequestEntry = new PutRecordsRequestEntry
                {
                    //需要上传的数据
                    Data = Encoding.UTF8.GetBytes(a),
                    //用于明确数据需要写入分区的哈希值,此哈希值将覆盖“PartitionKey”的哈希值
                    //ExplicitHashKey = "0",
                    //如果传了PartitionId参数,则优先使用PartitionId参数。如果PartitionId没有传,则使用PartitionKey
                    //PartitionKey = new Random().Next().ToString(),
                    //分区的唯一标识符
                    PartitionId = shardId,
                };
                putRecordsRequestEntries.Add(putRecordsRequestEntry);
            }
            putRecordsRequest.Records = putRecordsRequestEntries;

            PutRecordsResult response = dic.PutRecords(putRecordsRequest);

            foreach (var item in response.Records)
            {
                Console.WriteLine(item);
            }
            return(response);
        }
Ejemplo n.º 3
0
        private async Task <int> UploadFile(string streamName, string file, string fileName)
        {
            int failedRecordCount = 0;
            //最多上传128M的文件,这里先设置为最多上传500k
            int      splitFileSize = 500 * 1024;
            FileInfo fileInfo      = new FileInfo(file);
            //分几次传
            int    steps         = (int)Math.Ceiling((decimal)fileInfo.Length / splitFileSize);
            string deliverDataId = Guid.NewGuid().ToString("N");

            using (FileStream fs = new FileStream(file, FileMode.Open, FileAccess.Read))
            {
                using (BinaryReader br = new BinaryReader(fs))
                {
                    for (int i = 0; i < steps; i++)
                    {
                        byte[] input = br.ReadBytes(splitFileSize);
                        {
                            PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
                            putRecordsRequest.StreamName = streamName;
                            var putRecordsRequestEntries = new List <PutRecordsRequestEntry>();
                            var putRecordsRequestEntry   = new PutRecordsRequestEntry
                            {
                                Data        = input,
                                ExtenedInfo = new PutRecordsRequestEntryExtendedInfo()
                                {
                                    FileName       = fileName,
                                    DeliverDataId  = deliverDataId,
                                    SequenceNumber = i,
                                    EndFlag        = i == steps - 1 ? true : false,
                                }
                            };
                            putRecordsRequestEntries.Add(putRecordsRequestEntry);
                            putRecordsRequest.Records = putRecordsRequestEntries;

                            PutRecordsResult response = await this.UploadFile(putRecordsRequest);

                            failedRecordCount += response.FailedRecordCount;
                            //Thread.Sleep(500);
                            foreach (var item in response.Records)
                            {
                                Console.WriteLine("异步" + item);
                            }
                        }
                    }
                }
            }
            return(failedRecordCount);
        }
Ejemplo n.º 4
0
        /// <summary>
        /// 同步方式上传文件
        /// </summary>
        /// <param name="streamName">通道名称</param>
        /// <param name="file">文件名</param>
        public static int UploadFileDemo(string streamName, string file)
        {
            int failedRecordCount = 0;
            var dic = new DISIngestionClient();
            //最多上传128M的文件,这里先设置为最多上传500k
            int      splitFileSize = 500 * 1024;
            string   deliverDataId = Guid.NewGuid().ToString("N");
            FileInfo fileInfo      = new FileInfo(file);
            //分几次传
            int steps = (int)Math.Ceiling((decimal)fileInfo.Length / splitFileSize);

            using (FileStream fs = new FileStream(file, FileMode.Open, FileAccess.Read))
            {
                using (BinaryReader br = new BinaryReader(fs))
                {
                    for (int i = 0; i < steps; i++)
                    {
                        byte[] input = br.ReadBytes(splitFileSize);
                        {
                            PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
                            putRecordsRequest.StreamName = streamName;
                            var putRecordsRequestEntries = new List <PutRecordsRequestEntry>();
                            var putRecordsRequestEntry   = new PutRecordsRequestEntry
                            {
                                Data        = input,
                                ExtenedInfo = new PutRecordsRequestEntryExtendedInfo()
                                {
                                    FileName       = @"thisisshawnfolder/2018/07/04/14/56/Penguins2.jpg",
                                    DeliverDataId  = deliverDataId,
                                    SequenceNumber = i,
                                    EndFlag        = i == steps - 1 ? true : false,
                                }
                            };
                            putRecordsRequestEntries.Add(putRecordsRequestEntry);
                            putRecordsRequest.Records = putRecordsRequestEntries;
                            PutRecordsResult response = dic.PutFileRecords(putRecordsRequest);
                            failedRecordCount += response.FailedRecordCount;
                            //Thread.Sleep(500);
                            foreach (var item in response.Records)
                            {
                                Console.WriteLine("同步" + item);
                            }
                        }
                    }
                }
            }
            return(failedRecordCount);
        }
Ejemplo n.º 5
0
        protected PutRecordsResult InnerPutRecordsWithRetry(PutRecordsRequest putRecordsParam, PutRecordMethod putRecordMethod)
        {
            //数据上传的结果集
            PutRecordsResult putRecordsResult = null;

            //用该数组来汇总每次请求后的结果
            PutRecordsResultEntry[] putRecordsResultEntryList = null;

            //记录每次请求失败的下标位置
            int[] retryIndex = null;

            //每次需要上传的请求数据
            PutRecordsRequest retryPutRecordsRequest = putRecordsParam;

            int retryCount             = -1;
            int currentFailed          = 0;
            ExponentialBackOff backOff = null;

            try
            {
                do
                {
                    retryCount++;
                    if (retryCount > 0)
                    {
                        // 等待一段时间再发起重试
                        if (backOff == null)
                        {
                            Monitor.Enter(objlock);
                            logger.Info("Put records retry lock.");
                            backOff = new ExponentialBackOff(ExponentialBackOff.DEFAULT_INITIAL_INTERVAL,
                                                             ExponentialBackOff.DEFAULT_MULTIPLIER, _disConfig.GetBackOffMaxIntervalMs(),
                                                             ExponentialBackOff.DEFAULT_MAX_ELAPSED_TIME);
                        }

                        if (putRecordsResult != null && currentFailed != putRecordsResult.Records.Count)
                        {
                            // 部分失败则重置退避时间
                            backOff.resetCurrentInterval();
                        }

                        long sleepMs = backOff.getNextBackOff();

                        if (retryPutRecordsRequest.Records.Count > 0)
                        {
                            logger.DebugFormat(
                                "Put {0} records but {1} failed, will re-try after backoff {2} ms, current retry count is {3}.",
                                putRecordsResult != null ? putRecordsResult.Records.Count
                                    : putRecordsParam.Records.Count,
                                currentFailed,
                                sleepMs,
                                retryCount);
                        }

                        backOff.backOff(sleepMs);
                    }

                    try
                    {
                        putRecordsResult = putRecordMethod(retryPutRecordsRequest);
                    }
                    catch (Exception t)
                    {
                        if (putRecordsResultEntryList != null)
                        {
                            logger.Error(t.Message, t);
                            break;
                        }
                        throw t;
                    }

                    if (putRecordsResult != null)
                    {
                        currentFailed = putRecordsResult.FailedRecordCount;

                        if (putRecordsResultEntryList == null && currentFailed == 0 || _disConfig.GetRecordsRetries() == 0)
                        {
                            // 第一次发送全部成功或者不需要重试,则直接返回结果
                            return(putRecordsResult);
                        }

                        if (putRecordsResultEntryList == null)
                        {
                            // 存在发送失败的情况,需要重试,则使用数组来汇总每次请求后的结果。
                            putRecordsResultEntryList = new PutRecordsResultEntry[putRecordsParam.Records.Count];
                        }

                        // 需要重试发送数据的原始下标
                        List <int> retryIndexTemp = new List <int>(currentFailed);

                        if (currentFailed > 0)
                        {
                            // 初始化重试发送的数据请求
                            retryPutRecordsRequest            = new PutRecordsRequest();
                            retryPutRecordsRequest.StreamName = putRecordsParam.StreamName;
                            retryPutRecordsRequest.Records    = new List <PutRecordsRequestEntry>(currentFailed);
                        }

                        // 对每条结果分析,更新结果数据
                        for (int i = 0; i < putRecordsResult.Records.Count; i++)
                        {
                            // 获取重试数据在原始数据中的下标位置
                            int originalIndex = retryIndex == null ? i : retryIndex[i];
                            PutRecordsResultEntry putRecordsResultEntry = putRecordsResult.Records[i];
                            // 对所有异常进行重试 && "DIS.4303".equals(putRecordsResultEntry.getErrorCode())
                            if (!string.IsNullOrEmpty(putRecordsResultEntry.ErrorCode))
                            {
                                retryIndexTemp.Add(originalIndex);
                                retryPutRecordsRequest.Records.Add(putRecordsParam.Records[originalIndex]);
                            }
                            putRecordsResultEntryList[originalIndex] = putRecordsResultEntry;
                        }
                        retryIndex = retryIndexTemp.Count > 0 ? retryIndexTemp.ToArray()
                            : new int[0];
                    }
                } while ((retryIndex == null || retryIndex.Length > 0) && retryCount < _disConfig.GetRecordsRetries());
            }
            finally
            {
                if (retryCount > 0)
                {
                    Monitor.Exit(objlock);
                    logger.Info("Put records retry unlock.");
                }
            }
            putRecordsResult = new PutRecordsResult();
            if (retryIndex == null)
            {
                // 不可能存在此情况,完全没有发送出去会直接抛出异常
                putRecordsResult.FailedRecordCount = putRecordsParam.Records.Count;
            }
            else
            {
                putRecordsResult.FailedRecordCount = retryIndex.Length;
                putRecordsResult.Records           = new List <PutRecordsResultEntry>(putRecordsResultEntryList);
            }

            return(putRecordsResult);
        }