private static IReadOnlyCollection <m3u8_part_ts> download_m3u8File_parallel(m3u8_client mc, m3u8_file_t m3u8_file , CancellationTokenSource cts = null, int maxDegreeOfParallelism = 64) { var ct = (cts?.Token).GetValueOrDefault(CancellationToken.None); var baseAddress = m3u8_file.BaseAddress; var totalPatrs = m3u8_file.Parts.Count; var globalPartNumber = 0; var downloadPartsSet = new SortedSet <m3u8_part_ts>(default(m3u8_part_ts.comparer)); using (DefaultConnectionLimitSaver.Create(maxDegreeOfParallelism)) { Parallel.ForEach(m3u8_file.Parts, new ParallelOptions() { MaxDegreeOfParallelism = maxDegreeOfParallelism, CancellationToken = ct }, (part, loopState, idx) => { var n = Interlocked.Increment(ref globalPartNumber); try { CONSOLE.WriteLine($"#{n} of {totalPatrs}). '{part.RelativeUrlName}'..."); var downloadPart = mc.DownloadPart(part, baseAddress, ct).Result; if ((downloadPart.Error != null) && !ct.IsCancellationRequested) { CONSOLE.WriteLineError($"#{n} of {totalPatrs}). FAILED: {downloadPart.Error}{Environment.NewLine}"); } lock ( downloadPartsSet ) { downloadPartsSet.Add(downloadPart); } } catch (Exception ex) { #region [.code.] var aex = ex as AggregateException; if ((aex == null) || !aex.InnerExceptions.All(e => (e is OperationCanceledException))) { CONSOLE.WriteLineError("ERROR: " + ex); } #endregion } }); } ct.ThrowIfCancellationRequested(); CONSOLE.WriteLine($"\r\n total parts processed: {globalPartNumber} of {totalPatrs}"); return(downloadPartsSet); }
private static IEnumerable <m3u8_part_ts> download_m3u8File_parallel__v2(m3u8_client mc, m3u8_file_t m3u8_file , CancellationTokenSource cts = null, int maxDegreeOfParallelism = 64) { var ct = (cts?.Token).GetValueOrDefault(CancellationToken.None); var baseAddress = m3u8_file.BaseAddress; var totalPatrs = m3u8_file.Parts.Count; var successPartNumber = 0; var expectedPartNumber = m3u8_file.Parts.FirstOrDefault().OrderNumber; var maxPartNumber = m3u8_file.Parts.LastOrDefault().OrderNumber; var sourceQueue = new Queue <m3u8_part_ts>(m3u8_file.Parts); var downloadPartsSet = new SortedSet <m3u8_part_ts>(default(m3u8_part_ts.comparer)); using (DefaultConnectionLimitSaver.Create(maxDegreeOfParallelism)) using (var canExtractPartEvent = new AutoResetEvent(false)) using (var semaphore = new SemaphoreSlim(maxDegreeOfParallelism)) { var task_download = Task.Run(() => { for ( ; sourceQueue.Count != 0;) { semaphore.Wait(); var part = sourceQueue.Dequeue(); #if DEBUG CONSOLE.WriteLine($"start download part: {part}..."); #endif mc.DownloadPart(part, baseAddress, ct) .ContinueWith(t => { if (t.IsFaulted) { Interlocked.Increment(ref expectedPartNumber); CONSOLE.WriteLine($"'{t.Exception.GetType().Name}': '{t.Exception.Message}'.", ConsoleColor.Red); } else if (!t.IsCanceled) { Interlocked.Increment(ref successPartNumber); var downloadPart = t.Result; #if DEBUG CONSOLE.WriteLine($"end download part: {downloadPart}."); #endif lock ( downloadPartsSet ) { downloadPartsSet.Add(downloadPart); canExtractPartEvent.Set(); } } }); } }, ct); for ( ; expectedPartNumber <= maxPartNumber && !ct.IsCancellationRequested;) { #if DEBUG CONSOLE.WriteLine($"wait part #{expectedPartNumber}..."); #endif canExtractPartEvent.WaitOne(); lock ( downloadPartsSet ) { for ( ; downloadPartsSet.Count != 0;) { var min_part = downloadPartsSet.Min; if (expectedPartNumber == min_part.OrderNumber) { CONSOLE.WriteLine($"receive part #{expectedPartNumber}."); downloadPartsSet.Remove(min_part); Interlocked.Increment(ref expectedPartNumber); semaphore.Release(); yield return(min_part); } else { break; } } } } } ct.ThrowIfCancellationRequested(); CONSOLE.WriteLine($"\r\n total parts processed: {successPartNumber} of {totalPatrs}"); }
private static IEnumerable <m3u8_part_ts> download_m3u8File_parts_parallel(download_m3u8File_parts_parallel_params_t ip) { var ct = (ip.cts?.Token).GetValueOrDefault(CancellationToken.None); var baseAddress = ip.m3u8File.BaseAddress; var totalPatrs = ip.m3u8File.Parts.Count; var successReceivedPartCount = 0; var failedReceivedPartCount = 0; ip.responseStepAction?.Invoke(new ResponseStepActionParams(totalPatrs)); var expectedPartNumber = ip.m3u8File.Parts.FirstOrDefault().OrderNumber; var maxPartNumber = ip.m3u8File.Parts.LastOrDefault().OrderNumber; var sourceQueue = new Queue <m3u8_part_ts>(ip.m3u8File.Parts); var downloadPartsSet = new SortedSet <m3u8_part_ts>(default(m3u8_part_ts_comparer)); using (DefaultConnectionLimitSaver.Create(ip.maxDegreeOfParallelism)) using (var innerCts = new CancellationTokenSource()) using (var joinedCts = CancellationTokenSource.CreateLinkedTokenSource(ct, innerCts.Token)) using (var canExtractPartEvent = new AutoResetEvent(false)) using (var semaphore = new SemaphoreSlim(ip.maxDegreeOfParallelism)) { //-1-// var task_download = Task.Run(() => { for (var n = 1; sourceQueue.Count != 0; n++) { semaphore.Wait(joinedCts.Token); var part = sourceQueue.Dequeue(); var rq = RequestStepActionParams.CreateSuccess(totalPatrs, n, part); ip.requestStepAction?.Invoke(rq); ip.mc.DownloadPart(part, baseAddress, joinedCts.Token) .ContinueWith((continuationTask) => { var rsp = new ResponseStepActionParams(totalPatrs); if (continuationTask.IsFaulted) { Interlocked.Increment(ref expectedPartNumber); part.SetError(continuationTask.Exception); rsp.SuccessReceivedPartCount = successReceivedPartCount; rsp.FailedReceivedPartCount = Interlocked.Increment(ref failedReceivedPartCount); rsp.Part = part; ip.responseStepAction?.Invoke(rsp); innerCts.Cancel(); } else if (!continuationTask.IsCanceled) { var downloadPart = continuationTask.Result; if (downloadPart.Error != null) { rsp.SuccessReceivedPartCount = successReceivedPartCount; rsp.FailedReceivedPartCount = Interlocked.Increment(ref failedReceivedPartCount); } else { rsp.SuccessReceivedPartCount = Interlocked.Increment(ref successReceivedPartCount); rsp.FailedReceivedPartCount = failedReceivedPartCount; rsp.BytesLength = downloadPart.Bytes.Length; } rsp.Part = downloadPart; ip.responseStepAction?.Invoke(rsp); lock ( downloadPartsSet ) { downloadPartsSet.Add(downloadPart); canExtractPartEvent.Set(); } } }, joinedCts.Token); } }, joinedCts.Token); //-2-// for (var localReadyParts = new Queue <m3u8_part_ts>(Math.Min(0x1000, ip.maxDegreeOfParallelism)); expectedPartNumber <= maxPartNumber;) { var idx = WaitHandle.WaitAny(new[] { canExtractPartEvent /*0*/, joinedCts.Token.WaitHandle /*1*/, }); if (idx == 1) //[ct.IsCancellationRequested := 1] { break; } if (idx != 0) //[canExtractPartEvent := 0] { continue; } lock ( downloadPartsSet ) { for ( ; downloadPartsSet.Count != 0;) { var min_part = downloadPartsSet.Min; if (expectedPartNumber == min_part.OrderNumber) { downloadPartsSet.Remove(min_part); Interlocked.Increment(ref expectedPartNumber); semaphore.Release(); localReadyParts.Enqueue(min_part); } else { break; } } } for ( ; localReadyParts.Count != 0;) { var part = localReadyParts.Dequeue(); yield return(part); } } //-3.0-// if (innerCts.IsCancellationRequested) { throw (new m3u8_Exception("Canceled after part download error")); } //-3-// task_download.Wait(); } //-4-// ct.ThrowIfCancellationRequested(); }