private static IReadOnlyCollection <m3u8_part_ts> download_m3u8File_parallel(m3u8_client mc, m3u8_file_t m3u8_file
, CancellationTokenSource cts = null, int maxDegreeOfParallelism = 64)
{
var ct = (cts?.Token).GetValueOrDefault(CancellationToken.None);
var baseAddress = m3u8_file.BaseAddress;
var totalPatrs = m3u8_file.Parts.Count;
var globalPartNumber = 0;
var downloadPartsSet = new SortedSet <m3u8_part_ts>(default(m3u8_part_ts.comparer));
using (DefaultConnectionLimitSaver.Create(maxDegreeOfParallelism))
{
Parallel.ForEach(m3u8_file.Parts, new ParallelOptions()
{
MaxDegreeOfParallelism = maxDegreeOfParallelism, CancellationToken = ct
},
(part, loopState, idx) =>
{
var n = Interlocked.Increment(ref globalPartNumber);
try
{
CONSOLE.WriteLine($"#{n} of {totalPatrs}). '{part.RelativeUrlName}'...");
var downloadPart = mc.DownloadPart(part, baseAddress, ct).Result;
if ((downloadPart.Error != null) && !ct.IsCancellationRequested)
{
CONSOLE.WriteLineError($"#{n} of {totalPatrs}). FAILED: {downloadPart.Error}{Environment.NewLine}");
}
lock ( downloadPartsSet )
{
downloadPartsSet.Add(downloadPart);
}
}
catch (Exception ex)
{
#region [.code.]
var aex = ex as AggregateException;
if ((aex == null) || !aex.InnerExceptions.All(e => (e is OperationCanceledException)))
{
CONSOLE.WriteLineError("ERROR: " + ex);
}
#endregion
}
});
}
ct.ThrowIfCancellationRequested();
CONSOLE.WriteLine($"\r\n total parts processed: {globalPartNumber} of {totalPatrs}");
return(downloadPartsSet);
}
private static IEnumerable <m3u8_part_ts> download_m3u8File_parallel__v2(m3u8_client mc, m3u8_file_t m3u8_file
, CancellationTokenSource cts = null, int maxDegreeOfParallelism = 64)
{
var ct = (cts?.Token).GetValueOrDefault(CancellationToken.None);
var baseAddress = m3u8_file.BaseAddress;
var totalPatrs = m3u8_file.Parts.Count;
var successPartNumber = 0;
var expectedPartNumber = m3u8_file.Parts.FirstOrDefault().OrderNumber;
var maxPartNumber = m3u8_file.Parts.LastOrDefault().OrderNumber;
var sourceQueue = new Queue <m3u8_part_ts>(m3u8_file.Parts);
var downloadPartsSet = new SortedSet <m3u8_part_ts>(default(m3u8_part_ts.comparer));
using (DefaultConnectionLimitSaver.Create(maxDegreeOfParallelism))
using (var canExtractPartEvent = new AutoResetEvent(false))
using (var semaphore = new SemaphoreSlim(maxDegreeOfParallelism))
{
var task_download = Task.Run(() =>
{
for ( ; sourceQueue.Count != 0;)
{
semaphore.Wait();
var part = sourceQueue.Dequeue();
#if DEBUG
CONSOLE.WriteLine($"start download part: {part}...");
#endif
mc.DownloadPart(part, baseAddress, ct)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
Interlocked.Increment(ref expectedPartNumber);
CONSOLE.WriteLine($"'{t.Exception.GetType().Name}': '{t.Exception.Message}'.", ConsoleColor.Red);
}
else if (!t.IsCanceled)
{
Interlocked.Increment(ref successPartNumber);
var downloadPart = t.Result;
#if DEBUG
CONSOLE.WriteLine($"end download part: {downloadPart}.");
#endif
lock ( downloadPartsSet )
{
downloadPartsSet.Add(downloadPart);
canExtractPartEvent.Set();
}
}
});
}
}, ct);
for ( ; expectedPartNumber <= maxPartNumber && !ct.IsCancellationRequested;)
{
#if DEBUG
CONSOLE.WriteLine($"wait part #{expectedPartNumber}...");
#endif
canExtractPartEvent.WaitOne();
lock ( downloadPartsSet )
{
for ( ; downloadPartsSet.Count != 0;)
{
var min_part = downloadPartsSet.Min;
if (expectedPartNumber == min_part.OrderNumber)
{
CONSOLE.WriteLine($"receive part #{expectedPartNumber}.");
downloadPartsSet.Remove(min_part);
Interlocked.Increment(ref expectedPartNumber);
semaphore.Release();
yield return(min_part);
}
else
{
break;
}
}
}
}
}
ct.ThrowIfCancellationRequested();
CONSOLE.WriteLine($"\r\n total parts processed: {successPartNumber} of {totalPatrs}");
}
private static IEnumerable <m3u8_part_ts> download_m3u8File_parts_parallel(download_m3u8File_parts_parallel_params_t ip)
{
var ct = (ip.cts?.Token).GetValueOrDefault(CancellationToken.None);
var baseAddress = ip.m3u8File.BaseAddress;
var totalPatrs = ip.m3u8File.Parts.Count;
var successReceivedPartCount = 0;
var failedReceivedPartCount = 0;
ip.responseStepAction?.Invoke(new ResponseStepActionParams(totalPatrs));
var expectedPartNumber = ip.m3u8File.Parts.FirstOrDefault().OrderNumber;
var maxPartNumber = ip.m3u8File.Parts.LastOrDefault().OrderNumber;
var sourceQueue = new Queue <m3u8_part_ts>(ip.m3u8File.Parts);
var downloadPartsSet = new SortedSet <m3u8_part_ts>(default(m3u8_part_ts_comparer));
using (DefaultConnectionLimitSaver.Create(ip.maxDegreeOfParallelism))
using (var innerCts = new CancellationTokenSource())
using (var joinedCts = CancellationTokenSource.CreateLinkedTokenSource(ct, innerCts.Token))
using (var canExtractPartEvent = new AutoResetEvent(false))
using (var semaphore = new SemaphoreSlim(ip.maxDegreeOfParallelism))
{
//-1-//
var task_download = Task.Run(() =>
{
for (var n = 1; sourceQueue.Count != 0; n++)
{
semaphore.Wait(joinedCts.Token);
var part = sourceQueue.Dequeue();
var rq = RequestStepActionParams.CreateSuccess(totalPatrs, n, part);
ip.requestStepAction?.Invoke(rq);
ip.mc.DownloadPart(part, baseAddress, joinedCts.Token)
.ContinueWith((continuationTask) =>
{
var rsp = new ResponseStepActionParams(totalPatrs);
if (continuationTask.IsFaulted)
{
Interlocked.Increment(ref expectedPartNumber);
part.SetError(continuationTask.Exception);
rsp.SuccessReceivedPartCount = successReceivedPartCount;
rsp.FailedReceivedPartCount = Interlocked.Increment(ref failedReceivedPartCount);
rsp.Part = part;
ip.responseStepAction?.Invoke(rsp);
innerCts.Cancel();
}
else if (!continuationTask.IsCanceled)
{
var downloadPart = continuationTask.Result;
if (downloadPart.Error != null)
{
rsp.SuccessReceivedPartCount = successReceivedPartCount;
rsp.FailedReceivedPartCount = Interlocked.Increment(ref failedReceivedPartCount);
}
else
{
rsp.SuccessReceivedPartCount = Interlocked.Increment(ref successReceivedPartCount);
rsp.FailedReceivedPartCount = failedReceivedPartCount;
rsp.BytesLength = downloadPart.Bytes.Length;
}
rsp.Part = downloadPart;
ip.responseStepAction?.Invoke(rsp);
lock ( downloadPartsSet )
{
downloadPartsSet.Add(downloadPart);
canExtractPartEvent.Set();
}
}
}, joinedCts.Token);
}
}, joinedCts.Token);
//-2-//
for (var localReadyParts = new Queue <m3u8_part_ts>(Math.Min(0x1000, ip.maxDegreeOfParallelism));
expectedPartNumber <= maxPartNumber;)
{
var idx = WaitHandle.WaitAny(new[] { canExtractPartEvent /*0*/, joinedCts.Token.WaitHandle /*1*/, });
if (idx == 1) //[ct.IsCancellationRequested := 1]
{
break;
}
if (idx != 0) //[canExtractPartEvent := 0]
{
continue;
}
lock ( downloadPartsSet )
{
for ( ; downloadPartsSet.Count != 0;)
{
var min_part = downloadPartsSet.Min;
if (expectedPartNumber == min_part.OrderNumber)
{
downloadPartsSet.Remove(min_part);
Interlocked.Increment(ref expectedPartNumber);
semaphore.Release();
localReadyParts.Enqueue(min_part);
}
else
{
break;
}
}
}
for ( ; localReadyParts.Count != 0;)
{
var part = localReadyParts.Dequeue();
yield return(part);
}
}
//-3.0-//
if (innerCts.IsCancellationRequested)
{
throw (new m3u8_Exception("Canceled after part download error"));
}
//-3-//
task_download.Wait();
}
//-4-//
ct.ThrowIfCancellationRequested();
}
