private void Init()
{
TempBuffer = new byte[BufferSize];
#if true
RingBuffer = new ProducerConsumer<byte>();
#else
RingBuffer = new RingBuffer<byte>(BufferSize);
#endif
}
/// <summary>
/// Initializes a new instance of the <see cref="ConverterStream"/> class.
/// </summary>
/// <param name="sourceReader">The source reader.</param>
/// <param name="converter">The converter to use.</param>
public ConverterStream(
TextReader sourceReader,
TextConverter converter)
{
if (sourceReader == null)
{
throw new ArgumentNullException("sourceReader");
}
if (converter == null)
{
throw new ArgumentNullException("converter");
}
this.producer = converter.CreatePullChain(sourceReader, this);
this.sourceOrDestination = sourceReader;
this.maxLoopsWithoutProgress = 100000 + converter.InputStreamBufferSize + converter.OutputStreamBufferSize;
}
/// <summary>
/// Closes the current stream and releases any resources (such as sockets and file handles) associated with the current stream.
/// </summary>
public override void Close()
{
try
{
if (sourceOrDestination != null && consumer != null && !inconsistentState)
{
Flush();
}
if (producer != null && producer is IDisposable)
{
((IDisposable)producer).Dispose();
}
if (consumer != null && consumer is IDisposable)
{
((IDisposable)consumer).Dispose();
}
}
finally
{
if (sourceOrDestination != null)
{
if (sourceOrDestination is Stream)
{
((Stream)sourceOrDestination).Close();
}
else if (sourceOrDestination is TextReader)
{
((TextReader)sourceOrDestination).Close();
}
else
{
((TextWriter)sourceOrDestination).Close();
}
}
sourceOrDestination = null;
consumer = null;
producer = null;
chunkToReadBuffer = null;
writeBuffer = null;
byteSource = null;
}
}
private async Task MergeSpecificationInformationForAllProviders(SpecificationInformation specificationInformation)
{
string specificationId = specificationInformation.Id;
LogInformation($"Merging specification information for specification {specificationId} into summary for all providers with results currently tracking it");
ICosmosDbFeedIterator <ProviderWithResultsForSpecifications> providersWithResultsForSpecifications = GetProviderWithResultsBySpecificationId(specificationId);
await EnsureFundingPeriodEndDateQueried(specificationInformation);
MergeSpecificationInformationContext context = new MergeSpecificationInformationContext(providersWithResultsForSpecifications, specificationInformation);
IProducerConsumer producerConsumer = _producerConsumerFactory.CreateProducerConsumer(ProduceProviderWithResultsForSpecifications,
MergeSpecificationInformation,
200,
2,
_logger);
await producerConsumer.Run(context);
}
public ConverterWriter(TextWriter destinationWriter, TextConverter converter)
{
if (destinationWriter == null)
{
throw new ArgumentNullException("destinationWriter");
}
if (converter == null)
{
throw new ArgumentNullException("converter");
}
this.consumer = converter.CreatePushChain(this, destinationWriter);
this.destination = destinationWriter;
this.boundaryTesting = converter.TestBoundaryConditions;
this.maxLoopsWithoutProgress = 100000 + converter.InputStreamBufferSize + converter.OutputStreamBufferSize;
}
public async Task <IEnumerable <PublishedProviderFundingCsvData> > GetFundingData(IEnumerable <string> publishedProviderIds,
string specificationId,
params PublishedProviderStatus[] statuses)
{
Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
Guard.IsNotEmpty(publishedProviderIds, nameof(publishedProviderIds));
Guard.IsNotEmpty(statuses, nameof(statuses));
PublishedProviderFundingCsvDataProcessorContext context = new PublishedProviderFundingCsvDataProcessorContext(publishedProviderIds,
statuses,
specificationId);
IProducerConsumer producerConsumer = _producerConsumerFactory.CreateProducerConsumer(ProducePublishedProviderIds,
GetFundingForPublishedProviderIds,
20,
5,
_logger);
await producerConsumer.Run(context);
return(context.GetData());
}
/// <summary>
/// Releases the unmanaged resources used by the <see cref="T:System.IO.TextReader" /> and optionally releases the managed resources.
/// </summary>
/// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param>
protected override void Dispose(bool disposing)
{
if (disposing && source != null)
{
if (source is Stream)
{
((Stream)source).Close();
}
else
{
((TextReader)source).Close();
}
}
if (producer != null && producer is IDisposable)
{
((IDisposable)producer).Dispose();
}
source = null;
producer = null;
sourceOutputObject = null;
writeBuffer = null;
base.Dispose(disposing);
}
// Orphaned WPL code.
#if false
/// <summary>
/// Initializes a new instance of the <see cref="ConverterReader"/> class.
/// </summary>
/// <param name="sourceStream">The source stream.</param>
/// <param name="converter">The converter to use.</param>
public ConverterReader(Stream sourceStream, TextConverter converter)
{
if (sourceStream == null)
{
throw new ArgumentNullException("sourceStream");
}
if (converter == null)
{
throw new ArgumentNullException("converter");
}
if (!sourceStream.CanRead)
{
throw new ArgumentException(Strings.CannotReadFromSource, "sourceStream");
}
this.producer = converter.CreatePullChain(sourceStream, this);
this.source = sourceStream;
this.maxLoopsWithoutProgress = 100000 + converter.InputStreamBufferSize + converter.OutputStreamBufferSize;
}
public async Task Run(PublishedFundingUndoTaskContext taskContext)
{
LogStartingTask();
Guard.ArgumentNotNull(taskContext?.PublishedFundingVersionDetails, nameof(taskContext.PublishedFundingVersionDetails));
UndoTaskDetails details = taskContext.PublishedFundingVersionDetails;
ICosmosDbFeedIterator <PublishedFundingVersion> feed = GetPublishedFundingVersionsFeed(details);
FeedContext <PublishedFundingVersion> feedContext = new FeedContext <PublishedFundingVersion>(taskContext, feed);
IProducerConsumer producerConsumer = ProducerConsumerFactory.CreateProducerConsumer(ProducePublishedFundingVersions,
UndoPublishedFundingVersions,
200,
4,
Logger);
await producerConsumer.Run(feedContext);
await NotifyJobProgress(taskContext);
LogCompletedTask();
}
/// <summary>队列消费大循环，处理消息后自动确认</summary>
/// <typeparam name="T">消息类型</typeparam>
/// <param name="queue">队列</param>
/// <param name="onMessage">消息处理。如果处理消息时抛出异常，消息将延迟后回到队列</param>
/// <param name="cancellationToken">取消令牌</param>
/// <param name="log">日志对象</param>
/// <param name="idField">消息标识字段名，用于处理错误重试</param>
/// <returns></returns>
public static async Task ConsumeAsync <T>(this IProducerConsumer <String> queue, Func <T, String, CancellationToken, Task> onMessage, CancellationToken cancellationToken = default, ILog log = null, String idField = null)
{
// 大循环之前，打断性能追踪调用链
DefaultSpan.Current = null;
// 主题
var topic = (queue as RedisBase).Key;
if (topic.IsNullOrEmpty())
{
topic = queue.GetType().Name;
}
var rds = (queue as RedisBase).Redis;
var tracer = rds.Tracer;
var errLog = log ?? XTrace.Log;
var ids = new List <String> {
"Id", "guid", "OrderId", "Code"
};
if (!idField.IsNullOrEmpty() && !ids.Contains(idField))
{
ids.Insert(0, idField);
}
// 超时时间，用于阻塞等待
var timeout = rds.Timeout / 1000 - 1;
while (!cancellationToken.IsCancellationRequested)
{
var msgId = "";
var mqMsg = "";
ISpan span = null;
try
{
// 异步阻塞消费
mqMsg = await queue.TakeOneAsync(timeout);
if (mqMsg != null)
{
// 埋点
span = tracer?.NewSpan($"redismq:{topic}", mqMsg);
log?.Info($"[{topic}]消息内容为：{mqMsg}");
// 解码
var dic = JsonParser.Decode(mqMsg);
var msg = JsonHelper.Convert <T>(dic);
if (dic.TryGetValue("traceParent", out var tp))
{
span.Detach(tp + "");
}
// 消息标识
foreach (var item in ids)
{
if (dic.TryGetValue(item, out var id))
{
msgId = id + "";
if (!msgId.IsNullOrEmpty())
{
break;
}
}
}
// 处理消息
await onMessage(msg, mqMsg, cancellationToken);
// 确认消息
queue.Acknowledge(mqMsg);
}
else
{
// 没有消息，歇一会
await Task.Delay(1000, cancellationToken);
}
}
catch (ThreadAbortException) { break; }
catch (ThreadInterruptedException) { break; }
catch (Exception ex)
{
span?.SetError(ex, null);
// 消息处理错误超过10次则抛弃
if (!mqMsg.IsNullOrEmpty())
{
if (msgId.IsNullOrEmpty())
{
msgId = mqMsg.MD5();
}
errLog?.Error("[{0}/{1}]消息处理异常：{2} {3}", topic, msgId, mqMsg, ex);
var key = $"{topic}:Error:{msgId}";
var rs = rds.Increment(key, 1);
if (rs < 10)
{
rds.SetExpire(key, TimeSpan.FromHours(24));
}
else
{
queue.Acknowledge(mqMsg);
errLog?.Error("[{0}/{1}]错误过多，删除消息", topic, msgId);
}
}
}
finally
{
span?.Dispose();
}
}
}
/// <summary>队列消费大循环，处理消息后自动确认</summary>
/// <typeparam name="T">消息类型</typeparam>
/// <param name="queue">队列</param>
/// <param name="onMessage">消息处理。如果处理消息时抛出异常，消息将延迟后回到队列</param>
/// <param name="cancellationToken">取消令牌</param>
/// <param name="log">日志对象</param>
/// <param name="idField">消息标识字段名，用于处理错误重试</param>
/// <returns></returns>
public static async Task ConsumeAsync <T>(this IProducerConsumer <String> queue, Action <T> onMessage, CancellationToken cancellationToken = default, ILog log = null, String idField = null)
{
await ConsumeAsync <T>(queue, (m, k, t) => { onMessage(m); return(Task.FromResult(0)); }, cancellationToken, log, idField);
}
/// <inheritdoc />
public SynchronizedProducerQueue(ThreadQueueMode mode, [NotNull] ProducerConsumerQueueOptions <T> options, CancellationToken token = default(CancellationToken))
{
_queue = ProducerConsumerQueue.Create(mode, options, token);
_producerQueue = _queue as IProducerQueue <TQueue, T> ?? throw new NotSupportedException();
}
static async Task <IDictionary <Genders, IDictionary <string, string> > > DownloadUserImages(IList <User> users, IConfiguration configuration, IHostEnvironment environment, ILogger logger)
{
if (users.Count == 0)
{
return(null);
}
string imagesUrl = UriHelper.ToUri(configuration.GetValue <string>("images:users:url"), UriKind.Relative).String() ?? IMAGES_FOLDER_DEF;
string imagesPath = PathHelper.Trim(environment.ContentRootPath);
if (string.IsNullOrEmpty(imagesPath) || !Directory.Exists(imagesPath))
{
imagesPath = Directory.GetCurrentDirectory();
}
imagesPath = Path.Combine(imagesPath, imagesUrl.Replace('/', '\\').TrimStart('\\'));
logger?.LogInformation($"Initialized images directory as '{imagesPath}'.");
if (!DirectoryHelper.Ensure(imagesPath))
{
logger?.LogError($"Failed to create images directory '{imagesPath}'.");
return(null);
}
IDictionary <string, string> femalesNeeded = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
IDictionary <string, string> malesNeeded = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
IDictionary <string, string> females = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
IDictionary <string, string> males = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
string femalePattern = $"{IMAGES_PREFIX_FEMALE}??.jpg";
string malePattern = $"{IMAGES_PREFIX_MALE}??.jpg";
foreach (User user in users.Where(e => e.Gender == Genders.Female || e.Gender == Genders.Male))
{
string pattern;
IDictionary <string, string> needed, queue;
if (user.Gender == Genders.Female)
{
pattern = femalePattern;
needed = femalesNeeded;
queue = females;
}
else
{
pattern = malePattern;
needed = malesNeeded;
queue = males;
}
string path = Path.Combine(imagesPath, user.Id);
if (!Directory.Exists(path))
{
needed[user.Id] = null;
continue;
}
string file = Directory.EnumerateFiles(path, pattern, SearchOption.TopDirectoryOnly)
.FirstOrDefault();
if (string.IsNullOrEmpty(file))
{
needed[user.Id] = null;
continue;
}
queue[user.Id] = file;
}
IDictionary <Genders, IDictionary <string, string> > result = new Dictionary <Genders, IDictionary <string, string> >
{
[Genders.Female] = females,
[Genders.Male] = males
};
if (femalesNeeded.Count == 0 && malesNeeded.Count == 0)
{
return(result);
}
logger?.LogInformation($"Will download {femalesNeeded.Count} female images and {malesNeeded.Count} male images.");
Regex regex = new Regex("auto_[fm]_(?<x>\\d+)\\.jpg$", RegexHelper.OPTIONS_I);
HashSet <int> usedFemales = new HashSet <int>(females.Select(e =>
{
Match match = regex.Match(e.Value);
return(!match.Success
? -1
: int.Parse(match.Groups["x"].Value));
}).Where(e => e > -1));
HashSet <int> usedMales = new HashSet <int>(males.Select(e =>
{
Match match = regex.Match(e.Value);
return(!match.Success
? -1
: int.Parse(match.Groups["x"].Value));
}).Where(e => e > -1));
// download will timeout in x minutes where x is a number between 0 and 10 from the configuration
int timeout = configuration.GetValue("images:users:downloadTimeout", 5).Within(0, 10);
// use multi-thread to download the images
using (CancellationTokenSource cts = timeout > 0 ? new CancellationTokenSource(TimeSpan.FromMinutes(timeout)) : null)
{
CancellationToken token = cts?.Token ?? CancellationToken.None;
IOHttpDownloadFileWebRequestSettings downloadSettings = new IOHttpDownloadFileWebRequestSettings
{
BufferSize = Constants.BUFFER_256_KB,
Overwrite = true,
Timeout = TimeSpan.FromSeconds(configuration.GetValue("images:users:requestTimeout", 30).Within(0, 180)).TotalIntMilliseconds()
};
#if DEBUG
int threads = configuration.GetValue <bool>("limitThreads")
? 1
: TaskHelper.ProcessMaximum;
#else
int threads = TaskHelper.ProcessMaximum;
#endif
ProducerConsumerThreadQueueOptions <PersonDownloadData> options = new ProducerConsumerThreadQueueOptions <PersonDownloadData>(threads, (_, pdd) =>
{
// copy to local vars for threading issues
IDictionary <string, string> queue = result[pdd.Gender];
return(DownloadUserImage(imagesPath, pdd, queue, downloadSettings, logger, token));
})
{
WorkStartedCallback = _ => logger?.LogInformation($"Download started using {threads} threads..."),
WorkCompletedCallback = _ => logger?.LogInformation("Download completed.")
};
using (IProducerConsumer <PersonDownloadData> requests = ProducerConsumerQueue.Create(ThreadQueueMode.Task, options, token))
{
int number;
foreach (string id in femalesNeeded.Keys)
{
do
{
number = RNGRandomHelper.Next(0, 99);
}while (usedFemales.Contains(number));
usedFemales.Add(number);
requests.Enqueue(new PersonDownloadData
{
Id = id,
Gender = Genders.Female,
Number = number
});
}
foreach (string id in malesNeeded.Keys)
{
do
{
number = RNGRandomHelper.Next(0, 99);
}while (usedMales.Contains(number));
usedMales.Add(number);
requests.Enqueue(new PersonDownloadData
{
Id = id,
Gender = Genders.Male,
Number = number
});
}
requests.Complete();
await requests.WaitAsync();
}
}
return(result);
}
public ProducerConsumerStream(IProducerConsumer<byte> ProducerConsumer)
{
this.ProducerConsumer = ProducerConsumer;
}
/// <summary>从队列消费消息并推送到WebSocket客户端</summary>
/// <param name="socket"></param>
/// <param name="queue"></param>
/// <param name="source"></param>
/// <returns></returns>
public static async Task ConsumeAndPushAsync(this System.Net.WebSockets.WebSocket socket, IProducerConsumer <String> queue, CancellationTokenSource source)
{
var token = source.Token;
//var queue = _queue.GetQueue<String>($"cmd:{node.Code}");
try
{
while (!token.IsCancellationRequested && socket.State == System.Net.WebSockets.WebSocketState.Open)
{
var msg = await queue.TakeOneAsync(30_000);
if (msg != null)
{
await socket.SendAsync(new ArraySegment <Byte>(msg.GetBytes()), System.Net.WebSockets.WebSocketMessageType.Text, true, token);
}
else
{
await Task.Delay(100, token);
}
}
}
catch (Exception ex)
{
XTrace.WriteException(ex);
}
finally
{
//if (token.GetValue("_source") is CancellationTokenSource source) source.Cancel();
source.Cancel();
}
}
