public async Task Flush()
{
while (true)
{
var dequeueSize = events.Count < _settings.Value.BatchSize ?
events.Count :
_settings.Value.BatchSize;
var chunk = events
.DequeueChunk(dequeueSize)
.ToList();
if (!chunk.Any())
{
break;
}
var path = Path.Combine(
CommonHelper.MapPath(_packStoreOptions.TempStorePath),
$"{CommonHelper.NewSequentialGuid().ToString()}.log");
MessagePackHelper.ToPack(path, chunk);
await Task.CompletedTask;
}
}
public Task FromStreamAsync(int iteration)
{
var jilStreamAsync = JilHelper.ToStream(_testModel);
var messagePackStreamAsync = MessagePackHelper.ToStream(_testModel);
var msgPackStreamAsync = MsgPackHelper.ToStream(_testModel);
var newtonsoftJsonStreamAsync = NewtonsoftJsonHelper.ToStream(_testModel);
var systemTextJsonStreamAsync = SystemTextJsonHelper.ToStream(_testModel);
var utf8JsonStreamAsync = Utf8JsonHelper.ToStream(_testModel);
Console.WriteLine("FromStreamAsync go!");
Runner.Initialize();
Console.WriteLine(Runner.Time("JilHelper FromStreamAsync", iteration,
async() => await JilHelper.FromStreamAsync <TestModel>(jilStreamAsync)));
Console.WriteLine(Runner.Time("MessagePackHelper FromStreamAsync", iteration,
async() => await MessagePackHelper.FromStreamAsync <TestModel>(messagePackStreamAsync)));
Console.WriteLine(Runner.Time("MsgPackHelper FromStreamAsync", iteration,
async() => await MsgPackHelper.FromStreamAsync <TestModel>(msgPackStreamAsync)));
Console.WriteLine(Runner.Time("NewtonsoftJsonHelper FromStreamAsync", iteration,
async() => await NewtonsoftJsonHelper.FromStreamAsync <TestModel>(newtonsoftJsonStreamAsync)));
Console.WriteLine(Runner.Time("SystemTextJsonHelper FromStreamAsync", iteration,
async() => await SystemTextJsonHelper.FromStreamAsync <TestModel>(systemTextJsonStreamAsync)));
Console.WriteLine(Runner.Time("Utf8JsonHelper FromStreamAsync", iteration,
async() => await Utf8JsonHelper.FromStreamAsync <TestModel>(utf8JsonStreamAsync)));
Console.WriteLine("\r\nFromStreamAsync complete!\r\n");
return(Task.CompletedTask);
}
private static async Task Time(ILogger <Program> logger, MonitorOptions options)
{
try {
logger.LogInformation("开始批量处理TimeData Log文件");
var files = ListFiles(Path.Combine(options.FindTimeDataLogPath, "extract"));
foreach (var file in files)
{
logger.LogInformation("正在处理Time Log文件：" + file.Name);
var path = CommonHelper.MapPath(file.FullName);
var data = MessagePackHelper.ToObject <List <TimeData> >(path);
if (data.Any())
{
foreach (var g in data.GroupBy(g => g.RequestGroup))
{
timeStore.AddRang(g.Key, g.ToList());
}
data = null;
}
File.Delete(file.FullName);
}
logger.LogInformation("结束这批处理Time Log文件，共:" + files.Count());
}
catch (Exception ex) {
logger.LogError("Syinpo.Monitor.Processor Time Log 错误：" + ex.Message);
}
await Task.CompletedTask;
}
public void FromStream(int iteration)
{
#pragma warning disable CS0618
var binaryStream = BinaryHelper.ToStream(_testModel);
#pragma warning restore CS0618
var dataContractStream = DataContractHelper.ToStream(_testModel);
var jilStream = JilHelper.ToStream(_testModel);
var messagePackStream = MessagePackHelper.ToStream(_testModel);
var msgPackStream = MsgPackHelper.ToStream(_testModel);
var newtonsoftJsonStream = NewtonsoftJsonHelper.ToStream(_testModel);
var protobufStream = ProtobufHelper.ToStream(_testModel);
var sharpYamlStream = SharpYamlHelper.ToStream(_testModel);
var systemTextJsonStream = SystemTextJsonHelper.ToStream(_testModel);
var utf8JsonStream = Utf8JsonHelper.ToStream(_testModel);
var xmlStream = XmlHelper.ToStream(_testModel);
var yamlDotNetStream = YamlDotNetHelper.ToStream(_testModel);
var zeroFormatterStream = ZeroFormatterHelper.ToStream(_testModel);
Console.WriteLine("FromStream go!");
Runner.Initialize();
Console.WriteLine(Runner.Time("BinaryHelper FromStream", iteration,
#pragma warning disable CS0618
() => BinaryHelper.FromStream <TestModel>(binaryStream)));
#pragma warning restore CS0618
Console.WriteLine(Runner.Time("DataContractHelper FromStream", iteration,
() => DataContractHelper.FromStream <TestModel>(dataContractStream)));
Console.WriteLine(Runner.Time("JilHelper FromStream", iteration,
() => JilHelper.FromStream <TestModel>(jilStream)));
Console.WriteLine(Runner.Time("MessagePackHelper FromStream", iteration,
() => MessagePackHelper.FromStream <TestModel>(messagePackStream)));
Console.WriteLine(Runner.Time("MsgPackHelper FromStream", iteration,
() => MsgPackHelper.FromStream <TestModel>(msgPackStream)));
Console.WriteLine(Runner.Time("NewtonsoftJsonHelper FromStream", iteration,
() => NewtonsoftJsonHelper.FromStream <TestModel>(newtonsoftJsonStream)));
Console.WriteLine(Runner.Time("ProtobufHelper FromStream", iteration,
() => ProtobufHelper.FromStream <TestModel>(protobufStream)));
Console.WriteLine(Runner.Time("SharpYamlHelper FromStream", iteration,
() => SharpYamlHelper.FromStream <TestModel>(sharpYamlStream)));
Console.WriteLine(Runner.Time("SystemTextJsonHelper FromStream", iteration,
() => SystemTextJsonHelper.FromStream <TestModel>(systemTextJsonStream)));
Console.WriteLine(Runner.Time("Utf8JsonHelper FromStream", iteration,
() => Utf8JsonHelper.FromStream <TestModel>(utf8JsonStream)));
Console.WriteLine(Runner.Time("XmlHelper FromStream", iteration,
() => XmlHelper.FromStream <TestModel>(xmlStream)));
Console.WriteLine(Runner.Time("YamlDotNetHelper FromStream", iteration,
() => YamlDotNetHelper.FromStream <TestModel>(yamlDotNetStream)));
Console.WriteLine(Runner.Time("ZeroFormatterHelper FromStream", iteration,
() => ZeroFormatterHelper.FromStream <TestModel>(zeroFormatterStream)));
Console.WriteLine("\r\nFromStream complete!\r\n");
}
private static async Task Log(ILogger <Program> logger, MonitorOptions options)
{
try {
logger.LogInformation("开始批量处理Request Log文件");
var files = ListFiles(Path.Combine(options.FindRequestLogPath, "extract"));
foreach (var file in files)
{
logger.LogInformation("正在处理Request Log文件：" + file.Name);
try
{
var path = CommonHelper.MapPath(file.FullName);
var data = MessagePackHelper.ToObject <List <HttpLog> >(path);
if (data.Any())
{
var dbContext = IoC.Resolve <IDbContext>("monitor_dbcontext");
var result = data.MapToList <RequestLog>();
await IoC.Resolve <IGenericRepository <RequestLog> >().CreateRange(result);
}
}
catch (Exception ex)
{
logger.LogError("Syinpo.Monitor.Processor Request Log 错误：" + ex.Message + ex.InnerException?.Message);
}
File.Delete(file.FullName);
}
logger.LogInformation("结束这批处理Request Log文件，共:" + files.Count());
await Task.CompletedTask;
}
catch (Exception ex) {
logger.LogError("Syinpo.Monitor.Processor Request Log 错误：" + ex.Message);
}
}
public void ToStream(int iteration)
{
Console.WriteLine("ToStream go!");
Runner.Initialize();
Console.WriteLine(Runner.Time("BinaryHelper ToStream", iteration,
#pragma warning disable CS0618
() => BinaryHelper.ToStream(_testModel)));
#pragma warning restore CS0618
Console.WriteLine(Runner.Time("DataContractHelper ToStream", iteration,
() => DataContractHelper.ToStream(_testModel)));
Console.WriteLine(Runner.Time("JilHelper ToStream", iteration,
() => JilHelper.ToStream(_testModel)));
Console.WriteLine(Runner.Time("MessagePackHelper ToStream", iteration,
() => MessagePackHelper.ToStream(_testModel)));
Console.WriteLine(Runner.Time("MsgPackHelper ToStream", iteration,
() => MsgPackHelper.ToStream(_testModel)));
Console.WriteLine(Runner.Time("NewtonsoftJsonHelper ToStream", iteration,
() => NewtonsoftJsonHelper.ToStream(_testModel)));
Console.WriteLine(Runner.Time("ProtobufHelper ToStream", iteration,
() => ProtobufHelper.ToStream(_testModel)));
Console.WriteLine(Runner.Time("SharpYamlHelper ToStream", iteration,
() => SharpYamlHelper.ToStream(_testModel)));
Console.WriteLine(Runner.Time("SystemTextJsonHelper ToStream", iteration,
() => SystemTextJsonHelper.ToStream(_testModel)));
Console.WriteLine(Runner.Time("Utf8JsonHelper ToStream", iteration,
() => Utf8JsonHelper.ToStream(_testModel)));
Console.WriteLine(Runner.Time("XmlHelper ToStream", iteration,
() => XmlHelper.ToStream(_testModel)));
Console.WriteLine(Runner.Time("YamlDotNetHelper ToStream", iteration,
() => YamlDotNetHelper.ToStream(_testModel)));
Console.WriteLine(Runner.Time("ZeroFormatterHelper ToStream", iteration,
() => ZeroFormatterHelper.ToStream(_testModel)));
Console.WriteLine("\r\nToStream complete!\r\n");
}
public static byte[] ToBytes <TValue>(this TValue?value, MessagePackSerializerOptions?options = null,
CancellationToken cancellationToken = default) =>
MessagePackHelper.ToBytes(value, options, cancellationToken);
public static Task PackToAsync(this object?value, Type type, Stream?stream,
MessagePackSerializerOptions?options = null, CancellationToken cancellationToken = default) =>
MessagePackHelper.PackAsync(type, value, stream, options, cancellationToken);
public static Task PackToAsync <TValue>(this TValue?value, Stream?stream,
MessagePackSerializerOptions?options = null, CancellationToken cancellationToken = default) =>
MessagePackHelper.PackAsync(value, stream, options, cancellationToken);
public static MemoryStream ToStream(this object?value, Type type, MessagePackSerializerOptions?options = null,
CancellationToken cancellationToken = default) =>
MessagePackHelper.ToStream(type, value, options, cancellationToken);
public static ValueTask <object?> FromStreamAsync(this Stream?stream, Type type,
MessagePackSerializerOptions?options = null, CancellationToken cancellationToken = default) =>
MessagePackHelper.FromStreamAsync(type, stream, options, cancellationToken);
public static MemoryStream ToStream <TValue>(this TValue?value, MessagePackSerializerOptions?options = null,
CancellationToken cancellationToken = default) =>
MessagePackHelper.ToStream(value, options, cancellationToken);
public static TValue?FromBytes <TValue>(this ReadOnlyMemory <byte> bytes,
MessagePackSerializerOptions?options = null, CancellationToken cancellationToken = default) =>
MessagePackHelper.FromBytes <TValue>(bytes, options, cancellationToken);
public static ValueTask <TValue?> FromStreamAsync <TValue>(this Stream?stream,
MessagePackSerializerOptions?options = null, CancellationToken cancellationToken = default) =>
MessagePackHelper.FromStreamAsync <TValue>(stream, options, cancellationToken);
public static T FromMessagePack <T>(this byte[] bytes)
{
return(MessagePackHelper.Deserialize <T>(bytes));
}
public static void PackBy(this Stream?stream, Type type, object?value,
MessagePackSerializerOptions?options = null, CancellationToken cancellationToken = default) =>
MessagePackHelper.Pack(type, value, stream, options, cancellationToken);
public static void PackBy <TValue>(this Stream?stream, TValue?value, MessagePackSerializerOptions?options = null,
CancellationToken cancellationToken = default) =>
MessagePackHelper.Pack(value, stream, options, cancellationToken);
public static object?FromBytes(this ReadOnlyMemory <byte> bytes, Type type,
MessagePackSerializerOptions?options = null, CancellationToken cancellationToken = default) =>
MessagePackHelper.FromBytes(type, bytes, options, cancellationToken);
private static async Task Log(ILogger <Program> logger, MonitorOptions options)
{
logger.LogInformation("开始批量处理Log文件");
var files = ListFiles(Path.Combine(options.FindResponseLogPath, "extract"));
foreach (var file in files)
{
logger.LogInformation("正在处理Log文件：" + file.Name);
var path = CommonHelper.MapPath(file.FullName);
var data = MessagePackHelper.ToObject <List <MonitorEvent> >(path);
var resultResponse = new List <ResponseSnap>();
var resultSql = new List <SqlSnap>();
var resultError = new List <ExceptionSnap>();
var resultTrace = new List <TrackSnap>();
foreach (var item in data)
{
var traceId = GetValue <string>(item.Data, MonitorKeys.trace_id);
if (item.EventType == RequestTypeEnum.Http.ToString().ToLowerInvariant())
{
var snap = PreResponse(item, traceId);
resultResponse.Add(snap);
}
else if (item.EventType == RequestTypeEnum.Signalr.ToString().ToLowerInvariant())
{
var snap = PreResponse(item, traceId);
resultResponse.Add(snap);
}
else if (item.EventType == RequestTypeEnum.Sql.ToString().ToLowerInvariant())
{
var snap = PreSql(item, traceId);
resultSql.Add(snap);
}
else
{
var snap = PreTrace(item, traceId);
if (snap != null)
{
resultTrace.Add(snap);
}
else
{
logger.LogWarning("事件{0}，得EventType为{1}，不正确", traceId, item.EventType);
continue;
}
}
if (item.Data.ContainsKey(MonitorKeys.request_error))
{
var source = GetValue <string>(item.Data, MonitorKeys.request_error);
var errormessage = GetValue <string>(item.Data, MonitorKeys.request_errordetail);
var snap = new ExceptionSnap
{
TraceId = traceId,
ErrorSource = source,
ErrorDetail = errormessage,
CreateTime = DateTime.Now
};
resultError.Add(snap);
}
}
var dbContext = IoC.Resolve <IDbContext>("monitor_dbcontext");
if (resultResponse.Any())
{
await IoC.Resolve <IGenericRepository <ResponseSnap> >().CreateRange(resultResponse);
}
if (resultSql.Any())
{
await IoC.Resolve <IGenericRepository <SqlSnap> >().CreateRange(resultSql);
}
if (resultError.Any())
{
await IoC.Resolve <IGenericRepository <ExceptionSnap> >().CreateRange(resultError);
}
if (resultTrace.Any())
{
await IoC.Resolve <IGenericRepository <TrackSnap> >().CreateRange(resultTrace);
}
File.Delete(file.FullName);
}
logger.LogInformation("结束这批处理Log文件，共:" + files.Count());
await Task.CompletedTask;
}
public static byte[] ToMessagePack <T>(this T obj)
{
return(MessagePackHelper.Serialize(obj));
}
public static byte[] ToBytes(this object?value, Type type, MessagePackSerializerOptions?options = null,
CancellationToken cancellationToken = default) =>
MessagePackHelper.ToBytes(type, value, options, cancellationToken);
