/// <summary>读取状态数据</summary>
/// <param name="data"></param>
/// <returns>是否成功</returns>
public static StatModel Read(Object data)
{
var dic = data as IDictionary <String, Object>;
if (dic == null)
{
return(null);
}
var model = JsonHelper.Convert <StatModel>(dic);
if (dic["time"] is String st && st.EndsWithIgnoreCase(" UTC"))
{
model.Time = model.Time.ToLocalTime();
}
return(model);
}
private void FrmRedis_Load(Object sender, EventArgs e)
{
_config = new ControlConfig {
Control = this, FileName = "Redis.json"
};
_config.Load();
var nodes = _config.Items?["Nodes"];
if (nodes != null)
{
_rdsConfigs = JsonHelper.Convert <IList <RedisConfig> >(nodes);
}
if (_rdsConfigs == null)
{
_rdsConfigs = new List <RedisConfig>();
}
}
public async Task ExportAsync(DirectoryInfo directoryInfo, JsonHelper jsonHelper, SyncOptions options, ISession session)
{
var current = await session.Schemas.GetSchemasAsync(session.App);
var schemaMap = current.Items.ToDictionary(x => x.Name, x => x.Id);
jsonHelper.SetSchemaMap(schemaMap);
foreach (var schema in current.Items.OrderBy(x => x.Name))
{
await log.DoSafeAsync($"Exporting '{schema.Name}'", async() =>
{
var details = await session.Schemas.GetSchemaAsync(session.App, schema.Name);
var model = new SchemeModel
{
Name = schema.Name,
Schema = jsonHelper.Convert <SynchronizeSchemaDto>(details)
};
await jsonHelper.WriteWithSchema(directoryInfo, $"schemas/{schema.Name}.json", model, "../__json/schema");
});
}
}
/// <summary>队列消费大循环，处理消息后自动确认</summary>
/// <typeparam name="T">消息类型</typeparam>
/// <param name="queue">队列</param>
/// <param name="onMessage">消息处理。如果处理消息时抛出异常，消息将延迟后回到队列</param>
/// <param name="cancellationToken">取消令牌</param>
/// <param name="log">日志对象</param>
/// <param name="idField">消息标识字段名，用于处理错误重试</param>
/// <returns></returns>
public static async Task ConsumeAsync <T>(this IProducerConsumer <String> queue, Func <T, String, CancellationToken, Task> onMessage, CancellationToken cancellationToken = default, ILog log = null, String idField = null)
{
// 大循环之前，打断性能追踪调用链
DefaultSpan.Current = null;
// 主题
var topic = (queue as RedisBase).Key;
if (topic.IsNullOrEmpty())
{
topic = queue.GetType().Name;
}
var rds = (queue as RedisBase).Redis;
var tracer = rds.Tracer;
var errLog = log ?? XTrace.Log;
var ids = new List <String> {
"Id", "guid", "OrderId", "Code"
};
if (!idField.IsNullOrEmpty() && !ids.Contains(idField))
{
ids.Insert(0, idField);
}
// 超时时间，用于阻塞等待
var timeout = rds.Timeout / 1000 - 1;
while (!cancellationToken.IsCancellationRequested)
{
var msgId = "";
var mqMsg = "";
ISpan span = null;
try
{
// 异步阻塞消费
mqMsg = await queue.TakeOneAsync(timeout);
if (mqMsg != null)
{
// 埋点
span = tracer?.NewSpan($"redismq:{topic}", mqMsg);
log?.Info($"[{topic}]消息内容为：{mqMsg}");
// 解码
var dic = JsonParser.Decode(mqMsg);
var msg = JsonHelper.Convert <T>(dic);
if (dic.TryGetValue("traceParent", out var tp))
{
span.Detach(tp + "");
}
// 消息标识
foreach (var item in ids)
{
if (dic.TryGetValue(item, out var id))
{
msgId = id + "";
if (!msgId.IsNullOrEmpty())
{
break;
}
}
}
// 处理消息
await onMessage(msg, mqMsg, cancellationToken);
// 确认消息
queue.Acknowledge(mqMsg);
}
else
{
// 没有消息，歇一会
await Task.Delay(1000, cancellationToken);
}
}
catch (ThreadAbortException) { break; }
catch (ThreadInterruptedException) { break; }
catch (Exception ex)
{
span?.SetError(ex, null);
// 消息处理错误超过10次则抛弃
if (!mqMsg.IsNullOrEmpty())
{
if (msgId.IsNullOrEmpty())
{
msgId = mqMsg.MD5();
}
errLog?.Error("[{0}/{1}]消息处理异常：{2} {3}", topic, msgId, mqMsg, ex);
var key = $"{topic}:Error:{msgId}";
var rs = rds.Increment(key, 1);
if (rs < 10)
{
rds.SetExpire(key, TimeSpan.FromHours(24));
}
else
{
queue.Acknowledge(mqMsg);
errLog?.Error("[{0}/{1}]错误过多，删除消息", topic, msgId);
}
}
}
finally
{
span?.Dispose();
}
}
}
/// <summary>队列消费大循环，处理消息后自动确认</summary>
/// <typeparam name="T">消息类型</typeparam>
/// <param name="queue">队列</param>
/// <param name="onMessage">消息处理。如果处理消息时抛出异常，消息将延迟后回到队列</param>
/// <param name="cancellationToken">取消令牌</param>
/// <param name="log">日志对象</param>
/// <param name="idField">消息标识字段名，用于处理错误重试</param>
/// <returns></returns>
public static async Task ConsumeAsync <T>(this RedisReliableQueue <String> queue, Func <T, String, CancellationToken, Task> onMessage, CancellationToken cancellationToken = default, ILog log = null, String idField = null)
{
// 大循环之前，打断性能追踪调用链
DefaultSpan.Current = null;
// 主题
var topic = queue.Key;
if (topic.IsNullOrEmpty())
{
topic = queue.GetType().Name;
}
var rds = queue.Redis;
var tracer = rds.Tracer;
var errLog = log ?? XTrace.Log;
// 备用redis，容错、去重
var rds2 = new FullRedis
{
Name = rds.Name + "Bak",
Server = rds.Server,
UserName = rds.UserName,
Password = rds.Password,
Db = rds.Db == 15 ? 0 : (rds.Db + 1),
Tracer = rds.Tracer,
};
// 消息去重
if (queue.DuplicateExpire > 0 && idField.IsNullOrEmpty())
{
throw new ArgumentNullException(nameof(idField), $"队列[{topic}]消息[{queue.DuplicateExpire}]秒去重，需要指定消息唯一标识idField");
}
var ids = new List <String> {
"Id", "guid", "OrderId", "Code"
};
if (!idField.IsNullOrEmpty() && !ids.Contains(idField))
{
ids.Insert(0, idField);
}
// 超时时间，用于阻塞等待
var timeout = rds.Timeout / 1000 - 1;
while (!cancellationToken.IsCancellationRequested)
{
var msgId = "";
var mqMsg = "";
ISpan span = null;
try
{
// 异步阻塞消费
mqMsg = await queue.TakeOneAsync(timeout, cancellationToken);
if (mqMsg != null)
{
// 埋点
span = tracer?.NewSpan($"redismq:{topic}", mqMsg);
log?.Info($"[{topic}]消息内容为：{mqMsg}");
// 解码
var dic = JsonParser.Decode(mqMsg);
var msg = JsonHelper.Convert <T>(dic);
if (dic.TryGetValue("traceParent", out var tp))
{
span.Detach(tp + "");
}
// 消息标识
foreach (var item in ids)
{
if (dic.TryGetValue(item, out var id))
{
msgId = id + "";
if (!msgId.IsNullOrEmpty())
{
break;
}
}
}
// 消息去重
if (queue.DuplicateExpire > 0)
{
// 抢占msgId，处理异常时退出抢占
var dkey = $"{topic}:Duplicate:{msgId}";
if (!rds2.Add(dkey, queue.Status.Key, queue.DuplicateExpire))
{
log?.Info("队列[{0}]遇到重复消息[{1}]，自动跳过", topic, msgId);
}
else
{
try
{
await onMessage(msg, mqMsg, cancellationToken);
}
catch
{
rds2.Remove(dkey);
throw;
}
}
}
else
{
// 处理消息
await onMessage(msg, mqMsg, cancellationToken);
}
// 确认消息
queue.Acknowledge(mqMsg);
}
else
{
// 没有消息，歇一会
await Task.Delay(1000, cancellationToken);
}
}
catch (ThreadAbortException) { break; }
catch (ThreadInterruptedException) { break; }
catch (Exception ex)
{
span?.SetError(ex, null);
// 消息处理错误超过10次则抛弃
if (!mqMsg.IsNullOrEmpty())
{
if (msgId.IsNullOrEmpty())
{
msgId = mqMsg.MD5();
}
errLog?.Error("[{0}/{1}]消息处理异常：{2} {3}", topic, msgId, mqMsg, ex);
var key = $"{topic}:Error:{msgId}";
var rs = rds2.Increment(key, 1);
if (rs < 10)
{
rds2.SetExpire(key, TimeSpan.FromHours(24));
}
else
{
queue.Acknowledge(mqMsg);
errLog?.Error("[{0}/{1}]错误过多，删除消息", topic, msgId);
}
}
}
finally
{
span?.Dispose();
}
}
}
public async Task <HttpResponseMessage> ProcessRequestAsync(HttpRequestMessage request = null)
{
PSCommand psCommand;
if (request.Properties.TryGetValue("APP_PSCommand", out object psc) && psc is PSCommand)
{
psCommand = (PSCommand)psc;
}
else
{
PowerShellRestApiEvents.Raise.VerboseMessaging("Unable to retrieve the PowerShell command");
throw new PScommandNotFoundException("Unable to retrieve the PowerShell command");
}
Guid activityId = Guid.NewGuid();
bool asJob = psCommand.AsJob;
//var inParams = new Dictionary<string, object>();
var inParams = new List <KeyValuePair <string, object> >();
List <PSParameter> allowedParams;
#region ----- Body parameters -----
allowedParams = psCommand.GetParametersByLocation(RestLocation.Body);
string documentContents = await Request.Content.ReadAsStringAsync();
documentContents = documentContents.ToString().Trim();
if (!string.IsNullOrWhiteSpace(documentContents))
{
// If only one parameter is defined in parameter file, not name needed
/*if (allowedParams.Count == 1)
* {
* if (documentContents.StartsWith("["))
* {
* JArray tokenArray = JArray.Parse(documentContents);
* Object value = JsonHelper.Convert(tokenArray);
* inParams.Add(new KeyValuePair<string, object>(allowedParams.First().Name, value));
* }
* else if (documentContents.StartsWith("{"))
* {
* JObject obj = JObject.Parse(documentContents);
* Object value = JsonHelper.Convert(obj);
* inParams.Add(new KeyValuePair<string, object>(allowedParams.First().Name, value));
* }
* else
* {
* inParams.Add(new KeyValuePair<string, object>(allowedParams.First().Name, documentContents));
* }
* }
* else if(allowedParams.Count > 1)
* {
*/
if (documentContents.StartsWith("[")) // if more one parameter are defined in config file, array is not allow
{
PowerShellRestApiEvents.Raise.VerboseMessaging(String.Format("Body cannot be a json array."));
throw new MissingParametersException("Body cannot be a json array.");
}
else if (documentContents.StartsWith("{")) // it's an object. Let's just treat it as an object
{
JObject obj = JObject.Parse(documentContents);
foreach (var detail in obj)
{
String name = detail.Key;
Object value = JsonHelper.Convert(detail.Value);
if (allowedParams.FirstOrDefault(x => x.Name.Equals(name, StringComparison.CurrentCultureIgnoreCase)) != null)
{
inParams.Add(new KeyValuePair <string, object>(name, value));
}
else
{
PowerShellRestApiEvents.Raise.VerboseMessaging(String.Format("Parameter {0} is not allow in body.", name));
throw new MissingParametersException(String.Format("Parameter {0} is not allow in body.", name));
}
}
}
else
{
PowerShellRestApiEvents.Raise.VerboseMessaging(String.Format("Boby must be a json object with {0} properties.", allowedParams.Count));
throw new MissingParametersException(String.Format("Boby must be a json object with {0} properties.", allowedParams.Count));
}
//}
}
#endregion
#region ----- QueryString parameters -----
allowedParams = psCommand.GetParametersByLocation(RestLocation.Query);
foreach (var p in Request.GetQueryNameValuePairs())
{
var param = allowedParams.FirstOrDefault(x => x.Name.Equals(p.Key, StringComparison.CurrentCultureIgnoreCase));
if (param != null)
{
var value = Convert.ChangeType(p.Value, param.Type);
inParams.Add(new KeyValuePair <string, object>(param.Name, value));
}
else
{
PowerShellRestApiEvents.Raise.VerboseMessaging(String.Format("Parameter {0} is not allow in QueryString.", p.Key));
throw new MissingParametersException(String.Format("Parameter {0} is not allow in QueryString.", p.Key));
}
}
#endregion
#region ----- URI Path parameters -----
allowedParams = psCommand.GetParametersByLocation(RestLocation.Path);
if (Request.RequestUri.Segments.Length - 4 <= allowedParams.Count)
{
for (int i = 4; i < Request.RequestUri.Segments.Length; i++)
{
string uriValue = Request.RequestUri.Segments[i].Replace("/", string.Empty);
var param = allowedParams.Skip(i - 4).FirstOrDefault();
var value = Convert.ChangeType(uriValue, param.Type);
inParams.Add(new KeyValuePair <string, object>(param.Name, value));
}
}
else if (Request.RequestUri.Segments.Length - 4 > allowedParams.Count)
{
PowerShellRestApiEvents.Raise.VerboseMessaging(String.Format("Too many parameters in Path."));
throw new MissingParametersException(String.Format("Too many parameters in Path."));
}
#endregion
#region ----- Header parameters -----
List <string> allowedParamNames = psCommand.GetParametersByLocation(RestLocation.Header).Select(x => x.Name.ToLower()).ToList();
/*
* Request.Headers.Where(x => allowedParamNames.Contains(x.Key.ToLower()))
* .ToList()
* .ForEach(x => inParams.Add(new KeyValuePair<string, object>(x.Key, x.Value)));
*/
foreach (var p in Request.Headers)
{
var param = allowedParams.FirstOrDefault(x => x.Name.Equals(p.Key, StringComparison.CurrentCultureIgnoreCase));
if (param != null)
{
var value = Convert.ChangeType(p.Value, param.Type);
inParams.Add(new KeyValuePair <string, object>(param.Name, value));
}
}
#endregion
#region ----- ConfigFile parameters -----
foreach (var param in psCommand.GetParametersByLocation(RestLocation.ConfigFile, true))
{
inParams.Add(new KeyValuePair <string, object>(param.Name, param.Value));
}
#endregion
#region ----- User parameter -----
if (!string.IsNullOrWhiteSpace(psCommand.ParameterForUser))
{
string userName = this.User.Identity != null ? this.User.Identity.Name : "Anonymous";
inParams.Add(new KeyValuePair <string, object>(psCommand.ParameterForUser, userName));
}
#endregion
#region ----- Roles parameter -----
if (!string.IsNullOrWhiteSpace(psCommand.ParameterForRoles))
{
string[] userRoles = this.User.Identity == null
? new string[0]
: (this.User.Identity as ClaimsIdentity)
.Claims
.Where(c => c.Type == ClaimTypes.Role)
.Select(c => c.Value)
.ToArray();
inParams.Add(new KeyValuePair <string, object>(psCommand.ParameterForRoles, userRoles));
}
#endregion
#region ----- Claims parameter -----
if (!string.IsNullOrWhiteSpace(psCommand.ParameterForClaims))
{
Claim[] userClaims = this.User.Identity == null
? new Claim[0]
: (this.User.Identity as ClaimsIdentity)
.Claims
.ToArray();
inParams.Add(new KeyValuePair <string, object>(psCommand.ParameterForClaims, userClaims));
}
#endregion
#region ----- Check if all parameters required are found -----
if (psCommand.GetParametersRequired().Select(x => x.Name).Any(requiedName => inParams.All(q => q.Key != requiedName)))
{
PowerShellRestApiEvents.Raise.VerboseMessaging(String.Format("Cannot find all parameters required."));
throw new MissingParametersException("Cannot find all parameters required.");
}
#endregion
// We now catch an exception from the runner
try
{
PowerShellRestApiEvents.Raise.VerboseMessaging(String.Format("Started Executing the runner"));
if (!asJob)
{
PowershellReturn output =
await _powershellRunner.ExecuteAsync(psCommand.Name, psCommand.Snapin, psCommand.Module, inParams, asJob);
JToken token = string.IsNullOrWhiteSpace(output.ActualPowerShellData)
? new JObject()
: output.ActualPowerShellData.StartsWith("[")
? (JToken)JArray.Parse(output.ActualPowerShellData)
: output.ActualPowerShellData.StartsWith("{")
? JObject.Parse(output.ActualPowerShellData)
: JObject.Parse("{\"Message\":" + output.ActualPowerShellData + "}");
JToken token2 = "";
return(new HttpResponseMessage
{
Content = new JsonContent(token)
});
}
else // run as job.
{
Guid jobId = Guid.NewGuid();
string requestedHost = String.Empty;
if (Request.Properties.ContainsKey("MS_OwinContext"))
{
requestedHost = ((OwinContext)Request.Properties["MS_OwinContext"]).Request.RemoteIpAddress;
}
_jobListProvider.AddRequestedJob(jobId, requestedHost);
// Go off and run this job please sir.
var task = Task <bool> .Factory.StartNew(
() =>
{
try
{
Task <PowershellReturn> goTask =
_powershellRunner.ExecuteAsync(
psCommand.Name,
psCommand.Snapin,
psCommand.Module,
inParams,
true);
goTask.Wait();
var output = goTask.Result;
JToken token = output.ActualPowerShellData.StartsWith("[")
? (JToken)JArray.Parse(output.ActualPowerShellData)
: JObject.Parse(output.ActualPowerShellData);
_jobListProvider.CompleteJob(jobId, output.PowerShellReturnedValidData, String.Empty);
string outputPath = Path.Combine(WebApiConfiguration.Instance.Jobs.JobStorePath, jobId + ".json");
using (TextWriter writer = File.CreateText(outputPath))
{
JsonSerializer serializer = new JsonSerializer
{
Formatting = Formatting.Indented // Make it readable for Ops sake!
};
serializer.Serialize(writer, token);
}
return(true);
}
catch (PowerShellExecutionException poException)
{
CrashLogEntry entry = new CrashLogEntry
{
Exceptions = poException.Exceptions,
LogTime = poException.LogTime,
RequestAddress = requestedHost,
RequestMethod = psCommand.Name,
RequestUrl = Request.RequestUri.ToString()
};
entry.SetActivityId(activityId);
string logFile = _crashLogger.SaveLog(entry);
PowerShellRestApiEvents.Raise.InvalidPowerShellOutput(poException.Message + " logged to " + logFile);
ErrorResponse response =
new ErrorResponse
{
ActivityId = activityId,
LogFile = logFile,
Message = poException.Message
};
JToken token = new JObject(response);
_jobListProvider.CompleteJob(jobId, false, String.Empty);
string outputPath = Path.Combine(WebApiConfiguration.Instance.Jobs.JobStorePath, jobId + ".json");
using (TextWriter writer = File.CreateText(outputPath))
{
JsonSerializer serializer = new JsonSerializer
{
Formatting = Formatting.Indented // Make it readable for Ops sake!
};
serializer.Serialize(writer, token);
}
return(true);
}
}
);
// return the Job ID.
return(new HttpResponseMessage
{
Content = new JsonContent(new JValue(jobId))
});
}
}
catch (PowerShellClientException ex)
{
PowerShellRestApiEvents.Raise.InvalidPowerShellOutput("[" + ex.Category.ToString() + "]" + ex.Message);
switch (ex.Category)
{
case ErrorCategory.PermissionDenied: return(Request.CreateErrorResponse(HttpStatusCode.Forbidden, ex.Message));
case ErrorCategory.ObjectNotFound: return(Request.CreateErrorResponse(HttpStatusCode.NotFound, ex.Message));
case ErrorCategory.InvalidArgument: return(Request.CreateErrorResponse(HttpStatusCode.BadRequest, ex.Message));
case ErrorCategory.ResourceExists: return(Request.CreateErrorResponse(HttpStatusCode.Conflict, ex.Message));
}
return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "[" + ex.Category.ToString() + "]" + ex.Message));
}
catch (PowerShellExecutionException poException)
{
string requestedHost = String.Empty;
if (Request.Properties.ContainsKey("MS_OwinContext"))
{
requestedHost = ((OwinContext)Request.Properties["MS_OwinContext"]).Request.RemoteIpAddress;
}
CrashLogEntry entry = new CrashLogEntry
{
Exceptions = poException.Exceptions,
LogTime = poException.LogTime,
RequestAddress = requestedHost,
RequestMethod = psCommand.Name,
RequestUrl = Request.RequestUri.ToString()
};
entry.SetActivityId(activityId);
string logFile = _crashLogger.SaveLog(entry);
PowerShellRestApiEvents.Raise.InvalidPowerShellOutput(poException.Message + " logged to " + logFile);
HttpResponseMessage response = Request.CreateResponse(HttpStatusCode.InternalServerError,
new ErrorResponse
{
ActivityId = activityId,
LogFile = logFile,
Message = poException.Message
}
);
return(response);
}
catch (Exception ex)
{
CrashLogEntry entry = new CrashLogEntry
{
Exceptions = new List <PowerShellException>
{
new PowerShellException
{
ErrorMessage = ex.Message,
LineNumber = 0,
ScriptName = "GenericController.cs",
StackTrace = ex.StackTrace
}
},
LogTime = DateTime.Now,
RequestAddress = String.Empty, // TODO: Find a way of getting the request host.
RequestMethod = psCommand.Name,
RequestUrl = Request.RequestUri.ToString()
};
entry.SetActivityId(activityId);
string logFile = _crashLogger.SaveLog(entry);
PowerShellRestApiEvents.Raise.UnhandledException(ex.Message + " logged to " + logFile, ex.StackTrace ?? String.Empty);
HttpResponseMessage response = Request.CreateResponse(HttpStatusCode.InternalServerError,
new ErrorResponse
{
ActivityId = activityId,
LogFile = logFile,
Message = ex.Message
}
);
return(response);
}
}
