/// <summary>读取状态数据</summary> /// <param name="data"></param> /// <returns>是否成功</returns> public static StatModel Read(Object data) { var dic = data as IDictionary <String, Object>; if (dic == null) { return(null); } var model = JsonHelper.Convert <StatModel>(dic); if (dic["time"] is String st && st.EndsWithIgnoreCase(" UTC")) { model.Time = model.Time.ToLocalTime(); } return(model); }
private void FrmRedis_Load(Object sender, EventArgs e) { _config = new ControlConfig { Control = this, FileName = "Redis.json" }; _config.Load(); var nodes = _config.Items?["Nodes"]; if (nodes != null) { _rdsConfigs = JsonHelper.Convert <IList <RedisConfig> >(nodes); } if (_rdsConfigs == null) { _rdsConfigs = new List <RedisConfig>(); } }
public async Task ExportAsync(DirectoryInfo directoryInfo, JsonHelper jsonHelper, SyncOptions options, ISession session) { var current = await session.Schemas.GetSchemasAsync(session.App); var schemaMap = current.Items.ToDictionary(x => x.Name, x => x.Id); jsonHelper.SetSchemaMap(schemaMap); foreach (var schema in current.Items.OrderBy(x => x.Name)) { await log.DoSafeAsync($"Exporting '{schema.Name}'", async() => { var details = await session.Schemas.GetSchemaAsync(session.App, schema.Name); var model = new SchemeModel { Name = schema.Name, Schema = jsonHelper.Convert <SynchronizeSchemaDto>(details) }; await jsonHelper.WriteWithSchema(directoryInfo, $"schemas/{schema.Name}.json", model, "../__json/schema"); }); } }
/// <summary>队列消费大循环,处理消息后自动确认</summary> /// <typeparam name="T">消息类型</typeparam> /// <param name="queue">队列</param> /// <param name="onMessage">消息处理。如果处理消息时抛出异常,消息将延迟后回到队列</param> /// <param name="cancellationToken">取消令牌</param> /// <param name="log">日志对象</param> /// <param name="idField">消息标识字段名,用于处理错误重试</param> /// <returns></returns> public static async Task ConsumeAsync <T>(this IProducerConsumer <String> queue, Func <T, String, CancellationToken, Task> onMessage, CancellationToken cancellationToken = default, ILog log = null, String idField = null) { // 大循环之前,打断性能追踪调用链 DefaultSpan.Current = null; // 主题 var topic = (queue as RedisBase).Key; if (topic.IsNullOrEmpty()) { topic = queue.GetType().Name; } var rds = (queue as RedisBase).Redis; var tracer = rds.Tracer; var errLog = log ?? XTrace.Log; var ids = new List <String> { "Id", "guid", "OrderId", "Code" }; if (!idField.IsNullOrEmpty() && !ids.Contains(idField)) { ids.Insert(0, idField); } // 超时时间,用于阻塞等待 var timeout = rds.Timeout / 1000 - 1; while (!cancellationToken.IsCancellationRequested) { var msgId = ""; var mqMsg = ""; ISpan span = null; try { // 异步阻塞消费 mqMsg = await queue.TakeOneAsync(timeout); if (mqMsg != null) { // 埋点 span = tracer?.NewSpan($"redismq:{topic}", mqMsg); log?.Info($"[{topic}]消息内容为:{mqMsg}"); // 解码 var dic = JsonParser.Decode(mqMsg); var msg = JsonHelper.Convert <T>(dic); if (dic.TryGetValue("traceParent", out var tp)) { span.Detach(tp + ""); } // 消息标识 foreach (var item in ids) { if (dic.TryGetValue(item, out var id)) { msgId = id + ""; if (!msgId.IsNullOrEmpty()) { break; } } } // 处理消息 await onMessage(msg, mqMsg, cancellationToken); // 确认消息 queue.Acknowledge(mqMsg); } else { // 没有消息,歇一会 await Task.Delay(1000, cancellationToken); } } catch (ThreadAbortException) { break; } catch (ThreadInterruptedException) { break; } catch (Exception ex) { span?.SetError(ex, null); // 消息处理错误超过10次则抛弃 if (!mqMsg.IsNullOrEmpty()) { if (msgId.IsNullOrEmpty()) { msgId = mqMsg.MD5(); } errLog?.Error("[{0}/{1}]消息处理异常:{2} {3}", topic, msgId, mqMsg, ex); var key = $"{topic}:Error:{msgId}"; var rs = rds.Increment(key, 1); if (rs < 10) { rds.SetExpire(key, TimeSpan.FromHours(24)); } else { queue.Acknowledge(mqMsg); errLog?.Error("[{0}/{1}]错误过多,删除消息", topic, msgId); } } } finally { span?.Dispose(); } } }
/// <summary>队列消费大循环,处理消息后自动确认</summary> /// <typeparam name="T">消息类型</typeparam> /// <param name="queue">队列</param> /// <param name="onMessage">消息处理。如果处理消息时抛出异常,消息将延迟后回到队列</param> /// <param name="cancellationToken">取消令牌</param> /// <param name="log">日志对象</param> /// <param name="idField">消息标识字段名,用于处理错误重试</param> /// <returns></returns> public static async Task ConsumeAsync <T>(this RedisReliableQueue <String> queue, Func <T, String, CancellationToken, Task> onMessage, CancellationToken cancellationToken = default, ILog log = null, String idField = null) { // 大循环之前,打断性能追踪调用链 DefaultSpan.Current = null; // 主题 var topic = queue.Key; if (topic.IsNullOrEmpty()) { topic = queue.GetType().Name; } var rds = queue.Redis; var tracer = rds.Tracer; var errLog = log ?? XTrace.Log; // 备用redis,容错、去重 var rds2 = new FullRedis { Name = rds.Name + "Bak", Server = rds.Server, UserName = rds.UserName, Password = rds.Password, Db = rds.Db == 15 ? 0 : (rds.Db + 1), Tracer = rds.Tracer, }; // 消息去重 if (queue.DuplicateExpire > 0 && idField.IsNullOrEmpty()) { throw new ArgumentNullException(nameof(idField), $"队列[{topic}]消息[{queue.DuplicateExpire}]秒去重,需要指定消息唯一标识idField"); } var ids = new List <String> { "Id", "guid", "OrderId", "Code" }; if (!idField.IsNullOrEmpty() && !ids.Contains(idField)) { ids.Insert(0, idField); } // 超时时间,用于阻塞等待 var timeout = rds.Timeout / 1000 - 1; while (!cancellationToken.IsCancellationRequested) { var msgId = ""; var mqMsg = ""; ISpan span = null; try { // 异步阻塞消费 mqMsg = await queue.TakeOneAsync(timeout, cancellationToken); if (mqMsg != null) { // 埋点 span = tracer?.NewSpan($"redismq:{topic}", mqMsg); log?.Info($"[{topic}]消息内容为:{mqMsg}"); // 解码 var dic = JsonParser.Decode(mqMsg); var msg = JsonHelper.Convert <T>(dic); if (dic.TryGetValue("traceParent", out var tp)) { span.Detach(tp + ""); } // 消息标识 foreach (var item in ids) { if (dic.TryGetValue(item, out var id)) { msgId = id + ""; if (!msgId.IsNullOrEmpty()) { break; } } } // 消息去重 if (queue.DuplicateExpire > 0) { // 抢占msgId,处理异常时退出抢占 var dkey = $"{topic}:Duplicate:{msgId}"; if (!rds2.Add(dkey, queue.Status.Key, queue.DuplicateExpire)) { log?.Info("队列[{0}]遇到重复消息[{1}],自动跳过", topic, msgId); } else { try { await onMessage(msg, mqMsg, cancellationToken); } catch { rds2.Remove(dkey); throw; } } } else { // 处理消息 await onMessage(msg, mqMsg, cancellationToken); } // 确认消息 queue.Acknowledge(mqMsg); } else { // 没有消息,歇一会 await Task.Delay(1000, cancellationToken); } } catch (ThreadAbortException) { break; } catch (ThreadInterruptedException) { break; } catch (Exception ex) { span?.SetError(ex, null); // 消息处理错误超过10次则抛弃 if (!mqMsg.IsNullOrEmpty()) { if (msgId.IsNullOrEmpty()) { msgId = mqMsg.MD5(); } errLog?.Error("[{0}/{1}]消息处理异常:{2} {3}", topic, msgId, mqMsg, ex); var key = $"{topic}:Error:{msgId}"; var rs = rds2.Increment(key, 1); if (rs < 10) { rds2.SetExpire(key, TimeSpan.FromHours(24)); } else { queue.Acknowledge(mqMsg); errLog?.Error("[{0}/{1}]错误过多,删除消息", topic, msgId); } } } finally { span?.Dispose(); } } }
public async Task <HttpResponseMessage> ProcessRequestAsync(HttpRequestMessage request = null) { PSCommand psCommand; if (request.Properties.TryGetValue("APP_PSCommand", out object psc) && psc is PSCommand) { psCommand = (PSCommand)psc; } else { PowerShellRestApiEvents.Raise.VerboseMessaging("Unable to retrieve the PowerShell command"); throw new PScommandNotFoundException("Unable to retrieve the PowerShell command"); } Guid activityId = Guid.NewGuid(); bool asJob = psCommand.AsJob; //var inParams = new Dictionary<string, object>(); var inParams = new List <KeyValuePair <string, object> >(); List <PSParameter> allowedParams; #region ----- Body parameters ----- allowedParams = psCommand.GetParametersByLocation(RestLocation.Body); string documentContents = await Request.Content.ReadAsStringAsync(); documentContents = documentContents.ToString().Trim(); if (!string.IsNullOrWhiteSpace(documentContents)) { // If only one parameter is defined in parameter file, not name needed /*if (allowedParams.Count == 1) * { * if (documentContents.StartsWith("[")) * { * JArray tokenArray = JArray.Parse(documentContents); * Object value = JsonHelper.Convert(tokenArray); * inParams.Add(new KeyValuePair<string, object>(allowedParams.First().Name, value)); * } * else if (documentContents.StartsWith("{")) * { * JObject obj = JObject.Parse(documentContents); * Object value = JsonHelper.Convert(obj); * inParams.Add(new KeyValuePair<string, object>(allowedParams.First().Name, value)); * } * else * { * inParams.Add(new KeyValuePair<string, object>(allowedParams.First().Name, documentContents)); * } * } * else if(allowedParams.Count > 1) * { */ if (documentContents.StartsWith("[")) // if more one parameter are defined in config file, array is not allow { PowerShellRestApiEvents.Raise.VerboseMessaging(String.Format("Body cannot be a json array.")); throw new MissingParametersException("Body cannot be a json array."); } else if (documentContents.StartsWith("{")) // it's an object. Let's just treat it as an object { JObject obj = JObject.Parse(documentContents); foreach (var detail in obj) { String name = detail.Key; Object value = JsonHelper.Convert(detail.Value); if (allowedParams.FirstOrDefault(x => x.Name.Equals(name, StringComparison.CurrentCultureIgnoreCase)) != null) { inParams.Add(new KeyValuePair <string, object>(name, value)); } else { PowerShellRestApiEvents.Raise.VerboseMessaging(String.Format("Parameter {0} is not allow in body.", name)); throw new MissingParametersException(String.Format("Parameter {0} is not allow in body.", name)); } } } else { PowerShellRestApiEvents.Raise.VerboseMessaging(String.Format("Boby must be a json object with {0} properties.", allowedParams.Count)); throw new MissingParametersException(String.Format("Boby must be a json object with {0} properties.", allowedParams.Count)); } //} } #endregion #region ----- QueryString parameters ----- allowedParams = psCommand.GetParametersByLocation(RestLocation.Query); foreach (var p in Request.GetQueryNameValuePairs()) { var param = allowedParams.FirstOrDefault(x => x.Name.Equals(p.Key, StringComparison.CurrentCultureIgnoreCase)); if (param != null) { var value = Convert.ChangeType(p.Value, param.Type); inParams.Add(new KeyValuePair <string, object>(param.Name, value)); } else { PowerShellRestApiEvents.Raise.VerboseMessaging(String.Format("Parameter {0} is not allow in QueryString.", p.Key)); throw new MissingParametersException(String.Format("Parameter {0} is not allow in QueryString.", p.Key)); } } #endregion #region ----- URI Path parameters ----- allowedParams = psCommand.GetParametersByLocation(RestLocation.Path); if (Request.RequestUri.Segments.Length - 4 <= allowedParams.Count) { for (int i = 4; i < Request.RequestUri.Segments.Length; i++) { string uriValue = Request.RequestUri.Segments[i].Replace("/", string.Empty); var param = allowedParams.Skip(i - 4).FirstOrDefault(); var value = Convert.ChangeType(uriValue, param.Type); inParams.Add(new KeyValuePair <string, object>(param.Name, value)); } } else if (Request.RequestUri.Segments.Length - 4 > allowedParams.Count) { PowerShellRestApiEvents.Raise.VerboseMessaging(String.Format("Too many parameters in Path.")); throw new MissingParametersException(String.Format("Too many parameters in Path.")); } #endregion #region ----- Header parameters ----- List <string> allowedParamNames = psCommand.GetParametersByLocation(RestLocation.Header).Select(x => x.Name.ToLower()).ToList(); /* * Request.Headers.Where(x => allowedParamNames.Contains(x.Key.ToLower())) * .ToList() * .ForEach(x => inParams.Add(new KeyValuePair<string, object>(x.Key, x.Value))); */ foreach (var p in Request.Headers) { var param = allowedParams.FirstOrDefault(x => x.Name.Equals(p.Key, StringComparison.CurrentCultureIgnoreCase)); if (param != null) { var value = Convert.ChangeType(p.Value, param.Type); inParams.Add(new KeyValuePair <string, object>(param.Name, value)); } } #endregion #region ----- ConfigFile parameters ----- foreach (var param in psCommand.GetParametersByLocation(RestLocation.ConfigFile, true)) { inParams.Add(new KeyValuePair <string, object>(param.Name, param.Value)); } #endregion #region ----- User parameter ----- if (!string.IsNullOrWhiteSpace(psCommand.ParameterForUser)) { string userName = this.User.Identity != null ? this.User.Identity.Name : "Anonymous"; inParams.Add(new KeyValuePair <string, object>(psCommand.ParameterForUser, userName)); } #endregion #region ----- Roles parameter ----- if (!string.IsNullOrWhiteSpace(psCommand.ParameterForRoles)) { string[] userRoles = this.User.Identity == null ? new string[0] : (this.User.Identity as ClaimsIdentity) .Claims .Where(c => c.Type == ClaimTypes.Role) .Select(c => c.Value) .ToArray(); inParams.Add(new KeyValuePair <string, object>(psCommand.ParameterForRoles, userRoles)); } #endregion #region ----- Claims parameter ----- if (!string.IsNullOrWhiteSpace(psCommand.ParameterForClaims)) { Claim[] userClaims = this.User.Identity == null ? new Claim[0] : (this.User.Identity as ClaimsIdentity) .Claims .ToArray(); inParams.Add(new KeyValuePair <string, object>(psCommand.ParameterForClaims, userClaims)); } #endregion #region ----- Check if all parameters required are found ----- if (psCommand.GetParametersRequired().Select(x => x.Name).Any(requiedName => inParams.All(q => q.Key != requiedName))) { PowerShellRestApiEvents.Raise.VerboseMessaging(String.Format("Cannot find all parameters required.")); throw new MissingParametersException("Cannot find all parameters required."); } #endregion // We now catch an exception from the runner try { PowerShellRestApiEvents.Raise.VerboseMessaging(String.Format("Started Executing the runner")); if (!asJob) { PowershellReturn output = await _powershellRunner.ExecuteAsync(psCommand.Name, psCommand.Snapin, psCommand.Module, inParams, asJob); JToken token = string.IsNullOrWhiteSpace(output.ActualPowerShellData) ? new JObject() : output.ActualPowerShellData.StartsWith("[") ? (JToken)JArray.Parse(output.ActualPowerShellData) : output.ActualPowerShellData.StartsWith("{") ? JObject.Parse(output.ActualPowerShellData) : JObject.Parse("{\"Message\":" + output.ActualPowerShellData + "}"); JToken token2 = ""; return(new HttpResponseMessage { Content = new JsonContent(token) }); } else // run as job. { Guid jobId = Guid.NewGuid(); string requestedHost = String.Empty; if (Request.Properties.ContainsKey("MS_OwinContext")) { requestedHost = ((OwinContext)Request.Properties["MS_OwinContext"]).Request.RemoteIpAddress; } _jobListProvider.AddRequestedJob(jobId, requestedHost); // Go off and run this job please sir. var task = Task <bool> .Factory.StartNew( () => { try { Task <PowershellReturn> goTask = _powershellRunner.ExecuteAsync( psCommand.Name, psCommand.Snapin, psCommand.Module, inParams, true); goTask.Wait(); var output = goTask.Result; JToken token = output.ActualPowerShellData.StartsWith("[") ? (JToken)JArray.Parse(output.ActualPowerShellData) : JObject.Parse(output.ActualPowerShellData); _jobListProvider.CompleteJob(jobId, output.PowerShellReturnedValidData, String.Empty); string outputPath = Path.Combine(WebApiConfiguration.Instance.Jobs.JobStorePath, jobId + ".json"); using (TextWriter writer = File.CreateText(outputPath)) { JsonSerializer serializer = new JsonSerializer { Formatting = Formatting.Indented // Make it readable for Ops sake! }; serializer.Serialize(writer, token); } return(true); } catch (PowerShellExecutionException poException) { CrashLogEntry entry = new CrashLogEntry { Exceptions = poException.Exceptions, LogTime = poException.LogTime, RequestAddress = requestedHost, RequestMethod = psCommand.Name, RequestUrl = Request.RequestUri.ToString() }; entry.SetActivityId(activityId); string logFile = _crashLogger.SaveLog(entry); PowerShellRestApiEvents.Raise.InvalidPowerShellOutput(poException.Message + " logged to " + logFile); ErrorResponse response = new ErrorResponse { ActivityId = activityId, LogFile = logFile, Message = poException.Message }; JToken token = new JObject(response); _jobListProvider.CompleteJob(jobId, false, String.Empty); string outputPath = Path.Combine(WebApiConfiguration.Instance.Jobs.JobStorePath, jobId + ".json"); using (TextWriter writer = File.CreateText(outputPath)) { JsonSerializer serializer = new JsonSerializer { Formatting = Formatting.Indented // Make it readable for Ops sake! }; serializer.Serialize(writer, token); } return(true); } } ); // return the Job ID. return(new HttpResponseMessage { Content = new JsonContent(new JValue(jobId)) }); } } catch (PowerShellClientException ex) { PowerShellRestApiEvents.Raise.InvalidPowerShellOutput("[" + ex.Category.ToString() + "]" + ex.Message); switch (ex.Category) { case ErrorCategory.PermissionDenied: return(Request.CreateErrorResponse(HttpStatusCode.Forbidden, ex.Message)); case ErrorCategory.ObjectNotFound: return(Request.CreateErrorResponse(HttpStatusCode.NotFound, ex.Message)); case ErrorCategory.InvalidArgument: return(Request.CreateErrorResponse(HttpStatusCode.BadRequest, ex.Message)); case ErrorCategory.ResourceExists: return(Request.CreateErrorResponse(HttpStatusCode.Conflict, ex.Message)); } return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "[" + ex.Category.ToString() + "]" + ex.Message)); } catch (PowerShellExecutionException poException) { string requestedHost = String.Empty; if (Request.Properties.ContainsKey("MS_OwinContext")) { requestedHost = ((OwinContext)Request.Properties["MS_OwinContext"]).Request.RemoteIpAddress; } CrashLogEntry entry = new CrashLogEntry { Exceptions = poException.Exceptions, LogTime = poException.LogTime, RequestAddress = requestedHost, RequestMethod = psCommand.Name, RequestUrl = Request.RequestUri.ToString() }; entry.SetActivityId(activityId); string logFile = _crashLogger.SaveLog(entry); PowerShellRestApiEvents.Raise.InvalidPowerShellOutput(poException.Message + " logged to " + logFile); HttpResponseMessage response = Request.CreateResponse(HttpStatusCode.InternalServerError, new ErrorResponse { ActivityId = activityId, LogFile = logFile, Message = poException.Message } ); return(response); } catch (Exception ex) { CrashLogEntry entry = new CrashLogEntry { Exceptions = new List <PowerShellException> { new PowerShellException { ErrorMessage = ex.Message, LineNumber = 0, ScriptName = "GenericController.cs", StackTrace = ex.StackTrace } }, LogTime = DateTime.Now, RequestAddress = String.Empty, // TODO: Find a way of getting the request host. RequestMethod = psCommand.Name, RequestUrl = Request.RequestUri.ToString() }; entry.SetActivityId(activityId); string logFile = _crashLogger.SaveLog(entry); PowerShellRestApiEvents.Raise.UnhandledException(ex.Message + " logged to " + logFile, ex.StackTrace ?? String.Empty); HttpResponseMessage response = Request.CreateResponse(HttpStatusCode.InternalServerError, new ErrorResponse { ActivityId = activityId, LogFile = logFile, Message = ex.Message } ); return(response); } }