private NextStateRec(Type ofType, string name, TokenRef token, string identifier, ParseResponse response) { Name = name; Identifier = identifier; OfType = ofType; Token = token; Sequence = null; // end of list by default Alternate = null; // no alternate by default Response = response; }
/// <summary> /// информация перед запуском парсера /// </summary> /// <returns></returns> public async Task <ParseResponse> InfoBeforeParse() { using (var db = new JoogleContext()) { var model = new ParseResponse(); model.SitesNotParsed = db.Sites.Where(x => !x.IsDeleted && !x.IsParsed).Count(); model.Finished = false; return(model); } }
private ParseResponse Execute(WebRequest request, HttpStatusCode expectedCode = HttpStatusCode.OK) { var done = new ManualResetEvent(false); var response = new ParseResponse(); request.BeginGetResponse(ar1 => { try { var theRequest = (HttpWebRequest)ar1.AsyncState; HttpWebResponse httpResponse; try { httpResponse = (HttpWebResponse)theRequest.EndGetResponse(ar1); } catch (WebException we) { // server responses in the range of 4xx and 5xx throw a WebException httpResponse = (HttpWebResponse)we.Response; } response.StatusCode = httpResponse.StatusCode; try { using (var stream = httpResponse.GetResponseStream()) using (var sr = new StreamReader(stream)) { response.Content = sr.ReadToEnd(); } } catch { response.Content = null; } httpResponse.Dispose(); } finally { done.Set(); } }, request); done.WaitOne(Timeout); return(response); }
public static SourceStruct Build(SourceFile srcFile) { SourceObject start = srcFile.Sequence; // the start object in a sequence of Struct objects to process SourceObject next = start; // the next object to process SourceStruct first = null; // the 1st struct built upon this call SourceStruct previous, build = null; ParseResponse response = NextStateFunc.GoToNextState(next); do { } while (response != ParseResponse.Null); return(first); }
public ActionResult Parser(ParseResponse model) { try { joogleService.StartParseAllSites(model).GetAwaiter().GetResult(); return(View(model)); } catch { model.Sites = 0; model.Time = new TimeSpan(); model.Finished = false; return(View(model)); } }
public IActionResult Parse([FromBody] string data) { var response = new ParseResponse(); try { var deserializer = new Parser <Entity>(data); response.Entity = deserializer.GetObject(); Log.Information("Entity parse: SUCCESS."); } catch (ParsingException exception) { response.Error = exception; Log.Information("Entity parse: ERROR."); } return(Ok(response)); }
/// <summary> /// Delete http web request and parse return value to given type. /// </summary> /// <typeparam name="T">Result type of response</typeparam> /// <param name="httpClient">HttpClient to use for web request</param> /// <param name="requestUri">PUT URI of request</param> /// <param name="parseResponseMethod">Method to parse string based response to result</param> /// <returns>Typed result of response</returns> public static T Delete <T>(this HttpClient httpClient, Uri requestUri, ParseResponse <T> parseResponseMethod) { #region validation if (requestUri == null) { throw new ArgumentNullException(nameof(requestUri)); } if (parseResponseMethod == null) { throw new ArgumentNullException(nameof(parseResponseMethod)); } #endregion Log.Information($"DELETE HTTP request {requestUri}"); HttpResponseMessage response = httpClient.DeleteAsync(requestUri).Result; return(ParseResponseMessageAsString <T>(response, parseResponseMethod)); }
public static NextStateRec CreateTokenRef(string name, TokenRef.Type type, ParseResponse response) { NextStateRec nsRec; switch (type) { case TokenRef.Type.Null: throw new FormatException($"SourceObject Type {type.ToString()} is valid BUT NOT in the context as a Next State Token Ref!"); case TokenRef.Type.Empty: nsRec = new NextStateRec(Type.TokenRef, TokenRef.CreateEmpty(), null, response); break; case TokenRef.Type.Identifier: if (!(name is null)) { throw new ArgumentNullException(nameof(name), $"NextStateRecords can not contain Identifier Tokens with predefined names -{nameof(name)} must be set to 'null', not '{name}'!"); } nsRec = new NextStateRec(Type.TokenRef, TokenRef.CreateIdentifier(), response); break; case TokenRef.Type.Keyword: nsRec = new NextStateRec(Type.TokenRef, TokenRef.CreateKeyword(name), name, response); break; case TokenRef.Type.Operator: nsRec = new NextStateRec(Type.TokenRef, TokenRef.CreateOperator(name), name, response); break; case TokenRef.Type.Error: throw new NotImplementedException("Error types yet to be implemented!"); default: throw InvalidEnumValueException.InvalidArgument(typeof(TokenRef.Type), type, nameof(name)); } return(nsRec); }
/// <summary> /// запуск парсера /// </summary> /// <param name="model">модель парсера</param> /// <returns></returns> public async Task <ParseResponse> StartParseAllSites(ParseResponse model) { using (var db = new JoogleContext()) { var startTime = DateTime.UtcNow; var sites = db.Sites.Where(x => !x.IsDeleted && !x.IsParsed).ToList(); int maxConcurrency = 20; using (SemaphoreSlim concurrencySemaphore = new SemaphoreSlim(maxConcurrency)) { List <Task> tasks = new List <Task>(); foreach (var site in sites) { concurrencySemaphore.Wait(); var t = Task.Factory.StartNew(() => { try { SiteParse(site); } finally { concurrencySemaphore.Release(); } }); tasks.Add(t); } Task.WaitAll(tasks.ToArray()); concurrencySemaphore.Dispose(); } var endTime = DateTime.UtcNow; model.Sites = sites.Count; model.Time = endTime - startTime; model.Finished = true; model.SitesNotParsed = db.Sites.Where(x => !x.IsDeleted && !x.IsParsed).Count(); return(model); } }
/// <summary>Build the SourceStruct using the SoucreObject sequence from a SourceFile</summary> /// <param name="lastSrcStruct">Ref: loaded with the last SourceStruct created previously or null if none. Gets set to the last created this call</param> /// <param name="srcFile">The SourceFile to use for building the SourceStruct from</param> /// <returns>The 1st SourceStruct created by this call ...</returns> internal SourceStruct Build(ref SourceStruct lastSrcStruct, SourceFile srcFile) { BuildInstr instr = BuildInstr.Continue; // continue until a struct is identified SourceObject start = srcFile.Sequence; // the start object in a sequence of Struct objects to process SourceObject next = start; // the next object to process SourceStruct first = null; // the 1st struct built upon this call SourceStruct previous, build = null; NextStateFunc.BeginParse(); while (next != null) { ParseResponse response = NextStateFunc.GoToNextState(next); switch (response) { case ParseResponse.Accept: if (first == null) { first = build; } break; case ParseResponse.Resume: break; case ParseResponse.Next: case ParseResponse.Call: next = next.Sequence; break; default: throw new InvalidEnumValueException(typeof(ParseResponse), response); } } // TODO: add check for source file completion, i.e. nothing left on the stack return(first); }
private Task RunClientRequestAsync(TcpClient client) { return Task.Run(async () => { try { using (client) { WriteLine("client connected"); using (NetworkStream stream = client.GetStream()) { bool completed = false; do { byte[] readBuffer = new byte[1024]; int read = await stream.ReadAsync(readBuffer, 0, readBuffer.Length); string request = Encoding.ASCII.GetString(readBuffer, 0, read); WriteLine($"received {request}"); string sessionId; string result; byte[] writeBuffer = null; string response = string.Empty; ParseResponse resp = ParseRequest(request, out sessionId, out result); switch (resp) { case ParseResponse.OK: string content = $"{STATUSOK}::{SESSIONID}::{sessionId}"; if (!string.IsNullOrEmpty(result)) content += $"{SEPARATOR}{result}"; response = $"{STATUSOK}{SEPARATOR}{SESSIONID}{SEPARATOR}{sessionId}{SEPARATOR}{content}"; break; case ParseResponse.CLOSE: response = $"{STATUSCLOSED}"; completed = true; break; case ParseResponse.TIMEOUT: response = $"{STATUSTIMEOUT}"; break; case ParseResponse.ERROR: response = $"{STATUSINVALID}"; break; default: break; } writeBuffer = Encoding.ASCII.GetBytes(response); await stream.WriteAsync(writeBuffer, 0, writeBuffer.Length); await stream.FlushAsync(); WriteLine($"returned {Encoding.ASCII.GetString(writeBuffer, 0, writeBuffer.Length)}"); } while (!completed); } } } catch (Exception ex) { WriteLine($"Exception in client request handling of type {ex.GetType().Name}, Message: {ex.Message}"); } WriteLine("client disconnected"); }); }
public static NextStateRec CreateTokenRef(TokenRef.Type type, ParseResponse response) { return(CreateTokenRef(null, type, response)); }
public static NextStateRec CreateEmpty(ParseResponse response) { return(new NextStateRec(Type.TokenRef, "Empty", TokenRef.CreateEmpty(), "&Empty", response)); }
/// <summary> /// 进行客户端信息的获取 /// </summary> /// <param name="tcpClient"></param> /// <returns></returns> private Task RunClientRequestAsync(TcpClient tcpClient) { return(Task.Run(async() => { try { using (tcpClient) { //进行客户端流的获取 using NetworkStream stream = tcpClient.GetStream(); bool completed = false; do { //创建字节数组 byte[] readBuffer = new byte[1024]; //读取流到字节串中 int read = await stream.ReadAsync(readBuffer, 0, readBuffer.Length); //通过ASCII码转换到字符串 string request = Encoding.ASCII.GetString(readBuffer, 0, read); //创建写入的字节数组 byte[] writeBuffer = null; //创建返回的字符串 string response = string.Empty; //进行返回字符串的解析 ParseResponse resp = ParseRequest(request, out string sessionId, out string result); //进行返回值的解析 switch (resp) { case ParseResponse.OK: string content = $"{STATUSOK}::{SESSIONID}::{sessionId}"; if (!string.IsNullOrEmpty(result)) { content += $"{SEPARATOR}{result}"; } response = $"{STATUSOK}{SEPARATOR}{SESSIONID}{SEPARATOR}{sessionId}{SEPARATOR}{content}"; break; case ParseResponse.CLOSE: response = $"{STATUSCLOSED}"; completed = true; break; case ParseResponse.TIMEOUT: response = $"{STATUSTIMEOUT}"; break; case ParseResponse.ERROR: response = $"{STATUSINVALID}"; break; default: break; } //进行返回数据的读取 writeBuffer = Encoding.ASCII.GetBytes(response); //进行返回数据的写入 await stream.WriteAsync(writeBuffer, 0, writeBuffer.Length); //进行数据流的刷新 await stream.FlushAsync(); } while (!completed); } } catch (Exception ex) { throw ex; } })); }
/// <summary> /// Parse string based response message to result type. /// </summary> /// <typeparam name="T">Type of response result</typeparam> /// <param name="responseMessage">Response message from request</param> /// <param name="parseResponseMethod">Method to parse result object</param> /// <returns>Typed result of response</returns> private static T ParseResponseMessageAsString <T>(HttpResponseMessage responseMessage, ParseResponse <T> parseResponseMethod) { string responseBody = ParseResponseMessage(responseMessage, StringResult.IsEmpty, StringResult.ReadAsString); Log.Debug($"Response is {responseBody}"); try { return(parseResponseMethod(responseBody)); } catch (Exception exc) { Log.Error(exc, $"Parsing of response fails!"); throw new HttpResponseParseException(responseMessage, exc); } }
/// <summary> /// 为了在客户端上读写,TcpClient的Getstream方法返回NetworkStream。 /// 首先需要读取来自客户机的请求。为此,可以使用ReadAsync方法。 /// ReadAsync方法填充一个字节数组。这个字节数组使用Encoding类转换为字符串。 /// 收到的信息写入控制台,传递到ParseRequest辅助方法。 /// 根据ParseRequest方法的结果,创建客户端的回应,使用WriteAsync方法返回给客户端。 /// </summary> /// <param name="client"></param> /// <returns></returns> private Task RunClientRequest(TcpClient client) { return(Task.Run(async() => { try { using (client) { Console.WriteLine("与客户端已建立连接"); using (NetworkStream stream = client.GetStream()) { bool completed = false; do { byte[] readBuffer = new byte[1024]; int read = await stream.ReadAsync(readBuffer, 0, readBuffer.Length); string request = Encoding.ASCII.GetString(readBuffer, 0, read); Console.WriteLine($"已接收{request}"); string sessionId; string result = string.Empty; byte[] wiriteBuffer = null; string response = string.Empty; ParseResponse resp = ParseRequest(request, out sessionId, out response); switch (resp) { case ParseResponse.OK: string content = $"{CustomProtocol.STATUSOK}::{CustomProtocol.SESSIONID}::{sessionId}"; if (!string.IsNullOrEmpty(result)) { content += $"{CustomProtocol.SEPARATOR}{result}"; } response = $"{CustomProtocol.STATUSOK}{CustomProtocol.SEPARATOR}{CustomProtocol.SESSIONID}{CustomProtocol.SEPARATOR}" + $"{sessionId}{CustomProtocol.SEPARATOR}{content}"; break; case ParseResponse.CLOSE: response = $"{CustomProtocol.STATUSCLOSED}"; completed = true; break; case ParseResponse.TIMEOUT: response = $"{CustomProtocol.STATUSTIMEOUT}"; break; case ParseResponse.ERROR: response = $"{CustomProtocol.STATUSINVALID}"; break; default: break; } wiriteBuffer = Encoding.ASCII.GetBytes(response); await stream.WriteAsync(wiriteBuffer, 0, wiriteBuffer.Length); await stream.FlushAsync(); Console.WriteLine($"返回给客户端{Encoding.ASCII.GetString(wiriteBuffer, 0, wiriteBuffer.Length)}"); } while (!completed); } } } catch (Exception ex) { Console.WriteLine($"客户端请求异常 {ex.GetType().Name},message{ex.Message}"); } Console.WriteLine("与客户端断开连接"); })); }
public static NextStateRec CreateGrammaRef(string name, ParseResponse response) { return(new NextStateRec(Type.GrammaRef, name, TokenRef.CreateNonToken(), response)); }
public ParseResponse GoToNextState(SourceObject next) { ParseResponse response = ParseResponse.Null; NextStateRec state = CurrentState; TokenRef token = TokenRef.Create(next.OfType, next.Text); do { Debug.WriteIf(state != null, $"{{{token}}} against {{{state.Token}}}", "Check"); if (state == null) { response = ParseResponse.Reject; Debug.WriteLine(" -Match! Response = Reject"); } else if (token == state.Token) { response = state.Response; Debug.WriteLine(" -Match! Response = " + response); } else if (state.Token.OfType == TokenRef.Type.Empty) { response = state.Response; Debug.WriteLine(" -Match on Empty! Response = " + response); } else if (state.Token.OfType == TokenRef.Type.Null) { if (state.Response == ParseResponse.Next) { state = state.Sequence; response = ParseResponse.Null; } else { response = state.Response; } Debug.WriteLine(" -Special Response = " + response); } else { Debug.WriteLine(" -NO Match! Check Next"); state = state.Sequence; } } while (response == ParseResponse.Null); // Determine next state based on response Debug.Write($"{{{state.Token }}} -> ", "Result"); switch (response) { case ParseResponse.Next: CurrentState = state.Sequence; break; default: throw new InvalidEnumValueException(typeof(ParseResponse), response); } Debug.WriteLine($"{{{CurrentState.Token}}} -Response {response}!"); return(response); }