private NextStateRec(Type ofType, string name, TokenRef token, string identifier, ParseResponse response)
{
Name = name;
Identifier = identifier;
OfType = ofType;
Token = token;
Sequence = null; // end of list by default
Alternate = null; // no alternate by default
Response = response;
}
/// <summary>
/// информация перед запуском парсера
/// </summary>
/// <returns></returns>
public async Task <ParseResponse> InfoBeforeParse()
{
using (var db = new JoogleContext())
{
var model = new ParseResponse();
model.SitesNotParsed = db.Sites.Where(x => !x.IsDeleted && !x.IsParsed).Count();
model.Finished = false;
return(model);
}
}
private ParseResponse Execute(WebRequest request, HttpStatusCode expectedCode = HttpStatusCode.OK)
{
var done = new ManualResetEvent(false);
var response = new ParseResponse();
request.BeginGetResponse(ar1 =>
{
try
{
var theRequest = (HttpWebRequest)ar1.AsyncState;
HttpWebResponse httpResponse;
try
{
httpResponse = (HttpWebResponse)theRequest.EndGetResponse(ar1);
}
catch (WebException we)
{
// server responses in the range of 4xx and 5xx throw a WebException
httpResponse = (HttpWebResponse)we.Response;
}
response.StatusCode = httpResponse.StatusCode;
try
{
using (var stream = httpResponse.GetResponseStream())
using (var sr = new StreamReader(stream))
{
response.Content = sr.ReadToEnd();
}
}
catch
{
response.Content = null;
}
httpResponse.Dispose();
}
finally
{
done.Set();
}
}, request);
done.WaitOne(Timeout);
return(response);
}
public static SourceStruct Build(SourceFile srcFile)
{
SourceObject start = srcFile.Sequence; // the start object in a sequence of Struct objects to process
SourceObject next = start; // the next object to process
SourceStruct first = null; // the 1st struct built upon this call
SourceStruct previous, build = null;
ParseResponse response = NextStateFunc.GoToNextState(next);
do
{
} while (response != ParseResponse.Null);
return(first);
}
public ActionResult Parser(ParseResponse model)
{
try
{
joogleService.StartParseAllSites(model).GetAwaiter().GetResult();
return(View(model));
}
catch
{
model.Sites = 0;
model.Time = new TimeSpan();
model.Finished = false;
return(View(model));
}
}
public IActionResult Parse([FromBody] string data)
{
var response = new ParseResponse();
try
{
var deserializer = new Parser <Entity>(data);
response.Entity = deserializer.GetObject();
Log.Information("Entity parse: SUCCESS.");
}
catch (ParsingException exception)
{
response.Error = exception;
Log.Information("Entity parse: ERROR.");
}
return(Ok(response));
}
/// <summary>
/// Delete http web request and parse return value to given type.
/// </summary>
/// <typeparam name="T">Result type of response</typeparam>
/// <param name="httpClient">HttpClient to use for web request</param>
/// <param name="requestUri">PUT URI of request</param>
/// <param name="parseResponseMethod">Method to parse string based response to result</param>
/// <returns>Typed result of response</returns>
public static T Delete <T>(this HttpClient httpClient, Uri requestUri, ParseResponse <T> parseResponseMethod)
{
#region validation
if (requestUri == null)
{
throw new ArgumentNullException(nameof(requestUri));
}
if (parseResponseMethod == null)
{
throw new ArgumentNullException(nameof(parseResponseMethod));
}
#endregion
Log.Information($"DELETE HTTP request {requestUri}");
HttpResponseMessage response = httpClient.DeleteAsync(requestUri).Result;
return(ParseResponseMessageAsString <T>(response, parseResponseMethod));
}
public static NextStateRec CreateTokenRef(string name, TokenRef.Type type, ParseResponse response)
{
NextStateRec nsRec;
switch (type)
{
case TokenRef.Type.Null:
throw new FormatException($"SourceObject Type {type.ToString()} is valid BUT NOT in the context as a Next State Token Ref!");
case TokenRef.Type.Empty:
nsRec = new NextStateRec(Type.TokenRef, TokenRef.CreateEmpty(), null, response);
break;
case TokenRef.Type.Identifier:
if (!(name is null))
{
throw new ArgumentNullException(nameof(name), $"NextStateRecords can not contain Identifier Tokens with predefined names -{nameof(name)} must be set to 'null', not '{name}'!");
}
nsRec = new NextStateRec(Type.TokenRef, TokenRef.CreateIdentifier(), response);
break;
case TokenRef.Type.Keyword:
nsRec = new NextStateRec(Type.TokenRef, TokenRef.CreateKeyword(name), name, response);
break;
case TokenRef.Type.Operator:
nsRec = new NextStateRec(Type.TokenRef, TokenRef.CreateOperator(name), name, response);
break;
case TokenRef.Type.Error:
throw new NotImplementedException("Error types yet to be implemented!");
default:
throw InvalidEnumValueException.InvalidArgument(typeof(TokenRef.Type), type, nameof(name));
}
return(nsRec);
}
/// <summary>
/// запуск парсера
/// </summary>
/// <param name="model">модель парсера</param>
/// <returns></returns>
public async Task <ParseResponse> StartParseAllSites(ParseResponse model)
{
using (var db = new JoogleContext())
{
var startTime = DateTime.UtcNow;
var sites = db.Sites.Where(x => !x.IsDeleted && !x.IsParsed).ToList();
int maxConcurrency = 20;
using (SemaphoreSlim concurrencySemaphore = new SemaphoreSlim(maxConcurrency))
{
List <Task> tasks = new List <Task>();
foreach (var site in sites)
{
concurrencySemaphore.Wait();
var t = Task.Factory.StartNew(() =>
{
try
{
SiteParse(site);
}
finally
{
concurrencySemaphore.Release();
}
});
tasks.Add(t);
}
Task.WaitAll(tasks.ToArray());
concurrencySemaphore.Dispose();
}
var endTime = DateTime.UtcNow;
model.Sites = sites.Count;
model.Time = endTime - startTime;
model.Finished = true;
model.SitesNotParsed = db.Sites.Where(x => !x.IsDeleted && !x.IsParsed).Count();
return(model);
}
}
/// <summary>Build the SourceStruct using the SoucreObject sequence from a SourceFile</summary>
/// <param name="lastSrcStruct">Ref: loaded with the last SourceStruct created previously or null if none. Gets set to the last created this call</param>
/// <param name="srcFile">The SourceFile to use for building the SourceStruct from</param>
/// <returns>The 1st SourceStruct created by this call ...</returns>
internal SourceStruct Build(ref SourceStruct lastSrcStruct, SourceFile srcFile)
{
BuildInstr instr = BuildInstr.Continue; // continue until a struct is identified
SourceObject start = srcFile.Sequence; // the start object in a sequence of Struct objects to process
SourceObject next = start; // the next object to process
SourceStruct first = null; // the 1st struct built upon this call
SourceStruct previous, build = null;
NextStateFunc.BeginParse();
while (next != null)
{
ParseResponse response = NextStateFunc.GoToNextState(next);
switch (response)
{
case ParseResponse.Accept:
if (first == null)
{
first = build;
}
break;
case ParseResponse.Resume:
break;
case ParseResponse.Next:
case ParseResponse.Call:
next = next.Sequence;
break;
default:
throw new InvalidEnumValueException(typeof(ParseResponse), response);
}
}
// TODO: add check for source file completion, i.e. nothing left on the stack
return(first);
}
private Task RunClientRequestAsync(TcpClient client)
{
return Task.Run(async () =>
{
try
{
using (client)
{
WriteLine("client connected");
using (NetworkStream stream = client.GetStream())
{
bool completed = false;
do
{
byte[] readBuffer = new byte[1024];
int read = await stream.ReadAsync(readBuffer, 0, readBuffer.Length);
string request = Encoding.ASCII.GetString(readBuffer, 0, read);
WriteLine($"received {request}");
string sessionId;
string result;
byte[] writeBuffer = null;
string response = string.Empty;
ParseResponse resp = ParseRequest(request, out sessionId, out result);
switch (resp)
{
case ParseResponse.OK:
string content = $"{STATUSOK}::{SESSIONID}::{sessionId}";
if (!string.IsNullOrEmpty(result)) content += $"{SEPARATOR}{result}";
response = $"{STATUSOK}{SEPARATOR}{SESSIONID}{SEPARATOR}{sessionId}{SEPARATOR}{content}";
break;
case ParseResponse.CLOSE:
response = $"{STATUSCLOSED}";
completed = true;
break;
case ParseResponse.TIMEOUT:
response = $"{STATUSTIMEOUT}";
break;
case ParseResponse.ERROR:
response = $"{STATUSINVALID}";
break;
default:
break;
}
writeBuffer = Encoding.ASCII.GetBytes(response);
await stream.WriteAsync(writeBuffer, 0, writeBuffer.Length);
await stream.FlushAsync();
WriteLine($"returned {Encoding.ASCII.GetString(writeBuffer, 0, writeBuffer.Length)}");
} while (!completed);
}
}
}
catch (Exception ex)
{
WriteLine($"Exception in client request handling of type {ex.GetType().Name}, Message: {ex.Message}");
}
WriteLine("client disconnected");
});
}
public static NextStateRec CreateTokenRef(TokenRef.Type type, ParseResponse response)
{
return(CreateTokenRef(null, type, response));
}
public static NextStateRec CreateEmpty(ParseResponse response)
{
return(new NextStateRec(Type.TokenRef, "Empty", TokenRef.CreateEmpty(), "&Empty", response));
}
/// <summary>
/// 进行客户端信息的获取
/// </summary>
/// <param name="tcpClient"></param>
/// <returns></returns>
private Task RunClientRequestAsync(TcpClient tcpClient)
{
return(Task.Run(async() =>
{
try
{
using (tcpClient) {
//进行客户端流的获取
using NetworkStream stream = tcpClient.GetStream();
bool completed = false;
do
{
//创建字节数组
byte[] readBuffer = new byte[1024];
//读取流到字节串中
int read = await stream.ReadAsync(readBuffer, 0, readBuffer.Length);
//通过ASCII码转换到字符串
string request = Encoding.ASCII.GetString(readBuffer, 0, read);
//创建写入的字节数组
byte[] writeBuffer = null;
//创建返回的字符串
string response = string.Empty;
//进行返回字符串的解析
ParseResponse resp = ParseRequest(request, out string sessionId, out string result);
//进行返回值的解析
switch (resp)
{
case ParseResponse.OK:
string content = $"{STATUSOK}::{SESSIONID}::{sessionId}";
if (!string.IsNullOrEmpty(result))
{
content += $"{SEPARATOR}{result}";
}
response = $"{STATUSOK}{SEPARATOR}{SESSIONID}{SEPARATOR}{sessionId}{SEPARATOR}{content}";
break;
case ParseResponse.CLOSE:
response = $"{STATUSCLOSED}";
completed = true;
break;
case ParseResponse.TIMEOUT:
response = $"{STATUSTIMEOUT}";
break;
case ParseResponse.ERROR:
response = $"{STATUSINVALID}";
break;
default:
break;
}
//进行返回数据的读取
writeBuffer = Encoding.ASCII.GetBytes(response);
//进行返回数据的写入
await stream.WriteAsync(writeBuffer, 0, writeBuffer.Length);
//进行数据流的刷新
await stream.FlushAsync();
} while (!completed);
}
}
catch (Exception ex)
{
throw ex;
}
}));
}
/// <summary>
/// Parse string based response message to result type.
/// </summary>
/// <typeparam name="T">Type of response result</typeparam>
/// <param name="responseMessage">Response message from request</param>
/// <param name="parseResponseMethod">Method to parse result object</param>
/// <returns>Typed result of response</returns>
private static T ParseResponseMessageAsString <T>(HttpResponseMessage responseMessage, ParseResponse <T> parseResponseMethod)
{
string responseBody = ParseResponseMessage(responseMessage, StringResult.IsEmpty, StringResult.ReadAsString);
Log.Debug($"Response is {responseBody}");
try
{
return(parseResponseMethod(responseBody));
}
catch (Exception exc)
{
Log.Error(exc, $"Parsing of response fails!");
throw new HttpResponseParseException(responseMessage, exc);
}
}
/// <summary>
/// 为了在客户端上读写，TcpClient的Getstream方法返回NetworkStream。
/// 首先需要读取来自客户机的请求。为此，可以使用ReadAsync方法。
/// ReadAsync方法填充一个字节数组。这个字节数组使用Encoding类转换为字符串。
/// 收到的信息写入控制台，传递到ParseRequest辅助方法。
/// 根据ParseRequest方法的结果，创建客户端的回应，使用WriteAsync方法返回给客户端。
/// </summary>
/// <param name="client"></param>
/// <returns></returns>
private Task RunClientRequest(TcpClient client)
{
return(Task.Run(async() =>
{
try
{
using (client)
{
Console.WriteLine("与客户端已建立连接");
using (NetworkStream stream = client.GetStream())
{
bool completed = false;
do
{
byte[] readBuffer = new byte[1024];
int read = await stream.ReadAsync(readBuffer, 0, readBuffer.Length);
string request = Encoding.ASCII.GetString(readBuffer, 0, read);
Console.WriteLine($"已接收{request}");
string sessionId;
string result = string.Empty;
byte[] wiriteBuffer = null;
string response = string.Empty;
ParseResponse resp = ParseRequest(request, out sessionId, out response);
switch (resp)
{
case ParseResponse.OK:
string content = $"{CustomProtocol.STATUSOK}::{CustomProtocol.SESSIONID}::{sessionId}";
if (!string.IsNullOrEmpty(result))
{
content += $"{CustomProtocol.SEPARATOR}{result}";
}
response = $"{CustomProtocol.STATUSOK}{CustomProtocol.SEPARATOR}{CustomProtocol.SESSIONID}{CustomProtocol.SEPARATOR}" +
$"{sessionId}{CustomProtocol.SEPARATOR}{content}";
break;
case ParseResponse.CLOSE:
response = $"{CustomProtocol.STATUSCLOSED}";
completed = true;
break;
case ParseResponse.TIMEOUT:
response = $"{CustomProtocol.STATUSTIMEOUT}";
break;
case ParseResponse.ERROR:
response = $"{CustomProtocol.STATUSINVALID}";
break;
default:
break;
}
wiriteBuffer = Encoding.ASCII.GetBytes(response);
await stream.WriteAsync(wiriteBuffer, 0, wiriteBuffer.Length);
await stream.FlushAsync();
Console.WriteLine($"返回给客户端{Encoding.ASCII.GetString(wiriteBuffer, 0, wiriteBuffer.Length)}");
} while (!completed);
}
}
}
catch (Exception ex)
{
Console.WriteLine($"客户端请求异常 {ex.GetType().Name}，message{ex.Message}");
}
Console.WriteLine("与客户端断开连接");
}));
}
public static NextStateRec CreateGrammaRef(string name, ParseResponse response)
{
return(new NextStateRec(Type.GrammaRef, name, TokenRef.CreateNonToken(), response));
}
public ParseResponse GoToNextState(SourceObject next)
{
ParseResponse response = ParseResponse.Null;
NextStateRec state = CurrentState;
TokenRef token = TokenRef.Create(next.OfType, next.Text);
do
{
Debug.WriteIf(state != null, $"{{{token}}} against {{{state.Token}}}", "Check");
if (state == null)
{
response = ParseResponse.Reject;
Debug.WriteLine(" -Match! Response = Reject");
}
else if (token == state.Token)
{
response = state.Response;
Debug.WriteLine(" -Match! Response = " + response);
}
else if (state.Token.OfType == TokenRef.Type.Empty)
{
response = state.Response;
Debug.WriteLine(" -Match on Empty! Response = " + response);
}
else if (state.Token.OfType == TokenRef.Type.Null)
{
if (state.Response == ParseResponse.Next)
{
state = state.Sequence;
response = ParseResponse.Null;
}
else
{
response = state.Response;
}
Debug.WriteLine(" -Special Response = " + response);
}
else
{
Debug.WriteLine(" -NO Match! Check Next");
state = state.Sequence;
}
} while (response == ParseResponse.Null);
// Determine next state based on response
Debug.Write($"{{{state.Token }}} -> ", "Result");
switch (response)
{
case ParseResponse.Next:
CurrentState = state.Sequence;
break;
default:
throw new InvalidEnumValueException(typeof(ParseResponse), response);
}
Debug.WriteLine($"{{{CurrentState.Token}}} -Response {response}!");
return(response);
}
