public threadMsg(IAPI api, EventHandler <threadMsgEventArgs> on_message = null)
{
onMessageComplete = on_message;
_api = api;
_resetEvent = new ManualResetEvent(false);
_threadEvent = new ManualResetEvent(false);
_thread = new Thread(new ParameterizedThreadStart(delegate(object evt)
{
api.Init();
api.Open = true;
app.postToAPI(_API.MEDIA, _API.MEDIA_KEY_INITED, null);
threadMsgPara tm = (threadMsgPara)evt;
while (_exit == false)
{
tm.ResetEvent.WaitOne();
if (_exit)
{
break;
}
else
{
msg m = api.Execute(_msg);
//if (onMessageComplete != null) onMessageComplete.Invoke(this, new threadMsgEventArgs(m));
}
tm.ResetEvent.Reset();
}
}));
_thread.Start(new threadMsgPara(_resetEvent));
}
public msg Execute(msg m)
{
if (m == null || m.Input == null)
{
return(m);
}
bool hasUpdate = false;
switch (m.KEY)
{
case _API.SETTING_APP_KEY_UPDATE_FOLDER:
#region
string fol = (string)m.Input;
if (!string.IsNullOrEmpty(fol))
{
fol = fol.ToLower().Trim();
lock (_lock)
{
if (_setting.list_folder.IndexOf(fol) == -1)
{
_setting.list_folder.Add(fol);
hasUpdate = true;
app.postMessageToService(new msg()
{
API = _API.FOLDER_ANYLCTIC, Input = fol
});
}
}
}
#endregion
break;
case _API.SETTING_APP_KEY_UPDATE_NODE_OPENING:
oNode node = (oNode)m.Input;
lock (_lock)
_setting.node_opening = node.id;
hasUpdate = true;
break;
case _API.SETTING_APP_KEY_UPDATE_SIZE:
oAppSize app_size = (oAppSize)m.Input;
lock (_lock)
{
_setting.app_size = app_size;
hasUpdate = true;
}
break;
}
//if (hasUpdate)
//{
// using (var file = File.Create(file_name))
// {
// Serializer.Serialize<oSetting>(file, _setting);
// }
//}
m.Output.Ok = hasUpdate;
m.Output.Data = hasUpdate;
return(m);
}
public msg clone(object input)
{
msg m = Serializer.DeepClone <msg>(this);
m.Input = input;
return(m);
}
public void response_toMainRuntime(msg m)
{
if (fom == null)
{
fom = app.get_Main();
}
if (fom != null)
{
fom.api_responseMsg(null, new threadMsgEventArgs(m));
}
}
public api_base()
{
if (timer_api == null)
{
timer_api = new System.Threading.Timer(new System.Threading.TimerCallback((obj) =>
{
if (fom == null)
{
fom = app.get_Main();
}
lock (_lock_api)
{
if (cache_api.Count > 0)
{
msg m = null;
if (fom != null &&
cache_api.TryDequeue(out m) && m != null)
{
fom.api_responseMsg(null, new threadMsgEventArgs(m));
}
}
}
}), fom, 100, 100);
}
if (timer_msg == null)
{
timer_msg = new System.Threading.Timer(new System.Threading.TimerCallback((obj) =>
{
if (fom == null)
{
fom = app.get_Main();
}
lock (_lock_msg)
{
if (cache_msg.Count > 0)
{
msg m = null;
if (fom != null &&
cache_msg.TryDequeue(out m) && m != null)
{
fom.api_responseMsg(null, new threadMsgEventArgs(m));
}
}
}
}), fom, 500, 500);
}
}
msg f_CRAWLER_KEY_REQUEST_LINK(msg m)
{
if (CRAWLER_KEY_STOP)
{
f_CRAWLER_KEY_STOP_reset();
}
string[] urls = new string[] { };
string[] uri_ok = dicHtml.Keys.ToArray();
lock (_lockUrl)
{
listUrl = listUrl.Where(x => !uri_ok.Any(o => o == x)).Distinct().ToList();
urls = listUrl.Take(crawlMaxThread).ToArray();
listUrl = listUrl.Where(x => !urls.Any(o => o == x)).ToList();
Interlocked.Exchange(ref crawlPending, listUrl.Count);
Interlocked.Exchange(ref crawlCounter, urls.Length);
}
//if (Interlocked.CompareExchange(ref crawlPending, 0, 0) == 0)
if (Interlocked.CompareExchange(ref crawlCounter, 0, 0) == 0)
{
string[] rs_out = dicHtml.Keys.ToArray();
response_toMain(new msg()
{
API = _API.CRAWLER, KEY = _API.CRAWLER_KEY_REQUEST_LINK, Log = "Crawle complete result: " + rs_out.Length + " links. Writing file ..."
});
if (Interlocked.CompareExchange(ref crawlResult, 1, 1) > 1)
{
write_file_contentHTML();
}
response_toMain(new msg()
{
API = _API.CRAWLER, KEY = _API.CRAWLER_KEY_REQUEST_LINK_COMPLETE, Input = rs_out
});
return(m);
}
else
{
for (int i = 0; i < urls.Length; i++)
{
tasks[i].RunWorkerAsync(urls[i]);
}
}
return(m);
}
public static string postMessageToService(msg m)
{
if (m == null)
{
return(string.Empty);
}
if (m.KEY == string.Empty)
{
m.KEY = Guid.NewGuid().ToString();
}
if (dicService.ContainsKey(m.API))
{
dicService[m.API].Execute(m);
}
return(m.KEY);
}
public msg Execute(msg m)
{
if (m == null || Open == false)
{
return(m);
}
switch (m.KEY)
{
case _API.WORD_KEY_INITED:
break;
case _API.WORD_KEY_ANALYTIC:
#region
break;
#endregion
}
return(m);
}
msg f_CRAWLER_KEY_STOP(msg m)
{
CRAWLER_KEY_STOP = true;
string[] rs_out = dicHtml.Keys.ToArray();
response_toMain(new msg()
{
API = _API.CRAWLER, KEY = _API.CRAWLER_KEY_REQUEST_LINK, Log = "Crawle complete result: " + rs_out.Length + " links. Writing file ..."
});
if (Interlocked.CompareExchange(ref crawlResult, 1, 1) > 1)
{
write_file_contentHTML();
}
response_toMain(new msg()
{
API = _API.CRAWLER, KEY = _API.CRAWLER_KEY_REQUEST_LINK_COMPLETE, Input = rs_out
});
return(m);
}
public msg Execute(msg m)
{
if (m == null || m.Input == null)
{
return(m);
}
string path = (string)m.Input;
if (Directory.Exists(path))
{
m.KEY = string.Empty;
m.Output.Ok = false;
}
m.Output.Ok = true;
//m.Output.Data = wo.words;
return(m);
}
public msg Execute(msg m)
{
if (m == null || m.Input == null)
{
return(m);
}
oWordContent wo = (oWordContent)m.Input;
m.KEY = string.Empty;
m.Output.Ok = false;
string[] a_con = wo.words.Select(x => x.word.ToLower()).Distinct().ToArray();
string[] a_local = new string[] { };
lock (_lock)
a_local = dicWord.Keys.ToArray();
a_local = a_local.Where(x => a_con.Any(o => o == x)).ToArray();
string[] a_down;
if (a_local.Length == 0)
{
a_down = a_con;
}
else
{
a_down = a_local.Where(x => !a_con.Any(o => o == x)).ToArray();
}
if (a_down.Length > 0)
{
app.postMessageToService(new msg()
{
API = _API.WORD_DOWNLOAD, Input = a_down
});
}
m.Output.Ok = true;
m.Output.Data = a_con;
return(m);
}
public threadMsg(IAPI api, EventHandler <threadMsgEventArgs> on_message = null)
{
onMessageComplete = on_message;
_api = api;
_resetEvent = new ManualResetEvent(false);
_threadEvent = new ManualResetEvent(false);
_thread = new Thread(new ParameterizedThreadStart(delegate(object evt)
{
threadMsgPara tm = (threadMsgPara)evt;
while (_exit == false)
{
tm.ResetEvent.WaitOne();
if (_exit == false)
{
msg m = api.Execute(_msg);
//if (onMessageComplete != null) onMessageComplete.Invoke(this, new threadMsgEventArgs(m));
}
tm.ResetEvent.Reset();
}
}));
_thread.Start(new threadMsgPara(_resetEvent));
}
private static void f_api_init()
{
api_msg_queue = new ConcurrentQueue <msg>();
if (api_msg_timer == null)
{
api_msg_timer = new System.Threading.Timer(new System.Threading.TimerCallback((obj) =>
{
if (api_msg_queue.Count > 0)
{
msg m = null;
if (api_msg_queue.TryDequeue(out m) && m != null)
{
if (!string.IsNullOrEmpty(m.API) && dicService.ContainsKey(m.API))
{
IthreadMsg sv;
if (dicService.TryGetValue(m.API, out sv) && sv != null)
{
////new Thread(new ParameterizedThreadStart((object _sv) =>
////{
//// IthreadMsg so = (IthreadMsg)_sv;
//// so.Execute(m);
////})).Start(sv);
sv.Execute(m);
}
}
}
}
}), null, 100, 100);
}
dicResponses = new ConcurrentDictionary <string, msg>();
dicService = new ConcurrentDictionary <string, IthreadMsg>();
dicService.TryAdd(_API.CRAWLER, new threadMsg(new api_crawler()));
//dicService.TryAdd(_API.pro, new threadMsg(new api_pronunce()));
dicService.TryAdd(_API.WORD, new threadMsg(new api_word()));
dicService.TryAdd(_API.MEDIA, new threadMsg(new api_media()));
}
public msg Execute(msg m)
{
if (m == null || m.Input == null)
{
return(m);
}
//HttpWebRequest w = (HttpWebRequest)WebRequest.Create(new Uri(url));
//w.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36";
//w.BeginGetResponse(asyncResult =>
//{
// HttpWebResponse rs = (HttpWebResponse)w.EndGetResponse(asyncResult); //add a break point here
// StreamReader sr = new StreamReader(rs.GetResponseStream(), System.Text.Encoding.UTF8);
// s = sr.ReadToEnd();
// sr.Close();
// rs.Close();
// s = HttpUtility.HtmlDecode(s);
//}, w);
m.Output.Ok = true;
m.Output.Data = null;
return(m);
}
public void Execute(msg msg)
{
_msg = msg;
_resetEvent.Set();
}
msg f_CRAWLER_KEY_REGISTER_PATH(msg m)
{
CRAWLER_KEY_STOP = false;
domain_current = string.Empty;
setting_URL_CONTIANS = string.Empty;
setting_PARA1 = string.Empty;
setting_PARA2 = string.Empty;
if (m.Input != null)
{
Interlocked.Exchange(ref crawlResult, 0);
oLinkSetting st = (oLinkSetting)m.Input;
string para_url = st.Url;
if (st.Settings != null && st.Settings.Count > 0)
{
st.Settings.TryGetValue("URL_CONTIANS", out setting_URL_CONTIANS);
st.Settings.TryGetValue("PARA1", out setting_PARA1);
st.Settings.TryGetValue("PARA2", out setting_PARA2);
}
string[] a = para_url.Split('/');
domain_current = a[2].ToLower();
if (domain_current.StartsWith("www."))
{
domain_current = domain_current.Substring(4);
}
if (a.Length > 3)
{
url_sub_path_current = a[3];
}
dicHtml.Clear();
listUrl.Clear();
read_file_contentHTML();
HttpWebRequest w = (HttpWebRequest)WebRequest.Create(new Uri(para_url));
w.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36";
w.BeginGetResponse(asyncResult =>
{
HttpWebResponse rs = (HttpWebResponse)w.EndGetResponse(asyncResult); //add a break point here
string url = rs.ResponseUri.ToString();
response_toMain(new msg()
{
API = _API.CRAWLER, KEY = _API.CRAWLER_KEY_REQUEST_LINK, Log = url
});
if (rs.StatusCode == HttpStatusCode.OK)
{
string htm = string.Empty;
StreamReader sr = new StreamReader(rs.GetResponseStream(), Encoding.UTF8);
htm = sr.ReadToEnd();
sr.Close();
rs.Close();
if (!string.IsNullOrEmpty(htm))
{
htm = HttpUtility.HtmlDecode(htm);
htm = format_HTML(htm);
if (!dicHtml.ContainsKey(url))
{
dicHtml.TryAdd(url, htm);
Interlocked.Increment(ref crawlResult);
}
var us = get_Urls(url, htm);
if (CRAWLER_KEY_STOP)
{
f_CRAWLER_KEY_STOP_reset();
return;
}
if (us.Url_Html.Length > 0)
{
lock (_lockUrl)
listUrl.AddRange(us.Url_Html);
Execute(new msg()
{
API = _API.CRAWLER, KEY = _API.CRAWLER_KEY_REQUEST_LINK
});
}
else
{
Execute(new msg()
{
API = _API.CRAWLER, KEY = _API.CRAWLER_KEY_REQUEST_LINK_COMPLETE, Input = dicHtml.Keys.ToArray()
});
}
}
}
}, w);
}
return(m);
}
public msg Execute(msg m)
{
if (m == null || m.Input == null)
{
return(m);
}
string[] words = (string[])m.Input;
Encoding encoding = Encoding.UTF7;
string input = string.Join("\r\n", words);
m.KEY = input;
m.Output.Ok = false;
////string temp = HttpUtility.UrlEncode(input.Replace(" ", "---"));
//string temp = HttpUtility.UrlEncode(input);
////temp = temp.Replace("-----", "%20");
//string url = String.Format("http://www.google.com/translate_t?hl=en&ie=UTF8&text={0}&langpair={1}", temp, "en|vi");
//string s = String.Empty;
//using (WebClient webClient = new WebClient())
//{
// webClient.Encoding = encoding;
// s = webClient.DownloadString(url);
//}
//string ht = HttpUtility.HtmlDecode(s);
//string result = String.Empty;
//int p = s.IndexOf("id=result_box");
//if (p > 0)
// s = "<span " + s.Substring(p, s.Length - p);
//p = s.IndexOf("</div>");
//if (p > 0)
//{
// s = s.Substring(0, p);
// s = s.Replace("<br>", "¦");
// s = HttpUtility.HtmlDecode(s);
// result = Regex.Replace(s, @"<[^>]*>", String.Empty);
//}
//if (result != string.Empty)
//{
// string[] rs = result.Split('¦').Select(x => x.Trim()).ToArray();
// m.Output = new MsgOutput() { Ok = true, Data = rs, Total = rs.Length };
//
// lock (lockResponse)
// {
// if (dicResponses.ContainsKey(m.Key))
// dicResponses[m.Key] = m;
// else
// dicResponses.Add(m.Key, m);
// }
//}
//else
//{
// m.Output = new MsgOutput() { Data = "Can not translate" };
//}
//postMessageToFormUI(m.Key);
m.Output.Ok = true;
m.Output.Data = words;
return(m);
}
public msg Execute(msg msg)
{
return(msg);
}
public void api_initMsg(msg m)
{
}
public void response_toMain(msg m)
{
lock (_lock_api) cache_api.Enqueue(m);
}
public msg Execute(msg m)
{
if (m == null)
{
return(m);
}
switch (m.KEY)
{
case _API.CRAWLER_KEY_STOP:
f_CRAWLER_KEY_STOP(m);
break;
case _API.CRAWLER_KEY_REGISTER_PATH:
f_CRAWLER_KEY_REGISTER_PATH(m);
break;
case _API.CRAWLER_KEY_REQUEST_LINK:
f_CRAWLER_KEY_REQUEST_LINK(m);
break;
case _API.CRAWLER_KEY_CONVERT_PACKAGE_TO_HTML:
#region
////path_package = (string)m.Input;
////if (!string.IsNullOrEmpty(path_package) && File.Exists(path_package))
////{
//// //var dicRaw = new Dictionary<string, string>();
//// //var dicCon = new Dictionary<string, string>();
//// //var list_XPath = new List<string>();
//// //using (var fileStream = File.OpenRead(path_package))
//// // dicRaw = Serializer.Deserialize<Dictionary<string, string>>(fileStream);
//// ////foreach (var kv in dicRaw)
//// ////{
//// //// string s = kv.Value;
//// //// doc = new HtmlDocument();
//// //// doc.LoadHtml(s);
//// //// foreach (var h1 in doc.DocumentNode.SelectNodes("//h1"))
//// //// {
//// //// //d1.Add(kv.Key, h1.ParentNode.InnerText);
//// //// //d2.Add(kv.Key, h1.ParentNode.ParentNode.InnerText);
//// //// //d3.Add(kv.Key, h1.ParentNode.ParentNode.ParentNode.InnerText);
//// //// list_XPath.Add(h1.XPath);
//// //// break;
//// //// }
//// ////}
//// //foreach (var kv in dicRaw)
//// //{
//// // string s = kv.Value, si = string.Empty;
//// // doc = new HtmlDocument();
//// // doc.LoadHtml(s);
//// // var ns = doc.DocumentNode.SelectNodes("/html[1]/body[1]/div[3]/article[1]/div[1]/div[1]/div[1]/div[1]/article[1]");
//// // if (ns != null && ns.Count > 0)
//// // {
//// // si = ns[0].InnerHtml;
//// // dicCon.Add(kv.Key, si);
//// // }
//// //}
//// //using (var file = File.Create("crawler.htm.bin"))
//// // Serializer.Serialize<Dictionary<string, string>>(file, dicCon);
////}
#endregion
break;
case _API.CRAWLER_KEY_CONVERT_PACKAGE_TO_TEXT:
#region
////path_package = (string)m.Input;
////if (!string.IsNullOrEmpty(path_package) && File.Exists(path_package))
////{
//// var dicRaw = new Dictionary<string, string>();
//// var dicText = new Dictionary<string, string>();
//// using (var fileStream = File.OpenRead(path_package))
//// dicRaw = Serializer.Deserialize<Dictionary<string, string>>(fileStream);
//// foreach (var kv in dicRaw)
//// {
//// string s = new htmlToText().ConvertHtml(kv.Value).Trim();
//// dicText.Add(kv.Key, s);
//// }
//// using (var file = File.Create("crawler.txt.bin"))
//// Serializer.Serialize<Dictionary<string, string>>(file, dicText);
////}
#endregion
break;
}
m.Output.Ok = true;
m.Output.Data = null;
return(m);
}
public void notification_toMain(msg m)
{
lock (_lock_msg) cache_msg.Enqueue(m);
}
public msg Execute(msg m)
{
if (m == null || m.Input == null)
{
return(m);
}
string[] a = (string[])m.Input;
m.KEY = string.Empty;
m.Output.Ok = false;
////////https://s3.amazonaws.com/audio.oxforddictionaries.com/en/mp3/hello_gb_1.mp3
////////https://s3.amazonaws.com/audio.oxforddictionaries.com/en/mp3/you_gb_1.mp3
////////https://ssl.gstatic.com/dictionary/static/sounds/oxford/you--_gb_1.mp3
////////https://ssl.gstatic.com/dictionary/static/sounds/20160317/you--_gb_1.mp3
////////https://ssl.gstatic.com/dictionary/static/sounds/20160317/hello--_us_1.mp3
////////https://ssl.gstatic.com/dictionary/static/sounds/20160317/you--_us_1.mp3
////////https://ssl.gstatic.com/dictionary/static/sounds/20160317/ok--_us_1.mp3
////////https://ssl.gstatic.com/dictionary/static/sounds/20160317/you--_gb_1.mp3
////////https://ssl.gstatic.com/dictionary/static/sounds/20160317/ok--_gb_1.mp3
//////if (!NetworkInterface.GetIsNetworkAvailable())
//////{
////// // Network does not available.
//////}
//////var ping = new System.Net.NetworkInformation.Ping();
//////var result = ping.Send("www.google.com");
//////if (result.Status == System.Net.NetworkInformation.IPStatus.Success)
//////{
//////}
//////////string temp = HttpUtility.UrlEncode(input.Replace(" ", "---"));
////////string temp = HttpUtility.UrlEncode(input);
//////////temp = temp.Replace("-----", "%20");
////////string url = String.Format("http://www.google.com/translate_t?hl=en&ie=UTF8&text={0}&langpair={1}", temp, "en|vi");
////////string s = String.Empty;
////////using (WebClient webClient = new WebClient())
////////{
//////// webClient.Encoding = encoding;
//////// s = webClient.DownloadString(url);
////////}
////////string ht = HttpUtility.HtmlDecode(s);
////////string result = String.Empty;
////////int p = s.IndexOf("id=result_box");
////////if (p > 0)
//////// s = "<span " + s.Substring(p, s.Length - p);
////////p = s.IndexOf("</div>");
////////if (p > 0)
////////{
//////// s = s.Substring(0, p);
//////// s = s.Replace("<br>", "¦");
//////// s = HttpUtility.HtmlDecode(s);
//////// result = Regex.Replace(s, @"<[^>]*>", String.Empty);
////////}
////////if (result != string.Empty)
////////{
//////// string[] rs = result.Split('¦').Select(x => x.Trim()).ToArray();
//////// m.Output = new MsgOutput() { Ok = true, Data = rs, Total = rs.Length };
////////
//////// lock (lockResponse)
//////// {
//////// if (dicResponses.ContainsKey(m.Key))
//////// dicResponses[m.Key] = m;
//////// else
//////// dicResponses.Add(m.Key, m);
//////// }
////////}
////////else
////////{
//////// m.Output = new MsgOutput() { Data = "Can not translate" };
////////}
////////postMessageToFormUI(m.Key);
m.Output.Ok = true;
//m.Output.Data = wo.words;
return(m);
}
public msg Execute(msg m)
{
return(m);
}
public threadMsgEventArgs(msg msg)
{
Message = msg;
}
public static void postToAPI(msg m)
{
api_msg_queue.Enqueue(m);
}
