/// <summary>
/// Iterates through the provided HTML and fixes up image URLs.
/// </summary>
/// <param name="html">The HTML to iterate through.</param>
/// <param name="sourceUrl">The source URL that the HTML originated from.</param>
/// <returns>The fixed up HTML.</returns>
public string FixImageReferences(string html, string sourceUrl)
{
if (html == null)
{
throw new ArgumentNullException("html");
}
if (sourceUrl == null)
{
throw new ArgumentNullException("sourceUrl");
}
StringBuilder sb = new StringBuilder();
using (StringWriter writer = new StringWriter(sb, CultureInfo.InvariantCulture))
{
if (HtmlHandler.IsSharedCanvasTempUrl(sourceUrl))
{
HtmlReferenceFixer fixer = new HtmlReferenceFixer(html);
fixer.FixReferences(writer, _internalReferenceFixer, null);
}
else
{
return(html);
}
}
return(sb.ToString());
}
private async Task<string[]> MakeDlPage(string url) {
HtmlHandler ht = new HtmlHandler();
Helper hlp = new Helper();
await ht.DownloadHtml(url);
//Get RELEASE INFO
var temp = ht.GetNodeByAttr(ht.HtmlDoc,"li","class","release");
string releaseInfo = temp.InnerText.Replace("Release info:","").Trim()??"";
releaseInfo = hlp.FormatTxt(releaseInfo);
//Get Img Src
temp = ht.GetNodeByAttr(ht.HtmlDoc,"img","alt","Poster");
string Imgs = "";
try {
Imgs=ht.GetAttr(temp,"src") ?? "";
}
catch(Exception) { }
//Get Owner
temp = ht.GetNodeByAttr(ht.HtmlDoc,"li","class","author");
temp = ht.GetNodeByType(temp,"a")??null;
string owner = temp.InnerText.Trim()??"";
owner = hlp.FormatTxt(owner);
//Get Details
temp = ht.GetNodeByAttr(ht.HtmlDoc,"div","id","details");
string details = temp.InnerText.Replace("Subtitle details:","").Trim()??"";
details = hlp.FormatTxt(details);
//Get Downloadlink
temp = ht.GetNodeByAttr(ht.HtmlDoc,"a","id","downloadButton");
string dlUrl = ht.GetAttr(temp,"href")??"";
return new string []{Imgs,releaseInfo,owner,details,dlUrl};
}
private void GenerateReportFromHtmlToHtml(string htmlSource, string htmlTarget, Placeholders rep)
{
string html = File.ReadAllText(htmlSource);
html = HtmlHandler.ReplaceAll(html, rep);
File.WriteAllText(htmlTarget, html);
}
public async Task <IActionResult> Export(
[FromServices] BaseHandler json, [FromServices] XlsHandler xls, [FromServices] HtmlHandler html, [FromQuery] string format
)
{
try
{
IActionResult file = null;
switch (format.ToLower())
{
case "xlsx":
case "xls":
file = this.File(await xls.GetExportAsync(this.UserId), "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
break;
case "html":
file = this.File(await html.GetExportAsync(this.UserId), "text/html");
break;
default:
file = this.File(await json.GetExportAsync(this.UserId), "application/json");
break;
}
return(file);
}
catch (Exception e)
{
return(this.Ok(e));
}
}
public string GetFinalSearchresult(string unprocessedHtml, string ean)
{
try
{
HtmlDocument doc = new HtmlDocument();
HtmlDocument innerElement = new HtmlDocument();
doc.LoadHtml(unprocessedHtml);
string techInfo = doc.GetElementbyId("tabelleGroup").InnerHtml;
doc.LoadHtml(techInfo);
var result = doc.DocumentNode.SelectNodes("//a");
foreach (var r in result)
{
var temp = Regex.Replace(r.InnerText, @"\s+", "");
if (!temp.Equals(ean))
{
continue;
}
var link = r.GetAttributeValue("href", string.Empty);
return HtmlHandler.GetUnprocessedHtml("Knipex", "https://www.knipex.com" + link.Replace("&", "&"));
}
return null;
}
catch (Exception e)
{
Console.WriteLine("No final result " + ean);
return null;
}
}
private static void ExecuteLegrandOperations(string excelPath, int startIndex, int endIndex)
{
var wb = ExcelUtilities.CreateWorkbook(excelPath);
var excelSheet = ExcelUtilities.SelectExcelSheet(wb, "Legrand");
var eanDictionary = new Dictionary <string, string>();
ExcelUtilities.ReadFromExcel(eanDictionary, excelSheet, startIndex, endIndex);
var copiedDict = eanDictionary.ToDictionary(entry => entry.Key,
entry => entry.Value);
var fieldsDict = new Dictionary <string, int>();
foreach (var ean in eanDictionary.Keys)
{
var legrand = new Legrand();
var unprocessedHtml = HtmlHandler.GetUnprocessedHtml("Legrand", ean);
List <Dictionary <string, string> > fieldCollections = new List <Dictionary <string, string> >();
try
{
copiedDict = legrand.GetMasterData(unprocessedHtml, ean).ToDictionary(k => k.Key, v => v.Value);
}
catch (Exception)
{
startIndex++;
continue;
}
fieldCollections.Add(legrand.GetProductChars(unprocessedHtml, ean));
fieldCollections.Add(legrand.GetGeneralChars(unprocessedHtml, ean));
fieldCollections.Add(legrand.GetMoreInfo(unprocessedHtml, ean, 0));
fieldCollections.Add(legrand.GetMoreInfo(unprocessedHtml, ean, 1));
fieldCollections.Add(legrand.GetPicture(unprocessedHtml, ean));
copiedDict = fieldCollections
.Where(fieldCollection => fieldCollection != null)
.Aggregate(copiedDict, (current, fieldCollection) => current.Union(fieldCollection).ToDictionary(k => k.Key, v => v.Value));
Console.WriteLine(startIndex + ". " + ean + " downloaded");
charDB.AddDic(copiedDict);
foreach (var entry in copiedDict)
{
var fieldIndex = 0;
if (!fieldsDict.ContainsKey(entry.Key))
{
fieldsDict.Add(entry.Key, 7 + fieldsDict.Count);
}
fieldIndex = fieldsDict[entry.Key];
excelSheet.Cells.WrapText = true;
excelSheet.Cells[1, fieldIndex].Value = entry.Key;
excelSheet.Cells[startIndex, fieldIndex].Value = entry.Value;
}
Console.WriteLine(startIndex + ". " + ean + " written");
startIndex++;
}
wb.Close();
}
private static void registerChangeEvents(DispHTMLDocument document, IHTMLWindow2 window)
{
changeHandler = new HtmlHandler(recordChangeEvent, window);
focusHandler = new HtmlHandler(onFocusEvent, window);
blurHandler = new HtmlHandler(onBlurEvent, window);
Helper.getHelper().addEventListener(document, "change", changeHandler);
Helper.getHelper().addEventListener(document, "focus", focusHandler);
Helper.getHelper().addEventListener(document, "blur", blurHandler);
}
public void HandleTest(string str, string[] arr)
{
var handler = new HtmlHandler();
handler.HandleFragment(str);
var handled = handler.GetCompeleteContent();
Assert.IsTrue(handled.SequenceEqual(arr));
//AreArraysEqual(handled, arr);
}
private void InitCoreLogic()
{
htmlHandler = new HtmlHandler();
xlsHanlder = new ExcelHandler();
//TODO: Remove in production.
if (Debugger.IsAttached)
{
txtBoxToInputExcel.Text = @"C:\Users\artemm\Desktop\EmployeeInfoGrabber\InfoGrabber\bin\Debug\Resources\Input\input.xlsx";
txtBoxToHtmlReport.Text = @"C:\Users\artemm\Desktop\EmployeeInfoGrabber\InfoGrabber\bin\Debug\Resources\Output";
}
}
private void PrintHtml(string templateFile, string printername, Placeholders rep)
{
var htmlContent = File.ReadAllText(templateFile);
htmlContent = HtmlHandler.ReplaceAll(htmlContent, rep);
var tempFileToPrint = Path.ChangeExtension(Path.GetTempFileName(), ".html");
File.WriteAllText(tempFileToPrint, htmlContent);
LibreOfficeWrapper.Print(tempFileToPrint, printername, _locationOfLibreOfficeSoffice);
File.Delete(tempFileToPrint);
}
/// <summary>
/// Windows loaded event handler.
/// </summary>
/// <param name="sender">Sender object.</param>
/// <param name="e">Event argument.</param>
private void WindowsLoaded(object sender, RoutedEventArgs e)
{
this.IsEditBarEnabled = true;
var initialisation = new Initialization(this);
initialisation.Initialize();
this.htmlHandler = new HtmlHandler();
this.htmlHandler.CreateDocument(this.WebBrowserContainer, this.UpdateHtml);
}
private async void Page_Loaded(object sender,RoutedEventArgs e) {
try {
HtmlHandler h = new HtmlHandler();
await h.DownloadHtml("/browse/popular/" + url + "/1");
var tbody = h.GetNodeByType(h.HtmlDoc,"tbody");
ObservableCollection<IModel> ls = new ObservableCollection<IModel>();
Helper hlp = new Helper();
if(tbody != null)
foreach(var tr in tbody.ChildNodes) {
try {
if(tr == null)
continue;
var a1 = h.GetNodeByAttr(tr,"td","class","a1");
if(a1 == null)
continue;
IModel mdl = new IModel();
mdl.Url = h.GetAttr(h.GetNodeByType(a1,"a"),"href").ToString();
var l = h.GetNodeByAttr(a1,"span","class","l r neutral-icon");
mdl.Good =(l==null);
if(mdl.Good)
l = h.GetNodeByAttr(a1,"span","class","l r positive-icon");
mdl.Lang = hlp.FormatTxt(l?.InnerText ?? "");
var YearNode = h.GetNodeByAttr(a1,"div","class","subtle");
var NameNode = YearNode.ParentNode;
mdl.Year = Int32.Parse(YearNode.InnerText.Substring(1,4));
mdl.Name = hlp.FormatTxt(NameNode.InnerText.Replace(YearNode.InnerText,""));
mdl.NumDls = Int32.Parse(hlp.FormatTxt(h.GetNodeByAttr(tr,"td","class","a7")?.InnerText).Replace(",","")?? "0");
mdl.Date = h.GetNodeByAttr(tr,"td","class","a6")?.InnerText ?? "";
mdl.Date = hlp.FormatTxt(mdl.Date);
mdl.Owner = h.GetNodeByAttr(tr,"td","class","a5")?.InnerText??"";
mdl.Owner = hlp.FormatTxt(mdl.Owner);
ls.Add(mdl);
if(ls.Count > 25)
break;
}
catch(Exception) { }
}
GVName.ItemsSource = ls;
#region Fix Xaml
ring.IsActive = false;
ringPanel.Visibility = Visibility.Collapsed;
GVName.Visibility = Visibility.Visible;
#endregion
}
catch(Exception) { Frame.Navigate(typeof(Offline)); }
}
/// <summary>
/// Parse html to elements and adds elements to the Pdf
/// </summary>
/// <param name="html"><see cref="System.String"/>The HTML as string</param>
public Pdf AddHtml(string html)
{
var htmlHandler = new HtmlHandler();
using (var sr = new StringReader(html))
{
XMLWorkerHelper.GetInstance().ParseXHtml(htmlHandler, sr);
}
foreach (var e in htmlHandler.Elements)
{
this._elements.Add(new Builder <IElement>(e));
}
return(this);
}
public void ConternParsing()
{
HtmlHandler parser = new HtmlHandler();
string content;
using (StreamReader reader = new StreamReader("index.html"))
{
content = reader.ReadToEnd();
}
var data = parser.ParseHtml(content);
Assert.IsTrue(data.Length == 32, "Empty parsed data.");
}
public override async Task <object> ParseAsync(CancellationToken cancellationToken)
{
using (var scope = serviceScopeFactory.CreateScope())
{
var raceDB = scope.ServiceProvider.GetService <RaceDB.Models.RaceDBContext>();
var page = await _engine.newPage();
await page.GoToAsync(_settings.Bet365.Url.MainPage.ToString());
var waitOption = new WaitForSelectorOptions
{
Timeout = 30000,
Hidden = true
};
var preLoadOuter = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.PreLoader, waitOption);
waitOption.Hidden = false;
var selectSport = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.Soccer, waitOption);
Thread.Sleep(3000);
await selectSport.ClickAsync();
Thread.Sleep(3000);
var totalCatergoryHtml = await page.GetContentAsync();
var attrs = HtmlHandler.GetImplement("TotalCatergory", totalCatergoryHtml).GetsAttributes(_totalCounrtyFilter);
var values = HtmlHandler.GetImplement("TotalCatergory", totalCatergoryHtml).Gets(_totalCounrtyFilter);
var catergoryData = (from a in attrs
from b in values
where a.Key == b.Key
select new
{
CatergoryKey = a.Value.Where(x => x.Key == "data-sportskey").FirstOrDefault().Value,
CatergoryValue = b.Value
});
var differentDatas = catergoryData.Where(x => raceDB.Category.Any(g => g.CategoryName == x.CatergoryValue && g.CategoryKey != x.CatergoryKey)).ToList();
foreach (var data in differentDatas)
{
var category = raceDB.Category.Where(x => x.CategoryName == data.CatergoryValue).FirstOrDefault();
category.CategoryKey = data.CatergoryKey;
}
await raceDB.SaveChangesAsync();
}
return(null);
}
public void SaveOddsToRedis(Match match, string contents)
{
var rawFullTimeResult = HtmlHandler.GetImplement("Date", contents).Gets(_fullTimeResultFilter);
var date = DateTimeOffset.Now.ToString("yyyyMMdd");
var timeStamp = DateTimeOffset.Now;
var key = $"{date}:{match.MatchId}:odds:1101";
var db = _fact.Connection().GetDatabase();
var redis = new RedisVoteService <BetFieldType>(this._fact);
if (rawFullTimeResult != null)
{
redis.Delete(key);
}
var selection = new Selection();
selection.BetTypeSN = "1101";
selection.BetTypeNM = "Match Result";
var betFields = new List <BetFieldType>();
betFields.Add(new BetFieldType
{
BetFieldTypeSN = "1",
BetFieldTypeNM = "1",
Odds = rawFullTimeResult[0],
TimeStamp = timeStamp
});
betFields.Add(new BetFieldType
{
BetFieldTypeSN = "3",
BetFieldTypeNM = "Draw",
Odds = rawFullTimeResult[1],
TimeStamp = timeStamp
});
betFields.Add(new BetFieldType
{
BetFieldTypeSN = "2",
BetFieldTypeNM = "2",
Odds = rawFullTimeResult[2],
TimeStamp = timeStamp
});
selection.BetFieldList = betFields;
redis.SaveList(key, betFields);
}
private void GetData(object satte)
{
try
{
db.Execute("delete from " + tableName + "");
}
catch { }
string html = null;
string strField = null;
List <List <string> > list = null;
StringBuilder sbSql = new StringBuilder();
for (int page = minPage; page <= maxPage; page++)
{
lblTip.Dispatcher.Invoke(new Action <string>(ShowTip), "当前页码：" + (page + (1 - firstPageIndex)) + "/" + (maxPage + (1 - firstPageIndex)));
html = HtmlHandler.GetHtml(url, get, post, pageParamName, page, pageParamPos == "POST");
list = HtmlHandler.ParseBody(detailUrl, html, bodyRegEx, appendBodyRegEx, detailUrlIndex);
//sbSql.Clear();
foreach (List <string> item in list)
{
strField = "";
foreach (string value in item)
{
strField += "'" + FilterHelper.HtmlFilter(value) + "',";
}
if (item.Count != lstHeader.Count)
{
for (var i = item.Count + 1; i <= lstHeader.Count; i++)
{
strField += "'',";
}
}
//sbSql.Append(string.Format("insert into {0}({1}) values({2});", tableName, strHeader.TrimEnd(','), strField.TrimEnd(',')));
if ((int)db.GetScalar(string.Format("select count(1) from {0} where {1}='{2}'", tableName, key, item[keyIndex])) > 0)
{
continue;
}
db.Execute(string.Format("insert into {0}({1}) values({2});", tableName, strHeader.TrimEnd(','), strField.TrimEnd(',')));
}
//db.Execute(sbSql.ToString());
}
lblTip.Dispatcher.Invoke(new Action <string>(ShowTip), "操作完成");
btnBody.Dispatcher.Invoke(new Action(() => { btnBody.IsEnabled = true; }));
btnConn.Dispatcher.Invoke(new Action(() => { btnConn.IsEnabled = true; }));
btnHeader.Dispatcher.Invoke(new Action(() => { btnHeader.IsEnabled = true; }));
}
/// <summary>
/// Handle the DragEnter event for the presentation editor
/// </summary>
/// <param name="sender">sender</param>
/// <param name="e">event args</param>
protected override void DragEnter(object sender, DragEventArgs e)
{
if (!htmlMarshallingTarget.IsEditable) //disable drop operations when not in edit mode.
{
e.Effect = DragDropEffects.None;
return;
}
DataObjectMeister dataMeister = new DataObjectMeister(e.Data);
if (HtmlHandler.IsPasteFromSharedCanvas(dataMeister) && htmlMarshallingTarget.SelectionIsInvalid)
{
e.Effect = DragDropEffects.None;
return;
}
base.DragEnter(sender, e);
}
public string GetProductUrl(string EAN)
{
string url = "http://www.brennenstuhl.com/index.php?module=products&lang=en&index[products][action]=livesearch&index[products][pattern]=" + EAN;
try
{
string unprocessedHtml = HtmlHandler.GetPageByLink(url);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(unprocessedHtml);
var result = doc.DocumentNode.SelectNodes("//a[@href]").First();
var href = result.Attributes["href"].Value;
return("https://www.brennenstuhl.com/" + href);
}
catch (Exception e)
{
Console.WriteLine("No product found!");
return(null);
}
}
public string GetProductUrl(string EAN)
{
string url = "https://www.knipex.com/en/search/?s=" + EAN;
try
{
string unprocessedHtml = HtmlHandler.GetPageWithForm(url, EAN);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(unprocessedHtml);
string techInfo = doc.GetElementbyId("searchout").InnerHtml;
doc.LoadHtml(techInfo);
var result = doc.DocumentNode.SelectNodes("//a[@href]").First();
var href = result.Attributes["href"].Value;
href = HttpUtility.HtmlDecode(href);
return "http://www.knipex.com" + href;
}
catch (Exception e)
{
Console.WriteLine("No product found " + EAN);
}
return null;
}
public override async Task <object> ParseAsync(CancellationToken cancellationToken)
{
using (var scope = serviceScopeFactory.CreateScope())
{
var raceDB = scope.ServiceProvider.GetService <RaceDB.Models.RaceDBContext>();
var categoryKeys = raceDB.Category.Where(x => x.Status == 1).ToList();
foreach (var key in categoryKeys)
{
Func <XPathHandler, XPathHandler> categoryFilter = element => element.Find(TagEnum.Div, "@class='eventWrapper cc_34_8'");
var page = await _engine.newPage();
await page.GoToAsync(_settings.Bet365.Url.MainPage.ToString());
var waitOption = new WaitForSelectorOptions
{
Timeout = 30000,
Hidden = true
};
var preLoadOuter = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.PreLoader, waitOption);
waitOption.Hidden = false;
var selectSport = await page.WaitForXPathAsync(_settings.Bet365.ElementXpath.Soccer, waitOption);
Thread.Sleep(3000);
await selectSport.ClickAsync();
Thread.Sleep(3000);
var totalLeagueHtml = await page.GetContentAsync();
var html = HtmlHandler.GetImplement("TotalLeague", totalLeagueHtml).Gets(categoryFilter)[0];
var attrs = HtmlHandler.GetImplement("TotalLeague", html).GetsAttributes(_totalLeagueFilter);
var values = HtmlHandler.GetImplement("TotalLeague", html).Gets(_totalLeagueValueFilter);
var leagueData = (from a in attrs
from b in values
where a.Key == b.Key
select new
{
LeagueKey = a.Value.Where(x => x.Key == "data-sportskey").FirstOrDefault().Value,
LeagueValue = b.Value
});
var differentDatas = leagueData.Where(x => raceDB.League.Any(g => g.LeagueName != x.LeagueValue && g.LeagueKey != x.LeagueKey)).ToList();
foreach (var data in differentDatas)
{
var league = raceDB.League.Where(x => x.LeagueName == data.LeagueValue).FirstOrDefault();
if (league != null)
{
league.LeagueKey = data.LeagueKey;
}
else
{
RaceDB.Models.League leagueModel = new RaceDB.Models.League();
leagueModel.CategoryId = key.CategoryId;
leagueModel.LeagueKey = data.LeagueKey;
leagueModel.LeagueName = data.LeagueValue;
raceDB.Add(leagueModel);
}
//.LeagueKey = data.LeagueKey;
}
await raceDB.SaveChangesAsync();
}
}
return(null);
}
private static void registerClickEvents(DispHTMLDocument document, IHTMLWindow2 window)
{
clickHandler = new HtmlHandler(recordClickEvent, window);
Helper.getHelper().addEventListener(document, "click", clickHandler);
}
private static void ExecuteKnipexOperations(string excelPath, int startIndex, int endIndex)
{
var wb = ExcelUtilities.CreateWorkbook(excelPath);
var excelSheet = ExcelUtilities.SelectExcelSheet(wb, "Knipex");
var eanDictionary = new Dictionary <string, string>();
ExcelUtilities.ReadFromExcel(eanDictionary, excelSheet, startIndex, endIndex);
var copiedDict = eanDictionary.ToDictionary(entry => entry.Key,
entry => entry.Value);
var fieldsDict = new Dictionary <string, int>();
foreach (var ean in eanDictionary.Keys)
{
var knipex = new Knipex();
var newEan = ean.Remove(0, 1);
string knipexFullUrl = knipex.GetProductUrl(newEan);
if (string.IsNullOrEmpty(knipexFullUrl))
{
copiedDict[ean] = null;
startIndex++;
continue;
}
var unprocessedHtml = HtmlHandler.GetUnprocessedHtml("Knipex", knipexFullUrl);
var unprocessedHtml2 = knipex.GetFinalSearchresult(unprocessedHtml, newEan);
List <Dictionary <string, string> > fieldCollections = new List <Dictionary <string, string> >();
if (unprocessedHtml2 == null)
{
unprocessedHtml2 = unprocessedHtml;
}
copiedDict = knipex.GetTechData(unprocessedHtml2, ean).ToDictionary(k => k.Key, v => v.Value);
fieldCollections.Add(knipex.GetDescription(unprocessedHtml2, ean));
fieldCollections.Add(knipex.GetPicture(unprocessedHtml2, ean));
//fieldCollections.Add(knipex.GetApplications(unprocessedHtml, ean));
copiedDict = fieldCollections
.Where(fieldCollection => fieldCollection != null)
.Aggregate(copiedDict, (current, fieldCollection) => current.Union(fieldCollection).ToDictionary(k => k.Key, v => v.Value));
Console.WriteLine(startIndex + ". " + ean + " downloaded");
charDB.AddDic(copiedDict);
foreach (var entry in copiedDict)
{
var fieldIndex = 0;
if (!fieldsDict.ContainsKey(entry.Key))
{
fieldsDict.Add(entry.Key, 7 + fieldsDict.Count);
}
fieldIndex = fieldsDict[entry.Key];
excelSheet.Cells.WrapText = true;
excelSheet.Cells[1, fieldIndex].Value = entry.Key;
excelSheet.Cells[startIndex, fieldIndex].Value = entry.Value;
}
Console.WriteLine(startIndex + ". " + ean + " written");
startIndex++;
}
wb.Close();
}
private static void ExecuteBsOperations(string path, int startIndex, int endIndex)
{
var wb = ExcelUtilities.CreateWorkbook(path);
var excelSheet = ExcelUtilities.SelectExcelSheet(wb, "Brennenshtuhl");
var eanDictionary = new Dictionary <string, string>();
ExcelUtilities.ReadFromExcel(eanDictionary, excelSheet, startIndex, endIndex);
var copiedDict = eanDictionary.ToDictionary(entry => entry.Key,
entry => entry.Value);
var fieldsDict = new Dictionary <string, int>();
foreach (var ean in eanDictionary.Keys)
{
var bs = new Brennenstuhl();
//get link
string bsFullUrl = bs.GetProductUrl(ean);
if (string.IsNullOrEmpty(bsFullUrl))
{
copiedDict[ean] = null;
startIndex++;
continue;
}
//get link end
var unprocessedHtml = HtmlHandler.GetUnprocessedHtml("Brennenstuhl", bsFullUrl);
List <Dictionary <string, string> > fieldCollections = new List <Dictionary <string, string> >();
try
{
copiedDict = bs.GetTechData(unprocessedHtml, ean).ToDictionary(k => k.Key, v => v.Value);
}
catch (Exception)
{
startIndex++;
continue;
}
fieldCollections.Add(bs.GetDownloads(unprocessedHtml, ean));
fieldCollections.Add(bs.GetPicture(unprocessedHtml, ean));
copiedDict = fieldCollections
.Where(fieldCollection => fieldCollection != null)
.Aggregate(copiedDict, (current, fieldCollection) => current.Union(fieldCollection).ToDictionary(k => k.Key, v => v.Value));
Console.WriteLine(startIndex + ". " + ean + " downloaded");
charDB.AddDic(copiedDict);
foreach (var entry in copiedDict)
{
var fieldIndex = 0;
if (!fieldsDict.ContainsKey(entry.Key))
{
fieldsDict.Add(entry.Key, 7 + fieldsDict.Count);
}
fieldIndex = fieldsDict[entry.Key];
excelSheet.Cells.WrapText = true;
excelSheet.Cells[1, fieldIndex].Value = entry.Key;
excelSheet.Cells[startIndex, fieldIndex].Value = entry.Value;
}
Console.WriteLine(startIndex + ". " + ean + " written");
startIndex++;
}
wb.Close();
}
private async void Page_Loaded(object sender,RoutedEventArgs e)
{
if (_popularItems.Count > 2) return;
html = new HtmlHandler();
bool offline = false;
try { await html.DownloadHtml(); if(html.HtmlFile == "DIS") offline = true; }
catch(Exception) { offline = true; }
#region If Offline
if(offline) {
ring.IsActive = false;
Frame.Navigate(typeof(Offline));
}
#endregion
#region ONLINE
else {
#region popular
var popDiv = html.GetNodeByAttr(html.HtmlDoc,"div","class","popular-films");
foreach(var node in popDiv.ChildNodes) {
if(node.Name != "div")
continue;
var ul_details = html.GetNodeByAttr(node,"ul","class","details");
foreach(var li in ul_details.ChildNodes) {
if(li.Name != "li")
continue;
var a = html.GetNodeByType(li,"a");
string link = html.GetAttr(a,"href");
string poster = html.GetAttr(html.GetNodeByType(html.GetNodeByAttr(a,"div","class","poster"),"img"),"src").Trim();
var ulli = html.GetNodeByAttr(html.GetNodeByType(html.GetNodeByType(li,"ul"),"li"),"div","class","title");
string name = html.GetNodeByType(ulli,"a").InnerText;
name = name.Replace("'","'");
name = name.Replace(" "," ");
_popularItems.Add(new FilmItem() { Name = name,Url = link,Img = poster });
}
}
#endregion
#region Recent
var recentDiv = html.GetNodeByAttr(html.HtmlDoc,"div","class","recent-subtitles");
var ul = html.GetNodeByType(html.GetNodeByAttr(recentDiv,"div","class","content"),"ul");
foreach(var node in ul.ChildNodes) {
if(node.Name != "li")
continue;
var less = new FilmItem();
var a = html.GetNodeByType(html.GetNodeByAttr(node,"div","class","name"),"a");
less.Url = html.GetAttr(a,"href");
less.Name = a.InnerHtml.Substring(a.InnerHtml.IndexOf("</span>") + 7).Replace(" </a>","").Trim();
less.Name = less.Name.Replace("'","'");
less.Name = less.Name.Replace(" "," ");
less.Lang = html.GetNodeByType(a,"span").InnerText.Trim();
_recentItems.Add(less);
}
#endregion
#region Fix Xaml
ring.IsActive = false;
ringPanel.Visibility = Visibility.Collapsed;
FindName("mainStack");
mainStack.Visibility = Visibility.Visible;
try { HeightHelper.Width = Frame.RenderSize.Width - 28; }
catch(Exception) { }
#endregion
}
#endregion
}
private void btnHeader_Click(object sender, RoutedEventArgs e)
{
btnHeader.IsEnabled = false;
btnConn.IsEnabled = false;
url = txtUrl.Text.Trim();
detailUrl = txtDetailUrl.Text.Trim();
get = txtGET.Text.Trim();
post = txtPOST.Text.Trim();
pageParamName = txtPageParamName.Text.Trim();
pageParamPos = cbbPageParamPos.SelectedItem.ToString();
if (!int.TryParse(txtFirstPageIndex.Text.Trim(), out firstPageIndex))
{
firstPageIndex = 1;
}
bodyRegEx = txtBodyRegEx.Text.Trim();
headerRegEx = txtHeaderRegEx.Text.Trim();
maxPageRegEx = txtMaxPageRegEx.Text.Trim();
appendHeader = txtAppendHeader.Text.Trim();
appendBodyRegEx = txtAppendBodyRegEx.Text.Trim();
key = txtKey.Text.Trim();
tableName = txtTableName.Text.Trim();
string html = HtmlHandler.GetHtml(url, get, post, pageParamName, 1, pageParamPos == "POST");
lstHeader = HtmlHandler.ParseHeader(html, headerRegEx, maxPageRegEx, out minPage, out maxPage);
minPage -= (1 - firstPageIndex);
maxPage -= (1 - firstPageIndex);
if (!string.IsNullOrEmpty(appendHeader))
{
int index = appendHeader.IndexOf('-');
if (index > 0)
{
int.TryParse(appendHeader.Substring(0, index), out detailUrlIndex);
}
if (index + 1 < appendHeader.Length)
{
lstHeader.AddRange(appendHeader.Substring(index + 1).Split(','));
}
}
if (lstHeader.Count <= 0)
{
lblTip.Content = "未获取到表头。";
btnHeader.IsEnabled = true;
btnConn.IsEnabled = true;
return;
}
try
{
db.Execute("drop table " + tableName + "");
}
catch { }
string dataType = connType == "OleDb" ? "text" : "nvarchar(MAX)";
strHeader = "";
StringBuilder sbFields = new StringBuilder();
keyIndex = 0;
for (int index = 0; index < lstHeader.Count; index++)
{
sbFields.Append(string.Format("{0} {1},", lstHeader[index].Trim(), dataType));
strHeader += "" + lstHeader[index].Trim() + ",";
if (lstHeader[index].Trim() == key)
{
keyIndex = index;
}
}
string sql = string.Format(@"create table {0}(
{1}
)", tableName, sbFields.ToString().TrimEnd(','));
try
{
db.Execute(sql);
lblTip.Content = "表头获取成功，表创建成功，共" + (maxPage + (1 - firstPageIndex)) + "页。";
}
catch (Exception ex)
{
lblTip.Content = null == ex.InnerException ? ex.Message : ex.InnerException.Message;
}
btnBody.IsEnabled = true;
btnHeader.IsEnabled = true;
btnConn.IsEnabled = true;
}
internal static string getFrameName(object sender)
{
HtmlHandler htmlHandler = (HtmlHandler)sender;
return(htmlHandler.SourceHTMLWindow.name);
}
internal static IHTMLEventObj getEvent(Object sender)
{
HtmlHandler htmlHandler = (HtmlHandler)sender;
return(htmlHandler.SourceHTMLWindow.@event);
}
private static void registerMouseOut(DispHTMLDocument document, IHTMLWindow2 window)
{
mouseOutHandler = new HtmlHandler(deHighlightOnMouseOut, window);
Helper.getHelper().addEventListener(document, "mouseout", mouseOutHandler);
}
private static void registerContextEvent(DispHTMLDocument document, IHTMLWindow2 window)
{
contextHandler = new HtmlHandler(recordContextEvent, window);
Helper.getHelper().addEventListener(document, "contextmenu", contextHandler);
}
protected virtual bool CanCreateHtmlFormatHandler(DataObjectMeister dataObject)
{
return((editorContext.MarshalHtmlSupported || editorContext.MarshalTextSupported) && HtmlHandler.CanCreateFrom(dataObject));
}
public void CollectingAllReportsTest()
{
HtmlHandler parser = new HtmlHandler();
var data = parser.ReadReportData(".");
}
public override async Task <object> ParseAsync(CancellationToken cancellationToken)
{
using (var scope = serviceScopeFactory.CreateScope())
{
var raceDB = scope.ServiceProvider.GetService <RaceDB.Models.RaceDBContext>();
var leagues = raceDB.League.Where(x => x.Status == 1).ToList();
foreach (var league in leagues)
{
var totalMatchHtml = await GetMatchListAsync(league.LeagueKey);//await _engine.LoadHtml(string.Format(totalMatchUrl, league.LeagueKey.Trim()), JobTimeout);
var rawDate = HtmlHandler.GetImplement("Date", totalMatchHtml).Get(_DateFilter) + " 2018";
var attrs = HtmlHandler.GetImplement("TotalMatchAttrs", totalMatchHtml).GetsAttributes(_totalMatchValueFilter);
var values = HtmlHandler.GetImplement("TotalLeagueValues", totalMatchHtml).Gets(_totalMatchValueFilter);
if (values == null)
{
continue;
}
var matchDataList = (from a in attrs
from b in values
where a.Key == b.Key
where a.Value.Any(x => x.Key == "data-fixtureid")
select new
{
MatchKey = a.Value.Where(x => x.Key == "data-fixtureid").FirstOrDefault().Value,
MatchValue = b.Value
});
var redis = new RedisVoteService <int>(this._fact);
var date = Convert.ToDateTime(rawDate).ToString("yyyyMMdd");
foreach (var match in matchDataList)
{
var matchData = HtmlHandler.GetImplement("matchData", match.MatchValue).Gets(_matchCompetitor);
var matchDate = HtmlHandler.GetImplement("matchDate", match.MatchValue).Get(_matchDate);
var gameStartDate = new DateTimeOffset(Convert.ToDateTime(matchDate + " " + rawDate), new TimeSpan(1, 0, 0));
RaceDB.Models.Match matchModel = new RaceDB.Models.Match();
var existMatch = raceDB.Match.Where(x => x.MatchKey == match.MatchKey && x.StartDateTime == gameStartDate).FirstOrDefault();
if (existMatch != null)
{
matchModel = existMatch;
}
var homeCompetitor = matchData[0];
var awayCompetitor = matchData[1];
matchModel.MatchKey = match.MatchKey;
matchModel.LeagueId = league.LeagueId;
matchModel.CategoryId = league.CategoryId;
matchModel.HomeCompetitorName = homeCompetitor;
matchModel.AwayCompetitorName = awayCompetitor;
matchModel.Status = 2;
matchModel.InPlay = false;
matchModel.SportId = 0;
matchModel.StartDateTime = gameStartDate;
matchModel.CreateDate = DateTimeOffset.Now.ToOffset(new TimeSpan(-4, 0, 0));
matchModel.UpdateDate = DateTimeOffset.Now.ToOffset(new TimeSpan(-4, 0, 0));
matchModel.ResultStatus = 0;
if (existMatch == null)
{
raceDB.Add(matchModel);
}
raceDB.SaveChanges();
}
if (redis.GetList($"{date}:matches") == null)
{
var matchList = raceDB.Match.Where(x => x.StartDateTime.ToString("yyyyMMdd") == date).Select(x => x.MatchId).ToList();
redis.SaveList($"{date}:matches", matchList);
}
}
}
return(null);
}
