public bool AddShow(ShowBindingModel show)
{
string urlPath = String.Format(Constants.GetSearch, GetSlug(show.Name));
SearchInfoRoot data = WebParser <SearchInfoRoot> .GetInfo(urlPath);
Show newShow = new Show()
{
Id = data.data[0].id,
Title = data.data[0].seriesName,
CurrentEpisode = show.CurrentEpisode,
CurrentSeason = show.CurrentSeason,
Status = show.Status
};
ShowViewModel newView = new ShowViewModel(newShow);
if (!AddEpisodeInfo(newView))
{
if (data.data[0].status != "Continuing")
{
return(false);
}
}
Thread save = new Thread(AddAndSave);
save.Start(newShow);
//Insert the view into the collection
HelperFunctions.PutInTheRightPlace <ShowViewModel>(views, newView);
return(true);
}
public override bool AddShow(ShowBindingModel show)
{
string urlPath = String.Format(Constants.GetSearch, GetSlug(show.Name));
SearchInfoRoot data = WebParser <SearchInfoRoot> .GetInfo(urlPath);
BingeShow newShow = new BingeShow()
{
Id = data.data[0].id,
Title = data.data[0].seriesName,
CurrentEpisode = show.CurrentEpisode,
CurrentSeason = show.CurrentSeason,
};
BingeViewModel newView = new BingeViewModel(newShow);
if (BingeEnd(newView, newShow))
{
return(false);
}
Thread save = new Thread(AddAndSave);
save.Start(newShow);
views.Add(newView);
return(true);
}
public async Task <bool> Login(string userName, string password)
{
var loginPage = await client.GetStringAsync("https://twitter.com/login");
var form = WebParser.Parse(loginPage).Where(f => f.Action == "https://twitter.com/sessions").First();
if (form != null)
{
Dictionary <string, string> postData = new Dictionary <string, string>();
foreach (var input in form.Inputs)
{
postData[input.Name] = input.Value;
}
postData["session[username_or_email]"] = userName;
postData["session[password]"] = password;
postData["redirect_after_login"] = "******";
authenticity_token = postData["authenticity_token"];
HttpContent content = new FormUrlEncodedContent(postData);
var result = await client.PostAsync(form.Action, content);
string response = await result.Content.ReadAsStringAsync();
return(!response.Contains("https://twitter.com/sessions"));
}
return(false);
}
public void ExecuteUnit_ExecuteNotRegisteredUnit_ArgumentExceptionInExceptionProperty()
{
var eventBus = new WebParser <ParsingResult>();
eventBus.RegisterUnit <ModelInA, ModelInB>(new UnitA(eventBus));
eventBus.ExecuteUnit <ModelInB, ModelInC>(new ModelInB());
Assert.AreEqual(typeof(ArgumentException), eventBus.Exception.GetType());
StringAssert.EndsWith(eventBus.Exception.Message, "is not registered");
}
public void RegisterUnit_RegisterSameUnitTwice_ArgumentException()
{
var eventBus = new WebParser <ParsingResult>();
var exception = Assert.ThrowsException <ArgumentException>(() =>
{
eventBus.RegisterUnit <ModelInA, ModelInB>(new UnitA(eventBus));
eventBus.RegisterUnit <ModelInA, ModelInB>(new UnitA(eventBus));
});
StringAssert.EndsWith(exception.Message, "already set");
}
public void SampleOfParserSetupAndExecution()
{
var webParser = new WebParser <ParsingResult>();
webParser.RegisterUnit <ModelInA, ModelInB>(new UnitA(webParser));
webParser.RegisterUnit <ModelInB, ModelInC>(new UnitB(webParser));
webParser.ExecuteUnit <ModelInA, ModelInB>(new ModelInA());
Assert.AreEqual("a", webParser.Result.A);
Assert.AreEqual("b", webParser.Result.B);
Assert.AreEqual("UnitA > UnitB", webParser.ExecutionPath.ToString());
}
public void GetDataTest()
{
string data = new WebParser().GetData("https://petrathecat.github.io/smth");
string rightDataBegin = "<html lang=\"ru\"><head>";
string vitalPart = "https://youtu.be/dQw4w9WgXcQ";
Assert.IsTrue(data.Contains(rightDataBegin));
Assert.IsTrue(data.Contains(vitalPart));
Assert.Pass();
}
public Site911ParsingResult GetEventBusImplementation([FromUri] string id)
{
var httpClient = new HttpClient();
var parser = new WebParser <Site911ParsingResult>();
parser.RegisterUnit <A_SearchModelByIdRequestInput, A_SearchModelByIdRequestOutput>(new A_SearchModelByIdRequest(parser, httpClient));
parser.RegisterUnit <A_SearchModelByIdRequestOutput, B_ParseHtmlOutput>(new B_ParseHtml(parser, new HtmlDocument()));
parser.RegisterUnit <B_ParseHtmlOutput, C_QueryResultOutput>(new C_RequestPartDetails(parser, httpClient));
parser.RegisterUnit <C_QueryResultOutput, Site911ParsingResult>(new D_ParseHtmlPartDetails(parser, new HtmlDocument()));
parser.ExecuteUnit <A_SearchModelByIdRequestInput, A_SearchModelByIdRequestOutput>(new A_SearchModelByIdRequestInput {
Id = id
});
return(parser.Result);
}
private bool AddEpisodeInfo(ShowViewModel model)
{
string urlPath = String.Format(Constants.GetEpisode, model.Id, model.CurrentSeason, model.CurrentEpisode);
EpisodeInfoRoot ep;
try
{
ep = WebParser <EpisodeInfoRoot> .GetInfo(urlPath);
}
catch (Exception)
{
return(false);
}
model.UpdateEpisodeInfo(ep.data);
return(true);
}
private bool IsOngoing(BingeShow chosen)
{
string path = String.Format(Constants.GetSeries, chosen.Id);
try
{
ShowInfoRoot data = WebParser <ShowInfoRoot> .GetInfo(path);
if (data.data.status == "Continuing")
{
return(true);
}
}
catch (Exception)
{
//Unautorized
return(false);
}
return(false);
}
private bool AddEpisodeInfo(BaseViewModel model)
{
string epPath = String.Format(Constants.GetAllEpisodes, model.Id, model.CurrentSeason);
EpisodeInfoRoot ep;
string totalPath = String.Format(Constants.GetTotal, model.Id);
SeasonDataRoot total;
try
{
ep = WebParser <EpisodeInfoRoot> .GetInfo(epPath);
total = WebParser <SeasonDataRoot> .GetInfo(totalPath);
}
catch (HttpRequestException e)
{
if (e.HResult == 404)
{
return(false);
}
return(true);
}
catch (Exception)
{
return(false);
}
try
{
model.UpdateEpisodeInfo(ep.data, total.data);
}
catch (Exception)
{
return(false);
}
return(true);
}
public static string GetPosterSorce(int showId, int showSeason)
{
string srcBase = "https://www.thetvdb.com/banners/";
while (showSeason > 0)
{
string path = String.Format(Constants.GetPoster, showId, showSeason);
try
{
ImageInfoRoot data = WebParser <ImageInfoRoot> .GetInfo(path);
data.data.OrderByDescending(p => p.ratingsInfo.average);
return(srcBase + data.data[0].fileName);
}
catch (Exception)
{
--showSeason;
}
}
return("");
}
/// <summary>
/// Method updates estate objects in database
/// </summary>
public void Update()
{
connection.Open();
WebParser parser = new WebParser();
//try
//{
for (int page = 0; page < parser.GetCountPages(); page++)
{
for (int estateObject = 0; estateObject < 50; estateObject++)
{
string avitoId = parser.GetEstateObjectId(estateObject + 1, 1);
string sourcePage = parser.GetSourcePage(avitoId);
command = new SQLiteCommand("insert into EstateObject (avitoId, dealType, price, locality, address, area, roomNumber, " +
"floor, maxFloor, material, seller, description) values ('" + avitoId + "', " +
"'" + parser.GetDealType(sourcePage) + "', " +
"'" + parser.GetPrice(sourcePage) + "', " +
"'" + parser.GetLocality(sourcePage) + "', " +
"'" + parser.GetAddress(sourcePage) + "', " +
"'" + parser.GetArea(sourcePage) + "', " +
"'" + parser.GetRoomNumber(sourcePage) + "', " +
"'" + parser.GetFloor(sourcePage) + "', " +
"'" + parser.GetMaxFloor(sourcePage) + "', " +
"'" + parser.GetMaterial(sourcePage) + "', " +
"'" + parser.GetSellerName(sourcePage) + "', " +
"'" + parser.GetDescription(sourcePage) + "');", connection);
command.ExecuteNonQuery();
}
}
connection.Close();
//}
//catch (IndexOutOfRangeException)
//{
// return;
//}
}
/// <summary>
/// Gets the program given index number.
/// </summary>
/// <param name="index">The index.</param>
/// <returns>the tv program data</returns>
private ProgramData GetProgram(int index)
{
ProgramData guideData = (ProgramData)_parser.GetData(index);
if (guideData == null ||
guideData.StartTime == null || guideData.Title == string.Empty)
{
return(null);
}
// Set ChannelId
guideData.ChannelId = _strID;
if (_grabber.Actions != null && guideData.IsRemoved(_grabber.Actions))
{
_discarded++;
return(null);
}
//Log.Debug("WebEPG: Guide, Program title: {0}", guideData.Title);
//Log.Debug("WebEPG: Guide, Program start: {0}:{1} - {2}/{3}/{4}", guideData.StartTime.Hour, guideData.StartTime.Minute, guideData.StartTime.Day, guideData.StartTime.Month, guideData.StartTime.Year);
//if (guideData.EndTime != null)
// Log.Debug("WebEPG: Guide, Program end : {0}:{1} - {2}/{3}/{4}", guideData.EndTime.Hour, guideData.EndTime.Minute, guideData.EndTime.Day, guideData.EndTime.Month, guideData.EndTime.Year);
//Log.Debug("WebEPG: Guide, Program desc.: {0}", guideData.Description);
//Log.Debug("WebEPG: Guide, Program genre: {0}", guideData.Genre);
// Adjust Time
if (guideData.StartTime.Day == 0 || guideData.StartTime.Month == 0 || guideData.StartTime.Year == 0)
{
if (!_timeControl.CheckAdjustTime(ref guideData))
{
_discarded++;
return(null);
}
}
//Set TimeZone
guideData.StartTime.TimeZone = _siteTimeZone;
if (guideData.EndTime != null)
{
guideData.EndTime.TimeZone = _siteTimeZone;
Log.Info("WebEPG: Guide, Program Info: {0} / {1} - {2}",
guideData.StartTime.ToLocalLongDateTime(), guideData.EndTime.ToLocalLongDateTime(), guideData.Title);
}
else
{
Log.Info("WebEPG: Guide, Program Info: {0} - {1}", guideData.StartTime.ToLocalLongDateTime(),
guideData.Title);
}
if (guideData.StartTime.ToLocalTime() < _grabStart.AddHours(-2))
{
Log.Info("WebEPG: Program starts in the past, ignoring it.");
_discarded++;
return(null);
}
// Check TV db if program exists
if (_dblookup)
{
Program dbProg = dbProgram(guideData.Title, guideData.StartTime.ToLocalTime());
if (dbProg != null)
{
Log.Info("WebEPG: Program already in db");
Channel chan = Broker.TryRetrieveInstance <Channel>(new Key(true, "ExternalId", _strID));
if (chan != null)
{
ProgramData dbProgramData = new ProgramData();
dbProgramData.InitFromProgram(dbProg);
dbProgramData.ChannelId = _strID;
return(dbProgramData);
}
}
}
// SubLink
if (guideData.HasSublink())
{
if (_parser is WebParser)
{
// added: delay info
Log.Info("WebEPG: SubLink Request {0} - Delay: {1}ms", guideData.SublinkRequest.ToString(),
guideData.SublinkRequest.Delay);
WebParser webParser = (WebParser)_parser;
if (!webParser.GetLinkedData(ref guideData))
{
Log.Info("WebEPG: Getting sublinked data failed");
}
else
{
Log.Debug("WebEPG: Getting sublinked data sucessful");
}
}
}
if (_grabber.Actions != null)
{
guideData.Replace(_grabber.Actions);
}
return(guideData /*.ToTvProgram()*/);
}
public BaseWebRetriever(WebParser <OutType> parser)
{
mWebGetter = new WebGetter();
mParser = parser;
}
public void Getelement_ElementHasContent_ContentOfElement()
{
WebParserResult result = WebParser.GetElementById("http://laboru.co/hello.tx");
Assert.AreEqual(true, result.Status == WebParserResultStatus.Success);
}
private void Generating(object sender, DoWorkEventArgs e)
{
_sw.Reset();
_sw.Start();
_totalRows = 0;
base.Dispatcher.BeginInvoke((Action) delegate
{
GenerateProjectsButton.IsEnabled = false;
});
List <string> ext = new List <string>
{
".txt"
};
IEnumerable <string> enumerable = from s in Directory.GetFiles(_linksPath, "*.*", SearchOption.TopDirectoryOnly)
where ext.Contains(Path.GetExtension(s))
select s;
string[][] validMails = GetValidMails(_emailsFilePath);
int num = 0;
_linksFilePaths = ((enumerable as List <string>) ?? enumerable.ToList());
if (_linksFilePaths.Count > validMails.Length)
{
System.Windows.MessageBox.Show($"Email list contains `{validMails.Length}` unused emails, you trying to generate `{_linksFilePaths.Count}`", "Warning!", MessageBoxButton.OK);
}
else
{
int linesTotal = _linksFilePaths.SelectMany(File.ReadAllLines).Count();
ProcessedLinesProgressBar.Dispatcher.BeginInvoke((Action) delegate
{
ProcessedLinesProgressBar.Maximum = (double)linesTotal;
});
int num2 = 0;
foreach (string linksFilePath in _linksFilePaths)
{
ProjectName = Path.GetFileNameWithoutExtension(linksFilePath).Replace('.', '_') + "_Generated";
string text = GenerateNickName();
string text2 = RandomStringAlpha(11);
string text3 = validMails[num % validMails.Length][0];
string text4 = validMails[num % validMails.Length][1];
string[] array = GenerateCountryCity();
string path = Path.Combine(_outputPath, ProjectName + ".xml");
string text5 = "C:\\Temp\\" + ProjectName + "_URLs.txt";
string[] array2 = File.ReadAllLines(linksFilePath);
string contents = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + Environment.NewLine +
"<XRumerProject>" + Environment.NewLine +
" <PrimarySection>" + Environment.NewLine +
" <ProjectName>" + ProjectName + "</ProjectName>" + Environment.NewLine +
" <ProjectFormat>ANSI</ProjectFormat>" + Environment.NewLine +
" <NickName>" + text + "</NickName>" + Environment.NewLine +
" <RealName>" + text + RandomString(2) + "</RealName>" + Environment.NewLine +
" <Password>" + text2 + "</Password>" + Environment.NewLine +
" <EmailAddress>" + text3 + "</EmailAddress>" + Environment.NewLine +
" <EmailPassword>" + text4 + "</EmailPassword>" + Environment.NewLine +
" <EmailLogin>" + text3 + "</EmailLogin>" + Environment.NewLine +
" <EmailPOP>" + SetEmailPop(text3) + "</EmailPOP>" + Environment.NewLine +
" <Homepage>#file_links_A[\"C:\\Temp\\" + ProjectName + "_URLs.txt\",1,N]</Homepage>" + Environment.NewLine +
$" <ICQ>{_rnd.Next(111111111, 999999999)}</ICQ>" + Environment.NewLine +
" <City>" + array[1] + "</City>" + Environment.NewLine +
" <Country>" + array[0] + "</Country>" + Environment.NewLine +
" <Occupation>" + GenerateOccupation() + "</Occupation>" + Environment.NewLine +
" <Interests>" + GenerateInterests() + "</Interests>" + Environment.NewLine +
" <Signature>#file_links_A[\"" + text5 + "\",1,N]</Signature>" + Environment.NewLine +
" <Gender>0</Gender>" + Environment.NewLine +
" <UnknownFields></UnknownFields>" + Environment.NewLine +
" <PollTitle></PollTitle>" + Environment.NewLine +
" <PollOption1></PollOption1>" + Environment.NewLine +
" <PollOption2></PollOption2>" + Environment.NewLine +
" <PollOption3></PollOption3>" + Environment.NewLine +
" <PollOption4></PollOption4>" + Environment.NewLine +
" <PollOption5></PollOption5>" + Environment.NewLine +
" </PrimarySection>\r\n" +
" <SecondarySection>\r\n" +
" <Subject1>#file_links_A[\"C:\\Temp\\" + ProjectName + "_Titles.txt\",1,N]</Subject1>\r\n" +
" <Subject2>#file_links_A[\"C:\\Temp\\" + ProjectName + "_Titles.txt\",1,N]</Subject2>\r\n" +
" <PostText>#file_links_A[\"C:\\Temp\\" + ProjectName + "_Texts.txt\",1,N]</PostText>\r\n" +
" <Prior>бизнес\r\n" +
"досуг\r\n" +
"объяв\r\n" +
"курилка\r\n" +
"флейм\r\n" +
"флэйм\r\n" +
"основн\r\n" +
"развлеч\r\n" +
"оффтопик\r\n" +
"офтопик\r\n" +
"офф-топик\r\n" +
"прочее\r\n" +
"разное\r\n" +
"обо всём\r\n" +
"flood\r\n" +
"flame\r\n" +
"stuff\r\n" +
"blah\r\n" +
"off-topic\r\n" +
"off topic\r\n" +
"offtopic\r\n" +
"oftopic\r\n" +
"general\r\n" +
"common\r\n" +
"business\r\n" +
"обща\r\n" +
"общий\r\n" +
"общее\r\n" +
"общие\r\n" +
"реклам\r\n" +
"adver</Prior>\r\n" +
" <OnlyPriors>false</OnlyPriors>\r\n" +
" </SecondarySection>\r\n" +
"</XRumerProject>";
if (Regex.IsMatch(array2.First(), "http://.+"))
{
Task <List <string[]> > task = WebParser.AccessTheWebAsync(array2.ToList(), num2, _threads, backgroundWorker);
task.Wait();
List <string[]> result = task.Result;
num2 = (_totalRows = num2 + array2.Length);
backgroundWorker.ReportProgress(num2);
if (result != null && result.Count > 0)
{
string contents2 = string.Join(Environment.NewLine, from l in result
select l[0]);
string contents3 = string.Join(Environment.NewLine, from l in result
select l[1]);
string contents4 = string.Join(Environment.NewLine, result?.Select(PerformText));
File.WriteAllText("C:\\Temp\\" + ProjectName + "_URLs.txt", contents2);
File.WriteAllText("C:\\Temp\\" + ProjectName + "_Texts.txt", contents4);
File.WriteAllText("C:\\Temp\\" + ProjectName + "_Titles.txt", contents3);
File.AppendAllText(GetUsedEmailsFilePath(_emailsFilePath), text3 + Environment.NewLine);
File.WriteAllText(path, contents);
}
}
else
{
try
{
JArray jArray;
if ((jArray = (JsonConvert.DeserializeObject(array2.First()) as JArray)) != null)
{
List <string> values = (from x in jArray.Children()
select((string)x["link"]).Replace("\n", "").Replace("\r", "")).ToList();
List <string> values2 = (from x in jArray.Children()
select((string)x["title"]).Replace("\n", "").Replace("\r", "")).ToList();
List <string> values3 = (from x in jArray.Children()
select PerformText(new string[2]
{
((string)x["link"]).Replace("\n", "").Replace("\r", ""),
((string)x["description"]).Replace("\n", "").Replace("\r", "")
})).ToList();
File.WriteAllText("C:\\Temp\\" + ProjectName + "_URLs.txt", string.Join(Environment.NewLine, values));
using (StreamWriter streamWriter = new StreamWriter(File.Open("C:\\Temp\\" + ProjectName + "_Texts.txt", FileMode.Create), Encoding.GetEncoding("Windows-1251")))
{
streamWriter.Write(string.Join(Environment.NewLine, values3));
}
using (StreamWriter streamWriter2 = new StreamWriter(File.Open("C:\\Temp\\" + ProjectName + "_Titles.txt", FileMode.Create), Encoding.GetEncoding("Windows-1251")))
{
streamWriter2.Write(string.Join(Environment.NewLine, values2));
}
File.AppendAllText(GetUsedEmailsFilePath(_emailsFilePath), text3 + Environment.NewLine);
File.WriteAllText(path, contents);
num2++;
_totalRows += jArray.Count;
backgroundWorker.ReportProgress(num2);
}
}
catch (Exception ex)
{
System.Windows.MessageBox.Show("Fail to deserialize content in file `" + linksFilePath + "` as json " + Environment.NewLine + ex.Message, "Error", MessageBoxButton.OK, MessageBoxImage.Hand);
}
}
num++;
}
}
}
public ParsingService(WebParser parser) : base(parser.GetHttpHandler())
{
Parser = parser;
}
