private void RegisterBufferMenuItem_Click(object sender, RoutedEventArgs e)
{
Task.Run(() =>
{
int indexProxy = 0;
for (int i = 0; i < myDataContext.EmailsIua.Count; i++)
{
if (indexProxy > myDataContext.FreeProxys.Count)
{
indexProxy = 0;
}
FreeProxy p = myDataContext.FreeProxys[indexProxy];
RegistrationIua r = new RegistrationIua(p.Ip, p.Port);
bool registered = false;
try
{
registered = r.OpenRegistration(myDataContext.EmailsIua[i]);
}
catch
{
r.CloseDriver();
}
if (!registered)
{
i--;
indexProxy++;
}
}
});
//RegistrationIua r = new RegistrationIua();
//r.RegistrationContainer(myDataContext.EmailsIua);
}
protected override void ServiceActionCallback()
{
if (FreeProxies == null)
{
FreeProxies = new Queue <FreeProxy>();
}
FreeProxy freeProxy = null;
if (ConsumeSelf && FreeProxies.Any())
{
freeProxy = FreeProxies.ToArray()[0];
}
_pgCounter += 1;
var hostPgUrlPattern = MakeHostPageUrlPattern(HostPageUrlPattern, _pgCounter);
if (string.IsNullOrEmpty(hostPgUrlPattern))
{
throw new InvalidOperationException("Failed to parse HostPgUrlPattern to produce valid url.");
}
try
{
var contentDoc = LoadIPProxyDocumentContent(hostPgUrlPattern, freeProxy);
if (!string.IsNullOrEmpty(contentDoc))
{
var freeProxies = LoadUpIPProxies(contentDoc, (prxy) =>
{
FreeProxies.Enqueue(prxy);
InvokeEventFreeIPProxyFetched(new EventHandlers.FreeIPProxyFetchedEventArgs(prxy));
});
if (freeProxies == null || !freeProxies.Any())
{
return;
}
//FreeProxies = FreeProxies.Concat(freeProxies).ToList();
InvokeEventFreeIPProxiesFetched(new EventHandlers.FreeIPProxiesFetchedEventArgs(freeProxies.ToList()));
}
}
catch (Exception ex)
{
if (freeProxy != null)
{
lock (FreeProxies)
RemoveFreeProxy(freeProxy);
}
_pgCounter--;
return;
}
if (_pgCounter == HostPageMax)
{
_pgCounter = 0;
}
}
/// <summary>
///
/// </summary>
/// <param name="hostPgUrlPattern"></param>
/// <param name="freeProxy"></param>
/// <returns></returns>
protected string LoadIPProxyDocumentContent(string hostPgUrlPattern, FreeProxy freeProxy = null)
{
/*
* REFERENCES:
* other approach:
* //http://stackoverflow.com/questions/18921099/add-proxy-to-phantomjsdriver-selenium-c
* PhantomJSOptions phoptions = new PhantomJSOptions();
* phoptions.AddAdditionalCapability(CapabilityType.Proxy, "http://localhost:9999");
* driver = new PhantomJSDriver(phoptions);
*
* - (24/04/2019) https://stackoverflow.com/a/52446115/3796898
*/
try
{
//var options = new ChromeOptions();
//var userAgent = "user_agent_string";
//options.AddArgument("--user-agent=" + userAgent);
//IWebDriver driver = new ChromeDriver(options);
var service = PhantomJSDriverService.CreateDefaultService();
service.HideCommandPromptWindow = true;
if (freeProxy != null)
{
service.ProxyType = freeProxy.Protocol == ProxyProtocolsEnum.HTTP ? "http" : "https:";
var proxy = new Proxy
{
HttpProxy = $"{freeProxy.IPAddress}:{freeProxy.PortNo}",
};
service.Proxy = proxy.HttpProxy;
}
_driver = new PhantomJSDriver(service)
{
Url = hostPgUrlPattern
};
_driver.Navigate();
// The driver can now provide you with what we
// need (it will execute the script)
// get the source of the page
var content = _driver.PageSource;
_driver.Quit();
return(content);
}
catch (Exception ex)
{
throw;
}
}
/// <summary>
///
/// </summary>
/// <returns></returns>
public FreeProxy GetFreeProxy(bool onlyValid = false)
{
FreeProxy exFreeProxy = null;
var ok = false;
if (FreeProxies == null)
{
throw new Exception("FreeProxies is null");
}
while (!ok)
{
try
{
if (RotationId == Guid.Empty | (FreeProxies.Count(p => p.RotationTokenId != RotationId) == 0))
{
RotationId = Guid.NewGuid();
}
FreeProxy[] proxies = null;
proxies = onlyValid
? FreeProxies.ToArray().Where(p => p.RotationTokenId != RotationId && p.Valid).ToArray()
: FreeProxies.ToArray().Where(p => p.RotationTokenId != RotationId).ToArray();
if (proxies.Any())
{
exFreeProxy = proxies[0];
if (exFreeProxy != null)
{
exFreeProxy.RotationTokenId = RotationId;
}
}
ok = true; // _exFreeProxy != null && (_exFreeProxy.Valid && onlyValid);
}
catch
{
throw;
}
}
return(exFreeProxy);
}
/// <summary>
///
/// </summary>
/// <param name="hmaPgDoc"></param>
private int ExtractProxies(string hmaPgDoc)
{
var rowCount = 0;
var doc = new HtmlDocument();
doc.LoadHtml(hmaPgDoc);
var tblNode = HtmlUtil.GetNodeByAttribute(doc.DocumentNode, "table", "id", "listtable");
if (tblNode == null)
{
return(rowCount);
}
var tRowNodes = tblNode.Descendants("tr");
var htmlNodes = tRowNodes as HtmlNode[] ?? tRowNodes.ToArray();
rowCount = htmlNodes.Count();
if (tRowNodes == null || rowCount <= 0)
{
return(rowCount);
}
for (var ctr = 1; ctr < htmlNodes.Count(); ctr++)
{
var protocolOk = true;
var anonymityLevelOk = true;
var speedOk = true;
var connectionTimeOk = true;
FreeProxy freeProxy = null;
try
{
if (ParseRowToProxy(htmlNodes.ToArray()[ctr], ref freeProxy))
{
if (freeProxy != null)
{
if (_protocolOptions != null && _protocolOptions.Count() > 0)
{
protocolOk = _protocolOptions.Any(x => x != freeProxy.Protocol);
}
else if (Protocol.HasValue)
{
protocolOk = (Protocol.Value == freeProxy.Protocol);
}
if (AnonymityLevel.HasValue)
{
if (AnonymityLevel.Value == ProxyAnonymityLevelEnum.High__KA)
{
anonymityLevelOk = freeProxy.AnonymityLevel == ProxyAnonymityLevelEnum.High__KA;
}
else if (AnonymityLevel.Value == ProxyAnonymityLevelEnum.PlanetLab)
{
anonymityLevelOk = freeProxy.AnonymityLevel == ProxyAnonymityLevelEnum.PlanetLab;
}
else
{
if (AnonymityLevel.Value == ProxyAnonymityLevelEnum.High)
{
anonymityLevelOk = freeProxy.AnonymityLevel == ProxyAnonymityLevelEnum.High;
}
else if (AnonymityLevel.Value == ProxyAnonymityLevelEnum.Medium)
{
anonymityLevelOk = freeProxy.AnonymityLevel == ProxyAnonymityLevelEnum.High |
freeProxy.AnonymityLevel == ProxyAnonymityLevelEnum.Medium;
}
else if (AnonymityLevel.Value == ProxyAnonymityLevelEnum.Low)
{
anonymityLevelOk = freeProxy.AnonymityLevel == ProxyAnonymityLevelEnum.High |
freeProxy.AnonymityLevel == ProxyAnonymityLevelEnum.Medium |
freeProxy.AnonymityLevel == ProxyAnonymityLevelEnum.Low;
}
else if (AnonymityLevel.Value == ProxyAnonymityLevelEnum.None)
{
anonymityLevelOk = freeProxy.AnonymityLevel == ProxyAnonymityLevelEnum.High |
freeProxy.AnonymityLevel == ProxyAnonymityLevelEnum.Medium |
freeProxy.AnonymityLevel == ProxyAnonymityLevelEnum.Low |
freeProxy.AnonymityLevel == ProxyAnonymityLevelEnum.None;
}
}
}
if (Speed.HasValue)
{
if (Speed.Value == ProxySpeedEnum.Fast)
{
speedOk = freeProxy.Speed == ProxySpeedEnum.Fast;
}
else if (Speed.Value == ProxySpeedEnum.Medium)
{
speedOk = freeProxy.Speed == ProxySpeedEnum.Medium | freeProxy.Speed == ProxySpeedEnum.Fast;
}
else
{
speedOk = true;
}
}
if (ConnectionTime.HasValue)
{
if (ConnectionTime.Value == ProxyConnectionSpeedEnum.Fast)
{
connectionTimeOk = freeProxy.ConnectionTime == ProxyConnectionSpeedEnum.Fast;
}
else if (ConnectionTime.Value == ProxyConnectionSpeedEnum.Medium)
{
connectionTimeOk = freeProxy.ConnectionTime == ProxyConnectionSpeedEnum.Medium | freeProxy.ConnectionTime == ProxyConnectionSpeedEnum.Fast;
}
else
{
connectionTimeOk = true;
}
}
}
}
if (protocolOk && anonymityLevelOk && speedOk && connectionTimeOk)
{
var exFreeProxy = FreeProxies
.ToArray().FirstOrDefault(p => p.HostIP == freeProxy.HostIP && p.PortNo == freeProxy.PortNo);
if (exFreeProxy == null)
{
lock (FreeProxies)
{
FreeProxies.Add(freeProxy);
}
InvokeEventFreeIPProxyFetched(new EventHandlers.FreeIPProxyFetchedEventArgs(freeProxy));
}
}
}
catch (Exception ex)
{
throw;
}
}
return(rowCount);
}
/// <summary>
///
/// </summary>
/// <param name="rowNode"></param>
/// <param name="freeProxy"></param>
/// <returns></returns>
private bool ParseRowToProxy(HtmlNode rowNode, ref FreeProxy freeProxy)
{
try
{
var prxCells = rowNode.Descendants("td");
var htmlNodes = prxCells as HtmlNode[] ?? prxCells.ToArray();
if (prxCells != null && htmlNodes.Any())
{
freeProxy = new FreeProxy();
freeProxy.HostIP = SnatchIPAddress(htmlNodes.ToArray()[1].InnerHtml);
freeProxy.PortNo = int.Parse(htmlNodes.ToArray()[2].InnerText);
var country = htmlNodes.ToArray()[3].InnerText.Trim().ToUpper();
country = country.Replace("; ", "___")
.Replace(", ", "__")
.Replace(" ", "_")
.Replace("'", "____");
freeProxy.Country = (ProxyCountry)Enum.Parse(typeof(ProxyCountry), country);
var connspeedNode = htmlNodes.ToArray()[5];
connspeedNode = HtmlUtil.GetNodeByAttribute(connspeedNode, "div", "class", "speedbar connection_time");
connspeedNode = connspeedNode.Descendants("div").ToArray()[0];
var rate = connspeedNode.Attributes["style"].Value.Replace("width:", "").Replace("%", "");
freeProxy.ResponseRate = int.Parse(rate);
if (freeProxy.ResponseRate <= 35)
{
freeProxy.ConnectionTime = ProxyConnectionSpeedEnum.Slow;
}
else if (freeProxy.ResponseRate > 35 && freeProxy.ResponseRate <= 65)
{
freeProxy.ConnectionTime = ProxyConnectionSpeedEnum.Medium;
}
else if (freeProxy.ResponseRate > 65)
{
freeProxy.ConnectionTime = ProxyConnectionSpeedEnum.Medium;
}
var speedNode = htmlNodes.ToArray()[4];
speedNode = HtmlUtil.GetNodeByAttribute(speedNode, "div", "class", "speedbar response_time");
speedNode = speedNode.Descendants("div").ToArray()[0];
rate = speedNode.Attributes["style"].Value.Replace("width:", "").Replace("%", "");
freeProxy.SpeedRate = int.Parse(rate);
if (freeProxy.SpeedRate <= 35)
{
freeProxy.Speed = ProxySpeedEnum.Slow;
}
else if (freeProxy.SpeedRate > 35 && freeProxy.SpeedRate <= 65)
{
freeProxy.Speed = ProxySpeedEnum.Medium;
}
else if (freeProxy.SpeedRate > 65)
{
freeProxy.Speed = ProxySpeedEnum.Medium;
}
var protocol = htmlNodes.ToArray()[6].InnerText.Replace("/", "_");
freeProxy.Protocol = (ProxyProtocolEnum)Enum.Parse(typeof(ProxyProtocolEnum), protocol);
var anonymity = htmlNodes.ToArray()[7].InnerText.Replace(" +", "__");
freeProxy.AnonymityLevel = (ProxyAnonymityLevelEnum)Enum.Parse(typeof(ProxyAnonymityLevelEnum), anonymity);
freeProxy.Valid = true;
return(true);
}
}
catch (Exception ex)
{
throw;
}
return(false);
}
/// <summary>
///
/// </summary>
/// <param name="hmaPgDoc"></param>
private int ExtractProxies(string hmaPgDoc)
{
int _rowCount = 0;
HtmlAgilityPack.HtmlDocument _doc = new HtmlAgilityPack.HtmlDocument();
_doc.LoadHtml(hmaPgDoc);
var _tblNode = HtmlUtil.GetNode(_doc.DocumentNode, "table", "id", "listtable");
if (_tblNode != null)
{
var _tRowNodes = _tblNode.Descendants("tr");
_rowCount = _tRowNodes.Count();
if (_tRowNodes != null && _rowCount > 0)
{
Guid _usageId = Guid.NewGuid();
for (int ctr = 1; ctr < _tRowNodes.Count(); ctr++)
{
bool _ProtocolOK = true;
bool _AnonymityLevelOK = true;
bool _SpeedOK = true;
bool _ConnectionTimeOK = true;
FreeProxy _freeProxy = null;
try
{
if (ParseRowToProxy(_tRowNodes.ToArray()[ctr], ref _freeProxy))
{
if (_freeProxy != null)
{
_freeProxy.RotationTokenId = _usageId;
if (this.Protocol.HasValue)
{
_ProtocolOK = (this.Protocol.Value == _freeProxy.Protocol);
}
if (this.AnonymityLevel.HasValue)
{
_AnonymityLevelOK = this.AnonymityLevel.Value == _freeProxy.AnonymityLevel;
}
if (this.Speed.HasValue)
{
if (this.Speed.Value == ProxySpeedEnum.Fast)
{
_SpeedOK = _freeProxy.Speed == ProxySpeedEnum.Fast;
}
else if (this.Speed.Value == ProxySpeedEnum.Medium)
{
_SpeedOK = _freeProxy.Speed == ProxySpeedEnum.Medium | _freeProxy.Speed == ProxySpeedEnum.Fast;
}
else
{
_SpeedOK = true;
}
}
if (this.ConnectionTime.HasValue)
{
if (this.ConnectionTime.Value == ProxyConnectionSpeedEnum.Fast)
{
_ConnectionTimeOK = _freeProxy.ConnectionTime == ProxyConnectionSpeedEnum.Fast;
}
else if (this.ConnectionTime.Value == ProxyConnectionSpeedEnum.Medium)
{
_ConnectionTimeOK = _freeProxy.ConnectionTime == ProxyConnectionSpeedEnum.Medium | _freeProxy.ConnectionTime == ProxyConnectionSpeedEnum.Fast;
}
else
{
_ConnectionTimeOK = true;
}
}
}
}
if (_ProtocolOK && _AnonymityLevelOK && _SpeedOK && _ConnectionTimeOK)
{
this.InvokeEventFreeIPProxyFetched(new EventHandlers.FreeIPProxyFetchedEventArgs(_freeProxy));
}
}
catch (Exception ex)
{
//Requested value 'LEBANON' was not found.
}
}
}
}
return(_rowCount);
}
/// <summary>
///
/// </summary>
/// <param name="rowNode"></param>
/// <param name="freeProxy"></param>
/// <returns></returns>
private bool ParseRowToProxy(HtmlNode rowNode, ref FreeProxy freeProxy)
{
try
{
var _prxCells = rowNode.Descendants("td");
if (_prxCells != null && _prxCells.Count() > 0)
{
freeProxy = new FreeProxy();
freeProxy.Host = SnatchIPAddress(_prxCells.ToArray()[1].InnerHtml);
freeProxy.Port = int.Parse(_prxCells.ToArray()[2].InnerText);
string _country = _prxCells.ToArray()[3].InnerText.Trim().ToUpper();
_country = _country.Replace("; ", "___").Replace(", ", "__").Replace(" ", "_");
freeProxy.Country = (ProxyCountry)Enum.Parse(typeof(ProxyCountry), _country);
var _connspeedNode = _prxCells.ToArray()[5];
_connspeedNode = HtmlUtil.GetNode(_connspeedNode, "div", "class", "speedbar connection_time");
_connspeedNode = _connspeedNode.Descendants("div").ToArray()[0];
string _rate = _connspeedNode.Attributes["style"].Value.Replace("width:", "").Replace("%", "");
freeProxy.ConnectionTimeRate = int.Parse(_rate);
if (freeProxy.ConnectionTimeRate <= 35)
{
freeProxy.ConnectionTime = ProxyConnectionSpeedEnum.Slow;
}
else if (freeProxy.ConnectionTimeRate > 35 && freeProxy.ConnectionTimeRate <= 65)
{
freeProxy.ConnectionTime = ProxyConnectionSpeedEnum.Medium;
}
else if (freeProxy.ConnectionTimeRate > 65)
{
freeProxy.ConnectionTime = ProxyConnectionSpeedEnum.Medium;
}
var _speedNode = _prxCells.ToArray()[4];
_speedNode = HtmlUtil.GetNode(_speedNode, "div", "class", "speedbar response_time");
_speedNode = _speedNode.Descendants("div").ToArray()[0];
_rate = _speedNode.Attributes["style"].Value.Replace("width:", "").Replace("%", "");
freeProxy.SpeedRate = int.Parse(_rate);
if (freeProxy.SpeedRate <= 35)
{
freeProxy.Speed = ProxySpeedEnum.Slow;
}
else if (freeProxy.SpeedRate > 35 && freeProxy.SpeedRate <= 65)
{
freeProxy.Speed = ProxySpeedEnum.Medium;
}
else if (freeProxy.SpeedRate > 65)
{
freeProxy.Speed = ProxySpeedEnum.Medium;
}
string _protocol = _prxCells.ToArray()[6].InnerText.Replace("/", "_");
freeProxy.Protocol = (ProxyProtocolEnum)Enum.Parse(typeof(ProxyProtocolEnum), _protocol);
string _anonymity = _prxCells.ToArray()[7].InnerText.Replace(" +", "__");
freeProxy.AnonymityLevel = (ProxyAnonymityLevelEnum)Enum.Parse(typeof(ProxyAnonymityLevelEnum), _anonymity);
return(true);
}
}
catch (Exception ex)
{
throw ex;
}
return(false);
}
/// <summary>
///
/// </summary>
/// <param name="pageContentDocument"></param>
/// <param name="freeProxyFetched"></param>
/// <returns></returns>
public override Queue <FreeProxy> LoadUpIPProxies(string pageContentDocument, Action <FreeProxy> freeProxyFetched)
{
Queue <FreeProxy> freeProxies = null;
try
{
var doc = new HtmlDocument();
doc.LoadHtml(pageContentDocument);
var document = doc.DocumentNode;
var target = HtmlUtil.GetNodeByAttribute(document, "table", "id", "proxylist");
if (target == null)
{
return(null);
}
var lines = HtmlUtil.GetNodeCollection(target, "tr");
lines.ToList().ForEach(e =>
{
if (freeProxies == null)
{
freeProxies = new Queue <FreeProxy>();
}
if (e.Descendants("td").Count() <= 1)
{
return;
}
try
{
var proxy = new FreeProxy();
var cells = e.Descendants("td").ToArray();
var unwanted = cells[0].Descendants("script");
var enumerable = unwanted as HtmlNode[] ?? unwanted.ToArray();
if (unwanted != null && enumerable.ToArray().Any())
{
enumerable.ToArray()[0].Remove();
}
var address = cells[0];
var addressParts = address.InnerText.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
proxy.IPAddress = addressParts[0];
proxy.PortNo = int.Parse(addressParts[1]);
// get anonymity level
//proxy.AnonymityLevel = cells[1].InnerText.Contains("high")
// ? ProxyAnonymityLevelEnum.High
// : ProxyAnonymityLevelEnum.Medium;
proxy.AnonymityLevel = cells[1].InnerText;
// get last checked time
var checkdate = cells[2].InnerText;
//todo:
//proxy.LastValidationCheck = DateTime.Parse(checkdate);
// get the country
var countryPartial = cells[3].InnerText.ToLower().Replace(" ", "_");
countryPartial = countryPartial.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)[0];
var pxycountry = FindProxyCountryFromPartial(countryPartial);
proxy.Country = pxycountry;
//if (!ProxyTestHelper.CanPing(string.Format("{0}://{1}:{2}", proxy.Protocol == ProxyProtocolEnum.HTTP ? "http" : "https", proxy.HostIP, proxy.PortNo)))
//if (!ProxyTestHelper.ProxyIsGood(proxy.HostIP, proxy.PortNo)) return;
freeProxies.Enqueue(proxy);
freeProxyFetched(proxy);
}
catch (Exception ex)
{
throw;
}
});
}
catch (Exception ex)
{
throw;
}
return(freeProxies);
}
/// <summary>
///
/// </summary>
/// <param name="pageContentDocument"></param>
/// <param name="freeProxyFetched"></param>
/// <returns></returns>
public override Queue <FreeProxy> LoadUpIPProxies(string pageContentDocument, Action <FreeProxy> freeProxyFetched)
{
Queue <FreeProxy> freeProxies = null;
try
{
var doc = new HtmlDocument();
doc.LoadHtml(pageContentDocument);
var document = doc.DocumentNode;
var target = HtmlUtil.GetNodeByAttribute(document, "table", "class", "proxytbl");
if (target == null)
{
return(null);
}
var lines = HtmlUtil.GetNodeCollection(target, "tr");
lines.ToList().ForEach(e =>
{
if (freeProxies == null)
{
freeProxies = new Queue <FreeProxy>();
}
if (e.Descendants("td").Count() <= 1)
{
return;
}
var proxy = new FreeProxy();
var cells = e.Descendants("td").ToArray();
//ip
var ip = cells[0].InnerText;
//port
var scriptPart = cells[1].Descendants("script");
var htmlNodes = scriptPart as HtmlNode[] ?? scriptPart.ToArray();
if (htmlNodes.Any())
{
htmlNodes.ToArray()[0].Remove();
}
var port = HtmlUtil.Resolve(cells[1].InnerText);
// country
var country = HtmlUtil.Resolve(cells[2].InnerText);
// anon
var anon = HtmlUtil.Resolve(cells[3].InnerText);
//https
var http = HtmlUtil.Resolve(cells[4].Attributes["class"].Value);
//last check
var lastChecked = HtmlUtil.Resolve(cells[5].InnerText);
freeProxies.Enqueue(proxy);
freeProxyFetched(proxy);
});
}
catch (Exception ex)
{
throw;
}
return(freeProxies);
}
public bool RemoveFreeProxy(FreeProxy proxy)
{
return(FreeProxies.ToList().Remove(proxy));
}
public void RemoveProxy(FreeProxy proxy)
{
}
