public override async Task <object> Run_Internal(ProxyObserver observer, RestClient http, string asset, IScanRepository repository, object args)
{
var img = await http.GetAsync <JsonObject>($"screenshot?site={asset}");
observer?.Notify("WebpageScreenshot", "got screnshot", null);
return(null != img ? img.screenshot?.data : null);
}
public override async Task <object> Run_Internal(ProxyObserver observer, RestClient http, string asset, IScanRepository repository, object args)
{
Result result = await http.GetAsync <Result>($"getHtmlSource?site={asset}");
observer?.Notify("WebPageHthm", "got Html Source", null);
return(result?.html);
}
public override async Task <object> Run_Internal(ProxyObserver observer, RestClient http, string asset, IScanRepository repository, object args)
{
var proxy = new RestClient(http.Host);
List <Result> result = await proxy.GetAsync <List <Result> >("getDollars");
observer?.Notify("Dollars", "got dollar values", null);
return(result);
}
/// <summary>
/// Returns an IRemoteObserver used to send messages to the remote host at
/// the specified IPEndpoint.
/// </summary>
/// <param name="remoteEndpoint">The IPEndpoint of the remote host</param>
/// <returns>An IRemoteObserver used to send messages to the remote host</returns>
public IRemoteObserver <T> GetUnmanagedObserver(IPEndPoint remoteEndpoint)
{
if (remoteEndpoint == null)
{
throw new ArgumentNullException("remoteEndpoint");
}
ProxyObserver remoteObserver = CreateRemoteObserver(remoteEndpoint);
return(remoteObserver);
}
private ProxyObserver CreateRemoteObserver(IPEndPoint remoteEndpoint)
{
StreamingTransportClient <IRemoteEvent <T> > client =
new StreamingTransportClient <IRemoteEvent <T> >(remoteEndpoint,
_observerContainer,
_remoteEventCodec,
_tcpClientFactory);
ProxyObserver remoteObserver = new ProxyObserver(client);
return(remoteObserver);
}
private ProxyObserver CreateRemoteObserver(IPEndPoint remoteEndpoint)
{
TransportClient <IRemoteEvent <T> > client =
new TransportClient <IRemoteEvent <T> >(remoteEndpoint, _codec, _observerContainer, _tcpClientFactory);
var msg = string.Format("NewClientConnection: Local {0} connected to Remote {1}",
client.Link.LocalEndpoint.ToString(),
client.Link.RemoteEndpoint.ToString());
LOGGER.Log(Level.Info, msg);
ProxyObserver remoteObserver = new ProxyObserver(client);
return(remoteObserver);
}
/// <summary>
/// Returns an IObserver used to send messages to the remote host at
/// the specified IPEndpoint.
/// </summary>
/// <param name="remoteEndpoint">The IPEndpoint of the remote host</param>
/// <returns>An IObserver used to send messages to the remote host</returns>
public IObserver <T> GetRemoteObserver(IPEndPoint remoteEndpoint)
{
if (remoteEndpoint == null)
{
throw new ArgumentNullException("remoteEndpoint");
}
ProxyObserver remoteObserver;
if (!_cachedClients.TryGetValue(remoteEndpoint, out remoteObserver))
{
StreamingTransportClient <IRemoteEvent <T> > client =
new StreamingTransportClient <IRemoteEvent <T> >(remoteEndpoint, _observerContainer, _remoteEventCodec);
remoteObserver = new ProxyObserver(client);
_cachedClients[remoteEndpoint] = remoteObserver;
}
return(remoteObserver);
}
/// <summary>
/// Returns an IObserver used to send messages to the remote host at
/// the specified IPEndpoint.
/// </summary>
/// <param name="remoteEndpoint">The IPEndpoint of the remote host</param>
/// <returns>An IObserver used to send messages to the remote host</returns>
public IObserver <T> GetRemoteObserver(IPEndPoint remoteEndpoint)
{
if (remoteEndpoint == null)
{
throw new ArgumentNullException("remoteEndpoint");
}
ProxyObserver remoteObserver;
if (!_cachedClients.TryGetValue(remoteEndpoint, out remoteObserver))
{
TransportClient <IRemoteEvent <T> > client =
new TransportClient <IRemoteEvent <T> >(remoteEndpoint, _codec, _observerContainer);
var msg = string.Format("NewClientConnection: Local {0} connected to Remote {1}",
client.Link.LocalEndpoint.ToString(),
client.Link.RemoteEndpoint.ToString());
LOGGER.Log(Level.Info, msg);
remoteObserver = new ProxyObserver(client);
_cachedClients[remoteEndpoint] = remoteObserver;
}
return(remoteObserver);
}
public override async Task <object> Run_Internal(ProxyObserver observer, RestClient http, string asset, IScanRepository repository, object args)
{
if (String.IsNullOrWhiteSpace(asset) || IsLinkFile(asset))
{
return(new List <string>());
}
// asset = CleanURL(asset);
HashSet <string> temp = await http.GetAsync <HashSet <string> >($"getAllLinks?site={asset}");
if (null == temp)
{
return(new List <string>());
}
HashSet <string> temp2 = new HashSet <string>();
foreach (var link in temp)
{
if (!String.IsNullOrEmpty(link))
{
if (_badChars.Contains(link[link.Length - 1]))
{
string fixedLink = link.Remove(link.Length - 1);
temp2.Add(fixedLink);
}
else
{
temp2.Add(link);
}
}
}
IEnumerable <string> links = temp2;
if (null != links && links.Any())
{
var yuri = new Uri(asset);
string host = yuri.Scheme + "://" + yuri.Host;
links = links.Where(link => link.StartsWith(host) && !IsLinkFile(link));//sadece host adıyla başlayanları al.
if (links.Any())
{
IEnumerable <string> previousResults = await repository.GetResults(true, true);
HashSet <string> doNotIncludeThose = new HashSet <string>();
foreach (string result in previousResults)
{
try
{
var list = JsonConvert.DeserializeObject <List <string> >(result);
list.ForEach(i => doNotIncludeThose.Add(i));
}
catch { }
}
links = links.Where(l => !doNotIncludeThose.Contains(l));
_ = repository.AddChildScan(asset, links);//clild' lari ekle asekron olarak.
observer.Notify(nameof(Worker) + "_" + nameof(Run), "The Spider is working now.", links);
}
}
return(links);
}
public override async Task <object> Run_Internal(ProxyObserver observer, RestClient http, string asset, IScanRepository repository, object args)
{
return(await http.GetAsync <HashSet <string> >($"getAllLinks?site={asset}"));
}