protected override void BuildRequestHeaders(StringStringKeyValuePairContainer kvpsContainer, Request req, MobileApplication app, bool ssl, bool redirect)
{
kvpsContainer.Put(HeaderKeys.Host, app.Domain);
kvpsContainer.Put(HeaderKeys.Connection, HeaderValues.ConnectionKeepAlive);
kvpsContainer.Put(HeaderKeys.Accept, req.ResponseEncType.AcceptValue);
kvpsContainer.Put(HeaderKeys.UserAgent, BrowserCfg.UserAgent);
if (app.HasReferer)
{
kvpsContainer.Put(HeaderKeys.Referer, app.Referer);
}
kvpsContainer.Put(HeaderKeys.AcceptEncoding, HeaderValues.AcceptEncoding);
kvpsContainer.Put(HeaderKeys.AcceptLanguage, BrowserCfg.Language);
if (req.RequestType == RequestType.Options) //Sets the request method for options request
{
kvpsContainer.Put(HeaderKeys.AccessControlRequestMethod, ((OptionsRequest)req).RequestTypeOption.Notation); //Sets the option request header
}
if (req.RequestType == RequestType.Options || req is ContentRequest)
{
kvpsContainer.Put(HeaderKeys.Origin, HttpUtil.GetBaseUrl(req.Url));
}
if (req.RequestType == RequestType.Options || req is ContentRequest)
{
kvpsContainer.Put(HeaderKeys.Origin, HttpUtil.GetBaseUrl(req.Url));
}
if (req.HasBody())
{
((ContentRequest)req).ContentBody.Prepare(kvpsContainer);
}
}
private static void PerformTunnelHandshake(HttpSocket httpSocket, Proxy proxy, string hostAddress, int port, BrowserConfig browserCfg)
{
StringStringKeyValuePairContainer headersContainer = new StringStringKeyValuePairContainer();
headersContainer.Put(HeaderKeys.Host, hostAddress + ":" + port);
headersContainer.Put(HeaderKeys.UserAgent, browserCfg.UserAgent);
headersContainer.Put(HeaderKeys.Connection, HeaderValues.ConnectionKeepAlive);
if (proxy != null && proxy.HasAuthentication)
{
headersContainer.Put(HeaderKeys.ProxyConnection, HeaderValues.ConnectionKeepAlive);
headersContainer.Put(HeaderKeys.ProxyAuthorization, "Basic " + proxy.ProxyCreds.GetBase64Auth());
}
httpSocket.WriteLine(RequestType.Connect.Notation + " " + hostAddress + ":" + port + " " + browserCfg.HttpVersion.Notation);
foreach (StringStringKeyValuePair kvp in headersContainer.Kvps)
{
httpSocket.WriteLine(kvp.Key + ": " + kvp.Value);
}
httpSocket.WriteLine();
httpSocket.FlushUnderlying();
HttpHeaderDecoder hd = new HttpHeaderDecoder();
hd.Decode(httpSocket.Stream);
if (hd.ResponseStatus.HttpCode != HttpCode.Ok)
{
throw new IOException("Unable to tunnel through proxy => " + hd.ResponseStatus);
}
logger.Debug("Successfully tunneled through proxy");
}
public Application(string domain)
{
Domain = domain;
LocalDatabase = new StringStringKeyValuePairContainer();
LocalStorage = new StringStringKeyValuePairContainer();
SessionStorage = new StringStringKeyValuePairContainer();
Cache = new StringStringKeyValuePairContainer();
}
public RequestResponse(string endpoint, HttpCode expectedHttpCode, ResponseStatus responseStatus
, IBaseResponseBody responseBody, StringStringKeyValuePairContainer responseHeaders)
{
Endpoint = endpoint;
ExpectedHttpCode = expectedHttpCode;
ResponseStatus = responseStatus;
ResponseBody = responseBody;
ResponseHeaders = responseHeaders;
}
public Request(RequestType requestType, string url, HttpCode expectedResponseCode, params RequestProperty[] props)
{
RequestType = requestType;
Url = url;
ExpectedResponseCode = expectedResponseCode;
CustomHeaders = new StringStringKeyValuePairContainer();
ResponseEncType = EncType.UrlEncoded;
Port = 80;
Props = props;
}
public ContentBody Prepare(StringStringKeyValuePairContainer headers)
{
using (HttpMemoryStream httpMemStream = new HttpMemoryStream()) {
Build(httpMemStream, false);
SetContentLength(httpMemStream.Length);
SetContentHeaders(headers);
}
return(this);
}
private void BuildSecFetchHeaders(StringStringKeyValuePairContainer reqHeaders, Request request, WebApplication app)
{
string lastNavigatedUrlDomain = app.LastNavigatedUrl is null ? null : HttpUtil.GetDomain(app.LastNavigatedUrl);
reqHeaders.Put(HeaderKeys.SecFetchMode, request.HasProperty(RequestProperty.XMLHttpRequest)
? HeaderValues.SecFetchCors : HeaderValues.SecFetchNavigate);
//request.getSecFetchMode()
//TODO nested-navigate redirect to internal page by clicking button etc, no link? Form <<<
//TODO same-origin/same-site/cross-site when called inside the page, like js scripts being called
//boolean firstHostContact = !accessedHosts.contains(host) || (request.getRequestType() == RequestType.GET && ((GetRequest)request).isNoRef());
bool sameOrigin = IsSameOrigin(app.Domain, lastNavigatedUrlDomain);
bool sameSite = !sameOrigin && IsSameSiteDifferentHost(app.Domain, lastNavigatedUrlDomain);
bool crossSite = !sameOrigin && !sameSite;
if (AccessedHosts.Count > 0 && !crossSite && RedirectionChain.Count > 1)
{
for (int i = 0; i < RedirectionChain.Count; i++)
{
Request prevReq = RedirectionChain[i];
string prevHost = HttpUtil.GetDomain(prevReq.Url);
if (IsSameOrigin(app.Domain, prevHost))
{
continue;
}
if (IsSameSiteDifferentHost(app.Domain, prevHost))
{
sameSite = true;
continue;
}
crossSite = true;
break;
}
}
if (AccessedHosts.Count == 0)
{
reqHeaders.Put(HeaderKeys.SecFetchSite, HeaderValues.SecFetchNone);
}
else if (crossSite)
{
reqHeaders.Put(HeaderKeys.SecFetchSite, HeaderValues.SecFetchCrossSite);
}
else if (sameSite)
{
reqHeaders.Put(HeaderKeys.SecFetchSite, HeaderValues.SecFetchSameSite);
}
else
{
reqHeaders.Put(HeaderKeys.SecFetchSite, HeaderValues.SecFetchSameOrigin);
}
if (!request.HasProperty(RequestProperty.XMLHttpRequest))
{
reqHeaders.Put(HeaderKeys.SecFetchUser, HeaderValues.SecFetchUser);
}
}
public FormBody() : base(EncType.UrlEncoded)
{
kvpContainer = new StringStringKeyValuePairContainer();
}
protected override void SetContentHeaders(StringStringKeyValuePairContainer headers)
{
headers.Put(HeaderKeys.ContentType, raw ? EncType.ContentValue : MimeType.GetByFile(fileLink).Notation);
headers.Put(HeaderKeys.ContentLength, ContentLength.ToString());
}
protected override void SetContentHeaders(StringStringKeyValuePairContainer headers)
{
base.SetContentHeaders(headers);
headers.Put(HeaderKeys.ContentType, EncType.ContentValue + "; boundary=" + Boundary);
}
public static IBaseResponseBody ParseResponseBody(Request request, StringStringKeyValuePairContainer responseHeaders, Stream inputStream)
{
if (request.HasProperty(RequestProperty.NoDecode))
{
return(new StringResponseBody("Body decoding is turned off for this request"));
}
string contentEncoding = responseHeaders.GetValue(HeaderKeys.ContentEncoding); //The content encoding of the response, will be null when it's plain text
string contentLength = responseHeaders.GetValue(HeaderKeys.ContentLength);
string contentType = responseHeaders.GetValue(HeaderKeys.ContentType);
string chunkedValue = responseHeaders.GetValue(HeaderKeys.TransferEncoding);
bool chunked = chunkedValue != null && contentLength == null &&
chunkedValue.Equals(HeaderValues.TransferEncodingChunked, StringComparison.OrdinalIgnoreCase);
long bodySize = 0;
if (contentLength != null)
{
if (!MathUtil.IsInteger(contentLength))
{
throw new Exception("Content length for " + request.Url + " is not a valid integer: " + contentLength);
}
bodySize = MathUtil.ParseInt(contentLength, true, 0);
}
if (!chunked && contentLength == null)
{
logger.Warning("Response for " + request.Url + " is not chunked but has no Content-Length header.");
}
Console.WriteLine("Parsing server response body for [" + request.Url + "] with properties [Content-Encoding: " + contentEncoding
+ ", Transfer-Encoding: " + chunkedValue + ", Content-Length: " + bodySize + ", Content-Type: " + contentType + "]"); //TODO
bool download = request.SavePath != null && contentType != null;
if (chunked) //Read the chunks if the response is chunked
{
if (responseHeaders.GetValue(HeaderKeys.Location) != null) //When it's a redirect we don't need to read any body
{
return(new StringResponseBody(""));
}
logger.Debug("Reading response body chunks");
inputStream = ReadChunks(inputStream);
if (inputStream is null)
{
logger.Error("Failed to read chunks on " + request.Url);
if (HttpConfig.DebugMode)
{
throw new Exception("Failed to read chunks on " + request.Url);
}
return(download ? new FileResponseBody(null) : (IBaseResponseBody) new StringResponseBody("Failed to read chunks"));
}
if (bodySize > 0 && bodySize != inputStream.Length)
{
logger.Warning("Unexpected body size for " + request.Url + ", received " + inputStream.Length + " instead of " + bodySize);
}
bodySize = inputStream.Length;
}
if (contentLength != null && contentLength.Equals("0", StringComparison.Ordinal))
{
logger.Debug("Content length is zero for " + request.Url + ", no body to parse");
return(new StringResponseBody(""));
}
BaseResponseInterpretor interpretor = download ? new FileInterpretor(bodySize, request.SavePath)
: (BaseResponseInterpretor) new StringInterpretor(bodySize, contentEncoding, contentType);
if (request.HasProgressListener)
{
Task.Run(() => TrackProgress(request.ProgressListener, interpretor));
}
IBaseResponseBody respBody;
if (download)
{
FileInterpretor fi = interpretor as FileInterpretor;
respBody = fi.Interpret(inputStream);
}
else
{
StringInterpretor si = interpretor as StringInterpretor;
respBody = si.Interpret(inputStream);
}
return(respBody);
}
protected override void BuildRequestHeaders(StringStringKeyValuePairContainer kvpsContainer, Request req, WebApplication app, bool ssl, bool redirect)
{
bool xmlHttpRequest = req.HasProperty(RequestProperty.XMLHttpRequest);
kvpsContainer.Put(HeaderKeys.Host, app.Domain);
kvpsContainer.Put(HeaderKeys.Connection, HeaderValues.ConnectionKeepAlive);
if (ssl)
{
kvpsContainer.Put(HeaderKeys.Dpr, "1");
}
kvpsContainer.Put(HeaderKeys.UserAgent, BrowserCfg.UserAgent);
kvpsContainer.Put(HeaderKeys.Accept, req.ResponseEncType.AcceptValue);
if ((req.RequestType == RequestType.Post && !xmlHttpRequest) || redirect || !string.IsNullOrEmpty(req.CachePolicy))
{
kvpsContainer.Put(HeaderKeys.CacheControl, !string.IsNullOrEmpty(req.CachePolicy) ? req.CachePolicy : HeaderValues.MaxZeroCache);
}
if (!xmlHttpRequest)
{
kvpsContainer.Put(HeaderKeys.UpgradeInsecureRequests, "1");
}
if (xmlHttpRequest)
{
kvpsContainer.Put(HeaderKeys.XRequestedWith, HeaderValues.XmlHttpRequest);
}
if (BrowserCfg.DoNotTrack)
{
kvpsContainer.Put(HeaderKeys.Dnt, "1");
}
if (app.HasNavigated)
{
kvpsContainer.Put(HeaderKeys.Referer, app.LastNavigatedUrl);
}
kvpsContainer.Put(HeaderKeys.AcceptEncoding, HeaderValues.AcceptEncoding);
kvpsContainer.Put(HeaderKeys.AcceptLanguage, BrowserCfg.Language);
if (ssl)
{
BuildSecFetchHeaders(kvpsContainer, req, app);
}
if (req.RequestType == RequestType.Options) //Sets the request method for options request
{
kvpsContainer.Put(HeaderKeys.AccessControlRequestMethod, ((OptionsRequest)req).RequestTypeOption.Notation); //Sets the option request header
}
if (req.RequestType == RequestType.Options || xmlHttpRequest || req is ContentRequest)
{
kvpsContainer.Put(HeaderKeys.Origin, HttpUtil.GetBaseUrl(req.Url));
}
if (req.RequestType != RequestType.Options && app.CookieJar != null &&
app.CookieJar.Cookies.Count > 0 && SessionPolicy != HttpSessionPolicy.NoData)
{
StringBuilder cookieSb = new StringBuilder();
for (int i = 0; i < app.CookieJar.Cookies.Count; i++)
{
cookieSb.Append(app.CookieJar.Cookies[i].Name + "=" + app.CookieJar.Cookies[i].Value);
if (i < (app.CookieJar.Cookies.Count - 1))
{
cookieSb.Append("; ");
}
}
kvpsContainer.Put(HeaderKeys.Cookie, cookieSb.ToString());
}
if (req.HasBody())
{
((ContentRequest)req).ContentBody.Prepare(kvpsContainer);
}
}
protected abstract void BuildRequestHeaders(StringStringKeyValuePairContainer reqHeadersContainer, Request req, TA app, bool ssl, bool redirect);
protected virtual void SetContentHeaders(StringStringKeyValuePairContainer headers)
{
headers.Put(HeaderKeys.ContentType, EncType.ContentValue);
headers.Put(HeaderKeys.ContentLength, ContentLength.ToString());
}
