public MatchCollection TokenizeQueryString(Session s)
{
MatchCollection mc = new MatchCollection();
int offsetCount = 0;
string rawQueryString = s.Request.Path;
string[] tmp = rawQueryString.Split('?');
if (tmp.Length < 2) throw new ArgumentException();
//Keep track of the offset. (+1 for the ?)
offsetCount += tmp[0].Length + 1;
string[] paramaters = tmp[1].Split('&');
foreach (string param in paramaters) {
string[] rawParams = param.Split('=');
//BUGBUG: Sometimes this returns when i fail to parse the query string =( i could add better handlings.. maybe another layer of splittings.. i'll think about it.
if (rawParams.Length == 0 || rawParams.Length > 2) return mc;
// if the length is equal to two then we have a name value pair.. otherwise we have a "single" param.
if (rawParams.Length == 2) {
//then add the value as a token along with it's offset..
offsetCount += rawParams[0].Length + 1;
mc.Add(new QueryStringMatch(new Token(rawParams[1]), offsetCount, s.Id));
offsetCount += rawParams[1].Length + 1;
} else {
//offset = offsetCount;
//add the token..
mc.Add(new QueryStringMatch(new Token(rawParams[0]), offsetCount, s.Id));
offsetCount += rawParams[0].Length + 1;//the plus one is for the & symbol..
}
}
return mc;
}
public static List<Session> ProcessBodyResults(Session s, List<BodyMatch> bodyMatches, Token tt)
{
List<Session> ret = new List<Session>();
foreach (BodyMatch match in bodyMatches) {
//HACK: This is to support token Id's infront of the canary for tracing back persistent Xss vulns.
string id = String.Format("{0:x4}", ResultProcessingEngine.TokenId);
Token t = new Token(id + tt.Identifier);
ResultProcessingEngine.TokenId++;
Session newSession = s.deepClone();
StringBuilder sb = new StringBuilder(newSession.Request.GetBodyEncodedAs(Encoding.UTF8));
if (match.Token.TokenLength > 0) {
sb.Replace(match.Token.Identifier, t.Identifier, match.Offset, match.Token.TokenLength);
} else {
sb.Insert(match.Offset, t.Identifier);
}
newSession.Request.BodyBytes = Encoding.UTF8.GetBytes(sb.ToString());
ret.Add(newSession);
}
return ret;
}
public static List<Session> ProcessHeaderResults(Session s, List<HeaderMatch> matches, Token tt)
{
List<Session> ret = new List<Session>();
foreach (HeaderMatch match in matches) {
Session newSession = s.deepClone();
List<string> headersForKey = s.Request.Headers[match.HeaderName];
foreach (string header in headersForKey) {
//HACK: This is to support token Id's infront of the canary for tracing back persistent Xss vulns.
string id = String.Format("{0:x4}", ResultProcessingEngine.TokenId);
Token t = new Token(id + tt.Identifier);
ResultProcessingEngine.TokenId++;
StringBuilder sb = new StringBuilder(header);
if (match.Token.TokenLength > 0) {
sb.Replace(match.Token.Identifier, t.Identifier, match.Offset, match.Token.TokenLength);
} else {
sb.Insert(match.Offset, t.Identifier);
}
headersForKey.Remove(header);
headersForKey.Add(sb.ToString());
}
ret.Add(newSession);
}
return ret;
}
public MatchCollection InspectResponse(Session s, Token t)
{
MatchCollection retList = new MatchCollection();
retList.AddRange(s.Response.FindTokenInHeaders(t, s.Id));
retList.AddRange(s.Response.FindTokenInBody(t, s.Id));
return retList;
}
public MatchCollection TokenizeBody(Session s)
{
MatchCollection mc = new MatchCollection();
int offsetCount = 0;
string body = s.Request.GetBodyEncodedAs(Encoding.UTF8);
string[] paramaters = body.Split('&');
foreach (string param in paramaters) {
string[] rawParams = param.Split('=');
if (rawParams.Length == 0 || rawParams.Length > 2) throw new ArgumentException();
// if the length is equal to two then we have a name value pair.. otherwise we have a "single" param.
if (rawParams.Length == 2) {
//then add the value as a token along with it's offset..
offsetCount += rawParams[0].Length + 1;
mc.Add(new BodyMatch(new Token(rawParams[1]), offsetCount, s.Id));
offsetCount += rawParams[1].Length + 1;
} else {
//offset = offsetCount;
//add the token..
mc.Add(new BodyMatch(new Token(rawParams[0]), offsetCount, s.Id));
offsetCount += rawParams[0].Length + 1;//the plus one is for the & symbol..
}
}
return mc;
}
//below is for dealing with "Request checking" when looking for a token in a request to replace with a test case.
public MatchCollection LocateTokensInRequest(Session s, Token canary)
{
MatchCollection retList = new MatchCollection();
retList.AddRange(s.Request.FindTokenInHeaders(canary, s.Id));
retList.AddRange(s.Request.FindTokenInBody(canary, s.Id));
return retList;
}
public MatchCollection TokenizeBody(Session s)
{
MatchCollection mc = new MatchCollection();
string body = s.Request.GetBodyEncodedAs(Encoding.UTF8);
parseJsonObject(body, mc, 0, s);
return mc;
}
private MatchCollection parseSession(Session s)
{
MatchCollection ret = new MatchCollection();
//Call the appropriate parser here.. I can add logic to call them based off of criteria or other mechanisms.
//Lets get the default case.. AutoRequestParser
foreach (IRequestParser parser in parsers) {
ret.AddRange(parser.TokenizeRequest(s));
}
return ret;
}
/// <summary>
/// Tokenize Query Strings..
/// </summary>
/// <param name="s"></param>
/// <returns></returns>
public MatchCollection TokenizeQueryString(Session s)
{
MatchCollection mc = new MatchCollection();
foreach (IQueryStringParser parser in queryStringParsers) {
if (parser is ParserBase && UAUtilities.isMatch(((ParserBase)parser).ContentTypePatterns, s.Request.ContentType)) {
mc.AddRange(parser.TokenizeQueryString(s));
}
}
return mc;
}
private void parseJsonObject(string json, MatchCollection mc, int sIndex, Session s)
{
//this should put us at the seperator for the name/value pairs.
int k = json.IndexOf(":", sIndex);
if (k == -1)
return;
//add one to be at the start of the value
k++;
//if the first char in the value is a { then we have another object.. call recursive..
if (json[k] == '{')
{
parseJsonObject(json, mc, k, s);
}
else
{
int x = k;
//else we can go ahead and handle the value.
while (json[x] != '}' && json[x] != ',')
x++;
int len = x - k;
string token = json.Substring(k, len);
//if the token starts with a " then we actually have a value "string" value.. add 1 to the "start" and strip the ".
if (token[0] == '\"')
{
k++;
token = token.Substring(1, token.Length -2);
mc.Add(new BodyMatch(new Token(token), k, s.Id));
}
k = k + len;
parseJsonObject(json, mc, k, s);
}
}
/// <summary>
/// This method creates the session list to be injected..
/// </summary>
/// <param name="s"></param>
/// <param name="queryStringMatches"></param>
/// <param name="t">This token is the replacement value..</param>
/// <param name="encodeQueryStringParams"></param>
/// <returns></returns>
public static List<Session> ProcessQueryStringResults(Session s, List<QueryStringMatch> queryStringMatches, Token tt, bool encodeQueryStringParams)
{
List<Session> ret = new List<Session>();
foreach (QueryStringMatch match in queryStringMatches) {
//HACK: This is to support token Id's infront of the canary for tracing back persistent Xss vulns.
string id = String.Format("{0:x4}", ResultProcessingEngine.TokenId);
Token t = new Token(id + tt.Identifier);
ResultProcessingEngine.TokenId++;
Session newSession = s.deepClone();
StringBuilder sb = new StringBuilder(newSession.Request.Path);
if (match.Token.TokenLength > 0) {
sb.Replace(match.Token.Identifier, t.Identifier, match.Offset, match.Token.TokenLength);
} else {
sb.Insert(match.Offset, t.Identifier);
}
newSession.Request.Path = sb.ToString();
ret.Add(newSession);
}
return ret;
}
public MatchCollection ProcessSession(Session s)
{
return this.parseSession(s);
}
public MatchCollection TokenizeRequest(Session oSession)
{
return this.TokenizeRequest(oSession, true, true, true);
}
/// <summary>
/// Method responsible for Inspecting a response and processing results. Main entry point into response processing.
/// </summary>
/// <param name="s"></param>
/// <returns></returns>
public List<ResponseResult> InspectResponse(Session s)
{
MatchCollection mc = new MatchCollection();
mc.AddRange(this.ResponseEngine.FindTokenMatchesInResponse(s, new Token(this.Settings.canary)));
//CalcResults is where the actual "smart logic" is performed.
List<ResponseResult> list = this.ResponseEngine.CalcResults(s, mc, this.Settings.UnicodeTestMappings,this.Settings.canary ,this.Settings.intelLookAhead);
return list;
}
/// <summary>
/// Attempt to determine the character set used by the response document. If the character
/// set cannot be determined, return UTF-8 (a reasonable guess).
/// </summary>
/// <remarks>TODO: Extract XML/XHtml character sets?</remarks>
/// <param name="session">The Fiddler HTTP session to examine.</param>
/// <returns>The character set specified by the session content or a reasonable guess.</returns>
public static String GetHtmlCharset(Session session)
{
const String DefaultCharacterSet = "utf-8"; // Return UTF-8 if unsure, ASCII is preserved.
// Favor the character set from the HTTP Content-Type header if it exists.
String CharacterSet = session.Response.Headers.GetTokenValue("Content-Type", "charset");
if (!String.IsNullOrEmpty(CharacterSet))
{
// Found the character set in the header: normalize and return.
return CharacterSet.Trim().ToLower();
}
// If there is no content, return the default character set.
if (session.Response.BodyBytes == null || session.Request.BodyBytes.Length == 0)
{
return DefaultCharacterSet;
}
// Otherwise, parse the document returned for character set hints.
String ResponseBody = String.Empty;
try
{
// TODO: Pretty hokey here, defaulting to 7-bit ASCII Encoding
ResponseBody = Encoding.ASCII.GetString(session.Response.BodyBytes);
}
catch (DecoderFallbackException e)
{
// Thrown if a character cannot be decoded
Trace.TraceError("Error: DecoderFallbackException: {0}", e.Message);
Trace.TraceWarning("Warning: Assuming default character set due to previous error.");
return DefaultCharacterSet;
}
String Temp;
// Find Meta tags specifying the content type, e.g.
// <meta http-equiv="content-type" content="text/html; charset=utf-8"/>.
foreach (Match m in Utility.GetHtmlTags(ResponseBody, "meta"))
{
Temp = Utility.GetHtmlTagAttribute(m.ToString(), "http-equiv");
if (!String.IsNullOrEmpty(Temp))
{
if (Temp.Trim().ToLower(CultureInfo.InvariantCulture) == "content-type")
{
CharacterSet = Utility.GetHtmlTagAttribute(m.ToString(), "content");
}
}
}
// ... and return the last content type attribute if found
// TODO: Extract the character set from the content type
if (!String.IsNullOrEmpty(CharacterSet))
{
// Found the character set in the response body: normalize and return.
return CharacterSet.Trim().ToLower();
}
// Return the default character set if unsure
return DefaultCharacterSet;
}
/// <summary>
/// This method returns the decompressed, dechunked, and normalized HTTP response body.
/// </summary>
/// <param name="session">The Fiddler HTTP session to examine.</param>
/// <returns>Normalized HTTP response body.</returns>
public static String GetResponseText(Session session)
{
// Ensure the response body is available
if (session.Response.BodyBytes == null || session.Response.BodyBytes.Length == 0)
{
Trace.TraceWarning("Warning: Response body is empty.");
return String.Empty;
}
// Remove chunking and compression from the HTTP response
// Logging the return value may result in excessive verbosity: avoid it.
//session.utilDecodeResponse();
// Attempt to determine the character set used by the response document
String CharacterSet = Utility.GetHtmlCharset(session);
String ResponseBody = String.Empty;
try
{
// Get the decoded session response.
ResponseBody = Encoding.GetEncoding(CharacterSet).GetString(session.Response.BodyBytes);
}
catch (DecoderFallbackException e)
{
// Thrown if a character cannot be decoded
Trace.TraceError("Error: DecoderFallbackException: {0}", e.Message);
}
catch (ArgumentException e)
{
// Thrown if the GetEncoding argument is invalid
Trace.TraceError("Error: ArgumentException: {0}", e.Message);
}
try
{
// Fallback to UTF-8 if we failed from a booty CharacterSet name.
if (ResponseBody == String.Empty)
{
Trace.TraceInformation("Falling back to UTF-8 encoding.");
ResponseBody = Encoding.UTF8.GetString(session.Response.BodyBytes);
}
}
catch (DecoderFallbackException e)
{
// Thrown if a character cannot be decoded
Trace.TraceError("Error: DecoderFallbackException: {0}", e.Message);
}
return ResponseBody;
}
/// <summary>
///
/// </summary>
/// <param name="s">Original Session</param>
/// <param name="mc">Collection which contains the tokens to be replaced</param>
/// <param name="t">The token to replace with</param>
/// <returns></returns>
private List<Session> GetSessionsForToken(Session s, MatchCollection mc, Token t)
{
List<Session> sessions = new List<Session>();
if (Settings.checkRequestForCanary || Settings.injectIntoPost)
{
if (Settings.urlEncodeBodyMatches)
{
t = new Token(HttpUtility.UrlEncode(t.Identifier));
}
sessions.AddRange(ResultProcessingEngine.ProcessBodyResults(s, mc.GetMatchesInBody(), t));
}
if (Settings.checkRequestForCanary || Settings.injectIntoHeaders)
{
if (Settings.urlEncodeHeaderMatches)
{
t = new Token(HttpUtility.UrlEncode(t.Identifier));
}
sessions.AddRange(ResultProcessingEngine.ProcessHeaderResults(s, mc.GetMatchesInHeaders(), t));
}
if (Settings.injectIntoQueryString)
{
if (Settings.urlEncodeQueryStringMatches)
{ //This code encodes the query string param if the value is set to true in the settings.
t = new Token(HttpUtility.UrlEncode(t.Identifier));
}
sessions.AddRange(ResultProcessingEngine.ProcessQueryStringResults(s, mc.GetMatchesInQueryString(), t, this.Settings.urlEncodeQueryStringMatches));
}
return sessions;
}
/// <summary>
/// TODO: Fix up to support other variations of text/xml
/// </summary>
/// <param name="session"></param>
/// <returns></returns>
public static bool IsResponseXml(Session session)
{
return (IsResponseContentType(session, "text/xml") || IsResponseContentType(session, "application/xml"));
}
/// <summary>
/// This method is where result "confidence" is calculated. A list of all the occurences of the "canary" should be passed in.
/// </summary>
/// <param name="s">Session object with request/response</param>
/// <param name="matches">These are the canary matches..</param>
/// <param name="lookAheadFromMatchLoc"></param>
/// <returns></returns>
public List<ResponseResult> CalcResults(Session s, MatchCollection matches, UnicodeTestCases mappings, string canary,int lookAheadFromMatchLoc)
{
List<ResponseResult> ret = new List<ResponseResult>();
//BUGBUG: Chris says he's getting an exception because the unicode char is not being associated with the session..
if (s.Chr == null)
{
return ret;
}
foreach (ResponseHeaderMatch hMatch in matches.GetMatchesInHeaders()) {
ResponseResult rr = new ResponseResult(hMatch);
//Get the match context in each header. Multipule headers could have the same key.. so
//a list is returned that contains each "string value" to a given key. Each element representing a different
//value for the same key.
foreach(string headerValue in s.Response.Headers[hMatch.HeaderName]){
int forwardDif = 0;
int offset = hMatch.Offset + hMatch.Token.TokenLength + lookAheadFromMatchLoc;
if (offset > headerValue.Length) {
forwardDif = offset - headerValue.Length;
}
string context = headerValue.Substring(hMatch.Offset - 4, hMatch.Token.TokenLength + lookAheadFromMatchLoc - forwardDif);
UnicodeTestCase mapping = mappings.GetMappingFromSourceCodePoint(s.Chr.CodePoint);
// At this point i have the context of the "canary" + 5 chars ahead.. this is where logic can be introduced
// to determine "the type of match"
rr.Chr = mapping.SourcePoint;
rr.Transformation = this.DeduceTransformation(context, mapping, canary);
rr.Context = context;
rr.TestCase = mapping;
//Add to result list
System.Diagnostics.Debug.Write(s.Fsession.fullUrl);
ret.Add(rr);
}
}
//This will change.. the response should be responsible for returning bytes based off detected encoding..
//That logic will be moved into the Response class at a later point.
string body = Encoding.UTF8.GetString(s.Response.BodyBytes);
//Now we find matches in the body..
foreach (BodyMatch bMatch in matches.GetMatchesInBody()) {
ResponseResult rr = new ResponseResult(bMatch);
//Get the match context.
int dif = 0;
int offset = bMatch.Offset + lookAheadFromMatchLoc + bMatch.Token.TokenLength;
if (offset > body.Length) {
dif = offset - body.Length;
}
string context = body.Substring(bMatch.Offset - 4, bMatch.Token.TokenLength + lookAheadFromMatchLoc - dif);
UnicodeTestCase mapping = mappings.GetMappingFromSourceCodePoint(s.Chr.CodePoint);
rr.Context = context;
rr.Chr = mapping.SourcePoint;
rr.Transformation = this.DeduceTransformation(context, mapping, canary);
rr.TestCase = mapping;
//System.Diagnostics.Debug.Write(s.Fsession.fullUrl);
ret.Add(rr);
}
//Add to result list.
return ret;
}
/// <summary>
/// TODO: Fix up to support other variations of javascript
/// </summary>
/// <param name="session"></param>
/// <returns></returns>
public static bool IsResponseJavascript(Session session)
{
return (IsResponseContentType(session, "application/javascript") || IsResponseContentType(session, "application/x-javascript"));
}
public static bool IsResponsePlain(Session session)
{
return (IsResponseContentType(session, "text/plain"));
}
public MatchCollection FindTokenMatchesInResponse(Session s, Token t)
{
MatchCollection mc = null;
foreach (IResponseParser parser in parsers) {
mc = parser.InspectResponse(s, t);
}
return mc;
}
/// <summary>
/// TODO: Fix up to support other variations of text/css
/// </summary>
/// <param name="session"></param>
/// <returns></returns>
public static bool IsResponseCss(Session session)
{
return (IsResponseContentType(session, "text/css"));
}
public static bool IsResponseContentType(Session session, String contentType)
{
string tmp = GetResponseContentType(session);
return ((tmp != null && tmp.IndexOf(contentType) == 0) ? true : false);
}
public MatchCollection TokenizeRequest(Session oSession, bool bTokenizeQS, bool bTokenizeHeaders, bool bTokenizeBody)
{
MatchCollection mc = new MatchCollection();
//Is there a query string on this session? check for the presences of ?
//Essentually sanity checks.
try
{
if (oSession.Request.Path.Contains("?") && bTokenizeQS)
mc.AddRange(TokenizeQueryString(oSession));
if (oSession.Request.Headers.Count > 0 && bTokenizeHeaders)
mc.AddRange(TokenizeHeaders(oSession));
if (oSession.Request.BodyBytes.Length > 0 && bTokenizeBody)
mc.AddRange(TokenizeBody(oSession));
}
catch
{
//Swallowing parsing exceptions.. Maybe display a error message at some point.
}
return mc;
}
/// <summary>
/// Ok this method is the main entry into request parsing and handling.
/// </summary>
/// <param name="s"></param>
public void ProcessRequest(Session s)
{
//First we tell the engne to process the request.. returning us a match collection parsed based on the parsers.
//This list contains all the offsets in the request where we will be replacing.
MatchCollection mc = new MatchCollection();
if (settings.injectIntoPost || settings.injectIntoQueryString)
{
mc.AddRange(RequestEngine.ProcessSession(s));
}
/*
* This is where logic is introduced to do checking for canaries in the request.
*
*/
//If we want to check for token presence in the request. (We can borrow the response parser to do this
if (this.Settings.checkRequestForCanary) {
if (s.Flags.ContainsKey(UASettings.casabaFlag) == false)
{
mc.AddRange(this.RequestEngine.LocateTokensInRequest(s, new Token(this.Settings.canary)));
}
}
//Here we create the session objects to inject based on the matches returned from the processing.
List<Session> sessions = GetRequestsToInject(s, mc);
//Inject the sessions into the fiddler session.. this also transforms our session objects to fiddlers.
if (sessions.Count > 0)
{
FiddlerUtils.InjectSessions(sessions);
}
}
public Session deepClone()
{
Session s = new Session();
s.Host = new String(this.Host.ToCharArray());
s.Id = this.Id;
s.Request = (Request)this.Request.Clone();
s.Response = this.response;
s.UriScheme = this.UriScheme;
foreach (string key in this.flags.Keys) {
string value = "";
try {
this.flags.TryGetValue(key, out value);
} catch { }
s.flags.Add(new string(key.ToCharArray()), new string(value.ToCharArray()));
}
return s;
}
/// <summary>
/// Returns a list of session objects.. These session objects contain the *Replaced* matches.. This is where overlong logic occurs.. (if(tc==overlong)create new token as overlong)
/// </summary>
/// <param name="mc"></param>
private List<Session> GetRequestsToInject(Session s, MatchCollection mc)
{
List<Session> sessions = new List<Session>();
foreach(UnicodeTestCase tc in this.Settings.UnicodeTestMappings){
if (tc.Enabled)
{
Token replaceValue;
if (tc.Type == UnicodeTestCaseTypes.Overlong)
{
byte[] bytes = XNMD.UTF8Encoder.GetOverlongForCodePoint(tc.SourcePoint.CodePoint, 2);
string enc = HttpUtility.UrlEncode(bytes);
replaceValue = new Token(this.Settings.canary + enc);
}
else
{
replaceValue = new Token(this.Settings.canary + tc.SourcePoint.ToString());
}
List<Session> tmp = this.GetSessionsForToken(s, mc, replaceValue);
foreach (Session sess in tmp)
{
sess.ContainsCodePoint = true;
sess.Chr = tc.SourcePoint;
}
sessions.AddRange(tmp);
}
}
return sessions;
}
/// <summary>
/// TODO: Fix up to support other variations of text/html.
/// FIX: This will match Atom and RSS feeds now, which set text/html but use <?xml> in content
/// </summary>
/// <param name="session"></param>
/// <returns></returns>
public static bool IsResponseHtml(Session session)
{
return (IsResponseContentType(session, "text/html") || IsResponseXhtml(session));
}
public static String GetResponseContentType(Session session)
{
if (session.Response.Headers.ContainsKey("content-type"))
return (session.Response.Headers["content-type"][0].ToLower());
return (null);
}