public void NewTvf()
{
if (_trafficViewerFile != null)
{
//close the existing file
_trafficViewerFile.Close(false);
}
_trafficViewerFile = new TrafficViewerFile();
_trafficViewerFile.Profile = Options.GetDefaultProfile();
_trafficViewerFile.SaveUnpacked();
}
public void SaveAndOpen()
{
TrafficViewerFile tvf = MakeDummyTrafficFile();
TempFile temp = new TempFile(".tvf");
tvf.Save(temp.Path);
tvf.Close(false);
//verfiy that we can open
tvf.Open(temp.Path);
//run all validations
ValidateASEFile(tvf);
tvf.Close(false);
}
public void EditARequest()
{
string originalRequest = "GET / HTTP/1.1";
string originalResponse = "HTTP/1.1 200 OK";
TrafficViewerFile file = new TrafficViewerFile();
int reqId = file.AddRequestResponse(originalRequest, originalResponse);
Assert.AreEqual(1, file.RequestCount);
TVRequestInfo reqInfo = file.GetRequestInfo(reqId);
string newRequest = "POST /login HTTP/1.1";
string newResponse = "HTTP/1.1 302 Redirect";
file.SaveRequest(reqId, Encoding.UTF8.GetBytes(newRequest));
file.SaveResponse(reqId, Encoding.UTF8.GetBytes(newResponse));
//check the response info was updated
Assert.AreEqual(newRequest, reqInfo.RequestLine);
Assert.AreEqual("302", reqInfo.ResponseStatus);
Assert.AreEqual(newRequest.Length, reqInfo.RequestLength);
Assert.AreEqual(newResponse.Length, reqInfo.ResponseLength);
string loadedRequest = Encoding.UTF8.GetString(file.LoadRequestData(reqId));
Assert.AreEqual(newRequest, loadedRequest);
string loadedResponse = Encoding.UTF8.GetString(file.LoadResponseData(reqId));
Assert.AreEqual(newResponse, loadedResponse);
file.Close(false);
}
public void OpenMultipleFilesAtOnce()
{
string predefinedTempDir = Environment.GetEnvironmentVariable("temp");
TrafficViewerFile file1 = new TrafficViewerFile(predefinedTempDir);
Assert.IsTrue(Directory.Exists(file1.TempFileFolder));
TrafficViewerFile file2 = new TrafficViewerFile(predefinedTempDir);
Assert.IsTrue(Directory.Exists(file2.TempFileFolder));
Assert.IsTrue(Directory.Exists(file1.TempFileFolder));
TrafficViewerFile file3 = new TrafficViewerFile(predefinedTempDir);
Assert.IsTrue(Directory.Exists(file3.TempFileFolder));
//make sure that the 3 files have different temp file folders
Assert.AreNotEqual(file1.TempFileFolder, file2.TempFileFolder);
Assert.AreNotEqual(file2.TempFileFolder, file3.TempFileFolder);
file1.Close(false);
Assert.IsFalse(Directory.Exists(file1.TempFileFolder));
Assert.IsTrue(Directory.Exists(file2.TempFileFolder));
Assert.IsTrue(Directory.Exists(file3.TempFileFolder));
file2.Close(false);
file3.Close(false);
Assert.IsFalse(Directory.Exists(file2.TempFileFolder));
Assert.IsFalse(Directory.Exists(file3.TempFileFolder));
}
public void VerifyTempDir()
{
string originalRequest = "GET / HTTP/1.1";
string originalResponse = "HTTP/1.1 200 OK";
string predefinedTempDir = Environment.GetEnvironmentVariable("temp");
TrafficViewerFile file = new TrafficViewerFile(predefinedTempDir);
file.AddRequestResponse(originalRequest, originalResponse);
Assert.IsTrue(Directory.Exists(file.TempFileFolder));
file.Close(true);
Assert.IsTrue(Directory.Exists(file.TempFileFolder));
file = new TrafficViewerFile(predefinedTempDir);
file.AddRequestResponse(originalRequest, originalResponse);
Assert.IsTrue(Directory.Exists(file.TempFileFolder));
file.Close(false);
//after the file is closed the file gets closed with the option to not keep temp files
Assert.IsFalse(Directory.Exists(file.TempFileFolder));
}
public void RemoveARangeOfRequests()
{
string originalRequest = "GET / HTTP/1.1";
string originalResponse = "HTTP/1.1 200 OK";
List <int> addedRequests = new List <int>();
TrafficViewerFile file = new TrafficViewerFile();
addedRequests.Add(file.AddRequestResponse(originalRequest, originalResponse));
addedRequests.Add(file.AddRequestResponse(originalRequest, originalResponse));
//add one more
int savedRequestId = file.AddRequestResponse(originalRequest, originalResponse);
addedRequests.Add(file.AddRequestResponse(originalRequest, originalResponse));
addedRequests.Add(file.AddRequestResponse(originalRequest, originalResponse));
Assert.AreEqual(5, file.RequestCount);
file.RemoveRequestBatch(addedRequests);
Assert.AreEqual(1, file.RequestCount);
Assert.IsNotNull(file.GetRequestInfo(savedRequestId));
file.Close(false);
}
public void TestHarImport()
{
TempFile temp = new TempFile(".har");
temp.Write(Resources.demohar);
TrafficViewerFile tvf = new TrafficViewerFile();
var parser = new HarParser();
parser.Parse(temp.Path, tvf, ParsingOptions.GetDefaultProfile());
Assert.AreEqual(3, tvf.RequestCount);
tvf.Close(false);
}
static void Main(string[] args)
{
if (args.Length < 2)
{
Console.WriteLine("Usage: Har2Exd <HAR file path> <EXD file path>");
Console.WriteLine("Exit codes: 1 - No args, 2 - Incorrect har path, 3 - Parsing error, 4 - Export error.");
Environment.ExitCode = 1;
}
else
{
string harFilePath = args[0];
string exdFilePath = args[1];
if (!File.Exists(harFilePath))
{
Console.WriteLine("Could not find har file: '{0}'", harFilePath);
Environment.ExitCode = 2;
}
else
{
TrafficViewerFile tvf = new TrafficViewerFile();
try
{
Console.WriteLine("Importing from '{0}'...", harFilePath);
ITrafficParser harParser = new HarParser();
harParser.Parse(harFilePath, tvf, ParsingOptions.GetDefaultProfile());
}
catch (Exception ex)
{
Console.WriteLine("Parsing exception: '{0}'", ex.Message);
Environment.ExitCode = 3;
}
//now export
try
{
Console.WriteLine("Exporting to '{0}'...", exdFilePath);
var exporter = new ManualExploreExporter();
exporter.Export(tvf, new FileStream(exdFilePath, FileMode.Create, FileAccess.ReadWrite));
}
catch (Exception ex)
{
Console.WriteLine("Export exception: '{0}'", ex.Message);
Environment.ExitCode = 4;
}
tvf.Close(false);
Console.WriteLine("Done.");
}
}
}
public void SaveAndOpen()
{
string expectedRequest = "GET / HTTP/1.1";
string expectedResponse = "HTTP/1.1 200 OK";
TrafficViewerFile file = new TrafficViewerFile();
int reqId = file.AddRequestResponse(expectedRequest, expectedResponse);
file.GetRequestInfo(reqId).IsHttps = true;
Assert.AreEqual(1, file.RequestCount);
TempFile temp = new TempFile(".tvf");
file.Save(temp.Path);
//verify that the file can be saved
Assert.IsTrue(File.Exists(temp.Path), "Cannot save the file");
file.Close(false);
//make a new file and verify we can open
TrafficViewerFile file2 = new TrafficViewerFile();
file2.Open(temp.Path);
//verify actual file was open
Assert.AreEqual(1, file2.RequestCount, "Incorrect request count after opening saved file");
//verify request data is correct
int requestId = -1;
TVRequestInfo info = file2.GetNext(ref requestId);
Assert.IsNotNull(info, "Cannot obtain request info");
//veryfy transport info
Assert.IsTrue(info.IsHttps);
//verify request data
string loadedRequest = Encoding.UTF8.GetString(file2.LoadRequestData(info.Id));
Assert.AreEqual(expectedRequest, loadedRequest);
string loadedResponse = Encoding.UTF8.GetString(file2.LoadResponseData(info.Id));
Assert.AreEqual(expectedResponse, loadedResponse);
file2.Close(false);
}
public void Clear()
{
TrafficViewerFile tvf = MakeDummyTrafficFile();
TempFile temp = new TempFile(".tvf");
tvf.Save(temp.Path);
Assert.AreNotSame(0, tvf.RequestCount);
tvf.Clear(false);
Assert.AreEqual(0, tvf.RequestCount);
int i = -1;
Assert.IsNull(tvf.GetNext(ref i));
tvf.Close(false);
}
public void TestEncryptedRequest()
{
TrafficViewerFile file = new TrafficViewerFile();
string request1 = "GET /unencrypted HTTP/1.1";
string request2 = "GET /encrypted\r\n\r\nsecret=123456789 HTTP/1.1";
string response1 = "HTTP 200 OK\r\n\r\nUnencrypted Response";
string response2 = "HTTP 200 OK\r\n\r\nEncrypted Response (secret 1234567789)";
file.AddRequestResponse(request1, response1);
file.AddRequestResponse(request2, response2);
var reqInfo = file.GetRequestInfo(1);
Assert.IsFalse(reqInfo.IsEncrypted, "Default should be unencrypted");
reqInfo.IsEncrypted = true;
//resave the request
file.SaveRequestResponse(1, request2, response2);
TempFile tempFile = new TempFile();
file.EnableDefrag = true; //defrag the raw file
file.Save(tempFile.Path);
file = new TrafficViewerFile();
file.Open(tempFile.Path);
Assert.IsFalse(file.GetRequestInfo(0).IsEncrypted, "First request should not be encrypted");
Assert.IsTrue(file.GetRequestInfo(1).IsEncrypted, "Second request should be encrypted");
string testRequest = Constants.DefaultEncoding.GetString(file.LoadRequestData(1));
Assert.AreEqual(request2, testRequest);
string testResponse = Constants.DefaultEncoding.GetString(file.LoadResponseData(1));
Assert.AreEqual(response2, testResponse);
file.Close(false);
File.Delete(tempFile.Path);
}
//[TestMethod]
public void ExportExdUtil()
{
string sourcePath = @"c:\_transfer\jaguarmanualexplorefiltered.htd";
TrafficViewerFile source = new TrafficViewerFile();
source.Open(sourcePath);
int id = -1;
int index = 0;
int count = source.RequestCount;
int partNo = 1;
int numberOfParts = 6;
int partSize = count / numberOfParts;
TVRequestInfo info;
TrafficViewerFile currentPart = new TrafficViewerFile();
while ((info = source.GetNext(ref id)) != null)
{
if (index < partSize * partNo)
{
byte [] request = source.LoadRequestData(info.Id);
byte [] response = source.LoadResponseData(info.Id);
currentPart.AddRequestResponse(request, response);
}
else
{
ExportPart(partNo, currentPart);
currentPart.Close(false);
currentPart = new TrafficViewerFile();
partNo++;
}
index++;
}
if (currentPart.RequestCount > 0)
{
ExportPart(partNo, currentPart);
}
}
public void RemoveARequest()
{
string originalRequest = "GET / HTTP/1.1";
string originalResponse = "HTTP/1.1 200 OK";
TrafficViewerFile file = new TrafficViewerFile();
int reqId = file.AddRequestResponse(originalRequest, originalResponse);
//add one more
file.AddRequestResponse(originalRequest, originalResponse);
Assert.AreEqual(2, file.RequestCount);
file.RemoveRequest(reqId);
Assert.AreEqual(1, file.RequestCount);
Assert.IsNull(file.GetRequestInfo(reqId));
file.Close(false);
}
static void Main(string[] args)
{
TrafficViewerFile file = new TrafficViewerFile();
file.Profile.SetExclusions((IEnumerable <string>) new string[2]
{
"\\.(js|axd|zip|Z|tar|t?gz|sit|cab|pdf|ps|doc|ppt|xls|rtf|dot|mp(p|t|d|e|a|3|4|ga)|m4p|mdb|csv|pp(s|a)|xl(w|a)|dbf|slk|prn|dif|avi|mpe?g|mov(ie)?|qt|moov|rmi?|as(f|x)|m1v|wm(v|f|a)|wav|ra|au|aiff|midi?|m3u|gif|jpe?g|bmp|png|tif?f|ico|pcx|css|xml|dll)\\b",
ConvertorProperties.ExcludedDomainsFromRecordingPattern
});
try
{
// Create an instance of StreamReader to read from a file.
// The using statement also closes the StreamReader.
using (StreamReader sr = new StreamReader(args[0]))
{
String line;
// Read and display lines from the file until the end of
// the file is reached.
while ((line = sr.ReadLine()) != null)
{
Har har = JsonConvert.DeserializeObject <Har>(line);
int requestHeaderId = 0;
int counter = 0;
foreach (Entry tempEntry in har.log.entries)
{
if (HtdConvertorUtil.isRelevantRequest(tempEntry))
{
counter++;
requestHeaderId = file.AddRequestResponse(tempEntry.request.ToString(), tempEntry.response.ToString());
if (tempEntry.request.postData != null)
{
Console.WriteLine(tempEntry.request.postData.ToString());
}
file.GetRequestInfo(requestHeaderId).Description = "AppScan Proxy Request to Server";
if (tempEntry.request.isHttps)
{
file.GetRequestInfo(requestHeaderId).IsHttps = true;
}
}
}
Console.WriteLine(counter);
}
}
file.Save(args[1]);
file.Close(false);
Console.WriteLine("Recording has been done");
//Console.ReadLine();
}
catch (Exception e)
{
// Let the user know what went wrong.
Console.WriteLine("The file could not be read:");
Console.WriteLine(e.Message);
}
}
private HttpResponseInfo StopProxy(HttpRequestInfo requestInfo)
{
string report = "";
//get the port from the url
string portString = null;
requestInfo.QueryVariables.TryGetValue("port", out portString);
//optional secret to protect the recording session
string secret = null;
requestInfo.QueryVariables.TryGetValue("secret", out secret);
//optional flag indicating if similar requests should be skiped
string skipSimilar = null;
requestInfo.QueryVariables.TryGetValue("skipSimilar", out skipSimilar);
//the file to save to
string fileName = null;
requestInfo.QueryVariables.TryGetValue("fileName", out fileName);
//optional parameter to cancel the scan
string cancel = null;
requestInfo.QueryVariables.TryGetValue("cancel", out cancel);
if (fileName == null)
{
//assign a random file name
fileName = DateTime.Now.Ticks.ToString();
}
if (!Utils.IsMatch(fileName, "^[\\w._-]+$"))
{
return(GetResponse(400, "Bad Request", "Invalid file name."));
}
int port;
if (int.TryParse(portString, out port))
{
if (!CollectorProxyList.Instance.ProxyList.ContainsKey(port))
{
return(GetResponse(400, "Bad Request", "Port not found."));
}
else
{
IHttpProxy proxy = CollectorProxyList.Instance.ProxyList[port];
TrafficViewerFile trafficFile = (proxy as ManualExploreProxy).TrafficDataStore as TrafficViewerFile;
//check the secret if it exists
string configuredSecret = trafficFile.Profile.GetOption("secret") as String;
if (!String.IsNullOrWhiteSpace(configuredSecret) && !configuredSecret.Equals(secret))
{
return(GetResponse(401, "Unauthorized", "Invalid secret."));
}
string filePath = Path.Combine(TrafficCollectorSettings.Instance.DumpDir, fileName + ".htd");
if (proxy is DriveByAttackProxy)
{
DriveByAttackProxy dProx = proxy as DriveByAttackProxy;
int requestsLeft = dProx.RequestsLeft;
if (requestsLeft > 0 && (cancel == null || !cancel.Equals("true")))
{
return(GetResponse(206, "Partial Content", "Please wait... {0} request(s) left, {1} test job(s) in queue", requestsLeft, dProx.TestCount));
}
else
{
int id = -1;
TVRequestInfo info = null;
report = "\r\n\r\nVulnerability List\r\n";
report += "============================\r\n";
int count = 0;
while ((info = trafficFile.GetNext(ref id)) != null)
{
if (info.Description.Contains("Vulnerability"))
{
count++;
report += String.Format("Request {0} - {1} ({2})\r\n", info.RequestLine, info.Description, info.Validation);
}
}
report += String.Format("Total: {0}\r\n", count);
}
}
if (File.Exists(filePath)) //load the existing file and check the secret
{
TrafficViewerFile existingFile = new TrafficViewerFile();
existingFile.Open(filePath);
configuredSecret = existingFile.Profile.GetOption("secret") as String;
existingFile.Close(false);
if (String.IsNullOrWhiteSpace(configuredSecret) || String.IsNullOrWhiteSpace(secret) || !configuredSecret.Equals(secret))
{
return(GetResponse(401, "Unauthorized", "Cannot override existing file."));
}
}
proxy.Stop();
CollectorProxyList.Instance.ProxyList.Remove(port);
if (trafficFile.RequestCount > 0)
{
if (skipSimilar != null && skipSimilar.Equals("true", StringComparison.OrdinalIgnoreCase))
{
trafficFile = removeSimilar(trafficFile);
}
trafficFile.Save(filePath);
report += String.Format("Traffic file saved at '{0}'\r\n", filePath);
}
else
{
report += "Nothing recorded.";
}
}
}
else
{
return(GetResponse(400, "Bad Request", "Invalid 'port' parameter."));
}
return(GetResponse(200, "OK", "Proxy stopped. {0}", report));
}
static void Main(string[] args)
{
if (args.Length < 2)
{
Console.WriteLine("Usage: Traffic2Exd <traffic file path> <EXD file path>");
Console.WriteLine("Supported import formats: .har, .txt, .htd");
Console.WriteLine("If the EXD file already exists the tool will append to it.");
Console.WriteLine("Exit codes: 1 - No args, 2 - Incorrect file path, 3 - Parsing error, 4 - Export error, 5 - Unsupported Exception.");
Environment.ExitCode = 1;
}
else
{
string trafficFilePath = args[0];
string exdFilePath = args[1];
if (!File.Exists(trafficFilePath))
{
Console.WriteLine("Could not find har file: '{0}'", trafficFilePath);
Environment.ExitCode = 2;
}
else
{
TrafficViewerFile tvf = new TrafficViewerFile();
try
{
if (File.Exists(exdFilePath))
{
Console.WriteLine("EXD file {0} already exists. Appending to it.", exdFilePath);
ConfigurationParser exdParser = new ConfigurationParser();
exdParser.Parse(exdFilePath, tvf, ParsingOptions.GetDefaultProfile());
}
Console.WriteLine("Importing from '{0}'...", trafficFilePath);
ITrafficParser parser = null;
if (trafficFilePath.ToLower().EndsWith(".har"))
{
parser = new HarParser();
}
else if (trafficFilePath.ToLower().EndsWith(".txt"))
{
parser = new DefaultTrafficParser();
}
else if (trafficFilePath.ToLower().EndsWith(".htd"))
{
TrafficViewerFile tvf2 = new TrafficViewerFile();
tvf2.Open(trafficFilePath);
int id = -1;
TVRequestInfo info = null;
while ((info = tvf2.GetNext(ref id)) != null)
{
tvf.AddRequestResponse(tvf2.LoadRequestData(info.Id), tvf2.LoadResponseData(info.Id));
}
}
else
{
Console.WriteLine("File extension is unsupported. Supported extensions/formats: .har, .txt, .htd");
Environment.ExitCode = 5;
}
if (parser != null)
{
parser.Parse(trafficFilePath, tvf, ParsingOptions.GetRawProfile());
}
}
catch (Exception ex)
{
Console.WriteLine("Parsing exception: '{0}'", ex.Message);
Environment.ExitCode = 3;
}
//now export
try
{
Console.WriteLine("Exporting to '{0}'...", exdFilePath);
var exporter = new ManualExploreExporter();
exporter.Export(tvf, new FileStream(exdFilePath, FileMode.Create, FileAccess.ReadWrite));
}
catch (Exception ex)
{
Console.WriteLine("Export exception: '{0}'", ex.Message);
Environment.ExitCode = 4;
}
tvf.Close(false);
Console.WriteLine("Done.");
}
}
}