/// <summary>
/// Start a network capture
/// </summary>
public static void Start()
{
lock (startingLock)
{
try
{
// Retrieve the device list
devices = CaptureDeviceList.Instance;
// If no devices were found print an error
if (devices.Count < 1)
{
var error = "No devices were found on this machine";
TraceLog.TraceFatal(error);
throw new Exception(error);
}
// if capture is started on all devices, nothing to do
var started = true;
foreach (var dev in devices)
{
if (!dev.Started || !string.IsNullOrEmpty(dev.LastError))
{
started = false;
break;
}
}
if (started)
{
return;
}
// Print SharpPcap version
string ver = SharpPcap.Version.VersionString;
TraceLog.TraceInfo(string.Format("Starting collector with version {0}", ver));
foreach (var device in devices)
{
// reset device if already started
if (device.Started)
{
TraceLog.TraceInfo(string.Format("Stopping {0} {1}", device.Name, device.Description));
device.OnPacketArrival -= new PacketArrivalEventHandler(device_OnPacketArrival);
device.StopCapture();
device.Close();
}
// Register our handler function to the 'packet arrival' event
device.OnPacketArrival +=
new PacketArrivalEventHandler(device_OnPacketArrival);
// Open the device for capturing
int readTimeoutMilliseconds = 1000;
device.Open(DeviceMode.Promiscuous, readTimeoutMilliseconds);
TraceLog.TraceInfo(string.Format("Listening on {0} {1}",
device.Name, device.Description));
// DNS only
string filter = "udp dst port 53";
device.Filter = filter;
// Start the capturing process
device.StartCapture();
}
// get the device ID, and store it if it's not in the config file already
deviceId = ConfigClient.Read(ConfigClient.DeviceId);
if (deviceId == null)
{
deviceId = devices[0].MacAddress.ToString();
ConfigClient.Write(ConfigClient.DeviceId, deviceId);
}
}
catch (Exception ex)
{
TraceLog.TraceException("FATAL: Start caught exception", ex);
throw ex;
}
}
}
private static void GetVersionLoop()
{
while (loopFlag)
{
// sleep for timeout
Thread.Sleep(timeout);
// don't want two concurrent updates
if (checkingFlag || updatingFlag)
{
continue;
}
try
{
// locking is overkill here because only one thread that wakes up every <timeout>
checkingFlag = true;
WebServiceHelper.GetCurrentSoftwareVersion(
VersionUrl,
new WebServiceHelper.AccountDelegate((version) =>
{
if (string.IsNullOrEmpty(version))
{
TraceLog.TraceError("Could not download version file");
checkingFlag = false;
return;
}
try
{
// compare versions
var newVersion = new Version(version);
if (newVersion > CurrentVersion)
{
TraceLog.TraceInfo(string.Format(
"New version {0} more recent than current version {1}; downloading new version",
newVersion, CurrentVersion));
updatingFlag = true;
// download and install new software
WebServiceHelper.DownloadCurrentSoftware(
DownloadUrl,
new WebServiceHelper.AccountDelegate((filename) =>
{
// install the new software
TraceLog.TraceInfo(string.Format("Download successful; installing {0}", filename));
Install(filename);
}),
new WebServiceHelper.NetOpDelegate((inProgress, status) =>
{
// cleanup the checking flag based on network operation status (typically failures)
if (inProgress == false)
{
updatingFlag = false;
}
}));
}
checkingFlag = false;
}
catch (Exception ex)
{
TraceLog.TraceException("Version check failed", ex);
checkingFlag = false;
updatingFlag = false;
}
}),
new WebServiceHelper.NetOpDelegate((inProgress, status) =>
{
// cleanup the checking flag based on network operation status (typically failures)
if (inProgress == false)
{
checkingFlag = false;
}
}));
}
catch (Exception ex)
{
TraceLog.TraceException("GetVersionLoop failed", ex);
checkingFlag = false;
updatingFlag = false;
}
}
}
public static void WriteLine(string message, string level)
{
// create a json record
var record = new TraceRecord()
{
//Client = TraceLog.Client,
LogLevel = level,
Message = message,
Session = TraceLog.Session,
Timestamp = DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss.fff")
};
var json = JsonConvert.SerializeObject(record);
lock (writeLock)
{
// enter a retry loop writing the record to the trace file
int retryCount = 2;
while (retryCount > 0)
{
try
{
if (traceFilename == null)
{
// create the file
using (var stream = File.Create(TraceFilename))
using (var writer = new StreamWriter(stream))
{
// log the file creation
var createRecord = new TraceRecord()
{
//Client = TraceLog.Client,
LogLevel = TraceLog.LevelText(TraceLog.LogLevel.Info),
Message = "Created new trace file " + TraceFilename,
Session = TraceLog.Session,
Timestamp = DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss.fff")
};
var createJson = JsonConvert.SerializeObject(createRecord);
writer.WriteLine(createJson);
writer.Flush();
}
}
// open the file
using (var stream = File.Open(TraceFilename, FileMode.Append, FileAccess.Write, FileShare.Read))
using (var writer = new StreamWriter(stream))
{
writer.WriteLine(json);
writer.Flush();
// reset the trace filename if it exceeds the maximum file size
if (writer.BaseStream.Position > MaxFileSize)
{
traceFilename = null;
}
}
// success - terminate the enclosing retry loop
break;
}
catch (Exception)
{
// the file wasn't opened or written to correctly - try to start with a new file in the next iteration of the retry loop
traceFilename = null;
retryCount--;
}
}
}
}
private static void device_OnPacketArrival(object sender, CaptureEventArgs e)
{
try
{
var packet = PacketDotNet.Packet.ParsePacket(e.Packet.LinkLayerType, e.Packet.Data);
if (packet is PacketDotNet.EthernetPacket)
{
var eth = packet as EthernetPacket;
// this syntax is from PacketDotNet 0.13.0 which isn't supported in the nuget package (0.12.0)
//var udp = packet.Extract(typeof(UdpPacket)) as UdpPacket;
var udp = Extract(packet, typeof(UdpPacket)) as UdpPacket;
if (udp != null)
{
string websiteName = null;
if (udp.DestinationPort == 53) // DNS
{
// extract hostname. it is encoded in segments: one-byte length followed by the segment in ASCII
// the first segment starts at 0x0c
const int START = 0x0c;
int start = START;
byte len = udp.PayloadData[start];
var sb = new StringBuilder();
while (len > 0)
{
sb.Append(Encoding.ASCII.GetString(udp.PayloadData, start + 1, len));
start += len + 1;
len = udp.PayloadData[start];
if (len > 0)
{
sb.Append('.');
}
}
websiteName = sb.ToString();
}
var hostMacAddress = eth.SourceHwAddress.ToString();
#if DEBUG
// only log this level of detail if we're logging to the console
if (TraceLog.TraceDestination == TraceLog.Destination.Console)
{
// get the IP packet info
var ip = packet.PayloadPacket as IPv4Packet;
if (ip != null)
{
var hostIpAddress = ip.SourceAddress.ToString();
var destIpAddress = ip.DestinationAddress.ToString();
var sourcePort = udp.SourcePort;
var destPort = udp.DestinationPort;
var hostName = hostIpAddress;
try
{
IPHostEntry entry = null;
if (Environment.Version.CompareTo(new System.Version(4, 0)) < 0)
{
entry = Dns.GetHostByAddress(ip.SourceAddress);
}
else
{
entry = Dns.GetHostEntry(ip.SourceAddress);
}
if (entry != null)
{
hostName = entry.HostName;
}
}
catch (Exception ex)
{
TraceLog.TraceException(String.Format("GetHostByAddress failed for {0}", hostIpAddress), ex);
}
var destMacAddress = eth.DestinationHwAddress.ToString();
TraceLog.TraceInfo(String.Format("Source: [{0}; {1}:{2}; {3}]; Dest: [{4}; {5}:{6}; Website: {7}",
hostMacAddress, hostIpAddress, sourcePort, hostName,
destMacAddress, destIpAddress, destPort,
websiteName));
}
}
#endif
lock (records)
{
records.Add(new Record()
{
HostMacAddress = hostMacAddress,
WebsiteName = websiteName,
Timestamp = DateTime.Now.ToString("s")
});
}
}
}
}
catch (Exception ex)
{
TraceLog.TraceException("OnPacketArrival caught exception", ex);
}
}
// Common code to process the response from any web service call. This is invoked from the callback
// method for the web service, and passed a Type for deserializing the response body.
// This method will also invoke the delegate with the result of the Web Service call
private static void ProcessResponse <T>(IAsyncResult result)
{
WebServiceState state = result.AsyncState as WebServiceState;
if (state == null)
{
TraceLog.TraceError("Web Service State not found");
return;
}
// get the network operation status delegate
Delegate netOpInProgressDel = state.NetworkOperationInProgressDelegate as Delegate;
// get the web response and make sure it's not null (failed)
//HttpWebResponseWrapper<T> resp = GetWebResponse<T>(result);
HttpWebResponse resp = GetWebResponse(result);
if (resp == null)
{
TraceLog.TraceError("GetWebResponse failed");
// signal that the network operation completed unsuccessfully
if (netOpInProgressDel != null)
{
netOpInProgressDel.DynamicInvoke(false, OperationStatus.Failed);
}
return;
}
else
{
OperationStatus status = AsOperationStatus(resp.StatusCode);
if (resp.StatusCode == HttpStatusCode.Unauthorized)
{ // using this status code to indicate cookie has expired or is invalid
if (authCookie != null)
{ // remove cookie and retry with credentials
status = OperationStatus.Retry;
authCookie = null;
}
}
if (resp.StatusCode == HttpStatusCode.Forbidden)
{ // remove cookie and force authentication on next request
authCookie = null;
}
if (netOpInProgressDel != null)
{ // signal the network operation completed and whether it completed successfully
netOpInProgressDel.DynamicInvoke(false, status);
if (status == OperationStatus.Retry)
{ // delegate will retry, exit now
TraceLog.TraceInfo("Received a Retry response from Service");
return;
}
}
}
// get the method-specific delegate
Delegate del = state.Delegate as Delegate;
//if (del == null)
// return; // if no delegate was passed, the results can't be processed
// invoke the delegate with the response body
try
{
T resultObject = (T) new DataContractJsonSerializer(typeof(T)).ReadObject(resp.GetResponseStream());
//T resultObject = resp.GetBody();
if (del == null)
{
return; // if no delegate was passed, the results can't be processed
}
del.DynamicInvoke(resultObject);
}
catch (Exception ex)
{
TraceLog.TraceException("ProcessResponse: exception from GetBody or DynamicInvoke", ex);
// signal the operation is done and unsuccessful
if (netOpInProgressDel != null)
{
netOpInProgressDel.DynamicInvoke(false, OperationStatus.Failed);
}
if (del == null)
{
return; // if no delegate was passed, the results can't be processed
}
del.DynamicInvoke(null);
}
}
// This method is the callback for an HTTP request that returns a file. It will save the
// file to a temporary location and invoke the delegate with the location.
private static void ProcessFile(IAsyncResult result)
{
const string updateDir = @"updates";
const string msiname = @"WebTimer.msi";
WebServiceState state = result.AsyncState as WebServiceState;
if (state == null)
{
TraceLog.TraceError("Web Service State not found");
return;
}
// get the network operation status delegate
Delegate netOpInProgressDel = state.NetworkOperationInProgressDelegate as Delegate;
// get the web response and make sure it's not null (failed)
HttpWebResponse resp = GetWebResponse(result);
if (resp == null)
{
TraceLog.TraceError("GetWebResponse failed");
// signal that the network operation completed unsuccessfully
if (netOpInProgressDel != null)
{
netOpInProgressDel.DynamicInvoke(false, OperationStatus.Failed);
}
return;
}
// get the method-specific delegate
Delegate del = state.Delegate as Delegate;
if (del == null)
{
return; // if no delegate was passed, the results can't be processed
}
try
{
// create directory if it doesn't exist
if (!Directory.Exists(updateDir))
{
Directory.CreateDirectory(updateDir);
}
}
catch (Exception ex)
{
TraceLog.TraceException("Could not create updates directory", ex);
// signal the operation is done and unsuccessful
if (netOpInProgressDel != null)
{
netOpInProgressDel.DynamicInvoke(false, OperationStatus.Failed);
}
// can't process further
return;
}
// write the result of the GET to a temp file
//string updatepath = Path.GetRandomFileName();
var now = DateTime.Now;
string updatepath = Path.Combine(updateDir, string.Format("update-{0}", now.ToString("yyyy-MM-dd-HH-mm-ss")));
string updatefile = Path.Combine(updatepath, msiname);
try
{
TraceLog.TraceInfo(string.Format("Creating update directory {0}", updatepath));
Directory.CreateDirectory(updatepath);
TraceLog.TraceInfo(string.Format("Saving MSI in {0}", updatefile));
using (var inputStream = resp.GetResponseStream())
using (var fileStream = File.OpenWrite(updatefile))
{
byte[] buffer = new byte[8 * 1024];
int len;
while ((len = inputStream.Read(buffer, 0, buffer.Length)) > 0)
{
fileStream.Write(buffer, 0, len);
}
}
}
catch (Exception ex)
{
TraceLog.TraceException(string.Format("Could not create temporary file {0}", updatefile), ex);
// signal the operation is done and unsuccessful
if (netOpInProgressDel != null)
{
netOpInProgressDel.DynamicInvoke(false, OperationStatus.Failed);
}
return;
}
// invoke the delegate with the temp file name
try
{
del.DynamicInvoke(updatefile);
}
catch (Exception ex)
{
TraceLog.TraceException("ProcessFile: exception from DynamicInvoke", ex);
// signal the operation is done and unsuccessful
if (netOpInProgressDel != null)
{
netOpInProgressDel.DynamicInvoke(false, OperationStatus.Failed);
}
}
}
private static void InvokeWebServiceRequest_Inner(IAsyncResult res)
{
WebInvokeServiceState state = res.AsyncState as WebInvokeServiceState;
if (state == null)
{
TraceLog.TraceError("Web Service State not found");
return;
}
var request = state.Request;
var netOpInProgressDel = state.NetworkOperationInProgressDelegate as Delegate;
if (request == null)
{
TraceLog.TraceError("Web Service Request not found");
// signal the operation is done and unsuccessful
if (netOpInProgressDel != null)
{
netOpInProgressDel.DynamicInvoke(false, OperationStatus.Failed);
}
return;
}
Stream stream = null;
try
{
// this will throw if the connection can't be established
stream = request.EndGetRequestStream(res);
}
catch (Exception ex)
{
TraceLog.TraceException("Can't obtain stream", ex);
// signal the operation is done and unsuccessful
if (netOpInProgressDel != null)
{
netOpInProgressDel.DynamicInvoke(false, OperationStatus.Failed);
}
return;
}
// serialize a request body if one was passed in (and the verb will take it)
if (state.RequestBody != null && request.Method != "GET")
{
// a null request body means that the caller wants to get the stream back and write to it directly
if (state.RequestBody as Delegate != null)
{
Delegate streamDel = (Delegate)state.RequestBody;
// invoke the delegate passed in with the request stream, so that the external caller
// can push data into the stream as it becomes available
streamDel.DynamicInvoke(stream);
}
else
{
// the caller passed the complete object - so just serialize it onto the stream
if (state.RequestBody.GetType() == typeof(byte[]))
{
/*
* byte[] bytes = (byte[])state.RequestBody;
#if !IOS
* stream = new GZipStream(stream, CompressionMode.Compress);
* request.ContentType = "application/x-gzip";
#else
* stream = new MemoryStream();
* request.ContentType = "application/octet-stream";
#endif
* stream.Write(bytes, 0, bytes.Length);
*/
}
else
{
request.ContentType = "application/json";
DataContractJsonSerializer ser = new DataContractJsonSerializer(state.RequestBody.GetType());
ser.WriteObject(stream, state.RequestBody);
}
// we got all the data into the stream, so flush/close it
stream.Flush();
stream.Close();
}
}
// complete the invocation (this is not done inline because the InvokeWebServiceRequest_Inner_Complete() method
// is reused by external callers that want to write to a stream directly and then invoke the operation)
InvokeWebServiceRequest_Invoke(request, state.Delegate, state.NetworkOperationInProgressDelegate, state.Callback);
}
// Common code for invoking all the web service calls.
// GET requests will be served directly from this method,
// POST/PUT/DELETE requests are served from the InvokeWebServiceRequest_Inner method (which is an async callback)
private static void InvokeWebServiceRequest(string userCreds, string url, string verb, object obj, Delegate del, Delegate netOpInProgressDel, AsyncCallback callback)
{
// signal that a network operation is starting
if (netOpInProgressDel != null)
{
netOpInProgressDel.DynamicInvoke(true, OperationStatus.Started);
}
Uri uri = null;
if (Uri.TryCreate(url, UriKind.RelativeOrAbsolute, out uri) == false ||
uri.Scheme != "http" && uri.Scheme != "https")
{
TraceLog.TraceError("InvokeWebServiceRequest: bad URL: " + url);
return;
}
var request = (HttpWebRequest)WebRequest.Create(uri);
request.UserAgent = UserAgentString;
request.Accept = "application/json";
request.Method = verb == null ? "GET" : verb;
if (authCookie != null)
{ // send auth cookie
request.Headers[authRequestHeader] = authCookie;
}
else if (userCreds != null)
{ // send credentials in authorization header
// url form encoded
//string credentials = string.Format("UserName={0}&Password={1}", user.UserName, user.Password);
// basic auth encoded
request.Headers[authorizationHeader] = string.Format("Basic {0}", userCreds);
request.Headers[HttpApplicationHeaders.RequestedWith] = UserAgents.WebTimerWindows;
}
// set the session ID header
var sessionToken = TraceLog.Session;
if (sessionToken != null)
{
request.Headers[HttpApplicationHeaders.Session] = sessionToken;
}
// if this is a GET request, we can execute from here
if (request.Method == "GET")
{
// execute the web request and get the response
try
{
WebServiceState reqState = new WebServiceState()
{
Request = request,
Delegate = del,
NetworkOperationInProgressDelegate = netOpInProgressDel
};
request.BeginGetResponse(callback, reqState);
}
catch (Exception ex)
{
TraceLog.TraceException("Exception in BeginGetResponse", ex);
// signal that a network operation is done and unsuccessful
if (netOpInProgressDel != null)
{
netOpInProgressDel.DynamicInvoke(false, OperationStatus.Failed);
}
}
}
else
{
// this is a request that contains a body (PUT, POST, DELETE)
// need to nest another async call - this time to get the request stream
try
{
request.BeginGetRequestStream(
new AsyncCallback(InvokeWebServiceRequest_Inner),
new WebInvokeServiceState()
{
Request = request,
Callback = callback,
Delegate = del,
NetworkOperationInProgressDelegate = netOpInProgressDel,
RequestBody = obj
});
}
catch (Exception ex)
{
// trace the exception
TraceLog.TraceException("Exception in BeginGetResponse", ex);
// signal that a network operation is done and unsuccessful
if (netOpInProgressDel != null)
{
netOpInProgressDel.DynamicInvoke(false, OperationStatus.Failed);
}
}
}
}
private static void Send()
{
try
{
// snap the record list
var recordList = CollectorClient.GetRecords();
// if no records, may need to recover capture service
if (recordList.Count == 0)
{
if (recovering)
{
// this is the second timeout period that we didn't see records.
// attempt to restart the collector
CollectorClient.Start();
recovering = false;
}
else
{
// this will trigger recovery if the next run through also has zero records
recovering = true;
}
// nothing to do this time (no records to process)
return;
}
else
{
// reset recovery mode
recovering = false;
}
// compact the record list
var uniqueHosts = recordList.Select(r => r.HostMacAddress).Distinct();
foreach (var host in uniqueHosts)
{
var uniqueDests = recordList.Where(r => r.HostMacAddress == host).Select(r => r.WebsiteName).Distinct();
foreach (var dest in uniqueDests)
{
var records = recordList.Where(r => r.HostMacAddress == host && r.WebsiteName == dest).OrderBy(r => r.Timestamp);
// only keep the first and last record so that the duration calculated on the server will be > 0
var firstRecord = records.FirstOrDefault();
var lastRecord = records.LastOrDefault();
// if the last record is a second or more later, add the duration to the first record
TimeSpan duration;
if (lastRecord != null && lastRecord != firstRecord &&
(duration = Convert.ToDateTime(lastRecord.Timestamp) - Convert.ToDateTime(firstRecord.Timestamp))
>= TimeSpan.FromSeconds(1.0))
{
firstRecord.Duration = (int)duration.TotalSeconds;
}
lock (sendQueue)
{
var existingRecord = sendQueue.FirstOrDefault(r => r.HostMacAddress == host && r.WebsiteName == dest);
if (existingRecord == null ||
Convert.ToDateTime(firstRecord.Timestamp) - (Convert.ToDateTime(existingRecord.Timestamp) + TimeSpan.FromSeconds(existingRecord.Duration)) >= TimeSpan.FromSeconds(300.0)) // more than 5 minutes between records
{
// add the record to the send queue
sendQueue.Add(firstRecord);
}
else
{
// augment existing record with new duration
duration = Convert.ToDateTime(firstRecord.Timestamp) - Convert.ToDateTime(existingRecord.Timestamp) + TimeSpan.FromSeconds(firstRecord.Duration);
existingRecord.Duration = (int)duration.TotalSeconds;
}
}
}
}
if (sendQueue.Count > 0)
{
// snap the send queue
var recordBuffer = new List <Record>();
lock (sendQueue)
{
// clear this buffer if collection is suspended
if (collectionStatus == ControlMessage.SuspendCollection)
{
sendQueue.Clear();
TraceLog.TraceInfo("Probing for new collection status");
}
else
{
recordBuffer.AddRange(sendQueue);
TraceLog.TraceInfo(String.Format("Sending records: {0}", JsonConvert.SerializeObject(sendQueue)));
}
}
var sendList = new RecordList()
{
DeviceId = deviceId,
DeviceName = deviceName,
SoftwareVersion = UpdateClient.CurrentVersion.ToString(),
Records = recordBuffer
};
// send the queue to the web service
WebServiceHelper.PostRecords(
credentials,
sendList,
new WebServiceHelper.PostRecordsDelegate((response) =>
{
// a callback means the service processed the records successfully; we can free them now.
// note a slight race condition - if the callback takes longer than the sleep interval, we
// could lose a buffer. since the sleep interval is 60sec, this isn't an important issue
// to address.
lock (sendQueue)
{
sendQueue.Clear();
}
var msg = (ControlMessage)response.controlMessage;
if (collectionStatus == msg)
{
return;
}
// process the control message
lock (configLock)
{
switch (msg)
{
case ControlMessage.Normal:
TraceLog.TraceInfo("Changing collection status to Normal");
collectionStatus = msg;
ConfigClient.Write(ConfigClient.Disabled, Convert.ToString(false));
break;
case ControlMessage.SuspendCollection:
TraceLog.TraceInfo("Changing collection status to Suspended");
collectionStatus = msg;
ConfigClient.Write(ConfigClient.Disabled, Convert.ToString(true));
break;
case ControlMessage.DisableDevice:
TraceLog.TraceInfo("Disabling the device by wiping the config");
collectionStatus = msg;
deviceId = null;
deviceName = null;
// wipe the config
ConfigClient.Clear();
break;
default:
// new message we don't understand yet
TraceLog.TraceInfo(string.Format("Received unknown control message {0}", msg));
break;
}
}
}),
new WebServiceHelper.NetOpDelegate((inProgress, status) =>
{
if (status == OperationStatus.Retry)
{
// no need to do anything: next iteration will send original credentials
}
// no failure state to clean up
}));
}
}
catch (Exception ex)
{
TraceLog.TraceException("Upload processing failed", ex);
}
}
