/// <summary>
/// Create a Threat List Update Request Model.
/// </summary>
/// <param name="this">
/// A <see cref="ThreatListUpdateRequest" />.
/// </param>
/// <returns>
/// A <see cref="ThreatListUpdateRequestModel" /> if <paramref name="this" /> is not a null reference. A
/// null reference otherwise.
/// </returns>
internal static ThreatListUpdateRequestModel AsThreatListUpdateRequestModel(this ThreatListUpdateRequest @this)
{
ThreatListUpdateRequestModel threatListUpdateRequestModel = null;
if (@this != null)
{
threatListUpdateRequestModel = new ThreatListUpdateRequestModel();
threatListUpdateRequestModel.ClientMetadata = @this.ClientMetadata.AsClientMetadataModel();
threatListUpdateRequestModel.Queries = @this.Queries.Select(m => m.AsThreatListUpdateQueryModel());
}
return(threatListUpdateRequestModel);
}
/// <summary>
/// Get Threat List Updates Asynchronously.
/// </summary>
/// <param name="request">
/// A <see cref="ThreatListUpdateRequest" />.
/// </param>
/// <param name="cancellationToken">
/// A cancellation token to cancel the asynchronous operation with.
/// </param>
/// <returns>
/// A <see cref="ThreatListUpdateResponse" />.
/// </returns>
/// <exception cref="Gee.External.Browsing.Clients.BrowsingClientException">
/// Thrown if an error communicating with the Google Safe Browsing API occurs.
/// </exception>
/// <exception cref="System.ArgumentNullException">
/// Thrown if <paramref name="request" /> is a null reference.
/// </exception>
/// <exception cref="System.ObjectDisposedException">
/// Thrown if the object is disposed.
/// </exception>
/// <exception cref="System.OperationCanceledException">
/// Thrown if the asynchronous operation is cancelled.
/// </exception>
/// <exception cref="System.TimeoutException">
/// Thrown if communication with the Google Safe Browsing API times out.
/// </exception>
public Task <ThreatListUpdateResponse> GetThreatListUpdatesAsync(ThreatListUpdateRequest request, CancellationToken cancellationToken)
{
this.ThrowIfDisposed();
Guard.ThrowIf(nameof(request), request).Null();
var getThreatListUpdatesTask = GetThreatListUpdatesAsync(request, cancellationToken);
return(getThreatListUpdatesTask);
// <summary>
// Get Threat List Updates Asynchronously.
// </summary>
async Task <ThreatListUpdateResponse> GetThreatListUpdatesAsync(ThreatListUpdateRequest cRequest, CancellationToken cCancellationToken)
{
try {
// ...
//
// Throws an exception if the operation fails.
var cRequestModel = cRequest.AsThreatListUpdateRequestModel();
var cResponseModel = await HttpBrowsingClient.GetThreatListUpdatesUri
.PostJsonAsync(cRequestModel, cCancellationToken)
.ReceiveJson <ThreatListUpdateResponseModel>()
.ConfigureAwait(false);
var cResponse = cResponseModel.AsThreatListUpdateResponse(cRequest);
return(cResponse);
}
catch (FlurlHttpTimeoutException cEx) {
const string cDetailMessage = "An HTTP request to the Google Safe Browsing API timed out.";
throw new TimeoutException(cDetailMessage, cEx);
}
catch (FlurlHttpException cEx) {
var httpStatusCode = cEx.Call.HttpStatus ?? HttpStatusCode.BadRequest;
const string cDetailMessage = "An HTTP request to the Google Safe Browsing API failed.";
throw new BrowsingClientException(cDetailMessage, httpStatusCode, cEx);
}
}
}
/// <summary>
/// Create a Threat List Update Response.
/// </summary>
/// <param name="this">
/// A <see cref="ThreatListUpdateResponseModel" />.
/// </param>
/// <param name="request">
/// The <see cref="ThreatListUpdateRequest" /> made to the Google Safe Browsing API for which the
/// <see cref="ThreatListUpdateResponse" /> has been returned.
/// </param>
/// <returns>
/// A <see cref="ThreatListUpdateResponse" /> if <paramref name="this" /> is not a null reference. A null
/// reference otherwise.
/// </returns>
/// <exception cref="System.ArgumentNullException">
/// Thrown if <paramref name="request" /> is a null reference.
/// </exception>
internal static ThreatListUpdateResponse AsThreatListUpdateResponse(this ThreatListUpdateResponseModel @this, ThreatListUpdateRequest request)
{
ThreatListUpdateResponse threatListUpdateResponse = null;
if (@this != null)
{
// ...
//
// Throws an exception if the operation fails.
var threatListUpdateResponseBuilder = ThreatListUpdateResponse.Build();
threatListUpdateResponseBuilder.SetRequest(request);
SetResults(@this, threatListUpdateResponseBuilder, request);
threatListUpdateResponse = threatListUpdateResponseBuilder.Build();
}
return(threatListUpdateResponse);
// <summary>
// Create Wait to Date.
// </summary>
DateTime?CreateWaitToDate(ThreatListUpdateResponseModel cThis)
{
DateTime?cWaitToDate = null;
if (cThis.WaitDuration != null)
{
var cWaitDuration = cThis.WaitDuration.Substring(0, cThis.WaitDuration.Length - 1);
var cIsWaitDurationParsed = double.TryParse(cWaitDuration, out var cWaitDurationDouble);
if (cIsWaitDurationParsed)
{
cWaitToDate = DateTime.UtcNow.AddSeconds(cWaitDurationDouble);
}
}
return(cWaitToDate);
}
// <summary>
// Set Results.
// </summary>
void SetResults(ThreatListUpdateResponseModel cThis, ThreatListUpdateResponseBuilder cBuilder, ThreatListUpdateRequest cRequest)
{
if (cThis.Results != null)
{
var cThreatListRetrieveDate = DateTime.UtcNow;
var cThreatListWaitToDate = CreateWaitToDate(cThis);
foreach (var cResultModel in cThis.Results)
{
var cResult = cResultModel.AsThreatListUpdateResult(cRequest, cThreatListRetrieveDate, cThreatListWaitToDate);
cBuilder.AddResult(cResult);
}
}
}
}
private async Task SynchronizeDatabaseAsync()
{
var cancellationToken = this._synchronizationTaskCancellationTokenSource.Token;
while (!cancellationToken.IsCancellationRequested)
{
var getThreatListUpdatesTask = GetThreatListUpdatesAsync(this);
var(threatListUpdateResponse, delayToDate) = await getThreatListUpdatesTask.ConfigureAwait(false);
if (threatListUpdateResponse != null)
{
foreach (var threatListUpdateResult in threatListUpdateResponse.Results)
{
var threatListWaitToDate = threatListUpdateResult.RetrievedThreatList.WaitToDate;
if (threatListWaitToDate != null && threatListWaitToDate < delayToDate)
{
delayToDate = threatListWaitToDate.Value;
}
var synchronizeThreatListTask = SynchronizeThreatListAsync(this, threatListUpdateResult);
await synchronizeThreatListTask.ConfigureAwait(false);
}
}
var delayToTask = DelayToAsync(this, delayToDate);
await delayToTask.ConfigureAwait(false);
}
// <summary>
// Delay to a Date Asynchronously.
// </summary>
async Task DelayToAsync(BrowsingDatabaseManager @this, DateTime cDate)
{
try {
var cCancellationToken = @this._synchronizationTaskCancellationTokenSource.Token;
var cDelayToTask = TaskExtension.DelayTo(cDate, cCancellationToken);
await cDelayToTask.ConfigureAwait(false);
}
catch (OperationCanceledException) {
// ...
//
// We don't care if the cancellation token is cancelled.
}
}
// <summary>
// Get Threat List Updates Asynchronously.
// </summary>
async Task <(ThreatListUpdateResponse, DateTime)> GetThreatListUpdatesAsync(BrowsingDatabaseManager @this)
{
var cDelayToDate = DateTime.UtcNow.AddMinutes(30);
try {
IEnumerable <ThreatListDescriptor> cThreatListDescriptors = @this._updateConstraints.Keys;
if (@this._updateConstraints.Count == 0)
{
// ...
//
// If the database synchronizer is not restricted to specific threat lists, we retrieve all
// available threat lists from the Google Safe Browsing API. We don't cache the result in case
// new threat lists are made available between synchronization iterations.
//
// Throws an exception if the operation fails.
var cGetThreatListDescriptorsTask = @this._client.GetThreatListDescriptorsAsync();
cThreatListDescriptors = await cGetThreatListDescriptorsTask.ConfigureAwait(false);
}
// ...
//
// Retrieve the threat lists from the database. Throws an exception if the operation fails.
var cGetThreatListsTask = @this._database.GetThreatListsAsync(cThreatListDescriptors);
var cThreatLists = await cGetThreatListsTask.ConfigureAwait(false);
ThreatListUpdateRequestBuilder cThreatListUpdateRequestBuilder = null;
foreach (var cThreatList in cThreatLists)
{
if (cThreatList.Expired)
{
cThreatListUpdateRequestBuilder = cThreatListUpdateRequestBuilder ?? ThreatListUpdateRequest.Build();
var cThreatListDescriptor = cThreatList.Descriptor;
var cThreatListState = cThreatList.State;
@this._updateConstraints.TryGetValue(cThreatListDescriptor, out var cThreatListUpdateConstraints);
cThreatListUpdateRequestBuilder.AddQuery(b => {
b.SetThreatListDescriptor(cThreatListDescriptor);
b.SetThreatListState(cThreatListState);
b.SetUpdateConstraints(cThreatListUpdateConstraints);
return(b.Build());
});
}
else
{
if (cThreatList.WaitToDate != null && cThreatList.WaitToDate < cDelayToDate)
{
// ...
//
// If the current threat list's wait to date is earlier than the previous threat
// list's wait to date, take the current the threat list's wait to date instead.
cDelayToDate = cThreatList.WaitToDate.Value;
}
}
}
ThreatListUpdateResponse cThreatListUpdateResponse = null;
if (cThreatListUpdateRequestBuilder != null)
{
// ...
//
// Throws an exception if the operation fails.
var cThreatListUpdateRequest = cThreatListUpdateRequestBuilder.Build();
var cGetThreatListUpdatesTask = @this._client.GetThreatListUpdatesAsync(cThreatListUpdateRequest);
cThreatListUpdateResponse = await cGetThreatListUpdatesTask.ConfigureAwait(false);
}
return(cThreatListUpdateResponse, cDelayToDate);
}
catch {
return(null, cDelayToDate);
}
}
// <summary>
// Synchronize Threat List Asynchronously.
// </summary>
async Task SynchronizeThreatListAsync(BrowsingDatabaseManager @this, ThreatListUpdateResult cThreatListUpdateResult)
{
var cSynchronizationStartDate = DateTime.UtcNow;
var cThreatList = cThreatListUpdateResult.RetrievedThreatList;
try {
var cSha256HashPrefixes = cThreatListUpdateResult.ThreatsToAdd;
if (cThreatListUpdateResult.IsFullUpdate)
{
// ...
//
// Throws an exception if the operation fails.
var cModifyThreatListTask = @this._database.StoreThreatListAsync(cThreatList, cSha256HashPrefixes);
await cModifyThreatListTask.ConfigureAwait(false);
}
else if (cThreatListUpdateResult.IsPartialUpdate)
{
// ...
//
// Throws an exception if the operation fails.
var cIndices = cThreatListUpdateResult.ThreatsToRemove;
var cModifyThreatListTask = @this._database.ModifyThreatListAsync(cThreatList, cSha256HashPrefixes, cIndices);
await cModifyThreatListTask.ConfigureAwait(false);
}
// ...
//
// Throws an exception if the operation fails.
var cComputeThreatListChecksumTask = @this._database.ComputeThreatListChecksumAsync(cThreatList.Descriptor);
var cThreatListChecksum = await cComputeThreatListChecksumTask.ConfigureAwait(false);
if (cThreatListChecksum != cThreatListUpdateResult.RetrievedThreatListChecksum)
{
// ...
//
// Throws an exception if the operation fails.
cThreatList = ThreatList.CreateInvalid(cThreatList.Descriptor);
var cUpdateConstraints = cThreatListUpdateResult.Query.UpdateConstraints;
var cGetThreatListUpdatesTask = @this._client.GetThreatListUpdatesAsync(cThreatList, cUpdateConstraints);
var cThreatListUpdateRequest = await cGetThreatListUpdatesTask.ConfigureAwait(false);
cThreatListUpdateResult = cThreatListUpdateRequest.Results.First();
cSha256HashPrefixes = cThreatListUpdateResult.ThreatsToAdd;
if (cThreatListUpdateResult.IsFullUpdate)
{
// ...
//
// Throws an exception if the operation fails.
cThreatList = cThreatListUpdateResult.RetrievedThreatList;
var cModifyThreatListTask = @this._database.StoreThreatListAsync(cThreatList, cSha256HashPrefixes);
await cModifyThreatListTask.ConfigureAwait(false);
}
}
// ...
//
// Invoke threat list synchronization completed event.
var cSynchronizationCompletionDate = DateTime.UtcNow;
@this.OnThreatListSynchronizationCompleted(new ThreatListSynchronizationCompletedEventArgs(
cThreatList,
cSynchronizationStartDate,
cSynchronizationCompletionDate
));
}
catch (Exception cEx) {
// ...
//
// Invoke threat list synchronization failed event.
var cSynchronizationFailureDate = DateTime.UtcNow;
@this.OnThreatListSynchronizationFailed(new ThreatListSynchronizationFailedEventArgs(
cThreatList,
cSynchronizationStartDate,
cSynchronizationFailureDate,
cEx
));
}
}
}
/// <summary>
/// Create a Threat List Update Result.
/// </summary>
/// <param name="this">
/// A <see cref="ThreatListUpdateResultModel" />.
/// </param>
/// <param name="request">
/// The <see cref="ThreatListUpdateRequest" /> made to the Google Safe Browsing API for which the
/// <see cref="ThreatListUpdateResult" /> has been returned.
/// </param>
/// <param name="threatListRetrieveDate">
/// The date, in Coordinated Universal Time (UTC), the <see cref="ThreatList" /> was retrieved from the
/// Google Safe Browsing API. If the date is not expressed in UTC, it is converted to it.
/// </param>
/// <param name="threatListWaitToDate">
/// The date, in Coordinated Universal Time (UTC), a client must wait to before retrieving the
/// <see cref="ThreatList" /> from the Google Safe Browsing API again. If the date is not expressed in UTC,
/// it is converted to it. A null reference indicates a client does not need to wait.
/// </param>
/// <returns>
/// A <see cref="ThreatListUpdateResult" /> if <paramref name="this" /> is not a null reference. A null
/// reference otherwise.
/// </returns>
/// <exception cref="System.ArgumentNullException">
/// Thrown if <paramref name="request" /> is a null reference.
/// </exception>
internal static ThreatListUpdateResult AsThreatListUpdateResult(this ThreatListUpdateResultModel @this, ThreatListUpdateRequest request, DateTime threatListRetrieveDate, DateTime?threatListWaitToDate)
{
ThreatListUpdateResult threatListUpdateResult = null;
if (@this != null)
{
Guard.ThrowIf(nameof(request), request).Null();
var threatListUpdateResultBuilder = ThreatListUpdateResult.Build();
SetThreatsToAdd(@this, threatListUpdateResultBuilder);
SetThreatsToRemove(@this, threatListUpdateResultBuilder);
SetRetrievedThreatList(@this, threatListUpdateResultBuilder, threatListRetrieveDate, threatListWaitToDate);
SetRetrievedThreatListChecksum(@this, threatListUpdateResultBuilder);
SetUpdateType(@this, threatListUpdateResultBuilder);
// ...
//
// We need to set the query after the retrieved threat list is set.
SetQuery(threatListUpdateResultBuilder, request);
threatListUpdateResult = threatListUpdateResultBuilder.Build();
}
return(threatListUpdateResult);
// <summary>
// Set Query.
// </summary>
void SetQuery(ThreatListUpdateResultBuilder cBuilder, ThreatListUpdateRequest cRequest)
{
foreach (var cQuery in cRequest.Queries)
{
var cIsMatchedQuery = cQuery.ThreatListDescriptor.Equals(cBuilder.RetrievedThreatList.Descriptor);
if (cIsMatchedQuery)
{
cBuilder.SetQuery(cQuery);
break;
}
}
}
// <summary>
// Set Retrieved Threat List.
// </summary>
void SetRetrievedThreatList(ThreatListUpdateResultModel cModel, ThreatListUpdateResultBuilder cBuilder, DateTime cThreatListRetrieveDate, DateTime?cThreatListWaitToDate)
{
var cPlatformType = cModel.PlatformType.AsPlatformType();
var cState = cModel.ThreatListState.Base64Decode().HexadecimalEncode();
var cThreatEntryType = cModel.ThreatEntryType.AsThreatEntryType();
var cThreatType = cModel.ThreatType.AsThreatType();
cBuilder.SetRetrievedThreatList(b => {
b.SetDescriptor(cThreatType, cPlatformType, cThreatEntryType);
b.SetRetrieveDate(cThreatListRetrieveDate);
b.SetState(cState);
b.SetWaitToDate(cThreatListWaitToDate);
return(b.Build());
});
}
// <summary>
// Set Retrieved Threat List's Checksum.
// </summary>
void SetRetrievedThreatListChecksum(ThreatListUpdateResultModel cModel, ThreatListUpdateResultBuilder cBuilder)
{
var cRetrievedThreatListChecksum = cModel.Checksum.Sha256Hash.Base64Decode().HexadecimalEncode();
cBuilder.SetRetrievedThreatListChecksum(cRetrievedThreatListChecksum);
}
// <summary>
// Set Threats to Add.
// </summary>
void SetThreatsToAdd(ThreatListUpdateResultModel cModel, ThreatListUpdateResultBuilder cBuilder)
{
if (cModel.ThreatsToAdd != null)
{
foreach (var cThreatToAdd in cModel.ThreatsToAdd)
{
var cCompressionType = cThreatToAdd.CompressionType.AsCompressionType();
if (cCompressionType == CompressionType.Uncompressed)
{
// ...
//
// It's faster to 1) Base64 decode the SHA256 hashes, 2) hexadecimal encode them all, and
// 3) split them into subsets rather then 1) Base64 decode the SHA256 hashes and 2) encode
// them individually.
var cSha256Hashes = cThreatToAdd.UncompressedSha256HashPrefixes
.Sha256HashPrefixes
.Base64Decode()
.HexadecimalEncode();
// ...
//
// 1 byte is equal to 2 hexadecimal characters.
var cSha256HashPrefixSize = cThreatToAdd.UncompressedSha256HashPrefixes.Sha256HashPrefixSize * 2;
for (var cI = 0; cI < cSha256Hashes.Length; cI += cSha256HashPrefixSize)
{
var cSha256Hash = cSha256Hashes.Substring(cI, cSha256HashPrefixSize);
cBuilder.AddThreatToAdd(cSha256Hash);
}
}
}
}
}
// <summary>
// Set Threats to Remove.
// </summary>
void SetThreatsToRemove(ThreatListUpdateResultModel cModel, ThreatListUpdateResultBuilder cBuilder)
{
if (cModel.ThreatsToRemove != null)
{
foreach (var cThreatToRemove in cModel.ThreatsToRemove)
{
var cCompressionType = cThreatToRemove.CompressionType.AsCompressionType();
if (cCompressionType == CompressionType.Uncompressed)
{
var cIndices = cThreatToRemove.UncompressedIndices.Indices;
foreach (var cIndex in cIndices)
{
cBuilder.AddThreatToRemove(cIndex);
}
}
}
}
}
// <summary>
// Set Update Type.
// </summary>
void SetUpdateType(ThreatListUpdateResultModel cModel, ThreatListUpdateResultBuilder cBuilder)
{
var cUpdateType = cModel.UpdateType.AsThreatListUpdateType();
cBuilder.SetUpdateType(cUpdateType);
}
}