private static bool TryGetEasyRequest(IntPtr curlPtr, out EasyRequest easy)
{
Debug.Assert(curlPtr != IntPtr.Zero, "curlPtr is not null");
IntPtr gcHandlePtr;
CURLcode getInfoResult = Interop.Http.EasyGetInfoPointer(curlPtr, CURLINFO.CURLINFO_PRIVATE, out gcHandlePtr);
Debug.Assert(getInfoResult == CURLcode.CURLE_OK, "Failed to get info on a completing easy handle");
if (getInfoResult == CURLcode.CURLE_OK)
{
try
{
GCHandle handle = GCHandle.FromIntPtr(gcHandlePtr);
easy = (EasyRequest)handle.Target;
Debug.Assert(easy != null, "Expected non-null EasyRequest in GCHandle");
return(easy != null);
}
catch (Exception e)
{
EventSourceTrace("Error getting state from GCHandle: {0}", e);
Debug.Fail($"Exception in {nameof(TryGetEasyRequest)}", e.ToString());
}
}
easy = null;
return(false);
}
private static void SetSslOptionsForSupportedBackend(EasyRequest easy, ClientCertificateProvider certProvider, IntPtr userPointer)
{
CURLcode answer = easy.SetSslCtxCallback(s_sslCtxCallback, userPointer);
EventSourceTrace("Callback registration result: {0}", answer, easy: easy);
switch (answer)
{
case CURLcode.CURLE_OK:
// We successfully registered. If we'll be invoking a user-provided callback to verify the server
// certificate as part of that, disable libcurl's verification of the host name; we need to get
// the callback from libcurl even if the host name doesn't match, so we take on the responsibility
// of doing the host name match in the callback prior to invoking the user's delegate.
if (easy._handler.ServerCertificateCustomValidationCallback != null)
{
easy.SetCurlOption(Interop.Http.CURLoption.CURLOPT_SSL_VERIFYHOST, 0);
// But don't change the CURLOPT_SSL_VERIFYPEER setting, as setting it to 0 will
// cause SSL and libcurl to ignore the result of the server callback.
}
SetSslOptionsForCertificateStore(easy);
// The allowed SSL protocols will be set in the configuration callback.
break;
case CURLcode.CURLE_UNKNOWN_OPTION: // Curl 7.38 and prior
case CURLcode.CURLE_NOT_BUILT_IN: // Curl 7.39 and later
SetSslOptionsForUnsupportedBackend(easy, certProvider);
break;
default:
ThrowIfCURLEError(answer);
break;
}
}
private static void ThrowIfCURLEError(CURLcode error)
{
if (error != CURLcode.CURLE_OK)
{
var inner = new CurlException((int)error, isMulti: false);
VerboseTrace(inner.Message);
throw inner;
}
}
internal CURLcode SetSslCtxCallback(SslCtxCallback callback, IntPtr userPointer)
{
if (_callbackHandle == null)
{
_callbackHandle = new SafeCallbackHandle();
}
CURLcode result = Interop.Http.RegisterSslCtxCallback(_easyHandle, callback, userPointer, ref _callbackHandle);
return(result);
}
private static bool TryGetEasyRequest(IntPtr curlPtr, out EasyRequest easy)
{
Debug.Assert(curlPtr != IntPtr.Zero, "curlPtr is not null");
IntPtr gcHandlePtr;
CURLcode getInfoResult = Interop.Http.EasyGetInfoPointer(curlPtr, CURLINFO.CURLINFO_PRIVATE, out gcHandlePtr);
if (getInfoResult == CURLcode.CURLE_OK)
{
return(MultiAgent.TryGetEasyRequestFromGCHandle(gcHandlePtr, out easy));
}
Debug.Fail($"Failed to get info on a completing easy handle: {getInfoResult}");
easy = null;
return(false);
}
internal static void SetSslOptions(EasyRequest easy, ClientCertificateOption clientCertOption)
{
// Disable SSLv2/SSLv3, allow TLSv1.*
easy.SetCurlOption(Interop.Http.CURLoption.CURLOPT_SSLVERSION, (long)Interop.Http.CurlSslVersion.CURL_SSLVERSION_TLSv1);
IntPtr userPointer = IntPtr.Zero;
if (clientCertOption == ClientCertificateOption.Automatic)
{
ClientCertificateProvider certProvider = new ClientCertificateProvider();
userPointer = GCHandle.ToIntPtr(certProvider._gcHandle);
easy.Task.ContinueWith((_, state) => ((IDisposable)state).Dispose(),
certProvider,
CancellationToken.None,
TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Default);
}
else
{
Debug.Assert(clientCertOption == ClientCertificateOption.Manual, "ClientCertificateOption is manual or automatic");
}
CURLcode answer = easy.SetSslCtxCallback(s_sslCtxCallback, userPointer);
switch (answer)
{
case CURLcode.CURLE_OK:
break;
// Curl 7.38 and prior
case CURLcode.CURLE_UNKNOWN_OPTION:
// Curl 7.39 and later
case CURLcode.CURLE_NOT_BUILT_IN:
EventSourceTrace("CURLOPT_SSL_CTX_FUNCTION not supported. Platform default HTTPS chain building in use");
if (clientCertOption == ClientCertificateOption.Automatic)
{
throw new PlatformNotSupportedException(SR.net_http_unix_invalid_client_cert_option);
}
break;
default:
ThrowIfCURLEError(answer);
break;
}
}
private static bool TryGetEasyRequest(IntPtr curlPtr, out EasyRequest easy)
{
Debug.Assert(curlPtr != IntPtr.Zero, "curlPtr is not null");
IntPtr gcHandlePtr;
CURLcode getInfoResult = Interop.Http.EasyGetInfoPointer(curlPtr, CURLINFO.CURLINFO_PRIVATE, out gcHandlePtr);
Debug.Assert(getInfoResult == CURLcode.CURLE_OK, "Failed to get info on a completing easy handle");
if (getInfoResult == CURLcode.CURLE_OK)
{
GCHandle handle = GCHandle.FromIntPtr(gcHandlePtr);
easy = handle.Target as EasyRequest;
Debug.Assert(easy != null, "Expected non-null EasyRequest in GCHandle");
return easy != null;
}
easy = null;
return false;
}
private static void AddChannelBindingToken(X509Certificate2 certificate, IntPtr curlPtr)
{
Debug.Assert(curlPtr != IntPtr.Zero, "curlPtr is not null");
IntPtr gcHandlePtr;
CURLcode getInfoResult = Interop.Http.EasyGetInfoPointer(curlPtr, CURLINFO.CURLINFO_PRIVATE, out gcHandlePtr);
Debug.Assert(getInfoResult == CURLcode.CURLE_OK, "Failed to get info on a completing easy handle");
if (getInfoResult == CURLcode.CURLE_OK)
{
GCHandle handle = GCHandle.FromIntPtr(gcHandlePtr);
EasyRequest easy = handle.Target as EasyRequest;
Debug.Assert(easy != null, "Expected non-null EasyRequest in GCHandle");
if (easy._requestContentStream != null)
{
easy._requestContentStream.SetChannelBindingToken(certificate);
}
}
}
internal void StoreLastEffectiveUri()
{
IntPtr urlCharPtr; // do not free; will point to libcurl private memory
CURLcode urlResult = Interop.Http.EasyGetInfoPointer(_easyHandle, Interop.Http.CURLINFO.CURLINFO_EFFECTIVE_URL, out urlCharPtr);
if (urlResult == CURLcode.CURLE_OK && urlCharPtr != IntPtr.Zero)
{
string url = Marshal.PtrToStringAnsi(urlCharPtr);
Uri finalUri;
if (Uri.TryCreate(url, UriKind.Absolute, out finalUri))
{
_requestMessage.RequestUri = finalUri;
return;
}
}
Debug.Fail("Expected to be able to get the last effective Uri from libcurl");
}
private void HandleIncomingRequest(SafeCurlMultiHandle multiHandle, IncomingRequest request)
{
Debug.Assert(!Monitor.IsEntered(_incomingRequests), "Incoming requests lock should only be held while accessing the queue");
VerboseTrace("Type: " + request.Type, easy: request.Easy);
EasyRequest easy = request.Easy;
switch (request.Type)
{
case IncomingRequestType.New:
ActivateNewRequest(multiHandle, easy);
break;
case IncomingRequestType.Cancel:
Debug.Assert(easy._associatedMultiAgent == this, "Should only cancel associated easy requests");
Debug.Assert(easy._cancellationToken.IsCancellationRequested, "Cancellation should have been requested");
FindAndFailActiveRequest(multiHandle, easy, new OperationCanceledException(easy._cancellationToken));
break;
case IncomingRequestType.Unpause:
Debug.Assert(easy._associatedMultiAgent == this, "Should only unpause associated easy requests");
if (!easy._easyHandle.IsClosed)
{
IntPtr gcHandlePtr;
ActiveRequest ar;
Debug.Assert(FindActiveRequest(easy, out gcHandlePtr, out ar), "Couldn't find active request for unpause");
CURLcode unpauseResult = Interop.Http.EasyUnpause(easy._easyHandle);
try
{
ThrowIfCURLEError(unpauseResult);
}
catch (Exception exc)
{
FindAndFailActiveRequest(multiHandle, easy, exc);
}
}
break;
default:
Debug.Fail("Invalid request type: " + request.Type);
break;
}
}
private static void ThrowIfCURLEError(CURLcode error)
{
if (error != CURLcode.CURLE_OK) // success
{
string msg = CurlException.GetCurlErrorString((int)error, isMulti: false);
EventSourceTrace(msg);
switch (error)
{
case CURLcode.CURLE_OPERATION_TIMEDOUT:
throw new OperationCanceledException(msg);
case CURLcode.CURLE_OUT_OF_MEMORY:
throw new OutOfMemoryException(msg);
default:
throw new CurlException((int)error, msg);
}
}
}
private void FinishRequest(EasyRequest completedOperation, CURLcode messageResult)
{
VerboseTrace("messageResult: " + messageResult, easy: completedOperation);
if (completedOperation._responseMessage.StatusCode != HttpStatusCode.Unauthorized)
{
if (completedOperation._handler.PreAuthenticate)
{
long authAvailable;
if (Interop.Http.EasyGetInfoLong(completedOperation._easyHandle, CURLINFO.CURLINFO_HTTPAUTH_AVAIL, out authAvailable) == CURLcode.CURLE_OK)
{
completedOperation._handler.AddCredentialToCache(
completedOperation._requestMessage.RequestUri, (CURLAUTH)authAvailable, completedOperation._networkCredential);
}
// Ignore errors: no need to fail for the sake of putting the credentials into the cache
}
completedOperation._handler.AddResponseCookies(
completedOperation._requestMessage.RequestUri, completedOperation._responseMessage);
}
// Complete or fail the request
try
{
bool unsupportedProtocolRedirect = messageResult == CURLcode.CURLE_UNSUPPORTED_PROTOCOL && completedOperation._isRedirect;
if (!unsupportedProtocolRedirect)
{
ThrowIfCURLEError(messageResult);
}
completedOperation.EnsureResponseMessagePublished();
}
catch (Exception exc)
{
completedOperation.FailRequest(exc);
}
// At this point, we've completed processing the entire request, either due to error
// or due to completing the entire response.
completedOperation.Cleanup();
}
private void SetVersion()
{
Version v = _requestMessage.Version;
if (v != null)
{
// Try to use the requested version, if a known version was explicitly requested.
// If an unknown version was requested, we simply use libcurl's default.
// Only allow HTTP/2 when making https requests.
var curlVersion =
(v.Major == 1 && v.Minor == 1) ? Interop.Http.CurlHttpVersion.CURL_HTTP_VERSION_1_1 :
(v.Major == 1 && v.Minor == 0) ? Interop.Http.CurlHttpVersion.CURL_HTTP_VERSION_1_0 :
(v.Major == 2 && v.Minor == 0) ? Interop.Http.CurlHttpVersion.CURL_HTTP_VERSION_2TLS :
Interop.Http.CurlHttpVersion.CURL_HTTP_VERSION_NONE;
if (curlVersion != Interop.Http.CurlHttpVersion.CURL_HTTP_VERSION_NONE)
{
// Ask libcurl to use the specified version if possible.
CURLcode c = Interop.Http.EasySetOptionLong(_easyHandle, CURLoption.CURLOPT_HTTP_VERSION, (long)curlVersion);
if (c == CURLcode.CURLE_OK)
{
// Success. The requested version will be used.
EventSourceTrace("HTTP version: {0}", v);
}
else if (c == CURLcode.CURLE_UNSUPPORTED_PROTOCOL)
{
// The requested version is unsupported. Fall back to using the default version chosen by libcurl.
EventSourceTrace("Unsupported protocol: {0}", v);
}
else
{
// Some other error. Fail.
ThrowIfCURLEError(c);
}
}
}
}
internal static void SetSslOptions(EasyRequest easy)
{
CURLcode answer = Interop.Http.EasySetOptionPointer(
easy._easyHandle,
Interop.Http.CURLoption.CURLOPT_SSL_CTX_FUNCTION,
s_sslCtxCallback);
switch (answer)
{
case CURLcode.CURLE_OK:
break;
// Curl 7.38 and prior
case CURLcode.CURLE_UNKNOWN_OPTION:
// Curl 7.39 and later
case CURLcode.CURLE_NOT_BUILT_IN:
VerboseTrace("CURLOPT_SSL_CTX_FUNCTION is not supported, platform default https chain building in use");
break;
default:
ThrowIfCURLEError(answer);
break;
}
}
private static byte[] DownloadAsset(string uri, ref TimeSpan remainingDownloadTime)
{
if (remainingDownloadTime <= TimeSpan.Zero)
{
return(null);
}
List <byte[]> dataPieces = new List <byte[]>();
using (Interop.Http.SafeCurlHandle curlHandle = Interop.Http.EasyCreate())
{
GCHandle gcHandle = GCHandle.Alloc(dataPieces);
try
{
IntPtr dataHandlePtr = GCHandle.ToIntPtr(gcHandle);
Interop.Http.EasySetOptionString(curlHandle, Interop.Http.CURLoption.CURLOPT_URL, uri);
Interop.Http.EasySetOptionPointer(curlHandle, Interop.Http.CURLoption.CURLOPT_WRITEDATA, dataHandlePtr);
Interop.Http.EasySetOptionPointer(curlHandle, Interop.Http.CURLoption.CURLOPT_WRITEFUNCTION, s_writeCallback);
Interop.Http.EasySetOptionLong(curlHandle, Interop.Http.CURLoption.CURLOPT_FOLLOWLOCATION, 1L);
Stopwatch stopwatch = Stopwatch.StartNew();
Interop.Http.CURLcode res = Interop.Http.EasyPerform(curlHandle);
stopwatch.Stop();
// TimeSpan.Zero isn't a worrisome value on the subtraction, it only
// means "no limit" on the original input.
remainingDownloadTime -= stopwatch.Elapsed;
if (res != Interop.Http.CURLcode.CURLE_OK)
{
return(null);
}
}
finally
{
gcHandle.Free();
}
}
if (dataPieces.Count == 0)
{
return(null);
}
if (dataPieces.Count == 1)
{
return(dataPieces[0]);
}
int dataLen = 0;
for (int i = 0; i < dataPieces.Count; i++)
{
dataLen += dataPieces[i].Length;
}
byte[] data = new byte[dataLen];
int offset = 0;
for (int i = 0; i < dataPieces.Count; i++)
{
byte[] piece = dataPieces[i];
Buffer.BlockCopy(piece, 0, data, offset, piece.Length);
offset += piece.Length;
}
return(data);
}
internal static void SetSslOptions(EasyRequest easy, ClientCertificateOption clientCertOption)
{
Debug.Assert(clientCertOption == ClientCertificateOption.Automatic || clientCertOption == ClientCertificateOption.Manual);
// Create a client certificate provider if client certs may be used.
X509Certificate2Collection clientCertificates = easy._handler._clientCertificates;
ClientCertificateProvider certProvider =
clientCertOption == ClientCertificateOption.Automatic ? new ClientCertificateProvider(null) : // automatic
clientCertificates?.Count > 0 ? new ClientCertificateProvider(clientCertificates) : // manual with certs
null; // manual without certs
IntPtr userPointer = IntPtr.Zero;
if (certProvider != null)
{
// The client cert provider needs to be passed through to the callback, and thus
// we create a GCHandle to keep it rooted. This handle needs to be cleaned up
// when the request has completed, and a simple and pay-for-play way to do that
// is by cleaning it up in a continuation off of the request.
userPointer = GCHandle.ToIntPtr(certProvider._gcHandle);
easy.Task.ContinueWith((_, state) => ((IDisposable)state).Dispose(), certProvider,
CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
}
// Register the callback with libcurl. We need to register even if there's no user-provided
// server callback and even if there are no client certificates, because we support verifying
// server certificates against more than those known to OpenSSL.
CURLcode answer = easy.SetSslCtxCallback(s_sslCtxCallback, userPointer);
switch (answer)
{
case CURLcode.CURLE_OK:
// We successfully registered. If we'll be invoking a user-provided callback to verify the server
// certificate as part of that, disable libcurl's verification of the host name. The user's callback
// needs to be given the opportunity to examine the cert, and our logic will determine whether
// the host name matches and will inform the callback of that.
if (easy._handler.ServerCertificateValidationCallback != null)
{
easy.SetCurlOption(Interop.Http.CURLoption.CURLOPT_SSL_VERIFYHOST, 0); // don't verify the peer cert's hostname
// We don't change the SSL_VERIFYPEER setting, as setting it to 0 will cause
// SSL and libcurl to ignore the result of the server callback.
}
// The allowed SSL protocols will be set in the configuration callback.
break;
case CURLcode.CURLE_UNKNOWN_OPTION: // Curl 7.38 and prior
case CURLcode.CURLE_NOT_BUILT_IN: // Curl 7.39 and later
// It's ok if we failed to register the callback if all of the defaults are in play
// with relation to handling of certificates. But if that's not the case, failing to
// register the callback will result in those options not being factored in, which is
// a significant enough error that we need to fail.
EventSourceTrace("CURLOPT_SSL_CTX_FUNCTION not supported: {0}", answer);
if (certProvider != null ||
easy._handler.ServerCertificateValidationCallback != null ||
easy._handler.CheckCertificateRevocationList)
{
throw new PlatformNotSupportedException(
SR.Format(SR.net_http_unix_invalid_certcallback_option, CurlVersionDescription, CurlSslVersionDescription));
}
// Since there won't be a callback to configure the allowed SSL protocols, configure them here.
SetSslVersion(easy);
break;
default:
ThrowIfCURLEError(answer);
break;
}
}
internal void SetCurlCallbacks(
IntPtr easyGCHandle,
ReadWriteCallback receiveHeadersCallback,
ReadWriteCallback sendCallback,
SeekCallback seekCallback,
ReadWriteCallback receiveBodyCallback,
DebugCallback debugCallback)
{
if (_callbackHandle == null)
{
_callbackHandle = new SafeCallbackHandle();
}
// Add callback for processing headers
Interop.Http.RegisterReadWriteCallback(
_easyHandle,
ReadWriteFunction.Header,
receiveHeadersCallback,
easyGCHandle,
ref _callbackHandle);
// If we're sending data as part of the request, add callbacks for sending request data
if (_requestMessage.Content != null)
{
Interop.Http.RegisterReadWriteCallback(
_easyHandle,
ReadWriteFunction.Read,
sendCallback,
easyGCHandle,
ref _callbackHandle);
Interop.Http.RegisterSeekCallback(
_easyHandle,
seekCallback,
easyGCHandle,
ref _callbackHandle);
}
// If we're expecting any data in response, add a callback for receiving body data
if (_requestMessage.Method != HttpMethod.Head)
{
Interop.Http.RegisterReadWriteCallback(
_easyHandle,
ReadWriteFunction.Write,
receiveBodyCallback,
easyGCHandle,
ref _callbackHandle);
}
if (EventSourceTracingEnabled)
{
SetCurlOption(CURLoption.CURLOPT_VERBOSE, 1L);
CURLcode curlResult = Interop.Http.RegisterDebugCallback(
_easyHandle,
debugCallback,
easyGCHandle,
ref _callbackHandle);
if (curlResult != CURLcode.CURLE_OK)
{
EventSourceTrace("Failed to register debug callback.");
}
}
}
private static void ThrowIfCURLEError(CURLcode error)
{
if (error != CURLcode.CURLE_OK)
{
var inner = new CurlException((int)error, isMulti: false);
VerboseTrace(inner.Message);
throw inner;
}
}
private void WorkerLoop()
{
Debug.Assert(!Monitor.IsEntered(_incomingRequests), "No locks should be held while invoking Process");
Debug.Assert(_runningWorker != null && _runningWorker.Id == Task.CurrentId, "This is the worker, so it must be running");
Debug.Assert(_wakeupRequestedPipeFd != null && !_wakeupRequestedPipeFd.IsInvalid, "Should have a valid pipe for wake ups");
// Create the multi handle to use for this round of processing. This one handle will be used
// to service all easy requests currently available and all those that come in while
// we're processing other requests. Once the work quiesces and there are no more requests
// to process, this multi handle will be released as the worker goes away. The next
// time a request arrives and a new worker is spun up, a new multi handle will be created.
SafeCurlMultiHandle multiHandle = CreateAndConfigureMultiHandle();
// Clear our active operations table. This should already be clear, either because
// all previous operations completed without unexpected exception, or in the case of an
// unexpected exception we should have cleaned up gracefully anyway. But just in case...
Debug.Assert(_activeOperations.Count == 0, "We shouldn't have any active operations when starting processing.");
_activeOperations.Clear();
bool endingSuccessfully = false;
try
{
// Continue processing as long as there are any active operations
while (true)
{
// First handle any requests in the incoming requests queue.
while (true)
{
IncomingRequest request;
lock (_incomingRequests)
{
if (_incomingRequests.Count == 0) break;
request = _incomingRequests.Dequeue();
}
HandleIncomingRequest(multiHandle, request);
}
// If we have no active operations, we're done.
if (_activeOperations.Count == 0)
{
endingSuccessfully = true;
return;
}
// We have one or more active operations. Run any work that needs to be run.
ThrowIfCURLMError(Interop.Http.MultiPerform(multiHandle));
// Complete and remove any requests that have finished being processed.
CURLMSG message;
IntPtr easyHandle;
CURLcode result;
while (Interop.Http.MultiInfoRead(multiHandle, out message, out easyHandle, out result))
{
Debug.Assert(message == CURLMSG.CURLMSG_DONE, "CURLMSG_DONE is supposed to be the only message type");
if (message == CURLMSG.CURLMSG_DONE)
{
IntPtr gcHandlePtr;
CURLcode getInfoResult = Interop.Http.EasyGetInfoPointer(easyHandle, CURLINFO.CURLINFO_PRIVATE, out gcHandlePtr);
Debug.Assert(getInfoResult == CURLcode.CURLE_OK, "Failed to get info on a completing easy handle");
if (getInfoResult == CURLcode.CURLE_OK)
{
ActiveRequest completedOperation;
bool gotActiveOp = _activeOperations.TryGetValue(gcHandlePtr, out completedOperation);
Debug.Assert(gotActiveOp, "Expected to find GCHandle ptr in active operations table");
if (gotActiveOp)
{
DeactivateActiveRequest(multiHandle, completedOperation.Easy, gcHandlePtr, completedOperation.CancellationRegistration);
FinishRequest(completedOperation.Easy, result);
}
}
}
}
// Wait for more things to do.
bool isWakeupRequestedPipeActive;
bool isTimeout;
ThrowIfCURLMError(Interop.Http.MultiWait(multiHandle, _wakeupRequestedPipeFd, out isWakeupRequestedPipeActive, out isTimeout));
if (isWakeupRequestedPipeActive)
{
// We woke up (at least in part) because a wake-up was requested.
// Read the data out of the pipe to clear it.
Debug.Assert(!isTimeout, "should not have timed out if isExtraFileDescriptorActive");
VerboseTrace("curl_multi_wait wake-up notification");
ReadFromWakeupPipeWhenKnownToContainData();
}
VerboseTraceIf(isTimeout, "curl_multi_wait timeout");
// PERF NOTE: curl_multi_wait uses poll (assuming it's available), which is O(N) in terms of the number of fds
// being waited on. If this ends up being a scalability bottleneck, we can look into using the curl_multi_socket_*
// APIs, which would let us switch to using epoll by being notified when sockets file descriptors are added or
// removed and configuring the epoll context with EPOLL_CTL_ADD/DEL, which at the expense of a lot of additional
// complexity would let us turn the O(N) operation into an O(1) operation. The additional complexity would come
// not only in the form of additional callbacks and managing the socket collection, but also in the form of timer
// management, which is necessary when using the curl_multi_socket_* APIs and which we avoid by using just
// curl_multi_wait/perform.
}
}
finally
{
// If we got an unexpected exception, something very bad happened. We may have some
// operations that we initiated but that weren't completed. Make sure to clean up any
// such operations, failing them and releasing their resources.
if (_activeOperations.Count > 0)
{
Debug.Assert(!endingSuccessfully, "We should only have remaining operations if we got an unexpected exception");
foreach (KeyValuePair<IntPtr, ActiveRequest> pair in _activeOperations)
{
ActiveRequest failingOperation = pair.Value;
IntPtr failingOperationGcHandle = pair.Key;
DeactivateActiveRequest(multiHandle, failingOperation.Easy, failingOperationGcHandle, failingOperation.CancellationRegistration);
// Complete the operation's task and clean up any of its resources
failingOperation.Easy.FailRequest(CreateHttpRequestException());
failingOperation.Easy.Cleanup(); // no active processing remains, so cleanup
}
// Clear the table.
_activeOperations.Clear();
}
// Finally, dispose of the multi handle.
multiHandle.Dispose();
}
}
internal void SetCurlCallbacks(
IntPtr easyGCHandle,
ReadWriteCallback receiveHeadersCallback,
ReadWriteCallback sendCallback,
SeekCallback seekCallback,
ReadWriteCallback receiveBodyCallback,
DebugCallback debugCallback)
{
if (_callbackHandle == null)
{
_callbackHandle = new SafeCallbackHandle();
}
// Add callback for processing headers
Interop.Http.RegisterReadWriteCallback(
_easyHandle,
ReadWriteFunction.Header,
receiveHeadersCallback,
easyGCHandle,
ref _callbackHandle);
ThrowOOMIfInvalid(_callbackHandle);
// If we're sending data as part of the request and it wasn't already added as
// in-memory data, add callbacks for sending request data.
if (!_inMemoryPostContent && _requestMessage.Content != null)
{
Interop.Http.RegisterReadWriteCallback(
_easyHandle,
ReadWriteFunction.Read,
sendCallback,
easyGCHandle,
ref _callbackHandle);
Debug.Assert(!_callbackHandle.IsInvalid, $"Should have been allocated (or failed) when originally adding handlers");
Interop.Http.RegisterSeekCallback(
_easyHandle,
seekCallback,
easyGCHandle,
ref _callbackHandle);
Debug.Assert(!_callbackHandle.IsInvalid, $"Should have been allocated (or failed) when originally adding handlers");
}
// If we're expecting any data in response, add a callback for receiving body data
if (_requestMessage.Method != HttpMethod.Head)
{
Interop.Http.RegisterReadWriteCallback(
_easyHandle,
ReadWriteFunction.Write,
receiveBodyCallback,
easyGCHandle,
ref _callbackHandle);
Debug.Assert(!_callbackHandle.IsInvalid, $"Should have been allocated (or failed) when originally adding handlers");
}
if (NetEventSource.IsEnabled)
{
SetCurlOption(CURLoption.CURLOPT_VERBOSE, 1L);
CURLcode curlResult = Interop.Http.RegisterDebugCallback(
_easyHandle,
debugCallback,
easyGCHandle,
ref _callbackHandle);
Debug.Assert(!_callbackHandle.IsInvalid, $"Should have been allocated (or failed) when originally adding handlers");
if (curlResult != CURLcode.CURLE_OK)
{
EventSourceTrace("Failed to register debug callback.");
}
}
}
private void FinishRequest(EasyRequest completedOperation, CURLcode messageResult)
{
VerboseTrace("messageResult: " + messageResult, easy: completedOperation);
if (completedOperation._responseMessage.StatusCode != HttpStatusCode.Unauthorized)
{
if (completedOperation._handler.PreAuthenticate)
{
long authAvailable;
if (Interop.Http.EasyGetInfoLong(completedOperation._easyHandle, CURLINFO.CURLINFO_HTTPAUTH_AVAIL, out authAvailable) == CURLcode.CURLE_OK)
{
completedOperation._handler.AddCredentialToCache(
completedOperation._requestMessage.RequestUri, (CURLAUTH)authAvailable, completedOperation._networkCredential);
}
// Ignore errors: no need to fail for the sake of putting the credentials into the cache
}
completedOperation._handler.AddResponseCookies(
completedOperation._requestMessage.RequestUri, completedOperation._responseMessage);
}
// Complete or fail the request
try
{
bool unsupportedProtocolRedirect = messageResult == CURLcode.CURLE_UNSUPPORTED_PROTOCOL && completedOperation._isRedirect;
if (!unsupportedProtocolRedirect)
{
ThrowIfCURLEError(messageResult);
}
completedOperation.EnsureResponseMessagePublished();
}
catch (Exception exc)
{
completedOperation.FailRequest(exc);
}
// At this point, we've completed processing the entire request, either due to error
// or due to completing the entire response.
completedOperation.Cleanup();
}
private void WorkerLoop()
{
Debug.Assert(!Monitor.IsEntered(_incomingRequests), "No locks should be held while invoking Process");
Debug.Assert(_runningWorker != null && _runningWorker.Id == Task.CurrentId, "This is the worker, so it must be running");
Debug.Assert(_wakeupRequestedPipeFd != 0, "Should have a valid pipe for wake ups");
// Create the multi handle to use for this round of processing. This one handle will be used
// to service all easy requests currently available and all those that come in while
// we're processing other requests. Once the work quiesces and there are no more requests
// to process, this multi handle will be released as the worker goes away. The next
// time a request arrives and a new worker is spun up, a new multi handle will be created.
SafeCurlMultiHandle multiHandle = Interop.libcurl.curl_multi_init();
if (multiHandle.IsInvalid)
{
throw CreateHttpRequestException();
}
// Clear our active operations table. This should already be clear, either because
// all previous operations completed without unexpected exception, or in the case of an
// unexpected exception we should have cleaned up gracefully anyway. But just in case...
Debug.Assert(_activeOperations.Count == 0, "We shouldn't have any active operations when starting processing.");
_activeOperations.Clear();
bool endingSuccessfully = false;
try
{
// Continue processing as long as there are any active operations
while (true)
{
// First handle any requests in the incoming requests queue.
while (true)
{
IncomingRequest request;
lock (_incomingRequests)
{
if (_incomingRequests.Count == 0)
{
break;
}
request = _incomingRequests.Dequeue();
}
HandleIncomingRequest(multiHandle, request);
}
// If we have no active operations, we're done.
if (_activeOperations.Count == 0)
{
endingSuccessfully = true;
return;
}
// We have one or more active operations. Run any work that needs to be run.
int running_handles;
ThrowIfCURLMError(Interop.libcurl.curl_multi_perform(multiHandle, out running_handles));
// Complete and remove any requests that have finished being processed.
int pendingMessages;
IntPtr messagePtr;
while ((messagePtr = Interop.libcurl.curl_multi_info_read(multiHandle, out pendingMessages)) != IntPtr.Zero)
{
Interop.libcurl.CURLMsg message = Marshal.PtrToStructure <Interop.libcurl.CURLMsg>(messagePtr);
Debug.Assert(message.msg == Interop.libcurl.CURLMSG.CURLMSG_DONE, "CURLMSG_DONE is supposed to be the only message type");
if (message.msg == Interop.libcurl.CURLMSG.CURLMSG_DONE)
{
IntPtr gcHandlePtr;
CURLcode getInfoResult = Interop.Http.EasyGetInfoPointer(message.easy_handle, CURLINFO.CURLINFO_PRIVATE, out gcHandlePtr);
Debug.Assert(getInfoResult == CURLcode.CURLE_OK, "Failed to get info on a completing easy handle");
if (getInfoResult == CURLcode.CURLE_OK)
{
ActiveRequest completedOperation;
bool gotActiveOp = _activeOperations.TryGetValue(gcHandlePtr, out completedOperation);
Debug.Assert(gotActiveOp, "Expected to find GCHandle ptr in active operations table");
if (gotActiveOp)
{
DeactivateActiveRequest(multiHandle, completedOperation.Easy, gcHandlePtr, completedOperation.CancellationRegistration);
FinishRequest(completedOperation.Easy, message.result);
}
}
}
}
// Wait for more things to do. Even with our cancellation mechanism, we specify a timeout so that
// just in case something goes wrong we can recover gracefully. This timeout is relatively long.
// Note, though, that libcurl has its own internal timeout, which can be requested separately
// via curl_multi_timeout, but which is used implicitly by curl_multi_wait if it's shorter
// than the value we provide.
const int FailsafeTimeoutMilliseconds = 1000;
int numFds;
unsafe
{
Interop.libcurl.curl_waitfd extraFds = new Interop.libcurl.curl_waitfd {
fd = _wakeupRequestedPipeFd,
events = Interop.libcurl.CURL_WAIT_POLLIN,
revents = 0
};
ThrowIfCURLMError(Interop.libcurl.curl_multi_wait(multiHandle, &extraFds, 1, FailsafeTimeoutMilliseconds, out numFds));
if ((extraFds.revents & Interop.libcurl.CURL_WAIT_POLLIN) != 0)
{
// We woke up (at least in part) because a wake-up was requested.
// Read the data out of the pipe to clear it.
Debug.Assert(numFds >= 1, "numFds should have been at least one, as the extraFd was set");
VerboseTrace("curl_multi_wait wake-up notification");
ReadFromWakeupPipeWhenKnownToContainData();
}
VerboseTraceIf(numFds == 0, "curl_multi_wait timeout");
}
// PERF NOTE: curl_multi_wait uses poll (assuming it's available), which is O(N) in terms of the number of fds
// being waited on. If this ends up being a scalability bottleneck, we can look into using the curl_multi_socket_*
// APIs, which would let us switch to using epoll by being notified when sockets file descriptors are added or
// removed and configuring the epoll context with EPOLL_CTL_ADD/DEL, which at the expense of a lot of additional
// complexity would let us turn the O(N) operation into an O(1) operation. The additional complexity would come
// not only in the form of additional callbacks and managing the socket collection, but also in the form of timer
// management, which is necessary when using the curl_multi_socket_* APIs and which we avoid by using just
// curl_multi_wait/perform.
}
}
finally
{
// If we got an unexpected exception, something very bad happened. We may have some
// operations that we initiated but that weren't completed. Make sure to clean up any
// such operations, failing them and releasing their resources.
if (_activeOperations.Count > 0)
{
Debug.Assert(!endingSuccessfully, "We should only have remaining operations if we got an unexpected exception");
foreach (KeyValuePair <IntPtr, ActiveRequest> pair in _activeOperations)
{
ActiveRequest failingOperation = pair.Value;
IntPtr failingOperationGcHandle = pair.Key;
DeactivateActiveRequest(multiHandle, failingOperation.Easy, failingOperationGcHandle, failingOperation.CancellationRegistration);
// Complete the operation's task and clean up any of its resources
failingOperation.Easy.FailRequest(CreateHttpRequestException());
failingOperation.Easy.Cleanup(); // no active processing remains, so cleanup
}
// Clear the table.
_activeOperations.Clear();
}
// Finally, dispose of the multi handle.
multiHandle.Dispose();
}
}
