private static bool TryGetEasyRequest(IntPtr curlPtr, out EasyRequest easy) { Debug.Assert(curlPtr != IntPtr.Zero, "curlPtr is not null"); IntPtr gcHandlePtr; CURLcode getInfoResult = Interop.Http.EasyGetInfoPointer(curlPtr, CURLINFO.CURLINFO_PRIVATE, out gcHandlePtr); Debug.Assert(getInfoResult == CURLcode.CURLE_OK, "Failed to get info on a completing easy handle"); if (getInfoResult == CURLcode.CURLE_OK) { try { GCHandle handle = GCHandle.FromIntPtr(gcHandlePtr); easy = (EasyRequest)handle.Target; Debug.Assert(easy != null, "Expected non-null EasyRequest in GCHandle"); return(easy != null); } catch (Exception e) { EventSourceTrace("Error getting state from GCHandle: {0}", e); Debug.Fail($"Exception in {nameof(TryGetEasyRequest)}", e.ToString()); } } easy = null; return(false); }
private static void SetSslOptionsForSupportedBackend(EasyRequest easy, ClientCertificateProvider certProvider, IntPtr userPointer) { CURLcode answer = easy.SetSslCtxCallback(s_sslCtxCallback, userPointer); EventSourceTrace("Callback registration result: {0}", answer, easy: easy); switch (answer) { case CURLcode.CURLE_OK: // We successfully registered. If we'll be invoking a user-provided callback to verify the server // certificate as part of that, disable libcurl's verification of the host name; we need to get // the callback from libcurl even if the host name doesn't match, so we take on the responsibility // of doing the host name match in the callback prior to invoking the user's delegate. if (easy._handler.ServerCertificateCustomValidationCallback != null) { easy.SetCurlOption(Interop.Http.CURLoption.CURLOPT_SSL_VERIFYHOST, 0); // But don't change the CURLOPT_SSL_VERIFYPEER setting, as setting it to 0 will // cause SSL and libcurl to ignore the result of the server callback. } SetSslOptionsForCertificateStore(easy); // The allowed SSL protocols will be set in the configuration callback. break; case CURLcode.CURLE_UNKNOWN_OPTION: // Curl 7.38 and prior case CURLcode.CURLE_NOT_BUILT_IN: // Curl 7.39 and later SetSslOptionsForUnsupportedBackend(easy, certProvider); break; default: ThrowIfCURLEError(answer); break; } }
private static void ThrowIfCURLEError(CURLcode error) { if (error != CURLcode.CURLE_OK) { var inner = new CurlException((int)error, isMulti: false); VerboseTrace(inner.Message); throw inner; } }
internal CURLcode SetSslCtxCallback(SslCtxCallback callback, IntPtr userPointer) { if (_callbackHandle == null) { _callbackHandle = new SafeCallbackHandle(); } CURLcode result = Interop.Http.RegisterSslCtxCallback(_easyHandle, callback, userPointer, ref _callbackHandle); return(result); }
private static bool TryGetEasyRequest(IntPtr curlPtr, out EasyRequest easy) { Debug.Assert(curlPtr != IntPtr.Zero, "curlPtr is not null"); IntPtr gcHandlePtr; CURLcode getInfoResult = Interop.Http.EasyGetInfoPointer(curlPtr, CURLINFO.CURLINFO_PRIVATE, out gcHandlePtr); if (getInfoResult == CURLcode.CURLE_OK) { return(MultiAgent.TryGetEasyRequestFromGCHandle(gcHandlePtr, out easy)); } Debug.Fail($"Failed to get info on a completing easy handle: {getInfoResult}"); easy = null; return(false); }
internal static void SetSslOptions(EasyRequest easy, ClientCertificateOption clientCertOption) { // Disable SSLv2/SSLv3, allow TLSv1.* easy.SetCurlOption(Interop.Http.CURLoption.CURLOPT_SSLVERSION, (long)Interop.Http.CurlSslVersion.CURL_SSLVERSION_TLSv1); IntPtr userPointer = IntPtr.Zero; if (clientCertOption == ClientCertificateOption.Automatic) { ClientCertificateProvider certProvider = new ClientCertificateProvider(); userPointer = GCHandle.ToIntPtr(certProvider._gcHandle); easy.Task.ContinueWith((_, state) => ((IDisposable)state).Dispose(), certProvider, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); } else { Debug.Assert(clientCertOption == ClientCertificateOption.Manual, "ClientCertificateOption is manual or automatic"); } CURLcode answer = easy.SetSslCtxCallback(s_sslCtxCallback, userPointer); switch (answer) { case CURLcode.CURLE_OK: break; // Curl 7.38 and prior case CURLcode.CURLE_UNKNOWN_OPTION: // Curl 7.39 and later case CURLcode.CURLE_NOT_BUILT_IN: EventSourceTrace("CURLOPT_SSL_CTX_FUNCTION not supported. Platform default HTTPS chain building in use"); if (clientCertOption == ClientCertificateOption.Automatic) { throw new PlatformNotSupportedException(SR.net_http_unix_invalid_client_cert_option); } break; default: ThrowIfCURLEError(answer); break; } }
private static bool TryGetEasyRequest(IntPtr curlPtr, out EasyRequest easy) { Debug.Assert(curlPtr != IntPtr.Zero, "curlPtr is not null"); IntPtr gcHandlePtr; CURLcode getInfoResult = Interop.Http.EasyGetInfoPointer(curlPtr, CURLINFO.CURLINFO_PRIVATE, out gcHandlePtr); Debug.Assert(getInfoResult == CURLcode.CURLE_OK, "Failed to get info on a completing easy handle"); if (getInfoResult == CURLcode.CURLE_OK) { GCHandle handle = GCHandle.FromIntPtr(gcHandlePtr); easy = handle.Target as EasyRequest; Debug.Assert(easy != null, "Expected non-null EasyRequest in GCHandle"); return easy != null; } easy = null; return false; }
private static void AddChannelBindingToken(X509Certificate2 certificate, IntPtr curlPtr) { Debug.Assert(curlPtr != IntPtr.Zero, "curlPtr is not null"); IntPtr gcHandlePtr; CURLcode getInfoResult = Interop.Http.EasyGetInfoPointer(curlPtr, CURLINFO.CURLINFO_PRIVATE, out gcHandlePtr); Debug.Assert(getInfoResult == CURLcode.CURLE_OK, "Failed to get info on a completing easy handle"); if (getInfoResult == CURLcode.CURLE_OK) { GCHandle handle = GCHandle.FromIntPtr(gcHandlePtr); EasyRequest easy = handle.Target as EasyRequest; Debug.Assert(easy != null, "Expected non-null EasyRequest in GCHandle"); if (easy._requestContentStream != null) { easy._requestContentStream.SetChannelBindingToken(certificate); } } }
internal void StoreLastEffectiveUri() { IntPtr urlCharPtr; // do not free; will point to libcurl private memory CURLcode urlResult = Interop.Http.EasyGetInfoPointer(_easyHandle, Interop.Http.CURLINFO.CURLINFO_EFFECTIVE_URL, out urlCharPtr); if (urlResult == CURLcode.CURLE_OK && urlCharPtr != IntPtr.Zero) { string url = Marshal.PtrToStringAnsi(urlCharPtr); Uri finalUri; if (Uri.TryCreate(url, UriKind.Absolute, out finalUri)) { _requestMessage.RequestUri = finalUri; return; } } Debug.Fail("Expected to be able to get the last effective Uri from libcurl"); }
private void HandleIncomingRequest(SafeCurlMultiHandle multiHandle, IncomingRequest request) { Debug.Assert(!Monitor.IsEntered(_incomingRequests), "Incoming requests lock should only be held while accessing the queue"); VerboseTrace("Type: " + request.Type, easy: request.Easy); EasyRequest easy = request.Easy; switch (request.Type) { case IncomingRequestType.New: ActivateNewRequest(multiHandle, easy); break; case IncomingRequestType.Cancel: Debug.Assert(easy._associatedMultiAgent == this, "Should only cancel associated easy requests"); Debug.Assert(easy._cancellationToken.IsCancellationRequested, "Cancellation should have been requested"); FindAndFailActiveRequest(multiHandle, easy, new OperationCanceledException(easy._cancellationToken)); break; case IncomingRequestType.Unpause: Debug.Assert(easy._associatedMultiAgent == this, "Should only unpause associated easy requests"); if (!easy._easyHandle.IsClosed) { IntPtr gcHandlePtr; ActiveRequest ar; Debug.Assert(FindActiveRequest(easy, out gcHandlePtr, out ar), "Couldn't find active request for unpause"); CURLcode unpauseResult = Interop.Http.EasyUnpause(easy._easyHandle); try { ThrowIfCURLEError(unpauseResult); } catch (Exception exc) { FindAndFailActiveRequest(multiHandle, easy, exc); } } break; default: Debug.Fail("Invalid request type: " + request.Type); break; } }
private static void ThrowIfCURLEError(CURLcode error) { if (error != CURLcode.CURLE_OK) // success { string msg = CurlException.GetCurlErrorString((int)error, isMulti: false); EventSourceTrace(msg); switch (error) { case CURLcode.CURLE_OPERATION_TIMEDOUT: throw new OperationCanceledException(msg); case CURLcode.CURLE_OUT_OF_MEMORY: throw new OutOfMemoryException(msg); default: throw new CurlException((int)error, msg); } } }
private void FinishRequest(EasyRequest completedOperation, CURLcode messageResult) { VerboseTrace("messageResult: " + messageResult, easy: completedOperation); if (completedOperation._responseMessage.StatusCode != HttpStatusCode.Unauthorized) { if (completedOperation._handler.PreAuthenticate) { long authAvailable; if (Interop.Http.EasyGetInfoLong(completedOperation._easyHandle, CURLINFO.CURLINFO_HTTPAUTH_AVAIL, out authAvailable) == CURLcode.CURLE_OK) { completedOperation._handler.AddCredentialToCache( completedOperation._requestMessage.RequestUri, (CURLAUTH)authAvailable, completedOperation._networkCredential); } // Ignore errors: no need to fail for the sake of putting the credentials into the cache } completedOperation._handler.AddResponseCookies( completedOperation._requestMessage.RequestUri, completedOperation._responseMessage); } // Complete or fail the request try { bool unsupportedProtocolRedirect = messageResult == CURLcode.CURLE_UNSUPPORTED_PROTOCOL && completedOperation._isRedirect; if (!unsupportedProtocolRedirect) { ThrowIfCURLEError(messageResult); } completedOperation.EnsureResponseMessagePublished(); } catch (Exception exc) { completedOperation.FailRequest(exc); } // At this point, we've completed processing the entire request, either due to error // or due to completing the entire response. completedOperation.Cleanup(); }
private void SetVersion() { Version v = _requestMessage.Version; if (v != null) { // Try to use the requested version, if a known version was explicitly requested. // If an unknown version was requested, we simply use libcurl's default. // Only allow HTTP/2 when making https requests. var curlVersion = (v.Major == 1 && v.Minor == 1) ? Interop.Http.CurlHttpVersion.CURL_HTTP_VERSION_1_1 : (v.Major == 1 && v.Minor == 0) ? Interop.Http.CurlHttpVersion.CURL_HTTP_VERSION_1_0 : (v.Major == 2 && v.Minor == 0) ? Interop.Http.CurlHttpVersion.CURL_HTTP_VERSION_2TLS : Interop.Http.CurlHttpVersion.CURL_HTTP_VERSION_NONE; if (curlVersion != Interop.Http.CurlHttpVersion.CURL_HTTP_VERSION_NONE) { // Ask libcurl to use the specified version if possible. CURLcode c = Interop.Http.EasySetOptionLong(_easyHandle, CURLoption.CURLOPT_HTTP_VERSION, (long)curlVersion); if (c == CURLcode.CURLE_OK) { // Success. The requested version will be used. EventSourceTrace("HTTP version: {0}", v); } else if (c == CURLcode.CURLE_UNSUPPORTED_PROTOCOL) { // The requested version is unsupported. Fall back to using the default version chosen by libcurl. EventSourceTrace("Unsupported protocol: {0}", v); } else { // Some other error. Fail. ThrowIfCURLEError(c); } } } }
internal static void SetSslOptions(EasyRequest easy) { CURLcode answer = Interop.Http.EasySetOptionPointer( easy._easyHandle, Interop.Http.CURLoption.CURLOPT_SSL_CTX_FUNCTION, s_sslCtxCallback); switch (answer) { case CURLcode.CURLE_OK: break; // Curl 7.38 and prior case CURLcode.CURLE_UNKNOWN_OPTION: // Curl 7.39 and later case CURLcode.CURLE_NOT_BUILT_IN: VerboseTrace("CURLOPT_SSL_CTX_FUNCTION is not supported, platform default https chain building in use"); break; default: ThrowIfCURLEError(answer); break; } }
private static byte[] DownloadAsset(string uri, ref TimeSpan remainingDownloadTime) { if (remainingDownloadTime <= TimeSpan.Zero) { return(null); } List <byte[]> dataPieces = new List <byte[]>(); using (Interop.Http.SafeCurlHandle curlHandle = Interop.Http.EasyCreate()) { GCHandle gcHandle = GCHandle.Alloc(dataPieces); try { IntPtr dataHandlePtr = GCHandle.ToIntPtr(gcHandle); Interop.Http.EasySetOptionString(curlHandle, Interop.Http.CURLoption.CURLOPT_URL, uri); Interop.Http.EasySetOptionPointer(curlHandle, Interop.Http.CURLoption.CURLOPT_WRITEDATA, dataHandlePtr); Interop.Http.EasySetOptionPointer(curlHandle, Interop.Http.CURLoption.CURLOPT_WRITEFUNCTION, s_writeCallback); Interop.Http.EasySetOptionLong(curlHandle, Interop.Http.CURLoption.CURLOPT_FOLLOWLOCATION, 1L); Stopwatch stopwatch = Stopwatch.StartNew(); Interop.Http.CURLcode res = Interop.Http.EasyPerform(curlHandle); stopwatch.Stop(); // TimeSpan.Zero isn't a worrisome value on the subtraction, it only // means "no limit" on the original input. remainingDownloadTime -= stopwatch.Elapsed; if (res != Interop.Http.CURLcode.CURLE_OK) { return(null); } } finally { gcHandle.Free(); } } if (dataPieces.Count == 0) { return(null); } if (dataPieces.Count == 1) { return(dataPieces[0]); } int dataLen = 0; for (int i = 0; i < dataPieces.Count; i++) { dataLen += dataPieces[i].Length; } byte[] data = new byte[dataLen]; int offset = 0; for (int i = 0; i < dataPieces.Count; i++) { byte[] piece = dataPieces[i]; Buffer.BlockCopy(piece, 0, data, offset, piece.Length); offset += piece.Length; } return(data); }
internal static void SetSslOptions(EasyRequest easy, ClientCertificateOption clientCertOption) { Debug.Assert(clientCertOption == ClientCertificateOption.Automatic || clientCertOption == ClientCertificateOption.Manual); // Create a client certificate provider if client certs may be used. X509Certificate2Collection clientCertificates = easy._handler._clientCertificates; ClientCertificateProvider certProvider = clientCertOption == ClientCertificateOption.Automatic ? new ClientCertificateProvider(null) : // automatic clientCertificates?.Count > 0 ? new ClientCertificateProvider(clientCertificates) : // manual with certs null; // manual without certs IntPtr userPointer = IntPtr.Zero; if (certProvider != null) { // The client cert provider needs to be passed through to the callback, and thus // we create a GCHandle to keep it rooted. This handle needs to be cleaned up // when the request has completed, and a simple and pay-for-play way to do that // is by cleaning it up in a continuation off of the request. userPointer = GCHandle.ToIntPtr(certProvider._gcHandle); easy.Task.ContinueWith((_, state) => ((IDisposable)state).Dispose(), certProvider, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); } // Register the callback with libcurl. We need to register even if there's no user-provided // server callback and even if there are no client certificates, because we support verifying // server certificates against more than those known to OpenSSL. CURLcode answer = easy.SetSslCtxCallback(s_sslCtxCallback, userPointer); switch (answer) { case CURLcode.CURLE_OK: // We successfully registered. If we'll be invoking a user-provided callback to verify the server // certificate as part of that, disable libcurl's verification of the host name. The user's callback // needs to be given the opportunity to examine the cert, and our logic will determine whether // the host name matches and will inform the callback of that. if (easy._handler.ServerCertificateValidationCallback != null) { easy.SetCurlOption(Interop.Http.CURLoption.CURLOPT_SSL_VERIFYHOST, 0); // don't verify the peer cert's hostname // We don't change the SSL_VERIFYPEER setting, as setting it to 0 will cause // SSL and libcurl to ignore the result of the server callback. } // The allowed SSL protocols will be set in the configuration callback. break; case CURLcode.CURLE_UNKNOWN_OPTION: // Curl 7.38 and prior case CURLcode.CURLE_NOT_BUILT_IN: // Curl 7.39 and later // It's ok if we failed to register the callback if all of the defaults are in play // with relation to handling of certificates. But if that's not the case, failing to // register the callback will result in those options not being factored in, which is // a significant enough error that we need to fail. EventSourceTrace("CURLOPT_SSL_CTX_FUNCTION not supported: {0}", answer); if (certProvider != null || easy._handler.ServerCertificateValidationCallback != null || easy._handler.CheckCertificateRevocationList) { throw new PlatformNotSupportedException( SR.Format(SR.net_http_unix_invalid_certcallback_option, CurlVersionDescription, CurlSslVersionDescription)); } // Since there won't be a callback to configure the allowed SSL protocols, configure them here. SetSslVersion(easy); break; default: ThrowIfCURLEError(answer); break; } }
internal void SetCurlCallbacks( IntPtr easyGCHandle, ReadWriteCallback receiveHeadersCallback, ReadWriteCallback sendCallback, SeekCallback seekCallback, ReadWriteCallback receiveBodyCallback, DebugCallback debugCallback) { if (_callbackHandle == null) { _callbackHandle = new SafeCallbackHandle(); } // Add callback for processing headers Interop.Http.RegisterReadWriteCallback( _easyHandle, ReadWriteFunction.Header, receiveHeadersCallback, easyGCHandle, ref _callbackHandle); // If we're sending data as part of the request, add callbacks for sending request data if (_requestMessage.Content != null) { Interop.Http.RegisterReadWriteCallback( _easyHandle, ReadWriteFunction.Read, sendCallback, easyGCHandle, ref _callbackHandle); Interop.Http.RegisterSeekCallback( _easyHandle, seekCallback, easyGCHandle, ref _callbackHandle); } // If we're expecting any data in response, add a callback for receiving body data if (_requestMessage.Method != HttpMethod.Head) { Interop.Http.RegisterReadWriteCallback( _easyHandle, ReadWriteFunction.Write, receiveBodyCallback, easyGCHandle, ref _callbackHandle); } if (EventSourceTracingEnabled) { SetCurlOption(CURLoption.CURLOPT_VERBOSE, 1L); CURLcode curlResult = Interop.Http.RegisterDebugCallback( _easyHandle, debugCallback, easyGCHandle, ref _callbackHandle); if (curlResult != CURLcode.CURLE_OK) { EventSourceTrace("Failed to register debug callback."); } } }
private static void ThrowIfCURLEError(CURLcode error) { if (error != CURLcode.CURLE_OK) { var inner = new CurlException((int)error, isMulti: false); VerboseTrace(inner.Message); throw inner; } }
private void WorkerLoop() { Debug.Assert(!Monitor.IsEntered(_incomingRequests), "No locks should be held while invoking Process"); Debug.Assert(_runningWorker != null && _runningWorker.Id == Task.CurrentId, "This is the worker, so it must be running"); Debug.Assert(_wakeupRequestedPipeFd != null && !_wakeupRequestedPipeFd.IsInvalid, "Should have a valid pipe for wake ups"); // Create the multi handle to use for this round of processing. This one handle will be used // to service all easy requests currently available and all those that come in while // we're processing other requests. Once the work quiesces and there are no more requests // to process, this multi handle will be released as the worker goes away. The next // time a request arrives and a new worker is spun up, a new multi handle will be created. SafeCurlMultiHandle multiHandle = CreateAndConfigureMultiHandle(); // Clear our active operations table. This should already be clear, either because // all previous operations completed without unexpected exception, or in the case of an // unexpected exception we should have cleaned up gracefully anyway. But just in case... Debug.Assert(_activeOperations.Count == 0, "We shouldn't have any active operations when starting processing."); _activeOperations.Clear(); bool endingSuccessfully = false; try { // Continue processing as long as there are any active operations while (true) { // First handle any requests in the incoming requests queue. while (true) { IncomingRequest request; lock (_incomingRequests) { if (_incomingRequests.Count == 0) break; request = _incomingRequests.Dequeue(); } HandleIncomingRequest(multiHandle, request); } // If we have no active operations, we're done. if (_activeOperations.Count == 0) { endingSuccessfully = true; return; } // We have one or more active operations. Run any work that needs to be run. ThrowIfCURLMError(Interop.Http.MultiPerform(multiHandle)); // Complete and remove any requests that have finished being processed. CURLMSG message; IntPtr easyHandle; CURLcode result; while (Interop.Http.MultiInfoRead(multiHandle, out message, out easyHandle, out result)) { Debug.Assert(message == CURLMSG.CURLMSG_DONE, "CURLMSG_DONE is supposed to be the only message type"); if (message == CURLMSG.CURLMSG_DONE) { IntPtr gcHandlePtr; CURLcode getInfoResult = Interop.Http.EasyGetInfoPointer(easyHandle, CURLINFO.CURLINFO_PRIVATE, out gcHandlePtr); Debug.Assert(getInfoResult == CURLcode.CURLE_OK, "Failed to get info on a completing easy handle"); if (getInfoResult == CURLcode.CURLE_OK) { ActiveRequest completedOperation; bool gotActiveOp = _activeOperations.TryGetValue(gcHandlePtr, out completedOperation); Debug.Assert(gotActiveOp, "Expected to find GCHandle ptr in active operations table"); if (gotActiveOp) { DeactivateActiveRequest(multiHandle, completedOperation.Easy, gcHandlePtr, completedOperation.CancellationRegistration); FinishRequest(completedOperation.Easy, result); } } } } // Wait for more things to do. bool isWakeupRequestedPipeActive; bool isTimeout; ThrowIfCURLMError(Interop.Http.MultiWait(multiHandle, _wakeupRequestedPipeFd, out isWakeupRequestedPipeActive, out isTimeout)); if (isWakeupRequestedPipeActive) { // We woke up (at least in part) because a wake-up was requested. // Read the data out of the pipe to clear it. Debug.Assert(!isTimeout, "should not have timed out if isExtraFileDescriptorActive"); VerboseTrace("curl_multi_wait wake-up notification"); ReadFromWakeupPipeWhenKnownToContainData(); } VerboseTraceIf(isTimeout, "curl_multi_wait timeout"); // PERF NOTE: curl_multi_wait uses poll (assuming it's available), which is O(N) in terms of the number of fds // being waited on. If this ends up being a scalability bottleneck, we can look into using the curl_multi_socket_* // APIs, which would let us switch to using epoll by being notified when sockets file descriptors are added or // removed and configuring the epoll context with EPOLL_CTL_ADD/DEL, which at the expense of a lot of additional // complexity would let us turn the O(N) operation into an O(1) operation. The additional complexity would come // not only in the form of additional callbacks and managing the socket collection, but also in the form of timer // management, which is necessary when using the curl_multi_socket_* APIs and which we avoid by using just // curl_multi_wait/perform. } } finally { // If we got an unexpected exception, something very bad happened. We may have some // operations that we initiated but that weren't completed. Make sure to clean up any // such operations, failing them and releasing their resources. if (_activeOperations.Count > 0) { Debug.Assert(!endingSuccessfully, "We should only have remaining operations if we got an unexpected exception"); foreach (KeyValuePair<IntPtr, ActiveRequest> pair in _activeOperations) { ActiveRequest failingOperation = pair.Value; IntPtr failingOperationGcHandle = pair.Key; DeactivateActiveRequest(multiHandle, failingOperation.Easy, failingOperationGcHandle, failingOperation.CancellationRegistration); // Complete the operation's task and clean up any of its resources failingOperation.Easy.FailRequest(CreateHttpRequestException()); failingOperation.Easy.Cleanup(); // no active processing remains, so cleanup } // Clear the table. _activeOperations.Clear(); } // Finally, dispose of the multi handle. multiHandle.Dispose(); } }
internal void SetCurlCallbacks( IntPtr easyGCHandle, ReadWriteCallback receiveHeadersCallback, ReadWriteCallback sendCallback, SeekCallback seekCallback, ReadWriteCallback receiveBodyCallback, DebugCallback debugCallback) { if (_callbackHandle == null) { _callbackHandle = new SafeCallbackHandle(); } // Add callback for processing headers Interop.Http.RegisterReadWriteCallback( _easyHandle, ReadWriteFunction.Header, receiveHeadersCallback, easyGCHandle, ref _callbackHandle); ThrowOOMIfInvalid(_callbackHandle); // If we're sending data as part of the request and it wasn't already added as // in-memory data, add callbacks for sending request data. if (!_inMemoryPostContent && _requestMessage.Content != null) { Interop.Http.RegisterReadWriteCallback( _easyHandle, ReadWriteFunction.Read, sendCallback, easyGCHandle, ref _callbackHandle); Debug.Assert(!_callbackHandle.IsInvalid, $"Should have been allocated (or failed) when originally adding handlers"); Interop.Http.RegisterSeekCallback( _easyHandle, seekCallback, easyGCHandle, ref _callbackHandle); Debug.Assert(!_callbackHandle.IsInvalid, $"Should have been allocated (or failed) when originally adding handlers"); } // If we're expecting any data in response, add a callback for receiving body data if (_requestMessage.Method != HttpMethod.Head) { Interop.Http.RegisterReadWriteCallback( _easyHandle, ReadWriteFunction.Write, receiveBodyCallback, easyGCHandle, ref _callbackHandle); Debug.Assert(!_callbackHandle.IsInvalid, $"Should have been allocated (or failed) when originally adding handlers"); } if (NetEventSource.IsEnabled) { SetCurlOption(CURLoption.CURLOPT_VERBOSE, 1L); CURLcode curlResult = Interop.Http.RegisterDebugCallback( _easyHandle, debugCallback, easyGCHandle, ref _callbackHandle); Debug.Assert(!_callbackHandle.IsInvalid, $"Should have been allocated (or failed) when originally adding handlers"); if (curlResult != CURLcode.CURLE_OK) { EventSourceTrace("Failed to register debug callback."); } } }
private void FinishRequest(EasyRequest completedOperation, CURLcode messageResult) { VerboseTrace("messageResult: " + messageResult, easy: completedOperation); if (completedOperation._responseMessage.StatusCode != HttpStatusCode.Unauthorized) { if (completedOperation._handler.PreAuthenticate) { long authAvailable; if (Interop.Http.EasyGetInfoLong(completedOperation._easyHandle, CURLINFO.CURLINFO_HTTPAUTH_AVAIL, out authAvailable) == CURLcode.CURLE_OK) { completedOperation._handler.AddCredentialToCache( completedOperation._requestMessage.RequestUri, (CURLAUTH)authAvailable, completedOperation._networkCredential); } // Ignore errors: no need to fail for the sake of putting the credentials into the cache } completedOperation._handler.AddResponseCookies( completedOperation._requestMessage.RequestUri, completedOperation._responseMessage); } // Complete or fail the request try { bool unsupportedProtocolRedirect = messageResult == CURLcode.CURLE_UNSUPPORTED_PROTOCOL && completedOperation._isRedirect; if (!unsupportedProtocolRedirect) { ThrowIfCURLEError(messageResult); } completedOperation.EnsureResponseMessagePublished(); } catch (Exception exc) { completedOperation.FailRequest(exc); } // At this point, we've completed processing the entire request, either due to error // or due to completing the entire response. completedOperation.Cleanup(); }
private void WorkerLoop() { Debug.Assert(!Monitor.IsEntered(_incomingRequests), "No locks should be held while invoking Process"); Debug.Assert(_runningWorker != null && _runningWorker.Id == Task.CurrentId, "This is the worker, so it must be running"); Debug.Assert(_wakeupRequestedPipeFd != 0, "Should have a valid pipe for wake ups"); // Create the multi handle to use for this round of processing. This one handle will be used // to service all easy requests currently available and all those that come in while // we're processing other requests. Once the work quiesces and there are no more requests // to process, this multi handle will be released as the worker goes away. The next // time a request arrives and a new worker is spun up, a new multi handle will be created. SafeCurlMultiHandle multiHandle = Interop.libcurl.curl_multi_init(); if (multiHandle.IsInvalid) { throw CreateHttpRequestException(); } // Clear our active operations table. This should already be clear, either because // all previous operations completed without unexpected exception, or in the case of an // unexpected exception we should have cleaned up gracefully anyway. But just in case... Debug.Assert(_activeOperations.Count == 0, "We shouldn't have any active operations when starting processing."); _activeOperations.Clear(); bool endingSuccessfully = false; try { // Continue processing as long as there are any active operations while (true) { // First handle any requests in the incoming requests queue. while (true) { IncomingRequest request; lock (_incomingRequests) { if (_incomingRequests.Count == 0) { break; } request = _incomingRequests.Dequeue(); } HandleIncomingRequest(multiHandle, request); } // If we have no active operations, we're done. if (_activeOperations.Count == 0) { endingSuccessfully = true; return; } // We have one or more active operations. Run any work that needs to be run. int running_handles; ThrowIfCURLMError(Interop.libcurl.curl_multi_perform(multiHandle, out running_handles)); // Complete and remove any requests that have finished being processed. int pendingMessages; IntPtr messagePtr; while ((messagePtr = Interop.libcurl.curl_multi_info_read(multiHandle, out pendingMessages)) != IntPtr.Zero) { Interop.libcurl.CURLMsg message = Marshal.PtrToStructure <Interop.libcurl.CURLMsg>(messagePtr); Debug.Assert(message.msg == Interop.libcurl.CURLMSG.CURLMSG_DONE, "CURLMSG_DONE is supposed to be the only message type"); if (message.msg == Interop.libcurl.CURLMSG.CURLMSG_DONE) { IntPtr gcHandlePtr; CURLcode getInfoResult = Interop.Http.EasyGetInfoPointer(message.easy_handle, CURLINFO.CURLINFO_PRIVATE, out gcHandlePtr); Debug.Assert(getInfoResult == CURLcode.CURLE_OK, "Failed to get info on a completing easy handle"); if (getInfoResult == CURLcode.CURLE_OK) { ActiveRequest completedOperation; bool gotActiveOp = _activeOperations.TryGetValue(gcHandlePtr, out completedOperation); Debug.Assert(gotActiveOp, "Expected to find GCHandle ptr in active operations table"); if (gotActiveOp) { DeactivateActiveRequest(multiHandle, completedOperation.Easy, gcHandlePtr, completedOperation.CancellationRegistration); FinishRequest(completedOperation.Easy, message.result); } } } } // Wait for more things to do. Even with our cancellation mechanism, we specify a timeout so that // just in case something goes wrong we can recover gracefully. This timeout is relatively long. // Note, though, that libcurl has its own internal timeout, which can be requested separately // via curl_multi_timeout, but which is used implicitly by curl_multi_wait if it's shorter // than the value we provide. const int FailsafeTimeoutMilliseconds = 1000; int numFds; unsafe { Interop.libcurl.curl_waitfd extraFds = new Interop.libcurl.curl_waitfd { fd = _wakeupRequestedPipeFd, events = Interop.libcurl.CURL_WAIT_POLLIN, revents = 0 }; ThrowIfCURLMError(Interop.libcurl.curl_multi_wait(multiHandle, &extraFds, 1, FailsafeTimeoutMilliseconds, out numFds)); if ((extraFds.revents & Interop.libcurl.CURL_WAIT_POLLIN) != 0) { // We woke up (at least in part) because a wake-up was requested. // Read the data out of the pipe to clear it. Debug.Assert(numFds >= 1, "numFds should have been at least one, as the extraFd was set"); VerboseTrace("curl_multi_wait wake-up notification"); ReadFromWakeupPipeWhenKnownToContainData(); } VerboseTraceIf(numFds == 0, "curl_multi_wait timeout"); } // PERF NOTE: curl_multi_wait uses poll (assuming it's available), which is O(N) in terms of the number of fds // being waited on. If this ends up being a scalability bottleneck, we can look into using the curl_multi_socket_* // APIs, which would let us switch to using epoll by being notified when sockets file descriptors are added or // removed and configuring the epoll context with EPOLL_CTL_ADD/DEL, which at the expense of a lot of additional // complexity would let us turn the O(N) operation into an O(1) operation. The additional complexity would come // not only in the form of additional callbacks and managing the socket collection, but also in the form of timer // management, which is necessary when using the curl_multi_socket_* APIs and which we avoid by using just // curl_multi_wait/perform. } } finally { // If we got an unexpected exception, something very bad happened. We may have some // operations that we initiated but that weren't completed. Make sure to clean up any // such operations, failing them and releasing their resources. if (_activeOperations.Count > 0) { Debug.Assert(!endingSuccessfully, "We should only have remaining operations if we got an unexpected exception"); foreach (KeyValuePair <IntPtr, ActiveRequest> pair in _activeOperations) { ActiveRequest failingOperation = pair.Value; IntPtr failingOperationGcHandle = pair.Key; DeactivateActiveRequest(multiHandle, failingOperation.Easy, failingOperationGcHandle, failingOperation.CancellationRegistration); // Complete the operation's task and clean up any of its resources failingOperation.Easy.FailRequest(CreateHttpRequestException()); failingOperation.Easy.Cleanup(); // no active processing remains, so cleanup } // Clear the table. _activeOperations.Clear(); } // Finally, dispose of the multi handle. multiHandle.Dispose(); } }