/// <summary> /// Start up some HTTP GETs, within our limit on the maximum simultaneous number /// The entire method is not synchronized, only the portion pulling work off the list /// Important to not hold the synchronized block while we do network access /// </summary> public void PullRemoteRevisions() { //find the work to be done in a synchronized block var workToStartNow = new List <RevisionInternal>(); var bulkWorkToStartNow = new List <RevisionInternal>(); lock (locker) { while (httpConnectionCount + bulkWorkToStartNow.Count + workToStartNow.Count < ManagerOptions.Default.MaxOpenHttpConnections) { int nBulk = 0; if (bulkRevsToPull != null) { nBulk = (bulkRevsToPull.Count < ManagerOptions.Default.MaxRevsToGetInBulk) ? bulkRevsToPull.Count : ManagerOptions.Default.MaxRevsToGetInBulk; } if (nBulk == 1) { // Rather than pulling a single revision in 'bulk', just pull it normally: QueueRemoteRevision(bulkRevsToPull[0]); bulkRevsToPull.Remove(0); nBulk = 0; } if (nBulk > 0) { // Prefer to pull bulk revisions: var range = new Couchbase.Lite.Util.ArraySegment <RevisionInternal>(bulkRevsToPull.ToArray(), 0, nBulk); bulkWorkToStartNow.AddRange(range); bulkRevsToPull.RemoveAll(range); } else { // Prefer to pull an existing revision over a deleted one: IList <RevisionInternal> queue = revsToPull; if (queue == null || queue.Count == 0) { queue = deletedRevsToPull; if (queue == null || queue.Count == 0) { break; } } // both queues are empty workToStartNow.AddItem(queue[0]); queue.Remove(0); } } } //actually run it outside the synchronized block if (bulkWorkToStartNow.Count > 0) { PullBulkRevisions(bulkWorkToStartNow); } foreach (var rev in workToStartNow) { PullRemoteRevision(rev); } }
public void TestSimple() { var mime = Encoding.UTF8.GetBytes("--BOUNDARY\r\nFoo: Bar\r\n Header : Val ue \r\n\r\npart the first\r\n--BOUNDARY \r\n\r\n2nd part\r\n--BOUNDARY--"); var expectedParts = new List <IEnumerable <byte> > { Encoding.UTF8.GetBytes("part the first"), Encoding.UTF8.GetBytes("2nd part") }; var expectedHeaders = new List <IDictionary <string, object> > { new Dictionary <string, object> { { "Foo", "Bar" }, { "Header", "Val ue" } }, new Dictionary <string, object>() }; for (int chunkSize = 1; chunkSize <= mime.Length; ++chunkSize) { WriteDebug("--- chunkSize = {0}", chunkSize); Reset(); var reader = new MultipartReader("multipart/related; boundary=\"BOUNDARY\"", this); Assert.IsFalse(reader.Finished); Range r = new Range(0, 0); do { Assert.IsTrue(r.Location < mime.Length, "Parser didn't stop at end"); r.Length = Math.Min(chunkSize, mime.Length - r.Location); var sublist = new Couchbase.Lite.Util.ArraySegment <byte>(mime, r.Location, r.Length); reader.AppendData(sublist); r.Location += chunkSize; } while(!reader.Finished); } Assert.AreEqual(expectedHeaders, _headerList); Assert.AreEqual(expectedParts, _partList); }
private Task ExecuteRequest(CouchbaseLiteHttpClient httpClient, HttpRequestMessage request) { object fullBody = null; Exception error = null; HttpResponseMessage response = null; if (_tokenSource.IsCancellationRequested) { RespondWithResult(fullBody, new Exception(string.Format("{0}: Request {1} has been aborted", this, request)), response); var tcs = new TaskCompletionSource <bool>(); tcs.SetCanceled(); return(tcs.Task); } request.Headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip")); request.Headers.Add("X-Accept-Part-Encoding", "gzip"); Log.To.Sync.V(Tag, "Sending request: {0}", request); var requestTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_tokenSource.Token); httpClient.Authenticator = Authenticator; return(httpClient.SendAsync(request, requestTokenSource.Token).ContinueWith(t => { requestTokenSource.Dispose(); try { response = t.Result; } catch (Exception e) { var err = Misc.Flatten(e).First(); Log.To.Sync.E(Tag, "Unhandled exception while getting bulk documents", err); error = err; RespondWithResult(fullBody, err, response); return; } try { if (response == null) { Log.To.Sync.I(Tag, "Didn't get response for {0}", request); error = new HttpRequestException(); RespondWithResult(fullBody, error, response); } else if (!response.IsSuccessStatusCode) { HttpStatusCode status = response.StatusCode; Log.To.Sync.I(Tag, "Got error status: {0} for {1}. Reason: {2}", status.GetStatusCode(), request, response.ReasonPhrase); error = new HttpResponseException(status); RespondWithResult(fullBody, error, response); } else { Log.To.Sync.D(Tag, "Processing response: {0}", response); var entity = response.Content; var contentTypeHeader = entity.Headers.ContentType; Stream inputStream = null; if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/")) { Log.To.Sync.D(Tag, "contentTypeHeader = {0}", contentTypeHeader.ToString()); try { _topReader = new MultipartReader(contentTypeHeader.ToString(), this); inputStream = entity.ReadAsStreamAsync().Result; const int bufLen = 1024; var buffer = new byte[bufLen]; var numBytesRead = 0; while ((numBytesRead = inputStream.Read(buffer, 0, bufLen)) > 0) { if (numBytesRead != bufLen) { var bufferToAppend = new Couchbase.Lite.Util.ArraySegment <byte>(buffer, 0, numBytesRead).ToArray(); _topReader.AppendData(bufferToAppend); } else { _topReader.AppendData(buffer); } } RespondWithResult(fullBody, error, response); } finally { try { inputStream.Close(); } catch (IOException) { } } } else { Log.To.Sync.D(Tag, "contentTypeHeader is not multipart = {0}", contentTypeHeader.ToString()); if (entity != null) { try { inputStream = entity.ReadAsStreamAsync().Result; fullBody = Manager.GetObjectMapper().ReadValue <object>(inputStream); RespondWithResult(fullBody, error, response); } finally { try { inputStream.Close(); } catch (IOException) { } } } } } } catch (Exception e) { var err = (e is AggregateException) ? e.InnerException : e; Log.To.Sync.E(Tag, "Exception while processing bulk download response", err); error = err; RespondWithResult(fullBody, err, response); } })); }
internal void SendAsyncMultipartDownloaderRequest(HttpMethod method, string relativePath, object body, Database db, RemoteRequestCompletionBlock onCompletion) { try { var urlStr = BuildRelativeURLString(relativePath); var url = new Uri(urlStr); var message = new HttpRequestMessage(method, url); message.Headers.Add("Accept", "*/*"); AddRequestHeaders(message); var client = clientFactory.GetHttpClient(false); var challengeResponseAuth = Authenticator as IChallengeResponseAuthenticator; if (challengeResponseAuth != null) { var authHandler = clientFactory.Handler as DefaultAuthHandler; if (authHandler != null) { authHandler.Authenticator = challengeResponseAuth; } challengeResponseAuth.PrepareWithRequest(message); } var authHeader = AuthUtils.GetAuthenticationHeaderValue(Authenticator, message.RequestUri); if (authHeader != null) { client.DefaultRequestHeaders.Authorization = authHeader; } client.SendAsync(message, CancellationTokenSource.Token).ContinueWith(new Action<Task<HttpResponseMessage>>(responseMessage => { object fullBody = null; Exception error = null; try { var response = responseMessage.Result; // add in cookies to global store //CouchbaseLiteHttpClientFactory.Instance.AddCookies(clientFactory.HttpHandler.CookieContainer.GetCookies(url)); var status = response.StatusCode; if ((Int32)status.GetStatusCode() >= 300) { Log.E(TAG, "Got error " + Sharpen.Extensions.ToString(status.GetStatusCode())); Log.E(TAG, "Request was for: " + message); Log.E(TAG, "Status reason: " + response.ReasonPhrase); error = new WebException(response.ReasonPhrase); } else { var entity = response.Content; var contentTypeHeader = response.Content.Headers.ContentType; InputStream inputStream = null; if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/related")) { try { var reader = new MultipartDocumentReader(LocalDatabase); var contentType = contentTypeHeader.ToString(); reader.SetContentType(contentType); var inputStreamTask = entity.ReadAsStreamAsync(); //inputStreamTask.Wait(90000, CancellationTokenSource.Token); inputStream = inputStreamTask.Result; const int bufLen = 1024; var buffer = new byte[bufLen]; int numBytesRead = 0; while ((numBytesRead = inputStream.Read(buffer)) != -1) { if (numBytesRead != bufLen) { var bufferToAppend = new Couchbase.Lite.Util.ArraySegment<Byte>(buffer, 0, numBytesRead); reader.AppendData(bufferToAppend); } else { reader.AppendData(buffer); } } reader.Finish(); fullBody = reader.GetDocumentProperties(); if (onCompletion != null) { onCompletion(fullBody, error); } } catch (Exception ex) { Log.E(TAG, "SendAsyncMultipartDownloaderRequest has an error occurred.", ex); } finally { try { inputStream.Close(); } catch (Exception) { } } } else { if (entity != null) { try { var readTask = entity.ReadAsStreamAsync(); //readTask.Wait(); // TODO: This should be scaled based on content length. inputStream = readTask.Result; fullBody = Manager.GetObjectMapper().ReadValue<Object>(inputStream); if (onCompletion != null) onCompletion(fullBody, error); } catch (Exception ex) { Log.E(TAG, "SendAsyncMultipartDownloaderRequest has an error occurred.", ex); } finally { try { inputStream.Close(); } catch (Exception) { } } } } } } catch (System.Net.ProtocolViolationException e) { Log.E(TAG, "client protocol exception", e); error = e; } catch (IOException e) { Log.E(TAG, "IO Exception", e); error = e; } finally { client.Dispose(); } }), WorkExecutor.Scheduler); } catch (UriFormatException e) { Log.E(TAG, "Malformed URL for async request", e); } }
/// <summary> /// Start up some HTTP GETs, within our limit on the maximum simultaneous number /// The entire method is not synchronized, only the portion pulling work off the list /// Important to not hold the synchronized block while we do network access /// </summary> private void PullRemoteRevisions() { //find the work to be done in a synchronized block var workToStartNow = new List <RevisionInternal>(); var bulkWorkToStartNow = new List <RevisionInternal>(); lock (_locker) { while (LocalDatabase.IsOpen) { int nBulk = 0; if (_bulkRevsToPull != null) { nBulk = Math.Min(_bulkRevsToPull.Count, ReplicationOptions.MaxRevsToGetInBulk); } if (nBulk == 1) { // Rather than pulling a single revision in 'bulk', just pull it normally: QueueRemoteRevision(_bulkRevsToPull[0]); _bulkRevsToPull.RemoveAt(0); nBulk = 0; } if (nBulk > 0) { // Prefer to pull bulk revisions: var range = new Couchbase.Lite.Util.ArraySegment <RevisionInternal>(_bulkRevsToPull.ToArray(), 0, nBulk); bulkWorkToStartNow.AddRange(range); foreach (var val in range) { _bulkRevsToPull.Remove(val); } } else { // Prefer to pull an existing revision over a deleted one: IList <RevisionInternal> queue = _revsToPull; if (queue == null || queue.Count == 0) { queue = _deletedRevsToPull; if (queue == null || queue.Count == 0) { break; // both queues are empty } } workToStartNow.Add(queue[0]); queue.RemoveAt(0); } } } //actually run it outside the synchronized block if (bulkWorkToStartNow.Count > 0) { PullBulkRevisions(bulkWorkToStartNow); } foreach (var rev in workToStartNow) { PullRemoteRevision(rev); } }
protected override internal void ExecuteRequest(HttpClient httpClient, HttpRequestMessage request) { object fullBody = null; Exception error = null; HttpResponseMessage response = null; try { if (_tokenSource.IsCancellationRequested) { RespondWithResult(fullBody, new Exception(string.Format("{0}: Request {1} has been aborted", this, request)), response); return; } } catch (AggregateException e) { var err = e.InnerException; Log.E(Tag, "Unhandled Exception", err); error = err; RespondWithResult(fullBody, err, response); return; } catch (IOException e) { Log.E(Tag, "IO Exception", e); error = e; RespondWithResult(fullBody, e, response); return; } catch (Exception e) { Log.E(Tag, "ExecuteRequest Exception: ", e); error = e; RespondWithResult(fullBody, e, response); return; } try { Log.D(Tag + ".ExecuteRequest", "Sending request: {0}", request); var requestTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_tokenSource.Token); var responseTask = httpClient.SendAsync(request, requestTokenSource.Token); if (!responseTask.Wait((Int32)ManagerOptions.Default.RequestTimeout.TotalMilliseconds, requestTokenSource.Token)) { Log.E(Tag, "Response task timed out: {0}, {1}, {2}", responseTask, TaskScheduler.Current, Description()); throw new HttpResponseException(HttpStatusCode.RequestTimeout); } requestTokenSource.Dispose(); response = responseTask.Result; } catch (AggregateException e) { var err = e.InnerException; Log.E(Tag, "Unhandled Exception: {0}, {1}", TaskScheduler.Current, Description()); Log.E(Tag, "Unhandled Exception at Line 129 or 130", err); error = err; RespondWithResult(fullBody, err, response); return; } catch (IOException e) { Log.E(Tag, "IO Exception", e); error = e; RespondWithResult(fullBody, e, response); return; } catch (Exception e) { Log.E(Tag, "ExecuteRequest Exception: ", e); error = e; RespondWithResult(fullBody, e, response); return; } try { if (response == null) { Log.E(Tag, "Didn't get response for {0}", request); error = new HttpRequestException(); RespondWithResult(fullBody, error, response); } else if (!response.IsSuccessStatusCode) { HttpStatusCode status = response.StatusCode; Log.E(Tag, "Got error status: {0} for {1}. Reason: {2}", status.GetStatusCode(), request, response.ReasonPhrase); error = new HttpResponseException(status); RespondWithResult(fullBody, error, response); } else { Log.D(Tag, "Processing response: {0}", response); var entity = response.Content; var contentTypeHeader = entity.Headers.ContentType; Stream inputStream = null; if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/")) { Log.V(Tag, "contentTypeHeader = {0}", contentTypeHeader.ToString()); try { _topReader = new MultipartReader(contentTypeHeader.ToString(), this); inputStream = entity.ReadAsStreamAsync().Result; const int bufLen = 1024; var buffer = new byte[bufLen]; var numBytesRead = 0; while ((numBytesRead = inputStream.Read(buffer, 0, bufLen)) > 0) { if (numBytesRead != bufLen) { var bufferToAppend = new Couchbase.Lite.Util.ArraySegment<byte>(buffer, 0, numBytesRead).ToArray(); _topReader.AppendData(bufferToAppend); } else { _topReader.AppendData(buffer); } } _topReader.Finished(); RespondWithResult(fullBody, error, response); } finally { try { inputStream.Close(); } catch (IOException) { } } } else { Log.V(Tag, "contentTypeHeader is not multipart = {0}", contentTypeHeader.ToString()); if (entity != null) { try { inputStream = entity.ReadAsStreamAsync().Result; fullBody = Manager.GetObjectMapper().ReadValue<object>(inputStream); RespondWithResult(fullBody, error, response); } finally { try { inputStream.Close(); } catch (IOException) { } } } } } } catch (AggregateException e) { var err = e.InnerException; Log.E(Tag, "Unhandled Exception", err); error = err; RespondWithResult(fullBody, err, response); } catch (IOException e) { Log.E(Tag, "IO Exception", e); error = e; RespondWithResult(fullBody, e, response); } catch (Exception e) { Log.E(Tag, "ExecuteRequest Exception: ", e); error = e; RespondWithResult(fullBody, e, response); } }
public void AppendData(IEnumerable <byte> newData) { if (newData == null) { return; } var data = newData.ToArray(); if (_buffer == null || data.Length == 0) { return; } _buffer.AddRange(data); MultipartReader.MultipartReaderState nextState; do { nextState = MultipartReader.MultipartReaderState.Uninitialized; var bufLen = _buffer.Count; switch (state) { case MultipartReader.MultipartReaderState.AtStart: // The entire message might start with a boundary without a leading CRLF. var boundaryWithoutLeadingCRLF = TrimmedBoundary.ToArray(); if (bufLen >= boundaryWithoutLeadingCRLF.Length) { // if (Arrays.equals(buffer.toByteArray(), boundaryWithoutLeadingCRLF)) { if (Memcmp(_buffer.ToArray(), boundaryWithoutLeadingCRLF, boundaryWithoutLeadingCRLF.Length)) { DeleteUpThrough(boundaryWithoutLeadingCRLF.Length); nextState = MultipartReader.MultipartReaderState.InHeaders; } else { nextState = MultipartReader.MultipartReaderState.InPrologue; } } break; case MultipartReader.MultipartReaderState.InPrologue: case MultipartReader.MultipartReaderState.InBody: // Look for the next part boundary in the data we just added and the ending bytes of // the previous data (in case the boundary string is split across calls) if (bufLen < _boundary.Length) { break; } var start = Math.Max(0, bufLen - data.Length - _boundary.Length); var r = SearchFor(_boundary, start); if (r.Length > 0) { if (state == MultipartReader.MultipartReaderState.InBody) { var dataToAppend = new byte[r.Location]; Array.Copy(_buffer.ToArray(), 0, dataToAppend, 0, dataToAppend.Length); _readerDelegate.AppendToPart(dataToAppend); _readerDelegate.FinishedPart(); } DeleteUpThrough(r.Location + r.Length); nextState = MultipartReader.MultipartReaderState.InHeaders; } else { TrimBuffer(); } break; case MultipartReader.MultipartReaderState.InHeaders: // First check for the end-of-message string ("--" after separator): if (bufLen >= 2 && Memcmp(_buffer.ToArray(), EOM_BYTES, 2)) { state = MultipartReader.MultipartReaderState.AtEnd; Close(); return; } // Otherwise look for two CRLFs that delimit the end of the headers: var headerEnd = SearchFor(CRLF_CRLF, 0); if (headerEnd.Length > 0) { var headersBytes = new Couchbase.Lite.Util.ArraySegment <Byte>(_buffer.ToArray(), 0, headerEnd.Location); // <-- better? var headersString = Encoding.UTF8.GetString(headersBytes.ToArray()); ParseHeaders(headersString); DeleteUpThrough(headerEnd.Location + headerEnd.Length); _readerDelegate.StartedPart(_headers); nextState = MultipartReader.MultipartReaderState.InBody; } break; default: Log.To.Sync.E(Tag, "Unexpected data after end of MIME body, throwing..."); throw new InvalidOperationException("Unexpected data after end of MIME body"); } if (nextState != MultipartReader.MultipartReaderState.Uninitialized) { state = nextState; } } while (nextState != MultipartReader.MultipartReaderState.Uninitialized && _buffer.Count > 0); }
protected override internal Task ExecuteRequest(HttpClient httpClient, HttpRequestMessage request) { object fullBody = null; Exception error = null; HttpResponseMessage response = null; if (_tokenSource.IsCancellationRequested) { RespondWithResult(fullBody, new Exception(string.Format("{0}: Request {1} has been aborted", this, request)), response); var tcs = new TaskCompletionSource<bool>(); tcs.SetCanceled(); return tcs.Task; } Log.D(Tag + ".ExecuteRequest", "Sending request: {0}", request); var requestTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_tokenSource.Token); var retVal = httpClient.SendAsync(request, requestTokenSource.Token); retVal.ConfigureAwait(false).GetAwaiter().OnCompleted(() => { requestTokenSource.Dispose(); try { response = retVal.Result; } catch(Exception e) { var err = (e is AggregateException) ? e.InnerException : e; Log.E(Tag, "Unhandled Exception", err); error = err; RespondWithResult(fullBody, err, response); return; } try { if (response == null) { Log.E(Tag, "Didn't get response for {0}", request); error = new HttpRequestException(); RespondWithResult(fullBody, error, response); } else if (!response.IsSuccessStatusCode) { HttpStatusCode status = response.StatusCode; Log.E(Tag, "Got error status: {0} for {1}. Reason: {2}", status.GetStatusCode(), request, response.ReasonPhrase); error = new HttpResponseException(status); RespondWithResult(fullBody, error, response); } else { Log.D(Tag, "Processing response: {0}", response); var entity = response.Content; var contentTypeHeader = entity.Headers.ContentType; Stream inputStream = null; if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/")) { Log.V(Tag, "contentTypeHeader = {0}", contentTypeHeader.ToString()); try { _topReader = new MultipartReader(contentTypeHeader.ToString(), this); inputStream = entity.ReadAsStreamAsync().Result; const int bufLen = 1024; var buffer = new byte[bufLen]; var numBytesRead = 0; while ((numBytesRead = inputStream.Read(buffer, 0, bufLen)) > 0) { if (numBytesRead != bufLen) { var bufferToAppend = new Couchbase.Lite.Util.ArraySegment<byte>(buffer, 0, numBytesRead).ToArray(); _topReader.AppendData(bufferToAppend); } else { _topReader.AppendData(buffer); } } _topReader.Finished(); RespondWithResult(fullBody, error, response); } finally { try { inputStream.Close(); } catch (IOException) { } } } else { Log.V(Tag, "contentTypeHeader is not multipart = {0}", contentTypeHeader.ToString()); if (entity != null) { try { inputStream = entity.ReadAsStreamAsync().Result; fullBody = Manager.GetObjectMapper().ReadValue<object>(inputStream); RespondWithResult(fullBody, error, response); } finally { try { inputStream.Close(); } catch (IOException) { } } } } } } catch (AggregateException e) { var err = e.InnerException; Log.E(Tag, "Unhandled Exception", err); error = err; RespondWithResult(fullBody, err, response); } catch (IOException e) { Log.E(Tag, "IO Exception", e); error = e; RespondWithResult(fullBody, e, response); } catch (Exception e) { Log.E(Tag, "ExecuteRequest Exception: ", e); error = e; RespondWithResult(fullBody, e, response); } finally { response.Dispose(); } }); return retVal; }
public void TestSimple() { var mime = Encoding.UTF8.GetBytes("--BOUNDARY\r\nFoo: Bar\r\n Header : Val ue \r\n\r\npart the first\r\n--BOUNDARY \r\n\r\n2nd part\r\n--BOUNDARY--"); var expectedParts = new List<IEnumerable<byte>> { Encoding.UTF8.GetBytes("part the first"), Encoding.UTF8.GetBytes("2nd part") }; var expectedHeaders = new List<IDictionary<string, object>> { new Dictionary<string, object> { { "Foo", "Bar" }, { "Header", "Val ue" } }, new Dictionary<string, object>() }; for (int chunkSize = 1; chunkSize <= mime.Length; ++chunkSize) { WriteDebug("--- chunkSize = {0}", chunkSize); Reset(); var reader = new MultipartReader("multipart/related; boundary=\"BOUNDARY\"", this); Assert.IsFalse(reader.Finished); Range r = new Range(0, 0); do { Assert.IsTrue(r.Location < mime.Length, "Parser didn't stop at end"); r.Length = Math.Min(chunkSize, mime.Length - r.Location); var sublist = new Couchbase.Lite.Util.ArraySegment<byte>(mime, r.Location, r.Length); reader.AppendData(sublist); r.Location += chunkSize; } while(!reader.Finished); } Assert.AreEqual(expectedHeaders, _headerList); Assert.AreEqual(expectedParts, _partList); }
internal void SendAsyncMultipartDownloaderRequest(HttpMethod method, string relativePath, object body, Database db, RemoteRequestCompletionBlock onCompletion) { try { var url = _baseUrl.Append(relativePath); var message = new HttpRequestMessage(method, url); message.Headers.Add("Accept", "*/*"); AddRequestHeaders(message); var client = default(CouchbaseLiteHttpClient); if (!_client.AcquireFor(TimeSpan.FromSeconds(1), out client)) { Log.To.Sync.I(Tag, "Client is disposed, aborting request to {0}", new SecureLogString(relativePath, LogMessageSensitivity.PotentiallyInsecure)); return; } client.Authenticator = Authenticator; var request = client.SendAsync(message, _cancellationTokenSource.Token).ContinueWith(new Action <Task <HttpResponseMessage> >(responseMessage => { object fullBody = null; Exception error = null; try { if (responseMessage.IsFaulted) { error = responseMessage.Exception.InnerException; if (onCompletion != null) { onCompletion(null, error); } return; } var response = responseMessage.Result; // add in cookies to global store //CouchbaseLiteHttpClientFactory.Instance.AddCoIokies(clientFactory.HttpHandler.CookieContainer.GetCookies(url)); var status = response.StatusCode; if ((Int32)status.GetStatusCode() >= 300) { Log.To.Sync.W(Tag, "Got error {0}", status.GetStatusCode()); Log.To.Sync.W(Tag, "Request was for: " + message); Log.To.Sync.W(Tag, "Status reason: " + response.ReasonPhrase); Log.To.Sync.W(Tag, "Passing error onto callback..."); error = new HttpResponseException(status); if (onCompletion != null) { onCompletion(null, error); } } else { var entity = response.Content; var contentTypeHeader = response.Content.Headers.ContentType; Stream inputStream = null; if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/related")) { try { var reader = new MultipartDocumentReader(db); var contentType = contentTypeHeader.ToString(); reader.SetContentType(contentType); var inputStreamTask = entity.ReadAsStreamAsync(); //inputStreamTask.Wait(90000, CancellationTokenSource.Token); inputStream = inputStreamTask.Result; const int bufLen = 1024; var buffer = new byte[bufLen]; int numBytesRead = 0; while ((numBytesRead = inputStream.Read(buffer, 0, buffer.Length)) > 0) { if (numBytesRead != bufLen) { var bufferToAppend = new Couchbase.Lite.Util.ArraySegment <Byte>(buffer, 0, numBytesRead); reader.AppendData(bufferToAppend); } else { reader.AppendData(buffer); } } reader.Finish(); fullBody = reader.GetDocumentProperties(); if (onCompletion != null) { onCompletion(fullBody, error); } } catch (Exception ex) { Log.To.Sync.W(Tag, "SendAsyncMultipartDownloaderRequest got an exception, aborting...", ex); } finally { try { inputStream.Close(); } catch (Exception) { } } } else { if (entity != null) { try { var readTask = entity.ReadAsStreamAsync(); //readTask.Wait(); // TODO: This should be scaled based on content length. inputStream = readTask.Result; fullBody = Manager.GetObjectMapper().ReadValue <Object>(inputStream); if (onCompletion != null) { onCompletion(fullBody, error); } } catch (Exception ex) { Log.To.Sync.W(Tag, "SendAsyncMultipartDownloaderRequest got an exception, aborting...", ex); } finally { try { inputStream.Close(); } catch (Exception) { } } } } } } catch (Exception e) { Log.To.Sync.W(Tag, "Got exception during SendAsyncMultipartDownload, aborting..."); error = e; } finally { Task dummy; _requests.TryRemove(message, out dummy); responseMessage.Result.Dispose(); } }), _workExecutor.Scheduler); _requests.TryAdd(message, request); } catch (UriFormatException e) { Log.To.Sync.W(Tag, "Malformed URL for async request, aborting...", e); } }
public void AppendData(IEnumerable<byte> newData) { if (newData == null) { return; } var data = newData.ToArray(); if (_buffer == null || data.Length == 0) { return; } _buffer.AddRange(data); MultipartReader.MultipartReaderState nextState; do { nextState = MultipartReader.MultipartReaderState.Uninitialized; var bufLen = _buffer.Count; switch (state) { case MultipartReader.MultipartReaderState.AtStart: // The entire message might start with a boundary without a leading CRLF. var boundaryWithoutLeadingCRLF = TrimmedBoundary.ToArray(); if (bufLen >= boundaryWithoutLeadingCRLF.Length) { // if (Arrays.equals(buffer.toByteArray(), boundaryWithoutLeadingCRLF)) { if (Memcmp(_buffer.ToArray(), boundaryWithoutLeadingCRLF, boundaryWithoutLeadingCRLF.Length)) { DeleteUpThrough(boundaryWithoutLeadingCRLF.Length); nextState = MultipartReader.MultipartReaderState.InHeaders; } else { nextState = MultipartReader.MultipartReaderState.InPrologue; } } break; case MultipartReader.MultipartReaderState.InPrologue: case MultipartReader.MultipartReaderState.InBody: // Look for the next part boundary in the data we just added and the ending bytes of // the previous data (in case the boundary string is split across calls) if (bufLen < _boundary.Length) { break; } var start = Math.Max(0, bufLen - data.Length - _boundary.Length); var r = SearchFor(_boundary, start); if (r.Length > 0) { if (state == MultipartReader.MultipartReaderState.InBody) { var dataToAppend = new byte[r.Location]; Array.Copy(_buffer.ToArray(), 0, dataToAppend, 0, dataToAppend.Length); _readerDelegate.AppendToPart(dataToAppend); _readerDelegate.FinishedPart(); } DeleteUpThrough(r.Location + r.Length); nextState = MultipartReader.MultipartReaderState.InHeaders; } else { TrimBuffer(); } break; case MultipartReader.MultipartReaderState.InHeaders: // First check for the end-of-message string ("--" after separator): if (bufLen >= 2 && Memcmp(_buffer.ToArray(), EOM_BYTES, 2)) { state = MultipartReader.MultipartReaderState.AtEnd; Close(); return; } // Otherwise look for two CRLFs that delimit the end of the headers: var headerEnd = SearchFor(CRLF_CRLF, 0); if (headerEnd.Length > 0) { var headersBytes = new Couchbase.Lite.Util.ArraySegment<Byte>(_buffer.ToArray(), 0, headerEnd.Location); // <-- better? var headersString = Encoding.UTF8.GetString(headersBytes.ToArray()); ParseHeaders(headersString); DeleteUpThrough(headerEnd.Location + headerEnd.Length); _readerDelegate.StartedPart(_headers); nextState = MultipartReader.MultipartReaderState.InBody; } break; default: Log.To.Sync.E(Tag, "Unexpected data after end of MIME body, throwing..."); throw new InvalidOperationException("Unexpected data after end of MIME body"); } if (nextState != MultipartReader.MultipartReaderState.Uninitialized) { state = nextState; } } while (nextState != MultipartReader.MultipartReaderState.Uninitialized && _buffer.Count > 0); }
internal void SendAsyncMultipartDownloaderRequest(HttpMethod method, string relativePath, object body, Database db, RemoteRequestCompletionBlock onCompletion) { try { var url = _baseUrl.Append(relativePath); var message = new HttpRequestMessage(method, url); message.Headers.Add("Accept", "*/*"); AddRequestHeaders(message); var client = default(CouchbaseLiteHttpClient); if(!_client.AcquireFor(TimeSpan.FromSeconds(1), out client)) { Log.To.Sync.I(Tag, "Client is disposed, aborting request to {0}", new SecureLogString(relativePath, LogMessageSensitivity.PotentiallyInsecure)); return; } client.Authenticator = Authenticator; var request = client.SendAsync(message, _cancellationTokenSource.Token).ContinueWith(new Action<Task<HttpResponseMessage>>(responseMessage => { object fullBody = null; Exception error = null; try { if(responseMessage.IsFaulted) { error = responseMessage.Exception.InnerException; if(onCompletion != null) { onCompletion(null, error); } return; } var response = responseMessage.Result; // add in cookies to global store //CouchbaseLiteHttpClientFactory.Instance.AddCoIokies(clientFactory.HttpHandler.CookieContainer.GetCookies(url)); var status = response.StatusCode; if((Int32)status.GetStatusCode() >= 300) { Log.To.Sync.W(Tag, "Got error {0}", status.GetStatusCode()); Log.To.Sync.W(Tag, "Request was for: " + message); Log.To.Sync.W(Tag, "Status reason: " + response.ReasonPhrase); Log.To.Sync.W(Tag, "Passing error onto callback..."); error = new HttpResponseException(status); if(onCompletion != null) { onCompletion(null, error); } } else { var entity = response.Content; var contentTypeHeader = response.Content.Headers.ContentType; Stream inputStream = null; if(contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/related")) { try { var reader = new MultipartDocumentReader(db); var contentType = contentTypeHeader.ToString(); reader.SetContentType(contentType); var inputStreamTask = entity.ReadAsStreamAsync(); //inputStreamTask.Wait(90000, CancellationTokenSource.Token); inputStream = inputStreamTask.Result; const int bufLen = 1024; var buffer = new byte[bufLen]; int numBytesRead = 0; while((numBytesRead = inputStream.Read(buffer, 0, buffer.Length)) > 0) { if(numBytesRead != bufLen) { var bufferToAppend = new Couchbase.Lite.Util.ArraySegment<Byte>(buffer, 0, numBytesRead); reader.AppendData(bufferToAppend); } else { reader.AppendData(buffer); } } reader.Finish(); fullBody = reader.GetDocumentProperties(); if(onCompletion != null) { onCompletion(fullBody, error); } } catch(Exception ex) { Log.To.Sync.W(Tag, "SendAsyncMultipartDownloaderRequest got an exception, aborting...", ex); } finally { try { inputStream.Close(); } catch(Exception) { } } } else { if(entity != null) { try { var readTask = entity.ReadAsStreamAsync(); //readTask.Wait(); // TODO: This should be scaled based on content length. inputStream = readTask.Result; fullBody = Manager.GetObjectMapper().ReadValue<Object>(inputStream); if(onCompletion != null) onCompletion(fullBody, error); } catch(Exception ex) { Log.To.Sync.W(Tag, "SendAsyncMultipartDownloaderRequest got an exception, aborting...", ex); } finally { try { inputStream.Close(); } catch(Exception) { } } } } } } catch(Exception e) { Log.To.Sync.W(Tag, "Got exception during SendAsyncMultipartDownload, aborting..."); error = e; } finally { Task dummy; _requests.TryRemove(message, out dummy); responseMessage.Result.Dispose(); } }), _workExecutor.Scheduler); _requests.TryAdd(message, request); } catch(UriFormatException e) { Log.To.Sync.W(Tag, "Malformed URL for async request, aborting...", e); } }
protected override internal Task ExecuteRequest(HttpClient httpClient, HttpRequestMessage request) { object fullBody = null; Exception error = null; HttpResponseMessage response = null; if (_tokenSource.IsCancellationRequested) { RespondWithResult(fullBody, new Exception(string.Format("{0}: Request {1} has been aborted", this, request)), response); var tcs = new TaskCompletionSource <bool>(); tcs.SetCanceled(); return(tcs.Task); } Log.D(Tag + ".ExecuteRequest", "Sending request: {0}", request); var requestTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_tokenSource.Token); var retVal = httpClient.SendAsync(request, requestTokenSource.Token); retVal.ConfigureAwait(false).GetAwaiter().OnCompleted(() => { requestTokenSource.Dispose(); try { response = retVal.Result; } catch (Exception e) { var err = (e is AggregateException) ? e.InnerException : e; Log.E(Tag, "Unhandled Exception", err); error = err; RespondWithResult(fullBody, err, response); return; } try { if (response == null) { Log.E(Tag, "Didn't get response for {0}", request); error = new HttpRequestException(); RespondWithResult(fullBody, error, response); } else if (!response.IsSuccessStatusCode) { HttpStatusCode status = response.StatusCode; Log.E(Tag, "Got error status: {0} for {1}. Reason: {2}", status.GetStatusCode(), request, response.ReasonPhrase); error = new HttpResponseException(status); RespondWithResult(fullBody, error, response); } else { Log.D(Tag, "Processing response: {0}", response); var entity = response.Content; var contentTypeHeader = entity.Headers.ContentType; Stream inputStream = null; if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/")) { Log.V(Tag, "contentTypeHeader = {0}", contentTypeHeader.ToString()); try { _topReader = new MultipartReader(contentTypeHeader.ToString(), this); inputStream = entity.ReadAsStreamAsync().Result; const int bufLen = 1024; var buffer = new byte[bufLen]; var numBytesRead = 0; while ((numBytesRead = inputStream.Read(buffer, 0, bufLen)) > 0) { if (numBytesRead != bufLen) { var bufferToAppend = new Couchbase.Lite.Util.ArraySegment <byte>(buffer, 0, numBytesRead).ToArray(); _topReader.AppendData(bufferToAppend); } else { _topReader.AppendData(buffer); } } _topReader.Finished(); RespondWithResult(fullBody, error, response); } finally { try { inputStream.Close(); } catch (IOException) { } } } else { Log.V(Tag, "contentTypeHeader is not multipart = {0}", contentTypeHeader.ToString()); if (entity != null) { try { inputStream = entity.ReadAsStreamAsync().Result; fullBody = Manager.GetObjectMapper().ReadValue <object>(inputStream); RespondWithResult(fullBody, error, response); } finally { try { inputStream.Close(); } catch (IOException) { } } } } } } catch (AggregateException e) { var err = e.InnerException; Log.E(Tag, "Unhandled Exception", err); error = err; RespondWithResult(fullBody, err, response); } catch (IOException e) { Log.E(Tag, "IO Exception", e); error = e; RespondWithResult(fullBody, e, response); } catch (Exception e) { Log.E(Tag, "ExecuteRequest Exception: ", e); error = e; RespondWithResult(fullBody, e, response); } finally { response.Dispose(); } }); return(retVal); }
protected override internal void ExecuteRequest(HttpClient httpClient, HttpRequestMessage request) { object fullBody = null; Exception error = null; HttpResponseMessage response = null; try { if (_tokenSource.IsCancellationRequested) { RespondWithResult(fullBody, new Exception(string.Format("{0}: Request {1} has been aborted", this, request)), response); return; } } catch (AggregateException e) { var err = e.InnerException; Log.E(Tag, "Unhandled Exception", err); error = err; RespondWithResult(fullBody, err, response); return; } catch (IOException e) { Log.E(Tag, "IO Exception", e); error = e; RespondWithResult(fullBody, e, response); return; } catch (Exception e) { Log.E(Tag, "ExecuteRequest Exception: ", e); error = e; RespondWithResult(fullBody, e, response); return; } try { Log.D(Tag + ".ExecuteRequest", "Sending request: {0}", request); var requestTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_tokenSource.Token); var responseTask = httpClient.SendAsync(request, requestTokenSource.Token); if (!responseTask.Wait((Int32)ManagerOptions.Default.RequestTimeout.TotalMilliseconds, requestTokenSource.Token)) { Log.E(Tag, "Response task timed out: {0}, {1}, {2}", responseTask, TaskScheduler.Current, Description()); throw new HttpResponseException(HttpStatusCode.RequestTimeout); } requestTokenSource.Dispose(); response = responseTask.Result; } catch (AggregateException e) { var err = e.InnerException; Log.E(Tag, "Unhandled Exception: {0}, {1}", TaskScheduler.Current, Description()); Log.E(Tag, "Unhandled Exception at Line 129 or 130", err); error = err; RespondWithResult(fullBody, err, response); return; } catch (IOException e) { Log.E(Tag, "IO Exception", e); error = e; RespondWithResult(fullBody, e, response); return; } catch (Exception e) { Log.E(Tag, "ExecuteRequest Exception: ", e); error = e; RespondWithResult(fullBody, e, response); return; } try { if (response == null) { Log.E(Tag, "Didn't get response for {0}", request); error = new HttpRequestException(); RespondWithResult(fullBody, error, response); } else if (!response.IsSuccessStatusCode) { HttpStatusCode status = response.StatusCode; Log.E(Tag, "Got error status: {0} for {1}. Reason: {2}", status.GetStatusCode(), request, response.ReasonPhrase); error = new HttpResponseException(status); RespondWithResult(fullBody, error, response); } else { Log.D(Tag, "Processing response: {0}", response); var entity = response.Content; var contentTypeHeader = entity.Headers.ContentType; Stream inputStream = null; if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/")) { Log.V(Tag, "contentTypeHeader = {0}", contentTypeHeader.ToString()); try { _topReader = new MultipartReader(contentTypeHeader.ToString(), this); inputStream = entity.ReadAsStreamAsync().Result; const int bufLen = 1024; var buffer = new byte[bufLen]; var numBytesRead = 0; while ((numBytesRead = inputStream.Read(buffer, 0, bufLen)) > 0) { if (numBytesRead != bufLen) { var bufferToAppend = new Couchbase.Lite.Util.ArraySegment <byte>(buffer, 0, numBytesRead).ToArray(); _topReader.AppendData(bufferToAppend); } else { _topReader.AppendData(buffer); } } _topReader.Finished(); RespondWithResult(fullBody, error, response); } finally { try { inputStream.Close(); } catch (IOException) { } } } else { Log.V(Tag, "contentTypeHeader is not multipart = {0}", contentTypeHeader.ToString()); if (entity != null) { try { inputStream = entity.ReadAsStreamAsync().Result; fullBody = Manager.GetObjectMapper().ReadValue <object>(inputStream); RespondWithResult(fullBody, error, response); } finally { try { inputStream.Close(); } catch (IOException) { } } } } } } catch (AggregateException e) { var err = e.InnerException; Log.E(Tag, "Unhandled Exception", err); error = err; RespondWithResult(fullBody, err, response); } catch (IOException e) { Log.E(Tag, "IO Exception", e); error = e; RespondWithResult(fullBody, e, response); } catch (Exception e) { Log.E(Tag, "ExecuteRequest Exception: ", e); error = e; RespondWithResult(fullBody, e, response); } }
private Task ExecuteRequest(CouchbaseLiteHttpClient httpClient, HttpRequestMessage request) { object fullBody = null; Exception error = null; HttpResponseMessage response = null; if (_tokenSource.IsCancellationRequested) { RespondWithResult(fullBody, new Exception(string.Format("{0}: Request {1} has been aborted", this, request)), response); var tcs = new TaskCompletionSource<bool>(); tcs.SetCanceled(); return tcs.Task; } request.Headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip")); request.Headers.Add("X-Accept-Part-Encoding", "gzip"); Log.To.Sync.V(Tag, "Sending request: {0}", request); var requestTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_tokenSource.Token); httpClient.Authenticator = Authenticator; return httpClient.SendAsync(request, requestTokenSource.Token).ContinueWith(t => { requestTokenSource.Dispose(); try { response = t.Result; } catch(Exception e) { var err = Misc.Flatten(e).First(); Log.To.Sync.E(Tag, "Unhandled exception while getting bulk documents", err); error = err; RespondWithResult(fullBody, err, response); return; } try { if (response == null) { Log.To.Sync.I(Tag, "Didn't get response for {0}", request); error = new HttpRequestException(); RespondWithResult(fullBody, error, response); } else if (!response.IsSuccessStatusCode) { HttpStatusCode status = response.StatusCode; Log.To.Sync.I(Tag, "Got error status: {0} for {1}. Reason: {2}", status.GetStatusCode(), request, response.ReasonPhrase); error = new HttpResponseException(status); RespondWithResult(fullBody, error, response); } else { Log.To.Sync.D(Tag, "Processing response: {0}", response); var entity = response.Content; var contentTypeHeader = entity.Headers.ContentType; Stream inputStream = null; if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/")) { Log.To.Sync.D(Tag, "contentTypeHeader = {0}", contentTypeHeader.ToString()); try { _topReader = new MultipartReader(contentTypeHeader.ToString(), this); inputStream = entity.ReadAsStreamAsync().Result; const int bufLen = 1024; var buffer = new byte[bufLen]; var numBytesRead = 0; while ((numBytesRead = inputStream.Read(buffer, 0, bufLen)) > 0) { if (numBytesRead != bufLen) { var bufferToAppend = new Couchbase.Lite.Util.ArraySegment<byte>(buffer, 0, numBytesRead).ToArray(); _topReader.AppendData(bufferToAppend); } else { _topReader.AppendData(buffer); } } RespondWithResult(fullBody, error, response); } finally { try { inputStream.Close(); } catch (IOException) { } } } else { Log.To.Sync.D(Tag, "contentTypeHeader is not multipart = {0}", contentTypeHeader.ToString()); if (entity != null) { try { inputStream = entity.ReadAsStreamAsync().Result; fullBody = Manager.GetObjectMapper().ReadValue<object>(inputStream); RespondWithResult(fullBody, error, response); } finally { try { inputStream.Close(); } catch (IOException) { } } } } } } catch (Exception e) { var err = (e is AggregateException) ? e.InnerException : e; Log.To.Sync.E(Tag, "Exception while processing bulk download response", err); error = err; RespondWithResult(fullBody, err, response); } }); }