/// <summary>
/// Start up some HTTP GETs, within our limit on the maximum simultaneous number
/// The entire method is not synchronized, only the portion pulling work off the list
/// Important to not hold the synchronized block while we do network access
/// </summary>
public void PullRemoteRevisions()
{
//find the work to be done in a synchronized block
var workToStartNow = new List <RevisionInternal>();
var bulkWorkToStartNow = new List <RevisionInternal>();
lock (locker)
{
while (httpConnectionCount + bulkWorkToStartNow.Count + workToStartNow.Count < ManagerOptions.Default.MaxOpenHttpConnections)
{
int nBulk = 0;
if (bulkRevsToPull != null)
{
nBulk = (bulkRevsToPull.Count < ManagerOptions.Default.MaxRevsToGetInBulk) ? bulkRevsToPull.Count : ManagerOptions.Default.MaxRevsToGetInBulk;
}
if (nBulk == 1)
{
// Rather than pulling a single revision in 'bulk', just pull it normally:
QueueRemoteRevision(bulkRevsToPull[0]);
bulkRevsToPull.Remove(0);
nBulk = 0;
}
if (nBulk > 0)
{
// Prefer to pull bulk revisions:
var range = new Couchbase.Lite.Util.ArraySegment <RevisionInternal>(bulkRevsToPull.ToArray(), 0, nBulk);
bulkWorkToStartNow.AddRange(range);
bulkRevsToPull.RemoveAll(range);
}
else
{
// Prefer to pull an existing revision over a deleted one:
IList <RevisionInternal> queue = revsToPull;
if (queue == null || queue.Count == 0)
{
queue = deletedRevsToPull;
if (queue == null || queue.Count == 0)
{
break;
}
}
// both queues are empty
workToStartNow.AddItem(queue[0]);
queue.Remove(0);
}
}
}
//actually run it outside the synchronized block
if (bulkWorkToStartNow.Count > 0)
{
PullBulkRevisions(bulkWorkToStartNow);
}
foreach (var rev in workToStartNow)
{
PullRemoteRevision(rev);
}
}
public void TestSimple()
{
var mime = Encoding.UTF8.GetBytes("--BOUNDARY\r\nFoo: Bar\r\n Header : Val ue \r\n\r\npart the first\r\n--BOUNDARY \r\n\r\n2nd part\r\n--BOUNDARY--");
var expectedParts = new List <IEnumerable <byte> > {
Encoding.UTF8.GetBytes("part the first"),
Encoding.UTF8.GetBytes("2nd part")
};
var expectedHeaders = new List <IDictionary <string, object> > {
new Dictionary <string, object> {
{ "Foo", "Bar" },
{ "Header", "Val ue" }
},
new Dictionary <string, object>()
};
for (int chunkSize = 1; chunkSize <= mime.Length; ++chunkSize)
{
WriteDebug("--- chunkSize = {0}", chunkSize);
Reset();
var reader = new MultipartReader("multipart/related; boundary=\"BOUNDARY\"", this);
Assert.IsFalse(reader.Finished);
Range r = new Range(0, 0);
do
{
Assert.IsTrue(r.Location < mime.Length, "Parser didn't stop at end");
r.Length = Math.Min(chunkSize, mime.Length - r.Location);
var sublist = new Couchbase.Lite.Util.ArraySegment <byte>(mime, r.Location, r.Length);
reader.AppendData(sublist);
r.Location += chunkSize;
} while(!reader.Finished);
}
Assert.AreEqual(expectedHeaders, _headerList);
Assert.AreEqual(expectedParts, _partList);
}
private Task ExecuteRequest(CouchbaseLiteHttpClient httpClient, HttpRequestMessage request)
{
object fullBody = null;
Exception error = null;
HttpResponseMessage response = null;
if (_tokenSource.IsCancellationRequested)
{
RespondWithResult(fullBody, new Exception(string.Format("{0}: Request {1} has been aborted", this, request)), response);
var tcs = new TaskCompletionSource <bool>();
tcs.SetCanceled();
return(tcs.Task);
}
request.Headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip"));
request.Headers.Add("X-Accept-Part-Encoding", "gzip");
Log.To.Sync.V(Tag, "Sending request: {0}", request);
var requestTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_tokenSource.Token);
httpClient.Authenticator = Authenticator;
return(httpClient.SendAsync(request, requestTokenSource.Token).ContinueWith(t =>
{
requestTokenSource.Dispose();
try {
response = t.Result;
} catch (Exception e) {
var err = Misc.Flatten(e).First();
Log.To.Sync.E(Tag, "Unhandled exception while getting bulk documents", err);
error = err;
RespondWithResult(fullBody, err, response);
return;
}
try {
if (response == null)
{
Log.To.Sync.I(Tag, "Didn't get response for {0}", request);
error = new HttpRequestException();
RespondWithResult(fullBody, error, response);
}
else if (!response.IsSuccessStatusCode)
{
HttpStatusCode status = response.StatusCode;
Log.To.Sync.I(Tag, "Got error status: {0} for {1}. Reason: {2}", status.GetStatusCode(), request, response.ReasonPhrase);
error = new HttpResponseException(status);
RespondWithResult(fullBody, error, response);
}
else
{
Log.To.Sync.D(Tag, "Processing response: {0}", response);
var entity = response.Content;
var contentTypeHeader = entity.Headers.ContentType;
Stream inputStream = null;
if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/"))
{
Log.To.Sync.D(Tag, "contentTypeHeader = {0}", contentTypeHeader.ToString());
try {
_topReader = new MultipartReader(contentTypeHeader.ToString(), this);
inputStream = entity.ReadAsStreamAsync().Result;
const int bufLen = 1024;
var buffer = new byte[bufLen];
var numBytesRead = 0;
while ((numBytesRead = inputStream.Read(buffer, 0, bufLen)) > 0)
{
if (numBytesRead != bufLen)
{
var bufferToAppend = new Couchbase.Lite.Util.ArraySegment <byte>(buffer, 0, numBytesRead).ToArray();
_topReader.AppendData(bufferToAppend);
}
else
{
_topReader.AppendData(buffer);
}
}
RespondWithResult(fullBody, error, response);
} finally {
try {
inputStream.Close();
} catch (IOException) { }
}
}
else
{
Log.To.Sync.D(Tag, "contentTypeHeader is not multipart = {0}", contentTypeHeader.ToString());
if (entity != null)
{
try {
inputStream = entity.ReadAsStreamAsync().Result;
fullBody = Manager.GetObjectMapper().ReadValue <object>(inputStream);
RespondWithResult(fullBody, error, response);
} finally {
try {
inputStream.Close();
} catch (IOException) { }
}
}
}
}
}
catch (Exception e)
{
var err = (e is AggregateException) ? e.InnerException : e;
Log.To.Sync.E(Tag, "Exception while processing bulk download response", err);
error = err;
RespondWithResult(fullBody, err, response);
}
}));
}
internal void SendAsyncMultipartDownloaderRequest(HttpMethod method, string relativePath, object body, Database db, RemoteRequestCompletionBlock onCompletion)
{
try {
var urlStr = BuildRelativeURLString(relativePath);
var url = new Uri(urlStr);
var message = new HttpRequestMessage(method, url);
message.Headers.Add("Accept", "*/*");
AddRequestHeaders(message);
var client = clientFactory.GetHttpClient(false);
var challengeResponseAuth = Authenticator as IChallengeResponseAuthenticator;
if (challengeResponseAuth != null) {
var authHandler = clientFactory.Handler as DefaultAuthHandler;
if (authHandler != null) {
authHandler.Authenticator = challengeResponseAuth;
}
challengeResponseAuth.PrepareWithRequest(message);
}
var authHeader = AuthUtils.GetAuthenticationHeaderValue(Authenticator, message.RequestUri);
if (authHeader != null) {
client.DefaultRequestHeaders.Authorization = authHeader;
}
client.SendAsync(message, CancellationTokenSource.Token).ContinueWith(new Action<Task<HttpResponseMessage>>(responseMessage =>
{
object fullBody = null;
Exception error = null;
try {
var response = responseMessage.Result;
// add in cookies to global store
//CouchbaseLiteHttpClientFactory.Instance.AddCookies(clientFactory.HttpHandler.CookieContainer.GetCookies(url));
var status = response.StatusCode;
if ((Int32)status.GetStatusCode() >= 300) {
Log.E(TAG, "Got error " + Sharpen.Extensions.ToString(status.GetStatusCode()));
Log.E(TAG, "Request was for: " + message);
Log.E(TAG, "Status reason: " + response.ReasonPhrase);
error = new WebException(response.ReasonPhrase);
} else {
var entity = response.Content;
var contentTypeHeader = response.Content.Headers.ContentType;
InputStream inputStream = null;
if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/related")) {
try {
var reader = new MultipartDocumentReader(LocalDatabase);
var contentType = contentTypeHeader.ToString();
reader.SetContentType(contentType);
var inputStreamTask = entity.ReadAsStreamAsync();
//inputStreamTask.Wait(90000, CancellationTokenSource.Token);
inputStream = inputStreamTask.Result;
const int bufLen = 1024;
var buffer = new byte[bufLen];
int numBytesRead = 0;
while ((numBytesRead = inputStream.Read(buffer)) != -1) {
if (numBytesRead != bufLen) {
var bufferToAppend = new Couchbase.Lite.Util.ArraySegment<Byte>(buffer, 0, numBytesRead);
reader.AppendData(bufferToAppend);
} else {
reader.AppendData(buffer);
}
}
reader.Finish();
fullBody = reader.GetDocumentProperties();
if (onCompletion != null) {
onCompletion(fullBody, error);
}
} catch (Exception ex) {
Log.E(TAG, "SendAsyncMultipartDownloaderRequest has an error occurred.", ex);
} finally {
try {
inputStream.Close();
} catch (Exception) { }
}
} else {
if (entity != null) {
try {
var readTask = entity.ReadAsStreamAsync();
//readTask.Wait(); // TODO: This should be scaled based on content length.
inputStream = readTask.Result;
fullBody = Manager.GetObjectMapper().ReadValue<Object>(inputStream);
if (onCompletion != null)
onCompletion(fullBody, error);
} catch (Exception ex) {
Log.E(TAG, "SendAsyncMultipartDownloaderRequest has an error occurred.", ex);
} finally {
try {
inputStream.Close();
} catch (Exception) { }
}
}
}
}
} catch (System.Net.ProtocolViolationException e) {
Log.E(TAG, "client protocol exception", e);
error = e;
} catch (IOException e) {
Log.E(TAG, "IO Exception", e);
error = e;
} finally {
client.Dispose();
}
}), WorkExecutor.Scheduler);
} catch (UriFormatException e) {
Log.E(TAG, "Malformed URL for async request", e);
}
}
/// <summary>
/// Start up some HTTP GETs, within our limit on the maximum simultaneous number
/// The entire method is not synchronized, only the portion pulling work off the list
/// Important to not hold the synchronized block while we do network access
/// </summary>
private void PullRemoteRevisions()
{
//find the work to be done in a synchronized block
var workToStartNow = new List <RevisionInternal>();
var bulkWorkToStartNow = new List <RevisionInternal>();
lock (_locker)
{
while (LocalDatabase.IsOpen)
{
int nBulk = 0;
if (_bulkRevsToPull != null)
{
nBulk = Math.Min(_bulkRevsToPull.Count, ReplicationOptions.MaxRevsToGetInBulk);
}
if (nBulk == 1)
{
// Rather than pulling a single revision in 'bulk', just pull it normally:
QueueRemoteRevision(_bulkRevsToPull[0]);
_bulkRevsToPull.RemoveAt(0);
nBulk = 0;
}
if (nBulk > 0)
{
// Prefer to pull bulk revisions:
var range = new Couchbase.Lite.Util.ArraySegment <RevisionInternal>(_bulkRevsToPull.ToArray(), 0, nBulk);
bulkWorkToStartNow.AddRange(range);
foreach (var val in range)
{
_bulkRevsToPull.Remove(val);
}
}
else
{
// Prefer to pull an existing revision over a deleted one:
IList <RevisionInternal> queue = _revsToPull;
if (queue == null || queue.Count == 0)
{
queue = _deletedRevsToPull;
if (queue == null || queue.Count == 0)
{
break; // both queues are empty
}
}
workToStartNow.Add(queue[0]);
queue.RemoveAt(0);
}
}
}
//actually run it outside the synchronized block
if (bulkWorkToStartNow.Count > 0)
{
PullBulkRevisions(bulkWorkToStartNow);
}
foreach (var rev in workToStartNow)
{
PullRemoteRevision(rev);
}
}
protected override internal void ExecuteRequest(HttpClient httpClient, HttpRequestMessage request)
{
object fullBody = null;
Exception error = null;
HttpResponseMessage response = null;
try
{
if (_tokenSource.IsCancellationRequested)
{
RespondWithResult(fullBody, new Exception(string.Format("{0}: Request {1} has been aborted", this, request)), response);
return;
}
}
catch (AggregateException e)
{
var err = e.InnerException;
Log.E(Tag, "Unhandled Exception", err);
error = err;
RespondWithResult(fullBody, err, response);
return;
}
catch (IOException e)
{
Log.E(Tag, "IO Exception", e);
error = e;
RespondWithResult(fullBody, e, response);
return;
}
catch (Exception e)
{
Log.E(Tag, "ExecuteRequest Exception: ", e);
error = e;
RespondWithResult(fullBody, e, response);
return;
}
try
{
Log.D(Tag + ".ExecuteRequest", "Sending request: {0}", request);
var requestTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_tokenSource.Token);
var responseTask = httpClient.SendAsync(request, requestTokenSource.Token);
if (!responseTask.Wait((Int32)ManagerOptions.Default.RequestTimeout.TotalMilliseconds, requestTokenSource.Token))
{
Log.E(Tag, "Response task timed out: {0}, {1}, {2}", responseTask, TaskScheduler.Current, Description());
throw new HttpResponseException(HttpStatusCode.RequestTimeout);
}
requestTokenSource.Dispose();
response = responseTask.Result;
}
catch (AggregateException e)
{
var err = e.InnerException;
Log.E(Tag, "Unhandled Exception: {0}, {1}", TaskScheduler.Current, Description());
Log.E(Tag, "Unhandled Exception at Line 129 or 130", err);
error = err;
RespondWithResult(fullBody, err, response);
return;
}
catch (IOException e)
{
Log.E(Tag, "IO Exception", e);
error = e;
RespondWithResult(fullBody, e, response);
return;
}
catch (Exception e)
{
Log.E(Tag, "ExecuteRequest Exception: ", e);
error = e;
RespondWithResult(fullBody, e, response);
return;
}
try
{
if (response == null)
{
Log.E(Tag, "Didn't get response for {0}", request);
error = new HttpRequestException();
RespondWithResult(fullBody, error, response);
}
else if (!response.IsSuccessStatusCode)
{
HttpStatusCode status = response.StatusCode;
Log.E(Tag, "Got error status: {0} for {1}. Reason: {2}", status.GetStatusCode(), request, response.ReasonPhrase);
error = new HttpResponseException(status);
RespondWithResult(fullBody, error, response);
}
else
{
Log.D(Tag, "Processing response: {0}", response);
var entity = response.Content;
var contentTypeHeader = entity.Headers.ContentType;
Stream inputStream = null;
if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/"))
{
Log.V(Tag, "contentTypeHeader = {0}", contentTypeHeader.ToString());
try
{
_topReader = new MultipartReader(contentTypeHeader.ToString(), this);
inputStream = entity.ReadAsStreamAsync().Result;
const int bufLen = 1024;
var buffer = new byte[bufLen];
var numBytesRead = 0;
while ((numBytesRead = inputStream.Read(buffer, 0, bufLen)) > 0)
{
if (numBytesRead != bufLen)
{
var bufferToAppend = new Couchbase.Lite.Util.ArraySegment<byte>(buffer, 0, numBytesRead).ToArray();
_topReader.AppendData(bufferToAppend);
}
else
{
_topReader.AppendData(buffer);
}
}
_topReader.Finished();
RespondWithResult(fullBody, error, response);
}
finally
{
try
{
inputStream.Close();
}
catch (IOException)
{
}
}
}
else
{
Log.V(Tag, "contentTypeHeader is not multipart = {0}", contentTypeHeader.ToString());
if (entity != null)
{
try
{
inputStream = entity.ReadAsStreamAsync().Result;
fullBody = Manager.GetObjectMapper().ReadValue<object>(inputStream);
RespondWithResult(fullBody, error, response);
}
finally
{
try
{
inputStream.Close();
}
catch (IOException)
{
}
}
}
}
}
}
catch (AggregateException e)
{
var err = e.InnerException;
Log.E(Tag, "Unhandled Exception", err);
error = err;
RespondWithResult(fullBody, err, response);
}
catch (IOException e)
{
Log.E(Tag, "IO Exception", e);
error = e;
RespondWithResult(fullBody, e, response);
}
catch (Exception e)
{
Log.E(Tag, "ExecuteRequest Exception: ", e);
error = e;
RespondWithResult(fullBody, e, response);
}
}
public void AppendData(IEnumerable <byte> newData)
{
if (newData == null)
{
return;
}
var data = newData.ToArray();
if (_buffer == null || data.Length == 0)
{
return;
}
_buffer.AddRange(data);
MultipartReader.MultipartReaderState nextState;
do
{
nextState = MultipartReader.MultipartReaderState.Uninitialized;
var bufLen = _buffer.Count;
switch (state)
{
case MultipartReader.MultipartReaderState.AtStart:
// The entire message might start with a boundary without a leading CRLF.
var boundaryWithoutLeadingCRLF = TrimmedBoundary.ToArray();
if (bufLen >= boundaryWithoutLeadingCRLF.Length)
{
// if (Arrays.equals(buffer.toByteArray(), boundaryWithoutLeadingCRLF)) {
if (Memcmp(_buffer.ToArray(), boundaryWithoutLeadingCRLF, boundaryWithoutLeadingCRLF.Length))
{
DeleteUpThrough(boundaryWithoutLeadingCRLF.Length);
nextState = MultipartReader.MultipartReaderState.InHeaders;
}
else
{
nextState = MultipartReader.MultipartReaderState.InPrologue;
}
}
break;
case MultipartReader.MultipartReaderState.InPrologue:
case MultipartReader.MultipartReaderState.InBody:
// Look for the next part boundary in the data we just added and the ending bytes of
// the previous data (in case the boundary string is split across calls)
if (bufLen < _boundary.Length)
{
break;
}
var start = Math.Max(0, bufLen - data.Length - _boundary.Length);
var r = SearchFor(_boundary, start);
if (r.Length > 0)
{
if (state == MultipartReader.MultipartReaderState.InBody)
{
var dataToAppend = new byte[r.Location];
Array.Copy(_buffer.ToArray(), 0, dataToAppend, 0, dataToAppend.Length);
_readerDelegate.AppendToPart(dataToAppend);
_readerDelegate.FinishedPart();
}
DeleteUpThrough(r.Location + r.Length);
nextState = MultipartReader.MultipartReaderState.InHeaders;
}
else
{
TrimBuffer();
}
break;
case MultipartReader.MultipartReaderState.InHeaders:
// First check for the end-of-message string ("--" after separator):
if (bufLen >= 2 && Memcmp(_buffer.ToArray(), EOM_BYTES, 2))
{
state = MultipartReader.MultipartReaderState.AtEnd;
Close();
return;
}
// Otherwise look for two CRLFs that delimit the end of the headers:
var headerEnd = SearchFor(CRLF_CRLF, 0);
if (headerEnd.Length > 0)
{
var headersBytes = new Couchbase.Lite.Util.ArraySegment <Byte>(_buffer.ToArray(), 0, headerEnd.Location); // <-- better?
var headersString = Encoding.UTF8.GetString(headersBytes.ToArray());
ParseHeaders(headersString);
DeleteUpThrough(headerEnd.Location + headerEnd.Length);
_readerDelegate.StartedPart(_headers);
nextState = MultipartReader.MultipartReaderState.InBody;
}
break;
default:
Log.To.Sync.E(Tag, "Unexpected data after end of MIME body, throwing...");
throw new InvalidOperationException("Unexpected data after end of MIME body");
}
if (nextState != MultipartReader.MultipartReaderState.Uninitialized)
{
state = nextState;
}
} while (nextState != MultipartReader.MultipartReaderState.Uninitialized && _buffer.Count > 0);
}
protected override internal Task ExecuteRequest(HttpClient httpClient, HttpRequestMessage request)
{
object fullBody = null;
Exception error = null;
HttpResponseMessage response = null;
if (_tokenSource.IsCancellationRequested)
{
RespondWithResult(fullBody, new Exception(string.Format("{0}: Request {1} has been aborted", this, request)), response);
var tcs = new TaskCompletionSource<bool>();
tcs.SetCanceled();
return tcs.Task;
}
Log.D(Tag + ".ExecuteRequest", "Sending request: {0}", request);
var requestTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_tokenSource.Token);
var retVal = httpClient.SendAsync(request, requestTokenSource.Token);
retVal.ConfigureAwait(false).GetAwaiter().OnCompleted(() => {
requestTokenSource.Dispose();
try {
response = retVal.Result;
} catch(Exception e) {
var err = (e is AggregateException) ? e.InnerException : e;
Log.E(Tag, "Unhandled Exception", err);
error = err;
RespondWithResult(fullBody, err, response);
return;
}
try
{
if (response == null)
{
Log.E(Tag, "Didn't get response for {0}", request);
error = new HttpRequestException();
RespondWithResult(fullBody, error, response);
}
else if (!response.IsSuccessStatusCode)
{
HttpStatusCode status = response.StatusCode;
Log.E(Tag, "Got error status: {0} for {1}. Reason: {2}", status.GetStatusCode(), request, response.ReasonPhrase);
error = new HttpResponseException(status);
RespondWithResult(fullBody, error, response);
}
else
{
Log.D(Tag, "Processing response: {0}", response);
var entity = response.Content;
var contentTypeHeader = entity.Headers.ContentType;
Stream inputStream = null;
if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/"))
{
Log.V(Tag, "contentTypeHeader = {0}", contentTypeHeader.ToString());
try
{
_topReader = new MultipartReader(contentTypeHeader.ToString(), this);
inputStream = entity.ReadAsStreamAsync().Result;
const int bufLen = 1024;
var buffer = new byte[bufLen];
var numBytesRead = 0;
while ((numBytesRead = inputStream.Read(buffer, 0, bufLen)) > 0)
{
if (numBytesRead != bufLen)
{
var bufferToAppend = new Couchbase.Lite.Util.ArraySegment<byte>(buffer, 0, numBytesRead).ToArray();
_topReader.AppendData(bufferToAppend);
}
else
{
_topReader.AppendData(buffer);
}
}
_topReader.Finished();
RespondWithResult(fullBody, error, response);
}
finally
{
try
{
inputStream.Close();
}
catch (IOException)
{
}
}
}
else
{
Log.V(Tag, "contentTypeHeader is not multipart = {0}", contentTypeHeader.ToString());
if (entity != null)
{
try
{
inputStream = entity.ReadAsStreamAsync().Result;
fullBody = Manager.GetObjectMapper().ReadValue<object>(inputStream);
RespondWithResult(fullBody, error, response);
}
finally
{
try
{
inputStream.Close();
}
catch (IOException)
{
}
}
}
}
}
}
catch (AggregateException e)
{
var err = e.InnerException;
Log.E(Tag, "Unhandled Exception", err);
error = err;
RespondWithResult(fullBody, err, response);
}
catch (IOException e)
{
Log.E(Tag, "IO Exception", e);
error = e;
RespondWithResult(fullBody, e, response);
}
catch (Exception e)
{
Log.E(Tag, "ExecuteRequest Exception: ", e);
error = e;
RespondWithResult(fullBody, e, response);
}
finally {
response.Dispose();
}
});
return retVal;
}
public void TestSimple()
{
var mime = Encoding.UTF8.GetBytes("--BOUNDARY\r\nFoo: Bar\r\n Header : Val ue \r\n\r\npart the first\r\n--BOUNDARY \r\n\r\n2nd part\r\n--BOUNDARY--");
var expectedParts = new List<IEnumerable<byte>> {
Encoding.UTF8.GetBytes("part the first"),
Encoding.UTF8.GetBytes("2nd part")
};
var expectedHeaders = new List<IDictionary<string, object>> {
new Dictionary<string, object> {
{ "Foo", "Bar" },
{ "Header", "Val ue" }
},
new Dictionary<string, object>()
};
for (int chunkSize = 1; chunkSize <= mime.Length; ++chunkSize) {
WriteDebug("--- chunkSize = {0}", chunkSize);
Reset();
var reader = new MultipartReader("multipart/related; boundary=\"BOUNDARY\"", this);
Assert.IsFalse(reader.Finished);
Range r = new Range(0, 0);
do {
Assert.IsTrue(r.Location < mime.Length, "Parser didn't stop at end");
r.Length = Math.Min(chunkSize, mime.Length - r.Location);
var sublist = new Couchbase.Lite.Util.ArraySegment<byte>(mime, r.Location, r.Length);
reader.AppendData(sublist);
r.Location += chunkSize;
} while(!reader.Finished);
}
Assert.AreEqual(expectedHeaders, _headerList);
Assert.AreEqual(expectedParts, _partList);
}
internal void SendAsyncMultipartDownloaderRequest(HttpMethod method, string relativePath, object body, Database db, RemoteRequestCompletionBlock onCompletion)
{
try {
var url = _baseUrl.Append(relativePath);
var message = new HttpRequestMessage(method, url);
message.Headers.Add("Accept", "*/*");
AddRequestHeaders(message);
var client = default(CouchbaseLiteHttpClient);
if (!_client.AcquireFor(TimeSpan.FromSeconds(1), out client))
{
Log.To.Sync.I(Tag, "Client is disposed, aborting request to {0}", new SecureLogString(relativePath, LogMessageSensitivity.PotentiallyInsecure));
return;
}
client.Authenticator = Authenticator;
var request = client.SendAsync(message, _cancellationTokenSource.Token).ContinueWith(new Action <Task <HttpResponseMessage> >(responseMessage =>
{
object fullBody = null;
Exception error = null;
try {
if (responseMessage.IsFaulted)
{
error = responseMessage.Exception.InnerException;
if (onCompletion != null)
{
onCompletion(null, error);
}
return;
}
var response = responseMessage.Result;
// add in cookies to global store
//CouchbaseLiteHttpClientFactory.Instance.AddCoIokies(clientFactory.HttpHandler.CookieContainer.GetCookies(url));
var status = response.StatusCode;
if ((Int32)status.GetStatusCode() >= 300)
{
Log.To.Sync.W(Tag, "Got error {0}", status.GetStatusCode());
Log.To.Sync.W(Tag, "Request was for: " + message);
Log.To.Sync.W(Tag, "Status reason: " + response.ReasonPhrase);
Log.To.Sync.W(Tag, "Passing error onto callback...");
error = new HttpResponseException(status);
if (onCompletion != null)
{
onCompletion(null, error);
}
}
else
{
var entity = response.Content;
var contentTypeHeader = response.Content.Headers.ContentType;
Stream inputStream = null;
if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/related"))
{
try {
var reader = new MultipartDocumentReader(db);
var contentType = contentTypeHeader.ToString();
reader.SetContentType(contentType);
var inputStreamTask = entity.ReadAsStreamAsync();
//inputStreamTask.Wait(90000, CancellationTokenSource.Token);
inputStream = inputStreamTask.Result;
const int bufLen = 1024;
var buffer = new byte[bufLen];
int numBytesRead = 0;
while ((numBytesRead = inputStream.Read(buffer, 0, buffer.Length)) > 0)
{
if (numBytesRead != bufLen)
{
var bufferToAppend = new Couchbase.Lite.Util.ArraySegment <Byte>(buffer, 0, numBytesRead);
reader.AppendData(bufferToAppend);
}
else
{
reader.AppendData(buffer);
}
}
reader.Finish();
fullBody = reader.GetDocumentProperties();
if (onCompletion != null)
{
onCompletion(fullBody, error);
}
} catch (Exception ex) {
Log.To.Sync.W(Tag, "SendAsyncMultipartDownloaderRequest got an exception, aborting...", ex);
} finally {
try {
inputStream.Close();
} catch (Exception) { }
}
}
else
{
if (entity != null)
{
try {
var readTask = entity.ReadAsStreamAsync();
//readTask.Wait(); // TODO: This should be scaled based on content length.
inputStream = readTask.Result;
fullBody = Manager.GetObjectMapper().ReadValue <Object>(inputStream);
if (onCompletion != null)
{
onCompletion(fullBody, error);
}
} catch (Exception ex) {
Log.To.Sync.W(Tag, "SendAsyncMultipartDownloaderRequest got an exception, aborting...", ex);
} finally {
try {
inputStream.Close();
} catch (Exception) { }
}
}
}
}
} catch (Exception e) {
Log.To.Sync.W(Tag, "Got exception during SendAsyncMultipartDownload, aborting...");
error = e;
} finally {
Task dummy;
_requests.TryRemove(message, out dummy);
responseMessage.Result.Dispose();
}
}), _workExecutor.Scheduler);
_requests.TryAdd(message, request);
} catch (UriFormatException e) {
Log.To.Sync.W(Tag, "Malformed URL for async request, aborting...", e);
}
}
public void AppendData(IEnumerable<byte> newData)
{
if (newData == null) {
return;
}
var data = newData.ToArray();
if (_buffer == null || data.Length == 0) {
return;
}
_buffer.AddRange(data);
MultipartReader.MultipartReaderState nextState;
do {
nextState = MultipartReader.MultipartReaderState.Uninitialized;
var bufLen = _buffer.Count;
switch (state) {
case MultipartReader.MultipartReaderState.AtStart:
// The entire message might start with a boundary without a leading CRLF.
var boundaryWithoutLeadingCRLF = TrimmedBoundary.ToArray();
if (bufLen >= boundaryWithoutLeadingCRLF.Length) {
// if (Arrays.equals(buffer.toByteArray(), boundaryWithoutLeadingCRLF)) {
if (Memcmp(_buffer.ToArray(), boundaryWithoutLeadingCRLF, boundaryWithoutLeadingCRLF.Length)) {
DeleteUpThrough(boundaryWithoutLeadingCRLF.Length);
nextState = MultipartReader.MultipartReaderState.InHeaders;
} else {
nextState = MultipartReader.MultipartReaderState.InPrologue;
}
}
break;
case MultipartReader.MultipartReaderState.InPrologue:
case MultipartReader.MultipartReaderState.InBody:
// Look for the next part boundary in the data we just added and the ending bytes of
// the previous data (in case the boundary string is split across calls)
if (bufLen < _boundary.Length) {
break;
}
var start = Math.Max(0, bufLen - data.Length - _boundary.Length);
var r = SearchFor(_boundary, start);
if (r.Length > 0) {
if (state == MultipartReader.MultipartReaderState.InBody) {
var dataToAppend = new byte[r.Location];
Array.Copy(_buffer.ToArray(), 0, dataToAppend, 0, dataToAppend.Length);
_readerDelegate.AppendToPart(dataToAppend);
_readerDelegate.FinishedPart();
}
DeleteUpThrough(r.Location + r.Length);
nextState = MultipartReader.MultipartReaderState.InHeaders;
} else {
TrimBuffer();
}
break;
case MultipartReader.MultipartReaderState.InHeaders:
// First check for the end-of-message string ("--" after separator):
if (bufLen >= 2 && Memcmp(_buffer.ToArray(), EOM_BYTES, 2)) {
state = MultipartReader.MultipartReaderState.AtEnd;
Close();
return;
}
// Otherwise look for two CRLFs that delimit the end of the headers:
var headerEnd = SearchFor(CRLF_CRLF, 0);
if (headerEnd.Length > 0) {
var headersBytes = new Couchbase.Lite.Util.ArraySegment<Byte>(_buffer.ToArray(), 0, headerEnd.Location); // <-- better?
var headersString = Encoding.UTF8.GetString(headersBytes.ToArray());
ParseHeaders(headersString);
DeleteUpThrough(headerEnd.Location + headerEnd.Length);
_readerDelegate.StartedPart(_headers);
nextState = MultipartReader.MultipartReaderState.InBody;
}
break;
default:
Log.To.Sync.E(Tag, "Unexpected data after end of MIME body, throwing...");
throw new InvalidOperationException("Unexpected data after end of MIME body");
}
if (nextState != MultipartReader.MultipartReaderState.Uninitialized) {
state = nextState;
}
} while (nextState != MultipartReader.MultipartReaderState.Uninitialized && _buffer.Count > 0);
}
internal void SendAsyncMultipartDownloaderRequest(HttpMethod method, string relativePath, object body, Database db, RemoteRequestCompletionBlock onCompletion)
{
try {
var url = _baseUrl.Append(relativePath);
var message = new HttpRequestMessage(method, url);
message.Headers.Add("Accept", "*/*");
AddRequestHeaders(message);
var client = default(CouchbaseLiteHttpClient);
if(!_client.AcquireFor(TimeSpan.FromSeconds(1), out client)) {
Log.To.Sync.I(Tag, "Client is disposed, aborting request to {0}", new SecureLogString(relativePath, LogMessageSensitivity.PotentiallyInsecure));
return;
}
client.Authenticator = Authenticator;
var request = client.SendAsync(message, _cancellationTokenSource.Token).ContinueWith(new Action<Task<HttpResponseMessage>>(responseMessage =>
{
object fullBody = null;
Exception error = null;
try {
if(responseMessage.IsFaulted) {
error = responseMessage.Exception.InnerException;
if(onCompletion != null) {
onCompletion(null, error);
}
return;
}
var response = responseMessage.Result;
// add in cookies to global store
//CouchbaseLiteHttpClientFactory.Instance.AddCoIokies(clientFactory.HttpHandler.CookieContainer.GetCookies(url));
var status = response.StatusCode;
if((Int32)status.GetStatusCode() >= 300) {
Log.To.Sync.W(Tag, "Got error {0}", status.GetStatusCode());
Log.To.Sync.W(Tag, "Request was for: " + message);
Log.To.Sync.W(Tag, "Status reason: " + response.ReasonPhrase);
Log.To.Sync.W(Tag, "Passing error onto callback...");
error = new HttpResponseException(status);
if(onCompletion != null) {
onCompletion(null, error);
}
} else {
var entity = response.Content;
var contentTypeHeader = response.Content.Headers.ContentType;
Stream inputStream = null;
if(contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/related")) {
try {
var reader = new MultipartDocumentReader(db);
var contentType = contentTypeHeader.ToString();
reader.SetContentType(contentType);
var inputStreamTask = entity.ReadAsStreamAsync();
//inputStreamTask.Wait(90000, CancellationTokenSource.Token);
inputStream = inputStreamTask.Result;
const int bufLen = 1024;
var buffer = new byte[bufLen];
int numBytesRead = 0;
while((numBytesRead = inputStream.Read(buffer, 0, buffer.Length)) > 0) {
if(numBytesRead != bufLen) {
var bufferToAppend = new Couchbase.Lite.Util.ArraySegment<Byte>(buffer, 0, numBytesRead);
reader.AppendData(bufferToAppend);
} else {
reader.AppendData(buffer);
}
}
reader.Finish();
fullBody = reader.GetDocumentProperties();
if(onCompletion != null) {
onCompletion(fullBody, error);
}
} catch(Exception ex) {
Log.To.Sync.W(Tag, "SendAsyncMultipartDownloaderRequest got an exception, aborting...", ex);
} finally {
try {
inputStream.Close();
} catch(Exception) { }
}
} else {
if(entity != null) {
try {
var readTask = entity.ReadAsStreamAsync();
//readTask.Wait(); // TODO: This should be scaled based on content length.
inputStream = readTask.Result;
fullBody = Manager.GetObjectMapper().ReadValue<Object>(inputStream);
if(onCompletion != null)
onCompletion(fullBody, error);
} catch(Exception ex) {
Log.To.Sync.W(Tag, "SendAsyncMultipartDownloaderRequest got an exception, aborting...", ex);
} finally {
try {
inputStream.Close();
} catch(Exception) { }
}
}
}
}
} catch(Exception e) {
Log.To.Sync.W(Tag, "Got exception during SendAsyncMultipartDownload, aborting...");
error = e;
} finally {
Task dummy;
_requests.TryRemove(message, out dummy);
responseMessage.Result.Dispose();
}
}), _workExecutor.Scheduler);
_requests.TryAdd(message, request);
} catch(UriFormatException e) {
Log.To.Sync.W(Tag, "Malformed URL for async request, aborting...", e);
}
}
protected override internal Task ExecuteRequest(HttpClient httpClient, HttpRequestMessage request)
{
object fullBody = null;
Exception error = null;
HttpResponseMessage response = null;
if (_tokenSource.IsCancellationRequested)
{
RespondWithResult(fullBody, new Exception(string.Format("{0}: Request {1} has been aborted", this, request)), response);
var tcs = new TaskCompletionSource <bool>();
tcs.SetCanceled();
return(tcs.Task);
}
Log.D(Tag + ".ExecuteRequest", "Sending request: {0}", request);
var requestTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_tokenSource.Token);
var retVal = httpClient.SendAsync(request, requestTokenSource.Token);
retVal.ConfigureAwait(false).GetAwaiter().OnCompleted(() => {
requestTokenSource.Dispose();
try {
response = retVal.Result;
} catch (Exception e) {
var err = (e is AggregateException) ? e.InnerException : e;
Log.E(Tag, "Unhandled Exception", err);
error = err;
RespondWithResult(fullBody, err, response);
return;
}
try
{
if (response == null)
{
Log.E(Tag, "Didn't get response for {0}", request);
error = new HttpRequestException();
RespondWithResult(fullBody, error, response);
}
else if (!response.IsSuccessStatusCode)
{
HttpStatusCode status = response.StatusCode;
Log.E(Tag, "Got error status: {0} for {1}. Reason: {2}", status.GetStatusCode(), request, response.ReasonPhrase);
error = new HttpResponseException(status);
RespondWithResult(fullBody, error, response);
}
else
{
Log.D(Tag, "Processing response: {0}", response);
var entity = response.Content;
var contentTypeHeader = entity.Headers.ContentType;
Stream inputStream = null;
if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/"))
{
Log.V(Tag, "contentTypeHeader = {0}", contentTypeHeader.ToString());
try
{
_topReader = new MultipartReader(contentTypeHeader.ToString(), this);
inputStream = entity.ReadAsStreamAsync().Result;
const int bufLen = 1024;
var buffer = new byte[bufLen];
var numBytesRead = 0;
while ((numBytesRead = inputStream.Read(buffer, 0, bufLen)) > 0)
{
if (numBytesRead != bufLen)
{
var bufferToAppend = new Couchbase.Lite.Util.ArraySegment <byte>(buffer, 0, numBytesRead).ToArray();
_topReader.AppendData(bufferToAppend);
}
else
{
_topReader.AppendData(buffer);
}
}
_topReader.Finished();
RespondWithResult(fullBody, error, response);
}
finally
{
try
{
inputStream.Close();
}
catch (IOException)
{
}
}
}
else
{
Log.V(Tag, "contentTypeHeader is not multipart = {0}", contentTypeHeader.ToString());
if (entity != null)
{
try
{
inputStream = entity.ReadAsStreamAsync().Result;
fullBody = Manager.GetObjectMapper().ReadValue <object>(inputStream);
RespondWithResult(fullBody, error, response);
}
finally
{
try
{
inputStream.Close();
}
catch (IOException)
{
}
}
}
}
}
}
catch (AggregateException e)
{
var err = e.InnerException;
Log.E(Tag, "Unhandled Exception", err);
error = err;
RespondWithResult(fullBody, err, response);
}
catch (IOException e)
{
Log.E(Tag, "IO Exception", e);
error = e;
RespondWithResult(fullBody, e, response);
}
catch (Exception e)
{
Log.E(Tag, "ExecuteRequest Exception: ", e);
error = e;
RespondWithResult(fullBody, e, response);
}
finally {
response.Dispose();
}
});
return(retVal);
}
protected override internal void ExecuteRequest(HttpClient httpClient, HttpRequestMessage request)
{
object fullBody = null;
Exception error = null;
HttpResponseMessage response = null;
try
{
if (_tokenSource.IsCancellationRequested)
{
RespondWithResult(fullBody, new Exception(string.Format("{0}: Request {1} has been aborted", this, request)), response);
return;
}
}
catch (AggregateException e)
{
var err = e.InnerException;
Log.E(Tag, "Unhandled Exception", err);
error = err;
RespondWithResult(fullBody, err, response);
return;
}
catch (IOException e)
{
Log.E(Tag, "IO Exception", e);
error = e;
RespondWithResult(fullBody, e, response);
return;
}
catch (Exception e)
{
Log.E(Tag, "ExecuteRequest Exception: ", e);
error = e;
RespondWithResult(fullBody, e, response);
return;
}
try
{
Log.D(Tag + ".ExecuteRequest", "Sending request: {0}", request);
var requestTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_tokenSource.Token);
var responseTask = httpClient.SendAsync(request, requestTokenSource.Token);
if (!responseTask.Wait((Int32)ManagerOptions.Default.RequestTimeout.TotalMilliseconds, requestTokenSource.Token))
{
Log.E(Tag, "Response task timed out: {0}, {1}, {2}", responseTask, TaskScheduler.Current, Description());
throw new HttpResponseException(HttpStatusCode.RequestTimeout);
}
requestTokenSource.Dispose();
response = responseTask.Result;
}
catch (AggregateException e)
{
var err = e.InnerException;
Log.E(Tag, "Unhandled Exception: {0}, {1}", TaskScheduler.Current, Description());
Log.E(Tag, "Unhandled Exception at Line 129 or 130", err);
error = err;
RespondWithResult(fullBody, err, response);
return;
}
catch (IOException e)
{
Log.E(Tag, "IO Exception", e);
error = e;
RespondWithResult(fullBody, e, response);
return;
}
catch (Exception e)
{
Log.E(Tag, "ExecuteRequest Exception: ", e);
error = e;
RespondWithResult(fullBody, e, response);
return;
}
try
{
if (response == null)
{
Log.E(Tag, "Didn't get response for {0}", request);
error = new HttpRequestException();
RespondWithResult(fullBody, error, response);
}
else if (!response.IsSuccessStatusCode)
{
HttpStatusCode status = response.StatusCode;
Log.E(Tag, "Got error status: {0} for {1}. Reason: {2}", status.GetStatusCode(), request, response.ReasonPhrase);
error = new HttpResponseException(status);
RespondWithResult(fullBody, error, response);
}
else
{
Log.D(Tag, "Processing response: {0}", response);
var entity = response.Content;
var contentTypeHeader = entity.Headers.ContentType;
Stream inputStream = null;
if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/"))
{
Log.V(Tag, "contentTypeHeader = {0}", contentTypeHeader.ToString());
try
{
_topReader = new MultipartReader(contentTypeHeader.ToString(), this);
inputStream = entity.ReadAsStreamAsync().Result;
const int bufLen = 1024;
var buffer = new byte[bufLen];
var numBytesRead = 0;
while ((numBytesRead = inputStream.Read(buffer, 0, bufLen)) > 0)
{
if (numBytesRead != bufLen)
{
var bufferToAppend = new Couchbase.Lite.Util.ArraySegment <byte>(buffer, 0, numBytesRead).ToArray();
_topReader.AppendData(bufferToAppend);
}
else
{
_topReader.AppendData(buffer);
}
}
_topReader.Finished();
RespondWithResult(fullBody, error, response);
}
finally
{
try
{
inputStream.Close();
}
catch (IOException)
{
}
}
}
else
{
Log.V(Tag, "contentTypeHeader is not multipart = {0}", contentTypeHeader.ToString());
if (entity != null)
{
try
{
inputStream = entity.ReadAsStreamAsync().Result;
fullBody = Manager.GetObjectMapper().ReadValue <object>(inputStream);
RespondWithResult(fullBody, error, response);
}
finally
{
try
{
inputStream.Close();
}
catch (IOException)
{
}
}
}
}
}
}
catch (AggregateException e)
{
var err = e.InnerException;
Log.E(Tag, "Unhandled Exception", err);
error = err;
RespondWithResult(fullBody, err, response);
}
catch (IOException e)
{
Log.E(Tag, "IO Exception", e);
error = e;
RespondWithResult(fullBody, e, response);
}
catch (Exception e)
{
Log.E(Tag, "ExecuteRequest Exception: ", e);
error = e;
RespondWithResult(fullBody, e, response);
}
}
private Task ExecuteRequest(CouchbaseLiteHttpClient httpClient, HttpRequestMessage request)
{
object fullBody = null;
Exception error = null;
HttpResponseMessage response = null;
if (_tokenSource.IsCancellationRequested) {
RespondWithResult(fullBody, new Exception(string.Format("{0}: Request {1} has been aborted", this, request)), response);
var tcs = new TaskCompletionSource<bool>();
tcs.SetCanceled();
return tcs.Task;
}
request.Headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip"));
request.Headers.Add("X-Accept-Part-Encoding", "gzip");
Log.To.Sync.V(Tag, "Sending request: {0}", request);
var requestTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_tokenSource.Token);
httpClient.Authenticator = Authenticator;
return httpClient.SendAsync(request, requestTokenSource.Token).ContinueWith(t =>
{
requestTokenSource.Dispose();
try {
response = t.Result;
} catch(Exception e) {
var err = Misc.Flatten(e).First();
Log.To.Sync.E(Tag, "Unhandled exception while getting bulk documents", err);
error = err;
RespondWithResult(fullBody, err, response);
return;
}
try {
if (response == null) {
Log.To.Sync.I(Tag, "Didn't get response for {0}", request);
error = new HttpRequestException();
RespondWithResult(fullBody, error, response);
} else if (!response.IsSuccessStatusCode) {
HttpStatusCode status = response.StatusCode;
Log.To.Sync.I(Tag, "Got error status: {0} for {1}. Reason: {2}", status.GetStatusCode(), request, response.ReasonPhrase);
error = new HttpResponseException(status);
RespondWithResult(fullBody, error, response);
} else {
Log.To.Sync.D(Tag, "Processing response: {0}", response);
var entity = response.Content;
var contentTypeHeader = entity.Headers.ContentType;
Stream inputStream = null;
if (contentTypeHeader != null && contentTypeHeader.ToString().Contains("multipart/"))
{
Log.To.Sync.D(Tag, "contentTypeHeader = {0}", contentTypeHeader.ToString());
try {
_topReader = new MultipartReader(contentTypeHeader.ToString(), this);
inputStream = entity.ReadAsStreamAsync().Result;
const int bufLen = 1024;
var buffer = new byte[bufLen];
var numBytesRead = 0;
while ((numBytesRead = inputStream.Read(buffer, 0, bufLen)) > 0) {
if (numBytesRead != bufLen) {
var bufferToAppend = new Couchbase.Lite.Util.ArraySegment<byte>(buffer, 0, numBytesRead).ToArray();
_topReader.AppendData(bufferToAppend);
} else {
_topReader.AppendData(buffer);
}
}
RespondWithResult(fullBody, error, response);
} finally {
try {
inputStream.Close();
} catch (IOException) { }
}
} else {
Log.To.Sync.D(Tag, "contentTypeHeader is not multipart = {0}", contentTypeHeader.ToString());
if (entity != null) {
try {
inputStream = entity.ReadAsStreamAsync().Result;
fullBody = Manager.GetObjectMapper().ReadValue<object>(inputStream);
RespondWithResult(fullBody, error, response);
} finally {
try {
inputStream.Close();
} catch (IOException) { }
}
}
}
}
}
catch (Exception e)
{
var err = (e is AggregateException) ? e.InnerException : e;
Log.To.Sync.E(Tag, "Exception while processing bulk download response", err);
error = err;
RespondWithResult(fullBody, err, response);
}
});
}
