public void TestFindMissingRevisions()
        {
            var revs = new RevisionList();
            database.Storage.FindMissingRevisions(revs);

            var doc1r1 = PutDoc(new Dictionary<string, object> {
                ["_id"] = "11111",
                ["key"] = "one"
            });
            var doc2r1 = PutDoc(new Dictionary<string, object> {
                ["_id"] = "22222",
                ["key"] = "two"
            });
            PutDoc(new Dictionary<string, object> {
                ["_id"] = "33333",
                ["key"] = "three"
            });
            PutDoc(new Dictionary<string, object> {
                ["_id"] = "44444",
                ["key"] = "four"
            });
            PutDoc(new Dictionary<string, object> {
                ["_id"] = "55555",
                ["key"] = "five"
            });

            var doc1r2 = PutDoc(new Dictionary<string, object> {
                ["_id"] = "11111",
                ["_rev"] = doc1r1.RevID.ToString(),
                ["key"] = "one+"
            });
            var doc2r2 = PutDoc(new Dictionary<string, object> {
                ["_id"] = "22222",
                ["_rev"] = doc2r1.RevID.ToString(),
                ["key"] = "two+"
            });

            PutDoc(new Dictionary<string, object> {
                ["_id"] = "11111",
                ["_rev"] = doc1r2.RevID.ToString(),
                ["_deleted"] = true
            });

            // Now call FindMissingRevisions
            var revToFind1 = new RevisionInternal("11111", "3-6060".AsRevID(), false);
            var revToFind2 = new RevisionInternal("22222", doc2r2.RevID, false);
            var revToFind3 = new RevisionInternal("99999", "9-4141".AsRevID(), false);
            revs = new RevisionList(new List<RevisionInternal> { revToFind1, revToFind2, revToFind3 });
            database.Storage.FindMissingRevisions(revs);
            CollectionAssert.AreEqual(new List<RevisionInternal> { revToFind1, revToFind3 }, revs);

            // Check the possible ancestors
            ValueTypePtr<bool> haveBodies = false;
            CollectionAssert.AreEqual(new List<RevisionID> { doc1r2.RevID, doc1r1.RevID }, database.Storage.GetPossibleAncestors(revToFind1, 0, haveBodies));
            CollectionAssert.AreEqual(new List<RevisionID> { doc1r2.RevID }, database.Storage.GetPossibleAncestors(revToFind1, 1, haveBodies));
            CollectionAssert.AreEqual(new List<RevisionID>(), database.Storage.GetPossibleAncestors(revToFind3, 0, haveBodies));
        }
        internal override void ProcessInbox(RevisionList inbox)
        {
            // Generate a set of doc/rev IDs in the JSON format that _revs_diff wants:
            // <http://wiki.apache.org/couchdb/HttpPostRevsDiff>
            var diffs = new Dictionary<String, IList<String>>();
            foreach (var rev in inbox)
            {
                var docID = rev.GetDocId();
                var revs = diffs.Get(docID);
                if (revs == null)
                {
                    revs = new List<String>();
                    diffs[docID] = revs;
                }
                revs.AddItem(rev.GetRevId());
                AddPending(rev);
            }

            // Call _revs_diff on the target db:
            Log.D(Tag, "processInbox() calling asyncTaskStarted()");
            Log.D(Tag, "posting to /_revs_diff: {0}", String.Join(Environment.NewLine, Manager.GetObjectMapper().WriteValueAsString(diffs)));

            AsyncTaskStarted();
            SendAsyncRequest(HttpMethod.Post, "/_revs_diff", diffs, (response, e) =>
            {
                try {
                    var results = response.AsDictionary<string, object>();

                    Log.D(Tag, "/_revs_diff response: {0}\r\n{1}", response, results);

                    if (e != null) 
                    {
                        SetLastError(e);
                        RevisionFailed();
                    } else {
                        if (results.Count != 0) 
                        {
                            // Go through the list of local changes again, selecting the ones the destination server
                            // said were missing and mapping them to a JSON dictionary in the form _bulk_docs wants:
                            var docsToSend = new List<object> ();
                            var revsToSend = new RevisionList();
                            foreach (var rev in inbox)
                            {
                                // Is this revision in the server's 'missing' list?
                                IDictionary<string, object> properties = null;
                                var revResults = results.Get(rev.GetDocId()).AsDictionary<string, object>();

                                if (revResults == null)
                                {
                                    continue;
                                }

                                var revs = ((JArray)revResults.Get("missing")).Values<String>().ToList();
                                if (revs == null || !revs.Contains(rev.GetRevId()))
                                {
                                    RemovePending(rev);
                                    continue;
                                }

                                // Get the revision's properties:
                                var contentOptions = DocumentContentOptions.IncludeAttachments;

                                if (!dontSendMultipart && revisionBodyTransformationFunction == null)
                                {
                                    contentOptions &= DocumentContentOptions.BigAttachmentsFollow;
                                }


                                RevisionInternal loadedRev;
                                try {
                                    loadedRev = LocalDatabase.LoadRevisionBody (rev, contentOptions);
                                    properties = new Dictionary<string, object>(rev.GetProperties());
                                } catch (CouchbaseLiteException e1) {
                                    Log.W(Tag, string.Format("{0} Couldn't get local contents of {1}", rev, this), e1);
                                    RevisionFailed();
                                    continue;
                                }

                                var populatedRev = TransformRevision(loadedRev);

                                IList<string> possibleAncestors = null;
                                if (revResults.ContainsKey("possible_ancestors"))
                                {
                                    possibleAncestors = revResults["possible_ancestors"].AsList<string>();
                                }

                                properties = new Dictionary<string, object>(populatedRev.GetProperties());
                                var revisions = LocalDatabase.GetRevisionHistoryDictStartingFromAnyAncestor(populatedRev, possibleAncestors);
                                properties["_revisions"] = revisions;
                                populatedRev.SetProperties(properties);

                                // Strip any attachments already known to the target db:
                                if (properties.ContainsKey("_attachments")) 
                                {
                                    // Look for the latest common ancestor and stuf out older attachments:
                                    var minRevPos = FindCommonAncestor(populatedRev, possibleAncestors);

                                    Database.StubOutAttachmentsInRevBeforeRevPos(populatedRev, minRevPos + 1, false);

                                    properties = populatedRev.GetProperties();

                                    if (!dontSendMultipart && UploadMultipartRevision(populatedRev)) 
                                    {
                                        continue;
                                    }
                                }

                                if (properties == null || !properties.ContainsKey("_id"))
                                {
                                    throw new InvalidOperationException("properties must contain a document _id");
                                }
                                // Add the _revisions list:
                                revsToSend.Add(rev);

                                //now add it to the docs to send
                                docsToSend.AddItem (properties);
                            }

                            UploadBulkDocs(docsToSend, revsToSend);
                        } 
                        else 
                        {
                            foreach (var revisionInternal in inbox)
                            {
                                RemovePending(revisionInternal);
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    Log.E(Tag, "Unhandled exception in Pusher.ProcessInbox", ex);
                }
                finally
                {
                    Log.D(Tag, "processInbox() calling AsyncTaskFinished()");
                    AsyncTaskFinished(1);
                }
            });
        }
        private void UploadChanges(IList<RevisionInternal> changes, IDictionary<string, object> revsDiffResults)
        {

            // Go through the list of local changes again, selecting the ones the destination server
            // said were missing and mapping them to a JSON dictionary in the form _bulk_docs wants:
            var docsToSend = new List<object> ();
            var revsToSend = new RevisionList();
            IDictionary<string, object> revResults = null;
            foreach (var rev in changes) {
                // Is this revision in the server's 'missing' list?
                if (revsDiffResults != null) {
                    revResults = revsDiffResults.Get(rev.DocID).AsDictionary<string, object>(); 
                    if (revResults == null) {
                        continue;
                    }

                    var revs = revResults.Get("missing").AsList<string>();
                    if (revs == null || !revs.Any(id => id.Equals(rev.RevID.ToString()))) {
                        RemovePending(rev);
                        continue;
                    }
                }

                IDictionary<string, object> properties = null;
                RevisionInternal loadedRev;
                try {
                    loadedRev = LocalDatabase.LoadRevisionBody (rev);
                    if(loadedRev == null) {
                        throw Misc.CreateExceptionAndLog(Log.To.Sync, StatusCode.NotFound, TAG,
                            "Unable to load revision body");
                    }

                    properties = new Dictionary<string, object>(rev.GetProperties());
                } catch (Exception e1) {
                    Log.To.Sync.E(TAG, String.Format("Couldn't get local contents of {0}, marking revision failed",
                        rev), e1);
                    RevisionFailed();
                    continue;
                }

                if (properties.GetCast<bool> ("_removed")) {
                    RemovePending (rev);
                    continue;
                }

                var populatedRev = TransformRevision(loadedRev);
                var backTo = revResults?.Get("possible_ancestors")?.AsList<RevisionID>();

                try {
                    var history = LocalDatabase.GetRevisionHistory(populatedRev, backTo);
                    if(history == null) {
                        throw Misc.CreateExceptionAndLog(Log.To.Sync, StatusCode.DbError, TAG,
                            "Unable to load revision history");
                    }

                    properties["_revisions"] = TreeRevisionID.MakeRevisionHistoryDict(history);
                    populatedRev.SetPropertyForKey("_revisions", properties["_revisions"]);
                } catch(Exception e1) {
                    Log.To.Sync.E(TAG, "Error getting revision history, marking revision failed", e1);
                    RevisionFailed();
                    continue;
                }

                // Strip any attachments already known to the target db:
                if (properties.Get("_attachments") != null) {
                    // Look for the latest common ancestor and stuf out older attachments:
                    var minRevPos = FindCommonAncestor(populatedRev, backTo);
                    try {
                        LocalDatabase.ExpandAttachments(populatedRev, minRevPos + 1, !_dontSendMultipart, false);
                    } catch(Exception ex) {
                        Log.To.Sync.E(TAG, "Error expanding attachments, marking revision failed", ex);
                        RevisionFailed();
                        continue;
                    }

                    properties = populatedRev.GetProperties();
                    if (!_dontSendMultipart && UploadMultipartRevision(populatedRev)) {
                        continue;
                    }
                }

                if (properties == null || !properties.ContainsKey("_id")) {
                    throw Misc.CreateExceptionAndLog(Log.To.Sync, StatusCode.BadParam, TAG,
                        "properties must contain a document _id");
                }

                // Add the _revisions list:
                revsToSend.Add(rev);

                //now add it to the docs to send
                docsToSend.Add (properties);
            }

            UploadBulkDocs(docsToSend, revsToSend);
        }
Example #4
0
        private void UploadBulkDocs(IList <object> docsToSend, RevisionList revChanges)
        {
            // Post the revisions to the destination. "new_edits":false means that the server should
            // use the given _rev IDs instead of making up new ones.
            var numDocsToSend = docsToSend.Count;

            if (numDocsToSend == 0)
            {
                return;
            }

            Log.To.Sync.I(TAG, "{0} sending {1} revisions", this, numDocsToSend);
            Log.To.Sync.V(TAG, "{0} sending {1}", this, revChanges);

            var bulkDocsBody = new Dictionary <string, object>();

            bulkDocsBody["docs"]      = docsToSend;
            bulkDocsBody["new_edits"] = false;
            SafeAddToChangesCount(numDocsToSend);

            _remoteSession.SendAsyncRequest(HttpMethod.Post, "/_bulk_docs", bulkDocsBody, (result, e) => {
                if (e == null)
                {
                    var failedIds = new HashSet <string>();
                    // _bulk_docs response is really an array not a dictionary
                    var items = result.AsList <object>();
                    foreach (var item in items)
                    {
                        var itemObject = item.AsDictionary <string, object>();
                        var status     = StatusFromBulkDocsResponseItem(itemObject);
                        if (!status.IsSuccessful)
                        {
                            // One of the docs failed to save.
                            Log.To.Sync.I(TAG, "_bulk_docs got an error: " + item);

                            // 403/Forbidden means validation failed; don't treat it as an error
                            // because I did my job in sending the revision. Other statuses are
                            // actual replication errors.
                            if (status.Code != StatusCode.Forbidden)
                            {
                                var docId = itemObject.GetCast <string>("id");
                                failedIds.Add(docId);
                            }
                        }
                    }

                    // Remove from the pending list all the revs that didn't fail:
                    foreach (var revisionInternal in revChanges)
                    {
                        if (!failedIds.Contains(revisionInternal.DocID))
                        {
                            RemovePending(revisionInternal);
                        }
                    }
                }

                if (e != null)
                {
                    LastError = e;
                    RevisionFailed();
                }
                else
                {
                    Log.To.Sync.V(TAG, "{0} sent {1}", this, revChanges);
                }

                SafeAddToCompletedChangesCount(numDocsToSend);
            });
        }
        //Create a response body for an HTTP response from a given list of DB changes, including all conflicts
        private static IDictionary<string, object> ResponseBodyForChanges(RevisionList changes, long since, int limit, DBMonitorCouchbaseResponseState state)
        {
            string lastDocId = null;
            IDictionary<string, object> lastEntry = null;
            var entries = new List<IDictionary<string, object>>();
            foreach (var rev in changes) {
                string docId = rev.DocID;
                if (docId.Equals(lastDocId)) {
                    ((IList)lastEntry["changes"]).Add(new Dictionary<string, object> { { "rev", rev.RevID } });
                } else {
                    lastEntry = ChangesDictForRev(rev, state);
                    entries.Add(lastEntry);
                    lastDocId = docId;
                }
            }
                    
            entries.Sort((x, y) => (int)((long)x["seq"] - (long)y["seq"]));
            if (entries.Count > limit) {
                entries.RemoveRange(limit, entries.Count - limit);
            }

            long lastSequence = entries.Any() ? (long)entries.Last()["seq"] : since;
            return new Dictionary<string, object> {
                { "results", entries },
                { "last_seq", lastSequence }
            };
        }
        /// <exception cref="Couchbase.Lite.Storage.SQLException"></exception>
        internal Int32 FindMissingRevisions(RevisionList touchRevs)
        {
            var numRevisionsRemoved = 0;
            if (touchRevs.Count == 0)
            {
                return numRevisionsRemoved;
            }

            var quotedDocIds = JoinQuoted(touchRevs.GetAllDocIds());
            var quotedRevIds = JoinQuoted(touchRevs.GetAllRevIds());
            var sql = "SELECT docid, revid FROM revs, docs " + "WHERE docid IN (" + quotedDocIds
                         + ") AND revid in (" + quotedRevIds + ")" + " AND revs.doc_id == docs.doc_id";
            Cursor cursor = null;
            try
            {
                cursor = StorageEngine.RawQuery(sql);
                cursor.MoveToNext();
                while (!cursor.IsAfterLast())
                {
                    var rev = touchRevs.RevWithDocIdAndRevId(cursor.GetString(0), cursor.GetString(1));
                    if (rev != null)
                    {
                        touchRevs.Remove(rev);
                        numRevisionsRemoved += 1;
                    }
                    cursor.MoveToNext();
                }
            }
            finally
            {
                if (cursor != null)
                {
                    cursor.Close();
                }
            }
            return numRevisionsRemoved;
        }
        public RevisionList ChangesSince(Int64 lastSequence, ChangesOptions options, RevisionFilter filter)
        {
            // http://wiki.apache.org/couchdb/HTTP_database_API#Changes
            // Translate options to ForestDB:
            if (options.Descending) {
                // https://github.com/couchbase/couchbase-lite-ios/issues/641
                throw new CouchbaseLiteException(StatusCode.NotImplemented);
            }

            var forestOps = C4EnumeratorOptions.DEFAULT;
            forestOps.flags |= C4EnumeratorFlags.IncludeDeleted | C4EnumeratorFlags.IncludeNonConflicted;
            if (options.IncludeDocs || options.IncludeConflicts || filter != null) {
                forestOps.flags |= C4EnumeratorFlags.IncludeBodies;
            }

            var changes = new RevisionList();
            var e = new CBForestDocEnumerator(Forest, lastSequence, forestOps);
            foreach (var next in e) {
                var doc = next.Document;
                var revs = default(IEnumerable<RevisionInternal>);
                if (options.IncludeConflicts) {
                    using (var enumerator = new CBForestHistoryEnumerator(doc, true, false)) {
                        var includeBody = forestOps.flags.HasFlag(C4EnumeratorFlags.IncludeBodies);
                        revs = enumerator.Select(x => new RevisionInternal(x.Document, includeBody)).ToList();
                    }
                } else {
                    revs = new List<RevisionInternal> { new RevisionInternal(doc, forestOps.flags.HasFlag(C4EnumeratorFlags.IncludeBodies)) };
                }

                foreach (var rev in revs) {
                    Debug.Assert(rev != null);
                    if (filter == null || filter(rev)) {
                        if (!options.IncludeDocs) {
                            rev.SetBody(null);
                        }

                        if(filter == null || filter(rev)) {
                            changes.Add(rev);
                        }
                    }
                }
            }

            if (options.SortBySequence) {
                changes.SortBySequence(!options.Descending);
                changes.Limit(options.Limit);
            }

            return changes;
        }
        private void UploadBulkDocs(IList<object> docsToSend, RevisionList revChanges)
        {
            // Post the revisions to the destination. "new_edits":false means that the server should
            // use the given _rev IDs instead of making up new ones.
            var numDocsToSend = docsToSend.Count;
            if (numDocsToSend == 0)
            {
                return;
            }

            Log.V(TAG, string.Format("{0}: POSTing " + numDocsToSend + " revisions to _bulk_docs: {1}", this, docsToSend));

            var bulkDocsBody = new Dictionary<string, object>();
            bulkDocsBody["docs"] = docsToSend;
            bulkDocsBody["new_edits"] = false;
            SafeAddToChangesCount(numDocsToSend);

            SendAsyncRequest(HttpMethod.Post, "/_bulk_docs", bulkDocsBody, (result, e) => {
                if (e == null)
                {
                    var failedIds = new HashSet<string>();
                    // _bulk_docs response is really an array not a dictionary
                    var items = result.AsList<object>();
                    foreach(var item in items)
                    {
                        var itemObject = item.AsDictionary<string, object>();
                        var status = StatusFromBulkDocsResponseItem(itemObject);
                        if (!status.IsSuccessful)
                        {
                            // One of the docs failed to save.
                            Log.W(TAG, "_bulk_docs got an error: " + item);

                            // 403/Forbidden means validation failed; don't treat it as an error
                            // because I did my job in sending the revision. Other statuses are
                            // actual replication errors.
                            if (status.Code != StatusCode.Forbidden)
                            {
                                var docId = itemObject.GetCast<string>("id");
                                failedIds.Add(docId);
                            }
                        }
                    }

                    // Remove from the pending list all the revs that didn't fail:
                    foreach (var revisionInternal in revChanges)
                    {
                        if (!failedIds.Contains(revisionInternal.GetDocId()))
                        {
                            RemovePending(revisionInternal);
                        }
                    }
                }

                if (e != null) 
                {
                    LastError = e;
                    RevisionFailed();
                } 
                else 
                {
                    Log.V(TAG, string.Format("POSTed to _bulk_docs: {0}", docsToSend));
                }
                SafeAddToCompletedChangesCount(numDocsToSend);
            });
        }
        internal override void ProcessInbox(RevisionList inbox)
        {
            if (Status == ReplicationStatus.Offline)
            {
                Log.To.Sync.I(TAG, "Offline, so skipping inbox process");
                return;
            }

            if (ReplicationOptions.AllNew)
            {
                // If 'allNew' option is set, upload new revs without checking first:
                foreach (var rev in inbox)
                {
                    AddPending(rev);
                }

                UploadChanges(inbox, null);
                return;
            }

            // Generate a set of doc/rev IDs in the JSON format that _revs_diff wants:
            // <http://wiki.apache.org/couchdb/HttpPostRevsDiff>
            var diffs      = new Dictionary <String, IList <String> >();
            var inboxCount = inbox.Count;

            foreach (var rev in inbox)
            {
                var docID = rev.DocID;
                var revs  = diffs.Get(docID);
                if (revs == null)
                {
                    revs         = new List <String>();
                    diffs[docID] = revs;
                }
                revs.Add(rev.RevID.ToString());
                AddPending(rev);
            }

            // Call _revs_diff on the target db:
            Log.To.Sync.D(TAG, "posting to /_revs_diff: {0}", String.Join(Environment.NewLine, new[] { Manager.GetObjectMapper().WriteValueAsString(diffs) }));
            _remoteSession.SendAsyncRequest(HttpMethod.Post, "/_revs_diff", diffs, (response, e) =>
            {
                try {
                    if (!LocalDatabase.IsOpen)
                    {
                        return;
                    }

                    var results = response.AsDictionary <string, object>();

                    Log.To.Sync.D(TAG, "/_revs_diff response: {0}\r\n{1}", response, results);

                    if (e != null)
                    {
                        LastError = e;
                        for (int i = 0; i < inboxCount; i++)
                        {
                            RevisionFailed();
                        }

                        if (Continuous)
                        {
                            FireTrigger(ReplicationTrigger.WaitingForChanges);
                        }
                        else
                        {
                            FireTrigger(ReplicationTrigger.StopImmediate);
                        }
                    }
                    else
                    {
                        if (results.Count != 0)
                        {
                            UploadChanges(inbox, results);
                        }
                        else
                        {
                            foreach (var revisionInternal in inbox)
                            {
                                RemovePending(revisionInternal);
                            }

                            //SafeAddToCompletedChangesCount(inbox.Count);
                        }
                    }
                } catch (Exception ex) {
                    Log.To.Sync.E(TAG, "Unhandled exception in Pusher.ProcessInbox, continuing...", ex);
                }
            });
        }
Example #10
0
        private void UploadChanges(IList <RevisionInternal> changes, IDictionary <string, object> revsDiffResults)
        {
            // Go through the list of local changes again, selecting the ones the destination server
            // said were missing and mapping them to a JSON dictionary in the form _bulk_docs wants:
            var docsToSend = new List <object> ();
            var revsToSend = new RevisionList();
            IDictionary <string, object> revResults = null;

            foreach (var rev in changes)
            {
                // Is this revision in the server's 'missing' list?
                if (revsDiffResults != null)
                {
                    revResults = revsDiffResults.Get(rev.DocID).AsDictionary <string, object>();
                    if (revResults == null)
                    {
                        continue;
                    }

                    var revs = revResults.Get("missing").AsList <string>();
                    if (revs == null || !revs.Any(id => id.Equals(rev.RevID.ToString())))
                    {
                        RemovePending(rev);
                        continue;
                    }
                }

                IDictionary <string, object> properties = null;
                RevisionInternal             loadedRev;
                try {
                    loadedRev = LocalDatabase.LoadRevisionBody(rev);
                    if (loadedRev == null)
                    {
                        throw Misc.CreateExceptionAndLog(Log.To.Sync, StatusCode.NotFound, TAG,
                                                         "Unable to load revision body");
                    }

                    properties = rev.GetProperties();
                } catch (Exception e1) {
                    Log.To.Sync.E(TAG, String.Format("Couldn't get local contents of {0}, marking revision failed",
                                                     rev), e1);
                    RevisionFailed();
                    continue;
                }

                if (properties.GetCast <bool> ("_removed"))
                {
                    RemovePending(rev);
                    continue;
                }

                var populatedRev = TransformRevision(loadedRev);
                var backTo       = revResults?.Get("possible_ancestors")?.AsList <RevisionID>();

                try {
                    var history = LocalDatabase.GetRevisionHistory(populatedRev, backTo);
                    if (history == null)
                    {
                        throw Misc.CreateExceptionAndLog(Log.To.Sync, StatusCode.DbError, TAG,
                                                         "Unable to load revision history");
                    }

                    properties["_revisions"] = TreeRevisionID.MakeRevisionHistoryDict(history);
                    populatedRev.SetPropertyForKey("_revisions", properties["_revisions"]);
                } catch (Exception e1) {
                    Log.To.Sync.E(TAG, "Error getting revision history, marking revision failed", e1);
                    RevisionFailed();
                    continue;
                }

                // Strip any attachments already known to the target db:
                if (properties.Get("_attachments") != null)
                {
                    // Look for the latest common ancestor and stuf out older attachments:
                    var minRevPos = FindCommonAncestor(populatedRev, backTo);
                    try {
                        LocalDatabase.ExpandAttachments(populatedRev, minRevPos + 1, !_dontSendMultipart, false);
                    } catch (Exception ex) {
                        Log.To.Sync.E(TAG, "Error expanding attachments, marking revision failed", ex);
                        RevisionFailed();
                        continue;
                    }

                    properties = populatedRev.GetProperties();
                    if (!_dontSendMultipart && UploadMultipartRevision(populatedRev))
                    {
                        continue;
                    }
                }

                if (properties == null || !properties.ContainsKey("_id"))
                {
                    throw Misc.CreateExceptionAndLog(Log.To.Sync, StatusCode.BadParam, TAG,
                                                     "properties must contain a document _id");
                }

                // Add the _revisions list:
                revsToSend.Add(rev);

                //now add it to the docs to send
                docsToSend.Add(properties);
            }

            UploadBulkDocs(docsToSend, revsToSend);
        }
Example #11
0
        /// <summary>Process a bunch of remote revisions from the _changes feed at once</summary>
        internal override void ProcessInbox(RevisionList inbox)
        {
            Debug.Assert(inbox != null);

            if (!canBulkGet)
            {
                canBulkGet = CheckServerCompatVersion("0.81");
            }

            // Ask the local database which of the revs are not known to it:
            var lastInboxSequence = ((PulledRevision)inbox[inbox.Count - 1]).GetRemoteSequenceID();

            var numRevisionsRemoved = 0;

            try {
                // findMissingRevisions is the local equivalent of _revs_diff. it looks at the
                // array of revisions in inbox and removes the ones that already exist. So whatever's left in inbox
                // afterwards are the revisions that need to be downloaded.
                numRevisionsRemoved = LocalDatabase.FindMissingRevisions(inbox);
            } catch (Exception e) {
                Log.E(Tag, "Failed to look up local revs", e);
                inbox = null;
            }

            //introducing this to java version since inbox may now be null everywhere
            var inboxCount = 0;

            if (inbox != null)
            {
                inboxCount = inbox.Count;
            }

            if (numRevisionsRemoved > 0)
            {
                Log.V(Tag, "processInbox() setting changesCount to: " + (ChangesCount - numRevisionsRemoved));
                ChangesCount = ChangesCount - numRevisionsRemoved;
            }

            if (inboxCount == 0)
            {
                // Nothing to do. Just bump the lastSequence.
                Log.W(Tag, string.Format("{0} no new remote revisions to fetch", this));

                var seq = pendingSequences.AddValue(lastInboxSequence);
                pendingSequences.RemoveSequence(seq);
                LastSequence = pendingSequences.GetCheckpointedValue();
                return;
            }

            Log.V(Tag, "fetching " + inboxCount + " remote revisions...");

            // Dump the revs into the queue of revs to pull from the remote db:
            lock (locker) {
                int numBulked = 0;
                for (int i = 0; i < inbox.Count; i++)
                {
                    var rev = (PulledRevision)inbox [i];
                    //TODO: add support for rev isConflicted
                    if (canBulkGet || (rev.GetGeneration() == 1 && !rev.IsDeleted()))
                    {
                        // &&!rev.isConflicted)
                        //optimistically pull 1st-gen revs in bulk
                        if (bulkRevsToPull == null)
                        {
                            bulkRevsToPull = new List <RevisionInternal>(100);
                        }
                        bulkRevsToPull.AddItem(rev);
                        ++numBulked;
                    }
                    else
                    {
                        QueueRemoteRevision(rev);
                    }
                    rev.SetSequence(pendingSequences.AddValue(rev.GetRemoteSequenceID()));
                }
            }

            PullRemoteRevisions();
        }
Example #12
0
        protected internal override void ProcessInbox(RevisionList inbox)
        {
            if (canBulkGet == null)
            {
                canBulkGet = ServerIsSyncGatewayVersion("0.81");
            }
            // Ask the local database which of the revs are not known to it:
            string lastInboxSequence = ((PulledRevision)inbox[inbox.Count - 1]).GetRemoteSequenceID
                                           ();
            int numRevisionsRemoved = 0;

            try
            {
                // findMissingRevisions is the local equivalent of _revs_diff. it looks at the
                // array of revisions in ‘inbox’ and removes the ones that already exist. So whatever’s left in ‘inbox’
                // afterwards are the revisions that need to be downloaded.
                numRevisionsRemoved = db.FindMissingRevisions(inbox);
            }
            catch (SQLException e)
            {
                Log.E(Log.TagSync, string.Format("%s failed to look up local revs", this), e);
                inbox = null;
            }
            //introducing this to java version since inbox may now be null everywhere
            int inboxCount = 0;

            if (inbox != null)
            {
                inboxCount = inbox.Count;
            }
            if (numRevisionsRemoved > 0)
            {
                Log.V(Log.TagSync, "%s: processInbox() setting changesCount to: %s", this, GetChangesCount
                          () - numRevisionsRemoved);
                // May decrease the changesCount, to account for the revisions we just found out we don’t need to get.
                AddToChangesCount(-1 * numRevisionsRemoved);
            }
            if (inboxCount == 0)
            {
                // Nothing to do. Just bump the lastSequence.
                Log.W(Log.TagSync, "%s no new remote revisions to fetch", this);
                long seq = pendingSequences.AddValue(lastInboxSequence);
                pendingSequences.RemoveSequence(seq);
                SetLastSequence(pendingSequences.GetCheckpointedValue());
                return;
            }
            Log.V(Log.TagSync, "%s: fetching %s remote revisions...", this, inboxCount);
            // Dump the revs into the queue of revs to pull from the remote db:
            lock (this)
            {
                int numBulked = 0;
                for (int i = 0; i < inbox.Count; i++)
                {
                    PulledRevision rev = (PulledRevision)inbox[i];
                    //TODO: add support for rev isConflicted
                    if (canBulkGet || (rev.GetGeneration() == 1 && !rev.IsDeleted()))
                    {
                        // &&!rev.isConflicted)
                        //optimistically pull 1st-gen revs in bulk
                        if (bulkRevsToPull == null)
                        {
                            bulkRevsToPull = new AList <RevisionInternal>(100);
                        }
                        bulkRevsToPull.AddItem(rev);
                        ++numBulked;
                    }
                    else
                    {
                        QueueRemoteRevision(rev);
                    }
                    rev.SetSequence(pendingSequences.AddValue(rev.GetRemoteSequenceID()));
                }
            }
            PullRemoteRevisions();
        }
Example #13
0
        private void UploadBulkDocs(IList <object> docsToSend, RevisionList revChanges)
        {
            // Post the revisions to the destination. "new_edits":false means that the server should
            // use the given _rev IDs instead of making up new ones.
            var numDocsToSend = docsToSend.Count;

            if (numDocsToSend == 0)
            {
                return;
            }

            Log.V(Tag, string.Format("{0}: POSTing " + numDocsToSend + " revisions to _bulk_docs: {1}", this, docsToSend));

            SafeAddToChangesCount(numDocsToSend);

            var bulkDocsBody = new Dictionary <string, object>();

            bulkDocsBody["docs"]      = docsToSend;
            bulkDocsBody["new_edits"] = false;

            AsyncTaskStarted();

            SendAsyncRequest(HttpMethod.Post, "/_bulk_docs", bulkDocsBody, (result, e) => {
                try
                {
                    if (e == null)
                    {
                        var failedIds = new HashSet <string>();
                        // _bulk_docs response is really an array not a dictionary
                        var items = ((JArray)result).ToList();
                        foreach (var item in items)
                        {
                            var itemObject = item.AsDictionary <string, object>();
                            var status     = StatusFromBulkDocsResponseItem(itemObject);
                            if (!status.IsSuccessful)
                            {
                                // One of the docs failed to save.
                                Log.W(Tag, "_bulk_docs got an error: " + item);

                                // 403/Forbidden means validation failed; don't treat it as an error
                                // because I did my job in sending the revision. Other statuses are
                                // actual replication errors.
                                if (status.GetCode() != StatusCode.Forbidden)
                                {
                                    var docId = (string)item["id"];
                                    failedIds.Add(docId);
                                }
                            }
                        }

                        // Remove from the pending list all the revs that didn't fail:
                        foreach (var revisionInternal in revChanges)
                        {
                            if (!failedIds.Contains(revisionInternal.GetDocId()))
                            {
                                RemovePending(revisionInternal);
                            }
                        }
                    }

                    if (e != null)
                    {
                        SetLastError(e);
                        RevisionFailed();
                    }
                    else
                    {
                        Log.V(Tag, string.Format("POSTed to _bulk_docs: {0}", docsToSend));
                    }
                    SafeAddToCompletedChangesCount(numDocsToSend);
                }
                finally
                {
                    Log.D(Tag, "ProcessInbox() after _bulk_docs() calling AsyncTaskFinished()");
                    AsyncTaskFinished(1);
                }
            });
        }
Example #14
0
        internal override void ProcessInbox(RevisionList inbox)
        {
            // Generate a set of doc/rev IDs in the JSON format that _revs_diff wants:
            // <http://wiki.apache.org/couchdb/HttpPostRevsDiff>
            var diffs = new Dictionary <String, IList <String> >();

            foreach (var rev in inbox)
            {
                var docID = rev.GetDocId();
                var revs  = diffs.Get(docID);
                if (revs == null)
                {
                    revs         = new List <String>();
                    diffs[docID] = revs;
                }
                revs.AddItem(rev.GetRevId());
                AddPending(rev);
            }

            // Call _revs_diff on the target db:
            Log.D(Tag, "processInbox() calling asyncTaskStarted()");
            Log.D(Tag, "posting to /_revs_diff: {0}", String.Join(Environment.NewLine, Manager.GetObjectMapper().WriteValueAsString(diffs)));

            AsyncTaskStarted();
            SendAsyncRequest(HttpMethod.Post, "/_revs_diff", diffs, (response, e) =>
            {
                try {
                    var results = response.AsDictionary <string, object>();

                    Log.D(Tag, "/_revs_diff response: {0}\r\n{1}", response, results);

                    if (e != null)
                    {
                        SetLastError(e);
                        RevisionFailed();
                    }
                    else
                    {
                        if (results.Count != 0)
                        {
                            // Go through the list of local changes again, selecting the ones the destination server
                            // said were missing and mapping them to a JSON dictionary in the form _bulk_docs wants:
                            var docsToSend = new List <object> ();
                            var revsToSend = new RevisionList();
                            foreach (var rev in inbox)
                            {
                                // Is this revision in the server's 'missing' list?
                                IDictionary <string, object> properties = null;
                                var revResults = results.Get(rev.GetDocId()).AsDictionary <string, object>();

                                if (revResults == null)
                                {
                                    continue;
                                }

                                var revs = ((JArray)revResults.Get("missing")).Values <String>().ToList();
                                if (revs == null || !revs.Any(id => id.Equals(rev.GetRevId(), StringComparison.OrdinalIgnoreCase)))
                                {
                                    RemovePending(rev);
                                    continue;
                                }

                                // Get the revision's properties:
                                var contentOptions = DocumentContentOptions.IncludeAttachments;

                                if (!dontSendMultipart && revisionBodyTransformationFunction == null)
                                {
                                    contentOptions &= DocumentContentOptions.BigAttachmentsFollow;
                                }


                                RevisionInternal loadedRev;
                                try {
                                    loadedRev  = LocalDatabase.LoadRevisionBody(rev, contentOptions);
                                    properties = new Dictionary <string, object>(rev.GetProperties());
                                } catch (CouchbaseLiteException e1) {
                                    Log.W(Tag, string.Format("{0} Couldn't get local contents of {1}", rev, this), e1);
                                    RevisionFailed();
                                    continue;
                                }

                                var populatedRev = TransformRevision(loadedRev);

                                IList <string> possibleAncestors = null;
                                if (revResults.ContainsKey("possible_ancestors"))
                                {
                                    possibleAncestors = revResults["possible_ancestors"].AsList <string>();
                                }

                                properties               = new Dictionary <string, object>(populatedRev.GetProperties());
                                var revisions            = LocalDatabase.GetRevisionHistoryDictStartingFromAnyAncestor(populatedRev, possibleAncestors);
                                properties["_revisions"] = revisions;
                                populatedRev.SetProperties(properties);

                                // Strip any attachments already known to the target db:
                                if (properties.ContainsKey("_attachments"))
                                {
                                    // Look for the latest common ancestor and stuf out older attachments:
                                    var minRevPos = FindCommonAncestor(populatedRev, possibleAncestors);

                                    Database.StubOutAttachmentsInRevBeforeRevPos(populatedRev, minRevPos + 1, false);

                                    properties = populatedRev.GetProperties();

                                    if (!dontSendMultipart && UploadMultipartRevision(populatedRev))
                                    {
                                        continue;
                                    }
                                }

                                if (properties == null || !properties.ContainsKey("_id"))
                                {
                                    throw new InvalidOperationException("properties must contain a document _id");
                                }
                                // Add the _revisions list:
                                revsToSend.Add(rev);

                                //now add it to the docs to send
                                docsToSend.AddItem(properties);
                            }

                            UploadBulkDocs(docsToSend, revsToSend);
                        }
                        else
                        {
                            foreach (var revisionInternal in inbox)
                            {
                                RemovePending(revisionInternal);
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    Log.E(Tag, "Unhandled exception in Pusher.ProcessInbox", ex);
                }
                finally
                {
                    Log.D(Tag, "processInbox() calling AsyncTaskFinished()");
                    AsyncTaskFinished(1);
                }
            });
        }
        public void TestRevTree()
        {
            var rev = new RevisionInternal("MyDocId", "4-foxy", false, database);
            var revProperties = new Dictionary<string, object>();
            revProperties.Put("_id", rev.GetDocId());
            revProperties.Put("_rev", rev.GetRevId());
            revProperties["message"] = "hi";
            rev.SetProperties(revProperties);

            var revHistory = new AList<string>();
            revHistory.AddItem(rev.GetRevId());
            revHistory.AddItem("3-thrice");
            revHistory.AddItem("2-too");
            revHistory.AddItem("1-won");
            database.ForceInsert(rev, revHistory, null);
            Assert.AreEqual(1, database.DocumentCount);

            VerifyHistory(database, rev, revHistory);
            var conflict = new RevisionInternal("MyDocId", "5-epsilon", false, database);
            var conflictProperties = new Dictionary<string, object>();
            conflictProperties.Put("_id", conflict.GetDocId());
            conflictProperties.Put("_rev", conflict.GetRevId());
            conflictProperties["message"] = "yo";
            conflict.SetProperties(conflictProperties);
            
            var conflictHistory = new AList<string>();
            conflictHistory.AddItem(conflict.GetRevId());
            conflictHistory.AddItem("4-delta");
            conflictHistory.AddItem("3-gamma");
            conflictHistory.AddItem("2-too");
            conflictHistory.AddItem("1-won");
            database.ForceInsert(conflict, conflictHistory, null);
            Assert.AreEqual(1, database.DocumentCount);
            VerifyHistory(database, conflict, conflictHistory);
            
            // Add an unrelated document:
            var other = new RevisionInternal("AnotherDocID", "1-ichi", false, database);
            var otherProperties = new Dictionary<string, object>();
            otherProperties["language"] = "jp";
            other.SetProperties(otherProperties);
            var otherHistory = new AList<string>();
            otherHistory.AddItem(other.GetRevId());
            database.ForceInsert(other, otherHistory, null);
            
            // Fetch one of those phantom revisions with no body:
            var rev2 = database.GetDocumentWithIDAndRev(rev.GetDocId(), "2-too", 
                DocumentContentOptions.None);
            Assert.AreEqual(rev.GetDocId(), rev2.GetDocId());
            Assert.AreEqual("2-too", rev2.GetRevId());
            
            // Make sure no duplicate rows were inserted for the common revisions:
            Assert.AreEqual(8, database.GetLastSequenceNumber());
            // Make sure the revision with the higher revID wins the conflict:
            var current = database.GetDocumentWithIDAndRev(rev.GetDocId(), null, 
                DocumentContentOptions.None);
            Assert.AreEqual(conflict, current);
            
            // Get the _changes feed and verify only the winner is in it:
            var options = new ChangesOptions();
            var changes = database.ChangesSince(0, options, null);
            var expectedChanges = new RevisionList();
            expectedChanges.AddItem(conflict);
            expectedChanges.AddItem(other);
            Assert.AreEqual(changes, expectedChanges);
            options.SetIncludeConflicts(true);
            changes = database.ChangesSince(0, options, null);
            expectedChanges = new RevisionList();
            expectedChanges.AddItem(rev);
            expectedChanges.AddItem(conflict);
            expectedChanges.AddItem(other);
            Assert.AreEqual(changes, expectedChanges);
        }
Example #16
0
		public bool FindMissingRevisions(RevisionList touchRevs)
		{
			if (touchRevs.Count == 0)
			{
				return true;
			}
			string quotedDocIds = JoinQuoted(touchRevs.GetAllDocIds());
			string quotedRevIds = JoinQuoted(touchRevs.GetAllRevIds());
			string sql = "SELECT docid, revid FROM revs, docs " + "WHERE docid IN (" + quotedDocIds
				 + ") AND revid in (" + quotedRevIds + ")" + " AND revs.doc_id == docs.doc_id";
			Cursor cursor = null;
			try
			{
				cursor = database.RawQuery(sql, null);
				cursor.MoveToNext();
				while (!cursor.IsAfterLast())
				{
					RevisionInternal rev = touchRevs.RevWithDocIdAndRevId(cursor.GetString(0), cursor
						.GetString(1));
					if (rev != null)
					{
						touchRevs.Remove(rev);
					}
					cursor.MoveToNext();
				}
			}
			catch (SQLException e)
			{
				Log.E(Database.Tag, "Error finding missing revisions", e);
				return false;
			}
			finally
			{
				if (cursor != null)
				{
					cursor.Close();
				}
			}
			return true;
		}
Example #17
0
        internal override void ProcessInbox(RevisionList inbox)
        {
            if (Status == ReplicationStatus.Offline)
            {
                Log.To.Sync.I(TAG, "Offline, so skipping inbox process");
                return;
            }

            // Generate a set of doc/rev IDs in the JSON format that _revs_diff wants:
            // <http://wiki.apache.org/couchdb/HttpPostRevsDiff>
            var diffs      = new Dictionary <String, IList <String> >();
            var inboxCount = inbox.Count;

            foreach (var rev in inbox)
            {
                var docID = rev.DocID;
                var revs  = diffs.Get(docID);
                if (revs == null)
                {
                    revs         = new List <String>();
                    diffs[docID] = revs;
                }
                revs.Add(rev.RevID);
                AddPending(rev);
            }

            // Call _revs_diff on the target db:
            Log.To.Sync.D(TAG, "posting to /_revs_diff: {0}", String.Join(Environment.NewLine, new[] { Manager.GetObjectMapper().WriteValueAsString(diffs) }));
            SendAsyncRequest(HttpMethod.Post, "/_revs_diff", diffs, (response, e) =>
            {
                try {
                    if (!LocalDatabase.IsOpen)
                    {
                        return;
                    }

                    var results = response.AsDictionary <string, object>();

                    Log.To.Sync.D(TAG, "/_revs_diff response: {0}\r\n{1}", response, results);

                    if (e != null)
                    {
                        LastError = e;
                        for (int i = 0; i < inboxCount; i++)
                        {
                            RevisionFailed();
                        }

                        if (Continuous)
                        {
                            FireTrigger(ReplicationTrigger.WaitingForChanges);
                        }
                        else
                        {
                            FireTrigger(ReplicationTrigger.StopImmediate);
                        }
                    }
                    else
                    {
                        if (results.Count != 0)
                        {
                            // Go through the list of local changes again, selecting the ones the destination server
                            // said were missing and mapping them to a JSON dictionary in the form _bulk_docs wants:
                            var docsToSend = new List <object> ();
                            var revsToSend = new RevisionList();
                            foreach (var rev in inbox)
                            {
                                // Is this revision in the server's 'missing' list?
                                IDictionary <string, object> properties = null;
                                var revResults = results.Get(rev.DocID).AsDictionary <string, object>();
                                if (revResults == null)
                                {
                                    //SafeIncrementCompletedChangesCount();
                                    continue;
                                }

                                var revs = revResults.Get("missing").AsList <string>();
                                if (revs == null || !revs.Any(id => id.Equals(rev.RevID, StringComparison.OrdinalIgnoreCase)))
                                {
                                    RemovePending(rev);
                                    //SafeIncrementCompletedChangesCount();
                                    continue;
                                }

                                // Get the revision's properties:
                                var contentOptions = DocumentContentOptions.IncludeAttachments;
                                if (!_dontSendMultipart && RevisionBodyTransformationFunction == null)
                                {
                                    contentOptions |= DocumentContentOptions.BigAttachmentsFollow;
                                }

                                RevisionInternal loadedRev;
                                try {
                                    loadedRev = LocalDatabase.LoadRevisionBody(rev);
                                    if (loadedRev == null)
                                    {
                                        throw Misc.CreateExceptionAndLog(Log.To.Sync, StatusCode.DbError, TAG,
                                                                         "Unable to load revision body");
                                    }

                                    properties = new Dictionary <string, object>(rev.GetProperties());
                                } catch (Exception e1) {
                                    Log.To.Sync.E(TAG, String.Format("Couldn't get local contents of {0}, marking revision failed",
                                                                     rev), e1);
                                    RevisionFailed();
                                    continue;
                                }

                                var populatedRev = TransformRevision(loadedRev);
                                IList <string> possibleAncestors = null;
                                if (revResults.ContainsKey("possible_ancestors"))
                                {
                                    possibleAncestors = revResults["possible_ancestors"].AsList <string>();
                                }

                                properties = new Dictionary <string, object>(populatedRev.GetProperties());

                                try {
                                    var history = LocalDatabase.GetRevisionHistory(populatedRev, possibleAncestors);
                                    if (history == null)
                                    {
                                        throw Misc.CreateExceptionAndLog(Log.To.Sync, StatusCode.DbError, TAG,
                                                                         "Unable to load revision history");
                                    }

                                    properties["_revisions"] = Database.MakeRevisionHistoryDict(history);
                                } catch (Exception e1) {
                                    Log.To.Sync.E(TAG, "Error getting revision history, marking revision failed", e1);
                                    RevisionFailed();
                                    continue;
                                }

                                populatedRev.SetProperties(properties);
                                if (properties.GetCast <bool>("_removed"))
                                {
                                    RemovePending(rev);
                                    continue;
                                }

                                // Strip any attachments already known to the target db:
                                if (properties.ContainsKey("_attachments"))
                                {
                                    // Look for the latest common ancestor and stuf out older attachments:
                                    var minRevPos = FindCommonAncestor(populatedRev, possibleAncestors);
                                    try {
                                        LocalDatabase.ExpandAttachments(populatedRev, minRevPos + 1, !_dontSendMultipart, false);
                                    } catch (Exception ex) {
                                        Log.To.Sync.E(TAG, "Error expanding attachments, marking revision failed", ex);
                                        RevisionFailed();
                                        continue;
                                    }

                                    properties = populatedRev.GetProperties();
                                    if (!_dontSendMultipart && UploadMultipartRevision(populatedRev))
                                    {
                                        continue;
                                    }
                                }

                                if (properties == null || !properties.ContainsKey("_id"))
                                {
                                    throw Misc.CreateExceptionAndLog(Log.To.Sync, StatusCode.BadParam, TAG,
                                                                     "properties must contain a document _id");
                                }

                                // Add the _revisions list:
                                revsToSend.Add(rev);

                                //now add it to the docs to send
                                docsToSend.Add(properties);
                            }

                            UploadBulkDocs(docsToSend, revsToSend);
                        }
                        else
                        {
                            foreach (var revisionInternal in inbox)
                            {
                                RemovePending(revisionInternal);
                            }

                            //SafeAddToCompletedChangesCount(inbox.Count);
                        }
                    }
                } catch (Exception ex) {
                    Log.To.Sync.E(TAG, "Unhandled exception in Pusher.ProcessInbox, continuing...", ex);
                }
            });
        }
        public void TestRevTree()
        {
            var change = default(DocumentChange);
            database.Changed += (sender, args) =>
            {
                Assert.AreEqual(1, args.Changes.Count());
                Assert.IsNull(change, "Multiple notifications posted");
                change = args.Changes.First();
            };

            var rev = new RevisionInternal("MyDocId", "4-4444".AsRevID(), false);
            var revProperties = new Dictionary<string, object>();
            revProperties.SetDocRevID(rev.DocID, rev.RevID);
            revProperties["message"] = "hi";
            rev.SetProperties(revProperties);

            var revHistory = new List<RevisionID>();
            revHistory.Add(rev.RevID);
            revHistory.Add("3-3333".AsRevID());
            revHistory.Add("2-2222".AsRevID());
            revHistory.Add("1-1111".AsRevID());
            database.ForceInsert(rev, revHistory, null);
            Assert.AreEqual(1, database.GetDocumentCount());
            VerifyRev(rev, revHistory);
            Assert.AreEqual(Announcement(database, rev, rev), change);
            Assert.IsFalse(change.IsConflict);

            // No-op ForceInsert of already-existing revision
            var lastSeq = database.GetLastSequenceNumber();
            database.ForceInsert(rev, revHistory, null);
            Assert.AreEqual(lastSeq, database.GetLastSequenceNumber());
            
            var conflict = new RevisionInternal("MyDocId", "5-5555".AsRevID(), false);
            var conflictProperties = new Dictionary<string, object>();
            conflictProperties.SetDocRevID(conflict.DocID, conflict.RevID);
            conflictProperties["message"] = "yo";
            conflict.SetProperties(conflictProperties);
            
            var conflictHistory = new List<RevisionID>();
            conflictHistory.Add(conflict.RevID);
            conflictHistory.Add("4-4545".AsRevID());
            conflictHistory.Add("3-3030".AsRevID());
            conflictHistory.Add("2-2222".AsRevID());
            conflictHistory.Add("1-1111".AsRevID());
            change = null;
            database.ForceInsert(conflict, conflictHistory, null);
            Assert.AreEqual(1, database.GetDocumentCount());
            VerifyRev(conflict, conflictHistory);
            Assert.AreEqual(Announcement(database, conflict, conflict), change);
            Assert.IsTrue(change.IsConflict);

            // Add an unrelated document:
            var other = new RevisionInternal("AnotherDocID", "1-1010".AsRevID(), false);
            var otherProperties = new Dictionary<string, object>();
            otherProperties["language"] = "jp";
            other.SetProperties(otherProperties);
            var otherHistory = new List<RevisionID>();
            otherHistory.Add(other.RevID);
            change = null;
            database.ForceInsert(other, otherHistory, null);
            Assert.AreEqual(Announcement(database, other, other), change);
            Assert.IsFalse(change.IsConflict);

            // Fetch one of those phantom revisions with no body:
            var rev2 = database.GetDocument(rev.DocID, "2-2222".AsRevID(), 
                true);
            Assert.IsTrue(rev2.Missing);
            Assert.IsNull(rev2.GetBody());

            Assert.IsNull(database.GetDocument(rev.DocID, "666-6666".AsRevID(), true));

            // Make sure no duplicate rows were inserted for the common revisions:
            if(_storageType == StorageEngineTypes.SQLite) {
                Assert.AreEqual(8, database.GetLastSequenceNumber());
            } else {
                Assert.AreEqual(3, database.GetLastSequenceNumber());
            }
            // Make sure the revision with the higher revID wins the conflict:
            var current = database.GetDocument(rev.DocID, null, 
                true);
            Assert.AreEqual(conflict, current);

            // Check that the list of conflicts is accurate
            var conflictingRevs = database.Storage.GetAllDocumentRevisions(rev.DocID, true, false);
            CollectionAssert.AreEqual(new[] { conflict, rev }, conflictingRevs);
            
            // Get the _changes feed and verify only the winner is in it:
            var options = new ChangesOptions();
            var changes = database.ChangesSince(0, options, null, null);
            CollectionAssert.AreEqual(new[] { conflict, other }, changes);
            options.IncludeConflicts = true;
            changes = database.ChangesSince(0, options, null, null);
            var expectedChanges = new RevisionList();
            expectedChanges.Add(rev);
            expectedChanges.Add(conflict);
            expectedChanges.Add(other);
            var expectedChangesAlt = new RevisionList();
            expectedChangesAlt.Add(conflict);
            expectedChangesAlt.Add(rev);
            expectedChangesAlt.Add(other);
            Assert.IsTrue(expectedChanges.SequenceEqual(changes) || expectedChangesAlt.SequenceEqual(changes));
        }
 internal static void SortByDocID(this RevisionList list)
 {
     list.Sort((r1, r2) => r1.DocID.CompareTo(r2.DocID));
 }
        public void TestRevTree()
        {
            var rev = new RevisionInternal("MyDocId", "4-abcd", false);
            var revProperties = new Dictionary<string, object>();
            revProperties.Put("_id", rev.GetDocId());
            revProperties.Put("_rev", rev.GetRevId());
            revProperties["message"] = "hi";
            rev.SetProperties(revProperties);

            var revHistory = new List<string>();
            revHistory.AddItem(rev.GetRevId());
            revHistory.AddItem("3-abcd");
            revHistory.AddItem("2-abcd");
            revHistory.AddItem("1-abcd");
            database.ForceInsert(rev, revHistory, null);
            Assert.AreEqual(1, database.DocumentCount);

            VerifyHistory(database, rev, revHistory);
            var conflict = new RevisionInternal("MyDocId", "5-abcd", false);
            var conflictProperties = new Dictionary<string, object>();
            conflictProperties.Put("_id", conflict.GetDocId());
            conflictProperties.Put("_rev", conflict.GetRevId());
            conflictProperties["message"] = "yo";
            conflict.SetProperties(conflictProperties);
            
            var conflictHistory = new List<string>();
            conflictHistory.AddItem(conflict.GetRevId());
            conflictHistory.AddItem("4-bcde");
            conflictHistory.AddItem("3-bcde");
            conflictHistory.AddItem("2-abcd");
            conflictHistory.AddItem("1-abcd");
            database.ForceInsert(conflict, conflictHistory, null);
            Assert.AreEqual(1, database.DocumentCount);
            VerifyHistory(database, conflict, conflictHistory);
            
            // Add an unrelated document:
            var other = new RevisionInternal("AnotherDocID", "1-cdef", false);
            var otherProperties = new Dictionary<string, object>();
            otherProperties["language"] = "jp";
            other.SetProperties(otherProperties);
            var otherHistory = new List<string>();
            otherHistory.AddItem(other.GetRevId());
            database.ForceInsert(other, otherHistory, null);
            
            // Fetch one of those phantom revisions with no body:
            var rev2 = database.GetDocument(rev.GetDocId(), "2-abcd", 
                true);
            Assert.IsNull(rev2);

            // Make sure no duplicate rows were inserted for the common revisions:
            Assert.IsTrue(database.LastSequenceNumber <= 8);
            // Make sure the revision with the higher revID wins the conflict:
            var current = database.GetDocument(rev.GetDocId(), null, 
                true);
            Assert.AreEqual(conflict, current);
            
            // Get the _changes feed and verify only the winner is in it:
            var options = new ChangesOptions();
            var changes = database.ChangesSince(0, options, null, null);
            var expectedChanges = new RevisionList();
            expectedChanges.AddItem(conflict);
            expectedChanges.AddItem(other);
            Assert.AreEqual(expectedChanges, changes);
            options.IncludeConflicts = true;
            changes = database.ChangesSince(0, options, null, null);
            expectedChanges = new RevisionList();
            expectedChanges.AddItem(rev);
            expectedChanges.AddItem(conflict);
            expectedChanges.AddItem(other);
            var expectedChangesAlt = new RevisionList();
            expectedChangesAlt.AddItem(conflict);
            expectedChangesAlt.AddItem(rev);
            expectedChangesAlt.AddItem(other);
            Assert.IsTrue(expectedChanges.SequenceEqual(changes) || expectedChangesAlt.SequenceEqual(changes));
        }
Example #21
0
        internal override void ProcessInbox(RevisionList inbox)
        {
            if (Status == ReplicationStatus.Offline)
            {
                Log.To.Sync.I(TAG, "{0} is offline, so skipping inbox process", this);
                return;
            }

            Debug.Assert(inbox != null);
            if (!_canBulkGet)
            {
                _canBulkGet = CheckServerCompatVersion("0.81");
            }

            Log.To.SyncPerf.I(TAG, "{0} processing {1} changes", this, inbox.Count);
            Log.To.Sync.V(TAG, "{0} looking up {1}", this, inbox);

            // Ask the local database which of the revs are not known to it:
            var lastInboxSequence   = ((PulledRevision)inbox[inbox.Count - 1]).GetRemoteSequenceID();
            var numRevisionsRemoved = 0;

            try {
                // findMissingRevisions is the local equivalent of _revs_diff. it looks at the
                // array of revisions in inbox and removes the ones that already exist. So whatever's left in inbox
                // afterwards are the revisions that need to be downloaded.
                numRevisionsRemoved = LocalDatabase.Storage.FindMissingRevisions(inbox);
            } catch (Exception e) {
                Log.To.Sync.E(TAG, String.Format("{0} failed to look up local revs, aborting...", this), e);
                inbox = null;
            }

            var inboxCount = 0;

            if (inbox != null)
            {
                inboxCount = inbox.Count;
            }

            if (numRevisionsRemoved > 0)
            {
                // Some of the revisions originally in the inbox aren't missing; treat those as processed:
                SafeAddToCompletedChangesCount(numRevisionsRemoved);
            }

            if (inboxCount == 0)
            {
                // Nothing to do; just count all the revisions as processed.
                // Instead of adding and immediately removing the revs to _pendingSequences,
                // just do the latest one (equivalent but faster):
                Log.To.Sync.V(TAG, "{0} no new remote revisions to fetch", this);

                var seq = _pendingSequences.AddValue(lastInboxSequence);
                _pendingSequences.RemoveSequence(seq);
                LastSequence = _pendingSequences.GetCheckpointedValue();
                PauseOrResume();
                return;
            }

            Log.To.SyncPerf.I(TAG, "{0} queuing download requests for {1} revisions", this, inboxCount);
            Log.To.Sync.V(TAG, "{0} queuing remote revisions {1}", this, inbox);

            // Dump the revs into the queue of revs to pull from the remote db:
            lock (_locker) {
                int numBulked = 0;
                for (int i = 0; i < inboxCount; i++)
                {
                    var rev = (PulledRevision)inbox[i];
                    if (_canBulkGet || (rev.Generation == 1 && !rev.Deleted && !rev.IsConflicted))
                    {
                        //optimistically pull 1st-gen revs in bulk
                        if (_bulkRevsToPull == null)
                        {
                            _bulkRevsToPull = new List <RevisionInternal>(100);
                        }

                        _bulkRevsToPull.Add(rev);
                        ++numBulked;
                    }
                    else
                    {
                        QueueRemoteRevision(rev);
                    }
                    rev.Sequence = _pendingSequences.AddValue(rev.GetRemoteSequenceID());
                }

                Log.To.Sync.I(TAG, "{4} queued {0} remote revisions from seq={1} ({2} in bulk, {3} individually)", inboxCount,
                              ((PulledRevision)inbox[0]).GetRemoteSequenceID(), numBulked, inboxCount - numBulked, this);
            }

            PullRemoteRevisions();
            PauseOrResume();
        }
        /// <summary>
        /// Create a response body for an HTTP response from a given list of DB changes (no conflicts)
        /// </summary>
        /// <returns>The response body</returns>
        /// <param name="changes">The list of changes to be processed</param>
        /// <param name="since">The first change ID to be processed</param>
        /// <param name="responseState">The current response state</param>
        public static IDictionary<string, object> ResponseBodyForChanges(RevisionList changes, long since, DBMonitorCouchbaseResponseState responseState)
        {
            List<IDictionary<string, object>> results = new List<IDictionary<string, object>>();
            foreach (var change in changes) {
                results.Add(DatabaseMethods.ChangesDictForRev(change, responseState));
            }

            if (changes.Count > 0) {
                since = changes.Last().Sequence;
            }

            return new Dictionary<string, object> {
                { "results", results },
                { "last_seq", since }
            };
        }
        // Apply the options in the URL query to the specified revision and create a new revision object
        internal static RevisionInternal ApplyOptions(DocumentContentOptions options, RevisionInternal rev, ICouchbaseListenerContext context,
                                                      Database db, Status outStatus)
        {
            if ((options & (DocumentContentOptions.IncludeRevs | DocumentContentOptions.IncludeRevsInfo | DocumentContentOptions.IncludeConflicts |
                            DocumentContentOptions.IncludeAttachments | DocumentContentOptions.IncludeLocalSeq)) != 0)
            {
                var dst = rev.GetProperties();
                if (options.HasFlag(DocumentContentOptions.IncludeLocalSeq))
                {
                    dst["_local_seq"] = rev.GetSequence();
                }

                if (options.HasFlag(DocumentContentOptions.IncludeRevs))
                {
                    var revs = db.GetRevisionHistory(rev, null);
                    dst["_revisions"] = Database.MakeRevisionHistoryDict(revs);
                }

                if (options.HasFlag(DocumentContentOptions.IncludeRevsInfo))
                {
                    dst["_revs_info"] = db.Storage.GetRevisionHistory(rev, null).Select(x =>
                    {
                        string status = "available";
                        if (x.IsDeleted())
                        {
                            status = "deleted";
                        }
                        else if (x.IsMissing())
                        {
                            status = "missing";
                        }

                        return(new Dictionary <string, object> {
                            { "rev", x.GetRevId() },
                            { "status", status }
                        });
                    });
                }

                if (options.HasFlag(DocumentContentOptions.IncludeConflicts))
                {
                    RevisionList revs = db.Storage.GetAllDocumentRevisions(rev.GetDocId(), true);
                    if (revs.Count > 1)
                    {
                        dst["_conflicts"] = revs.Select(x =>
                        {
                            return(x.Equals(rev) || x.IsDeleted() ? null : x.GetRevId());
                        });
                    }
                }

                RevisionInternal nuRev = new RevisionInternal(dst);
                if (options.HasFlag(DocumentContentOptions.IncludeAttachments))
                {
                    bool attEncodingInfo = context != null && context.GetQueryParam <bool>("att_encoding_info", bool.TryParse, false);
                    db.ExpandAttachments(nuRev, 0, false, !attEncodingInfo);
                }

                rev = nuRev;
            }

            return(rev);
        }
Example #24
0
 public void OnCompletion(object response, Exception e)
 {
     try
     {
         Log.V(Log.TagSync, "%s: got /_revs_diff response");
         IDictionary <string, object> results = (IDictionary <string, object>)response;
         if (e != null)
         {
             this._enclosing.SetError(e);
             this._enclosing.RevisionFailed();
         }
         else
         {
             if (results.Count != 0)
             {
                 // Go through the list of local changes again, selecting the ones the destination server
                 // said were missing and mapping them to a JSON dictionary in the form _bulk_docs wants:
                 IList <object> docsToSend = new AList <object>();
                 RevisionList   revsToSend = new RevisionList();
                 foreach (RevisionInternal rev in changes)
                 {
                     // Is this revision in the server's 'missing' list?
                     IDictionary <string, object> properties = null;
                     IDictionary <string, object> revResults = (IDictionary <string, object>)results.Get
                                                                   (rev.GetDocId());
                     if (revResults == null)
                     {
                         continue;
                     }
                     IList <string> revs = (IList <string>)revResults.Get("missing");
                     if (revs == null || !revs.Contains(rev.GetRevId()))
                     {
                         this._enclosing.RemovePending(rev);
                         continue;
                     }
                     // Get the revision's properties:
                     EnumSet <Database.TDContentOptions> contentOptions = EnumSet.Of(Database.TDContentOptions
                                                                                     .TDIncludeAttachments);
                     if (!this._enclosing.dontSendMultipart && this._enclosing.revisionBodyTransformationBlock
                         == null)
                     {
                         contentOptions.AddItem(Database.TDContentOptions.TDBigAttachmentsFollow);
                     }
                     RevisionInternal loadedRev;
                     try
                     {
                         loadedRev  = this._enclosing.db.LoadRevisionBody(rev, contentOptions);
                         properties = new Dictionary <string, object>(rev.GetProperties());
                     }
                     catch (CouchbaseLiteException)
                     {
                         Log.W(Log.TagSync, "%s Couldn't get local contents of %s", rev, this._enclosing);
                         this._enclosing.RevisionFailed();
                         continue;
                     }
                     RevisionInternal populatedRev      = this._enclosing.TransformRevision(loadedRev);
                     IList <string>   possibleAncestors = (IList <string>)revResults.Get("possible_ancestors"
                                                                                         );
                     properties = new Dictionary <string, object>(populatedRev.GetProperties());
                     IDictionary <string, object> revisions = this._enclosing.db.GetRevisionHistoryDictStartingFromAnyAncestor
                                                                  (populatedRev, possibleAncestors);
                     properties.Put("_revisions", revisions);
                     populatedRev.SetProperties(properties);
                     // Strip any attachments already known to the target db:
                     if (properties.ContainsKey("_attachments"))
                     {
                         // Look for the latest common ancestor and stub out older attachments:
                         int minRevPos = Couchbase.Lite.Replicator.Pusher.FindCommonAncestor(populatedRev,
                                                                                             possibleAncestors);
                         Database.StubOutAttachmentsInRevBeforeRevPos(populatedRev, minRevPos + 1, false);
                         properties = populatedRev.GetProperties();
                         if (!this._enclosing.dontSendMultipart && this._enclosing.UploadMultipartRevision
                                 (populatedRev))
                         {
                             continue;
                         }
                     }
                     if (properties == null || !properties.ContainsKey("_id"))
                     {
                         throw new InvalidOperationException("properties must contain a document _id");
                     }
                     revsToSend.AddItem(rev);
                     docsToSend.AddItem(properties);
                 }
                 //TODO: port this code from iOS
                 // Post the revisions to the destination:
                 this._enclosing.UploadBulkDocs(docsToSend, revsToSend);
             }
             else
             {
                 // None of the revisions are new to the remote
                 foreach (RevisionInternal revisionInternal in changes)
                 {
                     this._enclosing.RemovePending(revisionInternal);
                 }
             }
         }
     }
     finally
     {
         Log.V(Log.TagSync, "%s | %s: processInbox.sendAsyncRequest() calling asyncTaskFinished()"
               , this, Sharpen.Thread.CurrentThread());
         this._enclosing.AsyncTaskFinished(1);
     }
 }
        /// <summary>
        /// Returns document by the specified docid from the specified db. Unless you request a
        /// specific revision, the latest revision of the document will always be returned.
        /// </summary>
        /// <returns>The response state for further HTTP processing</returns>
        /// <param name="context">The context of the Couchbase Lite HTTP request</param>
        /// <remarks>
        /// http://docs.couchdb.org/en/latest/api/document/common.html#get--db-docid
        /// <remarks>
        public static ICouchbaseResponseState GetDocument(ICouchbaseListenerContext context)
        {
            return(DatabaseMethods.PerformLogicWithDatabase(context, true, db => {
                var response = context.CreateResponse();
                string docId = context.DocumentName;
                bool isLocalDoc = docId.StartsWith("_local");

                DocumentContentOptions options = context.ContentOptions;
                string openRevsParam = context.GetQueryParam("open_revs");
                bool mustSendJson = context.ExplicitlyAcceptsType("application/json");
                if (openRevsParam == null || isLocalDoc)
                {
                    //Regular GET:
                    string revId = context.GetQueryParam("rev"); //often null
                    RevisionInternal rev;
                    bool includeAttachments = false, sendMultipart = false;
                    if (isLocalDoc)
                    {
                        rev = db.Storage.GetLocalDocument(docId, revId);
                    }
                    else
                    {
                        includeAttachments = options.HasFlag(DocumentContentOptions.IncludeAttachments);
                        if (includeAttachments)
                        {
                            sendMultipart = !mustSendJson;
                            options &= ~DocumentContentOptions.IncludeAttachments;
                        }

                        Status status = new Status();
                        rev = db.GetDocument(docId, revId, true, status);
                        if (rev != null)
                        {
                            rev = ApplyOptions(options, rev, context, db, status);
                        }

                        if (rev == null)
                        {
                            if (status.Code == StatusCode.Deleted)
                            {
                                response.StatusReason = "deleted";
                            }
                            else
                            {
                                response.StatusReason = "missing";
                            }

                            response.InternalStatus = status.Code;
                            return response;
                        }
                    }

                    if (rev == null)
                    {
                        response.InternalStatus = StatusCode.NotFound;
                        return response;
                    }

                    if (context.CacheWithEtag(rev.GetRevId()))
                    {
                        response.InternalStatus = StatusCode.NotModified;
                        return response;
                    }

                    if (!isLocalDoc && includeAttachments)
                    {
                        int minRevPos = 1;
                        IList <string> attsSince = context.GetJsonQueryParam("atts_since").AsList <string>();
                        string ancestorId = db.Storage.FindCommonAncestor(rev, attsSince);
                        if (ancestorId != null)
                        {
                            minRevPos = RevisionInternal.GenerationFromRevID(ancestorId) + 1;
                        }

                        bool attEncodingInfo = context.GetQueryParam <bool>("att_encoding_info", bool.TryParse, false);
                        db.ExpandAttachments(rev, minRevPos, sendMultipart, attEncodingInfo);
                    }

                    if (sendMultipart)
                    {
                        response.MultipartWriter = db.MultipartWriterForRev(rev, "multipart/related");
                    }
                    else
                    {
                        response.JsonBody = rev.GetBody();
                    }
                }
                else
                {
                    // open_revs query:
                    IList <IDictionary <string, object> > result;
                    if (openRevsParam.Equals("all"))
                    {
                        // ?open_revs=all returns all current/leaf revisions:
                        bool includeDeleted = context.GetQueryParam <bool>("include_deleted", bool.TryParse, false);
                        RevisionList allRevs = db.Storage.GetAllDocumentRevisions(docId, true);

                        result = new List <IDictionary <string, object> >();
                        foreach (var rev in allRevs)
                        {
                            if (!includeDeleted && rev.IsDeleted())
                            {
                                continue;
                            }

                            Status status = new Status();
                            RevisionInternal loadedRev = db.RevisionByLoadingBody(rev, status);
                            if (loadedRev != null)
                            {
                                ApplyOptions(options, loadedRev, context, db, status);
                            }

                            if (loadedRev != null)
                            {
                                result.Add(new Dictionary <string, object> {
                                    { "ok", loadedRev.GetProperties() }
                                });
                            }
                            else if (status.Code <= StatusCode.InternalServerError)
                            {
                                result.Add(new Dictionary <string, object> {
                                    { "missing", rev.GetRevId() }
                                });
                            }
                            else
                            {
                                response.InternalStatus = status.Code;
                                return response;
                            }
                        }
                    }
                    else
                    {
                        // ?open_revs=[...] returns an array of specific revisions of the document:
                        var openRevs = context.GetJsonQueryParam("open_revs").AsList <object>();
                        if (openRevs == null)
                        {
                            response.InternalStatus = StatusCode.BadParam;
                            return response;
                        }

                        result = new List <IDictionary <string, object> >();
                        foreach (var revIDObj in openRevs)
                        {
                            var revID = revIDObj as string;
                            if (revID == null)
                            {
                                response.InternalStatus = StatusCode.BadId;
                                return response;
                            }

                            Status status = new Status();
                            var rev = db.GetDocument(docId, revID, true);
                            if (rev != null)
                            {
                                rev = ApplyOptions(options, rev, context, db, status);
                            }

                            if (rev != null)
                            {
                                result.Add(new Dictionary <string, object> {
                                    { "ok", rev.GetProperties() }
                                });
                            }
                            else
                            {
                                result.Add(new Dictionary <string, object> {
                                    { "missing", revID }
                                });
                            }
                        }
                    }

                    if (mustSendJson)
                    {
                        response["Content-Type"] = "application/json";
                        response.JsonBody = new Body(result.Cast <object>().ToList());
                    }
                    else
                    {
                        response.SetMultipartBody(result.Cast <object>().ToList(), "multipart/mixed");
                    }
                }

                return response;
            }).AsDefaultState());
        }
        public RevisionList GetAllDocumentRevisions(string docId, bool onlyCurrent, bool includeDeleted)
        {
            var retVal = default(RevisionList);
            WithC4Document(docId, null, false, false, doc =>
            {
                using(var enumerator = new CBForestHistoryEnumerator(doc, onlyCurrent, false)) {
                    var expression = includeDeleted ?
                        enumerator.Select (x => new ForestRevisionInternal (x.GetDocument (), false)) :
                        enumerator.Where (x => !x.SelectedRev.IsDeleted).Select (x => new ForestRevisionInternal (x.GetDocument (), false));
                    retVal = new RevisionList(expression.Cast<RevisionInternal>().ToList());
                }
            });

            return retVal;
        }
        internal override void ProcessInbox(RevisionList inbox)
        {
            if (Status == ReplicationStatus.Offline)
            {
                Log.V(TAG, "Offline, so skipping inbox process");
                return;
            }

            if (_requests.Count > ManagerOptions.Default.MaxOpenHttpConnections)
            {
                Task.Delay(1000).ContinueWith(t => ProcessInbox(inbox), CancellationToken.None, TaskContinuationOptions.None, WorkExecutor.Scheduler);
                return;
            }

            // Generate a set of doc/rev IDs in the JSON format that _revs_diff wants:
            // <http://wiki.apache.org/couchdb/HttpPostRevsDiff>
            var diffs = new Dictionary <String, IList <String> >();

            foreach (var rev in inbox)
            {
                var docID = rev.GetDocId();
                var revs  = diffs.Get(docID);
                if (revs == null)
                {
                    revs         = new List <String>();
                    diffs[docID] = revs;
                }
                revs.AddItem(rev.GetRevId());
                AddPending(rev);
            }

            // Call _revs_diff on the target db:
            Log.D(TAG, "posting to /_revs_diff: {0}", String.Join(Environment.NewLine, new[] { Manager.GetObjectMapper().WriteValueAsString(diffs) }));

            SendAsyncRequest(HttpMethod.Post, "/_revs_diff", diffs, (response, e) =>
            {
                try {
                    var localDb = LocalDatabase;
                    if (localDb == null)
                    {
                        return;
                    }

                    var results = response.AsDictionary <string, object>();

                    Log.D(TAG, "/_revs_diff response: {0}\r\n{1}", response, results);

                    if (e != null)
                    {
                        LastError = e;
                        RevisionFailed();
                    }
                    else
                    {
                        if (results.Count != 0)
                        {
                            // Go through the list of local changes again, selecting the ones the destination server
                            // said were missing and mapping them to a JSON dictionary in the form _bulk_docs wants:
                            var docsToSend = new List <object> ();
                            var revsToSend = new RevisionList();
                            foreach (var rev in inbox)
                            {
                                // Is this revision in the server's 'missing' list?
                                IDictionary <string, object> properties = null;
                                var revResults = results.Get(rev.GetDocId()).AsDictionary <string, object>();
                                if (revResults == null)
                                {
                                    SafeIncrementCompletedChangesCount();
                                    continue;
                                }

                                var revs = revResults.Get("missing").AsList <string>();
                                if (revs == null || !revs.Any(id => id.Equals(rev.GetRevId(), StringComparison.OrdinalIgnoreCase)))
                                {
                                    RemovePending(rev);
                                    SafeIncrementCompletedChangesCount();
                                    continue;
                                }

                                // Get the revision's properties:
                                var contentOptions = DocumentContentOptions.IncludeAttachments;
                                if (!_dontSendMultipart && RevisionBodyTransformationFunction == null)
                                {
                                    contentOptions |= DocumentContentOptions.BigAttachmentsFollow;
                                }


                                RevisionInternal loadedRev;
                                try {
                                    loadedRev  = localDb.LoadRevisionBody(rev);
                                    properties = new Dictionary <string, object>(rev.GetProperties());
                                } catch (CouchbaseLiteException e1) {
                                    Log.W(TAG, string.Format("{0} Couldn't get local contents of {1}", rev, this), e1);
                                    RevisionFailed();
                                    continue;
                                }

                                var populatedRev = TransformRevision(loadedRev);
                                IList <string> possibleAncestors = null;
                                if (revResults.ContainsKey("possible_ancestors"))
                                {
                                    possibleAncestors = revResults["possible_ancestors"].AsList <string>();
                                }

                                properties  = new Dictionary <string, object>(populatedRev.GetProperties());
                                var history = localDb.GetRevisionHistory(populatedRev, possibleAncestors);
                                properties["_revisions"] = Database.MakeRevisionHistoryDict(history);
                                populatedRev.SetProperties(properties);
                                if (properties.GetCast <bool>("_removed"))
                                {
                                    RemovePending(rev);
                                    continue;
                                }

                                // Strip any attachments already known to the target db:
                                if (properties.ContainsKey("_attachments"))
                                {
                                    // Look for the latest common ancestor and stuf out older attachments:
                                    var minRevPos = FindCommonAncestor(populatedRev, possibleAncestors);
                                    try {
                                        localDb.ExpandAttachments(populatedRev, minRevPos + 1, !_dontSendMultipart, false);
                                    } catch (Exception ex) {
                                        Log.W(TAG, "Error expanding attachments!", ex);
                                        RevisionFailed();
                                        continue;
                                    }

                                    properties = populatedRev.GetProperties();
                                    if (!_dontSendMultipart && UploadMultipartRevision(populatedRev))
                                    {
                                        continue;
                                    }
                                }

                                if (properties == null || !properties.ContainsKey("_id"))
                                {
                                    throw new InvalidOperationException("properties must contain a document _id");
                                }

                                // Add the _revisions list:
                                revsToSend.Add(rev);

                                //now add it to the docs to send
                                docsToSend.AddItem(properties);
                            }

                            UploadBulkDocs(docsToSend, revsToSend);
                        }
                        else
                        {
                            foreach (var revisionInternal in inbox)
                            {
                                RemovePending(revisionInternal);
                            }

                            SafeAddToCompletedChangesCount(inbox.Count);
                        }
                    }
                } catch (Exception ex) {
                    Log.E(TAG, "Unhandled exception in Pusher.ProcessInbox", ex);
                }
            });
        }
        internal override void ProcessInbox(RevisionList inbox)
        {
            if (Status == ReplicationStatus.Offline) {
                Log.To.Sync.I(TAG, "Offline, so skipping inbox process");
                return;
            }
                
            if (ReplicationOptions.AllNew) {
                // If 'allNew' option is set, upload new revs without checking first:
                foreach (var rev in inbox) {
                    AddPending(rev);
                }

                UploadChanges(inbox, null);
                return;
            }

            // Generate a set of doc/rev IDs in the JSON format that _revs_diff wants:
            // <http://wiki.apache.org/couchdb/HttpPostRevsDiff>
            var diffs = new Dictionary<String, IList<String>>();
            var inboxCount = inbox.Count;
            foreach (var rev in inbox) {
                var docID = rev.DocID;
                var revs = diffs.Get(docID);
                if (revs == null) {
                    revs = new List<String>();
                    diffs[docID] = revs;
                }
                revs.Add(rev.RevID.ToString());
                AddPending(rev);
            }

            // Call _revs_diff on the target db:
            Log.To.Sync.D(TAG, "posting to /_revs_diff: {0}", String.Join(Environment.NewLine, new[] { Manager.GetObjectMapper().WriteValueAsString(diffs) }));
            _remoteSession.SendAsyncRequest(HttpMethod.Post, "/_revs_diff", diffs, (response, e) =>
            {
                try {
                    if(!LocalDatabase.IsOpen) {
                        return;
                    }

                    var results = response.AsDictionary<string, object>();

                    Log.To.Sync.D(TAG, "/_revs_diff response: {0}\r\n{1}", response, results);

                    if (e != null) {
                        LastError = e;
                        for(int i = 0; i < inboxCount; i++) {
                            RevisionFailed();
                        }

                        if(Continuous) {
                            FireTrigger(ReplicationTrigger.WaitingForChanges);
                        } else {
                            FireTrigger(ReplicationTrigger.StopImmediate);
                        }
                    } else {
                        if (results.Count != 0)  {
                            UploadChanges(inbox, results);
                        } else {
                            foreach (var revisionInternal in inbox) {
                                RemovePending(revisionInternal);
                            }

                            //SafeAddToCompletedChangesCount(inbox.Count);
                        }
                    }
                } catch (Exception ex) {
                    Log.To.Sync.E(TAG, "Unhandled exception in Pusher.ProcessInbox, continuing...", ex);
                }
            });
        }
        /// <summary>
        /// Returns a sorted list of changes made to documents in the database, in time order of application.
        /// </summary>
        /// <returns>The response state for further HTTP processing</returns>
        /// <param name="context">The context of the Couchbase Lite HTTP request</param>
        /// <remarks>
        /// http://docs.couchdb.org/en/latest/api/database/changes.html#get--db-_changes
        /// <remarks>
        public static ICouchbaseResponseState GetChanges(ICouchbaseListenerContext context)
        {
            DBMonitorCouchbaseResponseState responseState = new DBMonitorCouchbaseResponseState();

            var responseObject = PerformLogicWithDatabase(context, true, db =>
            {
                var response           = context.CreateResponse();
                responseState.Response = response;
                if (context.ChangesFeedMode < ChangesFeedMode.Continuous)
                {
                    if (context.CacheWithEtag(db.GetLastSequenceNumber().ToString()))
                    {
                        response.InternalStatus = StatusCode.NotModified;
                        return(response);
                    }
                }

                var options                           = ChangesOptions.Default;
                responseState.Db                      = db;
                responseState.ContentOptions          = context.ContentOptions;
                responseState.ChangesFeedMode         = context.ChangesFeedMode;
                responseState.ChangesIncludeDocs      = context.GetQueryParam <bool>("include_docs", bool.TryParse, false);
                options.IncludeDocs                   = responseState.ChangesIncludeDocs;
                responseState.ChangesIncludeConflicts = context.GetQueryParam("style") == "all_docs";
                options.IncludeConflicts              = responseState.ChangesIncludeConflicts;
                options.ContentOptions                = context.ContentOptions;
                options.SortBySequence                = !options.IncludeConflicts;
                options.Limit                         = context.GetQueryParam <int>("limit", int.TryParse, options.Limit);
                int since = context.GetQueryParam <int>("since", int.TryParse, 0);

                string filterName = context.GetQueryParam("filter");
                if (filterName != null)
                {
                    Status status = new Status();
                    responseState.ChangesFilter = db.GetFilter(filterName, status);
                    if (responseState.ChangesFilter == null)
                    {
                        return(context.CreateResponse(status.Code));
                    }

                    responseState.FilterParams = context.GetQueryParams();
                }


                RevisionList changes = db.ChangesSince(since, options, responseState.ChangesFilter, responseState.FilterParams);
                if ((context.ChangesFeedMode >= ChangesFeedMode.Continuous) ||
                    (context.ChangesFeedMode == ChangesFeedMode.LongPoll && changes.Count == 0))
                {
                    // Response is going to stay open (continuous, or hanging GET):
                    response.Chunked = true;
                    if (context.ChangesFeedMode == ChangesFeedMode.EventSource)
                    {
                        response["Content-Type"] = "text/event-stream; charset=utf-8";
                    }

                    if (context.ChangesFeedMode >= ChangesFeedMode.Continuous)
                    {
                        response.WriteHeaders();
                        foreach (var rev in changes)
                        {
                            var success = response.SendContinuousLine(ChangesDictForRev(rev, responseState), context.ChangesFeedMode);
                            if (!success)
                            {
                                return(context.CreateResponse(StatusCode.BadRequest));
                            }
                        }
                    }

                    responseState.SubscribeToDatabase(db);
                    string heartbeatParam = context.GetQueryParam("heartbeat");
                    if (heartbeatParam != null)
                    {
                        int heartbeat;
                        if (!int.TryParse(heartbeatParam, out heartbeat) || heartbeat <= 0)
                        {
                            responseState.IsAsync = false;
                            return(context.CreateResponse(StatusCode.BadParam));
                        }

                        var heartbeatSpan = TimeSpan.FromMilliseconds(heartbeat);
                        if (heartbeatSpan < MinHeartbeat)
                        {
                            heartbeatSpan = MinHeartbeat;
                        }

                        string heartbeatResponse = context.ChangesFeedMode == ChangesFeedMode.EventSource ? "\n\n" : "\r\n";
                        responseState.StartHeartbeat(heartbeatResponse, heartbeatSpan);
                    }

                    return(response);
                }
                else
                {
                    if (responseState.ChangesIncludeConflicts)
                    {
                        response.JsonBody = new Body(ResponseBodyForChanges(changes, since, options.Limit, responseState));
                    }
                    else
                    {
                        response.JsonBody = new Body(ResponseBodyForChanges(changes, since, responseState));
                    }

                    return(response);
                }
            });

            responseState.Response = responseObject;
            return(responseState);
        }
        public static ICouchbaseResponseState GetChangesPost(ICouchbaseListenerContext context)
        {
            DBMonitorCouchbaseResponseState responseState = new DBMonitorCouchbaseResponseState();

            var responseObject = PerformLogicWithDatabase(context, true, db =>
            {
                var response           = context.CreateResponse();
                responseState.Response = response;
                var body = context.BodyAs <Dictionary <string, object> >();
                ProcessBody(body);
                if (body.GetCast <ChangesFeedMode>("feed") < ChangesFeedMode.Continuous)
                {
                    if (context.CacheWithEtag(db.GetLastSequenceNumber().ToString()))
                    {
                        response.InternalStatus = StatusCode.NotModified;
                        return(response);
                    }
                }

                var options                           = ChangesOptions.Default;
                responseState.Db                      = db;
                responseState.ContentOptions          = body.GetCast <DocumentContentOptions>("content_options");
                responseState.ChangesFeedMode         = body.GetCast <ChangesFeedMode>("feed");
                responseState.ChangesIncludeDocs      = body.GetCast <bool>("include_docs");
                options.IncludeDocs                   = responseState.ChangesIncludeDocs;
                responseState.ChangesIncludeConflicts = body.GetCast <string>("style") == "all_docs";
                options.IncludeConflicts              = responseState.ChangesIncludeConflicts;
                options.ContentOptions                = responseState.ContentOptions;
                options.SortBySequence                = !options.IncludeConflicts;
                options.Limit                         = body.GetCast <int>("limit", options.Limit);
                int since = body.GetCast <int>("since");

                string filterName = body.GetCast <string>("filter");
                if (filterName != null)
                {
                    Status status = new Status();
                    responseState.ChangesFilter = db.GetFilter(filterName, status);
                    if (responseState.ChangesFilter == null)
                    {
                        return(context.CreateResponse(status.Code));
                    }

                    responseState.FilterParams = context.GetQueryParams();
                }


                RevisionList changes = db.ChangesSince(since, options, responseState.ChangesFilter, responseState.FilterParams);
                if ((responseState.ChangesFeedMode >= ChangesFeedMode.Continuous) ||
                    (responseState.ChangesFeedMode == ChangesFeedMode.LongPoll && changes.Count == 0))
                {
                    // Response is going to stay open (continuous, or hanging GET):
                    response.Chunked = true;
                    if (responseState.ChangesFeedMode == ChangesFeedMode.EventSource)
                    {
                        response["Content-Type"] = "text/event-stream; charset=utf-8";
                    }

                    if (responseState.ChangesFeedMode >= ChangesFeedMode.Continuous)
                    {
                        response.WriteHeaders();
                        foreach (var rev in changes)
                        {
                            response.SendContinuousLine(ChangesDictForRev(rev, responseState), context.ChangesFeedMode);
                        }
                    }

                    responseState.SubscribeToDatabase(db);
                    int heartbeat = body.GetCast <int>("heartbeat", Int32.MinValue);
                    if (heartbeat != Int32.MinValue)
                    {
                        if (heartbeat <= 0)
                        {
                            responseState.IsAsync = false;
                            return(context.CreateResponse(StatusCode.BadParam));
                        }

                        heartbeat = Math.Max(heartbeat, (int)MinHeartbeat.TotalMilliseconds);
                        string heartbeatResponse = context.ChangesFeedMode == ChangesFeedMode.EventSource ? "\n\n" : "\r\n";
                        responseState.StartHeartbeat(heartbeatResponse, TimeSpan.FromMilliseconds(heartbeat));
                    }

                    return(context.CreateResponse());
                }
                else
                {
                    if (responseState.ChangesIncludeConflicts)
                    {
                        response.JsonBody = new Body(ResponseBodyForChanges(changes, since, options.Limit, responseState));
                    }
                    else
                    {
                        response.JsonBody = new Body(ResponseBodyForChanges(changes, since, responseState));
                    }

                    return(response);
                }
            });

            responseState.Response = responseObject;
            return(responseState);
        }
        public static ICouchbaseResponseState RevsDiff(ICouchbaseListenerContext context)
        {
            // Collect all of the input doc/revision IDs as CBL_Revisions:
            var revs = new RevisionList();
            var body = context.BodyAs <Dictionary <string, object> >();

            if (body == null)
            {
                return(context.CreateResponse(StatusCode.BadJson).AsDefaultState());
            }

            foreach (var docPair in body)
            {
                var revIDs = docPair.Value.AsList <string>();
                if (revIDs == null)
                {
                    return(context.CreateResponse(StatusCode.BadParam).AsDefaultState());
                }

                foreach (var revID in revIDs)
                {
                    var rev = new RevisionInternal(docPair.Key, revID.AsRevID(), false);
                    revs.Add(rev);
                }
            }

            return(PerformLogicWithDatabase(context, true, db =>
            {
                var response = context.CreateResponse();
                // Look them up, removing the existing ones from revs:
                db.Storage.FindMissingRevisions(revs);

                // Return the missing revs in a somewhat different format:
                IDictionary <string, object> diffs = new Dictionary <string, object>();
                foreach (var rev in revs)
                {
                    var docId = rev.DocID;
                    IList <RevisionID> missingRevs = null;
                    if (!diffs.ContainsKey(docId))
                    {
                        missingRevs = new List <RevisionID>();
                        diffs[docId] = new Dictionary <string, IList <RevisionID> > {
                            { "missing", missingRevs }
                        };
                    }
                    else
                    {
                        missingRevs = ((Dictionary <string, IList <RevisionID> >)diffs[docId])["missing"];
                    }

                    missingRevs.Add(rev.RevID);
                }

                // Add the possible ancestors for each missing revision:
                foreach (var docPair in diffs)
                {
                    IDictionary <string, IList <RevisionID> > docInfo = (IDictionary <string, IList <RevisionID> >)docPair.Value;
                    int maxGen = 0;
                    RevisionID maxRevID = null;
                    foreach (var revId in docInfo["missing"])
                    {
                        if (revId.Generation > maxGen)
                        {
                            maxGen = revId.Generation;
                            maxRevID = revId;
                        }
                    }

                    var rev = new RevisionInternal(docPair.Key, maxRevID, false);
                    var ancestors = db.Storage.GetPossibleAncestors(rev, 0, ValueTypePtr <bool> .NULL)?.ToList();
                    if (ancestors != null && ancestors.Count > 0)
                    {
                        docInfo["possible_ancestors"] = ancestors;
                    }
                }

                response.JsonBody = new Body(diffs);
                return response;
            }).AsDefaultState());
        }
Example #32
0
		public RevisionList GetAllRevisionsOfDocumentID(string docId, long docNumericID, 
			bool onlyCurrent)
		{
			string sql = null;
			if (onlyCurrent)
			{
				sql = "SELECT sequence, revid, deleted FROM revs " + "WHERE doc_id=? AND current ORDER BY sequence DESC";
			}
			else
			{
				sql = "SELECT sequence, revid, deleted FROM revs " + "WHERE doc_id=? ORDER BY sequence DESC";
			}
			string[] args = new string[] { System.Convert.ToString(docNumericID) };
			Cursor cursor = null;
			cursor = database.RawQuery(sql, args);
			RevisionList result;
			try
			{
				cursor.MoveToNext();
				result = new RevisionList();
				while (!cursor.IsAfterLast())
				{
					RevisionInternal rev = new RevisionInternal(docId, cursor.GetString(1), (cursor.GetInt
						(2) > 0), this);
					rev.SetSequence(cursor.GetLong(0));
					result.AddItem(rev);
					cursor.MoveToNext();
				}
			}
			catch (SQLException e)
			{
				Log.E(Database.Tag, "Error getting all revisions of document", e);
				return null;
			}
			finally
			{
				if (cursor != null)
				{
					cursor.Close();
				}
			}
			return result;
		}
Example #33
0
        internal override void ProcessInbox(RevisionList inbox)
        {
            var lastInboxSequence = inbox[inbox.Count - 1].GetSequence();
            // Generate a set of doc/rev IDs in the JSON format that _revs_diff wants:
            // <http://wiki.apache.org/couchdb/HttpPostRevsDiff>
            var diffs = new Dictionary <String, IList <String> >();

            foreach (var rev in inbox)
            {
                var docID = rev.GetDocId();
                var revs  = diffs.Get(docID);
                if (revs == null)
                {
                    revs         = new AList <String>();
                    diffs[docID] = revs;
                }
                revs.AddItem(rev.GetRevId());
            }

            // Call _revs_diff on the target db:
            Log.D(Tag, this + "|" + Thread.CurrentThread() + ": processInbox() calling asyncTaskStarted()");
            Log.D(Tag, this + "|" + Thread.CurrentThread() + ": posting to /_revs_diff: " + diffs);

            AsyncTaskStarted();
            SendAsyncRequest(HttpMethod.Post, "/_revs_diff", diffs, (response, e) =>
            {
                try {
                    Log.D(Tag, this + "|" + Thread.CurrentThread() + ": /_revs_diff response: " + response);

                    var responseData = (JObject)response;
                    var results      = responseData.ToObject <IDictionary <string, object> >();

                    if (e != null)
                    {
                        LastError = e;
                        RevisionFailed();
                        //Stop ();
                    }
                    else
                    {
                        if (results.Count != 0)
                        {
                            // Go through the list of local changes again, selecting the ones the destination server
                            // said were missing and mapping them to a JSON dictionary in the form _bulk_docs wants:
                            var docsToSend = new AList <object> ();

                            foreach (var rev in inbox)
                            {
                                IDictionary <string, object> properties = null;
                                var resultDocData = (JObject)results.Get(rev.GetDocId());
                                var resultDoc     = resultDocData.ToObject <IDictionary <String, Object> >();
                                if (resultDoc != null)
                                {
                                    var revs = ((JArray)resultDoc.Get("missing")).Values <String>().ToList();
                                    if (revs != null && revs.Contains(rev.GetRevId()))
                                    {
                                        //remote server needs this revision
                                        // Get the revision's properties
                                        if (rev.IsDeleted())
                                        {
                                            properties = new Dictionary <string, object> ();
                                            properties.Put("_id", rev.GetDocId());
                                            properties.Put("_rev", rev.GetRevId());
                                            properties.Put("_deleted", true);
                                        }
                                        else
                                        {
                                            // OPT: Shouldn't include all attachment bodies, just ones that have changed
                                            var contentOptions = EnumSet.Of(TDContentOptions.TDIncludeAttachments, TDContentOptions.TDBigAttachmentsFollow);
                                            try {
                                                LocalDatabase.LoadRevisionBody(rev, contentOptions);
                                            } catch (CouchbaseLiteException e1) {
                                                Log.W(Tag, string.Format("%s Couldn't get local contents of %s", rev, this));
                                                RevisionFailed();
                                                continue;
                                            }
                                            properties = new Dictionary <String, Object> (rev.GetProperties());
                                        }
                                        if (properties.ContainsKey("_attachments"))
                                        {
                                            if (UploadMultipartRevision(rev))
                                            {
                                                continue;
                                            }
                                        }
                                        if (properties != null)
                                        {
                                            // Add the _revisions list:
                                            properties.Put("_revisions", LocalDatabase.GetRevisionHistoryDict(rev));
                                            //now add it to the docs to send
                                            docsToSend.AddItem(properties);
                                        }
                                    }
                                }
                            }
                            // Post the revisions to the destination. "new_edits":false means that the server should
                            // use the given _rev IDs instead of making up new ones.
                            var numDocsToSend = docsToSend.Count;
                            if (numDocsToSend > 0)
                            {
                                var bulkDocsBody = new Dictionary <String, Object> ();
                                bulkDocsBody.Put("docs", docsToSend);
                                bulkDocsBody.Put("new_edits", false);

                                Log.V(Tag, string.Format("%s: POSTing " + numDocsToSend + " revisions to _bulk_docs: %s", this, docsToSend));

                                ChangesCount += numDocsToSend;

                                Log.D(Tag, this + "|" + Thread.CurrentThread() + ": processInbox-before_bulk_docs() calling asyncTaskStarted()");

                                AsyncTaskStarted();
                                SendAsyncRequest(HttpMethod.Post, "/_bulk_docs", bulkDocsBody, (result, ex) => {
                                    try
                                    {
                                        if (ex != null)
                                        {
                                            LastError = ex;
                                            RevisionFailed();
                                        }
                                        else
                                        {
                                            Log.V(Tag, string.Format("%s: POSTed to _bulk_docs: %s", this, docsToSend));
                                            LastSequence = string.Format("{0}", lastInboxSequence);
                                        }
                                        CompletedChangesCount += numDocsToSend;
                                    }
                                    finally
                                    {
                                        AsyncTaskFinished(1);
                                    }
                                });
                            }
                        }
                        else
                        {
                            // If none of the revisions are new to the remote, just bump the lastSequence:
                            LastSequence = string.Format("{0}", lastInboxSequence);
                        }
                    }
                }
                finally
                {
                    Log.D(Tag, this + "|" + Thread.CurrentThread() + ": processInbox() calling asyncTaskFinished()");
                    AsyncTaskFinished(1);
                }
            });
        }
Example #34
0
		/// <exception cref="Couchbase.Lite.CouchbaseLiteException"></exception>
		public virtual void TestRevTree()
		{
			RevisionInternal rev = new RevisionInternal("MyDocId", "4-foxy", false, database);
			IDictionary<string, object> revProperties = new Dictionary<string, object>();
			revProperties.Put("_id", rev.GetDocId());
			revProperties.Put("_rev", rev.GetRevId());
			revProperties.Put("message", "hi");
			rev.SetProperties(revProperties);
			IList<string> revHistory = new AList<string>();
			revHistory.AddItem(rev.GetRevId());
			revHistory.AddItem("3-thrice");
			revHistory.AddItem("2-too");
			revHistory.AddItem("1-won");
			database.ForceInsert(rev, revHistory, null);
			NUnit.Framework.Assert.AreEqual(1, database.GetDocumentCount());
			VerifyHistory(database, rev, revHistory);
			RevisionInternal conflict = new RevisionInternal("MyDocId", "5-epsilon", false, database
				);
			IDictionary<string, object> conflictProperties = new Dictionary<string, object>();
			conflictProperties.Put("_id", conflict.GetDocId());
			conflictProperties.Put("_rev", conflict.GetRevId());
			conflictProperties.Put("message", "yo");
			conflict.SetProperties(conflictProperties);
			IList<string> conflictHistory = new AList<string>();
			conflictHistory.AddItem(conflict.GetRevId());
			conflictHistory.AddItem("4-delta");
			conflictHistory.AddItem("3-gamma");
			conflictHistory.AddItem("2-too");
			conflictHistory.AddItem("1-won");
			IList wasInConflict = new ArrayList();
			Database.ChangeListener listener = new _ChangeListener_84(wasInConflict);
			database.AddChangeListener(listener);
			database.ForceInsert(conflict, conflictHistory, null);
			NUnit.Framework.Assert.IsTrue(wasInConflict.Count > 0);
			database.RemoveChangeListener(listener);
			NUnit.Framework.Assert.AreEqual(1, database.GetDocumentCount());
			VerifyHistory(database, conflict, conflictHistory);
			// Add an unrelated document:
			RevisionInternal other = new RevisionInternal("AnotherDocID", "1-ichi", false, database
				);
			IDictionary<string, object> otherProperties = new Dictionary<string, object>();
			otherProperties.Put("language", "jp");
			other.SetProperties(otherProperties);
			IList<string> otherHistory = new AList<string>();
			otherHistory.AddItem(other.GetRevId());
			database.ForceInsert(other, otherHistory, null);
			// Fetch one of those phantom revisions with no body:
			RevisionInternal rev2 = database.GetDocumentWithIDAndRev(rev.GetDocId(), "2-too", 
				EnumSet.NoneOf<Database.TDContentOptions>());
			NUnit.Framework.Assert.AreEqual(rev.GetDocId(), rev2.GetDocId());
			NUnit.Framework.Assert.AreEqual("2-too", rev2.GetRevId());
			//Assert.assertNull(rev2.getContent());
			// Make sure no duplicate rows were inserted for the common revisions:
			NUnit.Framework.Assert.AreEqual(8, database.GetLastSequenceNumber());
			// Make sure the revision with the higher revID wins the conflict:
			RevisionInternal current = database.GetDocumentWithIDAndRev(rev.GetDocId(), null, 
				EnumSet.NoneOf<Database.TDContentOptions>());
			NUnit.Framework.Assert.AreEqual(conflict, current);
			// Get the _changes feed and verify only the winner is in it:
			ChangesOptions options = new ChangesOptions();
			RevisionList changes = database.ChangesSince(0, options, null);
			RevisionList expectedChanges = new RevisionList();
			expectedChanges.AddItem(conflict);
			expectedChanges.AddItem(other);
			NUnit.Framework.Assert.AreEqual(changes, expectedChanges);
			options.SetIncludeConflicts(true);
			changes = database.ChangesSince(0, options, null);
			expectedChanges = new RevisionList();
			expectedChanges.AddItem(rev);
			expectedChanges.AddItem(conflict);
			expectedChanges.AddItem(other);
			NUnit.Framework.Assert.AreEqual(changes, expectedChanges);
		}
 protected internal override void ProcessInbox(RevisionList changes)
 {
     // Generate a set of doc/rev IDs in the JSON format that _revs_diff wants:
     // <http://wiki.apache.org/couchdb/HttpPostRevsDiff>
     IDictionary<string, IList<string>> diffs = new Dictionary<string, IList<string>>(
         );
     foreach (RevisionInternal rev in changes)
     {
         string docID = rev.GetDocId();
         IList<string> revs = diffs.Get(docID);
         if (revs == null)
         {
             revs = new AList<string>();
             diffs.Put(docID, revs);
         }
         revs.AddItem(rev.GetRevId());
         AddPending(rev);
     }
     // Call _revs_diff on the target db:
     Log.V(Log.TagSync, "%s: posting to /_revs_diff", this);
     Log.V(Log.TagSync, "%s | %s: processInbox() calling asyncTaskStarted()", this, Sharpen.Thread
         .CurrentThread());
     AsyncTaskStarted();
     SendAsyncRequest("POST", "/_revs_diff", diffs, new _RemoteRequestCompletionBlock_280
         (this, changes));
 }
        internal override void ProcessInbox(RevisionList inbox)
        {
            if (Status == ReplicationStatus.Offline) {
                Log.V(TAG, "Offline, so skipping inbox process");
                return;
            }

            if(_requests.Count > ManagerOptions.Default.MaxOpenHttpConnections) {
                Task.Delay(1000).ContinueWith(t => ProcessInbox(inbox), CancellationToken.None, TaskContinuationOptions.None, WorkExecutor.Scheduler);
                return;
            }

            // Generate a set of doc/rev IDs in the JSON format that _revs_diff wants:
            // <http://wiki.apache.org/couchdb/HttpPostRevsDiff>
            var diffs = new Dictionary<String, IList<String>>();
            foreach (var rev in inbox) {
                var docID = rev.GetDocId();
                var revs = diffs.Get(docID);
                if (revs == null) {
                    revs = new List<String>();
                    diffs[docID] = revs;
                }
                revs.Add(rev.GetRevId());
                AddPending(rev);
            }

            // Call _revs_diff on the target db:
            Log.D(TAG, "posting to /_revs_diff: {0}", String.Join(Environment.NewLine, new[] { Manager.GetObjectMapper().WriteValueAsString(diffs) }));
            SendAsyncRequest(HttpMethod.Post, "/_revs_diff", diffs, (response, e) =>
            {
                try {
                    if(!LocalDatabase.IsOpen) {
                        return;
                    }

                    var results = response.AsDictionary<string, object>();

                    Log.D(TAG, "/_revs_diff response: {0}\r\n{1}", response, results);

                    if (e != null) {
                        LastError = e;
                        RevisionFailed();
                    } else {
                        if (results.Count != 0)  {
                            // Go through the list of local changes again, selecting the ones the destination server
                            // said were missing and mapping them to a JSON dictionary in the form _bulk_docs wants:
                            var docsToSend = new List<object> ();
                            var revsToSend = new RevisionList();
                            foreach (var rev in inbox) {
                                // Is this revision in the server's 'missing' list?
                                IDictionary<string, object> properties = null;
                                var revResults = results.Get(rev.GetDocId()).AsDictionary<string, object>(); 
                                if (revResults == null) {
                                    //SafeIncrementCompletedChangesCount();
                                    continue;
                                }

                                var revs = revResults.Get("missing").AsList<string>();
                                if (revs == null || !revs.Any( id => id.Equals(rev.GetRevId(), StringComparison.OrdinalIgnoreCase))) {
                                    RemovePending(rev);
                                    //SafeIncrementCompletedChangesCount();
                                    continue;
                                }

                                // Get the revision's properties:
                                var contentOptions = DocumentContentOptions.IncludeAttachments;
                                if (!_dontSendMultipart && RevisionBodyTransformationFunction == null)
                                {
                                    contentOptions |= DocumentContentOptions.BigAttachmentsFollow;
                                }

                                RevisionInternal loadedRev;
                                try {
                                    loadedRev = LocalDatabase.LoadRevisionBody (rev);
                                    if(loadedRev == null) {
                                        throw new CouchbaseLiteException("DB is closed", StatusCode.DbError);
                                    }

                                    properties = new Dictionary<string, object>(rev.GetProperties());
                                } catch (Exception e1) {
                                    Log.W(TAG, String.Format("{0} Couldn't get local contents of", rev), e);
                                    RevisionFailed();
                                    continue;
                                }

                                var populatedRev = TransformRevision(loadedRev);
                                IList<string> possibleAncestors = null;
                                if (revResults.ContainsKey("possible_ancestors")) {
                                    possibleAncestors = revResults["possible_ancestors"].AsList<string>();
                                }

                                properties = new Dictionary<string, object>(populatedRev.GetProperties());

                                try {
                                    var history = LocalDatabase.GetRevisionHistory(populatedRev, possibleAncestors);
                                    if(history == null) {
                                        throw new CouchbaseLiteException("DB closed", StatusCode.DbError);
                                    }

                                    properties["_revisions"] = Database.MakeRevisionHistoryDict(history);
                                } catch(Exception e1) {
                                    Log.W(TAG, "Error getting revision history", e1);
                                    RevisionFailed();
                                    continue;
                                }

                                populatedRev.SetProperties(properties);
                                if(properties.GetCast<bool>("_removed")) {
                                    RemovePending(rev);
                                    continue;
                                }

                                // Strip any attachments already known to the target db:
                                if (properties.ContainsKey("_attachments")) {
                                    // Look for the latest common ancestor and stuf out older attachments:
                                    var minRevPos = FindCommonAncestor(populatedRev, possibleAncestors);
                                    try {
                                        LocalDatabase.ExpandAttachments(populatedRev, minRevPos + 1, !_dontSendMultipart, false);
                                    } catch(Exception ex) {
                                        Log.W(TAG, "Error expanding attachments!", ex);
                                        RevisionFailed();
                                        continue;
                                    }

                                    properties = populatedRev.GetProperties();
                                    if (!_dontSendMultipart && UploadMultipartRevision(populatedRev)) {
                                        continue;
                                    }
                                }

                                if (properties == null || !properties.ContainsKey("_id")) {
                                    throw new InvalidOperationException("properties must contain a document _id");
                                }

                                // Add the _revisions list:
                                revsToSend.Add(rev);

                                //now add it to the docs to send
                                docsToSend.Add (properties);
                            }

                            UploadBulkDocs(docsToSend, revsToSend);
                        } else {
                            foreach (var revisionInternal in inbox) {
                                RemovePending(revisionInternal);
                            }

                            SafeAddToCompletedChangesCount(inbox.Count);
                        }
                    }
                } catch (Exception ex) {
                    Log.E(TAG, "Unhandled exception in Pusher.ProcessInbox", ex);
                }
            });
        }
 public _RemoteRequestCompletionBlock_280(Pusher _enclosing, RevisionList changes)
 {
     this._enclosing = _enclosing;
     this.changes = changes;
 }
        public RevisionList GetAllDocumentRevisions(string docId, bool onlyCurrent)
        {
            var retVal = default(RevisionList);
            WithC4Document(docId, null, false, false, doc =>
            {
                using(var enumerator = new CBForestHistoryEnumerator(doc, onlyCurrent, false)) {
                    retVal = new RevisionList(enumerator.Select(x => new RevisionInternal(x.Document, false)).ToList());
                }
            });

            return retVal;
        }
 public void OnCompletion(object response, Exception e)
 {
     try
     {
         Log.V(Log.TagSync, "%s: got /_revs_diff response");
         IDictionary<string, object> results = (IDictionary<string, object>)response;
         if (e != null)
         {
             this._enclosing.SetError(e);
             this._enclosing.RevisionFailed();
         }
         else
         {
             if (results.Count != 0)
             {
                 // Go through the list of local changes again, selecting the ones the destination server
                 // said were missing and mapping them to a JSON dictionary in the form _bulk_docs wants:
                 IList<object> docsToSend = new AList<object>();
                 RevisionList revsToSend = new RevisionList();
                 foreach (RevisionInternal rev in changes)
                 {
                     // Is this revision in the server's 'missing' list?
                     IDictionary<string, object> properties = null;
                     IDictionary<string, object> revResults = (IDictionary<string, object>)results.Get
                         (rev.GetDocId());
                     if (revResults == null)
                     {
                         continue;
                     }
                     IList<string> revs = (IList<string>)revResults.Get("missing");
                     if (revs == null || !revs.Contains(rev.GetRevId()))
                     {
                         this._enclosing.RemovePending(rev);
                         continue;
                     }
                     // Get the revision's properties:
                     EnumSet<Database.TDContentOptions> contentOptions = EnumSet.Of(Database.TDContentOptions
                         .TDIncludeAttachments);
                     if (!this._enclosing.dontSendMultipart && this._enclosing.revisionBodyTransformationBlock
                          == null)
                     {
                         contentOptions.AddItem(Database.TDContentOptions.TDBigAttachmentsFollow);
                     }
                     RevisionInternal loadedRev;
                     try
                     {
                         loadedRev = this._enclosing.db.LoadRevisionBody(rev, contentOptions);
                         properties = new Dictionary<string, object>(rev.GetProperties());
                     }
                     catch (CouchbaseLiteException)
                     {
                         Log.W(Log.TagSync, "%s Couldn't get local contents of %s", rev, this._enclosing);
                         this._enclosing.RevisionFailed();
                         continue;
                     }
                     RevisionInternal populatedRev = this._enclosing.TransformRevision(loadedRev);
                     IList<string> possibleAncestors = (IList<string>)revResults.Get("possible_ancestors"
                         );
                     properties = new Dictionary<string, object>(populatedRev.GetProperties());
                     IDictionary<string, object> revisions = this._enclosing.db.GetRevisionHistoryDictStartingFromAnyAncestor
                         (populatedRev, possibleAncestors);
                     properties.Put("_revisions", revisions);
                     populatedRev.SetProperties(properties);
                     // Strip any attachments already known to the target db:
                     if (properties.ContainsKey("_attachments"))
                     {
                         // Look for the latest common ancestor and stub out older attachments:
                         int minRevPos = Couchbase.Lite.Replicator.Pusher.FindCommonAncestor(populatedRev, 
                             possibleAncestors);
                         Database.StubOutAttachmentsInRevBeforeRevPos(populatedRev, minRevPos + 1, false);
                         properties = populatedRev.GetProperties();
                         if (!this._enclosing.dontSendMultipart && this._enclosing.UploadMultipartRevision
                             (populatedRev))
                         {
                             continue;
                         }
                     }
                     if (properties == null || !properties.ContainsKey("_id"))
                     {
                         throw new InvalidOperationException("properties must contain a document _id");
                     }
                     revsToSend.AddItem(rev);
                     docsToSend.AddItem(properties);
                 }
                 //TODO: port this code from iOS
                 // Post the revisions to the destination:
                 this._enclosing.UploadBulkDocs(docsToSend, revsToSend);
             }
             else
             {
                 // None of the revisions are new to the remote
                 foreach (RevisionInternal revisionInternal in changes)
                 {
                     this._enclosing.RemovePending(revisionInternal);
                 }
             }
         }
     }
     finally
     {
         Log.V(Log.TagSync, "%s | %s: processInbox.sendAsyncRequest() calling asyncTaskFinished()"
             , this, Sharpen.Thread.CurrentThread());
         this._enclosing.AsyncTaskFinished(1);
     }
 }
        public int FindMissingRevisions(RevisionList revs)
        {
            var sortedRevs = new RevisionList(revs);
            sortedRevs.SortByDocID();
            var lastDocId = (string)null;
            var doc = (C4Document*)null;
            var removedCount = 0;
            try {
                foreach (var rev in sortedRevs) {
                    if (rev.GetDocId() != lastDocId) {
                        lastDocId = rev.GetDocId();
                        Native.c4doc_free(doc);
                        doc = Native.c4doc_get(Forest, lastDocId, true, null);
                        if(doc == null) {
                            continue;
                        }
                    }

                    if (Native.c4doc_selectRevision(doc, rev.GetRevId(), false, null)) {
                        removedCount++;
                        revs.Remove(rev);
                    }
                }
            } finally {
                Native.c4doc_free(doc);
            }

            return removedCount;
        }
 protected internal void UploadBulkDocs(IList<object> docsToSend, RevisionList changes
     )
 {
     int numDocsToSend = docsToSend.Count;
     if (numDocsToSend == 0)
     {
         return;
     }
     Log.V(Log.TagSync, "%s: POSTing " + numDocsToSend + " revisions to _bulk_docs: %s"
         , this, docsToSend);
     AddToChangesCount(numDocsToSend);
     IDictionary<string, object> bulkDocsBody = new Dictionary<string, object>();
     bulkDocsBody.Put("docs", docsToSend);
     bulkDocsBody.Put("new_edits", false);
     Log.V(Log.TagSync, "%s | %s: uploadBulkDocs() calling asyncTaskStarted()", this, 
         Sharpen.Thread.CurrentThread());
     AsyncTaskStarted();
     SendAsyncRequest("POST", "/_bulk_docs", bulkDocsBody, new _RemoteRequestCompletionBlock_414
         (this, changes, numDocsToSend));
 }
        internal RevisionList ChangesSince(long lastSeq, ChangesOptions options, FilterDelegate filter)
        {
            // http://wiki.apache.org/couchdb/HTTP_database_API#Changes
            if (options == null)
            {
                options = new ChangesOptions();
            }

            var includeDocs = options.IsIncludeDocs() || (filter != null);
            var additionalSelectColumns = string.Empty;

            if (includeDocs)
            {
                additionalSelectColumns = ", json";
            }

            var sql = "SELECT sequence, revs.doc_id, docid, revid, deleted" + additionalSelectColumns
                      + " FROM revs, docs " + "WHERE sequence > ? AND current=1 " + "AND revs.doc_id = docs.doc_id "
                      + "ORDER BY revs.doc_id, revid DESC";
            var args = lastSeq;

            Cursor cursor = null;
            RevisionList changes = null;

            try
            {
                cursor = StorageEngine.RawQuery(sql, CommandBehavior.SequentialAccess, args);
                cursor.MoveToNext();

                changes = new RevisionList();
                long lastDocId = 0;

                while (!cursor.IsAfterLast())
                {
                    if (!options.IsIncludeConflicts())
                    {
                        // Only count the first rev for a given doc (the rest will be losing conflicts):
                        var docNumericId = cursor.GetLong(1);
                        if (docNumericId == lastDocId)
                        {
                            cursor.MoveToNext();
                            continue;
                        }
                        lastDocId = docNumericId;
                    }

                    var sequence = cursor.GetLong(0);
                    var rev = new RevisionInternal(cursor.GetString(2), cursor.GetString(3), (cursor.GetInt(4) > 0), this);
                    rev.SetSequence(sequence);

                    if (includeDocs)
                    {
                        ExpandStoredJSONIntoRevisionWithAttachments(cursor.GetBlob(5), rev, options.GetContentOptions());
                    }
                    IDictionary<string, object> paramsFixMe = null;
                    // TODO: these should not be null
                    if (RunFilter(filter, paramsFixMe, rev))
                    {
                        changes.AddItem(rev);
                    }
                    cursor.MoveToNext();
                }
            }
            catch (SQLException e)
            {
                Log.E(Tag, "Error looking for changes", e);
            }
            finally
            {
                if (cursor != null)
                {
                    cursor.Close();
                }
            }
            if (options.IsSortBySequence())
            {
                changes.SortBySequence();
            }
            changes.Limit(options.GetLimit());
            return changes;
        }
 public _RemoteRequestCompletionBlock_414(Pusher _enclosing, RevisionList changes, 
     int numDocsToSend)
 {
     this._enclosing = _enclosing;
     this.changes = changes;
     this.numDocsToSend = numDocsToSend;
 }
        private RevisionList GetAllRevisionsOfDocumentID(string docId, long docNumericID, bool onlyCurrent)
        {
            var sql = onlyCurrent 
                ? "SELECT sequence, revid, deleted FROM revs " + "WHERE doc_id=? AND current ORDER BY sequence DESC"
                : "SELECT sequence, revid, deleted FROM revs " + "WHERE doc_id=? ORDER BY sequence DESC";

            var args = new [] { Convert.ToString (docNumericID) };
            var cursor = StorageEngine.RawQuery(sql, args);

            RevisionList result;
            try
            {
                cursor.MoveToNext();
                result = new RevisionList();
                while (!cursor.IsAfterLast())
                {
                    var rev = new RevisionInternal(docId, cursor.GetString(1), (cursor.GetInt(2) > 0), this);
                    rev.SetSequence(cursor.GetLong(0));
                    result.AddItem(rev);
                    cursor.MoveToNext();
                }
            }
            catch (SQLException e)
            {
                Log.E(Tag, "Error getting all revisions of document", e);
                return null;
            }
            finally
            {
                if (cursor != null)
                {
                    cursor.Close();
                }
            }
            return result;
        }
Example #45
0
		protected internal override void ProcessInbox(RevisionList inbox)
		{
			long lastInboxSequence = inbox[inbox.Count - 1].GetSequence();
			// Generate a set of doc/rev IDs in the JSON format that _revs_diff wants:
			// <http://wiki.apache.org/couchdb/HttpPostRevsDiff>
			IDictionary<string, IList<string>> diffs = new Dictionary<string, IList<string>>(
				);
			foreach (RevisionInternal rev in inbox)
			{
				string docID = rev.GetDocId();
				IList<string> revs = diffs.Get(docID);
				if (revs == null)
				{
					revs = new AList<string>();
					diffs.Put(docID, revs);
				}
				revs.AddItem(rev.GetRevId());
			}
			// Call _revs_diff on the target db:
			Log.D(Database.Tag, this + "|" + Sharpen.Thread.CurrentThread() + ": processInbox() calling asyncTaskStarted()"
				);
			Log.D(Database.Tag, this + "|" + Sharpen.Thread.CurrentThread() + ": posting to /_revs_diff: "
				 + diffs);
			AsyncTaskStarted();
			SendAsyncRequest("POST", "/_revs_diff", diffs, new _RemoteRequestCompletionBlock_226
				(this, inbox, lastInboxSequence));
		}
        public static ICouchbaseResponseState RevsDiff(ICouchbaseListenerContext context)
        {
            // Collect all of the input doc/revision IDs as CBL_Revisions:
            var revs = new RevisionList();
            var body = context.BodyAs<Dictionary<string, object>>();
            if (body == null) {
                return context.CreateResponse(StatusCode.BadJson).AsDefaultState();
            }

            foreach (var docPair in body) {
                var revIDs = docPair.Value.AsList<string>();
                if (revIDs == null) {
                    return context.CreateResponse(StatusCode.BadParam).AsDefaultState();
                }

                foreach (var revID in revIDs) {
                    var rev = new RevisionInternal(docPair.Key, revID.AsRevID(), false);
                    revs.Add(rev);
                }
            }

            return PerformLogicWithDatabase(context, true, db =>
            {
                var response = context.CreateResponse();
                // Look them up, removing the existing ones from revs:
                db.Storage.FindMissingRevisions(revs);

                // Return the missing revs in a somewhat different format:
                IDictionary<string, object> diffs = new Dictionary<string, object>();
                foreach(var rev in revs) {
                    var docId = rev.DocID;
                    IList<RevisionID> missingRevs = null;
                    if(!diffs.ContainsKey(docId)) {
                        missingRevs = new List<RevisionID>();
                        diffs[docId] = new Dictionary<string, IList<RevisionID>> { { "missing", missingRevs } };
                    } else {
                        missingRevs = ((Dictionary<string, IList<RevisionID>>)diffs[docId])["missing"];
                    }

                    missingRevs.Add(rev.RevID);
                }

                // Add the possible ancestors for each missing revision:
                foreach(var docPair in diffs) {
                    IDictionary<string, IList<RevisionID>> docInfo = (IDictionary<string, IList<RevisionID>>)docPair.Value;
                    int maxGen = 0;
                    RevisionID maxRevID = null;
                    foreach(var revId in docInfo["missing"]) {
                        if(revId.Generation > maxGen) {
                            maxGen = revId.Generation;
                            maxRevID = revId;
                        }
                    }

                    var rev = new RevisionInternal(docPair.Key, maxRevID, false);
                    var ancestors = db.Storage.GetPossibleAncestors(rev, 0, ValueTypePtr<bool>.NULL)?.ToList();
                    if(ancestors != null && ancestors.Count > 0) {
                        docInfo["possible_ancestors"] = ancestors;
                    }
                }

                response.JsonBody = new Body(diffs);
                return response;
            }).AsDefaultState();

        }
Example #47
0
			public _RemoteRequestCompletionBlock_226(Pusher _enclosing, RevisionList inbox, long
				 lastInboxSequence)
			{
				this._enclosing = _enclosing;
				this.inbox = inbox;
				this.lastInboxSequence = lastInboxSequence;
			}
 internal abstract void ProcessInbox(RevisionList inbox);
Example #49
0
 public _RemoteRequestCompletionBlock_280(Pusher _enclosing, RevisionList changes)
 {
     this._enclosing = _enclosing;
     this.changes    = changes;
 }