public bool UpdateIndexes(IEnumerable <IViewStore> inputViews) { Log.D(TAG, "Checking indexes of ({0}) for {1}", ViewNames(inputViews.Cast <SqliteViewStore>()), Name); var db = _dbStorage; var status = false; status = db.RunInTransaction(() => { // If the view the update is for doesn't need any update, don't do anything: long dbMaxSequence = db.LastSequence; long forViewLastSequence = LastSequenceIndexed; if (forViewLastSequence >= dbMaxSequence) { return(true); } // Check whether we need to update at all, // and remove obsolete emitted results from the 'maps' table: long minLastSequence = dbMaxSequence; long[] viewLastSequence = new long[inputViews.Count()]; int deletedCount = 0; int i = 0; HashSet <string> docTypes = new HashSet <string>(); IDictionary <string, string> viewDocTypes = null; bool allDocTypes = false; IDictionary <int, int> viewTotalRows = new Dictionary <int, int>(); List <SqliteViewStore> views = new List <SqliteViewStore>(inputViews.Count()); List <MapDelegate> mapBlocks = new List <MapDelegate>(); foreach (var view in inputViews.Cast <SqliteViewStore>()) { var viewDelegate = view.Delegate; var mapBlock = viewDelegate == null ? null : viewDelegate.Map; if (mapBlock == null) { Debug.Assert(view != this, String.Format("Cannot index view {0}: no map block registered", view.Name)); Log.V(TAG, " {0} has no map block; skipping it", view.Name); continue; } views.Add(view); mapBlocks.Add(mapBlock); int viewId = view.ViewID; Debug.Assert(viewId > 0, String.Format("View '{0}' not found in database", view.Name)); int totalRows = view.TotalRows; viewTotalRows[viewId] = totalRows; long last = view == this ? forViewLastSequence : view.LastSequenceIndexed; viewLastSequence[i++] = last; if (last < 0) { throw new CouchbaseLiteException(StatusCode.DbError); } if (last < dbMaxSequence) { if (last == 0) { CreateIndex(); } minLastSequence = Math.Min(minLastSequence, last); Log.V(TAG, " {0} last indexed at #{1}", view.Name, last); string docType = viewDelegate.DocumentType; if (docType != null) { docTypes.Add(docType); if (viewDocTypes == null) { viewDocTypes = new Dictionary <string, string>(); } viewDocTypes[view.Name] = docType; } else { // can't filter by doc_type allDocTypes = true; } bool ok = true; int changes = 0; if (last == 0) { try { // If the lastSequence has been reset to 0, make sure to remove all map results: changes = db.StorageEngine.ExecSQL(view.QueryString("DELETE FROM 'maps_#'")); } catch (Exception) { ok = false; } } else { db.OptimizeSQLIndexes(); // ensures query will use the right indexes // Delete all obsolete map results (ones from since-replaced revisions): try { changes = db.StorageEngine.ExecSQL(view.QueryString("DELETE FROM 'maps_#' WHERE sequence IN (" + "SELECT parent FROM revs WHERE sequence>?" + "AND +parent>0 AND +parent<=?)"), last, last); } catch (Exception) { ok = false; } } if (!ok) { throw new CouchbaseLiteException(StatusCode.DbError); } // Update #deleted rows deletedCount += changes; // Only count these deletes as changes if this isn't a view reset to 0 if (last != 0) { viewTotalRows[viewId] -= changes; } } } if (minLastSequence == dbMaxSequence) { return(true); } Log.D(TAG, "Updating indexes of ({0}) from #{1} to #{2} ...", ViewNames(views), minLastSequence, dbMaxSequence); // This is the emit() block, which gets called from within the user-defined map() block // that's called down below. SqliteViewStore currentView = null; IDictionary <string, object> currentDoc = null; long sequence = minLastSequence; Status emitStatus = new Status(StatusCode.Ok); int insertedCount = 0; EmitDelegate emit = (key, value) => { StatusCode s = currentView.Emit(key, value, value == currentDoc, sequence); if (s != StatusCode.Ok) { emitStatus.Code = s; } else { viewTotalRows[currentView.ViewID] += 1; insertedCount++; } }; // Now scan every revision added since the last time the views were indexed: bool checkDocTypes = docTypes.Count > 1 || (allDocTypes && docTypes.Count > 0); var sql = new StringBuilder("SELECT revs.doc_id, sequence, docid, revid, json, deleted "); if (checkDocTypes) { sql.Append(", doc_type "); } sql.Append("FROM revs, docs WHERE sequence>? AND sequence <=? AND current!=0 "); if (minLastSequence == 0) { sql.Append("AND deleted=0 "); } if (!allDocTypes && docTypes.Count > 0) { sql.AppendFormat("AND doc_type IN ({0}) ", Database.JoinQuoted(docTypes)); } sql.Append("AND revs.doc_id = docs.doc_id " + "ORDER BY revs.doc_id, deleted, revid DESC"); Cursor c = null; Cursor c2 = null; try { c = db.StorageEngine.IntransactionRawQuery(sql.ToString(), minLastSequence, dbMaxSequence); bool keepGoing = c.MoveToNext(); while (keepGoing) { // Get row values now, before the code below advances 'c': long doc_id = c.GetLong(0); sequence = c.GetLong(1); string docId = c.GetString(2); if (docId.StartsWith("_design/")) // design documents don't get indexed { keepGoing = c.MoveToNext(); continue; } string revId = c.GetString(3); var json = c.GetBlob(4); bool deleted = c.GetInt(5) != 0; string docType = checkDocTypes ? c.GetString(6) : null; // Skip rows with the same doc_id -- these are losing conflicts. var conflicts = default(List <string>); while ((keepGoing = c.MoveToNext()) && c.GetLong(0) == doc_id) { if (conflicts == null) { conflicts = new List <string>(); } conflicts.Add(c.GetString(3)); } long realSequence = sequence; // because sequence may be changed, below if (minLastSequence > 0) { // Find conflicts with documents from previous indexings. using (c2 = db.StorageEngine.IntransactionRawQuery("SELECT revid, sequence FROM revs " + "WHERE doc_id=? AND sequence<=? AND current!=0 AND deleted=0 " + "ORDER BY revID DESC ", doc_id, minLastSequence)) { if (c2.MoveToNext()) { string oldRevId = c2.GetString(0); // This is the revision that used to be the 'winner'. // Remove its emitted rows: long oldSequence = c2.GetLong(1); foreach (var view in views) { int changes = db.StorageEngine.ExecSQL(QueryString("DELETE FROM 'maps_#' WHERE sequence=?"), oldSequence); deletedCount += changes; viewTotalRows[view.ViewID] -= changes; } if (deleted || RevisionInternal.CBLCompareRevIDs(oldRevId, revId) > 0) { // It still 'wins' the conflict, so it's the one that // should be mapped [again], not the current revision! revId = oldRevId; deleted = false; sequence = oldSequence; json = db.QueryOrDefault <byte[]>(x => x.GetBlob(0), true, null, "SELECT json FROM revs WHERE sequence=?", sequence); } if (!deleted) { // Conflict revisions: if (conflicts == null) { conflicts = new List <string>(); } conflicts.Add(oldRevId); while (c2.MoveToNext()) { conflicts.Add(c2.GetString(0)); } } } } } if (deleted) { continue; } // Get the document properties, to pass to the map function: currentDoc = db.GetDocumentProperties(json, docId, revId, deleted, sequence); if (currentDoc == null) { Log.W(TAG, "Failed to parse JSON of doc {0} rev {1}", docId, revId); continue; } currentDoc["_local_seq"] = sequence; if (conflicts != null) { currentDoc["_conflicts"] = conflicts; } // Call the user-defined map() to emit new key/value pairs from this revision: int viewIndex = -1; var e = views.GetEnumerator(); while (e.MoveNext()) { currentView = e.Current; ++viewIndex; if (viewLastSequence[viewIndex] < realSequence) { if (checkDocTypes) { var viewDocType = viewDocTypes[currentView.Name]; if (viewDocType != null && viewDocType != docType) { // skip; view's documentType doesn't match this doc continue; } } Log.V(TAG, " #{0}: map \"{1}\" for view {2}...", sequence, docId, e.Current.Name); try { mapBlocks[viewIndex](currentDoc, emit); } catch (Exception x) { Log.E(TAG, String.Format("Exception in map() block for view {0}", currentView.Name), x); emitStatus.Code = StatusCode.Exception; } if (emitStatus.IsError) { c.Dispose(); return(false); } } } currentView = null; } } catch (CouchbaseLiteException) { Log.W(TAG, "Failed to update index for {0}", currentView.Name); throw; } catch (Exception e) { throw new CouchbaseLiteException(String.Format("Error updating index for {0}", currentView.Name), e) { Code = StatusCode.Exception }; } finally { if (c != null) { c.Dispose(); } } // Finally, record the last revision sequence number that was indexed and update #rows: foreach (var view in views) { view.FinishCreatingIndex(); int newTotalRows = viewTotalRows[view.ViewID]; Debug.Assert(newTotalRows >= 0); var args = new ContentValues(); args["lastSequence"] = dbMaxSequence; args["total_docs"] = newTotalRows; try { db.StorageEngine.Update("views", args, "view_id=?", view.ViewID.ToString()); } catch (CouchbaseLiteException) { Log.W(TAG, "Failed to update view {0}", view.Name); throw; } catch (Exception e) { throw new CouchbaseLiteException(String.Format("Error updating view {0}", view.Name), e) { Code = StatusCode.Exception }; } } Log.D(TAG, "...Finished re-indexing ({0}) to #{1} (deleted {2}, added {3})", ViewNames(views), dbMaxSequence, deletedCount, insertedCount); return(true); }); if (!status) { Log.W(TAG, "CouchbaseLite: Failed to rebuild views ({0}): {1}", ViewNames(inputViews.Cast <SqliteViewStore>()), status); } return(status); }
private bool RevIdGreaterThanCurrent(string revId) { return(RevisionInternal.CBLCompareRevIDs(revId, currentRevision.Id) > 0); }
internal void UpdateIndex() { Log.I(Database.Tag, "Re-indexing view {0} ...", Name); System.Diagnostics.Debug.Assert((Map != null)); if (Id <= 0) { var msg = string.Format("View.Id <= 0"); throw new CouchbaseLiteException(msg, new Status(StatusCode.NotFound)); } var result = new Status(StatusCode.InternalServerError); Cursor cursor = null; Cursor cursor2 = null; try { Database.RunInTransaction(() => { var lastSequence = LastSequenceIndexed; var dbMaxSequence = Database.LastSequenceNumber; if (lastSequence >= dbMaxSequence) { // nothing to do (eg, kCBLStatusNotModified) Log.V(Database.Tag, "lastSequence ({0}) == dbMaxSequence ({1}), nothing to do", lastSequence, dbMaxSequence); result.SetCode(StatusCode.NotModified); return(false); } // First remove obsolete emitted results from the 'maps' table: var sequence = lastSequence; if (lastSequence < 0) { var msg = string.Format("lastSequence < 0 ({0})", lastSequence); throw new CouchbaseLiteException(msg, new Status(StatusCode.InternalServerError)); } if (lastSequence == 0) { // If the lastSequence has been reset to 0, make sure to remove // any leftover rows: var whereArgs = new string[] { Id.ToString() }; Database.StorageEngine.Delete("maps", "view_id=?", whereArgs); } else { Database.OptimizeSQLIndexes(); // Delete all obsolete map results (ones from since-replaced // revisions): var args = new [] { Id.ToString(), lastSequence.ToString(), lastSequence.ToString() }; Database.StorageEngine.ExecSQL( "DELETE FROM maps WHERE view_id=? AND sequence IN (" + "SELECT parent FROM revs WHERE sequence>? " + "AND +parent>0 AND +parent<=?)", args); } var deleted = 0; cursor = Database.StorageEngine.IntransactionRawQuery("SELECT changes()"); cursor.MoveToNext(); deleted = cursor.GetInt(0); cursor.Close(); // Find a better way to propagate this back // Now scan every revision added since the last time the view was indexed: var selectArgs = new[] { lastSequence.ToString(), dbMaxSequence.ToString() }; cursor = Database.StorageEngine.IntransactionRawQuery("SELECT revs.doc_id, sequence, docid, revid, json, no_attachments FROM revs, docs " + "WHERE sequence>? AND sequence<=? AND current!=0 AND deleted=0 " + "AND revs.doc_id = docs.doc_id " + "ORDER BY revs.doc_id, revid DESC", selectArgs); var lastDocID = 0L; var keepGoing = cursor.MoveToNext(); while (keepGoing) { long docID = cursor.GetLong(0); if (docID != lastDocID) { // Only look at the first-iterated revision of any document, // because this is the // one with the highest revid, hence the "winning" revision // of a conflict. lastDocID = docID; // Reconstitute the document as a dictionary: sequence = cursor.GetLong(1); string docId = cursor.GetString(2); if (docId.StartsWith("_design/", StringComparison.InvariantCultureIgnoreCase)) { // design docs don't get indexed! keepGoing = cursor.MoveToNext(); continue; } var revId = cursor.GetString(3); var json = cursor.GetBlob(4); var noAttachments = cursor.GetInt(5) > 0; // Skip rows with the same doc_id -- these are losing conflicts. while ((keepGoing = cursor.MoveToNext()) && cursor.GetLong(0) == docID) { } if (lastSequence > 0) { // Find conflicts with documents from previous indexings. var selectArgs2 = new[] { Convert.ToString(docID), Convert.ToString(lastSequence) }; cursor2 = Database.StorageEngine.IntransactionRawQuery("SELECT revid, sequence FROM revs " + "WHERE doc_id=? AND sequence<=? AND current!=0 AND deleted=0 " + "ORDER BY revID DESC " + "LIMIT 1", selectArgs2); if (cursor2.MoveToNext()) { var oldRevId = cursor2.GetString(0); // This is the revision that used to be the 'winner'. // Remove its emitted rows: var oldSequence = cursor2.GetLong(1); var args = new[] { Sharpen.Extensions.ToString(Id), Convert.ToString(oldSequence) }; Database.StorageEngine.ExecSQL("DELETE FROM maps WHERE view_id=? AND sequence=?", args); if (RevisionInternal.CBLCompareRevIDs(oldRevId, revId) > 0) { // It still 'wins' the conflict, so it's the one that // should be mapped [again], not the current revision! revId = oldRevId; sequence = oldSequence; var selectArgs3 = new[] { Convert.ToString(sequence) }; json = Misc.ByteArrayResultForQuery( Database.StorageEngine, "SELECT json FROM revs WHERE sequence=?", selectArgs3 ); } } cursor2.Close(); cursor2 = null; } // Get the document properties, to pass to the map function: var contentOptions = DocumentContentOptions.None; if (noAttachments) { contentOptions |= DocumentContentOptions.NoAttachments; } var properties = Database.DocumentPropertiesFromJSON( json, docId, revId, false, sequence, DocumentContentOptions.None ); if (properties != null) { // Call the user-defined map() to emit new key/value // pairs from this revision: // This is the emit() block, which gets called from within the // user-defined map() block // that's called down below. var enclosingView = this; var thisSequence = sequence; var map = Map; if (map == null) { throw new CouchbaseLiteException("Map function is missing."); } EmitDelegate emitBlock = (key, value) => { // TODO: Do we need to do any null checks on key or value? try { var keyJson = Manager.GetObjectMapper().WriteValueAsString(key); var valueJson = value == null ? null : Manager.GetObjectMapper().WriteValueAsString(value); var insertValues = new ContentValues(); insertValues.Put("view_id", enclosingView.Id); insertValues["sequence"] = thisSequence; insertValues["key"] = keyJson; insertValues["value"] = valueJson; enclosingView.Database.StorageEngine.Insert("maps", null, insertValues); } catch (Exception e) { Log.E(Database.Tag, "Error emitting", e); } }; map(properties, emitBlock); } } else { keepGoing = cursor.MoveToNext(); } } // Finally, record the last revision sequence number that was // indexed: var updateValues = new ContentValues(); updateValues["lastSequence"] = dbMaxSequence; var whereArgs_1 = new string[] { Id.ToString() }; Database.StorageEngine.Update("views", updateValues, "view_id=?", whereArgs_1); // FIXME actually count number added :) Log.V(Database.Tag, "...Finished re-indexing view {0} up to sequence {1} (deleted {2} added ?)", Name, Convert.ToString(dbMaxSequence), deleted); result.SetCode(StatusCode.Ok); return(true); }); } catch (Exception e) { throw new CouchbaseLiteException(e, new Status(StatusCode.DbError)); } finally { if (cursor2 != null) { cursor2.Close(); } if (cursor != null) { cursor.Close(); } if (!result.IsSuccessful) { Log.W(Database.Tag, "Failed to rebuild view {0}:{1}", Name, result.GetCode()); } } }
public void UpdateIndex() { Log.V(Log.TagView, "Re-indexing view: %s", name); System.Diagnostics.Debug.Assert((mapBlock != null)); if (GetViewId() <= 0) { string msg = string.Format("getViewId() < 0"); throw new CouchbaseLiteException(msg, new Status(Status.NotFound)); } database.BeginTransaction(); Status result = new Status(Status.InternalServerError); Cursor cursor = null; try { long lastSequence = GetLastSequenceIndexed(); long dbMaxSequence = database.GetLastSequenceNumber(); if (lastSequence == dbMaxSequence) { // nothing to do (eg, kCBLStatusNotModified) Log.V(Log.TagView, "lastSequence (%s) == dbMaxSequence (%s), nothing to do", lastSequence , dbMaxSequence); result.SetCode(Status.NotModified); return; } // First remove obsolete emitted results from the 'maps' table: long sequence = lastSequence; if (lastSequence < 0) { string msg = string.Format("lastSequence < 0 (%s)", lastSequence); throw new CouchbaseLiteException(msg, new Status(Status.InternalServerError)); } if (lastSequence == 0) { // If the lastSequence has been reset to 0, make sure to remove // any leftover rows: string[] whereArgs = new string[] { Sharpen.Extensions.ToString(GetViewId()) }; database.GetDatabase().Delete("maps", "view_id=?", whereArgs); } else { // Delete all obsolete map results (ones from since-replaced // revisions): string[] args = new string[] { Sharpen.Extensions.ToString(GetViewId()), System.Convert.ToString (lastSequence), System.Convert.ToString(lastSequence) }; database.GetDatabase().ExecSQL("DELETE FROM maps WHERE view_id=? AND sequence IN (" + "SELECT parent FROM revs WHERE sequence>? " + "AND parent>0 AND parent<=?)", args); } int deleted = 0; cursor = database.GetDatabase().RawQuery("SELECT changes()", null); cursor.MoveToNext(); deleted = cursor.GetInt(0); cursor.Close(); // This is the emit() block, which gets called from within the // user-defined map() block // that's called down below. AbstractTouchMapEmitBlock emitBlock = new _AbstractTouchMapEmitBlock_428(this); //Log.v(Log.TAG_VIEW, " emit(" + keyJson + ", " // + valueJson + ")"); // find a better way to propagate this back // Now scan every revision added since the last time the view was // indexed: string[] selectArgs = new string[] { System.Convert.ToString(lastSequence) }; cursor = database.GetDatabase().RawQuery("SELECT revs.doc_id, sequence, docid, revid, json, no_attachments FROM revs, docs " + "WHERE sequence>? AND current!=0 AND deleted=0 " + "AND revs.doc_id = docs.doc_id " + "ORDER BY revs.doc_id, revid DESC", selectArgs); long lastDocID = 0; bool keepGoing = cursor.MoveToNext(); while (keepGoing) { long docID = cursor.GetLong(0); if (docID != lastDocID) { // Only look at the first-iterated revision of any document, // because this is the // one with the highest revid, hence the "winning" revision // of a conflict. lastDocID = docID; // Reconstitute the document as a dictionary: sequence = cursor.GetLong(1); string docId = cursor.GetString(2); if (docId.StartsWith("_design/")) { // design docs don't get indexed! keepGoing = cursor.MoveToNext(); continue; } string revId = cursor.GetString(3); byte[] json = cursor.GetBlob(4); bool noAttachments = cursor.GetInt(5) > 0; while ((keepGoing = cursor.MoveToNext()) && cursor.GetLong(0) == docID) { } // Skip rows with the same doc_id -- these are losing conflicts. if (lastSequence > 0) { // Find conflicts with documents from previous indexings. string[] selectArgs2 = new string[] { System.Convert.ToString(docID), System.Convert.ToString (lastSequence) }; Cursor cursor2 = database.GetDatabase().RawQuery("SELECT revid, sequence FROM revs " + "WHERE doc_id=? AND sequence<=? AND current!=0 AND deleted=0 " + "ORDER BY revID DESC " + "LIMIT 1", selectArgs2); if (cursor2.MoveToNext()) { string oldRevId = cursor2.GetString(0); // This is the revision that used to be the 'winner'. // Remove its emitted rows: long oldSequence = cursor2.GetLong(1); string[] args = new string[] { Sharpen.Extensions.ToString(GetViewId()), System.Convert.ToString (oldSequence) }; database.GetDatabase().ExecSQL("DELETE FROM maps WHERE view_id=? AND sequence=?", args); if (RevisionInternal.CBLCompareRevIDs(oldRevId, revId) > 0) { // It still 'wins' the conflict, so it's the one that // should be mapped [again], not the current revision! revId = oldRevId; sequence = oldSequence; string[] selectArgs3 = new string[] { System.Convert.ToString(sequence) }; json = Utils.ByteArrayResultForQuery(database.GetDatabase(), "SELECT json FROM revs WHERE sequence=?" , selectArgs3); } } } // Get the document properties, to pass to the map function: EnumSet <Database.TDContentOptions> contentOptions = EnumSet.NoneOf <Database.TDContentOptions >(); if (noAttachments) { contentOptions.AddItem(Database.TDContentOptions.TDNoAttachments); } IDictionary <string, object> properties = database.DocumentPropertiesFromJSON(json , docId, revId, false, sequence, contentOptions); if (properties != null) { // Call the user-defined map() to emit new key/value // pairs from this revision: emitBlock.SetSequence(sequence); mapBlock.Map(properties, emitBlock); } } } // Finally, record the last revision sequence number that was // indexed: ContentValues updateValues = new ContentValues(); updateValues.Put("lastSequence", dbMaxSequence); string[] whereArgs_1 = new string[] { Sharpen.Extensions.ToString(GetViewId()) }; database.GetDatabase().Update("views", updateValues, "view_id=?", whereArgs_1); // FIXME actually count number added :) Log.V(Log.TagView, "Finished re-indexing view: %s " + " up to sequence %s" + " (deleted %s added ?)" , name, dbMaxSequence, deleted); result.SetCode(Status.Ok); } catch (SQLException e) { throw new CouchbaseLiteException(e, new Status(Status.DbError)); } finally { if (cursor != null) { cursor.Close(); } if (!result.IsSuccessful()) { Log.W(Log.TagView, "Failed to rebuild view %s. Result code: %d", name, result.GetCode ()); } if (database != null) { database.EndTransaction(result.IsSuccessful()); } } }