public async Task<BatchSet> GetChanges(string bucketName, Filter query, int limit, string anchor, string uploadAnchor)
{
var db = client.GetDatabase("local");
var collection = db.GetCollection<BsonDocument>("oplog.rs");
var filterBuilder = Builders<BsonDocument>.Filter;
FilterDefinition<BsonDocument> filter = filterBuilder.Eq("ns", DbName + "." + bucketName);
if (anchor != null)
{
filter = filter & filterBuilder.Gt("ts", new BsonTimestamp(Convert.ToInt64(anchor)));
}
var docs = await collection.Find(filter).Limit(limit).ToListAsync();
BatchSet bs = new BatchSet();
SyncLogItem logItem = null;
if (docs != null && docs.Count > 0 && !string.IsNullOrEmpty(uploadAnchor))
{
logItem = await GetSyncLogItem(uploadAnchor);
}
int i = 0;
foreach (var doc in docs)
{
if (i == docs.Count - 1)
{
bs.Anchor = doc["ts"].AsBsonTimestamp.ToString();
}
i++;
if (doc["op"] == "i" || doc["op"] == "u")
{
this.AddChangedDoc(bs, doc, logItem, query);
}
else if (doc["op"] == "d")
{
this.AddDeletedDoc(bs, doc, logItem);
}
}
return bs;
}
private void AddChangedDoc(BatchSet bs, BsonDocument doc, SyncLogItem logItem, Filter query)
{
if (bs.ChangedDocuments == null)
bs.ChangedDocuments = new List<SiaqodbDocument>();
var nestedDoc = doc["o"].AsBsonDocument;
if (logItem != null && logItem.KeyVersion != null && logItem.KeyVersion.ContainsKey(nestedDoc["_id"].AsString) && logItem.KeyVersion[nestedDoc["_id"].AsString] == nestedDoc["_rev"].AsString)
return;
if (OutOfFilter(query, nestedDoc))
return;
SiaqodbDocument siaqodbDoc = Mapper.ToSiaqodbDocument(nestedDoc);
bs.ChangedDocuments.Add(siaqodbDoc);
}
public async Task<BatchResponse> Store(string bucketName, BatchSet value)
{
var db = client.GetDatabase(DbName);
var collection = db.GetCollection<BsonDocument>(bucketName);
var response = new BatchResponse();
response.BatchItemResponses = new List<BatchItemResponse>();
SyncLogItem syncLogItem = new SyncLogItem();
syncLogItem.KeyVersion = new Dictionary<string, string>();
foreach (var document in value.ChangedDocuments)
{
var newDoc = Mapper.ToBsonDoc(document);
newDoc["_rev"] = this.GenerateNewVersion();
try
{
var exists = await collection.Find(filter: new BsonDocument { { "_id", document.Key } }).FirstOrDefaultAsync();
if (exists == null && !string.IsNullOrEmpty( document.Version))//somebody else deleted the doc-> conflict
{
BatchItemResponse respWithError = BuildResponseWithError(document.Key, document.Version, "conflict");
response.ItemsWithErrors++;
response.BatchItemResponses.Add(respWithError);
continue;
}
var result = await collection.ReplaceOneAsync(filter: new BsonDocument { { "_id", document.Key }, { "_rev", BsonTypeMapper.MapToBsonValue(document.Version) } },
options: new UpdateOptions { IsUpsert = true },
replacement: newDoc);
BatchItemResponse itemResp = new BatchItemResponse
{
Key = document.Key,
Version = newDoc["_rev"].AsString
};
response.BatchItemResponses.Add(itemResp);
syncLogItem.KeyVersion.Add(itemResp.Key, itemResp.Version);
}
catch (MongoWriteException ex)
{
string error = ex.Message;
if (ex.Message.Contains("duplicate key"))//conflict
{
error = "conflict";
}
var itemResp = BuildResponseWithError(document.Key, document.Version, error);
response.ItemsWithErrors++;
response.BatchItemResponses.Add(itemResp);
}
}
foreach (var document in value.DeletedDocuments)
{
BatchItemResponse itemResp = new BatchItemResponse()
{
Key = document.Key,
Version = document.Version
};
var del=await collection.DeleteOneAsync(filter: new BsonDocument { { "_id", document.Key }, { "_rev", BsonTypeMapper.MapToBsonValue(document.Version) } });
if (del.DeletedCount == 0)
{
var docFromDB= collection.Find(filter: new BsonDocument { { "_id", document.Key } }).FirstOrDefaultAsync();
if (docFromDB != null)
{
itemResp.Error = "conflict";
itemResp.ErrorDesc = "conflict";
response.ItemsWithErrors++;
}
}
response.BatchItemResponses.Add(itemResp);
}
//store uploadedAnchor
if (syncLogItem.KeyVersion.Count > 0)
{
syncLogItem.TimeInserted = DateTime.UtcNow;
BsonDocument syncLogDoc = new BsonDocument();
syncLogDoc["KeyVersion"] = new BsonDocument(syncLogItem.KeyVersion);
syncLogDoc["TimeInserted"] = syncLogItem.TimeInserted;
var collectionLog = db.GetCollection<BsonDocument>(SyncLogBucket);
await collectionLog.InsertOneAsync(syncLogDoc);
response.UploadAnchor = syncLogDoc["_id"].ToString();
}
return response;
}
private void AddDeletedDoc(BatchSet bs, BsonDocument doc, SyncLogItem logItem)
{
if (bs.DeletedDocuments == null)
bs.DeletedDocuments = new List<DeletedDocument>();
//check uploaded anchor- means cliet just uploaded this record and we should not return back
if (logItem != null && logItem.KeyVersion != null && logItem.KeyVersion.ContainsKey(doc["o"].AsBsonDocument["_id"].AsString))
return;
DeletedDocument delDoc = new DeletedDocument();
delDoc.Key = doc["o"].AsBsonDocument["_id"].AsString;
bs.DeletedDocuments.Add(delDoc);
}
public async Task<BatchResponse> Store(string bucketName, BatchSet batch)
{
using (var client = new MyCouchClient(DbServerUrl, bucketName))
{
var dbExists= await client.Database.HeadAsync();
if (dbExists.IsSuccess)
{
BulkRequest bulkRequest = new BulkRequest();
DateTime start = DateTime.Now;
int size = 0;
SiaqodbDocument crObjForUpdateViews = null;
if (batch.ChangedDocuments != null)
{
foreach (SiaqodbDocument obj in batch.ChangedDocuments)
{
if (obj != null)
{
if (crObjForUpdateViews == null)
crObjForUpdateViews = obj;
await CheckTagsViews(client, bucketName, obj.Tags);
CouchDBDocument doc = Mapper.ToCouchDBDoc(obj);
var serializedObject = client.Serializer.Serialize<CouchDBDocument>(doc);
bulkRequest.Include(serializedObject);
size += serializedObject.Length;
}
}
}
if (batch.DeletedDocuments != null)
{
foreach (DeletedDocument obj in batch.DeletedDocuments)
{
if (obj != null)
{
if (obj.Version != null)//otherwise means is a non-existing object
{
bulkRequest.Delete(obj.Key, obj.Version);
}
}
}
}
var response = await client.Documents.BulkAsync(bulkRequest);
if (response.IsSuccess)
{
var cnorResponse = new BatchResponse();
if (response.Rows != null)
{
cnorResponse.BatchItemResponses = new List<BatchItemResponse>();
SyncLogItem syncLogItem = new SyncLogItem();
syncLogItem.KeyVersion = new Dictionary<string, string>();
foreach (var row in response.Rows)
{
BatchItemResponse wresp = new BatchItemResponse();
if (!string.IsNullOrEmpty(row.Error))
{
cnorResponse.ItemsWithErrors++;
}
wresp.Error = row.Error;
wresp.ErrorDesc = row.Reason;
wresp.Key = row.Id;
wresp.Version = row.Rev;
cnorResponse.BatchItemResponses.Add(wresp);
if (string.IsNullOrEmpty(row.Error))
{
syncLogItem.KeyVersion.Add(row.Id, row.Rev);
}
}
if (syncLogItem.KeyVersion.Count > 0)
{
syncLogItem.TimeInserted = DateTime.UtcNow;
using (var clientLog = new MyCouchClient(DbServerUrl, SyncLogBucket))
{
string serLogItem = Newtonsoft.Json.JsonConvert.SerializeObject(syncLogItem);
var logResp = await clientLog.Documents.PostAsync(serLogItem);
cnorResponse.UploadAnchor = logResp.Id;
}
}
}
if (crObjForUpdateViews != null)
{
await this.StartRebuildViews(client, crObjForUpdateViews);
}
return cnorResponse;
}
else CheckBucketNotFound(bucketName, response);
}
else if (dbExists.StatusCode == System.Net.HttpStatusCode.NotFound)
{
throw new BucketNotFoundException(bucketName);
}
return null;
}
}
private void AddDeletedDoc(BatchSet changeSet, ChangesResponse<string>.Row row, SyncLogItem logItem)
{
if (changeSet.DeletedDocuments == null)
changeSet.DeletedDocuments = new List<DeletedDocument>();
DeletedDocument delObj = new DeletedDocument() { Key = row.Id, Version = row.Changes[0].Rev };
//check uploaded anchor- means cliet just uploaded this record and we should not return back
if (logItem != null && logItem.KeyVersion != null && logItem.KeyVersion.ContainsKey(delObj.Key) && logItem.KeyVersion[delObj.Key] == delObj.Version)
return;
changeSet.DeletedDocuments.Add(delObj);
}
private void AddChangedDoc(BatchSet changeSet, ChangesResponse<string>.Row row, SyncLogItem logItem, Filter query, MyCouch.Serialization.ISerializer serializer)
{
if (changeSet.ChangedDocuments == null)
changeSet.ChangedDocuments = new List<SiaqodbDocument>();
CouchDBDocument co = serializer.Deserialize<CouchDBDocument>(row.IncludedDoc);
if (co._id.StartsWith("_design/"))
return;
//check uploaded anchor- means cliet just uploaded this record and we should not return back
if (logItem != null && logItem.KeyVersion != null && logItem.KeyVersion.ContainsKey(co._id) && logItem.KeyVersion[co._id] == co._rev)
return;
if (OutOfFilter(query, co))
return;
changeSet.ChangedDocuments.Add(Mapper.ToSiaqodbDocument(co));
}
public async Task<BatchSet> GetChanges(string bucketName, Filter query, int limit, string anchor,string uploadAnchor)
{
using (var client = new MyCouchClient(DbServerUrl, bucketName))
{
GetChangesRequest changesReq = new GetChangesRequest();
changesReq.Since = anchor;
changesReq.Limit = limit;
changesReq.IncludeDocs = true;
var response = await client.Changes.GetAsync(changesReq);
if (response.IsSuccess)
{
BatchSet changeSet = new BatchSet();
if (response.Results != null)
{
SyncLogItem logItem = await this.GetSyncLogItem(uploadAnchor);
foreach (var row in response.Results)
{
if (row.Deleted)
{
this.AddDeletedDoc(changeSet, row, logItem);
}
else
{
this.AddChangedDoc(changeSet, row, logItem, query,client.Serializer );
}
}
changeSet.Anchor = response.LastSeq;
}
return changeSet;
}
else CheckBucketNotFound(bucketName, response);
return null;
}
}
