public Artist[] Init(TestEnvironment environment)
{
if(_artists != null && _artists.Any())
return _artists;
IntegrationTestsRuntime.EnsureCleanEnvironment();
_artists = ClientTestData.Artists.CreateArtists(2);
using (var client = IntegrationTestsRuntime.CreateDbClient())
{
var bulk = new BulkRequest();
bulk.Include(_artists.Select(i => client.Entities.Serializer.Serialize(i)).ToArray());
var bulkResponse = client.Documents.BulkAsync(bulk).Result;
foreach (var row in bulkResponse.Rows)
{
var artist = _artists.Single(i => i.ArtistId == row.Id);
client.Entities.Reflector.RevMember.SetValueTo(artist, row.Rev);
}
client.Documents.PostAsync(ClientTestData.Shows.ArtistsShows).Wait();
}
return _artists;
}
public ViewsFixture()
{
Artists = ClientTestData.Artists.CreateArtists(10);
Client = IntegrationTestsRuntime.CreateNormalClient();
var bulk = new BulkRequest();
bulk.Include(Artists.Select(i => Client.Entities.Serializer.Serialize(i)).ToArray());
var bulkResponse = Client.Documents.BulkAsync(bulk).Result;
foreach (var row in bulkResponse.Rows)
{
var artist = Artists.Single(i => i.ArtistId == row.Id);
Client.Entities.Reflector.RevMember.SetValueTo(artist, row.Rev);
}
var tmp = Client.Documents.PostAsync(ClientTestData.Views.ArtistsViews).Result;
var touchView1 = new QueryViewRequest(ClientTestData.Views.ArtistsAlbumsViewId).Configure(q => q.Stale(Stale.UpdateAfter));
var touchView2 = new QueryViewRequest(ClientTestData.Views.ArtistsNameNoValueViewId).Configure(q => q.Stale(Stale.UpdateAfter));
var touchView3 = new QueryViewRequest(ClientTestData.Views.ArtistsTotalNumOfAlbumsViewId).Configure(q => q.Stale(Stale.UpdateAfter));
Client.Views.QueryAsync(touchView1).Wait();
Client.Views.QueryAsync(touchView2).Wait();
Client.Views.QueryAsync(touchView3).Wait();
}
public Artist[] Init(TestEnvironment environment)
{
if(_artists != null && _artists.Any())
return _artists;
IntegrationTestsRuntime.EnsureCleanEnvironment();
_artists = ClientTestData.Artists.CreateArtists(10);
using (var client = IntegrationTestsRuntime.CreateDbClient())
{
var bulk = new BulkRequest();
bulk.Include(_artists.Select(i => client.Entities.Serializer.Serialize(i)).ToArray());
var bulkResponse = client.Documents.BulkAsync(bulk).Result;
foreach (var row in bulkResponse.Rows)
{
var artist = _artists.Single(i => i.ArtistId == row.Id);
client.Entities.Reflector.RevMember.SetValueTo(artist, row.Rev);
}
client.Documents.PostAsync(ClientTestData.Views.ArtistsViews).Wait();
var queryRequests = ClientTestData.Views.AllViewIds.Select(id => new QueryViewRequest(id).Configure(q => q.Stale(Stale.UpdateAfter)));
var queries = queryRequests.Select(q => client.Views.QueryAsync(q) as Task).ToArray();
Task.WaitAll(queries);
}
return _artists;
}
public virtual HttpRequest Create(BulkRequest request)
{
Ensure.That(request, "request").IsNotNull();
return new HttpRequest(HttpMethod.Post, GenerateRelativeUrl(request))
.SetRequestTypeHeader(request.GetType())
.SetJsonContent(GenerateRequestBody(request));
}
public virtual async Task<BulkResponse> BulkAsync(BulkRequest request)
{
Ensure.That(request, "request").IsNotNull();
using (var httpRequest = CreateHttpRequest(request))
{
using (var res = await SendAsync(httpRequest).ForAwait())
{
return ProcessBulkResponse(res);
}
}
}
public async Task buildIndex(StreamReader input) {
string line;
long pos = 0, lineNum = 0;
long bytePos = 0, lastBytePos = 0;
string id;
Stopwatch startWatch = new Stopwatch(), loopWatch = new Stopwatch();
startWatch.Start();
loopWatch.Start();
BulkRequest req = new BulkRequest();
while (!input.EndOfStream) {
line = input.ReadLine();
lastBytePos = bytePos;
bytePos += line.Length;
id = Regex.Match(line, regPattern).Value;
id = id.Trim("\"id\":\"".ToCharArray());
//await clientIndex.Entities.PostAsync(new Index { _id = id, line = lineNum, position = pos});
var index = new Index { _id = id, line = lineNum, position = pos };
string s = JsonConvert.SerializeObject(index);
req.Include(s);
//Console.WriteLine(line);
Console.WriteLine("ID: {0}\n line number: {1}\nbyte pos: {2}\nlast byte pos: {3}\nbyte difference: {4}\nline size: {5}",
id, lineNum, bytePos, lastBytePos, bytePos - lastBytePos, line.Length);
if (lineNum % 10000 == 0) {
await clientIndex.Documents.BulkAsync(req);
req = new BulkRequest();
//Console.Clear();
//Console.WriteLine("Stats\nProcessed: {0}", lineNum);
//Console.WriteLine("Loop Time: {0}", loopWatch.Elapsed);
//Console.WriteLine("Total Time: {0}", startWatch.Elapsed);
loopWatch.Restart();
}
lineNum++;
pos += line.Length + 1; //+1 for newline char
}
}
internal static void ClearAllDocuments(this IClient client)
{
var query = new QuerySystemViewRequest("_all_docs");
var response = client.Views.QueryAsync<dynamic>(query).Result;
if (!response.IsEmpty)
{
var bulkRequest = new BulkRequest();
foreach (var row in response.Rows)
bulkRequest.Delete(row.Id, row.Value.rev.ToString());
client.Documents.BulkAsync(bulkRequest).Wait();
}
}
protected virtual string GenerateRequestBody(BulkRequest request)
{
var sb = new StringBuilder();
var documents = request.GetDocuments();
sb.Append("{\"docs\":[");
for (var i = 0; i < documents.Length; i++)
{
sb.Append(documents[i]);
if (i < documents.Length - 1)
sb.Append(",");
}
sb.Append("]}");
return sb.ToString();
}
internal void Init(TestEnvironment environment)
{
environment.IsAgainstCloudant().Should().Be(true);
if (_blogs != null && _blogs.Any())
return;
IntegrationTestsRuntime.EnsureCleanEnvironment();
_blogs = CloudantTestData.Blogs.CreateAll();
using (var client = IntegrationTestsRuntime.CreateDbClient() as IMyCouchCloudantClient)
{
var bulk = new BulkRequest();
bulk.Include(_blogs.Select(i => client.Entities.Serializer.Serialize(i)).ToArray());
var bulkResponse = client.Documents.BulkAsync(bulk).Result;
CreateIndexes(client);
}
}
public SearchFixture()
{
Animals = CloudantTestData.Animals.CreateAll();
Client = IntegrationTestsRuntime.CreateCloudantClient();
var bulk = new BulkRequest();
bulk.Include(Animals.Select(i => Client.Entities.Serializer.Serialize(i)).ToArray());
var bulkResponse = Client.Documents.BulkAsync(bulk).Result;
foreach (var row in bulkResponse.Rows)
{
var animal = Animals.Single(i => i.AnimalId == row.Id);
Client.Entities.Reflector.RevMember.SetValueTo(animal, row.Rev);
}
Client.Documents.PostAsync(CloudantTestData.Views.Views101).Wait();
var queries = CloudantTestData.Views.AllViewIds.Select(id => new QueryViewRequest(id).Configure(q => q.Stale(Stale.UpdateAfter)));
foreach (var query in queries)
Client.Views.QueryAsync(query).Wait();
}
public void CleanupDatabase()
{
var view = this.db.Views.QueryAsync(new QueryViewRequest("person", "all_rev")).Result;
var request = new BulkRequest();
if (view.TotalRows > 0)
{
var rows = view.Rows.Select(x => new
{
Id = x.Id,
Rev = x.Value.TrimStart('\"').TrimEnd('\"'),
});
foreach (var row in rows)
{
request = request.Delete(row.Id, row.Rev);
}
/*
rows.Batch(1000).ForEach(x =>
{
Task.WaitAll(x.Select(w => this.db.Documents.DeleteAsync(new DeleteDocumentRequest(w.Id, w.Rev))).ToArray());
});
*/
}
this.db.Documents.BulkAsync(request).Wait();
}
protected virtual string GenerateRequestBody(BulkRequest request)
{
var sb = new StringBuilder();
var documents = request.GetDocuments();
sb.Append("{");
if (request.AllOrNothing)
sb.Append("\"all_or_nothing\":true,");
if (!request.NewEdits)
sb.Append("\"new_edits\":false,");
sb.Append("\"docs\":[");
for (var i = 0; i < documents.Length; i++)
{
sb.Append(documents[i]);
if (i < documents.Length - 1)
sb.Append(",");
}
sb.Append("]}");
return sb.ToString();
}
private static void BulkDelete(IMyCouchClient client, ViewQueryResponse<dynamic> response)
{
if (response.IsEmpty)
return;
var bulkRequest = new BulkRequest();
foreach (var row in response.Rows)
{
if (row.Id.ToLower() == "_design/_replicator")
continue;
bulkRequest.Delete(row.Id, row.Value.rev.ToString());
}
if(!bulkRequest.IsEmpty)
client.Documents.BulkAsync(bulkRequest).Wait();
}
public async Task buildCorpusIndex(FileStream input) {
long i = 0, line = 0, linePos = 0;
int bit = 0;
string indexStr, id;
BulkRequest req = new BulkRequest();
Stopwatch startWatch = new Stopwatch(), loopWatch = new Stopwatch();
startWatch.Start();
loopWatch.Start();
while (bit > -1) {
StringBuilder sb = new StringBuilder();
linePos = i;
while ((bit = input.ReadByte()) != '\n' && bit > -1) {
i++;
sb.Append((char)bit);
}
//+1 for new line bit
i++;
indexStr = sb.ToString();
id = Regex.Match(indexStr, regPattern).Value;
id = id.Trim("\"id\":\"".ToCharArray());
var index = new Index { _id = id, line = line, position = linePos };
//Console.WriteLine(index.ToString());
string s = JsonConvert.SerializeObject(index);
req.Include(s);
line++;
//Store and print stats every 1k lines
if (line % 100000 == 0) {
await clientIndex.Documents.BulkAsync(req);
req = new BulkRequest();
Console.Clear();
Console.WriteLine("Stats\nProcessed: {0}", line);
Console.WriteLine("Loop Time: {0}", loopWatch.Elapsed);
Console.WriteLine("Total Time: {0}", startWatch.Elapsed);
loopWatch.Restart();
}
}
}
protected virtual string GenerateRelativeUrl(BulkRequest request)
{
return "/_bulk_docs";
}
public async Task<BatchResponse> Store(string bucketName, BatchSet batch)
{
using (var client = new MyCouchClient(DbServerUrl, bucketName))
{
var dbExists= await client.Database.HeadAsync();
if (dbExists.IsSuccess)
{
BulkRequest bulkRequest = new BulkRequest();
DateTime start = DateTime.Now;
int size = 0;
SiaqodbDocument crObjForUpdateViews = null;
if (batch.ChangedDocuments != null)
{
foreach (SiaqodbDocument obj in batch.ChangedDocuments)
{
if (obj != null)
{
if (crObjForUpdateViews == null)
crObjForUpdateViews = obj;
await CheckTagsViews(client, bucketName, obj.Tags);
CouchDBDocument doc = Mapper.ToCouchDBDoc(obj);
var serializedObject = client.Serializer.Serialize<CouchDBDocument>(doc);
bulkRequest.Include(serializedObject);
size += serializedObject.Length;
}
}
}
if (batch.DeletedDocuments != null)
{
foreach (DeletedDocument obj in batch.DeletedDocuments)
{
if (obj != null)
{
if (obj.Version != null)//otherwise means is a non-existing object
{
bulkRequest.Delete(obj.Key, obj.Version);
}
}
}
}
var response = await client.Documents.BulkAsync(bulkRequest);
if (response.IsSuccess)
{
var cnorResponse = new BatchResponse();
if (response.Rows != null)
{
cnorResponse.BatchItemResponses = new List<BatchItemResponse>();
SyncLogItem syncLogItem = new SyncLogItem();
syncLogItem.KeyVersion = new Dictionary<string, string>();
foreach (var row in response.Rows)
{
BatchItemResponse wresp = new BatchItemResponse();
if (!string.IsNullOrEmpty(row.Error))
{
cnorResponse.ItemsWithErrors++;
}
wresp.Error = row.Error;
wresp.ErrorDesc = row.Reason;
wresp.Key = row.Id;
wresp.Version = row.Rev;
cnorResponse.BatchItemResponses.Add(wresp);
if (string.IsNullOrEmpty(row.Error))
{
syncLogItem.KeyVersion.Add(row.Id, row.Rev);
}
}
if (syncLogItem.KeyVersion.Count > 0)
{
syncLogItem.TimeInserted = DateTime.UtcNow;
using (var clientLog = new MyCouchClient(DbServerUrl, SyncLogBucket))
{
string serLogItem = Newtonsoft.Json.JsonConvert.SerializeObject(syncLogItem);
var logResp = await clientLog.Documents.PostAsync(serLogItem);
cnorResponse.UploadAnchor = logResp.Id;
}
}
}
if (crObjForUpdateViews != null)
{
await this.StartRebuildViews(client, crObjForUpdateViews);
}
return cnorResponse;
}
else CheckBucketNotFound(bucketName, response);
}
else if (dbExists.StatusCode == System.Net.HttpStatusCode.NotFound)
{
throw new BucketNotFoundException(bucketName);
}
return null;
}
}
protected virtual HttpRequest CreateHttpRequest(BulkRequest request)
{
return BulkHttpRequestFactory.Create(request);
}
