public Artist[] Init(TestEnvironment environment) { if(_artists != null && _artists.Any()) return _artists; IntegrationTestsRuntime.EnsureCleanEnvironment(); _artists = ClientTestData.Artists.CreateArtists(10); using (var client = IntegrationTestsRuntime.CreateDbClient()) { var bulk = new BulkRequest(); bulk.Include(_artists.Select(i => client.Entities.Serializer.Serialize(i)).ToArray()); var bulkResponse = client.Documents.BulkAsync(bulk).Result; foreach (var row in bulkResponse.Rows) { var artist = _artists.Single(i => i.ArtistId == row.Id); client.Entities.Reflector.RevMember.SetValueTo(artist, row.Rev); } client.Documents.PostAsync(ClientTestData.Views.ArtistsViews).Wait(); var queryRequests = ClientTestData.Views.AllViewIds.Select(id => new QueryViewRequest(id).Configure(q => q.Stale(Stale.UpdateAfter))); var queries = queryRequests.Select(q => client.Views.QueryAsync(q) as Task).ToArray(); Task.WaitAll(queries); } return _artists; }
public Artist[] Init(TestEnvironment environment) { if(_artists != null && _artists.Any()) return _artists; IntegrationTestsRuntime.EnsureCleanEnvironment(); _artists = ClientTestData.Artists.CreateArtists(2); using (var client = IntegrationTestsRuntime.CreateDbClient()) { var bulk = new BulkRequest(); bulk.Include(_artists.Select(i => client.Entities.Serializer.Serialize(i)).ToArray()); var bulkResponse = client.Documents.BulkAsync(bulk).Result; foreach (var row in bulkResponse.Rows) { var artist = _artists.Single(i => i.ArtistId == row.Id); client.Entities.Reflector.RevMember.SetValueTo(artist, row.Rev); } client.Documents.PostAsync(ClientTestData.Shows.ArtistsShows).Wait(); } return _artists; }
public ViewsFixture() { Artists = ClientTestData.Artists.CreateArtists(10); Client = IntegrationTestsRuntime.CreateNormalClient(); var bulk = new BulkRequest(); bulk.Include(Artists.Select(i => Client.Entities.Serializer.Serialize(i)).ToArray()); var bulkResponse = Client.Documents.BulkAsync(bulk).Result; foreach (var row in bulkResponse.Rows) { var artist = Artists.Single(i => i.ArtistId == row.Id); Client.Entities.Reflector.RevMember.SetValueTo(artist, row.Rev); } var tmp = Client.Documents.PostAsync(ClientTestData.Views.ArtistsViews).Result; var touchView1 = new QueryViewRequest(ClientTestData.Views.ArtistsAlbumsViewId).Configure(q => q.Stale(Stale.UpdateAfter)); var touchView2 = new QueryViewRequest(ClientTestData.Views.ArtistsNameNoValueViewId).Configure(q => q.Stale(Stale.UpdateAfter)); var touchView3 = new QueryViewRequest(ClientTestData.Views.ArtistsTotalNumOfAlbumsViewId).Configure(q => q.Stale(Stale.UpdateAfter)); Client.Views.QueryAsync(touchView1).Wait(); Client.Views.QueryAsync(touchView2).Wait(); Client.Views.QueryAsync(touchView3).Wait(); }
public async Task buildIndex(StreamReader input) { string line; long pos = 0, lineNum = 0; long bytePos = 0, lastBytePos = 0; string id; Stopwatch startWatch = new Stopwatch(), loopWatch = new Stopwatch(); startWatch.Start(); loopWatch.Start(); BulkRequest req = new BulkRequest(); while (!input.EndOfStream) { line = input.ReadLine(); lastBytePos = bytePos; bytePos += line.Length; id = Regex.Match(line, regPattern).Value; id = id.Trim("\"id\":\"".ToCharArray()); //await clientIndex.Entities.PostAsync(new Index { _id = id, line = lineNum, position = pos}); var index = new Index { _id = id, line = lineNum, position = pos }; string s = JsonConvert.SerializeObject(index); req.Include(s); //Console.WriteLine(line); Console.WriteLine("ID: {0}\n line number: {1}\nbyte pos: {2}\nlast byte pos: {3}\nbyte difference: {4}\nline size: {5}", id, lineNum, bytePos, lastBytePos, bytePos - lastBytePos, line.Length); if (lineNum % 10000 == 0) { await clientIndex.Documents.BulkAsync(req); req = new BulkRequest(); //Console.Clear(); //Console.WriteLine("Stats\nProcessed: {0}", lineNum); //Console.WriteLine("Loop Time: {0}", loopWatch.Elapsed); //Console.WriteLine("Total Time: {0}", startWatch.Elapsed); loopWatch.Restart(); } lineNum++; pos += line.Length + 1; //+1 for newline char } }
internal void Init(TestEnvironment environment) { environment.IsAgainstCloudant().Should().Be(true); if (_blogs != null && _blogs.Any()) return; IntegrationTestsRuntime.EnsureCleanEnvironment(); _blogs = CloudantTestData.Blogs.CreateAll(); using (var client = IntegrationTestsRuntime.CreateDbClient() as IMyCouchCloudantClient) { var bulk = new BulkRequest(); bulk.Include(_blogs.Select(i => client.Entities.Serializer.Serialize(i)).ToArray()); var bulkResponse = client.Documents.BulkAsync(bulk).Result; CreateIndexes(client); } }
public SearchFixture() { Animals = CloudantTestData.Animals.CreateAll(); Client = IntegrationTestsRuntime.CreateCloudantClient(); var bulk = new BulkRequest(); bulk.Include(Animals.Select(i => Client.Entities.Serializer.Serialize(i)).ToArray()); var bulkResponse = Client.Documents.BulkAsync(bulk).Result; foreach (var row in bulkResponse.Rows) { var animal = Animals.Single(i => i.AnimalId == row.Id); Client.Entities.Reflector.RevMember.SetValueTo(animal, row.Rev); } Client.Documents.PostAsync(CloudantTestData.Views.Views101).Wait(); var queries = CloudantTestData.Views.AllViewIds.Select(id => new QueryViewRequest(id).Configure(q => q.Stale(Stale.UpdateAfter))); foreach (var query in queries) Client.Views.QueryAsync(query).Wait(); }
public async Task<BatchResponse> Store(string bucketName, BatchSet batch) { using (var client = new MyCouchClient(DbServerUrl, bucketName)) { var dbExists= await client.Database.HeadAsync(); if (dbExists.IsSuccess) { BulkRequest bulkRequest = new BulkRequest(); DateTime start = DateTime.Now; int size = 0; SiaqodbDocument crObjForUpdateViews = null; if (batch.ChangedDocuments != null) { foreach (SiaqodbDocument obj in batch.ChangedDocuments) { if (obj != null) { if (crObjForUpdateViews == null) crObjForUpdateViews = obj; await CheckTagsViews(client, bucketName, obj.Tags); CouchDBDocument doc = Mapper.ToCouchDBDoc(obj); var serializedObject = client.Serializer.Serialize<CouchDBDocument>(doc); bulkRequest.Include(serializedObject); size += serializedObject.Length; } } } if (batch.DeletedDocuments != null) { foreach (DeletedDocument obj in batch.DeletedDocuments) { if (obj != null) { if (obj.Version != null)//otherwise means is a non-existing object { bulkRequest.Delete(obj.Key, obj.Version); } } } } var response = await client.Documents.BulkAsync(bulkRequest); if (response.IsSuccess) { var cnorResponse = new BatchResponse(); if (response.Rows != null) { cnorResponse.BatchItemResponses = new List<BatchItemResponse>(); SyncLogItem syncLogItem = new SyncLogItem(); syncLogItem.KeyVersion = new Dictionary<string, string>(); foreach (var row in response.Rows) { BatchItemResponse wresp = new BatchItemResponse(); if (!string.IsNullOrEmpty(row.Error)) { cnorResponse.ItemsWithErrors++; } wresp.Error = row.Error; wresp.ErrorDesc = row.Reason; wresp.Key = row.Id; wresp.Version = row.Rev; cnorResponse.BatchItemResponses.Add(wresp); if (string.IsNullOrEmpty(row.Error)) { syncLogItem.KeyVersion.Add(row.Id, row.Rev); } } if (syncLogItem.KeyVersion.Count > 0) { syncLogItem.TimeInserted = DateTime.UtcNow; using (var clientLog = new MyCouchClient(DbServerUrl, SyncLogBucket)) { string serLogItem = Newtonsoft.Json.JsonConvert.SerializeObject(syncLogItem); var logResp = await clientLog.Documents.PostAsync(serLogItem); cnorResponse.UploadAnchor = logResp.Id; } } } if (crObjForUpdateViews != null) { await this.StartRebuildViews(client, crObjForUpdateViews); } return cnorResponse; } else CheckBucketNotFound(bucketName, response); } else if (dbExists.StatusCode == System.Net.HttpStatusCode.NotFound) { throw new BucketNotFoundException(bucketName); } return null; } }
public async Task buildCorpusIndex(FileStream input) { long i = 0, line = 0, linePos = 0; int bit = 0; string indexStr, id; BulkRequest req = new BulkRequest(); Stopwatch startWatch = new Stopwatch(), loopWatch = new Stopwatch(); startWatch.Start(); loopWatch.Start(); while (bit > -1) { StringBuilder sb = new StringBuilder(); linePos = i; while ((bit = input.ReadByte()) != '\n' && bit > -1) { i++; sb.Append((char)bit); } //+1 for new line bit i++; indexStr = sb.ToString(); id = Regex.Match(indexStr, regPattern).Value; id = id.Trim("\"id\":\"".ToCharArray()); var index = new Index { _id = id, line = line, position = linePos }; //Console.WriteLine(index.ToString()); string s = JsonConvert.SerializeObject(index); req.Include(s); line++; //Store and print stats every 1k lines if (line % 100000 == 0) { await clientIndex.Documents.BulkAsync(req); req = new BulkRequest(); Console.Clear(); Console.WriteLine("Stats\nProcessed: {0}", line); Console.WriteLine("Loop Time: {0}", loopWatch.Elapsed); Console.WriteLine("Total Time: {0}", startWatch.Elapsed); loopWatch.Restart(); } } }