Esempio n. 1
0
        protected override Guid FlushBatch(List <RavenJObject> batch)
        {
            var sw = Stopwatch.StartNew();

            var results = _database.Batch(batch.Select(x =>
            {
                var metadata = x.Value <RavenJObject>("@metadata");
                var key      = metadata.Value <string>("@id");
                x.Remove("@metadata");
                return(new PutCommandData
                {
                    Document = x,
                    Etag = null,
                    Key = key,
                    Metadata = metadata
                });
            }).ToArray());

            ShowProgress("Wrote {0:#,#} documents in {1:#,#;;0} ms", batch.Count, sw.ElapsedMilliseconds);
            batch.Clear();


            if (results.Length == 0)
            {
                return(Guid.Empty);
            }

            return(results.Last().Etag.Value);
        }
Esempio n. 2
0
        protected override int SetupData()
        {
            DocumentDatabase.Put("Raven/Hilo/users", null, new RavenJObject(), new RavenJObject(), null);
            DocumentDatabase.Put("Raven/Hilo/posts", null, new RavenJObject(), new RavenJObject(), null);

            DocumentDatabase.Batch(new[]
            {
                new PutCommandData
                {
                    Document = new RavenJObject(),
                    Etag     = null,
                    Key      = "users/1",
                    Metadata = new RavenJObject(),
                    TransactionInformation = null
                },
                new PutCommandData
                {
                    Document = new RavenJObject(),
                    Etag     = null,
                    Key      = "posts/1",
                    Metadata = new RavenJObject(),
                    TransactionInformation = null
                },
                new PutCommandData
                {
                    Document = new RavenJObject(),
                    Etag     = null,
                    Key      = "posts/2",
                    Metadata = new RavenJObject(),
                    TransactionInformation = null
                },
                new PutCommandData
                {
                    Document = new RavenJObject(),
                    Etag     = null,
                    Key      = "posts/3",
                    Metadata = new RavenJObject(),
                    TransactionInformation = null
                },
                new PutCommandData
                {
                    Document = new RavenJObject(),
                    Etag     = null,
                    Key      = "posts/4",
                    Metadata = new RavenJObject(),
                    TransactionInformation = null
                },
                new PutCommandData
                {
                    Document = new RavenJObject(),
                    Etag     = null,
                    Key      = "posts/5",
                    Metadata = new RavenJObject(),
                    TransactionInformation = null
                },
            });

            return(8);
        }
Esempio n. 3
0
 public BatchResult[] Batch(ICommandData[] commandDatas)
 {
     foreach (var commandData in commandDatas)
     {
         commandData.TransactionInformation = GetTransactionInformation();
     }
     return(database.Batch(commandDatas));
 }
Esempio n. 4
0
 /// <summary>
 /// Executed the specified commands as a single batch
 /// </summary>
 /// <param name="commandDatas">The command data.</param>
 public BatchResult[] Batch(IEnumerable <ICommandData> commandDatas)
 {
     foreach (var commandData in commandDatas)
     {
         commandData.TransactionInformation = TransactionInformation;
     }
     CurrentOperationContext.Headers.Value = OperationsHeaders;
     return(database.Batch(commandDatas));
 }
 public BatchResult[] Batch(ICommandData[] commandDatas)
 {
     foreach (var commandData in commandDatas)
     {
         commandData.TransactionInformation = RavenTransactionAccessor.GetTransactionInformation();
     }
     CurrentRavenOperation.Headers.Value = OperationsHeaders;
     return(database.Batch(commandDatas));
 }
Esempio n. 6
0
        /// <summary>
        /// Executed the specified commands as a single batch
        /// </summary>
        /// <param name="commandDatas">The command data.</param>
        public BatchResult[] Batch(IEnumerable <ICommandData> commandDatas)
        {
            CurrentOperationContext.Headers.Value = OperationsHeaders;
            var batchResults = database.Batch(commandDatas.Select(cmd =>
            {
                cmd.TransactionInformation = TransactionInformation;
                return(cmd);
            }));

            if (batchResults != null)
            {
                foreach (var batchResult in batchResults.Where(batchResult => batchResult != null && batchResult.Metadata != null && batchResult.Metadata.IsSnapshot))
                {
                    batchResult.Metadata = (RavenJObject)batchResult.Metadata.CreateSnapshot();
                }
            }
            return(batchResults);
        }
Esempio n. 7
0
        protected override void FlushBatch(List <RavenJObject> batch)
        {
            var sw = Stopwatch.StartNew();

            _database.Batch(batch.Select(x =>
            {
                var metadata = x.Value <RavenJObject>("@metadata");
                var key      = metadata.Value <string>("@id");
                x.Remove("@metadata");
                return(new PutCommandData
                {
                    Document = x,
                    Etag = null,
                    Key = key,
                    Metadata = metadata
                });
            }).ToArray());

            ShowProgress("Wrote {0} documents in {1}", batch.Count, sw.ElapsedMilliseconds);
            ShowProgress(" in {0:#,#;;0} ms", sw.ElapsedMilliseconds);
            batch.Clear();
        }
Esempio n. 8
0
        public void GetDocumentAfterAnEtagWhileAddingDocsFromMultipleThreadsEnumeratesAllDocs()
        {
            var numberOfDocsAdded = 0;
            var threads           = new List <Thread>();

            try
            {
                for (var i = 0; i < 10; i++)
                {
                    var thread = new Thread(() =>
                    {
                        var cmds = new List <ICommandData>();
                        for (var k = 0; k < 100; k++)
                        {
                            var newId = Interlocked.Increment(ref numberOfDocsAdded);
                            cmds.Add(new PutCommandData
                            {
                                Document = new RavenJObject(),
                                Metadata = new RavenJObject(),
                                Key      = newId.ToString()
                            });
                        }
                        ;
                        db.Batch(cmds);
                    });
                    threads.Add(thread);
                    thread.Start();
                }

                var docs     = new List <string>();
                var lastEtag = Guid.Empty;
                var total    = 0;
                var stop     = false;
                do
                {
                    var etag          = lastEtag;
                    var jsonDocuments = new JsonDocument[0];
                    db.TransactionalStorage.Batch(actions =>
                    {
                        jsonDocuments = actions.Documents.GetDocumentsAfter(etag).Where(x => x != null).ToArray();
                    });
                    docs.AddRange(jsonDocuments.Select(x => x.Key));
                    total += jsonDocuments.Length;
                    if (jsonDocuments.Length > 0)
                    {
                        lastEtag = jsonDocuments.Last().Etag.Value;
                    }
                    if (stop)
                    {
                        break;
                    }
                    if (threads.All(x => !x.IsAlive))
                    {
                        stop = true;
                    }
                } while (true);

                Assert.Equal(numberOfDocsAdded, total);
            }
            finally
            {
                foreach (var thread in threads)
                {
                    thread.Join();
                }
            }
        }
        public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold)
        {
            var stopwatch = Stopwatch.StartNew();
            var items     = new List <ICommandData>(deletionBatchSize);

            try
            {
                var query = new IndexQuery
                {
                    Start          = 0,
                    DisableCaching = true,
                    Cutoff         = SystemTime.UtcNow,
                    PageSize       = deletionBatchSize,
                    Query          = $"LastModified:[* TO {expiryThreshold.Ticks}]",
                    FieldsToFetch  = new[]
                    {
                        "__document_id",
                    },
                    SortedFields = new[]
                    {
                        new SortedField("LastModified")
                        {
                            Field      = "LastModified",
                            Descending = false
                        }
                    }
                };
                var indexName = new ExpirySagaAuditIndex().IndexName;
                database.Query(indexName, query, database.WorkContext.CancellationToken,
                               doc =>
                {
                    var id = doc.Value <string>("__document_id");
                    if (string.IsNullOrEmpty(id))
                    {
                        return;
                    }

                    items.Add(new DeleteCommandData
                    {
                        Key = id
                    });
                });
            }
            catch (OperationCanceledException)
            {
                //Ignore
            }

            var deletionCount = 0;

            Chunker.ExecuteInChunks(items.Count, (s, e) =>
            {
                logger.InfoFormat("Batching deletion of {0}-{1} sagahistory documents.", s, e);
                var results = database.Batch(items.GetRange(s, e - s + 1), CancellationToken.None);
                logger.InfoFormat("Batching deletion of {0}-{1} sagahistory documents completed.", s, e);

                deletionCount += results.Count(x => x.Deleted == true);
            });

            if (deletionCount == 0)
            {
                logger.Info("No expired sagahistory documents found");
            }
            else
            {
                logger.InfoFormat("Deleted {0} expired sagahistory documents. Batch execution took {1}ms", deletionCount, stopwatch.ElapsedMilliseconds);
            }
        }
Esempio n. 10
0
        public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold)
        {
            var stopwatch   = Stopwatch.StartNew();
            var items       = new List <ICommandData>(deletionBatchSize);
            var attachments = new List <string>(deletionBatchSize);

            try
            {
                var query = new IndexQuery
                {
                    Start          = 0,
                    PageSize       = deletionBatchSize,
                    Cutoff         = SystemTime.UtcNow,
                    DisableCaching = true,
                    Query          = $"Status:[2 TO 4] AND LastModified:[* TO {expiryThreshold.Ticks}]",
                    FieldsToFetch  = new[]
                    {
                        "__document_id",
                        "ProcessingAttempts[0].MessageId"
                    },
                    SortedFields = new[]
                    {
                        new SortedField("LastModified")
                        {
                            Field      = "LastModified",
                            Descending = false
                        }
                    }
                };
                var indexName = new ExpiryErrorMessageIndex().IndexName;
                database.Query(indexName, query, database.WorkContext.CancellationToken,
                               null,
                               doc =>
                {
                    var id = doc.Value <string>("__document_id");
                    if (string.IsNullOrEmpty(id))
                    {
                        return;
                    }

                    items.Add(new DeleteCommandData
                    {
                        Key = id
                    });
                    var bodyid = doc.Value <string>("ProcessingAttempts[0].MessageId");
                    attachments.Add(bodyid);
                });
            }
            catch (OperationCanceledException)
            {
                //Ignore
            }

            var deletionCount = 0;

            Chunker.ExecuteInChunks(items.Count, (s, e) =>
            {
                logger.InfoFormat("Batching deletion of {0}-{1} error documents.", s, e);
                var results = database.Batch(items.GetRange(s, e - s + 1));
                logger.InfoFormat("Batching deletion of {0}-{1} error documents completed.", s, e);

                deletionCount += results.Count(x => x.Deleted == true);
            });

            Chunker.ExecuteInChunks(attachments.Count, (s, e) =>
            {
                database.TransactionalStorage.Batch(accessor =>
                {
                    logger.InfoFormat("Batching deletion of {0}-{1} attachment error documents.", s, e);
                    for (var idx = s; idx <= e; idx++)
                    {
                        accessor.Attachments.DeleteAttachment("messagebodies/" + attachments[idx], null);
                    }
                    logger.InfoFormat("Batching deletion of {0}-{1} attachment error documents completed.", s, e);
                });
            });

            if (deletionCount == 0)
            {
                logger.Info("No expired error documents found");
            }
            else
            {
                logger.InfoFormat("Deleted {0} expired error documents. Batch execution took {1}ms", deletionCount, stopwatch.ElapsedMilliseconds);
            }
        }
        public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold)
        {
            var stopwatch   = Stopwatch.StartNew();
            var items       = new List <ICommandData>(deletionBatchSize);
            var attachments = new List <string>(deletionBatchSize);

            try
            {
                var query = new IndexQuery
                {
                    Start          = 0,
                    PageSize       = deletionBatchSize,
                    Cutoff         = SystemTime.UtcNow,
                    DisableCaching = true,
                    Query          = $"ProcessedAt:[* TO {expiryThreshold.Ticks}]",
                    FieldsToFetch  = new[]
                    {
                        "__document_id",
                        "MessageMetadata"
                    },
                    SortedFields = new[]
                    {
                        new SortedField("ProcessedAt")
                        {
                            Field      = "ProcessedAt",
                            Descending = false
                        }
                    }
                };
                var indexName = new ExpiryProcessedMessageIndex().IndexName;
                database.Query(indexName, query, database.WorkContext.CancellationToken,
                               doc =>
                {
                    var id = doc.Value <string>("__document_id");
                    if (string.IsNullOrEmpty(id))
                    {
                        return;
                    }

                    items.Add(new DeleteCommandData
                    {
                        Key = id
                    });

                    string bodyId;
                    if (TryGetBodyId(doc, out bodyId))
                    {
                        attachments.Add(bodyId);
                    }
                });
            }
            catch (OperationCanceledException)
            {
                //Ignore
            }

            var deletionCount = 0;

            Chunker.ExecuteInChunks(items.Count, (s, e) =>
            {
                logger.InfoFormat("Batching deletion of {0}-{1} audit documents.", s, e);
                var results = database.Batch(items.GetRange(s, e - s + 1), CancellationToken.None);
                logger.InfoFormat("Batching deletion of {0}-{1} audit documents completed.", s, e);

                deletionCount += results.Count(x => x.Deleted == true);
            });

            Chunker.ExecuteInChunks(attachments.Count, (s, e) =>
            {
                database.TransactionalStorage.Batch(accessor =>
                {
                    logger.InfoFormat("Batching deletion of {0}-{1} attachment audit documents.", s, e);
                    for (var idx = s; idx <= e; idx++)
                    {
                        //We want to continue using attachments for now
#pragma warning disable 618
                        accessor.Attachments.DeleteAttachment(attachments[idx], null);
#pragma warning restore 618
                    }
                    logger.InfoFormat("Batching deletion of {0}-{1} attachment audit documents completed.", s, e);
                });
            });

            if (deletionCount == 0)
            {
                logger.Info("No expired audit documents found");
            }
            else
            {
                logger.InfoFormat("Deleted {0} expired audit documents. Batch execution took {1}ms", deletionCount, stopwatch.ElapsedMilliseconds);
            }
        }