public DatabaseBulkOperations(DocumentDatabase database, TransactionInformation transactionInformation, CancellationTokenSource tokenSource, CancellationTimeout timeout)
{
this.database = database;
this.transactionInformation = transactionInformation;
this.tokenSource = tokenSource;
this.timeout = timeout;
}
public StreamQueryContent(HttpRequestMessage req, QueryActions.DatabaseQueryOperation queryOp, IStorageActionsAccessor accessor, CancellationTimeout timeout, Action <string> contentTypeSetter)
{
this.req = req;
this.queryOp = queryOp;
this.accessor = accessor;
_timeout = timeout;
outputContentTypeSetter = contentTypeSetter;
}
public DatabaseBulkOperations(DocumentDatabase database, CancellationTokenSource tokenSource, CancellationTimeout timeout)
{
this.database = database;
this.tokenSource = tokenSource;
this.timeout = timeout;
}
public int BulkInsert(BulkInsertOptions options, IEnumerable <IEnumerable <JsonDocument> > docBatches, Guid operationId, CancellationToken token, CancellationTimeout timeout = null)
{
var documents = 0;
Database.Notifications.RaiseNotifications(new BulkInsertChangeNotification
{
OperationId = operationId,
Type = DocumentChangeTypes.BulkInsertStarted
});
using (var cts = CancellationTokenSource.CreateLinkedTokenSource(token, WorkContext.CancellationToken))
{
foreach (var docs in docBatches)
{
cts.Token.ThrowIfCancellationRequested();
var docsToInsert = docs.ToArray();
var batch = 0;
var keys = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
var collectionsAndEtags = new Dictionary <string, Etag>(StringComparer.OrdinalIgnoreCase);
timeout?.Pause();
using (Database.DocumentLock.Lock())
{
timeout?.Resume();
TransactionalStorage.Batch(accessor =>
{
var inserts = 0;
foreach (var doc in docsToInsert)
{
try
{
if (string.IsNullOrWhiteSpace(doc.Key))
{
throw new InvalidOperationException("Cannot try to bulk insert a document without a key");
}
doc.Key = doc.Key.Trim();
if (doc.Key[doc.Key.Length - 1] == '/')
{
doc.Key += GetNextIdentityValueWithoutOverwritingOnExistingDocuments(doc.Key, accessor);
}
RemoveReservedProperties(doc.DataAsJson);
RemoveMetadataReservedProperties(doc.Metadata);
if (options.CheckReferencesInIndexes)
{
keys.Add(doc.Key);
}
documents++;
batch++;
AssertPutOperationNotVetoed(doc.Key, doc.Metadata, doc.DataAsJson, null);
if (options.OverwriteExisting && options.SkipOverwriteIfUnchanged)
{
var existingDoc = accessor.Documents.DocumentByKey(doc.Key);
if (IsTheSameDocument(doc, existingDoc))
{
continue;
}
}
foreach (var trigger in Database.PutTriggers)
{
trigger.Value.OnPut(doc.Key, doc.DataAsJson, doc.Metadata, null);
}
var result = accessor.Documents.InsertDocument(doc.Key, doc.DataAsJson, doc.Metadata, options.OverwriteExisting);
if (result.Updated == false)
{
inserts++;
}
doc.Etag = result.Etag;
doc.Metadata.EnsureSnapshot(
"Metadata was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?");
doc.DataAsJson.EnsureSnapshot(
"Document was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?");
var entityName = doc.Metadata.Value <string>(Constants.RavenEntityName);
Etag highestEtagInCollection;
if (string.IsNullOrEmpty(entityName) == false && (collectionsAndEtags.TryGetValue(entityName, out highestEtagInCollection) == false ||
result.Etag.CompareTo(highestEtagInCollection) > 0))
{
collectionsAndEtags[entityName] = result.Etag;
}
foreach (var trigger in Database.PutTriggers)
{
trigger.Value.AfterPut(doc.Key, doc.DataAsJson, doc.Metadata, result.Etag, null);
}
Database.WorkContext.UpdateFoundWork();
}
catch (Exception e)
{
Database.Notifications.RaiseNotifications(new BulkInsertChangeNotification
{
OperationId = operationId,
Message = e.Message,
Etag = doc.Etag,
Id = doc.Key,
Type = DocumentChangeTypes.BulkInsertError
});
throw;
}
}
if (options.CheckReferencesInIndexes)
{
foreach (var key in keys)
{
Database.Indexes.CheckReferenceBecauseOfDocumentUpdate(key, accessor);
}
}
accessor.Documents.IncrementDocumentCount(inserts);
});
foreach (var collectionEtagPair in collectionsAndEtags)
{
Database.LastCollectionEtags.Update(collectionEtagPair.Key, collectionEtagPair.Value);
}
WorkContext.ShouldNotifyAboutWork(() => "BulkInsert batch of " + batch + " docs");
WorkContext.NotifyAboutWork(); // forcing notification so we would start indexing right away
WorkContext.UpdateFoundWork();
}
}
}
Database.Notifications.RaiseNotifications(new BulkInsertChangeNotification
{
OperationId = operationId,
Type = DocumentChangeTypes.BulkInsertEnded
});
if (documents > 0)
{
WorkContext.ShouldNotifyAboutWork(() => "BulkInsert of " + documents + " docs");
}
return(documents);
}
private IEnumerable <TimeSeriesAppend> YieldBatchItems(Stream partialStream, JsonSerializer serializer, CancellationTimeout timeout, Action <int> changeTimeSeriesFunc)
{
using (var stream = new GZipStream(partialStream, CompressionMode.Decompress, leaveOpen: true))
{
var reader = new BinaryReader(stream);
var count = reader.ReadInt32();
for (var i = 0; i < count; i++)
{
timeout.Delay();
var doc = (RavenJObject)RavenJToken.ReadFrom(new BsonReader(reader)
{
DateTimeKindHandling = DateTimeKind.Unspecified
});
yield return(doc.ToObject <TimeSeriesAppend>(serializer));
}
changeTimeSeriesFunc(count);
}
}
private IEnumerable <IEnumerable <TimeSeriesAppend> > YieldChangeBatches(Stream requestStream, CancellationTimeout timeout, Action <int> changeTimeSeriesFunc)
{
var serializer = JsonExtensions.CreateDefaultJsonSerializer();
try
{
using (requestStream)
{
var binaryReader = new BinaryReader(requestStream);
while (true)
{
timeout.ThrowIfCancellationRequested();
int batchSize;
try
{
batchSize = binaryReader.ReadInt32();
}
catch (EndOfStreamException)
{
break;
}
using (var stream = new PartialStream(requestStream, batchSize))
{
yield return(YieldBatchItems(stream, serializer, timeout, changeTimeSeriesFunc));
}
}
}
}
finally
{
requestStream.Close();
}
}
private HttpResponseMessage OnBulkOperation(Func <string, IndexQuery, BulkOperationOptions, Action <BulkOperationProgress>, RavenJArray> batchOperation, string index, CancellationTimeout timeout)
{
if (string.IsNullOrEmpty(index))
{
return(GetEmptyMessage(HttpStatusCode.BadRequest));
}
var option = new BulkOperationOptions
{
AllowStale = GetAllowStale(),
MaxOpsPerSec = GetMaxOpsPerSec(),
StaleTimeout = GetStaleTimeout(),
RetrieveDetails = GetRetrieveDetails()
};
var indexQuery = GetIndexQuery(maxPageSize: int.MaxValue);
var status = new BulkOperationStatus();
long id;
var task = Task.Factory.StartNew(() =>
{
using (DocumentCacher.SkipSetDocumentsInDocumentCache())
{
status.State["Batch"] = batchOperation(index, indexQuery, option, x =>
{
status.MarkProgress(x);
});
}
}).ContinueWith(t =>
{
if (timeout != null)
{
timeout.Dispose();
}
if (t.IsFaulted == false)
{
status.MarkCompleted($"Processed {status.OperationProgress.ProcessedEntries} items");
return;
}
var exception = t.Exception.ExtractSingleInnerException();
status.MarkFaulted(exception.Message);
});
Database.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription
{
StartTime = SystemTime.UtcNow,
TaskType = TaskActions.PendingTaskType.IndexBulkOperation,
Description = index
}, out id, timeout.CancellationTokenSource);
return(GetMessageWithObject(new { OperationId = id }, HttpStatusCode.Accepted));
}
private HttpResponseMessage OnBulkOperation(Func <string, IndexQuery, BulkOperationOptions, RavenJArray> batchOperation, string index, CancellationTimeout timeout)
{
if (string.IsNullOrEmpty(index))
{
return(GetEmptyMessage(HttpStatusCode.BadRequest));
}
var option = new BulkOperationOptions
{
AllowStale = GetAllowStale(),
MaxOpsPerSec = GetMaxOpsPerSec(),
StaleTimeout = GetStaleTimeout(),
RetrieveDetails = GetRetrieveDetails()
};
var indexQuery = GetIndexQuery(maxPageSize: int.MaxValue);
var status = new BulkOperationStatus();
long id;
var task = Task.Factory.StartNew(() =>
{
status.State = batchOperation(index, indexQuery, option);
}).ContinueWith(t =>
{
if (timeout != null)
{
timeout.Dispose();
}
if (t.IsFaulted == false)
{
status.Completed = true;
return;
}
var exception = t.Exception.ExtractSingleInnerException();
status.State = RavenJObject.FromObject(new { Error = exception.Message });
status.Faulted = true;
status.Completed = true;
});
Database.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription
{
StartTime = SystemTime.UtcNow,
TaskType = TaskActions.PendingTaskType.IndexBulkOperation,
Payload = index
}, out id, timeout.CancellationTokenSource);
return(GetMessageWithObject(new { OperationId = id }));
}
