public DatabaseBulkOperations(DocumentDatabase database, TransactionInformation transactionInformation, CancellationTokenSource tokenSource, CancellationTimeout timeout)
{
this.database = database;
this.transactionInformation = transactionInformation;
this.tokenSource = tokenSource;
this.timeout = timeout;
}
private static void WriteToStream(JsonWriter writer, RavenJObject item, CancellationTimeout timeout)
{
timeout.Delay();
item.WriteTo(writer);
}
public StreamQueryContent(HttpRequestMessage req, QueryActions.DatabaseQueryOperation queryOp, IStorageActionsAccessor accessor, CancellationTimeout timeout, Action<string> contentTypeSetter)
{
this.req = req;
this.queryOp = queryOp;
this.accessor = accessor;
_timeout = timeout;
outputContentTypeSetter = contentTypeSetter;
}
private IEnumerable<JsonDocument> YieldDocumentsInBatch(CancellationTimeout timeout, Stream partialStream, Action<int> increaseDocumentsCount)
{
using (var stream = new GZipStream(partialStream, CompressionMode.Decompress, leaveOpen: true))
{
var reader = new BinaryReader(stream);
var count = reader.ReadInt32();
for (var i = 0; i < count; i++)
{
timeout.Delay();
var doc = (RavenJObject)RavenJToken.ReadFrom(new BsonReader(reader)
{
DateTimeKindHandling = DateTimeKind.Unspecified
});
var metadata = doc.Value<RavenJObject>("@metadata");
if (metadata == null)
throw new InvalidOperationException("Could not find metadata for document");
var id = metadata.Value<string>("@id");
if (string.IsNullOrEmpty(id))
throw new InvalidOperationException("Could not get id from metadata");
if (id.Equals(Constants.BulkImportHeartbeatDocKey, StringComparison.InvariantCultureIgnoreCase))
continue; //its just a token document, should not get written into the database
//the purpose of the heartbeat document is to make sure that the connection doesn't time-out
//during long pauses in the bulk insert operation.
// Currently used by smuggler to make sure that the connection doesn't time out if there is a
//continuation token and lots of document skips
doc.Remove("@metadata");
yield return new JsonDocument
{
Key = id,
DataAsJson = doc,
Metadata = metadata
};
}
increaseDocumentsCount(count);
}
}
private IEnumerable<IEnumerable<JsonDocument>> YieldBatches(CancellationTimeout timeout, Stream inputStream, ManualResetEventSlim mre, Action<int> increaseDocumentsCount)
{
try
{
using (inputStream)
{
var binaryReader = new BinaryReader(inputStream);
while (true)
{
timeout.ThrowIfCancellationRequested();
int size;
try
{
size = binaryReader.ReadInt32();
}
catch (EndOfStreamException)
{
break;
}
using (var stream = new PartialStream(inputStream, size))
{
yield return YieldDocumentsInBatch(timeout, stream, increaseDocumentsCount);
}
}
}
}
finally
{
mre.Set();
inputStream.Close();
}
}
private IEnumerable<JsonDocument> YieldDocumentsInBatch(CancellationTimeout timeout, Stream partialStream, Action<int> increaseDocumentsCount)
{
using (var stream = new GZipStream(partialStream, CompressionMode.Decompress, leaveOpen: true))
{
var reader = new BinaryReader(stream);
var count = reader.ReadInt32();
for (var i = 0; i < count; i++)
{
timeout.Delay();
var doc = (RavenJObject)RavenJToken.ReadFrom(new BsonReader(reader)
{
DateTimeKindHandling = DateTimeKind.Unspecified
});
var metadata = doc.Value<RavenJObject>("@metadata");
if (metadata == null)
throw new InvalidOperationException("Could not find metadata for document");
var id = metadata.Value<string>("@id");
if (string.IsNullOrEmpty(id))
throw new InvalidOperationException("Could not get id from metadata");
doc.Remove("@metadata");
yield return new JsonDocument
{
Key = id,
DataAsJson = doc,
Metadata = metadata
};
}
increaseDocumentsCount(count);
}
}
private HttpResponseMessage OnBulkOperation(Func<string, IndexQuery, BulkOperationOptions, RavenJArray> batchOperation, string index, CancellationTimeout timeout)
{
if (string.IsNullOrEmpty(index))
return GetEmptyMessage(HttpStatusCode.BadRequest);
var option = new BulkOperationOptions
{
AllowStale = GetAllowStale(),
MaxOpsPerSec = GetMaxOpsPerSec(),
StaleTimeout = GetStaleTimeout(),
RetrieveDetails = GetRetrieveDetails()
};
var indexQuery = GetIndexQuery(maxPageSize: int.MaxValue);
var status = new BulkOperationStatus();
long id;
var task = Task.Factory.StartNew(() =>
{
status.State = batchOperation(index, indexQuery, option);
}).ContinueWith(t =>
{
if (timeout != null)
timeout.Dispose();
if (t.IsFaulted == false)
{
status.Completed = true;
return;
}
var exception = t.Exception.ExtractSingleInnerException();
status.State = RavenJObject.FromObject(new { Error = exception.Message });
status.Faulted = true;
status.Completed = true;
});
Database.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription
{
StartTime = SystemTime.UtcNow,
TaskType = TaskActions.PendingTaskType.IndexBulkOperation,
Payload = index
}, out id, timeout.CancellationTokenSource);
return GetMessageWithObject(new { OperationId = id });
}
public int BulkInsert(BulkInsertOptions options, IEnumerable<IEnumerable<JsonDocument>> docBatches, Guid operationId, CancellationToken token, CancellationTimeout timeout = null)
{
var documents = 0;
Database.Notifications.RaiseNotifications(new BulkInsertChangeNotification
{
OperationId = operationId,
Type = DocumentChangeTypes.BulkInsertStarted
});
using (var cts = CancellationTokenSource.CreateLinkedTokenSource(token, WorkContext.CancellationToken))
{
foreach (var docs in docBatches)
{
cts.Token.ThrowIfCancellationRequested();
var docsToInsert = docs.ToArray();
var batch = 0;
var keys = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var collectionsAndEtags = new Dictionary<string, Etag>(StringComparer.OrdinalIgnoreCase);
if (timeout != null)
timeout.Pause();
using (Database.DocumentLock.Lock())
{
if (timeout != null)
timeout.Resume();
TransactionalStorage.Batch(accessor =>
{
var inserts = 0;
foreach (var doc in docsToInsert)
{
try
{
if (string.IsNullOrEmpty(doc.Key))
throw new InvalidOperationException("Cannot try to bulk insert a document without a key");
RemoveReservedProperties(doc.DataAsJson);
RemoveMetadataReservedProperties(doc.Metadata);
if (options.CheckReferencesInIndexes)
keys.Add(doc.Key);
documents++;
batch++;
AssertPutOperationNotVetoed(doc.Key, doc.Metadata, doc.DataAsJson, null);
if (options.OverwriteExisting && options.SkipOverwriteIfUnchanged)
{
var existingDoc = accessor.Documents.DocumentByKey(doc.Key);
if (IsTheSameDocument(doc, existingDoc))
continue;
}
foreach (var trigger in Database.PutTriggers)
{
trigger.Value.OnPut(doc.Key, doc.DataAsJson, doc.Metadata, null);
}
var result = accessor.Documents.InsertDocument(doc.Key, doc.DataAsJson, doc.Metadata, options.OverwriteExisting);
if (result.Updated == false)
inserts++;
doc.Etag = result.Etag;
doc.Metadata.EnsureSnapshot(
"Metadata was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?");
doc.DataAsJson.EnsureSnapshot(
"Document was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?");
var entityName = doc.Metadata.Value<string>(Constants.RavenEntityName);
Etag highestEtagInCollection;
if (string.IsNullOrEmpty(entityName) == false && (collectionsAndEtags.TryGetValue(entityName, out highestEtagInCollection) == false ||
result.Etag.CompareTo(highestEtagInCollection) > 0))
{
collectionsAndEtags[entityName] = result.Etag;
}
foreach (var trigger in Database.PutTriggers)
{
trigger.Value.AfterPut(doc.Key, doc.DataAsJson, doc.Metadata, result.Etag, null);
}
Database.WorkContext.UpdateFoundWork();
}
catch (Exception e)
{
Database.Notifications.RaiseNotifications(new BulkInsertChangeNotification
{
OperationId = operationId,
Message = e.Message,
Etag = doc.Etag,
Id = doc.Key,
Type = DocumentChangeTypes.BulkInsertError
});
throw;
}
}
if (options.CheckReferencesInIndexes)
{
foreach (var key in keys)
{
Database.Indexes.CheckReferenceBecauseOfDocumentUpdate(key, accessor);
}
}
accessor.Documents.IncrementDocumentCount(inserts);
});
foreach (var collectionEtagPair in collectionsAndEtags)
{
Database.LastCollectionEtags.Update(collectionEtagPair.Key, collectionEtagPair.Value);
}
WorkContext.ShouldNotifyAboutWork(() => "BulkInsert batch of " + batch + " docs");
WorkContext.NotifyAboutWork(); // forcing notification so we would start indexing right away
WorkContext.UpdateFoundWork();
}
}
}
Database.Notifications.RaiseNotifications(new BulkInsertChangeNotification
{
OperationId = operationId,
Type = DocumentChangeTypes.BulkInsertEnded
});
if (documents > 0)
WorkContext.ShouldNotifyAboutWork(() => "BulkInsert of " + documents + " docs");
return documents;
}
