public void Store(object entity)
{
var shardId = shardResolutionStrategy.GenerateShardIdFor(entity, this);
DatabaseCommands = Shards[shardId].AsyncDatabaseCommands;
database = MultiDatabase.GetDatabaseName(Shards[shardId].Url);
string id;
if (generateEntityIdOnTheClient.TryGetIdFromInstance(entity, out id) == false)
{
id = generateEntityIdOnTheClient.GetOrGenerateDocumentKey(entity);
}
var modifyDocumentId = shardStrategy.ModifyDocumentId(shardedDocumentStore.Conventions, shardId, id);
BulkInsertOperation bulkInsertOperation;
if (Bulks.TryGetValue(shardId, out bulkInsertOperation) == false)
{
var shard = Shards[shardId];
bulkInsertOperation = new BulkInsertOperation(database, shard, shard.Listeners, options, shard.Changes());
Bulks.Add(shardId, bulkInsertOperation);
}
bulkInsertOperation.Store(entity, modifyDocumentId);
}
public override async Task ImportData(Stream stream, SmugglerOptions options)
{
SmugglerJintHelper.Initialize(options ?? SmugglerOptions);
var batchSize = options != null ? options.BatchSize : SmugglerOptions.BatchSize;
using (store = CreateStore())
{
Task disposeTask = null;
try
{
operation = store.BulkInsert(options: new BulkInsertOptions
{
BatchSize = batchSize,
CheckForUpdates = true
});
operation.Report += text => ShowProgress(text);
await base.ImportData(stream, options);
}
finally
{
disposeTask = operation.DisposeAsync();
}
if (disposeTask != null)
{
await disposeTask;
}
}
}
public void Store(object entity)
{
var shardId = shardResolutionStrategy.GenerateShardIdFor(entity, this);
var shard = shards[shardId];
BulkInsertOperation bulkInsertOperation;
if (Bulks.TryGetValue(shardId, out bulkInsertOperation) == false)
{
var actualDatabaseName = database ?? ((dynamic)shard).DefaultDatabase ?? MultiDatabase.GetDatabaseName(shard.Url);
bulkInsertOperation = new BulkInsertOperation(actualDatabaseName, shard, shard.Listeners, options, shard.Changes());
Bulks.Add(shardId, bulkInsertOperation);
}
DatabaseCommands = string.IsNullOrWhiteSpace(database)
? shard.AsyncDatabaseCommands
: shard.AsyncDatabaseCommands.ForDatabase(database);
string id;
if (generateEntityIdOnTheClient.TryGetIdFromInstance(entity, out id) == false)
{
id = generateEntityIdOnTheClient.GetOrGenerateDocumentKey(entity);
}
var modifyDocumentId = shardStrategy.ModifyDocumentId(shardedDocumentStore.Conventions, shardId, id);
bulkInsertOperation.Store(entity, modifyDocumentId);
}
public void Store(object entity)
{
var shardId = shardResolutionStrategy.GenerateShardIdFor(entity, this);
DatabaseCommands = Shards[shardId].AsyncDatabaseCommands;
database = MultiDatabase.GetDatabaseName(Shards[shardId].Url);
string id;
if (generateEntityIdOnTheClient.TryGetIdFromInstance(entity, out id) == false)
{
id = generateEntityIdOnTheClient.GetOrGenerateDocumentKey(entity);
}
var modifyDocumentId = shardStrategy.ModifyDocumentId(shardedDocumentStore.Conventions, shardId, id);
BulkInsertOperation bulkInsertOperation;
if (Bulks.TryGetValue(shardId, out bulkInsertOperation) == false)
{
var shard = Shards[shardId];
bulkInsertOperation = new BulkInsertOperation(database, shard, shard.Listeners, options, shard.Changes());
Bulks.Add(shardId, bulkInsertOperation);
}
bulkInsertOperation.Store(entity, modifyDocumentId);
}
public static void LoadRestaurants(string csvFile, BulkInsertOperation bulkInsert)
{
var wktReader = new WktReader();
using (var reader = new StreamReader(csvFile))
using (var csv = new CsvReader(reader, new CsvConfiguration {UseInvariantCulture = true}))
{
var restaurantCsvRows = csv.GetRecords<RestaurantCsvRow>();
foreach (var row in restaurantCsvRows)
{
Polygon deliveryArea = null;
if (!string.IsNullOrEmpty(row.DeliveryArea))
deliveryArea = (Polygon)wktReader.Read(row.DeliveryArea);
var restaurant = new Restaurant
{
Name = row.Name,
Street = row.Street,
City = row.City,
PostCode = row.PostCode,
Phone = row.Phone,
Location = new Point(row.Latitude, row.Longitude),
DeliveryArea = deliveryArea,
DriveThruArea = string.IsNullOrEmpty(row.DriveThruArea) ? null : row.DriveThruArea
};
bulkInsert.Store(restaurant);
}
}
}
static void Main(string[] args)
{
Console.WriteLine("Starting scan..");
ravenDBConnector = new RavenDBConnector();
Console.WriteLine("Iniating db connection..");
ravenDBStore = ravenDBConnector.InitDBConnection();
Console.WriteLine("Instantiating db session");
session = ravenDBStore.OpenSession();
Console.WriteLine("Getting the Bulk Insert object");
bulkInsert = ravenDBStore.BulkInsert();
string rootStartDrive = @"C:\anddev\";
Console.WriteLine("Drive for scanning is " + rootStartDrive);
DirectoryInfo directoryInfo = new DirectoryInfo(rootStartDrive);
AnalyzeDirectory(directoryInfo);
Console.WriteLine("Scanned {0} folders and {1} files under {2}", folderCount, fileCount,rootStartDrive);
Console.ReadLine();
ravenDBConnector.DisposeConnection();
}
public static void LoadRestaurants(string csvFile, BulkInsertOperation bulkInsert)
{
using (var reader = new StreamReader(csvFile))
using (var csv = new CsvReader(reader))
{
var restaurants = csv.GetRecords<Restaurant>();
foreach (var restaurant in restaurants)
{
if (string.IsNullOrEmpty(restaurant.DeliveryArea))
restaurant.DeliveryArea = null;
if (string.IsNullOrEmpty(restaurant.DriveThruArea))
restaurant.DriveThruArea = null;
bulkInsert.Store(restaurant);
}
}
}
private static void ParseDisks(BulkInsertOperation insert)
{
int i = 0;
var parser = new Parser();
var buffer = new byte[1024*1024];// more than big enough for all files
using (var bz2 = new BZip2InputStream(File.Open(@"D:\Scratch\freedb-complete-20150101.tar.bz2", FileMode.Open)))
using (var tar = new TarInputStream(bz2))
{
TarEntry entry;
while((entry=tar.GetNextEntry()) != null)
{
if(entry.Size == 0 || entry.Name == "README" || entry.Name == "COPYING")
continue;
var readSoFar = 0;
while(true)
{
var read = tar.Read(buffer, readSoFar, ((int) entry.Size) - readSoFar);
if (read == 0)
break;
readSoFar += read;
}
// we do it in this fashion to have the stream reader detect the BOM / unicode / other stuff
// so we can read the values properly
var fileText = new StreamReader(new MemoryStream(buffer,0, readSoFar)).ReadToEnd();
try
{
var disk = parser.Parse(fileText);
insert.Store(disk);
}
catch (Exception e)
{
Console.WriteLine();
Console.WriteLine(entry.Name);
Console.WriteLine(e);
}
}
}
}
public async Task StoreAsync(object entity)
{
var shardId = shardResolutionStrategy.GenerateShardIdFor(entity, this);
var shard = shards[shardId];
BulkInsertOperation bulkInsertOperation;
if (Bulks.TryGetValue(shardId, out bulkInsertOperation) == false)
{
var actualDatabaseName = database ?? ((DocumentStore)shard).DefaultDatabase ?? MultiDatabase.GetDatabaseName(shard.Url);
bulkInsertOperation = new BulkInsertOperation(actualDatabaseName, shard, shard.Listeners);
Bulks.Add(shardId, bulkInsertOperation);
}
DatabaseCommands = shard.AsyncDatabaseCommands;
string id;
if (generateEntityIdOnTheClient.TryGetIdFromInstance(entity, out id) == false)
{
id = generateEntityIdOnTheClient.GetOrGenerateDocumentKey(entity);
}
var modifyDocumentId = shardStrategy.ModifyDocumentId(shardedDocumentStore.Conventions, shardId, id);
await bulkInsertOperation.StoreAsync(entity, modifyDocumentId).ConfigureAwait(false);
}
private async Task CreateBulkInsertOperation()
{
if (operation != null)
await operation.DisposeAsync();
operation = new ChunkedBulkInsertOperation(store.DefaultDatabase, store, store.Listeners, new BulkInsertOptions
{
BatchSize = Options.BatchSize,
OverwriteExisting = true
}, store.Changes(), Options.ChunkSize, Options.TotalDocumentSizeInChunkLimitInBytes);
operation.Report += text => Operations.ShowProgress(text);
}
private void GenerateWorkItem(SqlConnection connection, WorkItem curWorkItem, Dictionary<int, ClosureReasonItem> closureReasonDictionary,
Dictionary<int, HandlingDepartment> handlingDepartmentsDictionary, Dictionary<int, ProductGroupItem> productGroupsDictionary, BulkInsertOperation workItemsSession,
Dictionary<int, User> usersDictionary)
{
var comments =
connection.Query<RavenFusion.Models.AtomSide.Comment>(@"SELECT * FROM COMMENT WHERE WORKITEM_ID=@ID",
new { ID = curWorkItem.Id });
var history =
connection.Query<RavenFusion.Models.AtomSide.WorkItemHistory>(
@"SELECT * FROM WORKITEMHISTORY WHERE WORKITEM_ID=@ID", new { ID = curWorkItem.Id });
var insuraceCompanies =
connection.Query<RavenFusion.Models.AtomSide.WorkItemInsuranceCompany>(
@"SELECT * FROM WorkItemInsuranceCompany WHERE WORKITEM_ID=@ID", new { ID = curWorkItem.Id });
var workItemSignOff =
connection.Query<RavenFusion.Models.AtomSide.WorkItemSignOff>(
@"SELECT * FROM WORKITEMSIGNOFF WHERE WORKITEM_ID=@ID", new { ID = curWorkItem.Id });
var workItemSubscription =
connection.Query<RavenFusion.Models.AtomSide.WorkItemSubscription>(
@"SELECT * FROM SUBSCRIPTION WHERE WORKITEM_ID=@ID", new { ID = curWorkItem.Id });
var workItemSupplier =
connection.Query<RavenFusion.Models.AtomSide.WorkItemSupplier>(
@"SELECT * FROM WORKITEMSUPPLIER WHERE WORKITEM_ID=@ID", new { ID = curWorkItem.Id });
var workItemDocuments = connection.Query<RavenFusion.Models.RavenSide.Document>(
@"SELECT * FROM DOCUMENT WHERE WORKITEM_ID=@ID", new { ID = curWorkItem.Id });
RavenFusion.Models.AtomSide.ClosureReasonItem workItemClosureReason = null;
closureReasonDictionary.TryGetValue(curWorkItem.ClosureReason_id, out workItemClosureReason);
RavenFusion.Models.AtomSide.HandlingDepartment workItemHandlingDepartment = null;
handlingDepartmentsDictionary.TryGetValue(curWorkItem.Department_id,
out workItemHandlingDepartment);
var workItemProductGroups = new List<RavenFusion.Models.RavenSide.ProductGroupDenormalized>();
foreach (var productGroupLink in connection.Query<RavenFusion.Models.AtomSide.WorkItemProductGroupLink>(
@"SELECT * FROM WorkItemProductGroup WHERE WORKITEM_ID=@ID",
new { ID = curWorkItem.Id }))
{
RavenFusion.Models.RavenSide.ProductGroupItem curProductGroup = null;
if (productGroupsDictionary.TryGetValue(productGroupLink.ProductGroup_id, out curProductGroup))
{
workItemProductGroups.Add(new RavenFusion.Models.RavenSide.ProductGroupDenormalized()
{
Id = curProductGroup.Id,
Name = curProductGroup.Name
});
}
}
workItemsSession.Store(
new RavenFusion.Models.RavenSide.WorkItem()
{
ActualUnitsOfWork = curWorkItem.ActualUnitsOfWork,
AssignedTo =
usersDictionary.ContainsKey(curWorkItem.AssignedTo_id)
? new RavenFusion.Models.RavenSide.UserDenormalized()
{
Id = usersDictionary[curWorkItem.AssignedTo_id].id,
Name = usersDictionary[curWorkItem.AssignedTo_id].Name,
NickName = usersDictionary[curWorkItem.AssignedTo_id].NickName
}
: null,
ClientRequirement = curWorkItem.ClientRequirement,
ClosedBy =
usersDictionary.ContainsKey(curWorkItem.ClosedBy_id)
? new RavenFusion.Models.RavenSide.UserDenormalized()
{
Id = usersDictionary[curWorkItem.ClosedBy_id].id,
Name = usersDictionary[curWorkItem.ClosedBy_id].Name,
NickName = usersDictionary[curWorkItem.ClosedBy_id].NickName
}
: null,
ClosedDate = curWorkItem.ClosedDate,
ClosureReason =
workItemClosureReason != null
? new RavenFusion.Models.RavenSide.ClosureReasonDenormalized()
{
Description = workItemClosureReason.Description,
Id = workItemClosureReason.Id.ToString()
}
: null,
Comments = comments.Select(x => new RavenFusion.Models.RavenSide.Comment()
{
CommentText = x.CommentText,
Type = x.Type,
UnitsOfWork = x.UnitOfWork
}).ToList(),
CompletionComment = curWorkItem.CompletionComment,
Department = workItemHandlingDepartment != null
? new RavenFusion.Models.RavenSide.HandlingDepartmentDenormalized()
{
Description = workItemHandlingDepartment.Description,
Id = workItemHandlingDepartment.id.ToString()
}
: null,
EstimatedStartDate = curWorkItem.EstimatedStartDate,
EstimatedUnitOfWork = curWorkItem.EstimatedUnitOfWork,
History = history.Select(x => x.Id).ToList(),
ImpactAnalysis = curWorkItem.ImpactAnalysis,
InHouseKeeping = curWorkItem.InHouseKeeping,
InsuranceCompanies = insuraceCompanies.Select(x => x.InsuranceCompany_id).ToList(),
InternalTesting = curWorkItem.InternalTesting,
ProductGroups = workItemProductGroups.Count > 0 ? workItemProductGroups : null,
Rejected = curWorkItem.Rejected,
RequestedCompletionDate = curWorkItem.RequestedCompletionDate,
Severity = curWorkItem.Severity,
SignOffs = workItemSignOff.Select(x => new RavenFusion.Models.RavenSide.WorkitemSignOff()
{
SignOffType = x.SignOffType,
SignedOff = x.SignedOff,
SignedOffBy =
usersDictionary.ContainsKey(x.SignedOffBy_id)
? new RavenFusion.Models.RavenSide.UserDenormalized()
{
Id = usersDictionary[x.SignedOffBy_id].id,
Name = usersDictionary[x.SignedOffBy_id].Name,
NickName = usersDictionary[x.SignedOffBy_id].NickName
}
: null,
}).Where(x => x.SignedOffBy != null).ToList(),
Subscriptions =
workItemSubscription.Select(
x =>
usersDictionary.ContainsKey(x.User_id)
? new RavenFusion.Models.RavenSide.UserDenormalized()
{
Id = usersDictionary[x.User_id].id,
Name = usersDictionary[x.User_id].Name,
NickName = usersDictionary[x.User_id].NickName
}
: null).Where(x => x != null).ToList(),
Summary = curWorkItem.Summary,
Suppliers = workItemSupplier.Select(x => x.Supplier_id).ToList(),
WorkItemType = curWorkItem.WorkItemType,
WorkStatus = curWorkItem.WorkStatus,
Documents = workItemDocuments.ToList()
}, @"WorkItems/" + curWorkItem.Id);
}
private static async Task<Tuple<int, Etag, DateTime>> TransferStreamedDocuments(DocumentStore exportStore,
SmugglerDatabaseOptions databaseOptions,
DateTime now,
SmugglerJintHelper jintHelper,
BulkInsertOperation bulkInsertOperation,
TimeSpan reportInterval,
int totalCount,
string fromEtag,
DateTime lastReport)
{
Etag lastReadEtag = fromEtag;
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(fromEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
lastReadEtag = etag;
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
if (databaseOptions.StripReplicationInformation)
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
if (databaseOptions.ShouldDisableVersioningBundle)
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
{
document = jintHelper.Transform(databaseOptions.TransformScript, document);
if (document == null)
continue;
metadata = document.Value<RavenJObject>("@metadata");
}
document.Remove("@metadata");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
return Tuple.Create(totalCount, lastReadEtag, lastReport);
}
private static async Task<Tuple<int, string, DateTime>> TransferDocumentsWithoutStreaming(DocumentStore exportStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize, OperationMetadata operationMetadata, DateTime now, BulkInsertOperation bulkInsertOperation, TimeSpan reportInterval, int totalCount, string fromEtag, DateTime lastReport)
{
var documents = await ((AsyncServerClient) exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, fromEtag, exportBatchSize, operationMetadata);
foreach (var jToken in documents)
{
var document = (RavenJObject) jToken;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
fromEtag = etag;
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
if (databaseOptions.StripReplicationInformation)
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
if (databaseOptions.ShouldDisableVersioningBundle)
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
return Tuple.Create(totalCount, fromEtag, lastReport);
}
public override async Task ImportData(SmugglerImportOptions importOptions, SmugglerOptions options, Stream stream)
{
SetSmugglerOptions(options);
SmugglerJintHelper.Initialize(options);
using (store = CreateStore(importOptions.To))
{
Task disposeTask;
try
{
operation = new ChunkedBulkInsertOperation(store.DefaultDatabase, store, store.Listeners, new BulkInsertOptions
{
BatchSize = options.BatchSize,
OverwriteExisting = true
}, store.Changes(), options.ChunkSize);
operation.Report += text => ShowProgress(text);
await base.ImportData(importOptions, options, stream);
}
finally
{
disposeTask = operation.DisposeAsync();
}
if (disposeTask != null)
{
await disposeTask;
}
}
}
protected override async Task PutDocument(RavenJObject document)
{
if (document == null)
return;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
document.Remove("@metadata");
operation.Store(document, metadata, id);
storedDocumentCountInBatch++;
if (storedDocumentCountInBatch >= currentBatchSize && currentBatchSize > 0)
{
storedDocumentCountInBatch = 0;
await operation.DisposeAsync();
operation = store.BulkInsert(options: new BulkInsertOptions
{
BatchSize = currentBatchSize,
CheckForUpdates = true
});
operation.Report += text => ShowProgress(text);
}
}
