internal static void StartSync()
{
//grab all tenants from configuration manager.
Console.WriteLine("Reading data from configuration manager");
var tenants = ConfigurationServiceClient.Instance.GetStructuredSetting("tenants");
List <ArtifactSyncJobConfig> configs = new List <ArtifactSyncJobConfig>();
foreach (string tenantId in tenants) // conversion from dynamic array
{
Console.WriteLine("Looking for configuration for tenant {0}", tenantId);
var artifactSyncJobConfig = new ArtifactSyncJobConfig(tenantId);
var tenantOriginalConnectionStringSettingName = tenantId + "-dest-ori";
ConnectionStringSettings destinationConnectionOriginal = ConfigurationManager.ConnectionStrings[tenantOriginalConnectionStringSettingName];
var tenantArtifactsConnectionStringSettingName = tenantId + "-dest-art";
ConnectionStringSettings destinationConnectionArtifacts = ConfigurationManager.ConnectionStrings[tenantArtifactsConnectionStringSettingName];
if (destinationConnectionOriginal == null)
{
Console.WriteLine("Destination connection for original [{1}], tenant {0} not specified, tenant skipped", tenantId, tenantOriginalConnectionStringSettingName);
continue;
}
if (destinationConnectionArtifacts == null)
{
Console.WriteLine("Destination connection for artifacts [{1}], tenant {0} not specified, tenant skipped", tenantId, tenantArtifactsConnectionStringSettingName);
continue;
}
Console.WriteLine("Configuration for tenant {0} valid!", tenantId);
Console.WriteLine("Destination db for tenant/Original {0}: {1}", tenantId, destinationConnectionOriginal.ConnectionString);
Console.WriteLine("Destination db for tenant/Artifacts {0}: {1}", tenantId, destinationConnectionArtifacts.ConnectionString);
Console.WriteLine("TENANT {0} sync started from checkpoint token {1}", tenantId, artifactSyncJobConfig.GetLastSyncedCheckpoint());
Console.Write("Do you want start syncronization for tenant {0} (s/n)?", tenantId);
Char answer;
do
{
answer = Console.ReadKey().KeyChar;
} while (answer != 's' && answer != 'n');
if (answer == 's')
{
artifactSyncJobConfig.SetDestination(destinationConnectionOriginal.ConnectionString, destinationConnectionArtifacts.ConnectionString);
configs.Add(artifactSyncJobConfig);
}
}
Console.WriteLine("Press a key to start sync");
Console.ReadLine();
List <Task> tasks = new List <Task>();
foreach (var config in configs)
{
var task = Task.Factory.StartNew(() => StartSyncingConfig(config));
tasks.Add(task);
}
Task.WaitAll(tasks.ToArray());
}
public SyncUnitConfiguration GetConfiguration(string fileId, ArtifactSyncJobConfig config)
{
string bucket = Path.GetFileNameWithoutExtension(fileId);
if (bucket == "original")
{
return(new SyncUnitConfiguration
{
FileId = fileId,
Bucket = bucket,
SourceConnectionString = config.SourceOriginalConnectionString,
DestConnectionString = config.DestinationOriginalConnectionString
});
}
return(new SyncUnitConfiguration
{
FileId = fileId,
Bucket = bucket,
SourceConnectionString = config.SourceArtifactsConnectionString,
DestConnectionString = config.DestinationArtifactsConnectionString
});
}
private static void StartSyncingConfig(ArtifactSyncJobConfig config)
{
var sourceMongoUrl = new MongoUrl(config.EventStoreConnectionString);
var sourceDatabase = new MongoClient(sourceMongoUrl).GetDatabase(sourceMongoUrl.DatabaseName);
var rmStream = sourceDatabase.GetCollection <BsonDocument>("Commits");
Int64 checkpoint = config.GetLastSyncedCheckpoint();
var syncUnit = new SyncUnit(new SyncUnitConfigurator());
while (true)
{
var files = rmStream
.Find(Builders <BsonDocument> .Filter.Gte("_id", checkpoint))
.Sort(Builders <BsonDocument> .Sort.Ascending("_id"))
.ToEnumerable();
foreach (var file in files)
{
BsonArray events = file["Events"].AsBsonArray;
foreach (var evt in events)
{
var body = evt["Payload"]["Body"].AsBsonDocument;
if (body.Names.Contains("BlobId"))
{
string fileId = body["BlobId"].AsString;
syncUnit.Sync(fileId, config);
}
}
var checkpointToken = file["_id"].AsInt64;
checkpoint = checkpointToken + 1;
config.SaveLastSyncedCheckpoint(checkpoint);
}
Console.WriteLine("No more commit to sync!!");
Thread.Sleep(2000);
}
}
/// <summary>
/// these information come from: rm.Stream
/// - FormatInfo.BlobId
/// - Filename.name
/// - Filename.ext
/// </summary>
/// <param name="fileId"></param>
/// <param name="dateTimeUploadFilter">I do not want to sync blob that are more recent than this
/// value, to avoid syncing blob while they are handled by DS</param>
public Boolean Sync(
string fileId,
ArtifactSyncJobConfig config,
DateTime?dateTimeUploadFilter = null)
{
var cfg = _configurator.GetConfiguration(fileId, config);
// initialize Mongo Databases
IGridFSBucket <string> sourceBucket;
MongoUrl sourceMongoUrl;
IGridFSBucket <string> destinationBucket;
MongoUrl destinationMongoUrl;
IMongoDatabase sourceDatabase;
sourceMongoUrl = new MongoUrl(cfg.SourceConnectionString);
sourceDatabase = new MongoClient(sourceMongoUrl).GetDatabase(sourceMongoUrl.DatabaseName);
sourceBucket = new GridFSBucket <string>(sourceDatabase, new GridFSBucketOptions
{
BucketName = cfg.Bucket,
ChunkSizeBytes = 1048576, // 1MB
});
IMongoDatabase destinationDatabase;
destinationMongoUrl = new MongoUrl(cfg.DestConnectionString);
destinationDatabase = new MongoClient(destinationMongoUrl).GetDatabase(destinationMongoUrl.DatabaseName);
IMongoCollection <BsonDocument> destinationCollection = destinationDatabase.GetCollection <BsonDocument>(cfg.Bucket + ".files");
destinationBucket = new GridFSBucket <string>(destinationDatabase, new GridFSBucketOptions
{
BucketName = cfg.Bucket,
ChunkSizeBytes = 1048576, // 1MB
});
// before uploading the new element check if it's already in the destination database (maybe it's an alias)
var findIdFilter = Builders <GridFSFileInfo <string> > .Filter.Eq(x => x.Id, fileId);
using (var cursor = destinationBucket.Find(findIdFilter))
{
var exists = cursor.FirstOrDefault();
if (exists != null)
{
return(true); //Already synced, true
}
}
var source = sourceBucket.Find(findIdFilter).FirstOrDefault();
if (source == null)
{
return(false); //source stream does not exists
}
if (dateTimeUploadFilter.HasValue && source.UploadDateTime > dateTimeUploadFilter)
{
return(false); //Consider this as not existing.
}
var sw = Stopwatch.StartNew();
Console.WriteLine("Sync needed Tenant {0}/{1}: ", config.Tenant, fileId);
GridFSUploadOptions options = new GridFSUploadOptions();
options.ChunkSizeBytes = source.ChunkSizeBytes;
options.ContentType = source.ContentType;
using (var destinationStream = destinationBucket.OpenUploadStream(fileId, source.Filename, options))
{
sourceBucket.DownloadToStream(fileId, destinationStream);
}
destinationCollection.UpdateOne(
Builders <BsonDocument> .Filter.Eq("_id", fileId),
Builders <BsonDocument> .Update.Set("uploadDate", source.UploadDateTime)
);
Console.WriteLine("DONE {0}/{1} ({2} ms)", config.Tenant, fileId, sw.ElapsedMilliseconds);
sw.Stop();
return(true);
}
