public AddSourceInfoHandler(MongoDbClient mongoDbClient, Authorizer authorizer, SourceStorage sourceStorage, ILog log)
{
this.mongoDbClient = mongoDbClient;
this.authorizer = authorizer;
this.log = log;
this.sourceStorage = sourceStorage;
}
private void IndexItemIntoDirectory(params IItem[] items)
{
var source = new Source {
Items = items
};
var indexer = new SourceStorage(_directory, _learningRepository, _converterRepository);
indexer.IndexItems(source, source.GetItems());
}
public SendMessageHandler(MessageSender messageSender,
MongoDbClient mongoClient,
SourceStorage sourceStorage,
Authorizer authorizer,
TimeSpan ttl)
{
this.messageSender = messageSender;
this.mongoClient = mongoClient;
this.sourceStorage = sourceStorage;
this.authorizer = authorizer;
this.ttl = ttl;
}
public void CannotFindItemWhenIndexIsEmpty()
{
var source = new Source();
var indexer = new SourceStorage(_directory, _learningRepository, _converterRepository);
source.Items = new TextItem[] {};
indexer.IndexItems(source, source.GetItems());
var searcher = GetAutocompleter();
var results = searcher.Autocomplete("simple");
Assert.False(results.HasAutoCompletion);
Assert.Null(results.AutoCompletedCommand);
}
public static async Task CheckStorage()
{
var storageData = CreateStorageData();
var immutableStorageData = storageData
.Select(kvp => new KeyValuePair <string, ImmutableDictionary <string, byte[]> >(kvp.Key, kvp.Value.ToImmutableDictionary()))
.ToImmutableDictionary();
var storage = new SourceStorage(immutableStorageData);
var testDirectoryName = storageData.Keys.Skip(storageData.Count / 2).First();
var testDirectoryInfo = await storage.GetDirectory(testDirectoryName);
Assert.Equal(testDirectoryName, testDirectoryInfo.NormalizedRelativePath);
await CheckDirectory(storageData[testDirectoryName].ToDictionary(kvp => kvp.Key, kvp => kvp.Value), testDirectoryInfo);
var syncObject = new object();
var actionBlock = new ActionBlock <ISourceDirectory>(async sourceDirectory =>
{
Dictionary <string, byte[]> originalDirectoryData;
lock (syncObject)
{
Assert.True(storageData.ContainsKey(sourceDirectory.NormalizedRelativePath), "Directory does not exist in original data");
originalDirectoryData = storageData[sourceDirectory.NormalizedRelativePath];
storageData.Remove(sourceDirectory.NormalizedRelativePath);
}
await CheckDirectory(originalDirectoryData, sourceDirectory);
});
await storage.GetAllDirectories(actionBlock, CancellationToken.None);
await actionBlock.Completion;
Assert.False(storageData.Any(), "Not all directories were enumerated");
}
public override void ExecuteCommand()
{
var sourceClient = SourceStorage.CreateCloudBlobClient();
var sourceContainer = sourceClient.GetContainerReference(SourceContainer);
if (!sourceContainer.Exists())
{
Log.Warn("No blobs in container!");
return;
}
var destClient = DestinationStorage.CreateCloudBlobClient();
var destContainer = destClient.GetContainerReference(DestinationContainer);
destContainer.CreateIfNotExists();
Log.Info("Collecting blob names in {0} to copy to {1}", SourceStorage.Credentials.AccountName, DestinationStorage.Credentials.AccountName);
var blobs = Util.CollectBlobs(
Log,
sourceContainer,
Prefix ?? String.Empty,
condition: b => (!PackageBlobsOnly || (!b.Name.StartsWith("/") && String.Equals(b.Name.ToLowerInvariant(), b.Name, StringComparison.Ordinal))),
countEstimate: 140000);
var count = blobs.Count;
int index = 0;
Parallel.ForEach(blobs, new ParallelOptions {
MaxDegreeOfParallelism = 10
}, blob =>
{
int currentIndex = Interlocked.Increment(ref index);
var percentage = (((double)currentIndex / (double)count) * 100);
var destBlob = destContainer.GetBlockBlobReference(blob.Name);
try
{
if (!destBlob.Exists() || Overwrite)
{
Log.Info("[{1:000000}/{2:000000}] ({3:000.00}%) Started Async Copy of {0}.", blob.Name, currentIndex, count, percentage);
if (!WhatIf)
{
destBlob.StartCopyFromBlob(blob);
}
}
else
{
Log.Info("[{1:000000}/{2:000000}] ({3:000.00}%) Skipped {0}. Blob already Exists", blob.Name, index, count, percentage);
}
}
catch (StorageException stex)
{
if (stex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict &&
stex.RequestInformation.ExtendedErrorInformation.ErrorCode == BlobErrorCodeStrings.PendingCopyOperation)
{
Log.Info("[{1:000000}/{2:000000}] ({3:000.00}%) Skipped {0}. Already being copied", blob.Name, index, count, percentage);
}
}
catch (Exception ex)
{
Log.Error("Error processing {0}: {1}", blob.Name, ex.ToString());
throw;
}
});
Log.Info("Copies started. Run checkblobcopies with similar parameters to wait on blob copy completion");
}
/// <summary>
/// コンストラクタ
/// </summary>
/// <param name="projectDir"></param>
public SiteGenerator(string src, string dest, bool isPreviewMode)
{
_srcRepository = new SourceStorage(src, isPreviewMode);
_outputRepository = new DestinationStorage(dest, isPreviewMode);
}
public Subscriber(Authorizer authorizer, SourceStorage sourceStorage, SseClient sseClient)
{
this.authorizer = authorizer;
this.sourceStorage = sourceStorage;
this.sseClient = sseClient;
}
public ExpirationDaemon(SourceStorage sourceStorage, TimeSpan period)
{
this.sourceStorage = sourceStorage;
this.period = period;
Run();
}
