public async Task CreatePoisonThumbnailSetAsync(
CreateThumbnailsMessage message,
ICloudStorageAccount cloudStorageAccount,
ILogger logger,
int dequeueCount,
CancellationToken cancellationToken)
{
try
{
logger.Warn("Failed to resize image from blob {0}/{1}", message.ContainerName, message.InputBlobName);
var startProcessingTime = DateTime.UtcNow;
await this.thumbnailProcessor.CreatePoisonThumbnailSetAsync(
message,
cloudStorageAccount,
logger,
cancellationToken);
var completeProcessingTime = DateTime.UtcNow;
await this.setFileProcessingComplete.ExecuteAsync(
message.FileId,
dequeueCount * -1,
startProcessingTime,
completeProcessingTime,
1,
1);
}
catch (Exception t)
{
logger.Error(t);
throw;
}
}
public AzureFunctionsFabric(
ISerializerFactorySelector serializerFactorySelector,
INumericIdGenerator idGenerator,
ITransitionRunner transitionRunner,
IAzureWebJobsEnviromentalSettings azureWebJobsEnviromentalSettings,
ICloudStorageAccountFactory cloudStorageAccountFactory,
AzureFunctionsFabricSettings settings,
IServiceProxyBuilder serviceProxyBuilder,
IRoutineMethodResolver routineMethodResolver,
IMethodInvokerFactory methodInvokerFactory,
IServiceRegistry serviceRegistry)
{
_transitionRunner = transitionRunner;
_idGenerator = idGenerator;
_settings = settings;
_serviceProxyBuilder = serviceProxyBuilder;
_routineMethodResolver = routineMethodResolver;
_methodInvokerFactory = methodInvokerFactory;
_serviceRegistry = serviceRegistry;
#warning Need configurable serializer
// Hard-code this for now.
_defaultSerializer = serializerFactorySelector.Select("dasyncjson").Create();
_storageAccountConnectionString = azureWebJobsEnviromentalSettings.DefaultStorageConnectionString;
_storageAccount = cloudStorageAccountFactory.Create(_storageAccountConnectionString);
//#warning make sure that site name is alpha-numeric and does not start with a number
// var prefix = azureWebJobsEnviromentalSettings.WebSiteName.ToLowerInvariant();
var prefix = "";
_routinesTable = _storageAccount.TableClient.GetTableReference(prefix + "routines");
_servicesTable = _storageAccount.TableClient.GetTableReference(prefix + "services");
}
public AzureStorageFactory(ICloudStorageAccount acc)
{
_account = acc;
_queueClient = _account.CreateCloudQueueClient();
_tableclient = _account.CreateCloudTableClient();
_blobClient = _account.CreateCloudBlobClient();
_client = _account.CreateDocumentClient();
}
public AzureTableStorageProvider( ICloudStorageAccount cloudStorageAccount )
{
_cloudStorageAccount = cloudStorageAccount;
if ( !_servicePointsUpdated.ContainsKey( _cloudStorageAccount.TableEndpoint ) || !_servicePointsUpdated[_cloudStorageAccount.TableEndpoint] )
{
ServicePoint servicePoint = ServicePointManager.FindServicePoint( new Uri( _cloudStorageAccount.TableEndpoint ) );
servicePoint.Expect100Continue = false;
servicePoint.ConnectionLimit = 48;
}
}
public void TestInitialize()
{
_storageAccount = Configuration.GetTestCosmosStorageAccount();
_tableStorageProvider = new AzureTableStorageProvider(_storageAccount);
_client = new CloudTableClient(new Uri(_storageAccount.TableEndpoint), _storageAccount.Credentials);
_tableName = _baseTableName + Guid.NewGuid().ToString().Replace("-", string.Empty);
var table = _client.GetTableReference(_tableName);
table.CreateAsync().Wait();
}
private static async Task ConfigureCors(ICloudStorageAccount storageAccount)
{
try
{
var blobClient = storageAccount.CreateCloudBlobClient();
var blobServiceProperties = await blobClient.GetServicePropertiesAsync();
ConfigureCors(blobServiceProperties);
await blobClient.SetServicePropertiesAsync(blobServiceProperties);
}
catch (Exception t)
{
ExceptionHandlerUtilities.ReportExceptionAsync(t, null);
}
}
public AzureTableStorageProvider( ICloudStorageAccount cloudStorageAccount )
: base(new AzureTableEntityTableContext( cloudStorageAccount ))
{
_cloudStorageAccount = cloudStorageAccount;
if ( _setConcurrentConnectionLimit && !_servicePointsUpdated.ContainsKey( _cloudStorageAccount.TableEndpoint ) )
{
UpdateServicePointConnectionLimit( new Uri( _cloudStorageAccount.TableEndpoint ), _tableStorageConcurrentConnectionLimit );
_servicePointsUpdated.AddOrUpdate( _cloudStorageAccount.TableEndpoint, true, ( s, v ) => true );
}
if ( !string.IsNullOrWhiteSpace( _cloudStorageAccount.ReadonlyFallbackTableEndpoint ) && _setConcurrentConnectionLimit && !_servicePointsUpdated.ContainsKey( _cloudStorageAccount.ReadonlyFallbackTableEndpoint ) )
{
UpdateServicePointConnectionLimit( new Uri( _cloudStorageAccount.ReadonlyFallbackTableEndpoint ), _tableStorageConcurrentConnectionLimit );
_servicePointsUpdated.AddOrUpdate( _cloudStorageAccount.ReadonlyFallbackTableEndpoint, true, ( s, v ) => true );
}
}
public async Task CreatePoisonThumbnailSetAsync(
CreateThumbnailsMessage message,
ICloudStorageAccount cloudStorageAccount,
ILogger logger,
CancellationToken cancellationToken)
{
if (message.Items == null || message.Items.Count == 0)
{
return;
}
var cache = this.GetItemsCache(message, cloudStorageAccount);
await this.PopulateExists(message, cancellationToken, cache);
if (message.Overwrite == false && cache.Values.All(v => v.Exists))
{
return;
}
// Create a small black 1x1 png.
using (var image = new MagickImage(new MagickColor(0, 0, 0), 1, 1))
{
image.Format = MagickFormat.Png;
foreach (var itemData in cache.Values)
{
if (itemData.ShouldCreate)
{
if (itemData.Exists)
{
await itemData.BlockBlob.FetchAttributesAsync(cancellationToken);
}
itemData.BlockBlob.Properties.ContentType = image.FormatInfo.MimeType;
using (var outputStream = await itemData.BlockBlob.OpenWriteAsync(cancellationToken))
{
image.Write(outputStream);
await Task.Factory.FromAsync(outputStream.BeginCommit(null, null), outputStream.EndCommit);
}
}
}
}
}
private static async Task ConfigureStorage(ICloudStorageAccount storageAccount)
{
try
{
var cloudQueueClient = storageAccount.CreateCloudQueueClient();
await CreateQueueIfNotExists(cloudQueueClient, WebJobs.Thumbnails.Shared.Constants.ThumbnailsQueueName);
await CreateQueueIfNotExists(cloudQueueClient, Fifthweek.GarbageCollection.Shared.Constants.GarbageCollectionQueueName);
await CreateQueueIfNotExists(cloudQueueClient, Payments.Shared.Constants.RequestSnapshotQueueName);
await CreateQueueIfNotExists(cloudQueueClient, Payments.Shared.Constants.RequestProcessPaymentsQueueName);
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
await CreateBlobContainerIfNotExists(cloudBlobClient, Payments.Shared.Constants.PaymentProcessingDataContainerName);
await CreateBlobIfNotExists(cloudBlobClient, Fifthweek.Azure.Constants.AzureLeaseObjectsContainerName, Payments.Shared.Constants.ProcessPaymentsLeaseObjectName);
await CreateBlobIfNotExists(cloudBlobClient, Fifthweek.Azure.Constants.AzureLeaseObjectsContainerName, WebJobs.GarbageCollection.Shared.Constants.LeaseObjectName);
await CreateBlobContainerIfNotExists(cloudBlobClient, Constants.PublicFileBlobContainerName);
}
catch (Exception t)
{
ExceptionHandlerUtilities.ReportExceptionAsync(t, null);
}
}
public async Task CreateThumbnailSetAsync(
CreateThumbnailsMessage message,
ICloudBlockBlob input,
ICloudStorageAccount cloudStorageAccount,
ILogger logger,
int dequeueCount,
CancellationToken cancellationToken)
{
try
{
var startProcessingTime = DateTime.UtcNow;
var result = await this.thumbnailProcessor.CreateThumbnailSetAsync(
message,
input,
cloudStorageAccount,
logger,
cancellationToken);
var completeProcessingTime = DateTime.UtcNow;
if (result != null)
{
await this.setFileProcessingComplete.ExecuteAsync(
message.FileId,
dequeueCount,
startProcessingTime,
completeProcessingTime,
result.RenderWidth,
result.RenderHeight);
}
}
catch (Exception t)
{
logger.Error(t);
throw;
}
}
/// <summary>
/// Gets a reference to a <see cref="ICloudBlobContainer"/> inside the <see cref="ICloudStorageAccount"/>
/// </summary>
/// <param name="storageAccount">The <see cref="ICloudStorageAccount"/></param>
/// <param name="containerName">The name of the container</param>
/// <returns>An instance of an <see cref="ICloudBlobContainer"/>.</returns>
public static ICloudBlobContainer GetContainerReference(this ICloudStorageAccount storageAccount, string containerName)
{
var blobClient = storageAccount.CreateCloudBlobClient();
return(blobClient.GetContainerReference(containerName));
}
public AzureTableStorageProvider(ICloudStorageAccount cloudStorageAccount)
: base(new AzureTableEntityTableContext(cloudStorageAccount))
{
}
public QueueService(ICloudStorageAccount cloudStorageAccount)
{
this.cloudStorageAccount = cloudStorageAccount;
}
public BlobService(ICloudStorageAccount cloudStorageAccount, IAzureConfiguration azureConfiguration)
{
this.cloudStorageAccount = cloudStorageAccount;
this.azureConfiguration = azureConfiguration;
}
public void TestInitialize()
{
_storageAccount = Configuration.GetTestStorageAccount();
_tableStorageProvider = new AzureTableStorageProvider( _storageAccount );
_client = new CloudTableClient( new Uri( _storageAccount.TableEndpoint ), _storageAccount.Credentials );
_tableName = _baseTableName + Guid.NewGuid().ToString().Replace( "-", string.Empty );
var table = _client.GetTableReference( _tableName );
table.CreateAsync().Wait();
}
public void TestInitialize()
{
_storageAccount = new ConnectionStringCloudStorageAccount( ConfigurationManager.AppSettings["storageConnectionString"] );
_tableStorageProvider = new AzureTableStorageProvider( _storageAccount );
_client = new CloudTableClient( new Uri( _storageAccount.TableEndpoint ), _storageAccount.Credentials );
_tableName = _baseTableName + Guid.NewGuid().ToString().Replace( "-", string.Empty );
var table = _client.GetTableReference( _tableName );
table.Create();
}
public AzureTableStorageProvider( ICloudStorageAccount cloudStorageAccount )
: base(new AzureTableEntityTableContext( cloudStorageAccount ))
{
}
public AzureTableStorageProvider( ICloudStorageAccount cloudStorageAccount )
{
_cloudStorageAccount = cloudStorageAccount;
}
private Dictionary <ThumbnailDefinition, ThumbnailSetItemData> GetItemsCache(CreateThumbnailsMessage message, ICloudStorageAccount cloudStorageAccount)
{
var client = cloudStorageAccount.CreateCloudBlobClient();
var container = client.GetContainerReference(message.ContainerName);
var cache = new Dictionary <ThumbnailDefinition, ThumbnailSetItemData>();
this.GetItemsCache(container, cache, message.Items);
return(cache);
}
public AzureStorageImpl(ICloudStorageAccount account)
{
_blobClient = account.CreateCloudBlobClient();
}
public async Task <CreateThumbnailSetResult> CreateThumbnailSetAsync(
CreateThumbnailsMessage message,
ICloudBlockBlob input,
ICloudStorageAccount cloudStorageAccount,
ILogger logger,
CancellationToken cancellationToken)
{
CreateThumbnailSetResult result = null;
var sw = new Stopwatch();
sw.Start();
logger.Info("StartFileProcessor: " + sw.ElapsedMilliseconds);
if (message.Items == null || message.Items.Count == 0)
{
return(null);
}
var cache = this.GetItemsCache(message, cloudStorageAccount);
logger.Info("GetBlockBlobs: " + sw.ElapsedMilliseconds);
await this.PopulateExists(message, cancellationToken, cache);
logger.Info("CheckExists: " + sw.ElapsedMilliseconds);
if (message.Overwrite == false && cache.Values.All(v => v.Exists))
{
return(null);
}
await input.FetchAttributesAsync(cancellationToken);
var cacheControl = input.Properties.CacheControl;
this.PopulateCacheControl(cacheControl, cache);
using (var image = await this.OpenImageAsync(input, cancellationToken, logger, sw))
{
logger.Info("OpenImage: " + sw.ElapsedMilliseconds);
var outputMimeType = image.FormatInfo.MimeType;
if (!SupportedOutputMimeTypes.Contains(outputMimeType))
{
outputMimeType = DefaultOutputMimeType;
image.Format = DefaultOutputFormat;
logger.Info("ConvertToJpeg: " + sw.ElapsedMilliseconds);
}
if (image.Format == MagickFormat.Jpg || image.Format == MagickFormat.Jpeg)
{
var colorProfile = image.GetColorProfile();
if (colorProfile == null)
{
image.AddProfile(ColorProfile.SRGB);
logger.Info("AddColorProfile: " + sw.ElapsedMilliseconds);
}
if (image.Quality > 85)
{
image.Quality = 85;
}
await this.OrientAndSetMetadataAsync(image, input, cancellationToken, logger, sw);
}
else
{
await SetAndSaveMetadata(input, image);
}
var jobData = new JobData(outputMimeType);
logger.Info("Processing: " + sw.ElapsedMilliseconds);
result = new CreateThumbnailSetResult(image.Width, image.Height);
await this.ProcessThumbnailItems(message.Items, cache, image, jobData, logger, cancellationToken);
logger.Info("ProcessingComplete: " + sw.ElapsedMilliseconds);
}
logger.Info("Disposed: " + sw.ElapsedMilliseconds);
return(result);
}
public AzureTableEntityTableContext(ICloudStorageAccount storageAccount)
{
_storageAccount = storageAccount;
}
