public async Task DownloadFileAsync(string bucket, string objectName, Stream destination)
{
using (StorageClient client = await StorageClient.CreateAsync(_credential))
{
await client.DownloadObjectAsync(bucket, objectName, destination);
}
}
public async Task DownloadToStreamAsync(string containerName, string fileName, Stream stream)
{
var credential = GoogleCredential.FromJson(_credentialsJson);
try
{
using (var storageClient = await StorageClient.CreateAsync(credential).ConfigureAwait(false))
{
var blockBlob = await storageClient.GetObjectAsync(containerName, fileName).ConfigureAwait(false);
await storageClient.DownloadObjectAsync(blockBlob, stream, new DownloadObjectOptions()
{
ChunkSize = ChunkSize
}).ConfigureAwait(false);
}
} catch (GoogleApiException e)
{
if (e.HttpStatusCode == HttpStatusCode.NotFound)
{
throw new ExternalServiceApiException(ExternalServiceApiException.CloudStorageFileNotFound, "The file was not found");
}
else if (e.HttpStatusCode == HttpStatusCode.Forbidden ||
e.HttpStatusCode == HttpStatusCode.Unauthorized)
{
throw new ExternalServiceApiException(ExternalServiceApiException.CloudStorageFileAccessDenied, "Access to the file was denied");
}
else
{
throw e;
}
}
}
public async Task WriteFileAsync(string name, Stream fileStream)
{
try
{
// Save to memory cache
var ms = new MemoryStream();
await fileStream.CopyToAsync(ms);
ms.Position = 0;
if (ms.Length <= Settings.MultimediaMemoryCacheMaxSizeInBytes)
{
await DistributedCache.SetAsync(name, ms.ToArray(), new DistributedCacheEntryOptions
{
SlidingExpiration = Settings.MultimediaMemoryCacheSlidingExpiration
});
ms.Position = 0;
}
// Upload file
_client = _client ?? await StorageClient.CreateAsync();
await _client.UploadObjectAsync(Settings.GoogleStorageBucket, name, null, ms);
}
catch
{
// File probably already exists
}
}
private async Task <string> ImageProcess(IFormFile file)
{
var credential = GoogleCredential.FromFile("./Secrets/b2732904e562.json");
var storage = await StorageClient.CreateAsync(credential);
await using (var ms = new MemoryStream())
{
file.CopyTo(ms);
ms.Seek(0, SeekOrigin.Begin);
var optimizer = new ImageOptimizer();
optimizer.OptimalCompression = true;
var r = optimizer.Compress(ms);
//fileByteArray = ms.ToArray();
// act on the Base64 data
using (var image = new MagickImage(ms, new MagickReadSettings()))
{
// Sets the output format to png
image.Format = MagickFormat.WebP;
// Write the image to the memorystream
image.Write(ms);
var response = storage.UploadObject("hp-cdn", "images/" + file.Name + "." + MagickFormat.WebP.ToString().ToLower(), "image/webp", ms);
return("https://url/" + response.Name);
}
}
}
public async Task <string> Run(IEnumerable <string> text)
{
// Creates an instance of a speech config with specified subscription key and service region.
// Replace with your own subscription key and service region (e.g., "westus").
// The default language is "en-us".
var config = SpeechConfig.FromSubscription(ConfigHelper.GetTextToSpeechKey(), "francecentral");
config.SetSpeechSynthesisOutputFormat(SpeechSynthesisOutputFormat.Audio16Khz64KBitRateMonoMp3);
// Creates a speech synthesizer using the default speaker as audio output.
using (var synthesizer = new SpeechSynthesizer(config))
{
var storageClient =
await StorageClient.CreateAsync(
GoogleCredential.FromFile(HttpContext.Current.Server.MapPath("~\\keys.json")));
using (var result = await synthesizer.SpeakTextAsync(text.Aggregate((i, j) => i + "\n" + j)))
{
var fileName = Guid.NewGuid() + ".mp3";
await storageClient.UploadObjectAsync("galeata_magica_123", fileName, null,
new MemoryStream(result.AudioData));
return(fileName);
}
}
}
public async Task <Dictionary <int, string> > RunG(DescRequest request)
{
var computerVision =
new ComputerVisionClient(new ApiKeyServiceClientCredentials(ConfigHelper.GetSubscriptionKey()))
{
Endpoint = "https://francecentral.api.cognitive.microsoft.com/"
};
var analysisList = new Dictionary <int, string>();
var storageClient =
await StorageClient.CreateAsync(
GoogleCredential.FromFile(HttpContext.Current.Server.MapPath("~\\keys.json")));
for (var i = 0; i < request.Count; i++)
{
var imageStream = new MemoryStream();
await storageClient.DownloadObjectAsync("galeata_magica_123", $"{request.Filename}/{i}.jpg",
imageStream);
var image = Image.FromStream(imageStream);
var analysis =
await computerVision.AnalyzeImageInStreamAsync(image.ToStream(ImageFormat.Jpeg), Features);
var text = analysis.Description.Captions.FirstOrDefault()?.Text ?? string.Empty;
analysisList.Add(i, text);
}
return(analysisList);
}
public async Task DeleteFileAsync(string bucket, string objectName)
{
using (StorageClient client = await StorageClient.CreateAsync(_credential))
{
await client.DeleteObjectAsync(bucket, objectName);
}
}
public async Task<StoragePutResult> PutAsync(string path, Stream content, string contentType, CancellationToken cancellationToken = default)
{
using (var storage = await StorageClient.CreateAsync())
using (var seekableContent = new MemoryStream())
{
await content.CopyToAsync(seekableContent, 65536, cancellationToken);
seekableContent.Position = 0;
var objectName = CoercePath(path);
try
{
// attempt to upload, succeeding only if the object doesn't exist
await storage.UploadObjectAsync(_bucketName, objectName, contentType, seekableContent, new UploadObjectOptions { IfGenerationMatch = 0 }, cancellationToken);
return StoragePutResult.Success;
}
catch (GoogleApiException e) when (e.HttpStatusCode == HttpStatusCode.PreconditionFailed)
{
// the object already exists; get the hash of its content from its metadata
var existingObject = await storage.GetObjectAsync(_bucketName, objectName, cancellationToken: cancellationToken);
var existingHash = Convert.FromBase64String(existingObject.Md5Hash);
// hash the content that was uploaded
seekableContent.Position = 0;
byte[] contentHash;
using (var md5 = MD5.Create())
contentHash = md5.ComputeHash(seekableContent);
// conflict if the two hashes are different
return existingHash.SequenceEqual(contentHash) ? StoragePutResult.AlreadyExists : StoragePutResult.Conflict;
}
}
}
public async Task <CloudFile> UploadFileAsync(Stream stream, string bucket, string objectName, string filename, string contentType, CancellationToken cancellationToken = default, IProgress <IUploadProgress> progress = null)
{
Progress <Google.Apis.Upload.IUploadProgress> googleProgress = new Progress <Google.Apis.Upload.IUploadProgress>();
if (progress != null)
{
googleProgress.ProgressChanged += (s, e) =>
{
progress.Report(new GoogleUploadProgress(e, stream.Length));
GC.Collect();
};
}
using (StorageClient client = await StorageClient.CreateAsync(_credential))
{
UploadObjectOptions options = new UploadObjectOptions
{
ChunkSize = _chunkSize
};
var dataObject = await client.UploadObjectAsync(bucket, objectName, contentType, stream, options, cancellationToken, googleProgress);
dataObject.ContentDisposition = $"filename=\"{filename}\"";
dataObject = await client.PatchObjectAsync(dataObject);
return(new CloudFile(GetDownloadLink(dataObject), dataObject.MediaLink));
}
}
/// <summary>
/// Helper method to validate that all arguments have been provided, obtain a StorageClient,
/// and then call the given command. We might want to make this an extension method at some point.
/// </summary>
private static void ConfigureForExecution(CommandLineApplication config, Func <StorageClient, Task> command)
{
config.OnExecute(async() =>
{
foreach (var argument in config.Arguments)
{
if (argument.Values.Count == 0)
{
Console.WriteLine($"Argument '{argument.Name}' not specified");
return(1);
}
}
try
{
var client = await StorageClient.CreateAsync();
await command(client);
return(0);
}
catch (Exception e)
{
Console.WriteLine(e);
return(1);
}
});
}
public async Task <IActionResult> Get()
{
var token = HttpContext.Request.Headers["Authorization"].Last().Split(" ").Last();
string[] roles = { "User", "Admin", "SchoolAdmin" };
if (RoleService.CheckRoles(token, roles, _userManager))
{
var candidacyRepo = new CandidacyRepository();
var detailsRepo = new UserDetailsRepository();
var handler = new JwtSecurityTokenHandler();
var sub = handler.ReadJwtToken(token).Payload.Sub;
var details = detailsRepo.GetByUserId(sub);
var candidacy = candidacyRepo.GetAll().Last(x => x.OwnerId.Value == details.Id);
var credentials =
GoogleCredential.FromFile(
PathHelper.GetCredentialsPath());
var storage = StorageClient.CreateAsync(credentials);
var url = SignedUrlHelper.GenerateV4SignedGetUrl("deep-castle-261418-user-photo-bucket",
candidacy.PhotoPath);
return(Ok(url));
}
return(Unauthorized());
}
public async Task DeleteTarget(string targetSetId, string targetId)
{
GoogleCredential cred = this.CreateCredentials();
var channel = new Channel(ProductSearchClient.DefaultEndpoint.Host, ProductSearchClient.DefaultEndpoint.Port, cred.ToChannelCredentials());
try
{
var client = ProductSearchClient.Create(channel);
var storage = await StorageClient.CreateAsync(cred);
IEnumerable <Google.Cloud.Vision.V1.ReferenceImage> referenceImages = await this.GetReferenceImages(client, targetId, 100);
await Task.WhenAll(referenceImages.Select(async r =>
{
await this.DeleteReferenceImage(client, targetId, r.ReferenceImageName.ReferenceImageId);
await this.DeleteFile(storage, this.options.Value.StorageBucketName, r.ReferenceImageName.ReferenceImageId);
}));
await this.RemoveProductFromProductSet(client, targetSetId, targetId);
await this.DeleteProduct(client, targetId);
}
finally
{
await channel.ShutdownAsync();
}
}
public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILogger <Startup> logger)
{
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
}
logger.LogInformation("Service is starting...");
app.UseRouting();
var thumbBucket = GetEnvironmentVariable("BUCKET_THUMBNAILS");
app.UseEndpoints(endpoints =>
{
endpoints.MapPost("/", async context =>
{
JToken pubSubMessage;
using (var reader = new StreamReader(context.Request.Body))
{
pubSubMessage = JValue.Parse(await reader.ReadToEndAsync());
logger.LogInformation("PubSub message: " + pubSubMessage);
}
var data = (string)pubSubMessage["message"]["data"];
var fileEvent = JValue.Parse(Encoding.UTF8.GetString(Convert.FromBase64String(data)));
logger.LogInformation("Base 64 decoded file event: " + fileEvent);
var bucket = (string)fileEvent["bucket"];
var name = (string)fileEvent["name"];
using (var inputStream = new MemoryStream())
{
var client = await StorageClient.CreateAsync();
await client.DownloadObjectAsync(bucket, name, inputStream);
logger.LogInformation($"Downloaded '{name}' from bucket '{bucket}'");
using (var outputStream = new MemoryStream())
{
inputStream.Position = 0; // Reset to read
using (Image image = Image.Load(inputStream))
{
image.Mutate(x => x
.Resize(ThumbWidth, ThumbHeight)
);
logger.LogInformation($"Resized image '{name}' to {ThumbWidth}x{ThumbHeight}");
image.SaveAsPng(outputStream);
}
await client.UploadObjectAsync(thumbBucket, name, "image/png", outputStream);
logger.LogInformation($"Uploaded '{name}' to bucket '{thumbBucket}'");
}
}
});
});
}
public async Task DeleteBookAsync(string userId, int bookId, string bookFileName)
{
var bookPath = $"{_baseBookPath}{userId}/{bookId}/{bookFileName}";
using (var storage = await StorageClient.CreateAsync(_credential))
{
await storage.DeleteObjectAsync(_googleCloudStorageSettings.BucketName, bookPath);
}
}
public async Task<Stream> GetAsync(string path, CancellationToken cancellationToken = default)
{
using (var storage = await StorageClient.CreateAsync())
{
var stream = new MemoryStream();
await storage.DownloadObjectAsync(_bucketName, CoercePath(path), stream, cancellationToken: cancellationToken);
stream.Position = 0;
return stream;
}
}
public async Task UploadBookAsync(Stream file, string userId, int bookId, string bookFileName)
{
var bookPath = $"{_baseBookPath}{userId}/{bookId}/{bookFileName}";
using (var storage = await StorageClient.CreateAsync(_credential))
{
var uploadedObject =
storage.UploadObject(_googleCloudStorageSettings.BucketName, bookPath, null, file);
}
}
public async Task DownloadAsync(string bucketName, string fileName, string destinationFilePath)
{
using (var storageClient = await StorageClient.CreateAsync())
{
using (var fileStream = _fileSystem.FileStream.Create(destinationFilePath, FileMode.OpenOrCreate))
{
await storageClient.DownloadObjectAsync(bucketName, fileName, fileStream);
}
}
}
public async Task <Target> CreateTarget(string displayName, string description, IReadOnlyDictionary <string, string> labels, byte[] referenceImageBinaries)
{
GoogleCredential cred = this.CreateCredentials();
var channel = new Channel(ProductSearchClient.DefaultEndpoint.Host, ProductSearchClient.DefaultEndpoint.Port, cred.ToChannelCredentials());
try
{
var client = ProductSearchClient.Create(channel);
var storage = await StorageClient.CreateAsync(cred);
string productId = Guid.NewGuid().ToString();
var createProductOptions = new CreateProductOptions
{
ProjectID = this.options.Value.ProjectId,
ComputeRegion = this.options.Value.LocationId,
ProductID = productId,
ProductCategory = "apparel",
DisplayName = displayName,
Description = description,
ProductLabels = labels,
};
Product product = await this.CreateProduct(client, createProductOptions);
var addProductOptions = new AddProductToProductSetOptions
{
ProjectID = this.options.Value.ProjectId,
ComputeRegion = this.options.Value.LocationId,
ProductID = product.ProductName.ProductId,
ProductSetId = this.options.Value.ProductSetId,
};
await this.AddProductToProductSet(client, addProductOptions);
string referenceImageId = Guid.NewGuid().ToString();
await this.UploadFile(storage, this.options.Value.StorageBucketName, referenceImageId, referenceImageBinaries);
var createReferenceImageOptions = new CreateReferenceImageOptions
{
ProjectID = this.options.Value.ProjectId,
ComputeRegion = this.options.Value.LocationId,
ProductID = product.ProductName.ProductId,
ReferenceImageID = referenceImageId,
ReferenceImageURI = $"gs://{this.options.Value.StorageBucketName}/{referenceImageId}",
};
Google.Cloud.Vision.V1.ReferenceImage referenceImage = await this.CreateReferenceImage(client, createReferenceImageOptions);
Target target = this.mapper.Map <Target>(product);
target.ReferenceImages = new ReferenceImage[] { this.mapper.Map <ReferenceImage>(referenceImage) };
return(target);
}
finally
{
await channel.ShutdownAsync();
}
}
private static async Task GetDataAsync(string path, GoogleCredential credential)
{
try
{
int pos = path.LastIndexOf("\\");
string strFileName = path.Substring(pos + 1, path.Length - pos - 1);
StorageClient storageClient = await StorageClient.CreateAsync(credential);
storageClient.Service.HttpClient.Timeout = new TimeSpan(0, 10, 0);
var bucket = await storageClient.GetBucketAsync("bucket ocr");
FileStream fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite);
using (MemoryStream memStream = new MemoryStream())
{
await fs.CopyToAsync(memStream);
Google.Apis.Storage.v1.Data.Object googleDataObject;
googleDataObject = await storageClient.UploadObjectAsync(bucket.Name, "sinProcesar/" + strFileName, "application/pdf", memStream);
}
var asyncRequest = new AsyncAnnotateFileRequest
{
InputConfig = new InputConfig
{
GcsSource = new GcsSource
{
Uri = $"gs://{bucket.Name}/sinProcesar/{strFileName}"
},
MimeType = "application/pdf"
},
OutputConfig = new OutputConfig
{
BatchSize = 2,
GcsDestination = new GcsDestination
{
Uri = $"gs://{bucket.Name}/procesados/{strFileName.Split('.')[0]}"
}
}
};
asyncRequest.Features.Add(new Feature
{
Type = Feature.Types.Type.DocumentTextDetection
});
List <AsyncAnnotateFileRequest> requests = new List <AsyncAnnotateFileRequest>();
requests.Add(asyncRequest);
var client = new ImageAnnotatorClientBuilder
{
CredentialsPath = @"D:\Google_API_key_TFM.json"
}.Build();
var operation = client.AsyncBatchAnnotateFiles(requests);
operation.PollUntilCompleted();
}
catch (Exception e) { }
}
private async void DownloadTableData(string bucketName, string objectName)
{
var credential = GoogleCredential.FromFile(" enter credentials");
StorageClient storageClient = await StorageClient.CreateAsync(credential);
using (var fstream = new FileStream(Path.Combine(WebHostEnvironment.WebRootPath, "data", objectName), FileMode.Create))
{
storageClient.DownloadObject(bucketName, objectName, fstream);
}
}
public async Task RenameFileAsync(string bucket, string objectName, string filename)
{
using (StorageClient client = await StorageClient.CreateAsync(_credential))
{
var dataObject = await client.GetObjectAsync(bucket, objectName);
dataObject.ContentDisposition = $"filename=\"{filename}\"";
await client.PatchObjectAsync(dataObject);
}
}
public async Task UploadCoverAsync(Stream cover, string userId, int bookId, string bookFileName)
{
var coverName = $"cover{bookId}.jpg";
var coverPath = $"{_baseBookPath}{userId}/{bookId}/{coverName}";
using (var storage = await StorageClient.CreateAsync(_credential))
{
var uploadedObject =
storage.UploadObject(_googleCloudStorageSettings.BucketName, coverPath, null, cover);
}
}
public async Task <Stream> DownloadBookAsync(string userId, int bookId, string bookFileName)
{
var bookPath = $"{_baseBookPath}{userId}/{bookId}/{bookFileName}";
using (var storage = await StorageClient.CreateAsync(_credential))
{
var stream = new MemoryStream();
await storage.DownloadObjectAsync(_googleCloudStorageSettings.BucketName, bookPath, stream);
return(stream);
}
}
public async Task InitializeAsync(CancellationToken ct = default)
{
try
{
storageClient = await StorageClient.CreateAsync();
await storageClient.GetBucketAsync(bucketName, cancellationToken : ct);
}
catch (Exception ex)
{
throw new ConfigurationException($"Cannot connect to google cloud bucket '{bucketName}'.", ex);
}
}
public async Task <bool> FileExistsAsync(string name)
{
var cachedBytes = await DistributedCache.GetAsync(name);
if (cachedBytes != null && cachedBytes.Length > 0)
{
return(true);
}
_client = _client ?? await StorageClient.CreateAsync();
var obj = await _client.GetObjectAsync(Settings.GoogleStorageBucket, name);
return(obj?.Size > 0);
}
public async Task DeleteAsync(string path, CancellationToken cancellationToken = default)
{
using (var storage = await StorageClient.CreateAsync())
{
try
{
var obj = await storage.GetObjectAsync(_bucketName, CoercePath(path), cancellationToken: cancellationToken);
await storage.DeleteObjectAsync(obj, cancellationToken: cancellationToken);
}
catch (GoogleApiException e) when (e.HttpStatusCode == HttpStatusCode.NotFound)
{
}
}
}
public async Task HandleAsync(HttpContext context)
{
_logger.LogInformation("Function received request");
try
{
var(bucket, file) = await _requestReader.ReadCloudStorageData(context);
using (var inputStream = new MemoryStream())
{
var client = await StorageClient.CreateAsync();
await client.DownloadObjectAsync(bucket, file, inputStream);
_logger.LogInformation($"Downloaded '{file}' from bucket '{bucket}'");
using (var outputStream = new MemoryStream())
{
inputStream.Position = 0; // Reset to read
using (Image image = Image.Load(inputStream))
{
image.Mutate(x => x
.Resize(ThumbWidth, ThumbHeight)
);
_logger.LogInformation($"Resized image '{file}' to {ThumbWidth}x{ThumbHeight}");
image.SaveAsPng(outputStream);
}
var outputFile = $"{Path.GetFileNameWithoutExtension(file)}-{ThumbWidth}x{ThumbHeight}.png";
await client.UploadObjectAsync(_outputBucket, outputFile, "image/png", outputStream);
_logger.LogInformation($"Uploaded '{outputFile}' to bucket '{_outputBucket}'");
var replyData = new { bucket = _outputBucket, file = outputFile };
var json = JsonConvert.SerializeObject(replyData);
_logger.LogInformation($"Replying back with json: {json}");
context.Response.ContentType = "application/json";
await context.Response.WriteAsync(json);
}
}
}
catch (Exception e)
{
_logger.LogError($"Error processing: " + e.Message);
throw e;
}
}
public async Task <IEnumerable <string> > GetListOfFiles(string bucketName, string bucketPath)
{
using (var storageClient = await StorageClient.CreateAsync())
{
var files = new List <string>();
var storageObjects = storageClient.ListObjects(bucketName, bucketPath);
foreach (var storageObject in storageObjects)
{
files.Add(storageObject.Name);
}
return(files);
}
}
static async Task AsyncMain()
{
var client = await StorageClient.CreateAsync();
var buckets = client.ListBucketsAsync("jonskeet-uberproject");
using (var iterator = buckets.GetEnumerator())
{
while (await iterator.MoveNext())
{
var bucket = iterator.Current;
Console.WriteLine(bucket.Name);
}
}
}
private async Task DeleteFromThumbnailsAsync(string objectName, ILogger logger)
{
var client = await StorageClient.CreateAsync();
try
{
await client.DeleteObjectAsync(_bucketThumbnails, objectName);
logger.LogInformation($"Deleted '{objectName}' from bucket '{_bucketThumbnails}'.");
}
catch (Exception e)
{
logger.LogInformation($"Failed to delete '{objectName}' from bucket '{_bucketThumbnails}': {e.Message}.");
}
}
