private GetObjectResponse GetObject()
{
return(_storageClient.GetObject(new GetObjectRequest
{
IncludeMeta = true,
Resource = new Uri(_objectId, UriKind.Relative)
}));
}
public ICloudFileMetadata GetFileMetadata(string fileName, string bucketName = "")
{
EnsureBucketExist(bucketName);
try
{
Object file = _storageClient.GetObject(_bucketName, fileName);
var output = new CloudFileMetadata
{
Bucket = file.Bucket,
Size = file.Size,
Name = file.Name,
LastModified = file.Updated,
ContentType = file.ContentType,
SignedUrl = _urlSigner.Sign(file.Bucket, file.Name, new TimeSpan(1, 0, 0), HttpMethod.Get)
};
return(output);
}
catch (GoogleApiException gex)
{
if (gex.HttpStatusCode == HttpStatusCode.NotFound)
{
_logger.LogError("Not Found - {fileName} in bucket {bucketName}", fileName, _bucketName);
//! References looks for starting part of the following message to identify file not found error
throw new ArgumentException($"Not Found - {fileName} in bucket {_bucketName}", gex);
}
_logger.LogError(-1, gex, gex.Error.Message);
throw;
}
}
// [END bigquery_copy_table]
public string GetFileNameFromCloudStorage(string bucket, string fileName)
{
StorageClient gcsClient = StorageClient.Create();
var file = gcsClient.GetObject(bucket, fileName);
return(file.Name);
}
/// <summary>
/// Returns true if the object exists into the bucket
/// </summary>
private bool ObjectExists(string objectName)
{
try
{
// Only encrypted objects are stored
objectName = $"{objectName}{Constants.EncryptedExt}";
// Remove root slash
if (objectName.StartsWith("/") || objectName.StartsWith("\\"))
{
objectName = objectName.Substring(1, objectName.Length - 1);
}
objectName = objectName.Replace("\\", "/");
return(_client.GetObject(_bucketName, objectName.ToLower()) != null);
}
catch (Google.GoogleApiException ex)
{
if (ex.HttpStatusCode == System.Net.HttpStatusCode.NotFound)
{
return(false);
}
else
{
throw;
}
}
}
public override string CopyAndExtractToTempFolder(string versionId, PackageInfo packageInfo, string tempFolder, string downloadFolder)
{
var objectFileName = Path.GetFileName(versionId);
if (objectFileName == null)
{
throw new InvalidOperationException($"Could not extract file name from object {versionId}");
}
var localFileName = Path.Combine(downloadFolder, packageInfo.Source + "_" + objectFileName);
var metadata = StorageClient.GetObject(Bucket, versionId);
if (DirectoryUtil.Exists(localFileName) && metadata.Md5Hash == DirectoryUtil.Md5(localFileName))
{
_log.Info("Using cached file: " + localFileName);
}
else
{
DownloadPackage(versionId, localFileName);
}
Extract(localFileName, tempFolder, packageInfo.InternalPath);
return(Path.Combine(tempFolder, packageInfo.InternalPath));
}
/// <summary>
/// Determine source file object matches with the remote object.
/// </summary>
public bool IsMetadataMatch(string fullpath, bool isEncrypted, string sourceOriginalMd5)
{
string destDataFile = $"{fullpath}{Constants.EncryptedExt}".Replace("\\", "/");
if (destDataFile.StartsWith("/"))
{
destDataFile = destDataFile.Substring(1, destDataFile.Length - 1);
}
try
{
var obj = _client.GetObject(_bucketName, destDataFile.ToLower());
string value;
if (obj.Metadata.TryGetValue(Constants.OriginalMD5Key, out value))
{
return(value == sourceOriginalMd5);
}
return(false);
}
catch (Google.GoogleApiException ex)
{
if (ex.HttpStatusCode == System.Net.HttpStatusCode.NotFound)
{
return(false);
}
else
{
throw;
}
}
}
public void TestExtractTableJson()
{
var snippet = new BigQueryExtractTableJson();
snippet.ExtractTableJson(_projectId, _bucketName);
var uploadedFile = _storage.GetObject(_bucketName, "shakespeare.json");
Assert.True(uploadedFile.Size > 0);
}
public string GetBlobUrl(string containerName, string blobName)
{
try
{
return(_client.GetObject(_bucket, ObjectName(containerName, blobName)).MediaLink);
}
catch (GoogleApiException gae)
{
throw Error(gae);
}
}
private bool FileExistsInCloud(string objectName)
{
try
{
storage.GetObject(bucketName, objectName);
}
catch (Google.GoogleApiException)
{
return(false);
}
return(true);
}
public static Google.Apis.Storage.v1.Data.Object TryGetObject(this StorageClient client, string bucket, string objectName)
{
if (client == null)
{
throw new ArgumentNullException("this");
}
try { return(client.GetObject(bucket, objectName)); }
catch (Exception ex) {
Debug.WriteLine(ex.GetType().Name);
throw;
}
}
/// <summary>
///
/// <para>GetFileSize:</para>
///
/// <para>Gets size of a file in bytes from File Service, caller thread will be blocked before it is done</para>
///
/// <para>Check <seealso cref="IBFileServiceInterface.GetFileSize"/> for detailed documentation</para>
///
/// </summary>
public bool GetFileSize(
string _BucketName,
string _KeyInBucket,
out ulong _FileSize,
Action <string> _ErrorMessageAction = null)
{
_FileSize = 0;
if (GSClient == null)
{
_ErrorMessageAction?.Invoke("BFileServiceGC->GetFileSize: GSClient is null.");
return(false);
}
try
{
var ResultObject = GSClient.GetObject(_BucketName, _KeyInBucket);
if (ResultObject == null || !ResultObject.Size.HasValue)
{
_ErrorMessageAction?.Invoke("BFileServiceGC->GetFileSize: GetObject Response is null or Size object does not have value.");
return(false);
}
_FileSize = ResultObject.Size.Value;
}
catch (Exception e)
{
_ErrorMessageAction?.Invoke("BFileServiceGC->GetFileSize: " + e.Message + ", Trace: " + e.StackTrace);
return(false);
}
return(true);
}
public static CacheValue Refresh(CacheValue previous, StorageClient client)
{
var containerObjects = client.ListObjects(BucketName, ContainerObjectsPrefix).Select(obj => obj.Name).ToList();
var newContainerNames = containerObjects.Except(previous.runsByStorageName.Keys).ToList();
// If there are no new runs to load, use the previous set even if there are new environments.
// An empty environment is fairly pointless... (and we don't mind too much if there are old ones that have been removed;
// they'll go when there's next a new one.)
if (!newContainerNames.Any())
{
return(previous);
}
// We won't reload everything from storage, but we'll come up with completely new set of objects each time, so we
// don't need to worry about the changes involved as we reattach links.
var environmentObject = client.GetObject(BucketName, EnvironmentObjectName);
var environments = environmentObject.Crc32c == previous.environmentCrc32c
? previous.Environments.Select(env => { var clone = env.Clone(); clone.Runs.Clear(); return(clone); }).ToList()
: LoadEnvironments(client).OrderBy(e => e.Machine).ThenBy(e => e.TargetFramework).ThenBy(e => e.OperatingSystem).ToList();
// Don't just use previous.runsByStorageName blindly - some may have been removed.
var runsByStorageName = new Dictionary <string, BenchmarkRun>();
foreach (var runStorageName in containerObjects)
{
if (previous.runsByStorageName.TryGetValue(runStorageName, out var runToAdd))
{
runToAdd = runToAdd.Clone();
}
else
{
runToAdd = LoadContainer(client, runStorageName);
}
runToAdd.Environment = environments.FirstOrDefault(env => env.BenchmarkEnvironmentId == runToAdd.BenchmarkEnvironmentId);
// If we don't have an environment, that's a bit worrying - skip and move on.
if (runToAdd.Environment == null)
{
// TODO: Use a proper logger
Console.WriteLine($"Run {runStorageName} has no environment. Skipping.");
continue;
}
runToAdd.PopulateLinks();
runsByStorageName[runStorageName] = runToAdd;
}
// Attach the runs to environments, in reverse chronological order
foreach (var run in runsByStorageName.Values.OrderByDescending(r => r.Start.ToInstant()))
{
run.Environment.Runs.Add(run);
}
return(new CacheValue(environments, environmentObject.Crc32c, runsByStorageName));
}
public void DeleteFile(string fileNameForStorage)
{
try
{
if (storageClient.GetObject(bucketName, fileNameForStorage) != null)
{
storageClient.DeleteObject(bucketName, fileNameForStorage);
}
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
/// <summary> Gets the Object at the path. If it's not found,
/// doesn't throw an exception; instead returns null. </summary>
Google.Apis.Storage.v1.Data.Object GetObject(ObjectPath path)
{
try
{
return(_storage.GetObject(path.BucketName,
path.ObjectName));
}
catch (Google.GoogleApiException e)
when(e.HttpStatusCode == HttpStatusCode.NotFound)
{
_logger.LogWarning("{0}/{1} not found.", path.BucketName,
path.ObjectName);
return(null);
}
}
public IActionResult GetImage([FromQuery] string name)
{
var stream = new MemoryStream();
try
{
stream.Position = 0;
var obj = _storageClient.GetObject("tenant-file-fc6de.appspot.com", name);
_storageClient.DownloadObject(obj, stream);
stream.Position = 0;
var response = File(stream, obj.ContentType, "file.png"); // FileStreamResult
return(response);
}
catch (Exception ex)
{
throw ex;
}
}
public async Task <string> LoadTextFileAsync(string filename)
{
Object item = _storageClient.GetObject(_configuration.BucketName, filename);
if (item != null)
{
using (Stream stream = new MemoryStream())
{
_storageClient.DownloadObject(item, stream);
stream.Position = 0;
using (var reader = new StreamReader(stream))
{
return(await reader.ReadToEndAsync());
}
}
}
return("");
}
public IActionResult GetImage([FromQuery] string name)
{
var stream = new MemoryStream();
try
{
stream.Position = 0;
var obj = _storageClient.GetObject("tenant-file-fc6de.appspot.com", name);
_storageClient.DownloadObject(obj, stream);
stream.Position = 0;
FileStreamResult response = File(stream, obj.ContentType, $"{name}"); // FileStreamResult
return(response);
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
return(new BadRequestResult());
}
}
public void CopyObject()
{
var projectId = _fixture.ProjectId;
var sourceBucket = _fixture.BucketName;
var destinationBucket = IdGenerator.FromGuid();
StorageClient.Create().CreateBucket(projectId, destinationBucket);
StorageSnippetFixture.SleepAfterBucketCreateDelete();
_fixture.RegisterBucketToDelete(destinationBucket);
// Snippet: CopyObject
StorageClient client = StorageClient.Create();
string sourceName = "greetings/hello.txt";
string destinationName = "copy.txt";
// This method actually uses the "rewrite" API operation, for added reliability
// when copying large objects across locations, storage classes or encryption keys.
client.CopyObject(sourceBucket, sourceName, destinationBucket, destinationName);
// End snippet
var obj = client.GetObject(destinationBucket, destinationName);
Assert.Equal((ulong)Encoding.UTF8.GetByteCount(_fixture.HelloWorldContent), obj.Size.Value);
}
static void Main()
{
var demoContent = File.ReadAllBytes(Path.Combine(Environment.CurrentDirectory, "DemoFile.txt"));
var storageClient = new StorageClient("5cd104e23fc947668a6c74fe63fd77e7/godbold_1310683369246",
"FGJeXUzxCz5poHoSzRzmMTceuek=");
var storedObjectResponse = storageClient.CreateObject(new CreateObjectRequest
{
Content = demoContent,
Resource =
new Uri("objects", UriKind.Relative),
GroupACL = "other=NONE",
ACL = "godbold=FULL_CONTROL",
Metadata = "part1=buy",
ListableMetadata =
"part4/part7/part8=quick"
});
Console.WriteLine("Object stored at {0}", storedObjectResponse.Location);
Console.ReadKey();
var namespaceCreateResponse = storageClient.CreateObject(new CreateObjectRequest
{
Resource =
new Uri("namespace/test/profiles of stuff/", UriKind.Relative),
GroupACL = "other=NONE",
ACL = "godbold=FULL_CONTROL",
Metadata = "part1=buy",
ListableMetadata =
"part4/part7/part8=quick"
});
Console.WriteLine("Namespace created at {0}", namespaceCreateResponse.Location);
Console.ReadKey();
var getNamespaceResponse =
storageClient.GetObject(new GetObjectRequest
{
Resource = new Uri(namespaceCreateResponse.Location, UriKind.Relative),
});
var namespaceContent = Encoding.ASCII.GetChars(getNamespaceResponse.Content, 0, getNamespaceResponse.Content.Length);
var namespaceContentString = new string(namespaceContent);
Console.WriteLine("Namespace {0} retrieved", namespaceCreateResponse.Location);
Console.WriteLine("Content: {0}", namespaceContentString);
Console.ReadKey();
Console.WriteLine("Deleting namespace at {0}", namespaceCreateResponse.Location);
storageClient.DeleteObject(new DeleteObjectRequest
{
Resource = new Uri(namespaceCreateResponse.Location, UriKind.Relative)
});
Console.WriteLine("Namespace at {0} deleted", namespaceCreateResponse.Location);
Console.ReadKey();
var getFullObjectResponse =
storageClient.GetObject(new GetObjectRequest
{
Resource = new Uri(storedObjectResponse.Location, UriKind.Relative),
});
var allContent = Encoding.ASCII.GetChars(getFullObjectResponse.Content, 0, getFullObjectResponse.Content.Length);
var allContentString = new string(allContent);
Console.WriteLine("Object {0} retrieved", storedObjectResponse.Location);
Console.WriteLine("Content: {0}", allContentString);
Console.ReadKey();
var getObjectResponse =
storageClient.GetObject(new GetObjectRequest
{
Resource = new Uri(storedObjectResponse.Location, UriKind.Relative),
LowerRange = 10
});
var content = Encoding.ASCII.GetChars(getObjectResponse.Content, 0, getObjectResponse.Content.Length);
var contentString = new string(content);
Console.WriteLine("Object {0} retrieved", storedObjectResponse.Location);
Console.WriteLine("Content: {0}", contentString);
Console.ReadKey();
Console.WriteLine("Updating object stored at {0}", storedObjectResponse.Location);
storageClient.UpdateObject(new UpdateObjectRequest
{
Resource = new Uri(storedObjectResponse.Location, UriKind.Relative),
Content = demoContent
});
Console.WriteLine("Object at {0} was updated", storedObjectResponse.Location);
Console.ReadKey();
Console.WriteLine("Deleting object stored at {0}", storedObjectResponse.Location);
storageClient.DeleteObject(new DeleteObjectRequest
{
Resource = new Uri(storedObjectResponse.Location, UriKind.Relative)
});
Console.WriteLine("Object at {0} deleted", storedObjectResponse.Location);
Console.ReadKey();
}
public StorageFile GetObject(string name) => ConvertObject(client.GetObject(bucket, name));
/// <summary>
/// Restore the object for a specific date time
/// </summary>
public bool Restore(ObjectVersion version, ref string destination)
{
try
{
using (StorageClient _client = StorageClient.Create(GoogleCredential.FromFile(_apiKey)))
{
var gen = new GetObjectOptions()
{
Generation = version.Generation
};
var obj = _client.GetObject(_bucketName, version.Name, gen);
string originalMD5, metadataMD5, metadataEncrypted;
if (!obj.Metadata.TryGetValue(Constants.OriginalMD5Key, out originalMD5) ||
!obj.Metadata.TryGetValue(Constants.MetadataMD5Key, out metadataMD5) ||
!obj.Metadata.TryGetValue(Constants.MetadataEncryptedKey, out metadataEncrypted))
{
_logger.WriteLog(ErrorCodes.GcsRestore_MissingMetadata,
string.Format(ErrorResources.GcsRestore_MissingMetadata, obj.Name),
Severity.Error, VerboseLevel.User);
return(false);
}
FileObject fo = new FileObject()
{
Metadata = new FileMetadata()
{
OriginalMD5 = originalMD5
},
DataContent = null,
IsSecured = true,
MetadataMD5 = metadataMD5,
MetadataContent = metadataEncrypted
};
DownloadObject(_client, obj, fo, version);
Helpers.WriteProgress("Decrypting...");
fo = new UnsecureTransform(_crypto_key, _crypto_iv, _logger).Process(fo);
Helpers.WriteProgress("Saving...");
Helpers.WriteProgress("");
// Force a destination name if not given by the user
if (string.IsNullOrEmpty(destination))
{
string destName = version.Name.Replace(Constants.EncryptedExt, "");
destination = Path.GetFileNameWithoutExtension(destName);
destination = $"{destination}.{version.Generation}.restore{Path.GetExtension(destName)}";
}
File.WriteAllBytes(destination, fo.DataContent);
File.SetAttributes(destination, fo.Metadata.Attributes);
File.SetLastWriteTime(destination, fo.Metadata.LastWriteTime);
return(true);
}
}
catch (Exception ex)
{
_logger.WriteLog(ErrorCodes.GcsRestore_RestoreObjectException,
ErrorResources.GcsRestore_RestoreObjectException + Environment.NewLine + ex.Message,
Severity.Error, VerboseLevel.User);
return(false);
}
}
public override AsimovVersion GetVersion(string versionId, PackageInfo packageInfo)
{
var @object = StorageClient.GetObject(Bucket, versionId);
return(ParseVersion(@object));
}
