/// <summary>
/// Writes the object describing the item to the output. Used by Get-Item.
/// </summary>
/// <param name="path">The path of the item to get.</param>
protected override void GetItem(string path)
{
var gcsPath = GcsPath.Parse(path);
switch (gcsPath.Type)
{
case GcsPath.GcsPathType.Drive:
WriteItemObject(PSDriveInfo, path, true);
break;
case GcsPath.GcsPathType.Bucket:
Bucket bucket;
var bucketCache = BucketCache.Value;
if (bucketCache.ContainsKey(gcsPath.Bucket))
{
bucket = bucketCache[gcsPath.Bucket];
}
else
{
bucket = Service.Buckets.Get(gcsPath.Bucket).Execute();
}
WriteItemObject(bucket, path, true);
break;
case GcsPath.GcsPathType.Object:
Object gcsObject = GetBucketModel(gcsPath.Bucket).GetGcsObject(gcsPath.ObjectPath);
WriteItemObject(gcsObject, path, IsItemContainer(path));
break;
default:
throw new InvalidOperationException($"Unknown Path Type {gcsPath.Type}");
}
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(GetItem));
}
public List <Google.Apis.Storage.v1.Data.Object> GetFoldersAndBlobs(string bucketName, string prefix, string delimiter)
{
List <string> folderNames = new List <string>()
{
};
List <Google.Apis.Storage.v1.Data.Object> blobs = new List <Google.Apis.Storage.v1.Data.Object>()
{
};
Google.Apis.Storage.v1.Data.Objects response = GetBlobsRaw(bucketName, prefix, delimiter);
if (response.Prefixes != null)
{
folderNames = response.Prefixes.ToList();
folderNames.ForEach(fn =>
{
Google.Apis.Storage.v1.Data.Object folderObj = GetBlob(bucketName, fn);
if (null != folderObj)
{
blobs.Add(folderObj);
}
});
}
if (response.Items != null)
{
blobs.AddRange(response.Items.ToList());
}
return(blobs);
}
public async Task <bool> UploadAsync(string filename, MemoryStream mem, string datatype, string dirName)
{
IConfigurableHttpClientInitializer credentials = GetApplicationDefaultCredentials();
StorageService service = new StorageService(
new BaseClientService.Initializer()
{
HttpClientInitializer = credentials,
ApplicationName = "NewsSwipes",
});
// To make public
var acl = new List <ObjectAccessControl>
{
new ObjectAccessControl
{
Role = "OWNER",
Entity = "allUsers"
}
};
filename = String.Format("{0}/{1}", dirName, filename);
var fileobj = new Google.Apis.Storage.v1.Data.Object()
{
Name = filename, Acl = acl
};
await service.Objects.Insert(fileobj, bucketName, mem, datatype).UploadAsync();
return(true);
}
public void Send(FileLogInfo file)
{
var bucketName = this.bucket ?? file.Bucket;
if (string.IsNullOrEmpty(bucketName))
{
return;
}
var contentType = Atoms.MimeType.GetMimeType(file.ContentType);
var contentEncoding = file.ContentEncoding;
var obj = new Google.Apis.Storage.v1.Data.Object
{
Name = file.FileName,
ContentType = contentType,
ContentEncoding = contentEncoding,
Bucket = bucketName,
};
if (dry)
{
Console.WriteLine($"DRY: push {file.FileName}");
var sReader = new StreamReader(file.Content);
var content = sReader.ReadToEnd();
Console.WriteLine(content);
}
else
{
client.UploadObject(obj, file.Content, this.opts, progress);
}
}
/// <summary>
/// Gets the Google Cloud Storage object of a given object name.
/// </summary>
public Object GetGcsObject(string objectName)
{
if (_objectMap.ContainsKey(objectName))
{
return(_objectMap[objectName]);
}
else if (_prefixes.ContainsKey(objectName))
{
if (_objectMap.ContainsKey(objectName + Separator))
{
return(_objectMap[objectName + Separator]);
}
else
{
return(new Object {
Bucket = _bucket, Name = objectName, ContentType = "Folder"
});
}
}
else
{
ObjectsResource.GetRequest request = _service.Objects.Get(_bucket, objectName);
request.Projection = ObjectsResource.GetRequest.ProjectionEnum.Full;
Object gcsObject = request.Execute();
_objectMap[gcsObject.Name] = gcsObject;
return(gcsObject);
}
}
private BlobDescriptor GetBlobDescriptor(Blob blob)
{
var match = Regex.Match(blob.Name, BlobNameRegex);
if (!match.Success)
{
throw new InvalidOperationException("Unable to match blob name with regex; all blob names");
}
var blobDescriptor = new BlobDescriptor
{
Container = match.Groups["Container"].Value,
ContentMD5 = blob.Md5Hash,
ContentType = blob.ContentType,
ETag = blob.ETag,
LastModified = blob.Updated,
Length = (long)blob.Size.GetValueOrDefault(),
Name = match.Groups["Blob"].Value,
Security = blob.Acl != null &&
blob.Acl.Any(acl => acl.Entity.ToLowerInvariant() == "allusers") ? BlobSecurity.Public : BlobSecurity.Private,
Url = blob.MediaLink
};
return(blobDescriptor);
}
public void UriEscaping()
{
StorageService client = new StorageService(new BaseClientService.Initializer
{
HttpClientInitializer = Helper.GetServiceCredential().CreateScoped(StorageService.Scope.DevstorageFullControl),
ApplicationName = "IntegrationTest",
});
var bucket = CreateTestBucket(client);
try
{
const string name = "foo [baz] ";
var obj = new Google.Apis.Storage.v1.Data.Object {
Name = name
};
var data = "Hello test!";
// Upload to GCS using a name with special characters
var stUpload = new MemoryStream(Encoding.ASCII.GetBytes(data));
var progress = client.Objects.Insert(obj, bucket.Name, stUpload, "text/plain").Upload();
Assert.Equal(UploadStatus.Completed, progress.Status);
Assert.Equal(stUpload.Length, progress.BytesSent);
// Download the same object and verify
var stDownload = new MemoryStream();
client.Objects.Get(bucket.Name, name).Download(stDownload);
Assert.Equal(stUpload.ToArray(), stDownload.ToArray());
}
finally
{
DeleteTestBuckets(client);
}
}
/// <summary>
/// Checks if the given object is a real object. An object could "exist" but not be "real" if it is a
/// prefix for another object (a logical folder that is not "real").
/// </summary>
/// <param name="objectName">The name of the object to check.</param>
/// <returns>True if the object actually exists.</returns>
public bool IsReal(string objectName)
{
if (_objectMap.ContainsKey(objectName))
{
return(true);
}
else if (!_pageLimited)
{
return(false);
}
else
{
try
{
ObjectsResource.GetRequest request = _service.Objects.Get(_bucket, objectName);
request.Projection = ObjectsResource.GetRequest.ProjectionEnum.Full;
Object gcsObject = request.Execute();
_objectMap[gcsObject.Name] = gcsObject;
return(true);
}
catch (GoogleApiException e)
{
if (e.HttpStatusCode == HttpStatusCode.NotFound)
{
return(false);
}
else
{
throw;
}
}
}
}
public Google.Apis.Storage.v1.Data.Object CreateSubDir(string bucketName, string blobName)
{
string subDir = blobName.EndsWith("/") ? blobName : blobName + "/";
Google.Apis.Storage.v1.Data.Object blob = CreateEmptyBlob(bucketName, subDir);
return(blob);
}
/// <summary>
/// Clears the content of an object. Used by Clear-Content.
/// </summary>
/// <param name="path">The path of the object to clear.</param>
public void ClearContent(string path)
{
if (!ShouldProcess(path, "Clear-Content"))
{
return;
}
var gcsPath = GcsPath.Parse(path);
Object body = new Object
{
Name = gcsPath.ObjectPath,
Bucket = gcsPath.Bucket
};
var memoryStream = new MemoryStream();
var contentType = GcsCmdlet.UTF8TextMimeType;
ObjectsResource.InsertMediaUpload request =
Service.Objects.Insert(body, gcsPath.Bucket, memoryStream, contentType);
IUploadProgress response = request.Upload();
if (response.Exception != null)
{
throw response.Exception;
}
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(ClearContent));
}
public void Delete(string objectName, GxFileType fileType)
{
Google.Apis.Storage.v1.Data.Object obj = new Google.Apis.Storage.v1.Data.Object();
obj.Name = objectName;
obj.Bucket = Bucket;
Client.DeleteObject(obj);
}
protected virtual ObjectsResource.InsertMediaUpload PrepareUploadRequest(string bucketName, string fileName, Stream fileStream, Action <IUploadProgress> onProgresChanged = null, Action <Object> success = null, Access access = Access.Public, string mimeType = null)
{
var newObject = new Object()
{
Bucket = bucketName,
Name = fileName,
};
mimeType = mimeType ?? MimeMapping.GetMimeMapping(fileName);
var uploadRequest = new ObjectsResource.InsertMediaUpload(Service, newObject, bucketName, fileStream,
mimeType);
uploadRequest.OauthToken = AccessToken;
uploadRequest.PredefinedAcl = access.ForUpload();
if (onProgresChanged != null)
{
uploadRequest.ProgressChanged += onProgresChanged;
}
if (success != null)
{
uploadRequest.ResponseReceived += success;
}
return(uploadRequest);
}
public string Upload(string localFile, string objectName, GxFileType fileType)
{
using (FileStream stream = new FileStream(localFile, FileMode.Open))
{
Google.Apis.Storage.v1.Data.Object obj = Client.UploadObject(Bucket, objectName, "application/octet-stream", stream, GetUploadOptions(fileType));
return(obj.MediaLink);
}
}
public Google.Apis.Storage.v1.Data.Object CreateEmptyBlob(string bucketName, string blobName)
{
var content = Encoding.UTF8.GetBytes("");
string contentType = MimeMapping.GetMimeMapping(Path.GetFileName(blobName));
Google.Apis.Storage.v1.Data.Object blob = this.storage.UploadObject(bucketName, blobName, contentType, new MemoryStream(content));
return(blob);
}
/// <summary>
/// Updates meta data of an object.
/// </summary>
/// <param name="obj">object to update, with new meta data</param>
/// <returns></returns>
public Object UpdateObject(Object obj)
{
var updateQuery = Service.Objects.Update(obj, obj.Bucket, obj.Name);
updateQuery.OauthToken = AccessToken;
var updated = updateQuery.Execute();
return(updated);
}
public string Copy(string url, string newName, string tableName, string fieldName, GxFileType fileType)
{
newName = Folder + StorageUtils.DELIMITER + tableName + StorageUtils.DELIMITER + fieldName + StorageUtils.DELIMITER + newName;
url = StorageUtils.DecodeUrl(url.Replace(StorageUri, string.Empty));
Google.Apis.Storage.v1.Data.Object obj = Client.CopyObject(Bucket, url, Bucket, newName, GetCopyOptions(fileType));
obj.Metadata = CreateObjectMetadata(tableName, fieldName, newName);
Client.UpdateObject(obj);
return(GetURL(newName, fileType, 0));
}
public UploadResult UploadFile(IFormFile file, string fileName, bool encryption, bool compressing)
{
var newObject = new Google.Apis.Storage.v1.Data.Object()
{
Bucket = BucketToUpload,
Name = fileName
};
string encryptionKey = null, iv = null;
// Actions with data here encrypting / compressing
var fileStream = file.OpenReadStream();
var fileBytes = ReadToEnd(fileStream);
#region Encryption and compressing
if (compressing)
{
fileBytes = Compress(fileBytes);
}
if (encryption)
{
var actionData = GenerateKeyAndIV(FileActionsConstants.AESFlavour);
fileBytes = Encrypt(fileBytes, actionData.Item1, actionData.Item2);
encryptionKey = Convert.ToBase64String(actionData.Item1);
iv = Convert.ToBase64String(actionData.Item2);
}
#endregion
Stream fileOutStream = new MemoryStream(fileBytes);
try
{
var uploadRequest = new ObjectsResource.InsertMediaUpload(StorageService, newObject,
BucketToUpload, fileOutStream, encryption || compressing ? "application/octet-stream" : file.ContentType);
var res = uploadRequest.Upload();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
return(new UploadResult {
Success = false
});
}
finally
{
if (fileStream != null)
{
fileStream.Dispose();
}
}
return(new UploadResult(true, encryptionKey, iv));
}
public void SetGcsProperties()
{
GcsService gcsService = GcsService.Instance();
this.pathType = GcsPathType.INVALID;
try
{
if ("/".Equals(this.FullPath))
{
/* root directory : /:*/
this.pathType = GcsPathType.ROOT;
return;
}
/* validate the bucket name */
if (!this.Bucket.Equals("") && Uri.CheckHostName(this.Bucket) == 0)
{
return;
}
if (!this.Bucket.Equals("") && this.Blob.Equals(""))
{
/* bucket: /bucket */
Bucket bucket = gcsService.GetBucket(this.Bucket);
if (null != bucket)
{
this.pathType = GcsPathType.BUCKET;
this.Created = bucket.TimeCreated;
this.Updated = bucket.Updated;
this.Size = 0L;
}
}
else if (!this.Bucket.Equals("") && !this.Blob.Equals(""))
{
Google.Apis.Storage.v1.Data.Object blob = gcsService.GetBlob(this.Bucket, this.Blob);
if (null != blob && !blob.Name.EndsWith("/"))
{
/* blob: /container/folder/file1 */
this.pathType = GcsPathType.BLOB;
this.Created = blob.TimeCreated;
this.Updated = blob.Updated;
this.Size = blob.Size;
}
else if (gcsService.SubDirExists(this.Bucket, this.Blob))
{
/* virtual directory : /container/folder/ */
this.pathType = GcsPathType.SUBDIR;
}
}
//Utils.WriteLine("pathType:" + this.pathType);
}
catch (Exception ex)
{
Logger.Log(Logger.Level.Error, ex.Message);
}
}
/// <summary>
/// Validates that the given Object has a "somewhat valid" (no URI encoding required) bucket name and an object name.
/// </summary>
/// <param name="obj">Object to validate</param>
/// <param name="paramName">The parameter name in the calling method</param>
private void ValidateObject(Object obj, string paramName)
{
GaxPreconditions.CheckNotNull(obj, paramName);
GaxPreconditions.CheckArgument(
obj.Name != null && obj.Bucket != null,
paramName,
"Object must have a name and bucket");
GaxPreconditions.CheckArgument(ValidBucketName.IsMatch(obj.Bucket),
paramName,
"Object bucket '{0}' is invalid", obj.Bucket);
}
public Google.Apis.Storage.v1.Data.Object GetBlob(string bucketName, string blobName)
{
Google.Apis.Storage.v1.Data.Object blob = null;
try
{
blob = this.storage.GetObject(bucketName, blobName);
}
catch (Google.GoogleApiException e)
when(e.Error.Code == 404)
{
return(null);
}
return(blob);
}
/// <summary>
/// Copies a Google Cloud Storage object or folder to another object or folder. Used by Copy-Item.
/// </summary>
/// <param name="path">The path to copy from.</param>
/// <param name="copyPath">The path to copy to.</param>
/// <param name="recurse">If true, will copy all decendent objects as well.</param>
protected override void CopyItem(string path, string copyPath, bool recurse)
{
if (!ShouldProcess($"Copy-Item from {path} to {copyPath}"))
{
return;
}
var dyanmicParameters = (GcsCopyItemDynamicParameters)DynamicParameters;
if (recurse)
{
path = path.TrimEnd('\\') + "\\";
copyPath = copyPath.TrimEnd('\\') + "\\";
}
var gcsPath = GcsPath.Parse(path);
var gcsCopyPath = GcsPath.Parse(copyPath);
if (recurse)
{
IEnumerable <Object> children = ListChildren(gcsPath, true);
foreach (Object child in children)
{
string objectSubPath = gcsPath.RelativePathToChild(child.Name);
string destinationObject = GcsPath.Parse(MakePath(copyPath, objectSubPath)).ObjectPath;
ObjectsResource.CopyRequest childRequest = Service.Objects.Copy(null, child.Bucket,
child.Name, gcsCopyPath.Bucket, destinationObject);
childRequest.SourceGeneration = dyanmicParameters.SourceGeneration;
childRequest.DestinationPredefinedAcl = dyanmicParameters.DestinationAcl;
childRequest.Projection = ObjectsResource.CopyRequest.ProjectionEnum.Full;
Object childObject = childRequest.Execute();
bool isContainer = (new GcsPath(childObject).Type != GcsPath.GcsPathType.Object);
WriteItemObject(childObject, copyPath, isContainer);
}
}
if (!recurse || GetBucketModel(gcsPath.Bucket).IsReal(gcsPath.ObjectPath))
{
ObjectsResource.CopyRequest request =
Service.Objects.Copy(null, gcsPath.Bucket, gcsPath.ObjectPath, gcsCopyPath.Bucket,
gcsCopyPath.ObjectPath);
request.SourceGeneration = dyanmicParameters.SourceGeneration;
request.DestinationPredefinedAcl = dyanmicParameters.DestinationAcl;
request.Projection = ObjectsResource.CopyRequest.ProjectionEnum.Full;
Object response = request.Execute();
WriteItemObject(response, copyPath, gcsCopyPath.Type != GcsPath.GcsPathType.Object);
}
BucketModels.Clear();
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(CopyItem));
}
public bool SubDirExists(string bucketName, string subDir)
{
subDir = subDir.EndsWith("/") ? subDir : subDir + "/";
Google.Apis.Storage.v1.Data.Object blob = this.GetBlob(bucketName, subDir);
if (null != blob && blob.Size == 0)
{
return(true);
}
List <Google.Apis.Storage.v1.Data.Object> blobs = this.GetBlobs(bucketName, subDir, "/");
if (blobs.Count > 0)
{
return(true);
}
return(false);
}
protected async Task CreateNewObject(string container, string blobName, Stream data, bool isPublic = false,
string contentType = null)
{
var blob = new Blob
{
Name = string.Format(ContainerBlobFormat, container, blobName),
//TODO: Figure out how the hell ACL has got to be tweaked to actually work. Currently this does not do it, and the .NET api does not expose the ability to set the query parameter "predefinedAcl" which would be perfect for our needs here.
ContentType = contentType ?? DefaultContentType
};
await _client.Objects.Insert(blob, Bucket, data, contentType ?? DefaultContentType).UploadAsync();
if (isPublic)
{
await _client.ObjectAccessControls.Insert(PublicAcl, Bucket, blob.Name).ExecuteAsync();
}
}
/// <summary>
/// Gets a content reader to read the contents of a downloaded Google Cloud Storage object.
/// Used by Get-Contents.
/// </summary>
/// <param name="path">The path to the object to read.</param>
/// <returns>A content reader of the contents of a given object.</returns>
public IContentReader GetContentReader(string path)
{
var gcsPath = GcsPath.Parse(path);
if (gcsPath.ObjectPath == null)
{
throw new InvalidOperationException($"Can not get the contents of a {gcsPath.Type}");
}
Object gcsObject = Service.Objects.Get(gcsPath.Bucket, gcsPath.ObjectPath).Execute();
var stream = Service.HttpClient.GetStreamAsync(gcsObject.MediaLink).Result;
IContentReader contentReader = new GcsStringReader(stream);
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(GetContentReader));
return(contentReader);
}
public string Upload(string fileName, Stream stream, GxFileType fileType)
{
Google.Apis.Storage.v1.Data.Object obj = new Google.Apis.Storage.v1.Data.Object();
obj.Name = fileName;
obj.Bucket = Bucket;
if (Path.GetExtension(fileName).Equals(".tmp"))
{
obj.ContentType = "image/jpeg";
}
else
{
obj.ContentType = MimeMapping.GetMimeMapping(fileName);
}
Client.UploadObject(obj, stream, GetUploadOptions(fileType));
return(StorageUri + StorageUtils.EncodeUrl(fileName));
}
/// <summary>
/// Writes the object describing the item to the output. Used by Get-Item.
/// </summary>
/// <param name="path">The path of the item to get.</param>
protected override void GetItem(string path)
{
var gcsPath = GcsPath.Parse(path);
switch (gcsPath.Type)
{
case GcsPath.GcsPathType.Drive:
WriteItemObject(PSDriveInfo, path, true);
break;
case GcsPath.GcsPathType.Bucket:
Dictionary <string, Bucket> bucketDict = null;
Bucket bucket;
// If the bucket cache is not initialized, then don't bother initializing it
// because that will cause a long wait time and we may not even know whether
// the user needs to use all the other buckets right away. Also, we should not
// refresh the whole cache right at this instance (which is why we call
// GetValueWithoutUpdate) for the same reason.
bucketDict = BucketCache.GetLastValueWithoutUpdate();
if (bucketDict != null && bucketDict.ContainsKey(gcsPath.Bucket))
{
bucket = bucketDict[gcsPath.Bucket];
break;
}
bucket = Service.Buckets.Get(gcsPath.Bucket).Execute();
if (bucketDict != null)
{
bucketDict[bucket.Name] = bucket;
}
WriteItemObject(bucket, path, true);
break;
case GcsPath.GcsPathType.Object:
Object gcsObject = GetBucketModel(gcsPath.Bucket).GetGcsObject(gcsPath.ObjectPath);
WriteItemObject(gcsObject, path, IsItemContainer(path));
break;
default:
throw new InvalidOperationException($"Unknown Path Type {gcsPath.Type}");
}
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(GetItem));
}
protected override async Task <Blob> GetBlobAsync(string fullPath, CancellationToken cancellationToken)
{
fullPath = StoragePath.Normalize(fullPath);
try
{
Object obj = await _client.GetObjectAsync(_bucketName, fullPath,
new GetObjectOptions
{
//todo
},
cancellationToken).ConfigureAwait(false);
return(GConvert.ToBlob(obj));
}
catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.NotFound)
{
return(null);
}
}
public List <string> GetFilesList()
{
var newObject = new Google.Apis.Storage.v1.Data.Object()
{
Bucket = BucketToUpload,
};
List <string> resultStatus = null;
try
{
var listRequest = new ObjectsResource.ListRequest(StorageService,
BucketToUpload);
listRequest.OauthToken = UserCredential.Token.AccessToken;
resultStatus = listRequest.Execute().Items.Select(x => x.Name).ToList();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
return(resultStatus);
}
private Google.Apis.Storage.v1.Data.Object BuildGoogleObject(string fileName, Permissions permissionLevel)
{
var fileobj = new Google.Apis.Storage.v1.Data.Object()
{
Name = fileName
};
if (permissionLevel == Permissions.PublicallyViewable)
{
fileobj.Acl = new List <ObjectAccessControl>
{
new ObjectAccessControl {
Role = "READER", Entity = "allUsers"
}
}
}
;
return(fileobj);
}
}
public bool DeleteFile(string fileId)
{
var newObject = new Google.Apis.Storage.v1.Data.Object()
{
Bucket = BucketToUpload,
Name = fileId
};
string resultStatus;
try
{
var deleteRequest = new ObjectsResource.DeleteRequest(StorageService,
BucketToUpload, fileId);
resultStatus = deleteRequest.Execute();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
return(true);
}
public void UploadQuiz(QuizDocument doc, bool useAsync = false)
{
var credential = GetCredentials();
StorageService service = new StorageService(
new BaseClientService.Initializer()
{
HttpClientInitializer = credential,
ApplicationName = "Quiz Plus Uploader",
});
foreach(var slide in doc.Slides)
{
var fileobj = new GSD.Object() { Name = slide.CanonicalName };
var uploadStream = new FileStream(slide.ImagePath, FileMode.Open, FileAccess.Read);
var uploadRequest = service.Objects.Insert(
fileobj,
BUCKET_NAME,
uploadStream,
"image/jpeg");
if (useAsync)
{
var task = uploadRequest.UploadAsync();
task.ContinueWith(t =>
{
// Remeber to clean the stream.
Logger.Log("Uploaded image " + slide.ImagePath);
uploadStream.Dispose();
});
}
else
{
uploadRequest.Upload();
Logger.Log("Uploaded image " + slide.ImagePath);
uploadStream.Dispose();
}
}
}
private async Task Run()
{
string projectId = "snowball-1053";
string bucketName = "snowball_test";
//Authentication Option 1: Credential for certificate-based service accounts.
string certificateFile = "c:\\snowball-544a05c14e78.p12";
string serviceAccountEmail = "*****@*****.**";
var certificate = new X509Certificate2(certificateFile, "notasecret", X509KeyStorageFlags.Exportable);
ServiceAccountCredential credential = new ServiceAccountCredential(
new ServiceAccountCredential.Initializer(serviceAccountEmail)
{
Scopes = new[] { StorageService.Scope.DevstorageReadWrite }
}.FromCertificate(certificate));
//Authentication Option 2: Credential when running in Google Compute Engine.
//Requires Google API CLient library >=1.9.1
//ComputeCredential credential = new ComputeCredential(new ComputeCredential.Initializer());
//Authentication Option 3: Installed application credentials, user interactive webflow.
/*
UserCredential credential;
string clientId = "YOUR_CLIENT_ID.apps.googleusercontent.com";
string clientSecret = "YOUR_CLIENT_SECRET";
credential = await GoogleWebAuthorizationBroker.AuthorizeAsync(new ClientSecrets{ClientId= clientId,ClientSecret=clientSecret},
new[] { StorageService.Scope.DevstorageFullControl }, Environment.UserName, CancellationToken.None);
//Console.WriteLine("Credential file saved at: " + Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData));
*/
var service = new StorageService(new BaseClientService.Initializer()
{
HttpClientInitializer = credential,
ApplicationName = "GCS Sample",
});
Console.WriteLine("List of buckets in current project");
Buckets buckets = await service.Buckets.List(projectId).ExecuteAsync();
foreach (var bkt in buckets.Items)
{
Console.WriteLine(bkt.Name);
}
Console.WriteLine("Total number of items in bucket: " + buckets.Items.Count);
Console.WriteLine("=============================");
//var sql =''
var _fileName = string.Format("cflows-{0}", DateTime.Now.ToString("yyyy-MM-dd"));
// using Google.Apis.Storage.v1.Data.Object to disambiguate from System.Object
Google.Apis.Storage.v1.Data.Object fileobj = new Google.Apis.Storage.v1.Data.Object() { Name = _fileName };
Console.WriteLine("Creating " + fileobj.Name + " in bucket " + bucketName);
#region prepare data
//var strBuilder = new StringBuilder();
//strBuilder.AppendLine("A,B,C");
//strBuilder.AppendLine("E,F,G");
//byte[] msgtxt = Encoding.UTF8.GetBytes(strBuilder.ToString());
//using (var streamOut = new MemoryStream(msgtxt))
//{
// await service.Objects.Insert(fileobj, bucketName, streamOut, "application/vnd.ms-excel").UploadAsync();
//}
_fileName = string.Format("cflows-{0}", DateTime.Now.ToString("yyyy-MM-dd"));
string strFilePath = string.Format("C:\\{0}.csv", _fileName);
using (FileStream fileStream = new FileStream(strFilePath, FileMode.Open, FileAccess.Read))
{
await service.Objects.Insert(fileobj, bucketName, fileStream, "application/vnd.ms-excel").UploadAsync();
}
#endregion
Console.WriteLine("Object created: " + fileobj.Name);
Console.WriteLine("=============================");
Console.WriteLine("Reading object " + fileobj.Name + " in bucket: " + bucketName);
var req = service.Objects.Get(bucketName, fileobj.Name);
Google.Apis.Storage.v1.Data.Object readobj = await req.ExecuteAsync();
Console.WriteLine("Object MediaLink: " + readobj.MediaLink);
// download using Google.Apis.Download and display the progress
string pathUser = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
var fileName = Path.Combine(pathUser, "Downloads") + "\\" + readobj.Name;
Console.WriteLine("Starting download to " + fileName);
var downloader = new MediaDownloader(service)
{
ChunkSize = DownloadChunkSize
};
// add a delegate for the progress changed event for writing to console on changes
downloader.ProgressChanged += Download_ProgressChanged;
using (var fileStream = new System.IO.FileStream(fileName,
System.IO.FileMode.Create, System.IO.FileAccess.Write))
{
var progress = await downloader.DownloadAsync(readobj.MediaLink, fileStream);
if (progress.Status == DownloadStatus.Completed)
{
Console.WriteLine(readobj.Name + " was downloaded successfully");
}
else
{
Console.WriteLine("Download {0} was interpreted in the middle. Only {1} were downloaded. ",
readobj.Name, progress.BytesDownloaded);
}
}
/*
// or download as a stream
Stream getStream = await service.HttpClient.GetStreamAsync(readobj.MediaLink);
Console.WriteLine("Object Content: ");
using (var reader = new StreamReader(getStream))
{
Console.WriteLine(reader.ReadToEnd());
}
*/
Console.WriteLine("=============================");
}
public async Task Run()
{
var certificate = new X509Certificate2(CertificateFile, "notasecret", X509KeyStorageFlags.Exportable);
var credential = new ServiceAccountCredential(new ServiceAccountCredential.Initializer(ServiceAccountEmail)
{
Scopes = new[]
{
StorageService.Scope.DevstorageReadWrite
}
}.FromCertificate(certificate));
var service = new StorageService(new BaseClientService.Initializer
{
HttpClientInitializer = credential,
ApplicationName = "Tile-ETL"
});
for (var level = StartLevel; level <= EndLevel; ++level)
{
Console.WriteLine("processing {0}", level);
var tileSize = WebMercatorDelta*Math.Pow(2, 1 - level);
var startRow = Convert.ToInt32(Math.Truncate((WebMercatorDelta - ExtentMaxY)/tileSize)) - TilePaddingY;
var endRow = Convert.ToInt32(Math.Truncate((WebMercatorDelta - ExtentMinY)/tileSize)) + 1 + TilePaddingY;
var startColumn = Convert.ToInt32(Math.Truncate((ExtentMinX + WebMercatorDelta)/tileSize)) -
TilePaddingX;
var endColumn = Convert.ToInt32(Math.Truncate((ExtentMaxX + WebMercatorDelta)/tileSize)) + 1 +
TilePaddingX;
var aclList = new[]
{
new ObjectAccessControl
{
Entity = "allUsers",
Role = "OWNER"
}
};
var acl = new ArraySegment<ObjectAccessControl>(aclList);
for (var r = startRow; r <= endRow; ++r)
{
for (var c = startColumn; c <= endColumn; ++c)
{
try
{
var imagePath = string.Format("{0}\\L{1:00}\\R{2:x8}\\C{3:x8}.{4}", TileDirectory, level,
r, c, "jpg");
if (!File.Exists(imagePath))
{
continue;
}
var file = File.ReadAllBytes(imagePath);
using (var streamOut = new MemoryStream(file))
{
var fileobj = new Object
{
Name = string.Format("{0}/{1}/{2}/{3}", MapName, level, r, c),
Acl = acl
};
await service.Objects.Insert(fileobj, BucketName, streamOut, "image/jpg").UploadAsync();
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
}
Console.WriteLine("Finished");
}
}
public GcsPath(Object input) : this(input.Bucket, input.Name) { }
private Google.Apis.Storage.v1.Data.Object BuildGoogleObject(string fileName, Permissions permissionLevel)
{
var fileobj = new Google.Apis.Storage.v1.Data.Object()
{
Name = fileName
};
if (permissionLevel == Permissions.PublicallyViewable)
fileobj.Acl = new List<ObjectAccessControl>
{
new ObjectAccessControl {Role = "READER", Entity = "allUsers"}
};
return fileobj;
}
/// <summary>
/// Adds or updates a Google Cloud Storage object to the model.
/// </summary>
/// <param name="gcsObject">The Google Cloud Storage object to add or update.</param>
public void AddObject(Object gcsObject)
{
_objectMap[gcsObject.Name] = gcsObject;
}
/// <summary>
/// Gets a writer used to upload data to a Google Cloud Storage object. Used by Set-Content.
/// </summary>
/// <param name="path">The path of the object to upload to.</param>
/// <returns>The writer.</returns>
public IContentWriter GetContentWriter(string path)
{
var gcsPath = GcsPath.Parse(path);
Object body = new Object
{
Name = gcsPath.ObjectPath,
Bucket = gcsPath.Bucket
};
var inputStream = new AnonymousPipeServerStream(PipeDirection.Out);
var outputStream = new AnonymousPipeClientStream(PipeDirection.In, inputStream.ClientSafePipeHandle);
var contentType = ((GcsGetContentWriterDynamicParameters)DynamicParameters).ContentType ?? GcsCmdlet.UTF8TextMimeType;
ObjectsResource.InsertMediaUpload request =
Service.Objects.Insert(body, gcsPath.Bucket, outputStream, contentType);
request.UploadAsync();
IContentWriter contentWriter = new GcsContentWriter(inputStream);
// Force the bucket models to refresh with the potentially new object.
BucketModels.Clear();
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(GetContentWriter));
return contentWriter;
}
/// <summary>
/// Clears the content of an object. Used by Clear-Content.
/// </summary>
/// <param name="path">The path of the object to clear.</param>
public void ClearContent(string path)
{
if (!ShouldProcess(path, "Clear-Content"))
{
return;
}
var gcsPath = GcsPath.Parse(path);
Object body = new Object
{
Name = gcsPath.ObjectPath,
Bucket = gcsPath.Bucket
};
var memoryStream = new MemoryStream();
var contentType = GcsCmdlet.UTF8TextMimeType;
ObjectsResource.InsertMediaUpload request =
Service.Objects.Insert(body, gcsPath.Bucket, memoryStream, contentType);
IUploadProgress response = request.Upload();
if (response.Exception != null)
{
throw response.Exception;
}
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(ClearContent));
}
private Object NewObject(GcsPath gcsPath, NewGcsObjectDynamicParameters dynamicParameters, Stream contentStream)
{
Object newGcsObject = new Object
{
Bucket = gcsPath.Bucket,
Name = gcsPath.ObjectPath,
ContentType = dynamicParameters.ContentType
};
ObjectsResource.InsertMediaUpload insertReq = Service.Objects.Insert(
newGcsObject, newGcsObject.Bucket, contentStream, newGcsObject.ContentType);
insertReq.PredefinedAcl = dynamicParameters.PredefinedAcl;
insertReq.Projection = ObjectsResource.InsertMediaUpload.ProjectionEnum.Full;
IUploadProgress finalProgress = insertReq.Upload();
if (finalProgress.Exception != null)
{
throw finalProgress.Exception;
}
return insertReq.ResponseBody;
}
public void Run(IConfigurableHttpClientInitializer credential,
string projectId, string bucketName)
{
StorageService service = new StorageService(
new BaseClientService.Initializer()
{
HttpClientInitializer = credential,
ApplicationName = "GCS Sample",
});
Console.WriteLine("List of buckets in current project");
Buckets buckets = service.Buckets.List(projectId).Execute();
foreach (var bucket in buckets.Items)
{
Console.WriteLine(bucket.Name);
}
Console.WriteLine("Total number of items in bucket: "
+ buckets.Items.Count);
Console.WriteLine("=============================");
// using Google.Apis.Storage.v1.Data.Object to disambiguate from
// System.Object
Google.Apis.Storage.v1.Data.Object fileobj =
new Google.Apis.Storage.v1.Data.Object()
{
Name = "somefile.txt"
};
Console.WriteLine("Creating " + fileobj.Name + " in bucket "
+ bucketName);
byte[] msgtxt = Encoding.UTF8.GetBytes("Lorem Ipsum");
service.Objects.Insert(fileobj, bucketName,
new MemoryStream(msgtxt), "text/plain").Upload();
Console.WriteLine("Object created: " + fileobj.Name);
Console.WriteLine("=============================");
Console.WriteLine("Reading object " + fileobj.Name + " in bucket: "
+ bucketName);
var req = service.Objects.Get(bucketName, fileobj.Name);
Google.Apis.Storage.v1.Data.Object readobj = req.Execute();
Console.WriteLine("Object MediaLink: " + readobj.MediaLink);
// download using Google.Apis.Download and display the progress
string pathUser = Environment.GetFolderPath(
Environment.SpecialFolder.UserProfile);
var fileName = Path.Combine(pathUser, "Downloads") + "\\"
+ readobj.Name;
Console.WriteLine("Starting download to " + fileName);
var downloader = new MediaDownloader(service)
{
ChunkSize = DownloadChunkSize
};
// add a delegate for the progress changed event for writing to
// console on changes
downloader.ProgressChanged += progress =>
Console.WriteLine(progress.Status + " "
+ progress.BytesDownloaded + " bytes");
using (var fileStream = new System.IO.FileStream(fileName,
System.IO.FileMode.Create, System.IO.FileAccess.Write))
{
var progress =
downloader.Download(readobj.MediaLink, fileStream);
if (progress.Status == DownloadStatus.Completed)
{
Console.WriteLine(readobj.Name
+ " was downloaded successfully");
}
else
{
Console.WriteLine("Download {0} was interrupted. Only {1} "
+ "were downloaded. ",
readobj.Name, progress.BytesDownloaded);
}
}
Console.WriteLine("=============================");
}
private async Task Run()
{
// Enter values here so you don't have to type them every time.
string projectId = @"";
string bucketName = @"";
if (String.IsNullOrWhiteSpace(projectId))
{
Console.Write("Enter your project id: ");
projectId = Console.ReadLine().Trim();
}
if (String.IsNullOrWhiteSpace(bucketName))
{
Console.Write("Enter your bucket name: ");
bucketName = Console.ReadLine().Trim();
}
GoogleCredential credential = await GoogleCredential.GetApplicationDefaultAsync();
if (credential.IsCreateScopedRequired)
{
credential = credential.CreateScoped(new[] { StorageService.Scope.DevstorageReadWrite });
}
StorageService service = new StorageService(new BaseClientService.Initializer()
{
HttpClientInitializer = credential,
ApplicationName = "GCS Sample",
});
Console.WriteLine("List of buckets in current project");
Buckets buckets = await service.Buckets.List(projectId).ExecuteAsync();
foreach (var bucket in buckets.Items)
{
Console.WriteLine(bucket.Name);
}
Console.WriteLine("Total number of items in bucket: " + buckets.Items.Count);
Console.WriteLine("=============================");
// using Google.Apis.Storage.v1.Data.Object to disambiguate from System.Object
Google.Apis.Storage.v1.Data.Object fileobj =
new Google.Apis.Storage.v1.Data.Object() { Name = "somefile.txt" };
Console.WriteLine("Creating " + fileobj.Name + " in bucket " + bucketName);
byte[] msgtxt = Encoding.UTF8.GetBytes("Lorem Ipsum");
await service.Objects.Insert(fileobj, bucketName, new MemoryStream(msgtxt),
"text/plain").UploadAsync();
Console.WriteLine("Object created: " + fileobj.Name);
Console.WriteLine("=============================");
Console.WriteLine("Reading object " + fileobj.Name + " in bucket: " + bucketName);
var req = service.Objects.Get(bucketName, fileobj.Name);
Google.Apis.Storage.v1.Data.Object readobj = await req.ExecuteAsync();
Console.WriteLine("Object MediaLink: " + readobj.MediaLink);
// download using Google.Apis.Download and display the progress
string pathUser = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
var fileName = Path.Combine(pathUser, "Downloads") + "\\" + readobj.Name;
Console.WriteLine("Starting download to " + fileName);
var downloader = new MediaDownloader(service) { ChunkSize = DownloadChunkSize };
// add a delegate for the progress changed event for writing to console on changes
downloader.ProgressChanged += progress =>
Console.WriteLine(progress.Status + " " + progress.BytesDownloaded + " bytes");
using (var fileStream = new System.IO.FileStream(fileName,
System.IO.FileMode.Create, System.IO.FileAccess.Write))
{
var progress = await downloader.DownloadAsync(readobj.MediaLink, fileStream);
if (progress.Status == DownloadStatus.Completed)
{
Console.WriteLine(readobj.Name + " was downloaded successfully");
}
else
{
Console.WriteLine("Download {0} was interrupted. Only {1} were downloaded. ",
readobj.Name, progress.BytesDownloaded);
}
}
Console.WriteLine("=============================");
}
private async Task Run()
{
string projectId = "causal-bus-95919";
string bucketName = "threat-detector-main-storage";
//Authentication Option 1: Credential for certificate-based service accounts.
string certificateFile = @"C:\Users\Алексей\Documents\Repositories\Course project the 4th\ThreatDetector\ThreatDetector-c02d8385fbd4.p12";
string serviceAccountEmail = "*****@*****.**";
var certificate = new X509Certificate2(certificateFile, "notasecret", X509KeyStorageFlags.Exportable);
var credential = new ServiceAccountCredential(
new ServiceAccountCredential.Initializer(serviceAccountEmail)
{
Scopes = new[] { StorageService.Scope.DevstorageReadWrite }
}.FromCertificate(certificate));
//Authentication Option 2: Credential when running in Google Compute Engine.
//Requires Google API CLient library >=1.9.1
//ComputeCredential credential = new ComputeCredential(new ComputeCredential.Initializer());
//Authentication Option 3: Installed application credentials, user interactive webflow.
/*
UserCredential credential;
string clientId = "YOUR_CLIENT_ID.apps.googleusercontent.com";
string clientSecret = "YOUR_CLIENT_SECRET";
credential = await GoogleWebAuthorizationBroker.AuthorizeAsync(new ClientSecrets{ClientId= clientId,ClientSecret=clientSecret},
new[] { StorageService.Scope.DevstorageFullControl }, Environment.UserName, CancellationToken.None);
//Console.WriteLine("Credential file saved at: " + Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData));
*/
var service = new StorageService(new BaseClientService.Initializer()
{
HttpClientInitializer = credential,
ApplicationName = "GCS Sample",
});
Console.WriteLine("List of buckets in current project");
Buckets buckets = await service.Buckets.List(projectId).ExecuteAsync();
foreach (var bkt in buckets.Items)
{
Console.WriteLine(bkt.Name);
}
Console.WriteLine("Total number of items in bucket: " + buckets.Items.Count);
Console.WriteLine("=============================");
// using Google.Apis.Storage.v1.Data.Object to disambiguate from System.Object
var fileobj = new Google.Apis.Storage.v1.Data.Object() { Name = "t/tt/somefile.txt" };
Console.WriteLine("Creating " + fileobj.Name + " in bucket " + bucketName);
byte[] msgtxt = Encoding.UTF8.GetBytes("Lorem Ipsum");
using (var streamOut = new MemoryStream(msgtxt))
{
await service.Objects.Insert(fileobj, bucketName, streamOut, "text/plain").UploadAsync();
}
Console.WriteLine("Object created: " + fileobj.Name);
Console.WriteLine("=============================");
Console.WriteLine("Reading object " + fileobj.Name + " in bucket: " + bucketName);
var req = service.Objects.Get(bucketName, fileobj.Name);
Google.Apis.Storage.v1.Data.Object readobj = await req.ExecuteAsync();
Console.WriteLine("Object MediaLink: " + readobj.MediaLink);
// download using Google.Apis.Download and display the progress
string pathUser = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
var fileName = Path.Combine(pathUser, "Downloads") + "\\" + readobj.Name;
Console.WriteLine("Starting download to " + fileName);
var downloader = new MediaDownloader(service)
{
ChunkSize = DownloadChunkSize
};
// add a delegate for the progress changed event for writing to console on changes
downloader.ProgressChanged += Download_ProgressChanged;
using (var fileStream = new System.IO.FileStream(fileName,
System.IO.FileMode.Create, System.IO.FileAccess.Write))
{
var progress = await downloader.DownloadAsync(readobj.MediaLink, fileStream);
if (progress.Status == DownloadStatus.Completed)
{
Console.WriteLine(readobj.Name + " was downloaded successfully");
}
else
{
Console.WriteLine("Download {0} was interpreted in the middle. Only {1} were downloaded. ",
readobj.Name, progress.BytesDownloaded);
}
}
/*
// or download as a stream
Stream getStream = await service.HttpClient.GetStreamAsync(readobj.MediaLink);
Console.WriteLine("Object Content: ");
using (var reader = new StreamReader(getStream))
{
Console.WriteLine(reader.ReadToEnd());
}
*/
Console.WriteLine("=============================");
}