public override async Task <BatchDeleteProcessor> RemovePrefixAsync(string bucketName, string prefix, int chunkSize, CancellationToken cancellationToken = default)
{
ValidateInstance();
var processor = new BatchDeleteProcessor(async(IEnumerable <string> keys) =>
{
var finishedDelete = false;
var observableDelete = await _minioClient.RemoveObjectAsync(bucketName, keys, cancellationToken).ConfigureAwait(false);
// Remove list of objects in objectNames from the bucket bucketName.
observableDelete.Subscribe(
deleteError => Console.WriteLine("Object: {0}", deleteError.Key),
ex => Console.WriteLine("OnError: {0}", ex),
() => finishedDelete = true
);
while (!finishedDelete)
{
await Task.Delay(250);
}
});
var bucketKeys = new List <string>();
var finishedList = false;
var prefixToFilter = (prefix.EndsWith("/") ? prefix : prefix + "/");
var observable = _minioClient.ListObjectsAsync(bucketName, prefixToFilter, true, cancellationToken);
observable.Subscribe
(
item =>
{
bucketKeys.Add(item.Key);
if (bucketKeys.Count >= chunkSize)
{
processor.EnqueueChunk(bucketKeys);
bucketKeys = new List <string>();
}
},
() =>
{
if (bucketKeys.Any())
{
processor.EnqueueChunk(bucketKeys);
}
finishedList = true;
}
);
while (!finishedList)
{
await Task.Delay(100);
}
return(processor);
}
private async static Task ListObjects_Test(MinioClient minio, string bucketName, string prefix, int numObjects, bool recursive = true)
{
int count = 0;
try
{
IObservable <Item> observable = minio.ListObjectsAsync(bucketName, prefix, recursive);
IDisposable subscription = observable.Subscribe(
item =>
{
Assert.IsTrue(item.Key.StartsWith(prefix));
count += 1;
Console.Out.WriteLine(item.Key + ":" + count.ToString());
},
ex => Console.WriteLine("OnError: {0}", ex),
() =>
{
Console.WriteLine("Listed all objects in bucket " + bucketName + "\n");
Assert.AreEqual(count, numObjects);
});
}
catch (Exception e)
{
Console.WriteLine("[Bucket] Exception: {0}", e);
}
}
/// <summary>
///
/// </summary>
/// <param name="_minio"></param>
/// <param name="bucketName"></param>
/// <param name="objectName"></param>
/// <param name="contentType"></param>
/// <param name="fileName"></param>
/// <param name="metaData"></param>
/// <returns></returns>
public async static Task <ObjectStat> PutObject_Tester(MinioClient _minio, string bucketName, string objectName, string contentType, string fileName = null, Dictionary <string, string> metaData = null)
{
int count = 0;
IObservable <Item> observable = _minio.ListObjectsAsync(bucketName, "objectName", true);
IDisposable subscription = observable.Subscribe(
item =>
{
if (item.Key.StartsWith("device3"))
{
count += 1;
Console.WriteLine($"count: {count}");
}
},
ex => Console.WriteLine($"OnError: {ex}"),
() => Console.WriteLine($"Listed all objects in bucket {bucketName}\n"));
ObjectStat statObject = await _minio.StatObjectAsync(bucketName, "device3");
//await _minio.PutObjectAsync(bucketName,objectName,objectName,contentType);
Assert.IsNotNull(statObject);
Assert.AreEqual(statObject.ObjectName, objectName);
//Assert.AreEqual(statObject.Size, file_read_size);
if (contentType != null)
{
Assert.AreEqual(statObject.ContentType, contentType);
}
return(statObject);
}
// List objects matching optional prefix in a specified bucket.
public static void Run(MinioClient minio,
string bucketName = "my-bucket-name",
string prefix = null,
bool recursive = true)
{
try
{
Console.WriteLine("Running example for API: ListObjectsAsync");
ListObjectsArgs listArgs = new ListObjectsArgs()
.WithBucket(bucketName)
.WithPrefix(prefix)
.WithRecursive(recursive);
IObservable <Item> observable = minio.ListObjectsAsync(listArgs);
IDisposable subscription = observable.Subscribe(
item => Console.WriteLine($"Object: {item.Key}"),
ex => Console.WriteLine($"OnError: {ex}"),
() => Console.WriteLine($"Listed all objects in bucket {bucketName}\n"));
listArgs.WithVersions(true);
IObservable <VersionItem> observableVer = minio.ListObjectVersionsAsync(listArgs);
IDisposable subscriptionVer = observableVer.Subscribe(
item => Console.WriteLine($"Object: {item.Key} Version: {item.VersionId}"),
ex => Console.WriteLine($"OnError: {ex}"),
() => Console.WriteLine($"Listed all objects in bucket {bucketName}\n"));
// subscription.Dispose();
}
catch (Exception e)
{
Console.WriteLine($"[Bucket] Exception: {e}");
}
}
// Lists all objects in a bucket or null case error or doesnt exists
public IObservable <Item> ListObjectsAsync(String BucketName, String Prefix = null, Boolean Recursive = true)
{
try
{
// Check whether 'mybucket' exists or not.
var found = minio.BucketExistsAsync(BucketName);
Task.WaitAll(found);
if (found.Result)
{
// List objects from 'my-bucketname'
IObservable <Item> observable = minio.ListObjectsAsync(BucketName, Prefix, Recursive);
return(observable);
/*IDisposable subscription = observable.Subscribe(
* item => Console.WriteLine("OnNext: {0}", item.Key),
* ex => Console.WriteLine("OnError: {0}", ex.Message),
* () => Console.WriteLine("OnComplete: {0}"));*/
}
else
{
throw new BucketNotFoundException();
}
}
catch (MinioException e)
{
throw;
}
}
public async Task Remove(string bucketName, string objectName, bool removeBucketIfEmpty = true)
{
await _client.RemoveObjectAsync(bucketName, objectName);
if (removeBucketIfEmpty && !await _client.ListObjectsAsync(bucketName).Any())
{
await _client.RemoveBucketAsync(bucketName);
}
}
public async Task <IEnumerable <string> > Get()
{
if (!await _blobClient.BucketExistsAsync(bucketName))
{
await _blobClient.MakeBucketAsync(bucketName);
}
return(_blobClient.ListObjectsAsync(bucketName).Select(x => x.Key).ToEnumerable());
}
private Task <IEnumerable <FileSpec> > GetFileListAsync(string searchPattern = null, int?limit = null, int?skip = null, CancellationToken cancellationToken = default)
{
if (limit.HasValue && limit.Value <= 0)
{
return(Task.FromResult(Enumerable.Empty <FileSpec>()));
}
var criteria = GetRequestCriteria(NormalizePath(searchPattern));
var objects = new List <Item>();
ExceptionDispatchInfo exception = null;
var resetEvent = new AutoResetEvent(false);
var observable = _client.ListObjectsAsync(new ListObjectsArgs().WithBucket(_bucket).WithPrefix(criteria.Prefix).WithRecursive(true), cancellationToken);
observable.Subscribe(item => {
if (!item.IsDir && (criteria.Pattern == null || criteria.Pattern.IsMatch(item.Key)))
{
objects.Add(item);
}
}, error => {
if (error.GetType().ToString() != "Minio.EmptyBucketOperation")
{
_logger.LogError(error, "Error trying to find files: {Pattern}", searchPattern);
exception = ExceptionDispatchInfo.Capture(error);
}
resetEvent.Set();
},
() => resetEvent.Set()
);
resetEvent.WaitOne();
if (exception != null)
{
if (exception.SourceException is ObjectNotFoundException ||
exception.SourceException is BucketNotFoundException)
{
return(Task.FromResult(Enumerable.Empty <FileSpec>()));
}
exception.Throw();
}
if (skip.HasValue)
{
objects = objects.Skip(skip.Value).ToList();
}
if (limit.HasValue)
{
objects = objects.Take(limit.Value).ToList();
}
return(Task.FromResult(objects.Select(blob => new FileSpec {
Path = blob.Key,
Size = (long)blob.Size,
Modified = DateTime.Parse(blob.LastModified),
Created = DateTime.Parse(blob.LastModified)
})));
}
public async Task Scan(string username)
{
var minio = new MinioClient(this.config.Endpoint,
this.config.AccessKey,
this.config.SecretKey);
if (this.config.WithSsl)
{
minio = minio.WithSSL();
}
if (await minio.BucketExistsAsync(this.config.BucketName))
{
var mediaLibrary =
this.GrainFactory.GetGrain <IMediaLibrary>(
$"{username}@{this.config.Endpoint}/{this.config.BucketName}");
var observable = minio.ListObjectsAsync(this.config.BucketName, recursive: true);
var streamProvider = this.GetStreamProvider(Constants.SmsProvider);
var scanEventStream = streamProvider.GetStream <ScanEventData>(this.GetPrimaryKey(), username);
var disposable = observable
.SubscribeOn(SynchronizationContext.Current)
.Subscribe(item =>
{
var mediaItem =
this.GrainFactory.GetGrain <IMediaItem>(
$"{this.config.Endpoint}/{this.config.BucketName}/{item.Key}");
mediaLibrary.AddMediaItem(mediaItem);
scanEventStream.OnNextAsync(new ScanEventData
{
ETag = item.ETag,
IsDir = item.IsDir,
Key = item.Key,
LastModifiedDateTime = item.LastModifiedDateTime,
LastModified = item.LastModified
});
Console.WriteLine(
$"Object: {item.Key}, {item.ETag}, {item.LastModifiedDateTime}, {((long) item.Size).Bytes()}");
},
ex =>
{
scanEventStream.OnErrorAsync(ex);
Console.WriteLine($"OnError: {ex}");
},
() =>
{
scanEventStream.OnCompletedAsync();
Console.WriteLine($"Listed all objects in bucket {this.config.BucketName}\n");
});
}
}
private void Button_Click(object sender, RoutedEventArgs e)
{
#region 20210326 komla
var open = new OpenFileDialog
{
CheckFileExists = true,
CheckPathExists = true,
};
if (open.ShowDialog(this) == false)
{
return;
}
string filePath = open.FileName;
#endregion
#region 20210326 流操作
//byte[] array = Encoding.ASCII.GetBytes(testStr);
//MemoryStream stream = new MemoryStream(array);
#endregion
#region 20210327
//byte[] bs = File.ReadAllBytes("G:\\手机文件\\半身照.jpg");
//MemoryStream filestream = new MemoryStream(bs);
#endregion
int count = 0;
IObservable <Item> observable = _minioClient.ListObjectsAsync("", "", true);
IDisposable subscription = observable.Subscribe(
item =>
{
if (item.Key.StartsWith(""))
{
count += 1;
Console.WriteLine($"count: {count}");
}
},
ex => Console.WriteLine($"OnError: {ex}"),
() => Console.WriteLine($"Listed all objects in bucket {""}\n"));
try
{
_minioClient.PutObjectAsync("company1", "device2", filePath);//20210327 这一步是不是封装了创建桶操作
//Dispatcher?.InvokeAsync(async () =>
//{
// await _minioClient.PutObjectAsync("company1", "device2", filestream, filestream.Length, "application/octet-stream");
//});
}
catch (MinioException ex)
{
throw;
}
}
/// <summary>
/// 返回指定bucketName桶中的对象集合
/// </summary>
/// <param name="bucketName">桶名</param>
/// <param name="prefix">前缀字符串，列出名称以prefix为前缀的对象</param>
/// <param name="recursive">如果为false，则模拟目录结构，其中返回的每个列表都是完整对象或对象键的一部分，直到第一个“ /”。所有具有相同前缀（直到第一个“ /”）的对象都将合并到一个条目中。默认为false</param>
/// <returns></returns>
public async Task <IObservable <Item> > ListObjectsAsync(string bucketName
, string prefix
, bool recursive = true)
{
try
{
return(await Task.Run(() =>
_minioClient.ListObjectsAsync(bucketName, prefix: prefix, recursive)));
}
catch (Exception ex)
{
throw ex;
}
}
/// <summary>
///
/// </summary>
/// <param name="containerName"></param>
/// <returns></returns>
public async Task <IEnumerable <string> > ListBlobsAsync(string containerName)
{
IList <Item> blobs = new List <Item>();
try
{
blobs = await _client.ListObjectsAsync(containerName).ToList();
return(blobs.Select(x => x.Key));
}
catch (MinioException e)
{
throw new Exception($"[Bucket] Exception: {e}");
}
}
public IEnumerable <IFileEntry> ListBucket(string bucketName, string prefix)
{
ThrowExceptionIfBucketDoesNotExist(bucketName);
var observable = m_client.ListObjectsAsync(bucketName, prefix, true);
foreach (var obj in observable.ToEnumerable())
{
yield return(new Common.IO.FileEntry(
obj.Key,
(long)obj.Size,
Convert.ToDateTime(obj.LastModified),
Convert.ToDateTime(obj.LastModified)
));
}
}
public async Task <IEnumerable <string> > EnumerateBucket(string bucket)
{
if (!await _client.BucketExistsAsync(bucket))
{
return(new string[0]);
}
var blobNames = new List <string>();
var observable = _client.ListObjectsAsync(bucket);
using (observable.Subscribe(item => blobNames.Add(item.Key)))
{
await observable;
return(blobNames.ToList());
}
}
public override async void OnApplicationShutdown(ApplicationShutdownContext context)
{
var minioClient = new MinioClient(_endPoint, _accessKey, _secretKey);
if (await minioClient.BucketExistsAsync(_randomContainerName))
{
var objects = await minioClient.ListObjectsAsync(_randomContainerName, null, true).ToList();
foreach (var item in objects)
{
await minioClient.RemoveObjectAsync(_randomContainerName, item.Key);
}
await minioClient.RemoveBucketAsync(_randomContainerName);
}
}
public IActionResult Index()
{
var list = minioClient.ListObjectsAsync(minioBucketName, null, true);
var objectName = new List <string>();
list.Subscribe(x => objectName.Add(x.Key));
var userFiles = new List <UserFiles>();
using (ApplicationDbContext context = new ApplicationDbContext())
{
userFiles = context.UserFiles.ToList();
}
return(View(userFiles));
}
/// <summary>
///
/// </summary>
/// <param name="minio"></param>
/// <returns></returns>
private static async Task GetTask(MinioClient minio, string bucket, string objName, bool recursive)
{
try
{
var iObservable = minio.ListObjectsAsync(bucket, objName, recursive);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
//ListObjectsArgs listArgs = new ListObjectsArgs()
// .WithBucket(bucket)
// .WithPrefix(objName)
// .WithRecursive(recursive);
//IObservable<Item> observable = minio.ListObjectsAsync(listArgs);
}
private async Task <IEnumerable <Minio.DataModel.Item> > GetBackupBlobs(bool sorted)
{
var observable = minio.ListObjectsAsync(bucketName);
var itemizedBlobs = new List <Minio.DataModel.Item>();
await observable.ForEachAsync(item => itemizedBlobs.Add(item));
Trace.TraceInformation("MinioBackupManager: Got {0} blobs", itemizedBlobs.Count);
if (sorted)
{
return(itemizedBlobs.OrderByDescending(x => x.LastModified));
}
else
{
return(itemizedBlobs);
}
}
public Task <List <Item> > ListObjectsAsync(string bucketName, string prefix = null)
{
if (string.IsNullOrEmpty(bucketName))
{
throw new ArgumentNullException(nameof(bucketName));
}
IObservable <Minio.DataModel.Item> observable = _client.ListObjectsAsync(
new ListObjectsArgs()
.WithBucket(bucketName)
.WithPrefix(prefix)
.WithRecursive(true));
List <Item> result = new List <Item>();
bool isFinish = false;
IDisposable subscription = observable.Subscribe(
item =>
{
result.Add(new Item()
{
Key = item.Key,
LastModified = item.LastModified,
ETag = item.ETag,
Size = item.Size,
BucketName = bucketName,
IsDir = item.IsDir,
LastModifiedDateTime = item.LastModifiedDateTime
});
},
ex =>
{
isFinish = true;
},
() =>
{
isFinish = true;
});
while (!isFinish)
{
Thread.Sleep(0);
}
return(Task.FromResult(result));
}
/// <summary>
/// Test ListObjectAsync function
/// </summary>
/// <param name="_minio"></param>
/// <returns></returns>
public async static Task ListObjects_Test1(MinioClient minio, string bucketName, string prefix, int numObjects, bool recursive = true)
{
int count = 0;
IObservable <Item> observable = minio.ListObjectsAsync(bucketName, prefix, recursive);
IDisposable subscription = observable.Subscribe(
item =>
{
if (item.Key.StartsWith(prefix))
{
count += 1;
Console.WriteLine($"count: {count}");
}
},
ex => Console.WriteLine($"OnError: {ex}"),
() => Console.WriteLine($"Listed all objects in bucket {bucketName}\n"));
//observable.Subscribe(
// item =>
// {
// Assert.IsTrue(item.Key.StartsWith(prefix));
// }
// );
//foreach (Item item in )
//{
// // Ignore
// continue;
//}
//IDisposable subscription = observable.Subscribe(
// item =>
// {
// Assert.IsTrue(item.Key.StartsWith(prefix));
// count += 1;
// },
// ex => throw ex,
// () =>
// {
// Assert.AreEqual(count, numObjects);
// });
}
static void Main(string[] args)
{
try
{
var bucketList = minio.ListBucketsAsync().Result;
//获取分类列表
foreach (Bucket bucket in bucketList.Buckets)
{
Console.WriteLine("-" + bucket.Name + " " + bucket.CreationDateDateTime);
//获取文件列表
minio.ListObjectsAsync(bucket.Name).Subscribe(new MyObserver());
}
Console.Read();
//上传文件
var bucketName = bucketList.Buckets.FirstOrDefault().Name;
/*
*
* var fileName = "test.txt";
* var filePath = @"D:\MinIO\test\test.txt";
* var contentType = "";
* minio.PutObjectAsync(bucketName, fileName, filePath).GetAwaiter().GetResult();
* Console.WriteLine("Successfully uploaded " + fileName);
*/
//bucket policy
var policy = minio.GetPolicyAsync(bucketName).Result;
Console.WriteLine(policy);
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
Console.Read();
}
// List objects matching optional prefix in a specified bucket.
public static void Run(MinioClient minio,
string bucketName = "my-bucket-name",
string prefix = null,
bool recursive = true)
{
try
{
Console.WriteLine("Running example for API: ListObjectsAsync");
IObservable <Item> observable = minio.ListObjectsAsync(bucketName, prefix, recursive);
IDisposable subscription = observable.Subscribe(
item => Console.WriteLine($"Object: {item.Key}"),
ex => Console.WriteLine($"OnError: {ex}"),
() => Console.WriteLine($"Listed all objects in bucket {bucketName}\n"));
// subscription.Dispose();
}
catch (Exception e)
{
Console.WriteLine($"[Bucket] Exception: {e}");
}
}
// List objects matching optional prefix in a specified bucket.
public static void Run(MinioClient minio,
string bucketName = "my-bucket-name",
string prefix = null,
bool recursive = true,
bool versions = false)
{
try
{
Console.WriteLine("Running example for API: ListObjectsAsync");
var listArgs = new ListObjectsArgs()
.WithBucket(bucketName)
.WithPrefix(prefix)
.WithRecursive(recursive);
var observable = minio.ListObjectsAsync(listArgs);
var subscription = observable.Subscribe(
item => Console.WriteLine($"Object: {item.Key}"),
ex => Console.WriteLine($"OnError: {ex}"),
() => Console.WriteLine($"Listed all objects in bucket {bucketName}\n"));
}
catch (Exception e)
{
Console.WriteLine($"[Bucket] Exception: {e}");
}
}
public async Task <(int, string)> Handle(HttpRequest request)
{
var reader = new StreamReader(request.Body);
var input = await reader.ReadToEndAsync();
var parms = input.Split('|');
input = parms[0];
var newBucket = (parms.Length > 1) ? parms[1] : $"{input}-copy";
Console.WriteLine($"NEWBUCKET: {newBucket}");
var minio = new MinioClient(Environment.GetEnvironmentVariable("minio_endpoint"),
Environment.GetEnvironmentVariable("minio_access_key"),
Environment.GetEnvironmentVariable("minio_secret_key")
);
var objectsJson = new JArray();
var objects = new List <string>();
var result = string.Empty;
var message = string.Empty;
JObject json = new JObject(new JProperty("result", "success"));
IObservable <Item> observable = minio.ListObjectsAsync(input);
IDisposable subscription = observable.Subscribe(
item =>
{
objects.Add(item.Key);
objectsJson.Add(new JObject(new JProperty("name", item.Key)));
},
e =>
{
json = new JObject();
json.Add(new JProperty("result", "fail"));
json.Add(new JProperty("exception", e.Message));
},
() =>
{
json.Add(new JProperty("objects", objectsJson));
foreach (var f in objects)
{
var key = Guid.NewGuid().ToString();
var callJson = new JObject(
new JProperty("_key", key),
new JProperty("params", new JObject(
new JProperty("bucket", input),
new JProperty("object", f),
new JProperty("newBucket", newBucket))
));
writeTaskToDb(callJson);
string callResponse = string.Empty;
makeCallAsync(key).ContinueWith(x =>
{
if (x.IsFaulted)
{
callResponse = $"Error calling objectmover for {f}: {x.Exception.Message}";
}
else
{
callResponse = x.Result;
}
Console.WriteLine($"{input}: {callResponse}");
}).Wait();
json.Add(new JProperty($"call-{f}", callResponse));
}
});
observable.Wait();
return(200, json.ToString());
}
