public async Task CanBrowseWithSharding()
{
var ms = new MemoryStream();
var streamWriter = new StreamWriter(ms);
var expected = new string('a', 1024);
streamWriter.Write(expected);
streamWriter.Flush();
ms.Position = 0;
await shardedClient.UploadAsync("a.txt", ms);
await shardedClient.UploadAsync("b.txt", ms);
await shardedClient.UploadAsync("c.txt", ms);
await shardedClient.UploadAsync("d.txt", ms);
await shardedClient.UploadAsync("e.txt", ms);
var pagingInfo = new ShardPagingInfo(shardedClient.NumberOfShards);
var result = await shardedClient.BrowseAsync(2, pagingInfo);
Assert.Equal(2, result.Length);
pagingInfo.CurrentPage++;
result = await shardedClient.BrowseAsync(2, pagingInfo);
Assert.Equal(2, result.Length);
pagingInfo.CurrentPage++;
result = await shardedClient.BrowseAsync(2, pagingInfo);
Assert.Equal(1, result.Length);
pagingInfo.CurrentPage++;
result = await shardedClient.BrowseAsync(2, pagingInfo);
Assert.Equal(0, result.Length);
}
public async Task <SearchResults> SearchAsync(string query, string[] sortFields = null, int pageSize = 25, ShardPagingInfo pagingInfo = null)
{
if (pagingInfo == null)
{
pagingInfo = new ShardPagingInfo(Clients.Count);
}
var indexes = pagingInfo.GetPagingInfo(pagingInfo.CurrentPage);
if (indexes == null)
{
var lastPage = pagingInfo.GetLastPageNumber();
if (pagingInfo.CurrentPage - lastPage > 10)
{
throw new InvalidOperationException("Not Enough info in order to calculate requested page in a timely fation, last page info is for page #" + lastPage + ", please go to a closer page");
}
var originalPage = pagingInfo.CurrentPage;
pagingInfo.CurrentPage = lastPage;
while (pagingInfo.CurrentPage < originalPage)
{
await SearchAsync(query, sortFields, pageSize, pagingInfo).ConfigureAwait(false);
pagingInfo.CurrentPage++;
}
indexes = pagingInfo.GetPagingInfo(pagingInfo.CurrentPage);
}
var result = new SearchResults
{
Files = new List <FileHeader>()
};
var applyAsync = await Strategy.ShardAccessStrategy.ApplyAsync(
Clients.Values.ToList(), new ShardRequestData(),
(client, i) => client.SearchAsync(query, sortFields, indexes[i], pageSize))
.ConfigureAwait(false);
var originalIndexes = pagingInfo.GetPagingInfo(pagingInfo.CurrentPage);
while (result.FileCount < pageSize)
{
var item = GetSmallest(applyAsync, indexes, originalIndexes, sortFields);
if (item == null)
{
break;
}
result.Files.Add(item);
result.FileCount++;
result.PageSize = pageSize;
result.Start = 0; //TODO: update start
}
pagingInfo.SetPagingInfo(indexes);
result.DurationMilliseconds += applyAsync.Sum(x => x.DurationMilliseconds);
return(result);
}
public async Task <string[]> GetFoldersAsync(string @from = null, int pageSize = 25, ShardPagingInfo pagingInfo = null)
{
if (pagingInfo == null)
{
pagingInfo = new ShardPagingInfo(Clients.Count);
}
var indexes = pagingInfo.GetPagingInfo(pagingInfo.CurrentPage);
if (indexes == null)
{
var lastPage = pagingInfo.GetLastPageNumber();
if (pagingInfo.CurrentPage - lastPage > 10)
{
throw new InvalidOperationException("Not Enough info in order to calculate requested page in a timely fation, last page info is for page #" + lastPage + ", please go to a closer page");
}
var originalPage = pagingInfo.CurrentPage;
pagingInfo.CurrentPage = lastPage;
while (pagingInfo.CurrentPage < originalPage)
{
await GetFoldersAsync(from, pageSize, pagingInfo).ConfigureAwait(false);
pagingInfo.CurrentPage++;
}
indexes = pagingInfo.GetPagingInfo(pagingInfo.CurrentPage);
}
var results = new List <string>();
var applyAsync =
await
Strategy.ShardAccessStrategy.ApplyAsync(Clients.Values.ToList(), new ShardRequestData(),
(client, i) => client.GetDirectoriesAsync(from, indexes[i], pageSize))
.ConfigureAwait(false);
var originalIndexes = pagingInfo.GetPagingInfo(pagingInfo.CurrentPage);
while (results.Count < pageSize)
{
var item = GetSmallest(applyAsync, indexes, originalIndexes);
if (item == null)
{
break;
}
results.Add(item);
}
pagingInfo.SetPagingInfo(indexes);
return(results.ToArray());
}
public Task <SearchResults> GetFilesAsync(string folder, FilesSortOptions options = FilesSortOptions.Default, string fileNameSearchPattern = "", int pageSize = 25, ShardPagingInfo pagingInfo = null)
{
var folderQueryPart = GetFolderQueryPart(folder);
if (string.IsNullOrEmpty(fileNameSearchPattern) == false && fileNameSearchPattern.Contains("*") == false &&
fileNameSearchPattern.Contains("?") == false)
{
fileNameSearchPattern = fileNameSearchPattern + "*";
}
var fileNameQueryPart = GetFileNameQueryPart(fileNameSearchPattern);
return(SearchAsync(folderQueryPart + fileNameQueryPart, GetSortFields(options), pageSize, pagingInfo));
}
public async Task CanNotBrowseToPageFarAway()
{
var ms = new MemoryStream();
var streamWriter = new StreamWriter(ms);
var expected = new string('a', 1024);
streamWriter.Write(expected);
streamWriter.Flush();
ms.Position = 0;
await shardedClient.UploadAsync("a.txt", ms);
await shardedClient.UploadAsync("b.txt", ms);
await shardedClient.UploadAsync("c.txt", ms);
await shardedClient.UploadAsync("d.txt", ms);
await shardedClient.UploadAsync("e.txt", ms);
var pagingInfo = new ShardPagingInfo(shardedClient.NumberOfShards) { CurrentPage = 20 };
try
{
await shardedClient.BrowseAsync(2, pagingInfo);
Assert.Equal(true, false);//Should not get here
}
catch (Exception exception)
{
Assert.IsType<InvalidOperationException>(exception);
}
}
public Task<SearchResults> GetFilesAsync(string folder, FilesSortOptions options = FilesSortOptions.Default, string fileNameSearchPattern = "", int pageSize = 25, ShardPagingInfo pagingInfo = null)
{
var folderQueryPart = GetFolderQueryPart(folder);
if (string.IsNullOrEmpty(fileNameSearchPattern) == false && fileNameSearchPattern.Contains("*") == false &&
fileNameSearchPattern.Contains("?") == false)
{
fileNameSearchPattern = fileNameSearchPattern + "*";
}
var fileNameQueryPart = GetFileNameQueryPart(fileNameSearchPattern);
return SearchAsync(folderQueryPart + fileNameQueryPart, GetSortFields(options), pageSize, pagingInfo);
}
public async Task<string[]> GetFoldersAsync(string @from = null, int pageSize = 25, ShardPagingInfo pagingInfo = null)
{
if (pagingInfo == null)
pagingInfo = new ShardPagingInfo(Clients.Count);
var indexes = pagingInfo.GetPagingInfo(pagingInfo.CurrentPage);
if (indexes == null)
{
var lastPage = pagingInfo.GetLastPageNumber();
if (pagingInfo.CurrentPage - lastPage > 10)
throw new InvalidOperationException("Not Enough info in order to calculate requested page in a timely fation, last page info is for page #" + lastPage + ", please go to a closer page");
var originalPage = pagingInfo.CurrentPage;
pagingInfo.CurrentPage = lastPage;
while (pagingInfo.CurrentPage < originalPage)
{
await GetFoldersAsync(from, pageSize, pagingInfo).ConfigureAwait(false);
pagingInfo.CurrentPage++;
}
indexes = pagingInfo.GetPagingInfo(pagingInfo.CurrentPage);
}
var results = new List<string>();
var applyAsync =
await
Strategy.ShardAccessStrategy.ApplyAsync(Clients.Values.ToList(), new ShardRequestData(),
(client, i) => client.GetDirectoriesAsync(from, indexes[i], pageSize))
.ConfigureAwait(false);
var originalIndexes = pagingInfo.GetPagingInfo(pagingInfo.CurrentPage);
while (results.Count < pageSize)
{
var item = GetSmallest(applyAsync, indexes, originalIndexes);
if (item == null)
break;
results.Add(item);
}
pagingInfo.SetPagingInfo(indexes);
return results.ToArray();
}
public async Task<SearchResults> SearchAsync(string query, string[] sortFields = null, int pageSize = 25, ShardPagingInfo pagingInfo = null)
{
if (pagingInfo == null)
pagingInfo = new ShardPagingInfo(Clients.Count);
var indexes = pagingInfo.GetPagingInfo(pagingInfo.CurrentPage);
if (indexes == null)
{
var lastPage = pagingInfo.GetLastPageNumber();
if (pagingInfo.CurrentPage - lastPage > 10)
throw new InvalidOperationException("Not Enough info in order to calculate requested page in a timely fation, last page info is for page #" + lastPage + ", please go to a closer page");
var originalPage = pagingInfo.CurrentPage;
pagingInfo.CurrentPage = lastPage;
while (pagingInfo.CurrentPage < originalPage)
{
await SearchAsync(query, sortFields, pageSize, pagingInfo).ConfigureAwait(false);
pagingInfo.CurrentPage++;
}
indexes = pagingInfo.GetPagingInfo(pagingInfo.CurrentPage);
}
var result = new SearchResults();
var applyAsync = await Strategy.ShardAccessStrategy.ApplyAsync(
Clients.Values.ToList(), new ShardRequestData(),
(client, i) => client.SearchAsync(query, sortFields, indexes[i], pageSize))
.ConfigureAwait(false);
var originalIndexes = pagingInfo.GetPagingInfo(pagingInfo.CurrentPage);
while (result.FileCount < pageSize)
{
var item = GetSmallest(applyAsync, indexes, originalIndexes, sortFields);
if (item == null)
break;
var files = new List<FileHeader>();
if (result.Files != null)
files.AddRange(result.Files);
if (item.Files != null)
files.AddRange(item.Files);
result.FileCount++;
result.Files = files;
result.PageSize = pageSize;
result.Start = 0; //TODO: update start
}
pagingInfo.SetPagingInfo(indexes);
result.Files = result.Files.Where(info => info != null).ToList();
result.FileCount = result.Files.Count;
return result;
}