public async Task <IEnumerable <SummaryActivity> > requestActivities(int totalRides_Runs_Swims, int perPage = 30)
{
var activities = new ConcurrentBag <SummaryActivity>();
int numPages = totalRides_Runs_Swims / perPage + 1;
//Loop for total/30 times to get at least the number of total rides, swims, and runs
Parallel.For(1, numPages, async page =>
{
var activitiesPage = await _activitiesApi.GetLoggedInAthleteActivitiesAsync(page: page, perPage: perPage);
if (activitiesPage != null && activitiesPage.Count != 0)
{
activities.AddRange(activitiesPage);
}
});
//Then sequentially request pages until returned page is empty
//This allows us to get the remaining actvities includes in the total (ski, yoga, workout, etc.)
while (true)
{
var activitesPage = await _activitiesApi.GetLoggedInAthleteActivitiesAsync(page : numPages, perPage : perPage);
if (activitesPage == null || activitesPage.Count == 0)
{
break;
}
activities.AddRange(activitesPage);
numPages++;
}
return(activities);
}
public async Task <Response> CreateResponseAsync()
{
var users = new ConcurrentBag <UserVm>();
var getUsersTasks = new List <Task>
{
Task.Run(async() =>
{
users.AddRange(privacyService.ApplyPrivacySettings(await loadUsersService.FindUsersByPhonesAsync(request.Phones).ConfigureAwait(false), request.Phones, clientConnection.UserId));
})
};
var nodeConnections = connectionsService.GetNodeConnections();
foreach (var connection in nodeConnections)
{
getUsersTasks.Add(Task.Run(async() =>
{
if (connection.IsEncryptedConnection && connection.NodeWebSocket.State == WebSocketState.Open)
{
users.AddRange(await nodeRequestSender.BatchPhonesSearchAsync(connection, request.Phones, clientConnection.UserId).ConfigureAwait(false));
}
}));
}
await Task.WhenAll(getUsersTasks).ConfigureAwait(false);
return(new UsersResponse(request.RequestId, users));
}
/// <inheritdoc />
public List <string> ValueFor([NotNull] string initialDirectory, [NotNull] FileListFromPathFilter fileListFromPathFilter)
{
if (initialDirectory == null)
{
throw new ArgumentNullException(nameof(initialDirectory));
}
_fileList = new ConcurrentBag <string>();
_fileListFromPathFilter = fileListFromPathFilter ?? throw new ArgumentNullException(nameof(fileListFromPathFilter));
if (!initialDirectory.IsAccessible())
{
return(_fileList.ToList());
}
//root directory.
var initialDirectoryFileList = Directory.GetFiles(initialDirectory).Select(item => item.ToLower()).ToList();
var dirList = initialDirectoryFileList.Where(FileIsValid).ToList();
//sub directories.
var initialDirectorySubdirectoriesFileList = GetSubdirectoriesContainingOnlyFiles(initialDirectory).SelectMany(Directory.GetFiles).Select(item => item.ToLower());
var dirSubList = initialDirectorySubdirectoriesFileList.Where(FileIsValid).ToList();
_fileList.AddRange(dirList);
_fileList.AddRange(dirSubList);
return(_fileList.ToList());
}
//TODO Fix Me
private static async Task <IEnumerable <DownloadResult> > GetDownloadables(string url, string extension, int levels)
{
var downloadResults = new ConcurrentBag <DownloadResult>();
var results = GetDownloadables(url, extension).Result.ToList();
downloadResults.AddRange(results);
var tempResults = new ConcurrentBag <DownloadResult>();
tempResults.AddRange(results);
for (int i = 0; i < levels; i++)
{
Parallel.ForEach(results.Where(r => r.IsDirectory),
//new ParallelOptions() { MaxDegreeOfParallelism = 1 },
async directory =>
{
tempResults.AddRange(await GetDownloadables(directory.Url, extension));
});
Console.WriteLine("Level {0}, {1} urls added", i, tempResults.Count);
downloadResults.AddRange(tempResults);
Console.WriteLine("Level {0}, {1} urls total", i, downloadResults.Count);
results = tempResults.Clone();
tempResults = new ConcurrentBag <DownloadResult>();
}
return(downloadResults.Distinct().OrderBy(x => x.Url));
}
/// <summary>
/// This is a internal class method that is used to get the products.
/// It is sperated from the public method to allow two versions of the public method to use this functionality
/// We require to versions of the public method so that we can make one Async for people using the api in such patterns
/// </summary>
/// <returns></returns>
private List <IProduct> InternalGetProducts()
{
//use a concurrent bag for the products since the loop is multi threaded
ConcurrentBag <IProduct> products = new ConcurrentBag <IProduct>();
//use a parallel for each to improve response time
Parallel.ForEach(mProviders, (prov) =>
{
IProvider provider;
switch (prov)
{
case Provider.Amazon:
provider = new Amazon.AmazonApi();
provider.setApiKeys(keystore.getKey(Provider.Amazon));
provider.setUPC(mUPC);
products.AddRange(provider.QueryProducts());
break;
case Provider.Ebay:
provider = new Ebay.EbayApi();
provider.setApiKeys(keystore.getKey(Provider.Ebay));
provider.setUPC(mUPC);
products.AddRange(provider.QueryProducts());
break;
case Provider.BestBuy:
provider = new BestBuy.BestBuy_Api();
provider.setApiKeys(keystore.getKey(Provider.BestBuy));
provider.setUPC(mUPC);
products.AddRange(provider.QueryProducts());
break;
case Provider.Kohls:
break;
case Provider.Target:
break;
case Provider.Walmart:
provider = new Walmart.WalmartApi();
provider.setApiKeys(keystore.getKey(Provider.Walmart));
provider.setUPC(mUPC);
products.AddRange(provider.QueryProducts());
break;
}
});
//use the to list method since we used a concurrentbag and need to return a list
return(products.ToList());
}
public void Given_TwoCollectionsAndConcurrentBag_When_AddRange_Then_ElementsAdded()
{
var concurrentBagExample = new ConcurrentBag <string>();
var testList1 = new List <string> {
"String1", "String2", "String3", "String4"
};
var testList2 = new List <string> {
"String5", "String6", "String7", "String8", "String9", "String0"
};
concurrentBagExample.AddRange(testList1);
concurrentBagExample.AddRange(testList2);
Assert.AreEqual(testList1.Count + testList2.Count, concurrentBagExample.Count);
}
private static void AddTypesToTypeList(IEnumerable <Type> types, ConcurrentBag <Type> typeList)
{
if (types.IsInstance())
{
typeList.AddRange(types);
}
}
// Gets the full game ranking entries
private async Task <List <RankingDto> > GetFullGameRankingAsync(RankingRequest request)
{
var rankingEntries = new ConcurrentBag <RankingDto>();
var tasks = new List <Task>();
foreach (var stage in request.Game.GetStages())
{
tasks.Add(Task.Run(async() =>
{
foreach (var level in SystemExtensions.Enumerate <Level>())
{
if (!request.SkipStages.Contains(stage))
{
var stageLevelRankings = await GetStageLevelRankingAsync(request, stage, level)
.ConfigureAwait(false);
rankingEntries.AddRange(stageLevelRankings);
}
}
}));
}
await Task.WhenAll(tasks).ConfigureAwait(false);
return(rankingEntries.ToList());
}
/// <summary>
/// Buils the flat content list.
/// </summary>
/// <param name="mediaContainer">The media container.</param>
/// <param name="folders">The folders.</param>
/// <param name="mediaTypeAlias">The media type alias.</param>
private void BuilFlatContentList(ConcurrentBag <IMedia> concurrentMediaContainer, ConcurrentBag <IMedia> concurrentFolders, string mediaTypeAlias, Nullable <DateTime> compareDate = null)
{
Parallel.ForEach(concurrentFolders, new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount / 2 * 10
}, (concurrentFolder) =>
{
switch (mediaTypeAlias)
{
case MediaTypeAlias.Folder:
if (compareDate.HasValue)
{
concurrentFolder.AddFolderIfContainsNewImages(concurrentMediaContainer, compareDate);
}
else
{
concurrentMediaContainer.Add(concurrentFolder);
}
break;
case MediaTypeAlias.Image:
concurrentMediaContainer.AddRange(concurrentFolder.Children().Where(p => p.ContentType.Alias == mediaTypeAlias));
break;
}
var folderMedia = new ConcurrentBag <IMedia>(concurrentFolder.Children().Where(p => p.ContentType.Alias == MediaTypeAlias.Folder));
BuilFlatContentList(concurrentMediaContainer, folderMedia, mediaTypeAlias, compareDate);
});
}
public async Task AddRange_Adds_Expected_Number_Of_Items()
{
var bag = new ConcurrentBag <int> {
1, 2, 3
};
var toAddA = new List <int> {
4, 5, 6
};
var toAddB = new List <int> {
7, 8, 9
};
var toAddLists = new List <List <int> > {
toAddA, toAddB
};
var expectedBag = new ConcurrentBag <int> {
1, 2, 3, 4, 5, 6, 7, 8, 9
};
// Doing things async, to emulate real-world usage of ConcurrentBag.
await toAddLists.ForEachAsync(
toAddLists.Count,
async t => { await Task.Run(() => bag.AddRange(t)); }
);
bag.Should().BeEquivalentTo(expectedBag);
}
public async Task SubscribingToAnEventOnAnyContract()
{
var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet);
//cancellation token to enable the listener to be stopped
//passing in a time limit as a safety valve for the unit test
var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1));
//somewhere to put matching events
//using ConcurrentBag because we'll be referencing the collection on different threads
var erc20Transfers = new ConcurrentBag <EventLog <TransferEventDto> >();
//initialise the processor with a blockchain url
var processor = web3.Eth.LogsProcessor <TransferEventDto>()
.OnEvents((events) => erc20Transfers.AddRange(events)) // subscribe to transfer events
.SetBlocksPerBatch(1) //optional: restrict batches to one block at a time
.SetMinimumBlockNumber(7540102) //optional: default is to start at current block on chain
// for test purposes we'll stop after processing a batch
.OnBatchProcessed((args) => cancellationTokenSource.Cancel())
.Build();
// run continually until cancellation token is fired
var rangesProcessed = await processor.ProcessContinuallyAsync(cancellationTokenSource.Token);
Assert.True(erc20Transfers.Any());
Assert.Equal((ulong)1, rangesProcessed);
}
internal static async Task GetSpecialAnime(User user, Token token, ConcurrentBag <SpecialUserAnimeRate> outContainer, AnimeStatus status = AnimeStatus.None)
{
using (HttpClient client = ClientWithHeaders(token.AccessToken))
{
string url;
if (!URI.ShikiUrls.TryGetValue(Link.AnimeListV2, out url))
{
throw new NoUriDictionaryException();
}
url = (status == AnimeStatus.None) ? string.Format(url, user.Id) : string.Concat(string.Format(url, user.Id), "&status=", AnimeParams.AnimeStatusString[status]);
var response = await client.GetAsync(url).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
var str = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
var array = JsonConvert.DeserializeObject <List <SpecialUserAnimeRate> >(str);
outContainer.AddRange(array);
}
else if (response.StatusCode == HttpStatusCode.Unauthorized)
{
throw new TokenExpiredException();
}
else
{
throw new FailedRequestException();
}
}
}
private static async Task <ConcurrentBag <T> > QueryAllTableEntities <T>(
string tableName, ICollection <string> partitionKeys, string rowFilter = "", int retries = -1)
where T : ITableEntity, new()
{
var allEntities = new ConcurrentBag <T>();
var table = await GetTable(tableName, retries).ConfigureAwait(false);
await partitionKeys.ForEachAsync(partitionKeys.Count, async partition =>
{
var partitionFilter =
TableQuery.GenerateFilterCondition(PartitionKey, QueryComparisons.Equal, partition.ToString(CultureInfo.InvariantCulture));
var filter = String.IsNullOrWhiteSpace(rowFilter)
? partitionFilter
: TableQuery.CombineFilters(partitionFilter, TableOperators.And, rowFilter);
var query = new TableQuery <T>().Where(filter);
await ProcessAllSegments(table, query,
batch => CommonEventSource.Log.RetrievePartitionBatchMappingsStart(partition, batch),
batch => CommonEventSource.Log.RetrievePartitionBatchMappingsStop(partition, batch),
segment => allEntities.AddRange(segment)).ConfigureAwait(false);
}).ConfigureAwait(false);
return(allEntities);
}
public async Task <Response> CreateResponseAsync()
{
try
{
var pollResults = await pollsService.GetPollVotedUsersAsync(
request.PollId,
request.ConversationId,
request.ConversationType,
request.OptionId,
clientConnection.UserId.GetValueOrDefault(),
30,
request.NavigationUserId.GetValueOrDefault()).ConfigureAwait(false);
var usersGroups = pollResults.GroupBy(opt => opt.FirstValue.NodeId);
List <Task> getUsersTasks = new List <Task>();
var resultUsers = new ConcurrentBag <VoteInfo>();
foreach (var group in usersGroups)
{
var nodeConnection = connectionsService.GetNodeConnection(group.Key.GetValueOrDefault());
if (nodeConnection != null)
{
await MetricsHelper.Instance.SetCrossNodeApiInvolvedAsync(request.RequestId).ConfigureAwait(false);
getUsersTasks.Add(Task.Run(async() =>
{
var usersInfo = await nodeRequestSender.GetUsersInfoAsync(group.Select(opt => opt.FirstValue.Id).ToList(), clientConnection.UserId, nodeConnection).ConfigureAwait(false);
resultUsers.AddRange(usersInfo.Select(user => new VoteInfo(user, pollResults.FirstOrDefault(opt => opt.FirstValue.Id == user.Id).SecondValue)));
}));
}
else if (group.Key == NodeSettings.Configs.Node.Id)
{
var users = UserConverter.GetUsersVm(group.Select(opt => opt.FirstValue).ToList(), clientConnection.UserId);
resultUsers.AddRange(users.Select(user => new VoteInfo(user, pollResults.FirstOrDefault(opt => opt.FirstValue.Id == user.Id).SecondValue)));
}
}
await Task.WhenAll(getUsersTasks).ConfigureAwait(false);
return(new PollResultsResponse(request.RequestId, resultUsers.OrderBy(opt => opt.User.Id)));
}
catch (PermissionDeniedException ex)
{
Logger.WriteLog(ex);
return(new ResultResponse(request.RequestId, "User does not have access to voted users list.", ErrorCode.PermissionDenied));
}
}
/// <summary>
/// Returns a list of test cases for a project.
/// </summary>
/// <param name="ids">A list of test ids to get test cases by.</param>
/// <returns>A collection of Rhino.Api.Contracts.AutomationProvider.RhinoTestCase</returns>
public override IEnumerable <RhinoTestCase> OnGetTestCases(params string[] ids)
{
// setup: issues map
var map = new ConcurrentDictionary <string, string>();
// build issues map
Parallel.ForEach(ids, options, id => map[id] = jiraClient.GetIssueType(idOrKey: id));
// entities
var byTests = map
.Where(i => i.Value.Equals($"{capabilities[AtlassianCapabilities.TestType]}", Compare))
.Select(i => i.Key);
var bySets = map
.Where(i => i.Value.Equals($"{capabilities[AtlassianCapabilities.SetType]}", Compare))
.Select(i => i.Key);
var byPlans = map
.Where(i => i.Value.Equals($"{capabilities[AtlassianCapabilities.PlanType]}", Compare))
.Select(i => i.Key);
var byExecutions = map
.Where(i => i.Value.Equals($"{capabilities[AtlassianCapabilities.ExecutionType]}", Compare))
.Select(i => i.Key);
// setup
var testCases = new ConcurrentBag <RhinoTestCase>();
// get and apply
var onTestCases = GetByTests(byTests);
testCases.AddRange(onTestCases);
onTestCases = GetBySets(bySets);
testCases.AddRange(onTestCases);
onTestCases = GetByPlans(byPlans);
testCases.AddRange(onTestCases);
onTestCases = GetByExecutions(byExecutions);
testCases.AddRange(onTestCases);
// results
return(testCases);
}
public void SetDao(ZclClusterDao dao)
{
_clusterId = dao.ClusterId;
_isClient = dao.IsClient;
_supportedAttributes.AddRange(dao.SupportedAttributes);
_supportedCommandsGenerated.AddRange(dao.SupportedCommandsGenerated);
_supportedCommandsReceived.AddRange(dao.SupportedCommandsReceived);
_attributes = dao.Attributes;
}
private void ProcessVsFiles(string path)
{
_slnFiles.AddRange(Directory.GetFiles(path, "*.sln", SearchOption.TopDirectoryOnly));
_projFiles.AddRange(Directory.GetFiles(path, "*.*proj", SearchOption.TopDirectoryOnly));
Parallel.ForEach(GetValidDirectories(path), (directory) =>
{
ProcessVsFiles(directory);
});
}
protected override IEnumerable <Product> GetProducts(string productName)
{
var products = new ConcurrentBag <Product>();
var tasks = new List <Task>();
var mainPageTask = Task.Run(() =>
{
var html = GetHtml($"https://www.jumbo.com/zoeken?SearchTerm={productName}");
var scrapedProducts = ScrapeProducts(html);
products.AddRange(scrapedProducts);
});
tasks.Add(mainPageTask);
var pageCount = 3; // DeterminePageCount(productName);
if (pageCount == 0)
{
return(products);
}
Parallel.For(1, pageCount, pageNumber =>
{
var task = Task.Run(() =>
{
var url = BuildUrl(productName, pageNumber);
var html = GetHtml(url);
var scrapedProducts = ScrapeProducts(html);
products.AddRange(scrapedProducts);
});
tasks.Add(task);
});
Task.WaitAll(tasks.ToArray());
return(products);
}
public void Given_CollectionSingleElementAndConcurrentBag_When_AddRange_Then_ElementsAdded()
{
var concurrentBagExample = new ConcurrentBag <string>();
var testList1 = new List <string> {
"String1", "String2", "String3", "String4"
};
concurrentBagExample.AddRange(testList1);
concurrentBagExample.Add("TEST");
Assert.AreEqual(testList1.Count + 1, concurrentBagExample.Count);
}
private async Task SetRatesForLeagues(IEnumerable <LeagueEntity> leagues)
{
var rates = new ConcurrentBag <BetRatesEntity>();
Parallel.ForEach(leagues, league =>
{
var leagueRates = _rateCalculator.GetLeagueGamesRates(league);
rates.AddRange(leagueRates);
});
await SaveRatesAsync(rates.ToList());
}
private async Task <IEnumerable <Adapters.Oal.dbo_normal_console_event_new> > GetEventRowsAsync(
Adapters.Oal.dbo_normal_console_event_new eventRow,
IList <string> pendingConsoles,
IList <string> consoleList,
string consignmentNotes,
int level = 0)
{
Console.Write("." + level);
var list = new ConcurrentBag <Adapters.Oal.dbo_normal_console_event_new>();
if (level > 4)
{
return(list);
}
var items = consignmentNotes.Split(new[] { ',', '\t' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
foreach (var c in items)
{
var itemConNote = c;
if (consoleList.Contains(itemConNote))
{
continue;
}
var row = await CloneChildAsync(eventRow, itemConNote);
list.Add(row);
if (!IsConsole(itemConNote))
{
continue;
}
consoleList.Add(itemConNote);
var childConsignments = await GetItemConsigmentsFromConsoleDetailsAsync(itemConNote);
if (null == childConsignments)
{
pendingConsoles.Add(itemConNote);
continue;
}
var children = await GetEventRowsAsync(eventRow,
pendingConsoles,
consoleList,
childConsignments.ToEmptyString().Trim(),
level + 1);
list.AddRange(children);
}
return(list);
}
public async Task SubscribingToMultipleEventsOnAContract()
{
var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet);
//cancellation token to enable the listener to be stopped
//passing in a time limit as a safety valve for the unit test
var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(2));
//somewhere to put matching events
//using ConcurrentBag because we'll be referencing the collection on different threads
var erc20Transfers = new ConcurrentBag <EventLog <TransferEventDto> >();
var approvals = new ConcurrentBag <EventLog <ApprovalEventDTO> >();
var all = new ConcurrentBag <FilterLog>();
//capture a fatal exception here (we're not expecting one!)
Exception fatalException = null;
//this is the contract we want to listen to
//the processor also accepts an array of addresses
const string ContractAddress = "0x9f8F72aA9304c8B593d555F12eF6589cC3A579A2";
//initialise the processor
//contract address or addresses is optional
//we don't need an account because this is read only
var processor = web3.Eth.LogsProcessor(ContractAddress)
.SetMinimumBlockNumber(7540000) //optional: default is to start at current block on chain
.SetBlocksPerBatch(100) //optional: number of blocks to scan at once, default is 100
.Add((events) => all.AddRange(events)) // any event for the contract/s - useful for logging
.Add <TransferEventDto>((events) => erc20Transfers.AddRange(events)) // transfer events
.Add <ApprovalEventDTO>((events) => approvals.AddRange(events)) // approval events
// optional: a handler for a fatal error which would stop processing
.OnFatalError((ex) => fatalException = ex)
// for test purposes we'll cancel after a batch or block range has been processed
// setting this is optional but is useful for monitoring progress
.OnBatchProcessed((args) => cancellationTokenSource.Cancel())
.Build();
// begin processing
var backgroundTask = processor.ProcessContinuallyInBackgroundAsync(cancellationTokenSource.Token);
//simulate doing something else whilst the listener works its magic!
while (!backgroundTask.IsCompleted)
{
await Task.Delay(1000);
}
Assert.True(backgroundTask.IsCanceled);
Assert.Equal(11, erc20Transfers.Count);
Assert.Equal(5, approvals.Count);
Assert.Equal(16, all.Count);
Assert.Null(fatalException);
}
public void Given_MultipleParallelizedCollectionsAndConcurrentBag_When_AddRange_Then_ElementsAdded()
{
var concurrentBagExample = new ConcurrentBag <string>();
Parallel.For(0, 10, i =>
{
concurrentBagExample.AddRange(new List <string> {
"Test1", "Test2"
});
});
Assert.AreEqual(10 * 2, concurrentBagExample.Count);
}
public async Task <Response> CreateResponseAsync()
{
List <ContactDto> contactsDto = await groupsService.GetGroupContactsAsync(
request.GroupId, clientConnection.UserId.GetValueOrDefault(), request.NavigationUserId.GetValueOrDefault()).ConfigureAwait(false);
List <ContactVm> contactsVm = ContactConverter.GetContactsVm(contactsDto);
IEnumerable <IGrouping <long, UserDto> > groupedUsers = contactsDto.Select(opt => opt.ContactUser).GroupBy(opt => opt.NodeId.GetValueOrDefault());
ConcurrentBag <UserVm> resultUsers = new ConcurrentBag <UserVm>();
List <Task> getUsersTasks = new List <Task>();
foreach (var group in groupedUsers)
{
getUsersTasks.Add(Task.Run(async() =>
{
var nodeConnection = connectionsService.GetNodeConnection(group.Key);
if (group.Key != NodeSettings.Configs.Node.Id && nodeConnection != null)
{
await MetricsHelper.Instance.SetCrossNodeApiInvolvedAsync(request.RequestId).ConfigureAwait(false);
var users = await nodeRequestSender.GetUsersInfoAsync(
group.Select(opt => opt.Id).ToList(),
clientConnection.UserId,
nodeConnection).ConfigureAwait(false);
resultUsers.AddRange(users);
resultUsers.AddRange(users);
}
else
{
resultUsers.AddRange(UserConverter.GetUsersVm(group.ToList(), clientConnection.UserId));
}
}));
}
await Task.WhenAll(getUsersTasks).ConfigureAwait(false);
foreach (var contact in contactsVm)
{
contact.ContactUser = resultUsers.FirstOrDefault(opt => opt.Id == contact.ContactUserId);
}
return(new ContactsResponse(request.RequestId, contactsVm));
}
static ParserRegistry()
{
var closedMethods = new ConcurrentBag <MethodInfo>();
var traverser = _traverser(t =>
{
var parsers = from m in t.GetDeclaredStaticMethods()
where m.HasAttribute <ParserAttribute>()
select m;
closedMethods.AddRange(parsers.Where(p => not(p.IsGenericMethod)));
});
traverser.Traverse(MetaFloor.Assembly);
Projector = ConversionSuite.FromMethods(closedMethods.ToArray());
}
/// <summary>
/// Splits all logfiles exceeding the configured size threshold into smaller chunks.
/// </summary>
/// <param name="files">The collection of files eligible for partitioning.</param>
/// <returns>Collection of all files in logset following the partitioning process.</returns>
public IEnumerable <LogFileContext> PartitionLargeFiles(IEnumerable <LogFileContext> files)
{
long maxBytes = tuningOptions.FilePartitionerThresholdMb * 1024 * 1024;
// Build list of files to chunk by searching for files exceeding the max size and consulting the parser factory to see if it's a single-line log type.
var processedFiles = new ConcurrentBag <LogFileContext>();
var filesToPartition = new List <LogFileContext>();
foreach (var file in files)
{
if (IsPartitionableFile(file, maxBytes))
{
filesToPartition.Add(file);
}
else
{
// Nothing to do; use it as-is.
processedFiles.Add(file);
}
}
if (!filesToPartition.Any())
{
Log.InfoFormat("No log files were found that are larger than {0}MB; skipping partitioning phase.", tuningOptions.FilePartitionerThresholdMb);
return(processedFiles);
}
Log.InfoFormat("Partitioning {0} log {1} larger than {2}MB to speed up processing. This may take some time..",
filesToPartition.Count, "file".Pluralize(filesToPartition.Count), tuningOptions.FilePartitionerThresholdMb);
// Set up task scheduler.
TaskFactory factory = GetFilePartitioningTaskFactory();
// Spin up partitioning tasks in parallel.
Task[] taskArray = new Task[filesToPartition.Count];
for (var i = 0; i < filesToPartition.Count; i++)
{
var fileToChunk = filesToPartition[i];
taskArray[i] = factory.StartNew(() =>
{
var partitions = PartitionFile(fileToChunk, maxBytes);
processedFiles.AddRange(partitions);
});
}
// Wait on any in-flight tasks.
Task.WaitAll(taskArray);
return(processedFiles);
}
public async Task UsingAzureTableStorageProgressRepository()
{
var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet);
// Requires: Nethereum.BlockchainStore.AzureTables
// Load config
// - this will contain the secrets and connection strings we don't want to hard code
var config = TestConfiguration.LoadConfig();
string azureStorageConnectionString = config["AzureStorageConnectionString"];
//cancellation token to enable the listener to be stopped
//passing in a time limit as a safety valve for the unit test
var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1));
//somewhere to put matching events
//using ConcurrentBag because we'll be referencing the collection on different threads
var erc20Transfers = new ConcurrentBag <EventLog <TransferEventDto> >();
//initialise the processor
var builder = web3.Eth.LogsProcessor <TransferEventDto>()
.OnEvents((events) => erc20Transfers.AddRange(events)) // transfer events
.SetBlocksPerBatch(1) //optional: restrict batches to one block at a time
.SetMinimumBlockNumber(7540102) //optional: default is to start at current block on chain
// for test purposes we'll stop after processing a batch
.OnBatchProcessed((args) => cancellationTokenSource.Cancel())
// tell the processor to reference an Azure Storage table for block progress
// this is an extension method from Nethereum.BlockchainStore.AzureTables
.UseAzureTableStorageForBlockProgress(azureStorageConnectionString, "EventLogProcessingSample");
var processor = builder.Build();
//we should have a BlockProgressRepository
Assert.NotNull(builder.BlockProgressRepository);
//there should be no prior progress
Assert.Null(await builder.BlockProgressRepository.GetLastBlockNumberProcessedAsync());
//run the processor for a while
var rangesProcessed = await processor.ProcessContinuallyAsync(cancellationTokenSource.Token);
//the last block processed should have been saved
Assert.NotNull(await builder.BlockProgressRepository.GetLastBlockNumberProcessedAsync());
//we should have captured some events
Assert.True(erc20Transfers.Any());
//clean up
await new CloudTableSetup(azureStorageConnectionString, "EventLogProcessingSample")
.GetCountersTable()
.DeleteIfExistsAsync();
}
public object Any(Search request)
{
var resultCollection = new ConcurrentBag <Comment>();
Parallel.Invoke(
() => resultCollection.AddRange(FacebookService.Client.Get(new Search {
Keyword = request.Keyword
})),
() => resultCollection.AddRange(FlickrService.Client.Get(new Search {
Keyword = request.Keyword
})),
() => resultCollection.AddRange(InstagramService.Client.Get(new Search {
Keyword = request.Keyword
})),
() => resultCollection.AddRange(TwitterService.Client.Get(new Search {
Keyword = request.Keyword
})),
() => resultCollection.AddRange(YouTubeService.Client.Get(new Search {
Keyword = request.Keyword
}))
);
return(resultCollection.ToList());// response;
}
public virtual async Task <IEnumerable <ITorrentSearchResult> > SearchMovie(string name, int?year, string imdbId, VideoQuality videoQuality = VideoQuality.Any,
string extraKeywords = null, string excludeKeywords = null,
int?minSize = null, int?maxSize = null, int?minSeed = null)
{
var results = new ConcurrentBag <ITorrentSearchResult>();
var tasks = MovieProviders.RunTasks(p =>
p.SearchMovie(name, year, imdbId, videoQuality, extraKeywords, excludeKeywords, minSize, maxSize, minSeed, this)
.ContinueWith(t => results.AddRange(t.Result)),
ExceptionHandler
);
await Task.WhenAll(tasks);
return(results.OrderByDescending(r => r.Seed));
}
public async Task UsingJsonFileProgressRepository()
{
var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet);
//cancellation token to enable the listener to be stopped
//passing in a time limit as a safety valve for the unit test
var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1));
//somewhere to put matching events
//using ConcurrentBag because we'll be referencing the collection on different threads
var erc20Transfers = new ConcurrentBag <EventLog <TransferEventDto> >();
// define a progress repository
// this sample uses an out of the box simple json file implementation
// if you want your own - the IBlockProgressRepository interface is easy to implement
// the processor will use this repo to define which block to start at and update it after each batch is complete
// it can prevent duplicate processing that could occur after a restart
var jsonFilePath = Path.Combine(Path.GetTempPath(), "EventProcessingBlockProgress.json");
//initialise the builder
var builder = web3.Eth.LogsProcessor <TransferEventDto>()
.OnEvents((events) => erc20Transfers.AddRange(events)) // transfer events
.SetBlocksPerBatch(1) //optional: restrict batches to one block at a time
.SetMinimumBlockNumber(7540102) //optional: default is to start at current block on chain
// for test purposes we'll stop after processing a batch
.OnBatchProcessed((args) => cancellationTokenSource.Cancel())
// tell the processor to use a Json File based Block Progress Repository
// for test purposes only we delete any existing file to ensure we start afresh with no previous state
.UseJsonFileForBlockProgress(jsonFilePath, deleteExistingFile: true);
var processor = builder.Build();
//we should have a BlockProgressRepository
Assert.NotNull(builder.BlockProgressRepository);
//there should be no prior progress
Assert.Null(await builder.BlockProgressRepository.GetLastBlockNumberProcessedAsync());
//run the processor for a while
var rangesProcessed = await processor.ProcessContinuallyAsync(cancellationTokenSource.Token);
//the last block processed should have been saved
Assert.NotNull(await builder.BlockProgressRepository.GetLastBlockNumberProcessedAsync());
//we should have captured some events
Assert.True(erc20Transfers.Any());
//clean up
File.Delete(jsonFilePath);
}
public virtual async Task<IEnumerable<ITorrentSearchResult>> SearchMovie(string name, int? year, string imdbId, VideoQuality videoQuality = VideoQuality.Any,
string extraKeywords = null, string excludeKeywords = null,
int? minSize = null, int? maxSize = null, int? minSeed = null) {
var results = new ConcurrentBag<ITorrentSearchResult>();
var tasks = MovieProviders.RunTasks(p =>
p.SearchMovie(name, year, imdbId, videoQuality, extraKeywords, excludeKeywords, minSize, maxSize, minSeed, this)
.ContinueWith(t => results.AddRange(t.Result)),
ExceptionHandler
);
await Task.WhenAll(tasks);
return results.OrderByDescending(r => r.Seed);
}
public virtual async Task<IEnumerable<ITorrentSearchResult>> Search(string query, VideoQuality videoQuality = VideoQuality.Any, string excludeKeywords = null,
int? minSize = null, int? maxSize = null, int? minSeed = null) {
var providers = ((IEnumerable<ITorrentProvider>)MovieProviders).Union(TvShowProviders);
var results = new ConcurrentBag<ITorrentSearchResult>();
var tasks = providers.RunTasks(p =>
p.Search(query, videoQuality, excludeKeywords, minSize, maxSize, minSeed, this)
.ContinueWith(t => results.AddRange(t.Result)),
ExceptionHandler
);
await Task.WhenAll(tasks);
return results.OrderByDescending(r => r.Seed);
}
public virtual async Task<IEnumerable<ITorrentSearchResult>> SearchTvShowEpisode(string name, int season, int episode, string episodeName, string imdbId,
VideoQuality videoQuality = VideoQuality.Any, string extraKeywords = null, string excludeKeywords = null,
int? minSize = null, int? maxSize = null, int? minSeed = null) {
var results = new ConcurrentBag<ITorrentSearchResult>();
var tasks = TvShowProviders.RunTasks(p =>
p.SearchTvShowEpisode(name, season, episode, episodeName, imdbId, videoQuality, extraKeywords, excludeKeywords, minSize, maxSize, minSeed, this)
.ContinueWith(t => results.AddRange(t.Result)),
ExceptionHandler
);
await Task.WhenAll(tasks);
return results
.Where(r => {
int? s, e;
Helper.DetectEpisodeInfo(r.Name, name, out s, out e);
return (s == null || s == season) && e == episode;
})
.OrderByDescending(r => r.Seed)
.ToList();
}
private static void AddTypesToTypeList(IEnumerable<Type> types, ConcurrentBag<Type> typeList)
{
if (types.IsInstance())
typeList.AddRange(types);
}