public async Task <IEnumerable <SummaryActivity> > requestActivities(int totalRides_Runs_Swims, int perPage = 30) { var activities = new ConcurrentBag <SummaryActivity>(); int numPages = totalRides_Runs_Swims / perPage + 1; //Loop for total/30 times to get at least the number of total rides, swims, and runs Parallel.For(1, numPages, async page => { var activitiesPage = await _activitiesApi.GetLoggedInAthleteActivitiesAsync(page: page, perPage: perPage); if (activitiesPage != null && activitiesPage.Count != 0) { activities.AddRange(activitiesPage); } }); //Then sequentially request pages until returned page is empty //This allows us to get the remaining actvities includes in the total (ski, yoga, workout, etc.) while (true) { var activitesPage = await _activitiesApi.GetLoggedInAthleteActivitiesAsync(page : numPages, perPage : perPage); if (activitesPage == null || activitesPage.Count == 0) { break; } activities.AddRange(activitesPage); numPages++; } return(activities); }
public async Task <Response> CreateResponseAsync() { var users = new ConcurrentBag <UserVm>(); var getUsersTasks = new List <Task> { Task.Run(async() => { users.AddRange(privacyService.ApplyPrivacySettings(await loadUsersService.FindUsersByPhonesAsync(request.Phones).ConfigureAwait(false), request.Phones, clientConnection.UserId)); }) }; var nodeConnections = connectionsService.GetNodeConnections(); foreach (var connection in nodeConnections) { getUsersTasks.Add(Task.Run(async() => { if (connection.IsEncryptedConnection && connection.NodeWebSocket.State == WebSocketState.Open) { users.AddRange(await nodeRequestSender.BatchPhonesSearchAsync(connection, request.Phones, clientConnection.UserId).ConfigureAwait(false)); } })); } await Task.WhenAll(getUsersTasks).ConfigureAwait(false); return(new UsersResponse(request.RequestId, users)); }
/// <inheritdoc /> public List <string> ValueFor([NotNull] string initialDirectory, [NotNull] FileListFromPathFilter fileListFromPathFilter) { if (initialDirectory == null) { throw new ArgumentNullException(nameof(initialDirectory)); } _fileList = new ConcurrentBag <string>(); _fileListFromPathFilter = fileListFromPathFilter ?? throw new ArgumentNullException(nameof(fileListFromPathFilter)); if (!initialDirectory.IsAccessible()) { return(_fileList.ToList()); } //root directory. var initialDirectoryFileList = Directory.GetFiles(initialDirectory).Select(item => item.ToLower()).ToList(); var dirList = initialDirectoryFileList.Where(FileIsValid).ToList(); //sub directories. var initialDirectorySubdirectoriesFileList = GetSubdirectoriesContainingOnlyFiles(initialDirectory).SelectMany(Directory.GetFiles).Select(item => item.ToLower()); var dirSubList = initialDirectorySubdirectoriesFileList.Where(FileIsValid).ToList(); _fileList.AddRange(dirList); _fileList.AddRange(dirSubList); return(_fileList.ToList()); }
//TODO Fix Me private static async Task <IEnumerable <DownloadResult> > GetDownloadables(string url, string extension, int levels) { var downloadResults = new ConcurrentBag <DownloadResult>(); var results = GetDownloadables(url, extension).Result.ToList(); downloadResults.AddRange(results); var tempResults = new ConcurrentBag <DownloadResult>(); tempResults.AddRange(results); for (int i = 0; i < levels; i++) { Parallel.ForEach(results.Where(r => r.IsDirectory), //new ParallelOptions() { MaxDegreeOfParallelism = 1 }, async directory => { tempResults.AddRange(await GetDownloadables(directory.Url, extension)); }); Console.WriteLine("Level {0}, {1} urls added", i, tempResults.Count); downloadResults.AddRange(tempResults); Console.WriteLine("Level {0}, {1} urls total", i, downloadResults.Count); results = tempResults.Clone(); tempResults = new ConcurrentBag <DownloadResult>(); } return(downloadResults.Distinct().OrderBy(x => x.Url)); }
/// <summary> /// This is a internal class method that is used to get the products. /// It is sperated from the public method to allow two versions of the public method to use this functionality /// We require to versions of the public method so that we can make one Async for people using the api in such patterns /// </summary> /// <returns></returns> private List <IProduct> InternalGetProducts() { //use a concurrent bag for the products since the loop is multi threaded ConcurrentBag <IProduct> products = new ConcurrentBag <IProduct>(); //use a parallel for each to improve response time Parallel.ForEach(mProviders, (prov) => { IProvider provider; switch (prov) { case Provider.Amazon: provider = new Amazon.AmazonApi(); provider.setApiKeys(keystore.getKey(Provider.Amazon)); provider.setUPC(mUPC); products.AddRange(provider.QueryProducts()); break; case Provider.Ebay: provider = new Ebay.EbayApi(); provider.setApiKeys(keystore.getKey(Provider.Ebay)); provider.setUPC(mUPC); products.AddRange(provider.QueryProducts()); break; case Provider.BestBuy: provider = new BestBuy.BestBuy_Api(); provider.setApiKeys(keystore.getKey(Provider.BestBuy)); provider.setUPC(mUPC); products.AddRange(provider.QueryProducts()); break; case Provider.Kohls: break; case Provider.Target: break; case Provider.Walmart: provider = new Walmart.WalmartApi(); provider.setApiKeys(keystore.getKey(Provider.Walmart)); provider.setUPC(mUPC); products.AddRange(provider.QueryProducts()); break; } }); //use the to list method since we used a concurrentbag and need to return a list return(products.ToList()); }
public void Given_TwoCollectionsAndConcurrentBag_When_AddRange_Then_ElementsAdded() { var concurrentBagExample = new ConcurrentBag <string>(); var testList1 = new List <string> { "String1", "String2", "String3", "String4" }; var testList2 = new List <string> { "String5", "String6", "String7", "String8", "String9", "String0" }; concurrentBagExample.AddRange(testList1); concurrentBagExample.AddRange(testList2); Assert.AreEqual(testList1.Count + testList2.Count, concurrentBagExample.Count); }
private static void AddTypesToTypeList(IEnumerable <Type> types, ConcurrentBag <Type> typeList) { if (types.IsInstance()) { typeList.AddRange(types); } }
// Gets the full game ranking entries private async Task <List <RankingDto> > GetFullGameRankingAsync(RankingRequest request) { var rankingEntries = new ConcurrentBag <RankingDto>(); var tasks = new List <Task>(); foreach (var stage in request.Game.GetStages()) { tasks.Add(Task.Run(async() => { foreach (var level in SystemExtensions.Enumerate <Level>()) { if (!request.SkipStages.Contains(stage)) { var stageLevelRankings = await GetStageLevelRankingAsync(request, stage, level) .ConfigureAwait(false); rankingEntries.AddRange(stageLevelRankings); } } })); } await Task.WhenAll(tasks).ConfigureAwait(false); return(rankingEntries.ToList()); }
/// <summary> /// Buils the flat content list. /// </summary> /// <param name="mediaContainer">The media container.</param> /// <param name="folders">The folders.</param> /// <param name="mediaTypeAlias">The media type alias.</param> private void BuilFlatContentList(ConcurrentBag <IMedia> concurrentMediaContainer, ConcurrentBag <IMedia> concurrentFolders, string mediaTypeAlias, Nullable <DateTime> compareDate = null) { Parallel.ForEach(concurrentFolders, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount / 2 * 10 }, (concurrentFolder) => { switch (mediaTypeAlias) { case MediaTypeAlias.Folder: if (compareDate.HasValue) { concurrentFolder.AddFolderIfContainsNewImages(concurrentMediaContainer, compareDate); } else { concurrentMediaContainer.Add(concurrentFolder); } break; case MediaTypeAlias.Image: concurrentMediaContainer.AddRange(concurrentFolder.Children().Where(p => p.ContentType.Alias == mediaTypeAlias)); break; } var folderMedia = new ConcurrentBag <IMedia>(concurrentFolder.Children().Where(p => p.ContentType.Alias == MediaTypeAlias.Folder)); BuilFlatContentList(concurrentMediaContainer, folderMedia, mediaTypeAlias, compareDate); }); }
public async Task AddRange_Adds_Expected_Number_Of_Items() { var bag = new ConcurrentBag <int> { 1, 2, 3 }; var toAddA = new List <int> { 4, 5, 6 }; var toAddB = new List <int> { 7, 8, 9 }; var toAddLists = new List <List <int> > { toAddA, toAddB }; var expectedBag = new ConcurrentBag <int> { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; // Doing things async, to emulate real-world usage of ConcurrentBag. await toAddLists.ForEachAsync( toAddLists.Count, async t => { await Task.Run(() => bag.AddRange(t)); } ); bag.Should().BeEquivalentTo(expectedBag); }
public async Task SubscribingToAnEventOnAnyContract() { var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet); //cancellation token to enable the listener to be stopped //passing in a time limit as a safety valve for the unit test var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1)); //somewhere to put matching events //using ConcurrentBag because we'll be referencing the collection on different threads var erc20Transfers = new ConcurrentBag <EventLog <TransferEventDto> >(); //initialise the processor with a blockchain url var processor = web3.Eth.LogsProcessor <TransferEventDto>() .OnEvents((events) => erc20Transfers.AddRange(events)) // subscribe to transfer events .SetBlocksPerBatch(1) //optional: restrict batches to one block at a time .SetMinimumBlockNumber(7540102) //optional: default is to start at current block on chain // for test purposes we'll stop after processing a batch .OnBatchProcessed((args) => cancellationTokenSource.Cancel()) .Build(); // run continually until cancellation token is fired var rangesProcessed = await processor.ProcessContinuallyAsync(cancellationTokenSource.Token); Assert.True(erc20Transfers.Any()); Assert.Equal((ulong)1, rangesProcessed); }
internal static async Task GetSpecialAnime(User user, Token token, ConcurrentBag <SpecialUserAnimeRate> outContainer, AnimeStatus status = AnimeStatus.None) { using (HttpClient client = ClientWithHeaders(token.AccessToken)) { string url; if (!URI.ShikiUrls.TryGetValue(Link.AnimeListV2, out url)) { throw new NoUriDictionaryException(); } url = (status == AnimeStatus.None) ? string.Format(url, user.Id) : string.Concat(string.Format(url, user.Id), "&status=", AnimeParams.AnimeStatusString[status]); var response = await client.GetAsync(url).ConfigureAwait(false); if (response.IsSuccessStatusCode) { var str = await response.Content.ReadAsStringAsync().ConfigureAwait(false); var array = JsonConvert.DeserializeObject <List <SpecialUserAnimeRate> >(str); outContainer.AddRange(array); } else if (response.StatusCode == HttpStatusCode.Unauthorized) { throw new TokenExpiredException(); } else { throw new FailedRequestException(); } } }
private static async Task <ConcurrentBag <T> > QueryAllTableEntities <T>( string tableName, ICollection <string> partitionKeys, string rowFilter = "", int retries = -1) where T : ITableEntity, new() { var allEntities = new ConcurrentBag <T>(); var table = await GetTable(tableName, retries).ConfigureAwait(false); await partitionKeys.ForEachAsync(partitionKeys.Count, async partition => { var partitionFilter = TableQuery.GenerateFilterCondition(PartitionKey, QueryComparisons.Equal, partition.ToString(CultureInfo.InvariantCulture)); var filter = String.IsNullOrWhiteSpace(rowFilter) ? partitionFilter : TableQuery.CombineFilters(partitionFilter, TableOperators.And, rowFilter); var query = new TableQuery <T>().Where(filter); await ProcessAllSegments(table, query, batch => CommonEventSource.Log.RetrievePartitionBatchMappingsStart(partition, batch), batch => CommonEventSource.Log.RetrievePartitionBatchMappingsStop(partition, batch), segment => allEntities.AddRange(segment)).ConfigureAwait(false); }).ConfigureAwait(false); return(allEntities); }
public async Task <Response> CreateResponseAsync() { try { var pollResults = await pollsService.GetPollVotedUsersAsync( request.PollId, request.ConversationId, request.ConversationType, request.OptionId, clientConnection.UserId.GetValueOrDefault(), 30, request.NavigationUserId.GetValueOrDefault()).ConfigureAwait(false); var usersGroups = pollResults.GroupBy(opt => opt.FirstValue.NodeId); List <Task> getUsersTasks = new List <Task>(); var resultUsers = new ConcurrentBag <VoteInfo>(); foreach (var group in usersGroups) { var nodeConnection = connectionsService.GetNodeConnection(group.Key.GetValueOrDefault()); if (nodeConnection != null) { await MetricsHelper.Instance.SetCrossNodeApiInvolvedAsync(request.RequestId).ConfigureAwait(false); getUsersTasks.Add(Task.Run(async() => { var usersInfo = await nodeRequestSender.GetUsersInfoAsync(group.Select(opt => opt.FirstValue.Id).ToList(), clientConnection.UserId, nodeConnection).ConfigureAwait(false); resultUsers.AddRange(usersInfo.Select(user => new VoteInfo(user, pollResults.FirstOrDefault(opt => opt.FirstValue.Id == user.Id).SecondValue))); })); } else if (group.Key == NodeSettings.Configs.Node.Id) { var users = UserConverter.GetUsersVm(group.Select(opt => opt.FirstValue).ToList(), clientConnection.UserId); resultUsers.AddRange(users.Select(user => new VoteInfo(user, pollResults.FirstOrDefault(opt => opt.FirstValue.Id == user.Id).SecondValue))); } } await Task.WhenAll(getUsersTasks).ConfigureAwait(false); return(new PollResultsResponse(request.RequestId, resultUsers.OrderBy(opt => opt.User.Id))); } catch (PermissionDeniedException ex) { Logger.WriteLog(ex); return(new ResultResponse(request.RequestId, "User does not have access to voted users list.", ErrorCode.PermissionDenied)); } }
/// <summary> /// Returns a list of test cases for a project. /// </summary> /// <param name="ids">A list of test ids to get test cases by.</param> /// <returns>A collection of Rhino.Api.Contracts.AutomationProvider.RhinoTestCase</returns> public override IEnumerable <RhinoTestCase> OnGetTestCases(params string[] ids) { // setup: issues map var map = new ConcurrentDictionary <string, string>(); // build issues map Parallel.ForEach(ids, options, id => map[id] = jiraClient.GetIssueType(idOrKey: id)); // entities var byTests = map .Where(i => i.Value.Equals($"{capabilities[AtlassianCapabilities.TestType]}", Compare)) .Select(i => i.Key); var bySets = map .Where(i => i.Value.Equals($"{capabilities[AtlassianCapabilities.SetType]}", Compare)) .Select(i => i.Key); var byPlans = map .Where(i => i.Value.Equals($"{capabilities[AtlassianCapabilities.PlanType]}", Compare)) .Select(i => i.Key); var byExecutions = map .Where(i => i.Value.Equals($"{capabilities[AtlassianCapabilities.ExecutionType]}", Compare)) .Select(i => i.Key); // setup var testCases = new ConcurrentBag <RhinoTestCase>(); // get and apply var onTestCases = GetByTests(byTests); testCases.AddRange(onTestCases); onTestCases = GetBySets(bySets); testCases.AddRange(onTestCases); onTestCases = GetByPlans(byPlans); testCases.AddRange(onTestCases); onTestCases = GetByExecutions(byExecutions); testCases.AddRange(onTestCases); // results return(testCases); }
public void SetDao(ZclClusterDao dao) { _clusterId = dao.ClusterId; _isClient = dao.IsClient; _supportedAttributes.AddRange(dao.SupportedAttributes); _supportedCommandsGenerated.AddRange(dao.SupportedCommandsGenerated); _supportedCommandsReceived.AddRange(dao.SupportedCommandsReceived); _attributes = dao.Attributes; }
private void ProcessVsFiles(string path) { _slnFiles.AddRange(Directory.GetFiles(path, "*.sln", SearchOption.TopDirectoryOnly)); _projFiles.AddRange(Directory.GetFiles(path, "*.*proj", SearchOption.TopDirectoryOnly)); Parallel.ForEach(GetValidDirectories(path), (directory) => { ProcessVsFiles(directory); }); }
protected override IEnumerable <Product> GetProducts(string productName) { var products = new ConcurrentBag <Product>(); var tasks = new List <Task>(); var mainPageTask = Task.Run(() => { var html = GetHtml($"https://www.jumbo.com/zoeken?SearchTerm={productName}"); var scrapedProducts = ScrapeProducts(html); products.AddRange(scrapedProducts); }); tasks.Add(mainPageTask); var pageCount = 3; // DeterminePageCount(productName); if (pageCount == 0) { return(products); } Parallel.For(1, pageCount, pageNumber => { var task = Task.Run(() => { var url = BuildUrl(productName, pageNumber); var html = GetHtml(url); var scrapedProducts = ScrapeProducts(html); products.AddRange(scrapedProducts); }); tasks.Add(task); }); Task.WaitAll(tasks.ToArray()); return(products); }
public void Given_CollectionSingleElementAndConcurrentBag_When_AddRange_Then_ElementsAdded() { var concurrentBagExample = new ConcurrentBag <string>(); var testList1 = new List <string> { "String1", "String2", "String3", "String4" }; concurrentBagExample.AddRange(testList1); concurrentBagExample.Add("TEST"); Assert.AreEqual(testList1.Count + 1, concurrentBagExample.Count); }
private async Task SetRatesForLeagues(IEnumerable <LeagueEntity> leagues) { var rates = new ConcurrentBag <BetRatesEntity>(); Parallel.ForEach(leagues, league => { var leagueRates = _rateCalculator.GetLeagueGamesRates(league); rates.AddRange(leagueRates); }); await SaveRatesAsync(rates.ToList()); }
private async Task <IEnumerable <Adapters.Oal.dbo_normal_console_event_new> > GetEventRowsAsync( Adapters.Oal.dbo_normal_console_event_new eventRow, IList <string> pendingConsoles, IList <string> consoleList, string consignmentNotes, int level = 0) { Console.Write("." + level); var list = new ConcurrentBag <Adapters.Oal.dbo_normal_console_event_new>(); if (level > 4) { return(list); } var items = consignmentNotes.Split(new[] { ',', '\t' }, StringSplitOptions.RemoveEmptyEntries).ToArray(); foreach (var c in items) { var itemConNote = c; if (consoleList.Contains(itemConNote)) { continue; } var row = await CloneChildAsync(eventRow, itemConNote); list.Add(row); if (!IsConsole(itemConNote)) { continue; } consoleList.Add(itemConNote); var childConsignments = await GetItemConsigmentsFromConsoleDetailsAsync(itemConNote); if (null == childConsignments) { pendingConsoles.Add(itemConNote); continue; } var children = await GetEventRowsAsync(eventRow, pendingConsoles, consoleList, childConsignments.ToEmptyString().Trim(), level + 1); list.AddRange(children); } return(list); }
public async Task SubscribingToMultipleEventsOnAContract() { var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet); //cancellation token to enable the listener to be stopped //passing in a time limit as a safety valve for the unit test var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(2)); //somewhere to put matching events //using ConcurrentBag because we'll be referencing the collection on different threads var erc20Transfers = new ConcurrentBag <EventLog <TransferEventDto> >(); var approvals = new ConcurrentBag <EventLog <ApprovalEventDTO> >(); var all = new ConcurrentBag <FilterLog>(); //capture a fatal exception here (we're not expecting one!) Exception fatalException = null; //this is the contract we want to listen to //the processor also accepts an array of addresses const string ContractAddress = "0x9f8F72aA9304c8B593d555F12eF6589cC3A579A2"; //initialise the processor //contract address or addresses is optional //we don't need an account because this is read only var processor = web3.Eth.LogsProcessor(ContractAddress) .SetMinimumBlockNumber(7540000) //optional: default is to start at current block on chain .SetBlocksPerBatch(100) //optional: number of blocks to scan at once, default is 100 .Add((events) => all.AddRange(events)) // any event for the contract/s - useful for logging .Add <TransferEventDto>((events) => erc20Transfers.AddRange(events)) // transfer events .Add <ApprovalEventDTO>((events) => approvals.AddRange(events)) // approval events // optional: a handler for a fatal error which would stop processing .OnFatalError((ex) => fatalException = ex) // for test purposes we'll cancel after a batch or block range has been processed // setting this is optional but is useful for monitoring progress .OnBatchProcessed((args) => cancellationTokenSource.Cancel()) .Build(); // begin processing var backgroundTask = processor.ProcessContinuallyInBackgroundAsync(cancellationTokenSource.Token); //simulate doing something else whilst the listener works its magic! while (!backgroundTask.IsCompleted) { await Task.Delay(1000); } Assert.True(backgroundTask.IsCanceled); Assert.Equal(11, erc20Transfers.Count); Assert.Equal(5, approvals.Count); Assert.Equal(16, all.Count); Assert.Null(fatalException); }
public void Given_MultipleParallelizedCollectionsAndConcurrentBag_When_AddRange_Then_ElementsAdded() { var concurrentBagExample = new ConcurrentBag <string>(); Parallel.For(0, 10, i => { concurrentBagExample.AddRange(new List <string> { "Test1", "Test2" }); }); Assert.AreEqual(10 * 2, concurrentBagExample.Count); }
public async Task <Response> CreateResponseAsync() { List <ContactDto> contactsDto = await groupsService.GetGroupContactsAsync( request.GroupId, clientConnection.UserId.GetValueOrDefault(), request.NavigationUserId.GetValueOrDefault()).ConfigureAwait(false); List <ContactVm> contactsVm = ContactConverter.GetContactsVm(contactsDto); IEnumerable <IGrouping <long, UserDto> > groupedUsers = contactsDto.Select(opt => opt.ContactUser).GroupBy(opt => opt.NodeId.GetValueOrDefault()); ConcurrentBag <UserVm> resultUsers = new ConcurrentBag <UserVm>(); List <Task> getUsersTasks = new List <Task>(); foreach (var group in groupedUsers) { getUsersTasks.Add(Task.Run(async() => { var nodeConnection = connectionsService.GetNodeConnection(group.Key); if (group.Key != NodeSettings.Configs.Node.Id && nodeConnection != null) { await MetricsHelper.Instance.SetCrossNodeApiInvolvedAsync(request.RequestId).ConfigureAwait(false); var users = await nodeRequestSender.GetUsersInfoAsync( group.Select(opt => opt.Id).ToList(), clientConnection.UserId, nodeConnection).ConfigureAwait(false); resultUsers.AddRange(users); resultUsers.AddRange(users); } else { resultUsers.AddRange(UserConverter.GetUsersVm(group.ToList(), clientConnection.UserId)); } })); } await Task.WhenAll(getUsersTasks).ConfigureAwait(false); foreach (var contact in contactsVm) { contact.ContactUser = resultUsers.FirstOrDefault(opt => opt.Id == contact.ContactUserId); } return(new ContactsResponse(request.RequestId, contactsVm)); }
static ParserRegistry() { var closedMethods = new ConcurrentBag <MethodInfo>(); var traverser = _traverser(t => { var parsers = from m in t.GetDeclaredStaticMethods() where m.HasAttribute <ParserAttribute>() select m; closedMethods.AddRange(parsers.Where(p => not(p.IsGenericMethod))); }); traverser.Traverse(MetaFloor.Assembly); Projector = ConversionSuite.FromMethods(closedMethods.ToArray()); }
/// <summary> /// Splits all logfiles exceeding the configured size threshold into smaller chunks. /// </summary> /// <param name="files">The collection of files eligible for partitioning.</param> /// <returns>Collection of all files in logset following the partitioning process.</returns> public IEnumerable <LogFileContext> PartitionLargeFiles(IEnumerable <LogFileContext> files) { long maxBytes = tuningOptions.FilePartitionerThresholdMb * 1024 * 1024; // Build list of files to chunk by searching for files exceeding the max size and consulting the parser factory to see if it's a single-line log type. var processedFiles = new ConcurrentBag <LogFileContext>(); var filesToPartition = new List <LogFileContext>(); foreach (var file in files) { if (IsPartitionableFile(file, maxBytes)) { filesToPartition.Add(file); } else { // Nothing to do; use it as-is. processedFiles.Add(file); } } if (!filesToPartition.Any()) { Log.InfoFormat("No log files were found that are larger than {0}MB; skipping partitioning phase.", tuningOptions.FilePartitionerThresholdMb); return(processedFiles); } Log.InfoFormat("Partitioning {0} log {1} larger than {2}MB to speed up processing. This may take some time..", filesToPartition.Count, "file".Pluralize(filesToPartition.Count), tuningOptions.FilePartitionerThresholdMb); // Set up task scheduler. TaskFactory factory = GetFilePartitioningTaskFactory(); // Spin up partitioning tasks in parallel. Task[] taskArray = new Task[filesToPartition.Count]; for (var i = 0; i < filesToPartition.Count; i++) { var fileToChunk = filesToPartition[i]; taskArray[i] = factory.StartNew(() => { var partitions = PartitionFile(fileToChunk, maxBytes); processedFiles.AddRange(partitions); }); } // Wait on any in-flight tasks. Task.WaitAll(taskArray); return(processedFiles); }
public async Task UsingAzureTableStorageProgressRepository() { var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet); // Requires: Nethereum.BlockchainStore.AzureTables // Load config // - this will contain the secrets and connection strings we don't want to hard code var config = TestConfiguration.LoadConfig(); string azureStorageConnectionString = config["AzureStorageConnectionString"]; //cancellation token to enable the listener to be stopped //passing in a time limit as a safety valve for the unit test var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1)); //somewhere to put matching events //using ConcurrentBag because we'll be referencing the collection on different threads var erc20Transfers = new ConcurrentBag <EventLog <TransferEventDto> >(); //initialise the processor var builder = web3.Eth.LogsProcessor <TransferEventDto>() .OnEvents((events) => erc20Transfers.AddRange(events)) // transfer events .SetBlocksPerBatch(1) //optional: restrict batches to one block at a time .SetMinimumBlockNumber(7540102) //optional: default is to start at current block on chain // for test purposes we'll stop after processing a batch .OnBatchProcessed((args) => cancellationTokenSource.Cancel()) // tell the processor to reference an Azure Storage table for block progress // this is an extension method from Nethereum.BlockchainStore.AzureTables .UseAzureTableStorageForBlockProgress(azureStorageConnectionString, "EventLogProcessingSample"); var processor = builder.Build(); //we should have a BlockProgressRepository Assert.NotNull(builder.BlockProgressRepository); //there should be no prior progress Assert.Null(await builder.BlockProgressRepository.GetLastBlockNumberProcessedAsync()); //run the processor for a while var rangesProcessed = await processor.ProcessContinuallyAsync(cancellationTokenSource.Token); //the last block processed should have been saved Assert.NotNull(await builder.BlockProgressRepository.GetLastBlockNumberProcessedAsync()); //we should have captured some events Assert.True(erc20Transfers.Any()); //clean up await new CloudTableSetup(azureStorageConnectionString, "EventLogProcessingSample") .GetCountersTable() .DeleteIfExistsAsync(); }
public object Any(Search request) { var resultCollection = new ConcurrentBag <Comment>(); Parallel.Invoke( () => resultCollection.AddRange(FacebookService.Client.Get(new Search { Keyword = request.Keyword })), () => resultCollection.AddRange(FlickrService.Client.Get(new Search { Keyword = request.Keyword })), () => resultCollection.AddRange(InstagramService.Client.Get(new Search { Keyword = request.Keyword })), () => resultCollection.AddRange(TwitterService.Client.Get(new Search { Keyword = request.Keyword })), () => resultCollection.AddRange(YouTubeService.Client.Get(new Search { Keyword = request.Keyword })) ); return(resultCollection.ToList());// response; }
public virtual async Task <IEnumerable <ITorrentSearchResult> > SearchMovie(string name, int?year, string imdbId, VideoQuality videoQuality = VideoQuality.Any, string extraKeywords = null, string excludeKeywords = null, int?minSize = null, int?maxSize = null, int?minSeed = null) { var results = new ConcurrentBag <ITorrentSearchResult>(); var tasks = MovieProviders.RunTasks(p => p.SearchMovie(name, year, imdbId, videoQuality, extraKeywords, excludeKeywords, minSize, maxSize, minSeed, this) .ContinueWith(t => results.AddRange(t.Result)), ExceptionHandler ); await Task.WhenAll(tasks); return(results.OrderByDescending(r => r.Seed)); }
public async Task UsingJsonFileProgressRepository() { var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet); //cancellation token to enable the listener to be stopped //passing in a time limit as a safety valve for the unit test var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1)); //somewhere to put matching events //using ConcurrentBag because we'll be referencing the collection on different threads var erc20Transfers = new ConcurrentBag <EventLog <TransferEventDto> >(); // define a progress repository // this sample uses an out of the box simple json file implementation // if you want your own - the IBlockProgressRepository interface is easy to implement // the processor will use this repo to define which block to start at and update it after each batch is complete // it can prevent duplicate processing that could occur after a restart var jsonFilePath = Path.Combine(Path.GetTempPath(), "EventProcessingBlockProgress.json"); //initialise the builder var builder = web3.Eth.LogsProcessor <TransferEventDto>() .OnEvents((events) => erc20Transfers.AddRange(events)) // transfer events .SetBlocksPerBatch(1) //optional: restrict batches to one block at a time .SetMinimumBlockNumber(7540102) //optional: default is to start at current block on chain // for test purposes we'll stop after processing a batch .OnBatchProcessed((args) => cancellationTokenSource.Cancel()) // tell the processor to use a Json File based Block Progress Repository // for test purposes only we delete any existing file to ensure we start afresh with no previous state .UseJsonFileForBlockProgress(jsonFilePath, deleteExistingFile: true); var processor = builder.Build(); //we should have a BlockProgressRepository Assert.NotNull(builder.BlockProgressRepository); //there should be no prior progress Assert.Null(await builder.BlockProgressRepository.GetLastBlockNumberProcessedAsync()); //run the processor for a while var rangesProcessed = await processor.ProcessContinuallyAsync(cancellationTokenSource.Token); //the last block processed should have been saved Assert.NotNull(await builder.BlockProgressRepository.GetLastBlockNumberProcessedAsync()); //we should have captured some events Assert.True(erc20Transfers.Any()); //clean up File.Delete(jsonFilePath); }
public virtual async Task<IEnumerable<ITorrentSearchResult>> SearchMovie(string name, int? year, string imdbId, VideoQuality videoQuality = VideoQuality.Any, string extraKeywords = null, string excludeKeywords = null, int? minSize = null, int? maxSize = null, int? minSeed = null) { var results = new ConcurrentBag<ITorrentSearchResult>(); var tasks = MovieProviders.RunTasks(p => p.SearchMovie(name, year, imdbId, videoQuality, extraKeywords, excludeKeywords, minSize, maxSize, minSeed, this) .ContinueWith(t => results.AddRange(t.Result)), ExceptionHandler ); await Task.WhenAll(tasks); return results.OrderByDescending(r => r.Seed); }
public virtual async Task<IEnumerable<ITorrentSearchResult>> Search(string query, VideoQuality videoQuality = VideoQuality.Any, string excludeKeywords = null, int? minSize = null, int? maxSize = null, int? minSeed = null) { var providers = ((IEnumerable<ITorrentProvider>)MovieProviders).Union(TvShowProviders); var results = new ConcurrentBag<ITorrentSearchResult>(); var tasks = providers.RunTasks(p => p.Search(query, videoQuality, excludeKeywords, minSize, maxSize, minSeed, this) .ContinueWith(t => results.AddRange(t.Result)), ExceptionHandler ); await Task.WhenAll(tasks); return results.OrderByDescending(r => r.Seed); }
public virtual async Task<IEnumerable<ITorrentSearchResult>> SearchTvShowEpisode(string name, int season, int episode, string episodeName, string imdbId, VideoQuality videoQuality = VideoQuality.Any, string extraKeywords = null, string excludeKeywords = null, int? minSize = null, int? maxSize = null, int? minSeed = null) { var results = new ConcurrentBag<ITorrentSearchResult>(); var tasks = TvShowProviders.RunTasks(p => p.SearchTvShowEpisode(name, season, episode, episodeName, imdbId, videoQuality, extraKeywords, excludeKeywords, minSize, maxSize, minSeed, this) .ContinueWith(t => results.AddRange(t.Result)), ExceptionHandler ); await Task.WhenAll(tasks); return results .Where(r => { int? s, e; Helper.DetectEpisodeInfo(r.Name, name, out s, out e); return (s == null || s == season) && e == episode; }) .OrderByDescending(r => r.Seed) .ToList(); }
private static void AddTypesToTypeList(IEnumerable<Type> types, ConcurrentBag<Type> typeList) { if (types.IsInstance()) typeList.AddRange(types); }