/// <summary> /// Initializes a new instance of the <see cref="AsynchronousTraceListenerWrapper" /> class. /// </summary> /// <param name="wrappedTraceListener">The wrapped trace listener.</param> /// <param name="ownsWrappedTraceListener">Indicates whether the wrapper should dispose the wrapped trace listener.</param> /// <param name="bufferSize">Size of the buffer for asynchronous requests.</param> /// <param name="maxDegreeOfParallelism">The max degree of parallelism for thread safe listeners. Specify <see langword="null"/> to use the current core count.</param> /// <param name="disposeTimeout">The timeout for waiting to complete buffered requests when disposing. When <see langword="null" /> the default of <see cref="Timeout.InfiniteTimeSpan" /> is used.</param> public AsynchronousTraceListenerWrapper( TraceListener wrappedTraceListener, bool ownsWrappedTraceListener = true, int? bufferSize = DefaultBufferSize, int? maxDegreeOfParallelism = null, TimeSpan? disposeTimeout = null) { Guard.ArgumentNotNull(wrappedTraceListener, "wrappedTraceListener"); CheckBufferSize(bufferSize); CheckMaxDegreeOfParallelism(maxDegreeOfParallelism); CheckDisposeTimeout(disposeTimeout); this.wrappedTraceListener = wrappedTraceListener; this.ownsWrappedTraceListener = ownsWrappedTraceListener; this.disposeTimeout = disposeTimeout ?? Timeout.InfiniteTimeSpan; this.closeSource = new CancellationTokenSource(); this.requests = bufferSize != null ? new BlockingCollection<Action<TraceListener>>(bufferSize.Value) : new BlockingCollection<Action<TraceListener>>(); if (this.wrappedTraceListener.IsThreadSafe) { this.maxDegreeOfParallelism = maxDegreeOfParallelism.HasValue ? maxDegreeOfParallelism.Value : Environment.ProcessorCount; this.asyncProcessingTask = Task.Factory.StartNew(this.ProcessRequestsInParallel, CancellationToken.None, TaskCreationOptions.HideScheduler | TaskCreationOptions.LongRunning, TaskScheduler.Default); } else { this.asyncProcessingTask = Task.Factory.StartNew(this.ProcessRequests, CancellationToken.None, TaskCreationOptions.HideScheduler | TaskCreationOptions.LongRunning, TaskScheduler.Default); } }
/// <summary> /// Creates a buffer pool. /// </summary> /// <param name="bufferSize">The size, in bytes, of each buffer.</param> /// <param name="maxBuffers">The maximum number of buffers to keep around, unused; by default, the number of unused buffers is unbounded.</param> private BufferPool(int bufferSize, int maxBuffers, int preallocationSize, string name) { Name = name; byteBufferSize = bufferSize; buffers = maxBuffers <= 0 ? new BlockingCollection<byte[]>() : new BlockingCollection<byte[]>(maxBuffers); var globalPoolSizeStat = IntValueStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_BUFFERS_INPOOL, () => Count); allocatedBufferCounter = CounterStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_ALLOCATED_BUFFERS); checkedOutBufferCounter = CounterStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_CHECKED_OUT_BUFFERS); checkedInBufferCounter = CounterStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_CHECKED_IN_BUFFERS); droppedBufferCounter = CounterStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_DROPPED_BUFFERS); droppedTooLargeBufferCounter = CounterStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_DROPPED_TOO_LARGE_BUFFERS); // Those 2 counters should be equal. If not, it means we don't release all buffers. IntValueStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_INUSE_CHECKED_OUT_NOT_CHECKED_IN_BUFFERS, () => checkedOutBufferCounter.GetCurrentValue() - checkedInBufferCounter.GetCurrentValue() - droppedBufferCounter.GetCurrentValue()); IntValueStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_INUSE_ALLOCATED_NOT_INPOOL_BUFFERS, () => allocatedBufferCounter.GetCurrentValue() - globalPoolSizeStat.GetCurrentValue() - droppedBufferCounter.GetCurrentValue()); if (preallocationSize <= 0) return; var dummy = GetMultiBuffer(preallocationSize * Size); Release(dummy); }
protected DuplicateRemovedScheduler(IBloomFilter<string> bloomFilter) { //_bloomFilter = new MemoryBloomFilter<string>(1000 * 10, 1000 * 10 * 20); _bloomFilter = bloomFilter; Queue = new BlockingCollection<RequestMessage>(new ConcurrentQueue<RequestMessage>()); _accumulatedMessageTotal = 0; }
public void TestStreamingTransportServer() { BlockingCollection<string> queue = new BlockingCollection<string>(); List<string> events = new List<string>(); IStreamingCodec<string> stringCodec = _injector.GetInstance<StringStreamingCodec>(); IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, 0); var remoteHandler = Observer.Create<TransportEvent<string>>(tEvent => queue.Add(tEvent.Data)); using (var server = new StreamingTransportServer<string>(endpoint.Address, remoteHandler, _tcpPortProvider, stringCodec)) { server.Run(); IPEndPoint remoteEndpoint = new IPEndPoint(IPAddress.Parse("127.0.0.1"), server.LocalEndpoint.Port); using (var client = new StreamingTransportClient<string>(remoteEndpoint, stringCodec)) { client.Send("Hello"); client.Send(", "); client.Send("World!"); events.Add(queue.Take()); events.Add(queue.Take()); events.Add(queue.Take()); } } Assert.Equal(3, events.Count); Assert.Equal(events[0], "Hello"); Assert.Equal(events[1], ", "); Assert.Equal(events[2], "World!"); }
/// <summary> /// Do the specified input and output. /// </summary> /// <param name="input">Input.</param> /// <param name="output">Output.</param> public void Do(BlockingCollection<ISkeleton> input, Action<IEnumerable<Result>> fireNewCommand) { var data = new Dictionary<JointType, InputVector>(); foreach (var skeleton in input.GetConsumingEnumerable()) { foreach (var joint in skeleton.Joints) { if (!data.ContainsKey(joint.JointType)) { data.Add(joint.JointType, new InputVector()); } data[joint.JointType].Stream.Add(joint.Point); } if (data.First().Value.Stream.Count < 5) { continue; } var results = Recognizer.Recognize(data); try { fireNewCommand(results); } catch (Exception) { if (data.First().Value.Stream.Count > 40) { data.Clear(); } continue; } data.Clear(); } }
public VideoClass(string url, string languageCode) { Url = url; Subtitles = new BlockingCollection<SubtitlesClass>(); Language = TEDdownloader.Language.GetLanguage(languageCode); LanguageCode = languageCode; }
public static void StartCollection() { if (_started) return; _started = true; conf = Config.CarbonatorSection.Current; if (conf == null) { if (conf.LogLevel >= 3) EventLog.WriteEntry(Program.EVENT_SOURCE, "Carbonator configuration is missing. This service cannot start", EventLogEntryType.Error); throw new InvalidOperationException("Carbonator configuration is missing. This service cannot start"); } Thread.CurrentThread.CurrentCulture = CultureInfo.GetCultureInfo(conf.DefaultCulture); Thread.CurrentThread.CurrentUICulture = CultureInfo.GetCultureInfo(conf.DefaultCulture); if (Config.CarbonatorSection.Current.MaxMetricBufferSize > 0) { _metricsList = new BlockingCollection<CollectedMetric>(Config.CarbonatorSection.Current.MaxMetricBufferSize); } else { _metricsList = new BlockingCollection<CollectedMetric>(); } // start collection and reporting timers _metricCollectorTimer = new Timer(collectMetrics, new StateControl(), conf.CollectionInterval, conf.CollectionInterval); _metricReporterTimer = new Timer(reportMetrics, new StateControl(), conf.ReportingInterval, conf.ReportingInterval); if (conf.LogLevel >= 1) EventLog.WriteEntry(Program.EVENT_SOURCE, "Carbonator service has been initialized and began reporting metrics", EventLogEntryType.Information); }
static void Main(string[] args) { Console.WriteLine("=== In Line Execution ==="); Task<string>[] tasks = GetTasks(); string[] all = Task.WhenAll(tasks).Result; Console.WriteLine(string.Join("", all)); Console.WriteLine("=== Standard Live Execution ==="); tasks = GetTasks(); BlockingCollection<string> results = new BlockingCollection<string>(); Task.WaitAll(tasks.Select(async t => { results.Add(await t); Console.WriteLine("[{0:hh:mm:ss.fff}] Current result: [{1}]", DateTime.Now, string.Join("", results)); }).ToArray()); Console.WriteLine("=== Pragmateek Live Execution ==="); results = new BlockingCollection<string>(); tasks = GetTasks(); Task.WaitAll(tasks.Select(t => t.ContinueWith(p => { results.Add(p.Result); Console.WriteLine("[{0:hh:mm:ss.fff}] Current result: [{1}]", DateTime.Now, string.Join("", results)); })).ToArray()); }
public Consumer(string groupName, ConsumerSetting setting) { if (groupName == null) { throw new ArgumentNullException("groupName"); } GroupName = groupName; Setting = setting ?? new ConsumerSetting(); _lockObject = new object(); _subscriptionTopics = new Dictionary<string, HashSet<string>>(); _topicQueuesDict = new ConcurrentDictionary<string, IList<MessageQueue>>(); _pullRequestDict = new ConcurrentDictionary<string, PullRequest>(); _remotingClient = new SocketRemotingClient(Setting.BrokerAddress, Setting.SocketSetting, Setting.LocalAddress); _adminRemotingClient = new SocketRemotingClient(Setting.BrokerAdminAddress, Setting.SocketSetting, Setting.LocalAdminAddress); _binarySerializer = ObjectContainer.Resolve<IBinarySerializer>(); _scheduleService = ObjectContainer.Resolve<IScheduleService>(); _allocateMessageQueueStragegy = ObjectContainer.Resolve<IAllocateMessageQueueStrategy>(); _logger = ObjectContainer.Resolve<ILoggerFactory>().Create(GetType().FullName); _remotingClient.RegisterConnectionEventListener(new ConnectionEventListener(this)); if (Setting.MessageHandleMode == MessageHandleMode.Sequential) { _consumingMessageQueue = new BlockingCollection<ConsumingMessage>(); _consumeMessageWorker = new Worker("ConsumeMessage", () => HandleMessage(_consumingMessageQueue.Take())); } _messageRetryQueue = new BlockingCollection<ConsumingMessage>(); }
public void TestWritableTransportServer() { BlockingCollection<WritableString> queue = new BlockingCollection<WritableString>(); List<string> events = new List<string>(); IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, 0); var remoteHandler = Observer.Create<TransportEvent<WritableString>>(tEvent => queue.Add(tEvent.Data)); using (var server = new WritableTransportServer<WritableString>(endpoint, remoteHandler, _tcpPortProvider, _injector)) { server.Run(); IPEndPoint remoteEndpoint = new IPEndPoint(IPAddress.Parse("127.0.0.1"), server.LocalEndpoint.Port); using (var client = new WritableTransportClient<WritableString>(remoteEndpoint, _injector)) { client.Send(new WritableString("Hello")); client.Send(new WritableString(", ")); client.Send(new WritableString("World!")); events.Add(queue.Take().Data); events.Add(queue.Take().Data); events.Add(queue.Take().Data); } } Assert.AreEqual(3, events.Count); Assert.AreEqual(events[0], "Hello"); Assert.AreEqual(events[1], ", "); Assert.AreEqual(events[2], "World!"); }
/// <summary>Initializes a new instance of the StaTaskScheduler class with the specified concurrency level.</summary> /// <param name="numberOfThreads">The number of threads that should be created and used by this scheduler.</param> public StaTaskScheduler(int numberOfThreads) { // Validate arguments if (numberOfThreads < 1) throw new ArgumentOutOfRangeException("concurrencyLevel"); // Initialize the tasks collection _tasks = new BlockingCollection<Task>(); // Create the threads to be used by this scheduler _threads = Enumerable.Range(0, numberOfThreads).Select(i => { var thread = new Thread(() => { // Continually get the next task and try to execute it. // This will continue until the scheduler is disposed and no more tasks remain. foreach (var t in _tasks.GetConsumingEnumerable()) { TryExecuteTask(t); } }); thread.IsBackground = true; thread.SetApartmentState(ApartmentState.STA); return thread; }).ToList(); // Start all of the threads _threads.ForEach(t => t.Start()); }
public BlockingCollectionQueue() { _writerQueue = new BlockingCollection<User>(); _readerQueue = new BlockingCollection<User>(); _currentQueue = _writerQueue; Task.Factory.StartNew(() => ConsumerFunc(), TaskCreationOptions.None); }
public void CanGetNotificationAboutDocumentIndexUpdate() { using (var server = GetNewServer()) using (var store = NewRemoteDocumentStore(ravenDbServer: server)) { var list = new BlockingCollection<IndexChangeNotification>(); var taskObservable = store.Changes(); taskObservable.Task.Wait(); var observableWithTask = taskObservable.ForIndex("Raven/DocumentsByEntityName"); observableWithTask.Task.Wait(); observableWithTask .Subscribe(list.Add); using (var session = store.OpenSession()) { session.Store(new Item(), "items/1"); session.SaveChanges(); } IndexChangeNotification indexChangeNotification; Assert.True(list.TryTake(out indexChangeNotification, TimeSpan.FromSeconds(5))); Assert.Equal("Raven/DocumentsByEntityName", indexChangeNotification.Name); Assert.Equal(indexChangeNotification.Type, IndexChangeTypes.MapCompleted); } }
public HardDiskCache(string cachePath) { CachePath = cachePath; indexList = new List<LocationInfo>(); imageQueue = new BlockingCollection<Image>(); StartConsumerThread(); }
public void CanGetNotificationsAboutConflictedAttachements() { using(var store1 = CreateStore()) using (var store2 = CreateStore()) { store1.DatabaseCommands.PutAttachment("attachment/1", null, new MemoryStream(new byte[] {1, 2, 3}), new RavenJObject()); store2.DatabaseCommands.PutAttachment("attachment/1", null, new MemoryStream(new byte[] {1, 2, 3}), new RavenJObject()); var list = new BlockingCollection<ReplicationConflictNotification>(); var taskObservable = store2.Changes(); taskObservable.Task.Wait(); var observableWithTask = taskObservable.ForAllReplicationConflicts(); observableWithTask.Task.Wait(); observableWithTask .Subscribe(list.Add); TellFirstInstanceToReplicateToSecondInstance(); ReplicationConflictNotification replicationConflictNotification; Assert.True(list.TryTake(out replicationConflictNotification, TimeSpan.FromSeconds(10))); Assert.Equal("attachment/1", replicationConflictNotification.Id); Assert.Equal(replicationConflictNotification.ItemType, ReplicationConflictTypes.AttachmentReplicationConflict); Assert.Equal(2, replicationConflictNotification.Conflicts.Length); Assert.Equal(ReplicationOperationTypes.Put, replicationConflictNotification.OperationType); } }
public void CanGetNotificationAboutDocumentDelete() { using (GetNewServer()) using (var store = new DocumentStore { Url = "http://localhost:8079" }.Initialize()) { var list = new BlockingCollection<DocumentChangeNotification>(); var taskObservable = store.Changes(); taskObservable.Task.Wait(); var observableWithTask = taskObservable.ForDocument("items/1"); observableWithTask.Task.Wait(); observableWithTask .Where(x => x.Type == DocumentChangeTypes.Delete) .Subscribe(list.Add); using (var session = store.OpenSession()) { session.Store(new Item(), "items/1"); session.SaveChanges(); } store.DatabaseCommands.Delete("items/1", null); DocumentChangeNotification DocumentChangeNotification; Assert.True(list.TryTake(out DocumentChangeNotification, TimeSpan.FromSeconds(2))); Assert.Equal("items/1", DocumentChangeNotification.Id); Assert.Equal(DocumentChangeNotification.Type, DocumentChangeTypes.Delete); ((RemoteDatabaseChanges) taskObservable).DisposeAsync().Wait(); } }
private SubstManager() { FindAvailableDrives(); _freeDriveQ = new BlockingCollection<char>(_driveSet.Length); foreach (char drive in _driveSet) _freeDriveQ.Add(drive); }
public void Write(BlockingCollection<AddressInfo> queue, String filePath, CancellationToken cancelToken) { Count = 0; using (var fs = new FileStream(filePath, FileMode.Create)) using (var fileWriter = new StreamWriter(fs, Encoding.GetEncoding(1251))) { while (true) { AddressInfo addressInfo = null; try { addressInfo = queue.Take(cancelToken); } catch (InvalidOperationException) { } if (cancelToken.IsCancellationRequested || queue.IsCompleted) return; if (addressInfo != null) { fileWriter.WriteLine(String.Format("{0}\t{1}\t{2}", addressInfo.Address, addressInfo.FlatCount, addressInfo.Uik)); Count++; } } } }
public DynamicSaveStrategy() { _decayBag = new ConcurrentBag<Item>(); _itemThreadWriters = new BlockingCollection<QueuedMemoryWriter>(); _mobileThreadWriters = new BlockingCollection<QueuedMemoryWriter>(); _guildThreadWriters = new BlockingCollection<QueuedMemoryWriter>(); }
static void Main(string[] args) { // create a blocking collection BlockingCollection<int> blockingCollection = new BlockingCollection<int>(); // create and start a producer Task.Factory.StartNew(() => { // put the producer to sleep System.Threading.Thread.Sleep(500); for (int i = 0; i < 100; i++) { // add the item to the collection blockingCollection.Add(i); } // mark the collection as finished blockingCollection.CompleteAdding(); }); // create and start a consumer Task consumer = Task.Factory.StartNew(() => { // use a foreach loop to consume the blocking collection foreach (int i in blockingCollection) { Console.WriteLine("Item {0}", i); } Console.WriteLine("Collection is fully consumed"); }); // wait for the consumer to finish consumer.Wait(); // wait for input before exiting Console.WriteLine("Press enter to finish"); Console.ReadLine(); }
public CAudioPlayer() { _effects = new BlockingCollection<CSound>(); System.Threading.ThreadStart threadStarter = _checkForThingsToPlay; _audioThread = new Thread(threadStarter); _audioThread.Start(); }
/// <summary>Initializes a new instance of the MTATaskScheduler class with the specified concurrency level.</summary> /// <param name="numberOfThreads">The number of threads that should be created and used by this scheduler.</param> /// <param name="nameFormat">The template name form to use to name threads.</param> public MTATaskScheduler(int numberOfThreads, string nameFormat) { // Validate arguments if (numberOfThreads < 1) throw new ArgumentOutOfRangeException("numberOfThreads"); // Initialize the tasks collection tasks = new BlockingCollection<Task>(); // Create the threads to be used by this scheduler _threads = Enumerable.Range(0, numberOfThreads).Select(i => { var thread = new Thread(() => { // Continually get the next task and try to execute it. // This will continue until the scheduler is disposed and no more tasks remain. foreach (var t in tasks.GetConsumingEnumerable()) { TryExecuteTask(t); } }) { IsBackground = true }; thread.SetApartmentState(ApartmentState.MTA); thread.Name = String.Format("{0} - {1}", nameFormat, thread.ManagedThreadId); return thread; }).ToList(); // Start all of the threads _threads.ForEach(t => t.Start()); }
public void TestStreamingOneWayCommunication() { IPAddress listeningAddress = IPAddress.Parse("127.0.0.1"); BlockingCollection<string> queue = new BlockingCollection<string>(); List<string> events = new List<string>(); IStreamingCodec<string> codec = TangFactory.GetTang().NewInjector().GetInstance<StringStreamingCodec>(); using (var remoteManager1 = _remoteManagerFactory1.GetInstance<string>(listeningAddress, codec)) using (var remoteManager2 = _remoteManagerFactory1.GetInstance<string>(listeningAddress, codec)) { var observer = Observer.Create<string>(queue.Add); IPEndPoint endpoint1 = new IPEndPoint(listeningAddress, 0); remoteManager2.RegisterObserver(endpoint1, observer); var remoteObserver = remoteManager1.GetRemoteObserver(remoteManager2.LocalEndpoint); remoteObserver.OnNext("abc"); remoteObserver.OnNext("def"); remoteObserver.OnNext("ghi"); events.Add(queue.Take()); events.Add(queue.Take()); events.Add(queue.Take()); } Assert.AreEqual(3, events.Count); }
public void CanGetNotificationAboutDocumentPut() { using(GetNewServer()) {using (var store = new DocumentStore { Url = "http://localhost:8079", Conventions = { FailoverBehavior = FailoverBehavior.FailImmediately } }.Initialize()) { var list = new BlockingCollection<DocumentChangeNotification>(); var taskObservable = store.Changes(); taskObservable.Task.Wait(); var observableWithTask = taskObservable.ForDocument("items/1"); observableWithTask.Task.Wait(); observableWithTask.Subscribe(list.Add); using (var session = store.OpenSession()) { session.Store(new Item(), "items/1"); session.SaveChanges(); } DocumentChangeNotification documentChangeNotification; Assert.True(list.TryTake(out documentChangeNotification, TimeSpan.FromSeconds(3))); Assert.Equal("items/1", documentChangeNotification.Id); Assert.Equal(documentChangeNotification.Type, DocumentChangeTypes.Put); Assert.NotNull(documentChangeNotification.Etag); } Thread.Sleep(1000); } }
public CommandProcessor(BlockingCollection<Job> allJobs, Scheduler scheduler, JobQueue jobQueue, QueueProcessor queueProcessor) { _allJobs = allJobs; _scheduler = scheduler; _jobQueue = jobQueue; _queueProcessor = queueProcessor; }
public void TestOneWayCommunicationClientOnly() { IPAddress listeningAddress = IPAddress.Parse("127.0.0.1"); BlockingCollection<string> queue = new BlockingCollection<string>(); List<string> events = new List<string>(); using (var remoteManager1 = _remoteManagerFactory.GetInstance(new StringCodec())) using (var remoteManager2 = _remoteManagerFactory.GetInstance(listeningAddress, new StringCodec())) { IPEndPoint remoteEndpoint = new IPEndPoint(listeningAddress, 0); var observer = Observer.Create<string>(queue.Add); remoteManager2.RegisterObserver(remoteEndpoint, observer); var remoteObserver = remoteManager1.GetRemoteObserver(remoteManager2.LocalEndpoint); remoteObserver.OnNext("abc"); remoteObserver.OnNext("def"); remoteObserver.OnNext("ghi"); events.Add(queue.Take()); events.Add(queue.Take()); events.Add(queue.Take()); } Assert.Equal(3, events.Count); }
public OutBuffer(int maxSizeAsPowerOfTwo) { //_completeds = new BlockingCollection<TaskCompleted>((int)Math.Pow(2, maxSizeAsPowerOfTwo)); //This is for output, and not spcifying the size revert to ConcurrentQueue. //Bounding slows down considerably. Should remove when ring array in place. _completeds = new BlockingCollection<TaskCompleted>(); }
/// <summary> /// Constructor /// </summary> public Reducer() { _distinctWordList = new ConcurrentDictionary<string, int>(); var concurrentBag = new ConcurrentBag<string>(); _wordChunks = new BlockingCollection<string>(concurrentBag); Numwords = 0; }
/// <summary>Parameterized constructor. /// </summary> /// <param name="loggerFactory"></param> public DefaultActionExecutionService(ILoggerFactory loggerFactory) { _logger = loggerFactory.Create(GetType().Name); _actionQueue = new BlockingCollection<ActionInfo>(new ConcurrentQueue<ActionInfo>()); _worker = new Worker(TryTakeAndExecuteAction, DefaultPeriod); _worker.Start(); }
public static BlockingCollection<LogMsg> GetLogQueue() { if (m_logQueue == null) { m_logQueue = new BlockingCollection<LogMsg> (); } return m_logQueue; }
public MapTask(TaskFactory taskFactory, BlockingCollection <TIn> inCollection, BlockingCollection <TOut> outCollection, IMapper <TIn, TOut> mapper) : base(taskFactory) { _inCollection = inCollection; _outCollection = outCollection; _mapper = mapper; }
public BlockingAggregator(BlockingCollection <InputType> input, int chunkSize, Func <List <InputType>, OutputType> factory) { this.inputQueue = input; this.chunkSize = chunkSize; this.factory = factory; }
protected AbstractDownloader(IShellService shellService, CancellationToken ct, PauseToken pt, IProgress <DownloadProgress> progress, BlockingCollection <TumblrPost> producerConsumerCollection, FileDownloader fileDownloader, ICrawlerService crawlerService = null, IBlog blog = null, IFiles files = null) { this.shellService = shellService; this.crawlerService = crawlerService; this.blog = blog; this.files = files; this.ct = ct; this.pt = pt; this.progress = progress; this.producerConsumerCollection = producerConsumerCollection; this.fileDownloader = fileDownloader; }
public LifeCycleEventPublisher(ILifeCycleEventHub eventHub, ILoggerFactory loggerFactory) { _eventHub = eventHub; _outbox = new BlockingCollection <LifeCycleEvent>(); _logger = loggerFactory.CreateLogger(GetType()); }
public static void SaveMatrix(IZone[] zones, float[][] data, string fileName) { StringBuilder[] zoneLines = new StringBuilder[zones.Length]; var dir = Path.GetDirectoryName(fileName); if (!String.IsNullOrWhiteSpace(dir)) { if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); } } using (StreamWriter writer = new StreamWriter(fileName)) { BlockingCollection <SaveTask> toWrite = new BlockingCollection <SaveTask>(); var saveTask = Task.Run(() => { int nextRow = 0; SortedList <int, SaveTask> backlog = new SortedList <int, SaveTask>(); foreach (var newTask in toWrite.GetConsumingEnumerable()) { var task = newTask; do { string currentString = task.Text; int currentRow = task.RowNumber; if (nextRow == currentRow) { writer.WriteLine(currentString); nextRow++; } else { backlog.Add(currentRow, new SaveTask() { RowNumber = currentRow, Text = currentString }); break; } if (backlog.Count == 0) { break; } if (backlog.TryGetValue(nextRow, out task)) { backlog.Remove(nextRow); continue; } break; } while (true); } }); var stringBuilder = new StringBuilder(); stringBuilder.Append("Zones O\\D"); for (int i = 0; i < zones.Length; i++) { stringBuilder.Append(','); stringBuilder.Append(zones[i].ZoneNumber); } toWrite.Add(new SaveTask() { RowNumber = 0, Text = stringBuilder.ToString() }); Parallel.For(0, zones.Length, () => new StringBuilder(), (int i, ParallelLoopState _, StringBuilder strBuilder) => { strBuilder.Clear(); strBuilder.Append(zones[i].ZoneNumber); var row = data[i]; if (row == null) { for (int j = 0; j < zones.Length; j++) { strBuilder.Append(",0"); } } else { for (int j = 0; j < row.Length; j++) { strBuilder.Append(','); strBuilder.Append(row[j]); } } toWrite.Add(new SaveTask() { RowNumber = i + 1, Text = strBuilder.ToString() }); return(strBuilder); }, (StringBuilder _) => { }); toWrite.CompleteAdding(); saveTask.Wait(); } }
public BackgroundThreadProducer(BlockingCollection <T> source) : this(source, false) { }
public XboxTransferWorker(IXboxGameRepositoryFactory xboxGameRepositoryFactory, string gameName, BlockingCollection <IXboxTransferRequest> requests, BlockingCollection <IXboxTransferRequest> finishedRequests, IProgressNotifier notifier) : base(xboxGameRepositoryFactory, gameName) { _requests = requests ?? throw new ArgumentNullException(nameof(requests)); _finishedRequests = finishedRequests; _notifier = notifier; }
public override void CancelRunningOperations() { waitingIndexes = new BlockingCollection <TileIndex>(); }
public SpySink() { this.events = new BlockingCollection <LogEvent>(); }
public override int CaptureWithCursor(FrameInfo frame) { var res = new Result(-1); try { //Try to get the duplicated output frame within given time. res = DuplicatedOutput.TryAcquireNextFrame(0, out var info, out var resource); //Checks how to proceed with the capture. It could have failed, or the screen, cursor or both could have been captured. if (res.Failure || resource == null || (info.AccumulatedFrames == 0 && info.LastMouseUpdateTime <= LastProcessTime)) { //Somehow, it was not possible to retrieve the resource, frame or metadata. //frame.WasDropped = true; //BlockingCollection.Add(frame); resource?.Dispose(); return(FrameCount); } else if (info.AccumulatedFrames == 0 && info.LastMouseUpdateTime > LastProcessTime) { //Gets the cursor shape if the screen hasn't changed in between, so the cursor will be available for the next frame. GetCursor(null, info, frame); resource.Dispose(); return(FrameCount); } //Saves the most recent capture time. LastProcessTime = Math.Max(info.LastPresentTime, info.LastMouseUpdateTime); //Copy resource into memory that can be accessed by the CPU. using (var screenTexture = resource.QueryInterface <Texture2D>()) { //Copies from the screen texture only the area which the user wants to capture. Device.ImmediateContext.CopySubresourceRegion(screenTexture, 0, new ResourceRegion(TrueLeft, TrueTop, 0, TrueRight, TrueBottom, 1), BackingTexture, 0); //Copy the captured desktop texture into a staging texture, in order to show the mouse cursor and not make the captured texture dirty with it. Device.ImmediateContext.CopyResource(BackingTexture, StagingTexture); //Gets the cursor image and merges with the staging texture. GetCursor(StagingTexture, info, frame); } //Get the desktop capture texture. var data = Device.ImmediateContext.MapSubresource(StagingTexture, 0, MapMode.Read, MapFlags.None); if (data.IsEmpty) { //frame.WasDropped = true; //BlockingCollection.Add(frame); Device.ImmediateContext.UnmapSubresource(StagingTexture, 0); resource.Dispose(); return(FrameCount); } #region Get image data var bitmap = new System.Drawing.Bitmap(Width, Height, PixelFormat.Format32bppArgb); var boundsRect = new System.Drawing.Rectangle(0, 0, Width, Height); //Copy pixels from screen capture Texture to the GDI bitmap. var mapDest = bitmap.LockBits(boundsRect, ImageLockMode.WriteOnly, bitmap.PixelFormat); var sourcePtr = data.DataPointer; var destPtr = mapDest.Scan0; for (var y = 0; y < Height; y++) { //Copy a single line. Utilities.CopyMemory(destPtr, sourcePtr, Width * 4); //Advance pointers. sourcePtr = IntPtr.Add(sourcePtr, data.RowPitch); destPtr = IntPtr.Add(destPtr, mapDest.Stride); } //Release source and dest locks. bitmap.UnlockBits(mapDest); //Set frame details. FrameCount++; frame.Path = $"{Project.FullPath}{FrameCount}.png"; frame.Delay = FrameRate.GetMilliseconds(SnapDelay); frame.Image = bitmap; BlockingCollection.Add(frame); #endregion Device.ImmediateContext.UnmapSubresource(StagingTexture, 0); resource.Dispose(); return(FrameCount); } catch (SharpDXException se) when(se.ResultCode.Code == SharpDX.DXGI.ResultCode.WaitTimeout.Result.Code) { return(FrameCount); } catch (SharpDXException se) when(se.ResultCode.Code == SharpDX.DXGI.ResultCode.DeviceRemoved.Result.Code || se.ResultCode.Code == SharpDX.DXGI.ResultCode.DeviceReset.Result.Code) { //When the device gets lost or reset, the resources should be instantiated again. DisposeInternal(); Initialize(); return(FrameCount); } catch (Exception ex) { LogWriter.Log(ex, "It was not possible to finish capturing the frame with DirectX."); MajorCrashHappened = true; OnError.Invoke(ex); return(FrameCount); } finally { try { //Only release the frame if there was a success in capturing it. if (res.Success) { DuplicatedOutput.ReleaseFrame(); } } catch (Exception e) { LogWriter.Log(e, "It was not possible to release the frame."); } } }
/// <summary> Starts capturing and processing video frames. </summary> /// <param name="frameGrabDelay"> The frame grab delay. </param> /// <param name="timestampFn"> Function to generate the timestamp for each frame. This /// function will get called once per frame. </param> protected void StartProcessing(TimeSpan frameGrabDelay, Func <DateTime> timestampFn) { OnProcessingStarting(); _resetTrigger = true; _frameGrabTimer.Reset(); _analysisTaskQueue = new BlockingCollection <Task <NewResultEventArgs> >(); var timerIterations = 0; // Create a background thread that will grab frames in a loop. _producerTask = Task.Factory.StartNew(() => { var frameCount = 0; while (!_stopping) { LogMessage("Producer: waiting for timer to trigger frame-grab"); // Wait to get released by the timer. _frameGrabTimer.WaitOne(); LogMessage("Producer: grabbing frame..."); var startTime = DateTime.Now; // Grab single frame. var timestamp = timestampFn(); Mat image = new Mat(); bool success = _reader.Read(image); LogMessage("Producer: frame-grab took {0} ms", (DateTime.Now - startTime).Milliseconds); if (!success) { // If we've reached the end of the video, stop here. if (_reader.CaptureType == CaptureType.File) { LogMessage("Producer: null frame from video file, stop!"); // This will call StopProcessing on a new thread. var stopTask = StopProcessingAsync(); // Break out of the loop to make sure we don't try grabbing more // frames. break; } else { // If failed on live camera, try again. LogMessage("Producer: null frame from live camera, continue!"); continue; } } // Package the image for submission. VideoFrameMetadata meta; meta.Index = frameCount; meta.Timestamp = timestamp; VideoFrame vframe = new VideoFrame(image, meta); // Raise the new frame event LogMessage("Producer: new frame provided, should analyze? Frame num: {0}", meta.Index); OnNewFrameProvided(vframe); if (_analysisPredicate(vframe)) { LogMessage("Producer: analyzing frame"); // Call the analysis function on a threadpool thread var analysisTask = DoAnalyzeFrame(vframe); LogMessage("Producer: adding analysis task to queue {0}", analysisTask.Id); // Push the frame onto the queue _analysisTaskQueue.Add(analysisTask); } else { LogMessage("Producer: not analyzing frame"); } LogMessage("Producer: iteration took {0} ms", (DateTime.Now - startTime).Milliseconds); ++frameCount; } LogMessage("Producer: stopping, destroy reader and timer"); _analysisTaskQueue.CompleteAdding(); // We reach this point by breaking out of the while loop. So we must be stopping. _reader.Dispose(); _reader = null; // Make sure the timer stops, then get rid of it. var h = new ManualResetEvent(false); _timer.Dispose(h); h.WaitOne(); _timer = null; LogMessage("Producer: stopped"); }, TaskCreationOptions.LongRunning); _consumerTask = Task.Factory.StartNew(async() => { while (!_analysisTaskQueue.IsCompleted) { LogMessage("Consumer: waiting for task to get added"); // Get the next processing task. Task <NewResultEventArgs> nextTask = null; // Blocks if m_analysisTaskQueue.Count == 0 // IOE means that Take() was called on a completed collection. // Some other thread can call CompleteAdding after we pass the // IsCompleted check but before we call Take. // In this example, we can simply catch the exception since the // loop will break on the next iteration. // See https://msdn.microsoft.com/en-us/library/dd997371(v=vs.110).aspx try { nextTask = _analysisTaskQueue.Take(); } catch (InvalidOperationException) { } if (nextTask != null) { // Block until the result becomes available. LogMessage("Consumer: waiting for next result to arrive for task {0}", nextTask.Id); var result = await nextTask; // Raise the new result event. LogMessage("Consumer: got result for frame {0}. {1} tasks in queue", result.Frame.Metadata.Index, _analysisTaskQueue.Count); OnNewResultAvailable(result); } } LogMessage("Consumer: stopped"); }, TaskCreationOptions.LongRunning); // Set up a timer object that will trigger the frame-grab at a regular interval. _timer = new Timer(async s /* state */ => { await _timerMutex.WaitAsync(); try { // If the handle was not reset by the producer, then the frame-grab was missed. bool missed = _frameGrabTimer.WaitOne(0); _frameGrabTimer.Set(); if (missed) { LogMessage("Timer: missed frame-grab {0}", timerIterations - 1); } LogMessage("Timer: grab frame num {0}", timerIterations); ++timerIterations; } finally { _timerMutex.Release(); } }, null, TimeSpan.Zero, frameGrabDelay); OnProcessingStarted(); }
public LightThreadPool(int maxthreadcount = 256) { _maxThreadCount = maxthreadcount; _actions = new BlockingCollection <Action>(); }
public Mailbox() { _messageQueue = new BlockingCollection <T>(1); _responseQueue = new BlockingCollection <T>(1); }
private void FillingSectors(int numberOfSector) { _logger.WriteProtocol("started", $"filling sectors,{numberOfSector}", $"{Task.CurrentId}"); TextBlock ref_textBlock = null; ListBox ref_listbox = null; BlockingCollection <Fan> ref_BagFillingTheSector = null; switch (numberOfSector) { case 1: { ref_BagFillingTheSector = _bagFillingTheSector1; ref_textBlock = tBlockBusyPlacesSecotr1; ref_listbox = lBoxSector1; break; } case 2: { ref_BagFillingTheSector = _bagFillingTheSector2; ref_textBlock = tBlockBusyPlacesSecotr2; ref_listbox = lBoxSector2; break; } case 3: { ref_BagFillingTheSector = _bagFillingTheSector3; ref_textBlock = tBlockBusyPlacesSecotr3; ref_listbox = lBoxSector3; break; } case 4: { ref_BagFillingTheSector = _bagFillingTheSector4; ref_textBlock = tBlockBusyPlacesSecotr4; ref_listbox = lBoxSector4; break; } case 5: { ref_BagFillingTheSector = _bagFillingTheSector5; ref_textBlock = tBlockBusyPlacesSecotr5; ref_listbox = lBoxSector5; break; } case 6: { ref_BagFillingTheSector = _bagFillingTheSector6; ref_textBlock = tBlockBusyPlacesSecotr6; ref_listbox = lBoxSector6; break; } } while (_queueInFrontOfTheSectors.IsCompleted != true && ref_BagFillingTheSector.Count != 10000) { Thread.Sleep(1); Fan tmpFan = null; _queueInFrontOfTheSectors.TryTake(out tmpFan, 100, _cts.Token); if (tmpFan != null && tmpFan.NumberOfSector == numberOfSector) { ref_BagFillingTheSector.Add(tmpFan); _context.Send( //Метод работает синхронно, текущий поток дождется его завершения //Post будет работать асинхронно, текущий поток его не ждет state => { ref_textBlock.Text = ref_BagFillingTheSector.Count.ToString(); ref_listbox.Items.Add(tmpFan.NumberOfPlace.ToString()); tBlockQueueBeforeSectors.Text = (_queueInFrontOfTheSectors.Count.ToString()); tBlockFansOnTheirPlaces.Text = (_bagFillingTheSector1.Count + _bagFillingTheSector2.Count + _bagFillingTheSector3.Count + _bagFillingTheSector4.Count + _bagFillingTheSector5.Count + _bagFillingTheSector6.Count).ToString(); }, null ); } else if (tmpFan != null) { _queueInFrontOfTheSectors.Add(tmpFan); } } _logger.WriteProtocol("finished", $"filling sectors,{numberOfSector}", $"{Task.CurrentId}"); }
public PlayersMapping() { _playersCollection = new BlockingCollection <Player>(); _tasks = new List <Task>(); _players = new List <Player>(); }
// // //public DataTreatment(BlockingCollection<RawData> collection, BlockingCollection<RawData> graphCollection, BlockingCollection<RawData> filterCollection, IData iData, ConvertAlgo conv) public DataTreatment(BlockingCollection <double> collection, BlockingCollection <double> graphCollection, BlockingCollection <double> filterCollection, BlockingCollection <double> alarmCollection, ConvertAlgo conv, IData data, IAlarm alarm, UC1M1_ZeroAdjustment adj) { _collection = collection; _graphCollection = graphCollection; _filterCollection = filterCollection; _alarmCollection = alarmCollection; DataInterface = data; //_filterCollection = filterCollection; AlarmController = alarm; AdjustmentController = adj; //DataInterface = iData; ConvertAlgorithm = conv; FullList = new List <double>(); DownsampledRawList = new List <double>(); GraphList = new List <double>(); PulseList = new List <double>(); }
public RabbitMqClientFixture(ScenarioContext scenatioContext) { this.scenatioContext = scenatioContext; collectionOfSignal = new BlockingCollection <JObject>(); }
public async Task SizeMixParallel() { var st = new StringTable(0); ConcurrentDictionary <StringId, string> map = new ConcurrentDictionary <StringId, string>(); var sb = new StringBuilder(3000000); var strings = new BlockingCollection <string>(); var stringIds = new BlockingCollection <StringId>(); var createStringsTask = Task.Run(() => { // zip through small sizes for (int i = 0; i < 3000; i++) { sb.Length = 0; sb.Append('x', i); string str = sb.ToString(); strings.Add(str); } // now use increasingly large amounts, including exceeding the size of a single buffer's worth for (int i = 0; i < 100; i++) { sb.Length = 0; sb.Append('x', i * 10000); string str = sb.ToString(); } strings.CompleteAdding(); }); var validateStringsTask = Task.Run(() => { for (int i = 0; i < 2; i++) { // make sure all the right strings come out int startBias = 0; var buf = new char[4000000]; foreach (StringId sd in stringIds.GetConsumingEnumerable()) { // get the actual string we kept string str = map[sd]; // does the table report the right length? int length = st.GetLength(sd); XAssert.AreEqual(str.Length, length); // put sentinel bytes in the char buffer, extract the string from the table, and check the sentinels if (startBias > 0) { buf[startBias - 1] = (char)42; } buf[startBias + length] = (char)31415; st.CopyString(sd, buf, startBias); if (startBias > 0) { XAssert.AreEqual(42, buf[startBias - 1]); } XAssert.AreEqual(31415, buf[startBias + length]); // make sure we got all the characters out that we should have for (int j = 0; j < str.Length; j++) { XAssert.AreEqual(str[j], buf[startBias + j]); } startBias++; } // make sure the same behavior occurs after freezing st.Freeze(); } }); Parallel.ForEach(strings.GetConsumingEnumerable(), str => { StringId sd = st.AddString(str); map.TryAdd(sd, str); stringIds.Add(sd); }); stringIds.CompleteAdding(); await createStringsTask; await validateStringsTask; }
public static IIntCodeProgram New(BlockingCollection <IntCodeValue> input) { return(New(input, new BlockingCollection <IntCodeValue>())); }
/// <summary> /// Flushes all existing messages in the <see cref="BlockingCollection"/> before /// continuing. This is useful before prompting for user input so that log messages are /// written out before the user prompt text. /// </summary> protected void FlushBlockingCollection() { BlockingCollection.Add(new FlushMessage()); WaitHandle.WaitAny(new WaitHandle[] { _flushSemaphore, Token.WaitHandle }); }
public void SaveData(Stream stream, IDataView data, params int[] colIndices) { _host.CheckValue(stream, nameof(stream)); _host.CheckValue(data, nameof(data)); _host.CheckValueOrNull(colIndices); _host.CheckParam(stream.CanWrite, nameof(stream), "cannot save to non-writable stream"); _host.CheckParam(stream.CanSeek, nameof(stream), "cannot save to non-seekable stream"); _host.CheckParam(stream.Position == 0, nameof(stream), "stream must be positioned at head of stream"); using (IChannel ch = _host.Start("Saving")) using (ExceptionMarshaller exMarshaller = new ExceptionMarshaller()) { var toWrite = new BlockingCollection <Block>(16); var toCompress = new BlockingCollection <Block>(16); var activeColumns = GetActiveColumns(data.Schema, colIndices); int rowsPerBlock = RowsPerBlockHeuristic(data, activeColumns); ch.Assert(rowsPerBlock > 0); Stopwatch sw = new Stopwatch(); // Set up the compression and write workers that consume the input information first. Task compressionTask = null; if (activeColumns.Length > 0) { OrderedWaiter waiter = _deterministicBlockOrder ? new OrderedWaiter() : null; Task[] compressionThreads = new Task[Environment.ProcessorCount]; for (int i = 0; i < compressionThreads.Length; ++i) { compressionThreads[i] = Utils.RunOnBackgroundThread( () => CompressionWorker(toCompress, toWrite, activeColumns.Length, waiter, exMarshaller)); } compressionTask = Task.WhenAll(compressionThreads); } // While there is an advantage to putting the IO into a separate thread, there is not an // advantage to having more than one worker. Task writeThread = Utils.RunOnBackgroundThread( () => WriteWorker(stream, toWrite, activeColumns, data.Schema, rowsPerBlock, _host, exMarshaller)); sw.Start(); // REVIEW: For now the fetch worker just works in the main thread. If it's // a fairly large view through, it may be advantageous to consider breaking up the // fetchwrite operations on the pipes, somehow. // Despite running in the main thread for now, the fetch worker follows the same // pattern of utilizing exMarshaller. using (var pch = _silent ? null : _host.StartProgressChannel("BinarySaver")) { FetchWorker(toCompress, data, activeColumns, rowsPerBlock, sw, ch, pch, exMarshaller); } _host.Assert(compressionTask != null || toCompress.IsCompleted); if (compressionTask != null) { compressionTask.Wait(); } toWrite.CompleteAdding(); writeThread.Wait(); exMarshaller.ThrowIfSet(ch); if (!_silent) { ch.Info("Wrote {0} rows across {1} columns in {2}", _rowCount, activeColumns.Length, sw.Elapsed); } // When we dispose the exception marshaller, this will set the cancellation token when we internally // dispose the cancellation token source, so one way or another those threads are being cancelled, even // if an exception is thrown in the main body of this function. } }
public List <TerrainPatch> ExportTemperatureProfiles() { _precache = new Dictionary <Point, bool>(); _cache = new Dictionary <Point, TerrainPatch>(); var sieglers = ReadSieglerProduct2().ToList(); Console.WriteLine($"There are {sieglers.Count} lines in the siegler product."); // Filter for sieglers near 12420,17549 var center = new Point(12420, 17549); sieglers = sieglers.Where(s => Distance(s.Pixel, center) < 30d).ToList(); Console.WriteLine($"There are {sieglers.Count} pixels close to the test center"); //var sieglers2 = sieglers.Where(s => s.Distance < d).ToList(); //Console.WriteLine($"filtered by distance {d} count={sieglers2.Count} fraction={sieglers2.Count / (double)sieglers.Count}"); //sieglers = sieglers.Where(s=>s.Depth<=1f && s.Depth>=0f).ToList(); var group1 = sieglers.GroupBy(s => DepthEquivalenceClass(s.Depth)).ToList(); var sieglers2 = new List <Siegler>(); foreach (var g in group1) { sieglers2.AddRange(g.Take(10)); } Console.WriteLine($"bunched by percent [0-1] count={sieglers2.Count} fraction={sieglers2.Count / (double)sieglers.Count}"); var groups = sieglers2.GroupBy(s => s.Pixel).ToList(); Console.WriteLine($"There are {groups.Count} grouped pixels within the map"); var consolidated = groups.Select(g => new ConsolidatedLine(g.ToList())).ToList(); consolidated = consolidated.Take(100).ToList(); // smaller set for now var filtered = consolidated.Where(HorizonsHaveBeenCalculated).ToList(); Console.WriteLine($"There are {filtered.Count} pixels with horizons already calculated"); Console.WriteLine($"There are {_cache.Count} tiles involved."); Console.WriteLine($"Beginning to load patches"); Parallel.ForEach(_cache.Values, patch => GetPatch(patch.Id)); Console.WriteLine($"Finished loading patches"); var expected_count = consolidated.Count; var count = 0; var stopwatch = new Stopwatch(); var starttime = DateTime.Now; stopwatch.Start(); var queue = new BlockingCollection <ConsolidatedLine>(10000); Task.Run(() => { Parallel.ForEach(consolidated, new ParallelOptions { MaxDegreeOfParallelism = Math.Max(Environment.ProcessorCount - 1, 1) }, c => { var r = DoCalculation(c); if (r) { queue.Add(c); Console.WriteLine(@"[{0:D4},{1:D4}].Lighting.Max()={2:F8} Depth={3:F8}", c.Pixel.X, c.Pixel.Y, c.Lighting.Max(), c.Depth); } else { Console.WriteLine($"Calculation failed for [{c.Pixel.X},{c.Pixel.Y}]"); } }); queue.CompleteAdding(); }); using (var fs = File.Create(LightingCacheFilename, 4096, FileOptions.None)) using (var bw = new BinaryWriter(fs)) foreach (var c in queue.GetConsumingEnumerable()) { c.Write(bw); Math.DivRem(count++, 100, out int rem); if (rem == 0) { var ticks_per_solution = stopwatch.Elapsed.Ticks / count; var timespan_per_solution = new TimeSpan(ticks_per_solution); var estimated_elapsed_ticks = ticks_per_solution * expected_count; var estimated_stop = starttime.AddTicks(estimated_elapsed_ticks); Console.WriteLine($"Processed {count} pixels. {timespan_per_solution} per solution. Est stop={estimated_stop}"); } } stopwatch.Stop(); Console.WriteLine($"The lighting calculations took {stopwatch.Elapsed}"); return(_cache.Values.ToList()); }
private void WriteWorker(Stream stream, BlockingCollection <Block> toWrite, ColumnCodec[] activeColumns, DataViewSchema sourceSchema, int rowsPerBlock, IChannelProvider cp, ExceptionMarshaller exMarshaller) { _host.AssertValue(exMarshaller); try { _host.AssertValue(cp); cp.AssertValue(stream); cp.AssertValue(toWrite); cp.AssertValue(activeColumns); cp.AssertValue(sourceSchema); cp.Assert(rowsPerBlock > 0); using (IChannel ch = cp.Start("Write")) { var blockLookups = new List <BlockLookup> [activeColumns.Length]; for (int c = 0; c < blockLookups.Length; ++c) { blockLookups[c] = new List <BlockLookup>(); } var deadLookups = new int[activeColumns.Length]; // Reserve space for the header at the start. This will be filled // in with valid values once writing has completed. ch.CheckIO(stream.Position == 0); stream.Write(new byte[Header.HeaderSize], 0, Header.HeaderSize); ch.CheckIO(stream.Position == Header.HeaderSize); long expectedPosition = stream.Position; BlockLookup deadLookup = new BlockLookup(); foreach (Block block in toWrite.GetConsumingEnumerable(exMarshaller.Token)) { ch.CheckIO(stream.Position == expectedPosition); MemoryStream compressed = block.BlockData; ArraySegment <byte> buffer; bool tmp = compressed.TryGetBuffer(out buffer); ch.Assert(tmp); stream.Write(buffer.Array, buffer.Offset, buffer.Count); BlockLookup currLookup = new BlockLookup(expectedPosition, (int)compressed.Length, block.UncompressedLength); expectedPosition += compressed.Length; _memPool.Return(ref compressed); ch.CheckIO(stream.Position == expectedPosition); // Record the position. We have this "lookups" list per column. Yet, it may be that sometimes // the writer receives things out of order. // REVIEW: The format and the rest of the pipeline supposedly supports a long number // of blocks, but the writing scheme does not yet support that. int blockIndex = (int)block.BlockIndex; var lookups = blockLookups[block.ColumnIndex]; if (lookups.Count == block.BlockIndex) // Received in order. { lookups.Add(currLookup); } else if (lookups.Count < block.BlockIndex) // Received a block a little bit early. { // Add a bunch of dead filler lookups, until these late blocks come in. int deadToAdd = (int)block.BlockIndex - lookups.Count; for (int i = 0; i < deadToAdd; ++i) { lookups.Add(deadLookup); } deadLookups[block.ColumnIndex] += deadToAdd; ch.Assert(lookups.Count == block.BlockIndex); lookups.Add(currLookup); } else // Received a block a little bit late. { // This should be a dead block unless the compressors are buggy and somehow // yielding duplicate blocks or something. ch.Assert(lookups[blockIndex].BlockOffset == 0); deadLookups[block.ColumnIndex]--; lookups[blockIndex] = currLookup; } } // We have finished writing all blocks. We will now write the block lookup tables (so we can // find the blocks), the slot names (for any columns that have them), the column table of // contents (so we know how to decode the blocks, and where the lookups and names are), // and the header (so we know dataview wide information and where to find the table of // contents) in that order. long[] lookupOffsets = new long[blockLookups.Length]; using (BinaryWriter writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen: true)) { // Write the block lookup directories. These are referenced from the table of contents, // so that someone knows where to look for some block data. for (int c = 0; c < blockLookups.Length; ++c) { ch.Assert(deadLookups[c] == 0); // The block lookup directories are written uncompressed and in fixed length // to enable rapid seeking. lookupOffsets[c] = stream.Position; foreach (BlockLookup lookup in blockLookups[c]) { // *** Lookup table entry format *** // long: Offset to the start of a block // int: Byte length of block as written // int: Byte length of block when uncompressed ch.Assert(lookup.BlockOffset > 0); writer.Write(lookup.BlockOffset); writer.Write(lookup.BlockLength); writer.Write(lookup.DecompressedBlockLength); } ch.CheckIO(stream.Position == lookupOffsets[c] + (16 * blockLookups[c].Count), "unexpected offsets after block lookup table write"); } // Write the metadata for each column. long[] metadataTocOffsets = new long[activeColumns.Length]; for (int c = 0; c < activeColumns.Length; ++c) { metadataTocOffsets[c] = WriteMetadata(writer, sourceSchema, activeColumns[c].SourceIndex, ch); } // Write the table of contents. long tocOffset = stream.Position; { int c = 0; expectedPosition = stream.Position; foreach (var active in activeColumns) { // *** Column TOC entry format *** // string: column name // codec (as interpretable by CodecFactory.TryGetCodec): column block codec // CompressionKind(byte): block compression strategy // LEB128 int: Rows per block // long: Offset to the start of the lookup table // long: Offset to the start of the metadata TOC entries, or 0 if this has no metadata string name = sourceSchema[active.SourceIndex].Name; writer.Write(name); int nameLen = Encoding.UTF8.GetByteCount(name); expectedPosition += Utils.Leb128IntLength((uint)nameLen) + nameLen; ch.CheckIO(stream.Position == expectedPosition, "unexpected offsets after table of contents name"); expectedPosition += _factory.WriteCodec(stream, active.Codec); ch.CheckIO(stream.Position == expectedPosition, "unexpected offsets after table of contents type description"); writer.Write((byte)_compression); expectedPosition++; // REVIEW: Right now the number of rows per block is fixed, so we // write the same value each time. In some future state, it may be that this // is relaxed, with possibly some tradeoffs (for example, inability to randomly seek). writer.WriteLeb128Int((ulong)rowsPerBlock); expectedPosition += Utils.Leb128IntLength((uint)rowsPerBlock); // Offset of the lookup table. writer.Write(lookupOffsets[c]); expectedPosition += sizeof(long); // Offset of the metadata table of contents. writer.Write(metadataTocOffsets[c]); expectedPosition += sizeof(long); ch.CheckIO(stream.Position == expectedPosition, "unexpected offsets after table of contents"); c++; } } // Write the tail signature. long tailOffset = stream.Position; writer.Write(Header.TailSignatureValue); // Now move back to the beginning of the stream, and write out the now completed header. Header header = new Header() { Signature = Header.SignatureValue, Version = Header.WriterVersion, CompatibleVersion = Header.CanBeReadByVersion, TableOfContentsOffset = tocOffset, TailOffset = tailOffset, RowCount = _rowCount, ColumnCount = activeColumns.Length }; byte[] headerBytes = new byte[Header.HeaderSize]; unsafe { Marshal.Copy(new IntPtr(&header), headerBytes, 0, Marshal.SizeOf(typeof(Header))); } writer.Seek(0, SeekOrigin.Begin); writer.Write(headerBytes); } } } catch (Exception ex) { exMarshaller.Set("writing", ex); } }
private void FetchWorker(BlockingCollection <Block> toCompress, IDataView data, ColumnCodec[] activeColumns, int rowsPerBlock, Stopwatch sw, IChannel ch, IProgressChannel pch, ExceptionMarshaller exMarshaller) { Contracts.AssertValue(ch); Contracts.AssertValueOrNull(pch); ch.AssertValue(exMarshaller); try { ch.AssertValue(toCompress); ch.AssertValue(data); ch.AssertValue(activeColumns); ch.AssertValue(sw); ch.Assert(rowsPerBlock > 0); // The main thread handles fetching from the cursor, and storing it into blocks passed to toCompress. HashSet <int> activeSet = new HashSet <int>(activeColumns.Select(col => col.SourceIndex)); long blockIndex = 0; int remainingInBlock = rowsPerBlock; using (DataViewRowCursor cursor = data.GetRowCursor(data.Schema.Where(c => activeSet.Contains(c.Index)))) { WritePipe[] pipes = new WritePipe[activeColumns.Length]; for (int c = 0; c < activeColumns.Length; ++c) { pipes[c] = WritePipe.Create(this, cursor, activeColumns[c]); } for (int c = 0; c < pipes.Length; ++c) { pipes[c].BeginBlock(); } long rows = 0; if (pch != null) { pch.SetHeader(new ProgressHeader(new[] { "rows" }), e => e.SetProgress(0, rows)); } while (cursor.MoveNext()) { for (int c = 0; c < pipes.Length; ++c) { pipes[c].FetchAndWrite(); } if (--remainingInBlock == 0) { for (int c = 0; c < pipes.Length; ++c) { // REVIEW: It may be better if EndBlock got moved to a different worker thread. toCompress.Add(new Block(pipes[c].EndBlock(), c, blockIndex), exMarshaller.Token); pipes[c].BeginBlock(); } remainingInBlock = rowsPerBlock; blockIndex++; } rows++; } if (remainingInBlock < rowsPerBlock) { for (int c = 0; c < pipes.Length; ++c) { toCompress.Add(new Block(pipes[c].EndBlock(), c, blockIndex), exMarshaller.Token); } } Contracts.Assert(rows == (blockIndex + 1) * rowsPerBlock - remainingInBlock); _rowCount = rows; if (pch != null) { pch.Checkpoint(rows); } } toCompress.CompleteAdding(); } catch (Exception ex) { exMarshaller.Set("cursoring", ex); } }
public CompletionPort() { m_queue = new BlockingCollection <CompletionStatus>(); }
public BoundedMessageQueue() { _queue = new BlockingCollection <Envelope>(); }
public LoggerFactory(BlockingCollection <LogMessage> pendingMessages) { _pendingMessages = pendingMessages; }
private void IndexBalances(ChainBase chain, string checkpointName, Func <uint256, Transaction, uint256, BlockHeader, int, IEnumerable <OrderedBalanceChange> > extract) { SetThrottling(); BlockingCollection <OrderedBalanceChange[]> indexedEntries = new BlockingCollection <OrderedBalanceChange[]>(100); var tasks = CreateTaskPool(indexedEntries, (entries) => Index(entries.Select(e => e.ToEntity()), this.Configuration.GetBalanceTable()), 30); using (IndexerTrace.NewCorrelation("Import balances " + checkpointName + " to azure started").Open()) { this.Configuration.GetBalanceTable().CreateIfNotExists(); var buckets = new MultiValueDictionary <string, OrderedBalanceChange>(); using (var storedBlocks = Enumerate(checkpointName, chain)) { foreach (var block in storedBlocks) { foreach (var tx in block.Block.Transactions) { var txId = tx.GetHash(); try { var entries = extract(txId, tx, block.BlockId, block.Block.Header, block.Height); foreach (var entry in entries) { buckets.Add(entry.PartitionKey, entry); var bucket = buckets[entry.PartitionKey]; if (bucket.Count == 100) { indexedEntries.Add(bucket.ToArray()); buckets.Remove(entry.PartitionKey); } } if (storedBlocks.NeedSave) { foreach (var kv in buckets.AsLookup().ToArray()) { indexedEntries.Add(kv.ToArray()); } buckets.Clear(); tasks.Stop(); storedBlocks.SaveCheckpoint(); tasks.Start(); } } catch (Exception ex) { IndexerTrace.ErrorWhileImportingBalancesToAzure(ex, txId); throw; } } } foreach (var kv in buckets.AsLookup().ToArray()) { indexedEntries.Add(kv.ToArray()); } tasks.Stop(); storedBlocks.SaveCheckpoint(); } } }
public UdpThreadInfos(BlockingCollection <T> dataTransferer, int port) { DataTransferer = dataTransferer; Port = port; }
public void ShouldStreamAllDocumentsAfterSubscriptionCreation() { using (var store = GetDocumentStore()) { using (var session = store.OpenSession()) { session.Store(new User { Age = 31 }, "users/1"); session.Store(new User { Age = 27 }, "users/12"); session.Store(new User { Age = 25 }, "users/3"); session.SaveChanges(); } var id = store.Subscriptions.Create <User>(); using (var subscription = store.Subscriptions.GetSubscriptionWorker <User>(new SubscriptionWorkerOptions(id) { TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5) })) { var keys = new BlockingCollection <string>(); var ages = new BlockingCollection <int>(); subscription.Run(batch => { batch.Items.ForEach(x => keys.Add(x.Id)); batch.Items.ForEach(x => ages.Add(x.Result.Age)); }); Assert.True(keys.TryTake(out string key, _reasonableWaitTime)); Assert.Equal("users/1", key); Assert.True(keys.TryTake(out key, _reasonableWaitTime)); Assert.Equal("users/12", key); Assert.True(keys.TryTake(out key, _reasonableWaitTime)); Assert.Equal("users/3", key); Assert.True(ages.TryTake(out int age, _reasonableWaitTime)); Assert.Equal(31, age); Assert.True(ages.TryTake(out age, _reasonableWaitTime)); Assert.Equal(27, age); Assert.True(ages.TryTake(out age, _reasonableWaitTime)); Assert.Equal(25, age); var expectedSendBufferSize = SubscriptionWorkerOptions.DefaultSendBufferSizeInBytes; var expectedReceiveBufferSize = SubscriptionWorkerOptions.DefaultReceiveBufferSizeInBytes; if (PlatformDetails.RunningOnLinux) { // linux is doubling that value by design expectedSendBufferSize *= 2; expectedReceiveBufferSize *= 2; } Assert.Equal(expectedSendBufferSize, subscription._tcpClient.SendBufferSize); Assert.Equal(expectedReceiveBufferSize, subscription._tcpClient.ReceiveBufferSize); } } }