/// <summary>
/// Initializes a new instance of the <see cref="AsynchronousTraceListenerWrapper" /> class.
/// </summary>
/// <param name="wrappedTraceListener">The wrapped trace listener.</param>
/// <param name="ownsWrappedTraceListener">Indicates whether the wrapper should dispose the wrapped trace listener.</param>
/// <param name="bufferSize">Size of the buffer for asynchronous requests.</param>
/// <param name="maxDegreeOfParallelism">The max degree of parallelism for thread safe listeners. Specify <see langword="null"/> to use the current core count.</param>
/// <param name="disposeTimeout">The timeout for waiting to complete buffered requests when disposing. When <see langword="null" /> the default of <see cref="Timeout.InfiniteTimeSpan" /> is used.</param>
public AsynchronousTraceListenerWrapper(
TraceListener wrappedTraceListener,
bool ownsWrappedTraceListener = true,
int? bufferSize = DefaultBufferSize,
int? maxDegreeOfParallelism = null,
TimeSpan? disposeTimeout = null)
{
Guard.ArgumentNotNull(wrappedTraceListener, "wrappedTraceListener");
CheckBufferSize(bufferSize);
CheckMaxDegreeOfParallelism(maxDegreeOfParallelism);
CheckDisposeTimeout(disposeTimeout);
this.wrappedTraceListener = wrappedTraceListener;
this.ownsWrappedTraceListener = ownsWrappedTraceListener;
this.disposeTimeout = disposeTimeout ?? Timeout.InfiniteTimeSpan;
this.closeSource = new CancellationTokenSource();
this.requests = bufferSize != null ? new BlockingCollection<Action<TraceListener>>(bufferSize.Value) : new BlockingCollection<Action<TraceListener>>();
if (this.wrappedTraceListener.IsThreadSafe)
{
this.maxDegreeOfParallelism = maxDegreeOfParallelism.HasValue ? maxDegreeOfParallelism.Value : Environment.ProcessorCount;
this.asyncProcessingTask = Task.Factory.StartNew(this.ProcessRequestsInParallel, CancellationToken.None, TaskCreationOptions.HideScheduler | TaskCreationOptions.LongRunning, TaskScheduler.Default);
}
else
{
this.asyncProcessingTask = Task.Factory.StartNew(this.ProcessRequests, CancellationToken.None, TaskCreationOptions.HideScheduler | TaskCreationOptions.LongRunning, TaskScheduler.Default);
}
}
/// <summary>
/// Creates a buffer pool.
/// </summary>
/// <param name="bufferSize">The size, in bytes, of each buffer.</param>
/// <param name="maxBuffers">The maximum number of buffers to keep around, unused; by default, the number of unused buffers is unbounded.</param>
private BufferPool(int bufferSize, int maxBuffers, int preallocationSize, string name)
{
Name = name;
byteBufferSize = bufferSize;
buffers = maxBuffers <= 0 ? new BlockingCollection<byte[]>() : new BlockingCollection<byte[]>(maxBuffers);
var globalPoolSizeStat = IntValueStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_BUFFERS_INPOOL,
() => Count);
allocatedBufferCounter = CounterStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_ALLOCATED_BUFFERS);
checkedOutBufferCounter = CounterStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_CHECKED_OUT_BUFFERS);
checkedInBufferCounter = CounterStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_CHECKED_IN_BUFFERS);
droppedBufferCounter = CounterStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_DROPPED_BUFFERS);
droppedTooLargeBufferCounter = CounterStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_DROPPED_TOO_LARGE_BUFFERS);
// Those 2 counters should be equal. If not, it means we don't release all buffers.
IntValueStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_INUSE_CHECKED_OUT_NOT_CHECKED_IN_BUFFERS,
() => checkedOutBufferCounter.GetCurrentValue()
- checkedInBufferCounter.GetCurrentValue()
- droppedBufferCounter.GetCurrentValue());
IntValueStatistic.FindOrCreate(StatisticNames.SERIALIZATION_BUFFERPOOL_INUSE_ALLOCATED_NOT_INPOOL_BUFFERS,
() => allocatedBufferCounter.GetCurrentValue()
- globalPoolSizeStat.GetCurrentValue()
- droppedBufferCounter.GetCurrentValue());
if (preallocationSize <= 0) return;
var dummy = GetMultiBuffer(preallocationSize * Size);
Release(dummy);
}
protected DuplicateRemovedScheduler(IBloomFilter<string> bloomFilter)
{
//_bloomFilter = new MemoryBloomFilter<string>(1000 * 10, 1000 * 10 * 20);
_bloomFilter = bloomFilter;
Queue = new BlockingCollection<RequestMessage>(new ConcurrentQueue<RequestMessage>());
_accumulatedMessageTotal = 0;
}
public void TestStreamingTransportServer()
{
BlockingCollection<string> queue = new BlockingCollection<string>();
List<string> events = new List<string>();
IStreamingCodec<string> stringCodec = _injector.GetInstance<StringStreamingCodec>();
IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, 0);
var remoteHandler = Observer.Create<TransportEvent<string>>(tEvent => queue.Add(tEvent.Data));
using (var server = new StreamingTransportServer<string>(endpoint.Address, remoteHandler, _tcpPortProvider, stringCodec))
{
server.Run();
IPEndPoint remoteEndpoint = new IPEndPoint(IPAddress.Parse("127.0.0.1"), server.LocalEndpoint.Port);
using (var client = new StreamingTransportClient<string>(remoteEndpoint, stringCodec))
{
client.Send("Hello");
client.Send(", ");
client.Send("World!");
events.Add(queue.Take());
events.Add(queue.Take());
events.Add(queue.Take());
}
}
Assert.Equal(3, events.Count);
Assert.Equal(events[0], "Hello");
Assert.Equal(events[1], ", ");
Assert.Equal(events[2], "World!");
}
/// <summary>
/// Do the specified input and output.
/// </summary>
/// <param name="input">Input.</param>
/// <param name="output">Output.</param>
public void Do(BlockingCollection<ISkeleton> input, Action<IEnumerable<Result>> fireNewCommand)
{
var data = new Dictionary<JointType, InputVector>();
foreach (var skeleton in input.GetConsumingEnumerable())
{
foreach (var joint in skeleton.Joints)
{
if (!data.ContainsKey(joint.JointType))
{
data.Add(joint.JointType, new InputVector());
}
data[joint.JointType].Stream.Add(joint.Point);
}
if (data.First().Value.Stream.Count < 5)
{
continue;
}
var results = Recognizer.Recognize(data);
try
{
fireNewCommand(results);
}
catch (Exception)
{
if (data.First().Value.Stream.Count > 40)
{
data.Clear();
}
continue;
}
data.Clear();
}
}
public VideoClass(string url, string languageCode)
{
Url = url;
Subtitles = new BlockingCollection<SubtitlesClass>();
Language = TEDdownloader.Language.GetLanguage(languageCode);
LanguageCode = languageCode;
}
public static void StartCollection()
{
if (_started)
return;
_started = true;
conf = Config.CarbonatorSection.Current;
if (conf == null)
{
if (conf.LogLevel >= 3)
EventLog.WriteEntry(Program.EVENT_SOURCE, "Carbonator configuration is missing. This service cannot start", EventLogEntryType.Error);
throw new InvalidOperationException("Carbonator configuration is missing. This service cannot start");
}
Thread.CurrentThread.CurrentCulture = CultureInfo.GetCultureInfo(conf.DefaultCulture);
Thread.CurrentThread.CurrentUICulture = CultureInfo.GetCultureInfo(conf.DefaultCulture);
if (Config.CarbonatorSection.Current.MaxMetricBufferSize > 0)
{
_metricsList = new BlockingCollection<CollectedMetric>(Config.CarbonatorSection.Current.MaxMetricBufferSize);
}
else
{
_metricsList = new BlockingCollection<CollectedMetric>();
}
// start collection and reporting timers
_metricCollectorTimer = new Timer(collectMetrics, new StateControl(), conf.CollectionInterval, conf.CollectionInterval);
_metricReporterTimer = new Timer(reportMetrics, new StateControl(), conf.ReportingInterval, conf.ReportingInterval);
if (conf.LogLevel >= 1)
EventLog.WriteEntry(Program.EVENT_SOURCE, "Carbonator service has been initialized and began reporting metrics", EventLogEntryType.Information);
}
static void Main(string[] args)
{
Console.WriteLine("=== In Line Execution ===");
Task<string>[] tasks = GetTasks();
string[] all = Task.WhenAll(tasks).Result;
Console.WriteLine(string.Join("", all));
Console.WriteLine("=== Standard Live Execution ===");
tasks = GetTasks();
BlockingCollection<string> results = new BlockingCollection<string>();
Task.WaitAll(tasks.Select(async t =>
{
results.Add(await t);
Console.WriteLine("[{0:hh:mm:ss.fff}] Current result: [{1}]", DateTime.Now, string.Join("", results));
}).ToArray());
Console.WriteLine("=== Pragmateek Live Execution ===");
results = new BlockingCollection<string>();
tasks = GetTasks();
Task.WaitAll(tasks.Select(t => t.ContinueWith(p =>
{
results.Add(p.Result);
Console.WriteLine("[{0:hh:mm:ss.fff}] Current result: [{1}]", DateTime.Now, string.Join("", results));
})).ToArray());
}
public Consumer(string groupName, ConsumerSetting setting)
{
if (groupName == null)
{
throw new ArgumentNullException("groupName");
}
GroupName = groupName;
Setting = setting ?? new ConsumerSetting();
_lockObject = new object();
_subscriptionTopics = new Dictionary<string, HashSet<string>>();
_topicQueuesDict = new ConcurrentDictionary<string, IList<MessageQueue>>();
_pullRequestDict = new ConcurrentDictionary<string, PullRequest>();
_remotingClient = new SocketRemotingClient(Setting.BrokerAddress, Setting.SocketSetting, Setting.LocalAddress);
_adminRemotingClient = new SocketRemotingClient(Setting.BrokerAdminAddress, Setting.SocketSetting, Setting.LocalAdminAddress);
_binarySerializer = ObjectContainer.Resolve<IBinarySerializer>();
_scheduleService = ObjectContainer.Resolve<IScheduleService>();
_allocateMessageQueueStragegy = ObjectContainer.Resolve<IAllocateMessageQueueStrategy>();
_logger = ObjectContainer.Resolve<ILoggerFactory>().Create(GetType().FullName);
_remotingClient.RegisterConnectionEventListener(new ConnectionEventListener(this));
if (Setting.MessageHandleMode == MessageHandleMode.Sequential)
{
_consumingMessageQueue = new BlockingCollection<ConsumingMessage>();
_consumeMessageWorker = new Worker("ConsumeMessage", () => HandleMessage(_consumingMessageQueue.Take()));
}
_messageRetryQueue = new BlockingCollection<ConsumingMessage>();
}
public void TestWritableTransportServer()
{
BlockingCollection<WritableString> queue = new BlockingCollection<WritableString>();
List<string> events = new List<string>();
IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, 0);
var remoteHandler = Observer.Create<TransportEvent<WritableString>>(tEvent => queue.Add(tEvent.Data));
using (var server = new WritableTransportServer<WritableString>(endpoint, remoteHandler, _tcpPortProvider, _injector))
{
server.Run();
IPEndPoint remoteEndpoint = new IPEndPoint(IPAddress.Parse("127.0.0.1"), server.LocalEndpoint.Port);
using (var client = new WritableTransportClient<WritableString>(remoteEndpoint, _injector))
{
client.Send(new WritableString("Hello"));
client.Send(new WritableString(", "));
client.Send(new WritableString("World!"));
events.Add(queue.Take().Data);
events.Add(queue.Take().Data);
events.Add(queue.Take().Data);
}
}
Assert.AreEqual(3, events.Count);
Assert.AreEqual(events[0], "Hello");
Assert.AreEqual(events[1], ", ");
Assert.AreEqual(events[2], "World!");
}
/// <summary>Initializes a new instance of the StaTaskScheduler class with the specified concurrency level.</summary>
/// <param name="numberOfThreads">The number of threads that should be created and used by this scheduler.</param>
public StaTaskScheduler(int numberOfThreads)
{
// Validate arguments
if (numberOfThreads < 1) throw new ArgumentOutOfRangeException("concurrencyLevel");
// Initialize the tasks collection
_tasks = new BlockingCollection<Task>();
// Create the threads to be used by this scheduler
_threads = Enumerable.Range(0, numberOfThreads).Select(i =>
{
var thread = new Thread(() =>
{
// Continually get the next task and try to execute it.
// This will continue until the scheduler is disposed and no more tasks remain.
foreach (var t in _tasks.GetConsumingEnumerable())
{
TryExecuteTask(t);
}
});
thread.IsBackground = true;
thread.SetApartmentState(ApartmentState.STA);
return thread;
}).ToList();
// Start all of the threads
_threads.ForEach(t => t.Start());
}
public BlockingCollectionQueue()
{
_writerQueue = new BlockingCollection<User>();
_readerQueue = new BlockingCollection<User>();
_currentQueue = _writerQueue;
Task.Factory.StartNew(() => ConsumerFunc(), TaskCreationOptions.None);
}
public void CanGetNotificationAboutDocumentIndexUpdate()
{
using (var server = GetNewServer())
using (var store = NewRemoteDocumentStore(ravenDbServer: server))
{
var list = new BlockingCollection<IndexChangeNotification>();
var taskObservable = store.Changes();
taskObservable.Task.Wait();
var observableWithTask = taskObservable.ForIndex("Raven/DocumentsByEntityName");
observableWithTask.Task.Wait();
observableWithTask
.Subscribe(list.Add);
using (var session = store.OpenSession())
{
session.Store(new Item(), "items/1");
session.SaveChanges();
}
IndexChangeNotification indexChangeNotification;
Assert.True(list.TryTake(out indexChangeNotification, TimeSpan.FromSeconds(5)));
Assert.Equal("Raven/DocumentsByEntityName", indexChangeNotification.Name);
Assert.Equal(indexChangeNotification.Type, IndexChangeTypes.MapCompleted);
}
}
public HardDiskCache(string cachePath)
{
CachePath = cachePath;
indexList = new List<LocationInfo>();
imageQueue = new BlockingCollection<Image>();
StartConsumerThread();
}
public void CanGetNotificationsAboutConflictedAttachements()
{
using(var store1 = CreateStore())
using (var store2 = CreateStore())
{
store1.DatabaseCommands.PutAttachment("attachment/1", null, new MemoryStream(new byte[] {1, 2, 3}),
new RavenJObject());
store2.DatabaseCommands.PutAttachment("attachment/1", null, new MemoryStream(new byte[] {1, 2, 3}),
new RavenJObject());
var list = new BlockingCollection<ReplicationConflictNotification>();
var taskObservable = store2.Changes();
taskObservable.Task.Wait();
var observableWithTask = taskObservable.ForAllReplicationConflicts();
observableWithTask.Task.Wait();
observableWithTask
.Subscribe(list.Add);
TellFirstInstanceToReplicateToSecondInstance();
ReplicationConflictNotification replicationConflictNotification;
Assert.True(list.TryTake(out replicationConflictNotification, TimeSpan.FromSeconds(10)));
Assert.Equal("attachment/1", replicationConflictNotification.Id);
Assert.Equal(replicationConflictNotification.ItemType, ReplicationConflictTypes.AttachmentReplicationConflict);
Assert.Equal(2, replicationConflictNotification.Conflicts.Length);
Assert.Equal(ReplicationOperationTypes.Put, replicationConflictNotification.OperationType);
}
}
public void CanGetNotificationAboutDocumentDelete()
{
using (GetNewServer())
using (var store = new DocumentStore
{
Url = "http://localhost:8079"
}.Initialize())
{
var list = new BlockingCollection<DocumentChangeNotification>();
var taskObservable = store.Changes();
taskObservable.Task.Wait();
var observableWithTask = taskObservable.ForDocument("items/1");
observableWithTask.Task.Wait();
observableWithTask
.Where(x => x.Type == DocumentChangeTypes.Delete)
.Subscribe(list.Add);
using (var session = store.OpenSession())
{
session.Store(new Item(), "items/1");
session.SaveChanges();
}
store.DatabaseCommands.Delete("items/1", null);
DocumentChangeNotification DocumentChangeNotification;
Assert.True(list.TryTake(out DocumentChangeNotification, TimeSpan.FromSeconds(2)));
Assert.Equal("items/1", DocumentChangeNotification.Id);
Assert.Equal(DocumentChangeNotification.Type, DocumentChangeTypes.Delete);
((RemoteDatabaseChanges) taskObservable).DisposeAsync().Wait();
}
}
private SubstManager()
{
FindAvailableDrives();
_freeDriveQ = new BlockingCollection<char>(_driveSet.Length);
foreach (char drive in _driveSet)
_freeDriveQ.Add(drive);
}
public void Write(BlockingCollection<AddressInfo> queue, String filePath, CancellationToken cancelToken)
{
Count = 0;
using (var fs = new FileStream(filePath, FileMode.Create))
using (var fileWriter = new StreamWriter(fs, Encoding.GetEncoding(1251)))
{
while (true)
{
AddressInfo addressInfo = null;
try
{
addressInfo = queue.Take(cancelToken);
}
catch (InvalidOperationException) { }
if (cancelToken.IsCancellationRequested || queue.IsCompleted)
return;
if (addressInfo != null)
{
fileWriter.WriteLine(String.Format("{0}\t{1}\t{2}", addressInfo.Address, addressInfo.FlatCount, addressInfo.Uik));
Count++;
}
}
}
}
public DynamicSaveStrategy()
{
_decayBag = new ConcurrentBag<Item>();
_itemThreadWriters = new BlockingCollection<QueuedMemoryWriter>();
_mobileThreadWriters = new BlockingCollection<QueuedMemoryWriter>();
_guildThreadWriters = new BlockingCollection<QueuedMemoryWriter>();
}
static void Main(string[] args)
{
// create a blocking collection
BlockingCollection<int> blockingCollection
= new BlockingCollection<int>();
// create and start a producer
Task.Factory.StartNew(() => {
// put the producer to sleep
System.Threading.Thread.Sleep(500);
for (int i = 0; i < 100; i++) {
// add the item to the collection
blockingCollection.Add(i);
}
// mark the collection as finished
blockingCollection.CompleteAdding();
});
// create and start a consumer
Task consumer = Task.Factory.StartNew(() => {
// use a foreach loop to consume the blocking collection
foreach (int i in blockingCollection) {
Console.WriteLine("Item {0}", i);
}
Console.WriteLine("Collection is fully consumed");
});
// wait for the consumer to finish
consumer.Wait();
// wait for input before exiting
Console.WriteLine("Press enter to finish");
Console.ReadLine();
}
public CAudioPlayer()
{
_effects = new BlockingCollection<CSound>();
System.Threading.ThreadStart threadStarter = _checkForThingsToPlay;
_audioThread = new Thread(threadStarter);
_audioThread.Start();
}
/// <summary>Initializes a new instance of the MTATaskScheduler class with the specified concurrency level.</summary>
/// <param name="numberOfThreads">The number of threads that should be created and used by this scheduler.</param>
/// <param name="nameFormat">The template name form to use to name threads.</param>
public MTATaskScheduler(int numberOfThreads, string nameFormat)
{
// Validate arguments
if (numberOfThreads < 1) throw new ArgumentOutOfRangeException("numberOfThreads");
// Initialize the tasks collection
tasks = new BlockingCollection<Task>();
// Create the threads to be used by this scheduler
_threads = Enumerable.Range(0, numberOfThreads).Select(i =>
{
var thread = new Thread(() =>
{
// Continually get the next task and try to execute it.
// This will continue until the scheduler is disposed and no more tasks remain.
foreach (var t in tasks.GetConsumingEnumerable())
{
TryExecuteTask(t);
}
})
{
IsBackground = true
};
thread.SetApartmentState(ApartmentState.MTA);
thread.Name = String.Format("{0} - {1}", nameFormat, thread.ManagedThreadId);
return thread;
}).ToList();
// Start all of the threads
_threads.ForEach(t => t.Start());
}
public void TestStreamingOneWayCommunication()
{
IPAddress listeningAddress = IPAddress.Parse("127.0.0.1");
BlockingCollection<string> queue = new BlockingCollection<string>();
List<string> events = new List<string>();
IStreamingCodec<string> codec = TangFactory.GetTang().NewInjector().GetInstance<StringStreamingCodec>();
using (var remoteManager1 = _remoteManagerFactory1.GetInstance<string>(listeningAddress, codec))
using (var remoteManager2 = _remoteManagerFactory1.GetInstance<string>(listeningAddress, codec))
{
var observer = Observer.Create<string>(queue.Add);
IPEndPoint endpoint1 = new IPEndPoint(listeningAddress, 0);
remoteManager2.RegisterObserver(endpoint1, observer);
var remoteObserver = remoteManager1.GetRemoteObserver(remoteManager2.LocalEndpoint);
remoteObserver.OnNext("abc");
remoteObserver.OnNext("def");
remoteObserver.OnNext("ghi");
events.Add(queue.Take());
events.Add(queue.Take());
events.Add(queue.Take());
}
Assert.AreEqual(3, events.Count);
}
public void CanGetNotificationAboutDocumentPut()
{
using(GetNewServer())
{using (var store = new DocumentStore
{
Url = "http://localhost:8079",
Conventions =
{
FailoverBehavior = FailoverBehavior.FailImmediately
}
}.Initialize())
{
var list = new BlockingCollection<DocumentChangeNotification>();
var taskObservable = store.Changes();
taskObservable.Task.Wait();
var observableWithTask = taskObservable.ForDocument("items/1");
observableWithTask.Task.Wait();
observableWithTask.Subscribe(list.Add);
using (var session = store.OpenSession())
{
session.Store(new Item(), "items/1");
session.SaveChanges();
}
DocumentChangeNotification documentChangeNotification;
Assert.True(list.TryTake(out documentChangeNotification, TimeSpan.FromSeconds(3)));
Assert.Equal("items/1", documentChangeNotification.Id);
Assert.Equal(documentChangeNotification.Type, DocumentChangeTypes.Put);
Assert.NotNull(documentChangeNotification.Etag);
}
Thread.Sleep(1000);
}
}
public CommandProcessor(BlockingCollection<Job> allJobs, Scheduler scheduler, JobQueue jobQueue, QueueProcessor queueProcessor)
{
_allJobs = allJobs;
_scheduler = scheduler;
_jobQueue = jobQueue;
_queueProcessor = queueProcessor;
}
public void TestOneWayCommunicationClientOnly()
{
IPAddress listeningAddress = IPAddress.Parse("127.0.0.1");
BlockingCollection<string> queue = new BlockingCollection<string>();
List<string> events = new List<string>();
using (var remoteManager1 = _remoteManagerFactory.GetInstance(new StringCodec()))
using (var remoteManager2 = _remoteManagerFactory.GetInstance(listeningAddress, new StringCodec()))
{
IPEndPoint remoteEndpoint = new IPEndPoint(listeningAddress, 0);
var observer = Observer.Create<string>(queue.Add);
remoteManager2.RegisterObserver(remoteEndpoint, observer);
var remoteObserver = remoteManager1.GetRemoteObserver(remoteManager2.LocalEndpoint);
remoteObserver.OnNext("abc");
remoteObserver.OnNext("def");
remoteObserver.OnNext("ghi");
events.Add(queue.Take());
events.Add(queue.Take());
events.Add(queue.Take());
}
Assert.Equal(3, events.Count);
}
public OutBuffer(int maxSizeAsPowerOfTwo)
{
//_completeds = new BlockingCollection<TaskCompleted>((int)Math.Pow(2, maxSizeAsPowerOfTwo));
//This is for output, and not spcifying the size revert to ConcurrentQueue.
//Bounding slows down considerably. Should remove when ring array in place.
_completeds = new BlockingCollection<TaskCompleted>();
}
/// <summary>
/// Constructor
/// </summary>
public Reducer()
{
_distinctWordList = new ConcurrentDictionary<string, int>();
var concurrentBag = new ConcurrentBag<string>();
_wordChunks = new BlockingCollection<string>(concurrentBag);
Numwords = 0;
}
/// <summary>Parameterized constructor.
/// </summary>
/// <param name="loggerFactory"></param>
public DefaultActionExecutionService(ILoggerFactory loggerFactory)
{
_logger = loggerFactory.Create(GetType().Name);
_actionQueue = new BlockingCollection<ActionInfo>(new ConcurrentQueue<ActionInfo>());
_worker = new Worker(TryTakeAndExecuteAction, DefaultPeriod);
_worker.Start();
}
public static BlockingCollection<LogMsg> GetLogQueue()
{
if (m_logQueue == null) {
m_logQueue = new BlockingCollection<LogMsg> ();
}
return m_logQueue;
}
public MapTask(TaskFactory taskFactory, BlockingCollection <TIn> inCollection, BlockingCollection <TOut> outCollection, IMapper <TIn, TOut> mapper) : base(taskFactory)
{
_inCollection = inCollection;
_outCollection = outCollection;
_mapper = mapper;
}
public BlockingAggregator(BlockingCollection <InputType> input, int chunkSize, Func <List <InputType>, OutputType> factory)
{
this.inputQueue = input;
this.chunkSize = chunkSize;
this.factory = factory;
}
protected AbstractDownloader(IShellService shellService, CancellationToken ct, PauseToken pt, IProgress <DownloadProgress> progress, BlockingCollection <TumblrPost> producerConsumerCollection, FileDownloader fileDownloader, ICrawlerService crawlerService = null, IBlog blog = null, IFiles files = null)
{
this.shellService = shellService;
this.crawlerService = crawlerService;
this.blog = blog;
this.files = files;
this.ct = ct;
this.pt = pt;
this.progress = progress;
this.producerConsumerCollection = producerConsumerCollection;
this.fileDownloader = fileDownloader;
}
public LifeCycleEventPublisher(ILifeCycleEventHub eventHub, ILoggerFactory loggerFactory)
{
_eventHub = eventHub;
_outbox = new BlockingCollection <LifeCycleEvent>();
_logger = loggerFactory.CreateLogger(GetType());
}
public static void SaveMatrix(IZone[] zones, float[][] data, string fileName)
{
StringBuilder[] zoneLines = new StringBuilder[zones.Length];
var dir = Path.GetDirectoryName(fileName);
if (!String.IsNullOrWhiteSpace(dir))
{
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
}
using (StreamWriter writer = new StreamWriter(fileName))
{
BlockingCollection <SaveTask> toWrite = new BlockingCollection <SaveTask>();
var saveTask = Task.Run(() =>
{
int nextRow = 0;
SortedList <int, SaveTask> backlog = new SortedList <int, SaveTask>();
foreach (var newTask in toWrite.GetConsumingEnumerable())
{
var task = newTask;
do
{
string currentString = task.Text;
int currentRow = task.RowNumber;
if (nextRow == currentRow)
{
writer.WriteLine(currentString);
nextRow++;
}
else
{
backlog.Add(currentRow, new SaveTask()
{
RowNumber = currentRow, Text = currentString
});
break;
}
if (backlog.Count == 0)
{
break;
}
if (backlog.TryGetValue(nextRow, out task))
{
backlog.Remove(nextRow);
continue;
}
break;
} while (true);
}
});
var stringBuilder = new StringBuilder();
stringBuilder.Append("Zones O\\D");
for (int i = 0; i < zones.Length; i++)
{
stringBuilder.Append(',');
stringBuilder.Append(zones[i].ZoneNumber);
}
toWrite.Add(new SaveTask()
{
RowNumber = 0, Text = stringBuilder.ToString()
});
Parallel.For(0, zones.Length, () => new StringBuilder(),
(int i, ParallelLoopState _, StringBuilder strBuilder) =>
{
strBuilder.Clear();
strBuilder.Append(zones[i].ZoneNumber);
var row = data[i];
if (row == null)
{
for (int j = 0; j < zones.Length; j++)
{
strBuilder.Append(",0");
}
}
else
{
for (int j = 0; j < row.Length; j++)
{
strBuilder.Append(',');
strBuilder.Append(row[j]);
}
}
toWrite.Add(new SaveTask()
{
RowNumber = i + 1, Text = strBuilder.ToString()
});
return(strBuilder);
}, (StringBuilder _) => { });
toWrite.CompleteAdding();
saveTask.Wait();
}
}
public BackgroundThreadProducer(BlockingCollection <T> source) : this(source, false)
{
}
public XboxTransferWorker(IXboxGameRepositoryFactory xboxGameRepositoryFactory, string gameName,
BlockingCollection <IXboxTransferRequest> requests, BlockingCollection <IXboxTransferRequest> finishedRequests, IProgressNotifier notifier) : base(xboxGameRepositoryFactory, gameName)
{
_requests = requests ?? throw new ArgumentNullException(nameof(requests));
_finishedRequests = finishedRequests;
_notifier = notifier;
}
public override void CancelRunningOperations()
{
waitingIndexes = new BlockingCollection <TileIndex>();
}
public SpySink()
{
this.events = new BlockingCollection <LogEvent>();
}
public override int CaptureWithCursor(FrameInfo frame)
{
var res = new Result(-1);
try
{
//Try to get the duplicated output frame within given time.
res = DuplicatedOutput.TryAcquireNextFrame(0, out var info, out var resource);
//Checks how to proceed with the capture. It could have failed, or the screen, cursor or both could have been captured.
if (res.Failure || resource == null || (info.AccumulatedFrames == 0 && info.LastMouseUpdateTime <= LastProcessTime))
{
//Somehow, it was not possible to retrieve the resource, frame or metadata.
//frame.WasDropped = true;
//BlockingCollection.Add(frame);
resource?.Dispose();
return(FrameCount);
}
else if (info.AccumulatedFrames == 0 && info.LastMouseUpdateTime > LastProcessTime)
{
//Gets the cursor shape if the screen hasn't changed in between, so the cursor will be available for the next frame.
GetCursor(null, info, frame);
resource.Dispose();
return(FrameCount);
}
//Saves the most recent capture time.
LastProcessTime = Math.Max(info.LastPresentTime, info.LastMouseUpdateTime);
//Copy resource into memory that can be accessed by the CPU.
using (var screenTexture = resource.QueryInterface <Texture2D>())
{
//Copies from the screen texture only the area which the user wants to capture.
Device.ImmediateContext.CopySubresourceRegion(screenTexture, 0, new ResourceRegion(TrueLeft, TrueTop, 0, TrueRight, TrueBottom, 1), BackingTexture, 0);
//Copy the captured desktop texture into a staging texture, in order to show the mouse cursor and not make the captured texture dirty with it.
Device.ImmediateContext.CopyResource(BackingTexture, StagingTexture);
//Gets the cursor image and merges with the staging texture.
GetCursor(StagingTexture, info, frame);
}
//Get the desktop capture texture.
var data = Device.ImmediateContext.MapSubresource(StagingTexture, 0, MapMode.Read, MapFlags.None);
if (data.IsEmpty)
{
//frame.WasDropped = true;
//BlockingCollection.Add(frame);
Device.ImmediateContext.UnmapSubresource(StagingTexture, 0);
resource.Dispose();
return(FrameCount);
}
#region Get image data
var bitmap = new System.Drawing.Bitmap(Width, Height, PixelFormat.Format32bppArgb);
var boundsRect = new System.Drawing.Rectangle(0, 0, Width, Height);
//Copy pixels from screen capture Texture to the GDI bitmap.
var mapDest = bitmap.LockBits(boundsRect, ImageLockMode.WriteOnly, bitmap.PixelFormat);
var sourcePtr = data.DataPointer;
var destPtr = mapDest.Scan0;
for (var y = 0; y < Height; y++)
{
//Copy a single line.
Utilities.CopyMemory(destPtr, sourcePtr, Width * 4);
//Advance pointers.
sourcePtr = IntPtr.Add(sourcePtr, data.RowPitch);
destPtr = IntPtr.Add(destPtr, mapDest.Stride);
}
//Release source and dest locks.
bitmap.UnlockBits(mapDest);
//Set frame details.
FrameCount++;
frame.Path = $"{Project.FullPath}{FrameCount}.png";
frame.Delay = FrameRate.GetMilliseconds(SnapDelay);
frame.Image = bitmap;
BlockingCollection.Add(frame);
#endregion
Device.ImmediateContext.UnmapSubresource(StagingTexture, 0);
resource.Dispose();
return(FrameCount);
}
catch (SharpDXException se) when(se.ResultCode.Code == SharpDX.DXGI.ResultCode.WaitTimeout.Result.Code)
{
return(FrameCount);
}
catch (SharpDXException se) when(se.ResultCode.Code == SharpDX.DXGI.ResultCode.DeviceRemoved.Result.Code || se.ResultCode.Code == SharpDX.DXGI.ResultCode.DeviceReset.Result.Code)
{
//When the device gets lost or reset, the resources should be instantiated again.
DisposeInternal();
Initialize();
return(FrameCount);
}
catch (Exception ex)
{
LogWriter.Log(ex, "It was not possible to finish capturing the frame with DirectX.");
MajorCrashHappened = true;
OnError.Invoke(ex);
return(FrameCount);
}
finally
{
try
{
//Only release the frame if there was a success in capturing it.
if (res.Success)
{
DuplicatedOutput.ReleaseFrame();
}
}
catch (Exception e)
{
LogWriter.Log(e, "It was not possible to release the frame.");
}
}
}
/// <summary> Starts capturing and processing video frames. </summary>
/// <param name="frameGrabDelay"> The frame grab delay. </param>
/// <param name="timestampFn"> Function to generate the timestamp for each frame. This
/// function will get called once per frame. </param>
protected void StartProcessing(TimeSpan frameGrabDelay, Func <DateTime> timestampFn)
{
OnProcessingStarting();
_resetTrigger = true;
_frameGrabTimer.Reset();
_analysisTaskQueue = new BlockingCollection <Task <NewResultEventArgs> >();
var timerIterations = 0;
// Create a background thread that will grab frames in a loop.
_producerTask = Task.Factory.StartNew(() =>
{
var frameCount = 0;
while (!_stopping)
{
LogMessage("Producer: waiting for timer to trigger frame-grab");
// Wait to get released by the timer.
_frameGrabTimer.WaitOne();
LogMessage("Producer: grabbing frame...");
var startTime = DateTime.Now;
// Grab single frame.
var timestamp = timestampFn();
Mat image = new Mat();
bool success = _reader.Read(image);
LogMessage("Producer: frame-grab took {0} ms", (DateTime.Now - startTime).Milliseconds);
if (!success)
{
// If we've reached the end of the video, stop here.
if (_reader.CaptureType == CaptureType.File)
{
LogMessage("Producer: null frame from video file, stop!");
// This will call StopProcessing on a new thread.
var stopTask = StopProcessingAsync();
// Break out of the loop to make sure we don't try grabbing more
// frames.
break;
}
else
{
// If failed on live camera, try again.
LogMessage("Producer: null frame from live camera, continue!");
continue;
}
}
// Package the image for submission.
VideoFrameMetadata meta;
meta.Index = frameCount;
meta.Timestamp = timestamp;
VideoFrame vframe = new VideoFrame(image, meta);
// Raise the new frame event
LogMessage("Producer: new frame provided, should analyze? Frame num: {0}", meta.Index);
OnNewFrameProvided(vframe);
if (_analysisPredicate(vframe))
{
LogMessage("Producer: analyzing frame");
// Call the analysis function on a threadpool thread
var analysisTask = DoAnalyzeFrame(vframe);
LogMessage("Producer: adding analysis task to queue {0}", analysisTask.Id);
// Push the frame onto the queue
_analysisTaskQueue.Add(analysisTask);
}
else
{
LogMessage("Producer: not analyzing frame");
}
LogMessage("Producer: iteration took {0} ms", (DateTime.Now - startTime).Milliseconds);
++frameCount;
}
LogMessage("Producer: stopping, destroy reader and timer");
_analysisTaskQueue.CompleteAdding();
// We reach this point by breaking out of the while loop. So we must be stopping.
_reader.Dispose();
_reader = null;
// Make sure the timer stops, then get rid of it.
var h = new ManualResetEvent(false);
_timer.Dispose(h);
h.WaitOne();
_timer = null;
LogMessage("Producer: stopped");
}, TaskCreationOptions.LongRunning);
_consumerTask = Task.Factory.StartNew(async() =>
{
while (!_analysisTaskQueue.IsCompleted)
{
LogMessage("Consumer: waiting for task to get added");
// Get the next processing task.
Task <NewResultEventArgs> nextTask = null;
// Blocks if m_analysisTaskQueue.Count == 0
// IOE means that Take() was called on a completed collection.
// Some other thread can call CompleteAdding after we pass the
// IsCompleted check but before we call Take.
// In this example, we can simply catch the exception since the
// loop will break on the next iteration.
// See https://msdn.microsoft.com/en-us/library/dd997371(v=vs.110).aspx
try
{
nextTask = _analysisTaskQueue.Take();
}
catch (InvalidOperationException) { }
if (nextTask != null)
{
// Block until the result becomes available.
LogMessage("Consumer: waiting for next result to arrive for task {0}", nextTask.Id);
var result = await nextTask;
// Raise the new result event.
LogMessage("Consumer: got result for frame {0}. {1} tasks in queue", result.Frame.Metadata.Index, _analysisTaskQueue.Count);
OnNewResultAvailable(result);
}
}
LogMessage("Consumer: stopped");
}, TaskCreationOptions.LongRunning);
// Set up a timer object that will trigger the frame-grab at a regular interval.
_timer = new Timer(async s /* state */ =>
{
await _timerMutex.WaitAsync();
try
{
// If the handle was not reset by the producer, then the frame-grab was missed.
bool missed = _frameGrabTimer.WaitOne(0);
_frameGrabTimer.Set();
if (missed)
{
LogMessage("Timer: missed frame-grab {0}", timerIterations - 1);
}
LogMessage("Timer: grab frame num {0}", timerIterations);
++timerIterations;
}
finally
{
_timerMutex.Release();
}
}, null, TimeSpan.Zero, frameGrabDelay);
OnProcessingStarted();
}
public LightThreadPool(int maxthreadcount = 256)
{
_maxThreadCount = maxthreadcount;
_actions = new BlockingCollection <Action>();
}
public Mailbox()
{
_messageQueue = new BlockingCollection <T>(1);
_responseQueue = new BlockingCollection <T>(1);
}
private void FillingSectors(int numberOfSector)
{
_logger.WriteProtocol("started", $"filling sectors,{numberOfSector}", $"{Task.CurrentId}");
TextBlock ref_textBlock = null;
ListBox ref_listbox = null;
BlockingCollection <Fan> ref_BagFillingTheSector = null;
switch (numberOfSector)
{
case 1:
{
ref_BagFillingTheSector = _bagFillingTheSector1;
ref_textBlock = tBlockBusyPlacesSecotr1;
ref_listbox = lBoxSector1;
break;
}
case 2:
{
ref_BagFillingTheSector = _bagFillingTheSector2;
ref_textBlock = tBlockBusyPlacesSecotr2;
ref_listbox = lBoxSector2;
break;
}
case 3:
{
ref_BagFillingTheSector = _bagFillingTheSector3;
ref_textBlock = tBlockBusyPlacesSecotr3;
ref_listbox = lBoxSector3;
break;
}
case 4:
{
ref_BagFillingTheSector = _bagFillingTheSector4;
ref_textBlock = tBlockBusyPlacesSecotr4;
ref_listbox = lBoxSector4;
break;
}
case 5:
{
ref_BagFillingTheSector = _bagFillingTheSector5;
ref_textBlock = tBlockBusyPlacesSecotr5;
ref_listbox = lBoxSector5;
break;
}
case 6:
{
ref_BagFillingTheSector = _bagFillingTheSector6;
ref_textBlock = tBlockBusyPlacesSecotr6;
ref_listbox = lBoxSector6;
break;
}
}
while (_queueInFrontOfTheSectors.IsCompleted != true &&
ref_BagFillingTheSector.Count != 10000)
{
Thread.Sleep(1);
Fan tmpFan = null;
_queueInFrontOfTheSectors.TryTake(out tmpFan, 100, _cts.Token);
if (tmpFan != null && tmpFan.NumberOfSector == numberOfSector)
{
ref_BagFillingTheSector.Add(tmpFan);
_context.Send( //Метод работает синхронно, текущий поток дождется его завершения
//Post будет работать асинхронно, текущий поток его не ждет
state =>
{
ref_textBlock.Text = ref_BagFillingTheSector.Count.ToString();
ref_listbox.Items.Add(tmpFan.NumberOfPlace.ToString());
tBlockQueueBeforeSectors.Text = (_queueInFrontOfTheSectors.Count.ToString());
tBlockFansOnTheirPlaces.Text = (_bagFillingTheSector1.Count +
_bagFillingTheSector2.Count +
_bagFillingTheSector3.Count +
_bagFillingTheSector4.Count +
_bagFillingTheSector5.Count +
_bagFillingTheSector6.Count).ToString();
},
null
);
}
else if (tmpFan != null)
{
_queueInFrontOfTheSectors.Add(tmpFan);
}
}
_logger.WriteProtocol("finished", $"filling sectors,{numberOfSector}", $"{Task.CurrentId}");
}
public PlayersMapping()
{
_playersCollection = new BlockingCollection <Player>();
_tasks = new List <Task>();
_players = new List <Player>();
}
//
//
//public DataTreatment(BlockingCollection<RawData> collection, BlockingCollection<RawData> graphCollection, BlockingCollection<RawData> filterCollection, IData iData, ConvertAlgo conv)
public DataTreatment(BlockingCollection <double> collection, BlockingCollection <double> graphCollection, BlockingCollection <double> filterCollection,
BlockingCollection <double> alarmCollection, ConvertAlgo conv, IData data, IAlarm alarm, UC1M1_ZeroAdjustment adj)
{
_collection = collection;
_graphCollection = graphCollection;
_filterCollection = filterCollection;
_alarmCollection = alarmCollection;
DataInterface = data;
//_filterCollection = filterCollection;
AlarmController = alarm;
AdjustmentController = adj;
//DataInterface = iData;
ConvertAlgorithm = conv;
FullList = new List <double>();
DownsampledRawList = new List <double>();
GraphList = new List <double>();
PulseList = new List <double>();
}
public RabbitMqClientFixture(ScenarioContext scenatioContext)
{
this.scenatioContext = scenatioContext;
collectionOfSignal = new BlockingCollection <JObject>();
}
public async Task SizeMixParallel()
{
var st = new StringTable(0);
ConcurrentDictionary <StringId, string> map = new ConcurrentDictionary <StringId, string>();
var sb = new StringBuilder(3000000);
var strings = new BlockingCollection <string>();
var stringIds = new BlockingCollection <StringId>();
var createStringsTask = Task.Run(() =>
{
// zip through small sizes
for (int i = 0; i < 3000; i++)
{
sb.Length = 0;
sb.Append('x', i);
string str = sb.ToString();
strings.Add(str);
}
// now use increasingly large amounts, including exceeding the size of a single buffer's worth
for (int i = 0; i < 100; i++)
{
sb.Length = 0;
sb.Append('x', i * 10000);
string str = sb.ToString();
}
strings.CompleteAdding();
});
var validateStringsTask = Task.Run(() =>
{
for (int i = 0; i < 2; i++)
{
// make sure all the right strings come out
int startBias = 0;
var buf = new char[4000000];
foreach (StringId sd in stringIds.GetConsumingEnumerable())
{
// get the actual string we kept
string str = map[sd];
// does the table report the right length?
int length = st.GetLength(sd);
XAssert.AreEqual(str.Length, length);
// put sentinel bytes in the char buffer, extract the string from the table, and check the sentinels
if (startBias > 0)
{
buf[startBias - 1] = (char)42;
}
buf[startBias + length] = (char)31415;
st.CopyString(sd, buf, startBias);
if (startBias > 0)
{
XAssert.AreEqual(42, buf[startBias - 1]);
}
XAssert.AreEqual(31415, buf[startBias + length]);
// make sure we got all the characters out that we should have
for (int j = 0; j < str.Length; j++)
{
XAssert.AreEqual(str[j], buf[startBias + j]);
}
startBias++;
}
// make sure the same behavior occurs after freezing
st.Freeze();
}
});
Parallel.ForEach(strings.GetConsumingEnumerable(), str =>
{
StringId sd = st.AddString(str);
map.TryAdd(sd, str);
stringIds.Add(sd);
});
stringIds.CompleteAdding();
await createStringsTask;
await validateStringsTask;
}
public static IIntCodeProgram New(BlockingCollection <IntCodeValue> input)
{
return(New(input, new BlockingCollection <IntCodeValue>()));
}
/// <summary>
/// Flushes all existing messages in the <see cref="BlockingCollection"/> before
/// continuing. This is useful before prompting for user input so that log messages are
/// written out before the user prompt text.
/// </summary>
protected void FlushBlockingCollection()
{
BlockingCollection.Add(new FlushMessage());
WaitHandle.WaitAny(new WaitHandle[] { _flushSemaphore, Token.WaitHandle });
}
public void SaveData(Stream stream, IDataView data, params int[] colIndices)
{
_host.CheckValue(stream, nameof(stream));
_host.CheckValue(data, nameof(data));
_host.CheckValueOrNull(colIndices);
_host.CheckParam(stream.CanWrite, nameof(stream), "cannot save to non-writable stream");
_host.CheckParam(stream.CanSeek, nameof(stream), "cannot save to non-seekable stream");
_host.CheckParam(stream.Position == 0, nameof(stream), "stream must be positioned at head of stream");
using (IChannel ch = _host.Start("Saving"))
using (ExceptionMarshaller exMarshaller = new ExceptionMarshaller())
{
var toWrite = new BlockingCollection <Block>(16);
var toCompress = new BlockingCollection <Block>(16);
var activeColumns = GetActiveColumns(data.Schema, colIndices);
int rowsPerBlock = RowsPerBlockHeuristic(data, activeColumns);
ch.Assert(rowsPerBlock > 0);
Stopwatch sw = new Stopwatch();
// Set up the compression and write workers that consume the input information first.
Task compressionTask = null;
if (activeColumns.Length > 0)
{
OrderedWaiter waiter = _deterministicBlockOrder ? new OrderedWaiter() : null;
Task[] compressionThreads = new Task[Environment.ProcessorCount];
for (int i = 0; i < compressionThreads.Length; ++i)
{
compressionThreads[i] = Utils.RunOnBackgroundThread(
() => CompressionWorker(toCompress, toWrite, activeColumns.Length, waiter, exMarshaller));
}
compressionTask = Task.WhenAll(compressionThreads);
}
// While there is an advantage to putting the IO into a separate thread, there is not an
// advantage to having more than one worker.
Task writeThread = Utils.RunOnBackgroundThread(
() => WriteWorker(stream, toWrite, activeColumns, data.Schema, rowsPerBlock, _host, exMarshaller));
sw.Start();
// REVIEW: For now the fetch worker just works in the main thread. If it's
// a fairly large view through, it may be advantageous to consider breaking up the
// fetchwrite operations on the pipes, somehow.
// Despite running in the main thread for now, the fetch worker follows the same
// pattern of utilizing exMarshaller.
using (var pch = _silent ? null : _host.StartProgressChannel("BinarySaver"))
{
FetchWorker(toCompress, data, activeColumns, rowsPerBlock, sw, ch, pch, exMarshaller);
}
_host.Assert(compressionTask != null || toCompress.IsCompleted);
if (compressionTask != null)
{
compressionTask.Wait();
}
toWrite.CompleteAdding();
writeThread.Wait();
exMarshaller.ThrowIfSet(ch);
if (!_silent)
{
ch.Info("Wrote {0} rows across {1} columns in {2}", _rowCount, activeColumns.Length, sw.Elapsed);
}
// When we dispose the exception marshaller, this will set the cancellation token when we internally
// dispose the cancellation token source, so one way or another those threads are being cancelled, even
// if an exception is thrown in the main body of this function.
}
}
public List <TerrainPatch> ExportTemperatureProfiles()
{
_precache = new Dictionary <Point, bool>();
_cache = new Dictionary <Point, TerrainPatch>();
var sieglers = ReadSieglerProduct2().ToList();
Console.WriteLine($"There are {sieglers.Count} lines in the siegler product.");
// Filter for sieglers near 12420,17549
var center = new Point(12420, 17549);
sieglers = sieglers.Where(s => Distance(s.Pixel, center) < 30d).ToList();
Console.WriteLine($"There are {sieglers.Count} pixels close to the test center");
//var sieglers2 = sieglers.Where(s => s.Distance < d).ToList();
//Console.WriteLine($"filtered by distance {d} count={sieglers2.Count} fraction={sieglers2.Count / (double)sieglers.Count}");
//sieglers = sieglers.Where(s=>s.Depth<=1f && s.Depth>=0f).ToList();
var group1 = sieglers.GroupBy(s => DepthEquivalenceClass(s.Depth)).ToList();
var sieglers2 = new List <Siegler>();
foreach (var g in group1)
{
sieglers2.AddRange(g.Take(10));
}
Console.WriteLine($"bunched by percent [0-1] count={sieglers2.Count} fraction={sieglers2.Count / (double)sieglers.Count}");
var groups = sieglers2.GroupBy(s => s.Pixel).ToList();
Console.WriteLine($"There are {groups.Count} grouped pixels within the map");
var consolidated = groups.Select(g => new ConsolidatedLine(g.ToList())).ToList();
consolidated = consolidated.Take(100).ToList(); // smaller set for now
var filtered = consolidated.Where(HorizonsHaveBeenCalculated).ToList();
Console.WriteLine($"There are {filtered.Count} pixels with horizons already calculated");
Console.WriteLine($"There are {_cache.Count} tiles involved.");
Console.WriteLine($"Beginning to load patches");
Parallel.ForEach(_cache.Values, patch => GetPatch(patch.Id));
Console.WriteLine($"Finished loading patches");
var expected_count = consolidated.Count;
var count = 0;
var stopwatch = new Stopwatch();
var starttime = DateTime.Now;
stopwatch.Start();
var queue = new BlockingCollection <ConsolidatedLine>(10000);
Task.Run(() =>
{
Parallel.ForEach(consolidated, new ParallelOptions {
MaxDegreeOfParallelism = Math.Max(Environment.ProcessorCount - 1, 1)
}, c =>
{
var r = DoCalculation(c);
if (r)
{
queue.Add(c);
Console.WriteLine(@"[{0:D4},{1:D4}].Lighting.Max()={2:F8} Depth={3:F8}", c.Pixel.X, c.Pixel.Y, c.Lighting.Max(), c.Depth);
}
else
{
Console.WriteLine($"Calculation failed for [{c.Pixel.X},{c.Pixel.Y}]");
}
});
queue.CompleteAdding();
});
using (var fs = File.Create(LightingCacheFilename, 4096, FileOptions.None))
using (var bw = new BinaryWriter(fs))
foreach (var c in queue.GetConsumingEnumerable())
{
c.Write(bw);
Math.DivRem(count++, 100, out int rem);
if (rem == 0)
{
var ticks_per_solution = stopwatch.Elapsed.Ticks / count;
var timespan_per_solution = new TimeSpan(ticks_per_solution);
var estimated_elapsed_ticks = ticks_per_solution * expected_count;
var estimated_stop = starttime.AddTicks(estimated_elapsed_ticks);
Console.WriteLine($"Processed {count} pixels. {timespan_per_solution} per solution. Est stop={estimated_stop}");
}
}
stopwatch.Stop();
Console.WriteLine($"The lighting calculations took {stopwatch.Elapsed}");
return(_cache.Values.ToList());
}
private void WriteWorker(Stream stream, BlockingCollection <Block> toWrite, ColumnCodec[] activeColumns,
DataViewSchema sourceSchema, int rowsPerBlock, IChannelProvider cp, ExceptionMarshaller exMarshaller)
{
_host.AssertValue(exMarshaller);
try
{
_host.AssertValue(cp);
cp.AssertValue(stream);
cp.AssertValue(toWrite);
cp.AssertValue(activeColumns);
cp.AssertValue(sourceSchema);
cp.Assert(rowsPerBlock > 0);
using (IChannel ch = cp.Start("Write"))
{
var blockLookups = new List <BlockLookup> [activeColumns.Length];
for (int c = 0; c < blockLookups.Length; ++c)
{
blockLookups[c] = new List <BlockLookup>();
}
var deadLookups = new int[activeColumns.Length];
// Reserve space for the header at the start. This will be filled
// in with valid values once writing has completed.
ch.CheckIO(stream.Position == 0);
stream.Write(new byte[Header.HeaderSize], 0, Header.HeaderSize);
ch.CheckIO(stream.Position == Header.HeaderSize);
long expectedPosition = stream.Position;
BlockLookup deadLookup = new BlockLookup();
foreach (Block block in toWrite.GetConsumingEnumerable(exMarshaller.Token))
{
ch.CheckIO(stream.Position == expectedPosition);
MemoryStream compressed = block.BlockData;
ArraySegment <byte> buffer;
bool tmp = compressed.TryGetBuffer(out buffer);
ch.Assert(tmp);
stream.Write(buffer.Array, buffer.Offset, buffer.Count);
BlockLookup currLookup = new BlockLookup(expectedPosition, (int)compressed.Length, block.UncompressedLength);
expectedPosition += compressed.Length;
_memPool.Return(ref compressed);
ch.CheckIO(stream.Position == expectedPosition);
// Record the position. We have this "lookups" list per column. Yet, it may be that sometimes
// the writer receives things out of order.
// REVIEW: The format and the rest of the pipeline supposedly supports a long number
// of blocks, but the writing scheme does not yet support that.
int blockIndex = (int)block.BlockIndex;
var lookups = blockLookups[block.ColumnIndex];
if (lookups.Count == block.BlockIndex) // Received in order.
{
lookups.Add(currLookup);
}
else if (lookups.Count < block.BlockIndex) // Received a block a little bit early.
{
// Add a bunch of dead filler lookups, until these late blocks come in.
int deadToAdd = (int)block.BlockIndex - lookups.Count;
for (int i = 0; i < deadToAdd; ++i)
{
lookups.Add(deadLookup);
}
deadLookups[block.ColumnIndex] += deadToAdd;
ch.Assert(lookups.Count == block.BlockIndex);
lookups.Add(currLookup);
}
else // Received a block a little bit late.
{
// This should be a dead block unless the compressors are buggy and somehow
// yielding duplicate blocks or something.
ch.Assert(lookups[blockIndex].BlockOffset == 0);
deadLookups[block.ColumnIndex]--;
lookups[blockIndex] = currLookup;
}
}
// We have finished writing all blocks. We will now write the block lookup tables (so we can
// find the blocks), the slot names (for any columns that have them), the column table of
// contents (so we know how to decode the blocks, and where the lookups and names are),
// and the header (so we know dataview wide information and where to find the table of
// contents) in that order.
long[] lookupOffsets = new long[blockLookups.Length];
using (BinaryWriter writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen: true))
{
// Write the block lookup directories. These are referenced from the table of contents,
// so that someone knows where to look for some block data.
for (int c = 0; c < blockLookups.Length; ++c)
{
ch.Assert(deadLookups[c] == 0);
// The block lookup directories are written uncompressed and in fixed length
// to enable rapid seeking.
lookupOffsets[c] = stream.Position;
foreach (BlockLookup lookup in blockLookups[c])
{
// *** Lookup table entry format ***
// long: Offset to the start of a block
// int: Byte length of block as written
// int: Byte length of block when uncompressed
ch.Assert(lookup.BlockOffset > 0);
writer.Write(lookup.BlockOffset);
writer.Write(lookup.BlockLength);
writer.Write(lookup.DecompressedBlockLength);
}
ch.CheckIO(stream.Position == lookupOffsets[c] + (16 * blockLookups[c].Count),
"unexpected offsets after block lookup table write");
}
// Write the metadata for each column.
long[] metadataTocOffsets = new long[activeColumns.Length];
for (int c = 0; c < activeColumns.Length; ++c)
{
metadataTocOffsets[c] = WriteMetadata(writer, sourceSchema, activeColumns[c].SourceIndex, ch);
}
// Write the table of contents.
long tocOffset = stream.Position;
{
int c = 0;
expectedPosition = stream.Position;
foreach (var active in activeColumns)
{
// *** Column TOC entry format ***
// string: column name
// codec (as interpretable by CodecFactory.TryGetCodec): column block codec
// CompressionKind(byte): block compression strategy
// LEB128 int: Rows per block
// long: Offset to the start of the lookup table
// long: Offset to the start of the metadata TOC entries, or 0 if this has no metadata
string name = sourceSchema[active.SourceIndex].Name;
writer.Write(name);
int nameLen = Encoding.UTF8.GetByteCount(name);
expectedPosition += Utils.Leb128IntLength((uint)nameLen) + nameLen;
ch.CheckIO(stream.Position == expectedPosition, "unexpected offsets after table of contents name");
expectedPosition += _factory.WriteCodec(stream, active.Codec);
ch.CheckIO(stream.Position == expectedPosition, "unexpected offsets after table of contents type description");
writer.Write((byte)_compression);
expectedPosition++;
// REVIEW: Right now the number of rows per block is fixed, so we
// write the same value each time. In some future state, it may be that this
// is relaxed, with possibly some tradeoffs (for example, inability to randomly seek).
writer.WriteLeb128Int((ulong)rowsPerBlock);
expectedPosition += Utils.Leb128IntLength((uint)rowsPerBlock);
// Offset of the lookup table.
writer.Write(lookupOffsets[c]);
expectedPosition += sizeof(long);
// Offset of the metadata table of contents.
writer.Write(metadataTocOffsets[c]);
expectedPosition += sizeof(long);
ch.CheckIO(stream.Position == expectedPosition, "unexpected offsets after table of contents");
c++;
}
}
// Write the tail signature.
long tailOffset = stream.Position;
writer.Write(Header.TailSignatureValue);
// Now move back to the beginning of the stream, and write out the now completed header.
Header header = new Header()
{
Signature = Header.SignatureValue,
Version = Header.WriterVersion,
CompatibleVersion = Header.CanBeReadByVersion,
TableOfContentsOffset = tocOffset,
TailOffset = tailOffset,
RowCount = _rowCount,
ColumnCount = activeColumns.Length
};
byte[] headerBytes = new byte[Header.HeaderSize];
unsafe
{
Marshal.Copy(new IntPtr(&header), headerBytes, 0, Marshal.SizeOf(typeof(Header)));
}
writer.Seek(0, SeekOrigin.Begin);
writer.Write(headerBytes);
}
}
}
catch (Exception ex)
{
exMarshaller.Set("writing", ex);
}
}
private void FetchWorker(BlockingCollection <Block> toCompress, IDataView data,
ColumnCodec[] activeColumns, int rowsPerBlock, Stopwatch sw, IChannel ch, IProgressChannel pch, ExceptionMarshaller exMarshaller)
{
Contracts.AssertValue(ch);
Contracts.AssertValueOrNull(pch);
ch.AssertValue(exMarshaller);
try
{
ch.AssertValue(toCompress);
ch.AssertValue(data);
ch.AssertValue(activeColumns);
ch.AssertValue(sw);
ch.Assert(rowsPerBlock > 0);
// The main thread handles fetching from the cursor, and storing it into blocks passed to toCompress.
HashSet <int> activeSet = new HashSet <int>(activeColumns.Select(col => col.SourceIndex));
long blockIndex = 0;
int remainingInBlock = rowsPerBlock;
using (DataViewRowCursor cursor = data.GetRowCursor(data.Schema.Where(c => activeSet.Contains(c.Index))))
{
WritePipe[] pipes = new WritePipe[activeColumns.Length];
for (int c = 0; c < activeColumns.Length; ++c)
{
pipes[c] = WritePipe.Create(this, cursor, activeColumns[c]);
}
for (int c = 0; c < pipes.Length; ++c)
{
pipes[c].BeginBlock();
}
long rows = 0;
if (pch != null)
{
pch.SetHeader(new ProgressHeader(new[] { "rows" }), e => e.SetProgress(0, rows));
}
while (cursor.MoveNext())
{
for (int c = 0; c < pipes.Length; ++c)
{
pipes[c].FetchAndWrite();
}
if (--remainingInBlock == 0)
{
for (int c = 0; c < pipes.Length; ++c)
{
// REVIEW: It may be better if EndBlock got moved to a different worker thread.
toCompress.Add(new Block(pipes[c].EndBlock(), c, blockIndex), exMarshaller.Token);
pipes[c].BeginBlock();
}
remainingInBlock = rowsPerBlock;
blockIndex++;
}
rows++;
}
if (remainingInBlock < rowsPerBlock)
{
for (int c = 0; c < pipes.Length; ++c)
{
toCompress.Add(new Block(pipes[c].EndBlock(), c, blockIndex), exMarshaller.Token);
}
}
Contracts.Assert(rows == (blockIndex + 1) * rowsPerBlock - remainingInBlock);
_rowCount = rows;
if (pch != null)
{
pch.Checkpoint(rows);
}
}
toCompress.CompleteAdding();
}
catch (Exception ex)
{
exMarshaller.Set("cursoring", ex);
}
}
public CompletionPort()
{
m_queue = new BlockingCollection <CompletionStatus>();
}
public BoundedMessageQueue()
{
_queue = new BlockingCollection <Envelope>();
}
public LoggerFactory(BlockingCollection <LogMessage> pendingMessages)
{
_pendingMessages = pendingMessages;
}
private void IndexBalances(ChainBase chain, string checkpointName, Func <uint256, Transaction, uint256, BlockHeader, int, IEnumerable <OrderedBalanceChange> > extract)
{
SetThrottling();
BlockingCollection <OrderedBalanceChange[]> indexedEntries = new BlockingCollection <OrderedBalanceChange[]>(100);
var tasks = CreateTaskPool(indexedEntries, (entries) => Index(entries.Select(e => e.ToEntity()), this.Configuration.GetBalanceTable()), 30);
using (IndexerTrace.NewCorrelation("Import balances " + checkpointName + " to azure started").Open())
{
this.Configuration.GetBalanceTable().CreateIfNotExists();
var buckets = new MultiValueDictionary <string, OrderedBalanceChange>();
using (var storedBlocks = Enumerate(checkpointName, chain))
{
foreach (var block in storedBlocks)
{
foreach (var tx in block.Block.Transactions)
{
var txId = tx.GetHash();
try
{
var entries = extract(txId, tx, block.BlockId, block.Block.Header, block.Height);
foreach (var entry in entries)
{
buckets.Add(entry.PartitionKey, entry);
var bucket = buckets[entry.PartitionKey];
if (bucket.Count == 100)
{
indexedEntries.Add(bucket.ToArray());
buckets.Remove(entry.PartitionKey);
}
}
if (storedBlocks.NeedSave)
{
foreach (var kv in buckets.AsLookup().ToArray())
{
indexedEntries.Add(kv.ToArray());
}
buckets.Clear();
tasks.Stop();
storedBlocks.SaveCheckpoint();
tasks.Start();
}
}
catch (Exception ex)
{
IndexerTrace.ErrorWhileImportingBalancesToAzure(ex, txId);
throw;
}
}
}
foreach (var kv in buckets.AsLookup().ToArray())
{
indexedEntries.Add(kv.ToArray());
}
tasks.Stop();
storedBlocks.SaveCheckpoint();
}
}
}
public UdpThreadInfos(BlockingCollection <T> dataTransferer, int port)
{
DataTransferer = dataTransferer;
Port = port;
}
public void ShouldStreamAllDocumentsAfterSubscriptionCreation()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new User {
Age = 31
}, "users/1");
session.Store(new User {
Age = 27
}, "users/12");
session.Store(new User {
Age = 25
}, "users/3");
session.SaveChanges();
}
var id = store.Subscriptions.Create <User>();
using (var subscription = store.Subscriptions.GetSubscriptionWorker <User>(new SubscriptionWorkerOptions(id)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
}))
{
var keys = new BlockingCollection <string>();
var ages = new BlockingCollection <int>();
subscription.Run(batch =>
{
batch.Items.ForEach(x => keys.Add(x.Id));
batch.Items.ForEach(x => ages.Add(x.Result.Age));
});
Assert.True(keys.TryTake(out string key, _reasonableWaitTime));
Assert.Equal("users/1", key);
Assert.True(keys.TryTake(out key, _reasonableWaitTime));
Assert.Equal("users/12", key);
Assert.True(keys.TryTake(out key, _reasonableWaitTime));
Assert.Equal("users/3", key);
Assert.True(ages.TryTake(out int age, _reasonableWaitTime));
Assert.Equal(31, age);
Assert.True(ages.TryTake(out age, _reasonableWaitTime));
Assert.Equal(27, age);
Assert.True(ages.TryTake(out age, _reasonableWaitTime));
Assert.Equal(25, age);
var expectedSendBufferSize = SubscriptionWorkerOptions.DefaultSendBufferSizeInBytes;
var expectedReceiveBufferSize = SubscriptionWorkerOptions.DefaultReceiveBufferSizeInBytes;
if (PlatformDetails.RunningOnLinux)
{
// linux is doubling that value by design
expectedSendBufferSize *= 2;
expectedReceiveBufferSize *= 2;
}
Assert.Equal(expectedSendBufferSize, subscription._tcpClient.SendBufferSize);
Assert.Equal(expectedReceiveBufferSize, subscription._tcpClient.ReceiveBufferSize);
}
}
}
