public IConnectionMultiplexer GetConnection()
{
try
{
var isValueCreatedCount = Pool.Count(lazy => lazy.IsValueCreated);
if (isValueCreatedCount == this.Pool.Count)
{
return(Pool.OrderBy(x => x.Value.TotalOutstanding()).First().Value.Connection);
}
foreach (var item in Pool)
{
if (!item.IsValueCreated)
{
return(item.Value.Connection);
}
if (item.Value.TotalOutstanding() == 0)
{
return(item.Value.Connection);
}
}
return(this.Pool.First().Value.Connection);
}
catch
{
return(null);
}
}
public int GetCacheHitCount()
{
var count = _itemsQueue.Count(item => item.EventId == CacheableEventId.CacheHit);
_itemsQueue.Clear();
return(count);
}
public void ValidateLimitOrders()
{
var orderEventTracker = new ConcurrentBag <OrderEvent>();
var oanda = (OandaBrokerage)Brokerage;
var symbol = Symbol;
var quote = oanda.GetRates(new OandaSymbolMapper().GetBrokerageSymbol(symbol));
EventHandler <OrderEvent> orderStatusChangedCallback = (s, e) => {
orderEventTracker.Add(e);
};
oanda.OrderStatusChanged += orderStatusChangedCallback;
// Buy Limit order below market
var limitPrice = quote.BidPrice - 0.5m;
var order = new LimitOrder(symbol, 1, limitPrice, DateTime.Now);
OrderProvider.Add(order);
Assert.IsTrue(oanda.PlaceOrder(order));
// update Buy Limit order with no changes
Assert.IsTrue(oanda.UpdateOrder(order));
// move Buy Limit order above market
order.LimitPrice = quote.AskPrice + 0.5m;
Assert.IsTrue(oanda.UpdateOrder(order));
oanda.OrderStatusChanged -= orderStatusChangedCallback;
Assert.AreEqual(orderEventTracker.Count(x => x.Status == OrderStatus.Submitted), 1);
Assert.AreEqual(orderEventTracker.Count(x => x.Status == OrderStatus.Filled), 1);
}
public void VerifyDeduplicationMultithreaded()
{
var client = CreateClient();
// TODO: We need to look into why the ErrorPlugin causes data to sometimes calculate invalid hashcodes
var errorPlugin = new SimpleErrorPlugin();
var contexts = new ConcurrentBag <EventPluginContext>();
using (var duplicateCheckerPlugin = new DuplicateCheckerPlugin(TimeSpan.FromSeconds(1))) {
var result = Parallel.For(0, 10, index => {
var builder = GetException().ToExceptionless();
var context = new EventPluginContext(client, builder.Target, builder.PluginContextData);
contexts.Add(context);
errorPlugin.Run(context);
duplicateCheckerPlugin.Run(context);
});
while (!result.IsCompleted)
{
Thread.Yield();
}
}
Assert.Equal(1, contexts.Count(c => !c.Cancel));
Assert.Equal(9, contexts.Count(c => c.Cancel));
Assert.Equal(9, contexts.Sum(c => c.Event.Count.GetValueOrDefault()));
}
/// <summary>
/// use ConcurrectCollection
/// </summary>
public void Run4()
{
var products = GetProducts();
ConcurrentBag <NewProduct> newProducts = new ConcurrentBag <NewProduct>();
Console.WriteLine("來源資料筆數:{0}", products.Count());
Console.WriteLine("執行前結果資料筆數:{0}", newProducts.Count());
Parallel.ForEach(products, x =>
{
int id = x.Id;
string name = x.Name;
int stock = 100 + x.Id;
var newProduct = new NewProduct()
{
Id = id,
Name = name,
Stock = stock
};
newProducts.Add(newProduct);
});
Console.WriteLine("執行後結果資料筆數:{0}", newProducts.Count());
}
public async Task ReportsExceptionsWithReferenceTypes()
{
var inputValues = AsyncEnumerableProvider.GetObjects(100);
Func <DummyClass, Task <DummyClass> > action = async(DummyClass input) =>
{
await Task.Delay(10);
if (input.MyInt % 10 == 0)
{
throw new Exception("We don't like 10s");
}
return(new DummyClass(input.MyInt * 2));
};
var results = new ConcurrentBag <Result <DummyClass, DummyClass> >();
await foreach (var result in inputValues.SafeParallelAsyncWithResult(action))
{
results.Add(result);
}
results.Count().ShouldBe(100);
results.Count(r => r.Success).ShouldBe(90);
results.Count(r => !r.Success).ShouldBe(10);
results.First(r => !r.Success).Exception.ShouldNotBeNull();
results.First(r => !r.Success).Exception.Message.ShouldBe("We don't like 10s");
results.Select(r => r.Input.MyInt).ShouldBe(Enumerable.Range(1, 100), true);
results.First(r => !r.Success).Output.ShouldBe(null);
}
public decimal PValue(decimal logFoldChange, List <BiorepIntensity> allLights, List <BiorepIntensity> allHeavies)
{
if (allLights.Count != allHeavies.Count)
{
throw new ArgumentException("Error: Imputation has gone awry. Each biorep should have the same number of biorep intensities for NeuCode light and heavy at this point.");
}
int maxPermutations = 10000;
ConcurrentBag <decimal> permutedRatios = new ConcurrentBag <decimal>();
Parallel.For(0, maxPermutations, i =>
{
List <double> combined = allLights.Select(j => j.intensity).Concat(allHeavies.Select(j => j.intensity)).ToList();
combined.Shuffle();
double numerator = combined.Take(allLights.Count).Sum();
double denominator = combined.Skip(allLights.Count).Take(allHeavies.Count).Sum();
decimal someRatio = (decimal)Math.Log(numerator / denominator, 2);
permutedRatios.Add(someRatio);
});
decimal pValue = logFoldChange > 0 ?
(decimal)(1M / maxPermutations) + (decimal)permutedRatios.Count(x => x > logFoldChange) / (decimal)permutedRatios.Count : //adding a slight positive shift so that later logarithms don't produce fault
(decimal)(1M / maxPermutations) + (decimal)permutedRatios.Count(x => x < logFoldChange) / (decimal)permutedRatios.Count; //adding a slight positive shift so that later logarithms don't produce fault
return(pValue);
}
private static void TestCollection()
{
Console.WriteLine("Start to testing with generic collection");
SortedList <string, string> sortedListString = new SortedList <string, string>()
{
{ "hello", "hi" },
{ "hello1", "hi1" },
{ "hello2", "hi2" }
};
Console.WriteLine(sortedListString.GetValueOrDefault("hello"));
//Concurrent
ISysLogin sys = new LoginSystem();
int i = 0;
ConcurrentBag <bool> results = new ConcurrentBag <bool>();
ParallelLoopResult loopPar = Parallel.For(0, 100000, (j) => {
results.Add(sys.Register("Username" + i, "Password"));
i++;
});
while (!loopPar.IsCompleted)
{
}
Console.WriteLine("Number of Success process: {0}", results.Count(a => a));
Console.WriteLine("Number of failed process: {0}", results.Count(a => !a));
Console.WriteLine("Number of process: {0}", results.Count);
}
public void ValidateLimitOrders()
{
var orderEventTracker = new ConcurrentBag <OrderEvent>();
var alpaca = (AlpacaBrokerage)Brokerage;
var symbol = Symbol;
var lastPrice = GetLastPrice(symbol.Value);
EventHandler <OrderEvent> orderStatusChangedCallback = (s, e) =>
{
orderEventTracker.Add(e);
};
alpaca.OrderStatusChanged += orderStatusChangedCallback;
// Buy Limit order above market - should be filled immediately
var limitPrice = lastPrice + 0.5m;
var order = new LimitOrder(symbol, 1, limitPrice, DateTime.UtcNow);
OrderProvider.Add(order);
Assert.IsTrue(alpaca.PlaceOrder(order));
Thread.Sleep(10000);
alpaca.OrderStatusChanged -= orderStatusChangedCallback;
Assert.AreEqual(orderEventTracker.Count(x => x.Status == OrderStatus.Submitted), 1);
Assert.AreEqual(orderEventTracker.Count(x => x.Status == OrderStatus.Filled), 1);
}
private static void ShowStats(SynchedColouredConsoleTraceWriter synchedConsole)
{
synchedConsole.WriteLine($@"
Following results & performance based on a {100.0 / _perTestMeasureEveryN:#0.0}% sample ({MonitoredWorkBag.Count:#,##0}) + all timeouts & faulted requests.
Results & performance of sample...
Total: {MonitoredWorkBag.Count:#,##0},
Completed: {MonitoredWorkBag.Count(t => t.Item1.IsCompleted):#,##0},
Faulted: {MonitoredWorkBag.Count(t => t.Item1.IsFaulted):#,##0},
Cancelled: {MonitoredWorkBag.Count(t => t.Item1.IsCanceled):#,##0},
Performance (microseconds [us] - thousandaths of milliseconds / millionths of seconds)
Fastest: {MonitoredWorkBag.Min(t => t.Item2.TotalMilliseconds * 1000):#,##0.0} (us),
Slowest: {MonitoredWorkBag.Max(t => t.Item2.TotalMilliseconds * 1000):#,##0.0} (us),
Average time: {MonitoredWorkBag.Average(t => t.Item2.TotalMilliseconds * 1000):#,##0.0} (us),
Median: {Percentile(MonitoredWorkBag.Select(t => t.Item2.TotalMilliseconds * 1000), 0.50):#,##0.0} (us),
P75: {Percentile(MonitoredWorkBag.Select(t => t.Item2.TotalMilliseconds * 1000), 0.75):#,##0.0} (us),
P90: {Percentile(MonitoredWorkBag.Select(t => t.Item2.TotalMilliseconds * 1000), 0.90):#,##0.0} (us),
P95: {Percentile(MonitoredWorkBag.Select(t => t.Item2.TotalMilliseconds * 1000), 0.95):#,##0.0} (us),
P99: {Percentile(MonitoredWorkBag.Select(t => t.Item2.TotalMilliseconds * 1000), 0.99):#,##0.0} (us),
P3x9: {Percentile(MonitoredWorkBag.Select(t => t.Item2.TotalMilliseconds * 1000), 0.999):#,##0.0} (us),
P5x9: {Percentile(MonitoredWorkBag.Select(t => t.Item2.TotalMilliseconds * 1000), 0.99999):#,##0.0} (us),
", overrideShowOutput: true);
}
public void ValidateMarketOrders()
{
var orderEventTracker = new ConcurrentBag <OrderEvent>();
var alpaca = (AlpacaBrokerage)Brokerage;
var symbol = Symbol;
EventHandler <OrderEvent> orderStatusChangedCallback = (s, e) =>
{
orderEventTracker.Add(e);
};
alpaca.OrderStatusChanged += orderStatusChangedCallback;
const int numberOfOrders = 2;
for (var i = 0; i < numberOfOrders; i++)
{
var order = new MarketOrder(symbol, 10, DateTime.UtcNow);
OrderProvider.Add(order);
Console.WriteLine("Buy Order");
alpaca.PlaceOrder(order);
var orderr = new MarketOrder(symbol, -10, DateTime.UtcNow);
OrderProvider.Add(orderr);
Console.WriteLine("Sell Order");
alpaca.PlaceOrder(orderr);
}
// We want to verify the number of order events with OrderStatus.Filled sent
Thread.Sleep(14000);
alpaca.OrderStatusChanged -= orderStatusChangedCallback;
Assert.AreEqual(orderEventTracker.Count(x => x.Status == OrderStatus.Submitted), numberOfOrders * 2);
Assert.AreEqual(orderEventTracker.Count(x => x.Status == OrderStatus.Filled), numberOfOrders * 2);
}
public void Publish_1000_Messages_To_A_Queue_Using_Custom_Exchange()
{
var _busConfigurationBuilder = new BusConfigurationBuilder(Accessories.Configuration.ConnectionSettingsString);
_busConfigurationBuilder
.RegisterPublication <Accessories.MyEvent>(Accessories.Configuration.ExchangeName1, typeof(Accessories.MyEvent).Name, MessageDeliveryMode.Persistent, message => Accessories.Configuration.QueueName1);
var _SUT = new Bus(_busConfigurationBuilder.Build());
_SUT.Connect();
var _tasks = new ConcurrentBag <Task <PMCG.Messaging.PublicationResult> >();
for (int count = 0; count < 1000; count++)
{
var _message = new Accessories.MyEvent(Guid.NewGuid(), "", "R1", 1, "09:00", "DDD....");
_tasks.Add(_SUT.PublishAsync(_message));
}
Task.WhenAll(_tasks).ContinueWith(a =>
{
var _taskStatusCount = _tasks.Count(result => result.Status == TaskStatus.RanToCompletion);
var _messagePublishedCount = _tasks.Count(result => result.Result.Status == Messaging.PublicationResultStatus.Published);
Console.WriteLine(string.Format("RanToCompletionTaskCount expected: (1000), actual: ({0})", _taskStatusCount));
Console.WriteLine(string.Format("MessagePublishedCount expected: (1000), actual: ({0})", _messagePublishedCount));
Console.WriteLine("Verify that 1000 messages on queue on management ui");
});
Console.Read();
}
public void ValidateMarketOrders()
{
var orderEventTracker = new ConcurrentBag <OrderEvent>();
var oanda = (OandaBrokerage)Brokerage;
var symbol = Symbol;
EventHandler <OrderEvent> orderStatusChangedCallback = (s, e) => {
orderEventTracker.Add(e);
};
oanda.OrderStatusChanged += orderStatusChangedCallback;
const int numberOfOrders = 100;
Parallel.For(0, numberOfOrders, (i) =>
{
var order = new MarketOrder(symbol, 100, DateTime.Now);
OrderProvider.Add(order);
Assert.IsTrue(oanda.PlaceOrder(order));
Assert.IsTrue(order.Status == OrderStatus.Filled || order.Status == OrderStatus.PartiallyFilled);
var orderr = new MarketOrder(symbol, -100, DateTime.UtcNow);
OrderProvider.Add(orderr);
Assert.IsTrue(oanda.PlaceOrder(orderr));
Assert.IsTrue(orderr.Status == OrderStatus.Filled || orderr.Status == OrderStatus.PartiallyFilled);
});
// We want to verify the number of order events with OrderStatus.Filled sent
Thread.Sleep(4000);
oanda.OrderStatusChanged -= orderStatusChangedCallback;
Assert.AreEqual(orderEventTracker.Count(x => x.Status == OrderStatus.Submitted), numberOfOrders * 2);
Assert.AreEqual(orderEventTracker.Count(x => x.Status == OrderStatus.Filled), numberOfOrders * 2);
}
public async Task ReportsExceptions()
{
var inputValues = Enumerable.Range(1, 100);
Func <int, Task> action = async(int i) =>
{
await Task.Delay(10);
if (i % 10 == 0)
{
throw new Exception("We don't like 10s");
}
};
ConcurrentBag <Result <int> > results = new ConcurrentBag <Result <int> >();
await foreach (var result in inputValues.SafeParallelAsyncWithResult(action))
{
results.Add(result);
}
results.Count().ShouldBe(100);
results.Count(r => r.Success).ShouldBe(90);
results.Count(r => !r.Success).ShouldBe(10);
results.First(r => !r.Success).Exception.ShouldNotBeNull();
results.First(r => !r.Success).Exception.Message.ShouldBe("We don't like 10s");
results.Select(r => r.Input).ShouldBe(inputValues, true);
}
public void VerifyDeduplicationMultithreaded()
{
var client = CreateClient();
var errorPlugin = new ErrorPlugin();
var contexts = new ConcurrentBag <EventPluginContext>();
using (var duplicateCheckerPlugin = new DuplicateCheckerPlugin(TimeSpan.FromMilliseconds(100))) {
var result = Parallel.For(0, 10, index => {
var builder = GetException().ToExceptionless();
var context = new EventPluginContext(client, builder.Target, builder.PluginContextData);
contexts.Add(context);
errorPlugin.Run(context);
duplicateCheckerPlugin.Run(context);
});
while (!result.IsCompleted)
{
Thread.Sleep(1);
}
}
Thread.Sleep(150);
Assert.Equal(1, contexts.Count(c => !c.Cancel));
Assert.Equal(9, contexts.Count(c => c.Cancel));
Assert.Equal(9, contexts.Sum(c => c.Event.Count.GetValueOrDefault()));
}
public static List <ResultSummary> GetResultSummaryList(int suiteId, string suiteName)
{
var resDetail = new ConcurrentBag <ResultSummary>();
Stp.Restart();
MtmInteraction.ClearPerformanceCounters();
MtmInteraction.GetResultDetails(suiteId, resDetail, suiteName);
Stp.Stop();
Diagnostic.AppendLine("Fetch Suite data for " + suiteName + "-" + suiteId + ": " +
Stp.Elapsed.TotalSeconds);
Diagnostic.AppendLine("Outcometime: " + MtmInteraction.OutcomeTime);
Diagnostic.AppendLine("Priority Time: " + MtmInteraction.PriorityTime);
Diagnostic.AppendLine("Tester Time: " + MtmInteraction.TesterTime);
Diagnostic.AppendLine("Title Time: " + MtmInteraction.TitleTime);
Diagnostic.AppendLine("Initialize Time: " + MtmInteraction.Initialize);
Diagnostic.AppendLine("Test Case Id Time: " + MtmInteraction.TcidTime);
Diagnostic.AppendLine("Automation Status Time: " + MtmInteraction.AutomationTime);
Diagnostic.AppendLine("Plan: " + MtmInteraction.SelectedPlanName);
Diagnostic.AppendLine("Total count: " + resDetail.Count() + " Blocked Count: " +
resDetail.Count(l => l.Outcome.Equals("Blocked", StringComparison.OrdinalIgnoreCase)));
Diagnostic.AppendLine("---------------------------------------------------");
return(resDetail.ToList());
}
/// <summary>
/// Queries the Octopart website for information related to the query string
/// </summary>
/// <remarks>
/// - If the query string has already been searched, calling this function will bring up the next set
/// of results (usually in increments of 20 items, with a limit of up to 100 items).
/// - The query string will have all whitespace removed, and be brought to lower case
/// - If a query with the specified query string is already in progress, nothing will be done
/// </remarks>
/// <param name="mpn_or_sku">The query string</param>
public void QueryNext(string mpn_or_sku)
{
mpn_or_sku = mpn_or_sku.Sanitize();
// Verify that there already exists a query for the part. If not, simply create a new query
if (_queryList.Count(query => query.Q == mpn_or_sku) == 0)
{
EnqueueQuery(mpn_or_sku, 0);
}
else
{
// Only add a new query if there is not already one processing, and we haven't hit max hits
// and if previously error'd, re-request it
var existingQueries = _queryList.Where(i => i.Q == mpn_or_sku);
int maxstart = existingQueries.Max(i => i.Start);
if (existingQueries.Count(i => i.State == CacheItem.ProcessingState.Error) != 0)
{
existingQueries.ToList().ForEach(i => { i.State = CacheItem.ProcessingState.Await; i.Error = string.Empty; });
_queryTimer.Enabled = true;
}
else if ((existingQueries.Count(i => i.State == CacheItem.ProcessingState.Processing || i.State == CacheItem.ProcessingState.Await) == 0) && (maxstart < ApiV4.RECORD_START_MAX) && !IsQueryLimitMaxed(mpn_or_sku))
{
EnqueueQuery(mpn_or_sku, maxstart + ApiV4.RECORD_LIMIT_PER_QUERY);
}
}
}
public static void Execute()
{
for (int i = 1; i <= 1000; i++)
{
employees.Add(new Employee()
{
Id = i, Dob = DateTime.Now.Date.AddDays((new Random()).Next(-18250, -6570))
});
}
var stopwatchSync = System.Diagnostics.Stopwatch.StartNew();
foreach (var emp in employees)
{
if (emp.Dob <= DateTime.Now.Date.AddDays(-10950))
{
employeesOver30s.Add(emp);
Thread.Sleep(1);
}
}
stopwatchSync.Stop();
Console.WriteLine($"Count: {employeesOver30s.Count()} in: {stopwatchSync.ElapsedMilliseconds} ms");
var stopwatchAync = System.Diagnostics.Stopwatch.StartNew();
Parallel.ForEach(employees, emp =>
{
if (emp.Dob <= DateTime.Now.Date.AddDays(-10950))
{
employeesOver30sAsync.Add(emp);
Thread.Sleep(1);
}
});
stopwatchAync.Stop();
Console.WriteLine($"Count: {employeesOver30sAsync.Count()} in: {stopwatchAync.ElapsedMilliseconds} ms");
var stopwatchAyncLocVar = System.Diagnostics.Stopwatch.StartNew();
Parallel.ForEach <Employee, List <Employee> >(employees, () => new List <Employee>(), (emp, loop, subList) =>
{
if (emp.Dob <= DateTime.Now.Date.AddDays(-10950))
{
subList.Add(emp);
Thread.Sleep(1);
}
return(subList);
},
(empSublist) => empSublist.ForEach(e => employeesOver30sAsyncLocVar.Add(e)));
stopwatchAyncLocVar.Stop();
Console.WriteLine($"Count: {employeesOver30sAsync.Count()} in: {stopwatchAyncLocVar.ElapsedMilliseconds} ms");
}
private void LogTemplates(ConcurrentBag <long> speedLogs, int totalCount)
{
Log($"Max: {speedLogs.Max():n0} ms");
Log($"Min: {speedLogs.Min():n0} ms");
Log($"Avg: {speedLogs.Average():n3} ms");
Log($"Total: {speedLogs.Sum():n0} ms");
Log($"InsertsInSecond: {speedLogs.Count() / (speedLogs.Sum() / (double) 1000):n3}");
Log($"Success: {speedLogs.Count():n0}/{totalCount:n0}\n");
}
/// <summary>
/// returns the total number of tweets
/// </summary>
/// <returns></returns>
private int GetTwitterCount()
{
int result = 0;
if (metricsBag != null)
{
result = metricsBag.Count();
}
return(result);
}
public void RoundRobinShouldHandleMultiThreadedRollOver()
{
var selector = new DefaultPartitionSelector();
var bag = new ConcurrentBag<Partition>();
Parallel.For(0, 100, x => bag.Add(selector.Select(_topicA, null)));
Assert.That(bag.Count(x => x.PartitionId == 0), Is.EqualTo(50));
Assert.That(bag.Count(x => x.PartitionId == 1), Is.EqualTo(50));
}
public void RoundRobinShouldHandleMultiThreadedRollOver()
{
var selector = new PartitionSelector();
var bag = new ConcurrentBag <MetadataResponse.Partition>();
Parallel.For(0, 100, x => bag.Add(selector.Select(_topicA, null)));
Assert.That(bag.Count(x => x.PartitionId == 0), Is.EqualTo(50));
Assert.That(bag.Count(x => x.PartitionId == 1), Is.EqualTo(50));
}
public void RoundRobinShouldHandleMultiThreadedRollOver()
{
RoundRobinPartitionSelector.Singleton.Reset();
var bag = new ConcurrentBag <MetadataResponse.Partition>();
Parallel.For(0, 100, x => bag.Add(RoundRobinPartitionSelector.Singleton.Select(_topicA, new ArraySegment <byte>())));
Assert.That(bag.Count(x => x.partition_id == 0), Is.EqualTo(50));
Assert.That(bag.Count(x => x.partition_id == 1), Is.EqualTo(50));
}
public static void Stats()
{
Console.ForegroundColor = ConsoleColor.DarkCyan;
Console.WriteLine(
"{0}Stats: Total={1} Passed={2} Failed={3}{0}",
Environment.NewLine,
list.Count,
list.Count(r => r.Result),
list.Count(r => !r.Result)
);
}
public void Save(MiniProfiler profiler)
{
if (Bag.Count() > Limit)
{
Reset();
}
if (Bag.All(p => p.Id != profiler.Id))
{
Bag.Add(profiler);
}
}
/// <summary>
/// Gets the object.
/// </summary>
/// <returns>T.</returns>
public T GetObject()
{
T item;
if (_objects.TryTake(out item))
{
return(item);
}
Console.WriteLine(_objects.Count());
return(_objectGenerator());
}
public void ShowCompleted(LeaseRowVM row)
{
_bag.Add(row.Errors.IsBlank());
var done = _bag.Count;
var guds = _bag.Count(_ => _);
var bads = _bag.Count(_ => !_);
var totl = LeasesList.Count;
Status = $"Fail: {bads:N0} : "
+ $"Success: {guds:N0}/{done:N0} : "
+ $"({DateTime.Now:h:mm}) "
+ $"Done: {done:N0}/{totl:N0} : {row.Lease}";
}
private async Task AutoTest()
{
/*
* 需要硬件环境
* rs232连接2-3, 自发自回
*/
if (SerialPort.GetPortNames().Length == 0)
{
return;
}
var rst = new ConcurrentBag <string>();
var auto = new SerialPortProtocol(1000);
var disposables = new CompositeDisposable();
Observable.Merge
(
auto.History.Select(p => $"History: {p}"),
auto.ConnectionState.Select(p => $"ConnectionState: {p}"),
auto.Received.Select(p => $"Received: {p.ByteToHex()}")
)
.Subscribe(p => { rst.Add(p); })
.DisposeWith(disposables);
auto.Init(SerialPort.GetPortNames()[0], 115200);
await Task.Delay(100);
//send when connected
auto.ConnectionState
.Where(p => p)
.Delay(TimeSpan.FromSeconds(0.5))
.Subscribe(async _ =>
{
await auto.SendAsync(new CommContext("AA00 0888 44BB", "AA00$2$1BB", 200));
await auto.SendAsync(new CommContext("CC00 1111 DD", "CC00$2DD", 200));
});
await Task.Delay(2000);
if (rst.Count(p => p.StartsWith("Received")) > 0)
{
Debugger.Break();
Assert.True(rst.Count(p => p.Contains("Success")) == 2 && rst.Count(p => p.Contains("Error")) == 0);
}
auto.Dispose();
disposables.Dispose();
rst.Clear();
}
public void ReturnsTokenUptoLimitAndThenRaisesMaxTokenIssuedEvent()
{
var sut = CreateInstance(_serverName, 0, TimeSpan.FromMilliseconds(20), TimeSpan.FromMilliseconds(50), 5, false);
sut.TokenIssued += Sut_TokenIssued;
sut.MaxTokenIssued += Sut_MaxTokenIssued;
var randomClientId = Guid.NewGuid().ToString();
var threadCount = 10;
using (var countdownEvent = new CountdownEvent(threadCount))
{
for (var i = 0; i < threadCount; i++)
{
ThreadPool.QueueUserWorkItem(new WaitCallback(PullToken_OnAThreadPool_Thread),
new ThreadFuncObj {
TokenRepo = sut, Tokens = _tokensIssued, SyncObj = countdownEvent, ExtApiCallDuration = TimeSpan.FromMilliseconds(500)
});
}
countdownEvent.Wait();
}
Assert.Equal(10, _tokensIssued.Count);
Assert.True(_tokensIssued.Count(x => x != null) > 0); //a few tokens were issues
Assert.True(_tokensIssued.Count(x => x == null) > 0); //a few failed too.
Assert.True(_callbackTokensIssued.Count > 0);
Assert.True(_callbackMaxTokensIssued.Count > 0);
}
private static void SampleConnections(CrankArguments arguments, ConcurrentBag <Connection> connections, TimeSpan elapsed)
{
var connecting = connections.Count(c => c.State == ConnectionState.Connecting);
var connected = connections.Count(c => c.State == ConnectionState.Connected);
var reconnecting = connections.Count(c => c.State == ConnectionState.Reconnecting);
var disconnected = connections.Count(c => c.State == ConnectionState.Disconnected);
Mark(arguments, (ulong)connecting, "Connections Connecting");
Mark(arguments, (ulong)connected, "Connections Connected");
Mark(arguments, (ulong)reconnecting, "Connections Reconnecting");
Mark(arguments, (ulong)disconnected, "Connections Disconnected");
var transportState = "";
if (connections.First().Transport.GetType() == typeof(AutoTransport))
{
transportState = String.Format(", Transport={0}ws|{1}ss|{2}lp",
connections.Count(c => c.Transport.Name.Equals("webSockets", StringComparison.InvariantCultureIgnoreCase)),
connections.Count(c => c.Transport.Name.Equals("serverSentEvents", StringComparison.InvariantCultureIgnoreCase)),
connections.Count(c => c.Transport.Name.Equals("longPolling", StringComparison.InvariantCultureIgnoreCase)));
}
Console.WriteLine(String.Format("[{0}] Connections: {1}/{2}, State={3}|{4}c|{5}r|{6}d",
elapsed,
connections.Count(),
arguments.NumClients,
connecting,
connected,
reconnecting,
disconnected)
+ transportState);
}
/// <summary>
/// Deals with the console logging from the text file differences
/// </summary>
/// <param name="linesAdded"></param>
/// <param name="linesRemovedFromOriginal"></param>
public void TextDifferencesConsoleLogging(ConcurrentBag <string> linesAdded, ConcurrentBag <string> linesRemovedFromOriginal)
{
if (linesAdded.Count() > 0)
{
Console.WriteLine("");
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine($"The following lines have been added to {_config.RootUrl}:");
Console.WriteLine("");
Console.ForegroundColor = ConsoleColor.Gray;
foreach (var line in linesAdded)
{
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("");
Console.WriteLine(line);
Console.ForegroundColor = ConsoleColor.Gray;
}
}
if (linesRemovedFromOriginal.Count() > 0)
{
Console.WriteLine("");
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine($"The following lines have been removed from {_config.RootUrl}:");
Console.WriteLine("");
Console.ForegroundColor = ConsoleColor.White;
foreach (var line in linesRemovedFromOriginal)
{
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("");
Console.WriteLine(line);
Console.ForegroundColor = ConsoleColor.Gray;
}
}
if (linesRemovedFromOriginal.Count() == 0 && linesAdded.Count() == 0)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine($"The files in both folder are the same!");
Console.WriteLine("");
Console.ForegroundColor = ConsoleColor.Gray;
}
if (!_config.RunScheduled)
{
Console.WriteLine("");
Console.WriteLine("Please return to proceed...");
Console.WriteLine("");
Console.ReadLine();
}
}
public void GetService_can_be_accessed_from_multiple_threads_concurrently()
{
for (var i = 0; i < 30; i++)
{
var bag = new ConcurrentBag<MigrationSqlGenerator>();
var resolver = new MigrationsConfigurationResolver();
ExecuteInParallel(() => bag.Add(resolver.GetService<MigrationSqlGenerator>("System.Data.SqlClient")));
Assert.Equal(20, bag.Count);
foreach (var generator in bag)
{
Assert.IsType<SqlServerMigrationSqlGenerator>(generator);
Assert.Equal(19, bag.Count(c => generator != c));
}
}
}
public async Task A_partitioned_stream_can_be_mapped_at_query_time()
{
for (var i = 1; i <= 9; i++)
{
store.WriteEvents(
streamId: Guid.NewGuid().ToString(),
howMany: 10,
bucketId: i.ToString());
}
var partitionedStream = Stream.Partitioned<EventMessage, int, string>(async (q, p) =>
{
var bucketId = ((IStreamQueryValuePartition<string>) p).Value;
var streamsToSnapshot = store.Advanced.GetStreamsToSnapshot(bucketId, 0);
var streamId = streamsToSnapshot.Select(s => s.StreamId).Single();
var stream = NEventStoreStream.ByAggregate(store, streamId, bucketId);
var batch = await stream.CreateQuery(q.Cursor, q.BatchSize).NextBatch();
return batch;
}).Trace();
var domainEvents = partitionedStream.Map(es => es.Select(e => e.Body).OfType<IDomainEvent>());
// catch up
var catchup = domainEvents.DistributeAmong(Enumerable.Range(1, 10)
.Select(i => Partition.ByValue(i.ToString())),
batchSize: 2);
var receivedEvents = new ConcurrentBag<IDomainEvent>();
catchup.Subscribe(async b =>
{
foreach (var e in b)
{
receivedEvents.Add(e);
}
});
await catchup.RunUntilCaughtUp().Timeout();
receivedEvents.Count().Should().Be(90);
}
public void ShouldEmitSubscriberExceptionsOnExceptionObservable()
{
FakeServiceEndpoint serviceEndpoint1 = new FakeServiceEndpoint(typeof(ITestServiceMessage1));
FakeServiceEndpoint serviceEndpoint2 = new FakeServiceEndpoint(typeof(ITestServiceMessage2));
IServiceBus serviceBus = ServiceBus.Configure()
.WithEndpoint((IServiceEndpointClient) serviceEndpoint1)
.WithEndpoint((IServiceEndpoint)serviceEndpoint1)
.WithEndpoint((IServiceEndpointClient) serviceEndpoint2)
.WithEndpoint((IServiceEndpoint)serviceEndpoint2)
.UsingConsoleLogging().Create();
ConcurrentBag<Exception> exceptions = new ConcurrentBag<Exception>();
serviceBus.Exceptions.Subscribe(exceptions.Add);
FakeSubscriber subscriber = new FakeSubscriber();
var testScheduler = new TestScheduler();
var subscription = serviceBus.Subscribe(subscriber, testScheduler);
subscriber.ThrowExceptions = true;
serviceBus.PublishAsync(new TestServiceEvent1());
testScheduler.AdvanceBy(1);
serviceBus.PublishAsync(new TestServiceEvent2());
testScheduler.AdvanceBy(1);
serviceBus.SendAsync(new TestServiceCommand1());
testScheduler.AdvanceBy(1);
serviceBus.SendAsync(new TestServiceCommand2());
testScheduler.AdvanceBy(1);
subscription.Dispose();
Assert.That(exceptions.Count(), Is.EqualTo(4));
Assert.That(subscriber.Received.Count(), Is.EqualTo(0));
}
public void ShouldCatchAndHandleExceptionsThrownByEndpointObservables()
{
FakeServiceEndpoint erroringEndpoint = new FakeServiceEndpoint(typeof(ITestServiceMessage1)) { ThrowException = true };
FakeServiceEndpoint serviceEndpoint = new FakeServiceEndpoint(typeof(ITestServiceMessage2));
IServiceBus serviceBus = ServiceBus.Configure()
.WithEndpoint((IServiceEndpointClient) erroringEndpoint)
.WithEndpoint((IServiceEndpoint)erroringEndpoint)
.WithEndpoint((IServiceEndpointClient) serviceEndpoint)
.WithEndpoint((IServiceEndpoint)serviceEndpoint)
.UsingConsoleLogging().Create();
ConcurrentBag<IMessage> messages = new ConcurrentBag<IMessage>();
ConcurrentBag<Exception> exceptions = new ConcurrentBag<Exception>();
serviceBus.Events.Subscribe(messages.Add);
serviceBus.Commands.Subscribe(messages.Add);
serviceBus.Requests.Subscribe(messages.Add);
serviceBus.Exceptions.Subscribe(exceptions.Add);
// trigger exception
serviceBus.PublishAsync(new TestServiceEvent1());
TestServiceEvent2 message1 = new TestServiceEvent2();
serviceBus.PublishAsync(message1);
// trigger another exception
serviceBus.PublishAsync(new TestServiceEvent1());
TestServiceEvent2 message2 = new TestServiceEvent2();
serviceBus.PublishAsync(message2);
Assert.That(exceptions.Count(), Is.EqualTo(2));
Assert.That(messages.Contains(message1), "message1 not received");
Assert.That(messages.Contains(message2), "message2 not received");
}
public void ShouldSendAllMessagesToSubscribers()
{
FakeServiceEndpoint serviceEndpoint1 = new FakeServiceEndpoint(typeof(ITestServiceMessage1));
FakeServiceEndpoint serviceEndpoint2 = new FakeServiceEndpoint(typeof(ITestServiceMessage2));
IServiceBus serviceBus = ServiceBus.Configure()
.WithEndpoint((IServiceEndpointClient) serviceEndpoint1)
.WithEndpoint((IServiceEndpoint)serviceEndpoint1)
.WithEndpoint((IServiceEndpointClient) serviceEndpoint2)
.WithEndpoint((IServiceEndpoint)serviceEndpoint2)
.UsingConsoleLogging()
.Create();
ConcurrentBag<Exception> exceptions = new ConcurrentBag<Exception>();
serviceBus.Exceptions.Subscribe(exceptions.Add);
FakeSubscriber subscriber = new FakeSubscriber();
FakeSubscriber2 subscriber2 = new FakeSubscriber2();
var testScheduler = new TestScheduler();
var subscription = serviceBus.Subscribe(subscriber, testScheduler);
var subscription2 = serviceBus.Subscribe(subscriber2, testScheduler);
serviceBus.PublishAsync(new TestServiceEvent1());
testScheduler.AdvanceBy(1);
serviceBus.PublishAsync(new TestServiceEvent2());
testScheduler.AdvanceBy(1);
serviceBus.SendAsync(new TestServiceCommand1());
testScheduler.AdvanceBy(1);
serviceBus.SendAsync(new TestServiceCommand2());
testScheduler.AdvanceBy(1);
serviceEndpoint1.Messages.OnNext(new TestServiceRequest1());
testScheduler.AdvanceBy(1);
subscription.Dispose();
subscription2.Dispose();
Assert.That(exceptions.Count(), Is.EqualTo(0));
Assert.That(subscriber.Received.Count(), Is.EqualTo(5));
Assert.That(subscriber.Received[0].GetType(), Is.EqualTo(typeof(TestServiceEvent1)));
Assert.That(subscriber.Received[1].GetType(), Is.EqualTo(typeof(TestServiceEvent2)));
Assert.That(subscriber.Received[2].GetType(), Is.EqualTo(typeof(TestServiceCommand1)));
Assert.That(subscriber.Received[3].GetType(), Is.EqualTo(typeof(TestServiceCommand2)));
Assert.That(subscriber.Received[4].GetType(), Is.EqualTo(typeof(TestServiceRequest1)));
Assert.That(subscriber2.Received.Count(), Is.EqualTo(5));
Assert.That(subscriber2.Received[0].GetType(), Is.EqualTo(typeof(TestServiceEvent1)));
Assert.That(subscriber2.Received[1].GetType(), Is.EqualTo(typeof(TestServiceEvent2)));
Assert.That(subscriber2.Received[2].GetType(), Is.EqualTo(typeof(TestServiceCommand1)));
Assert.That(subscriber2.Received[3].GetType(), Is.EqualTo(typeof(TestServiceCommand2)));
Assert.That(subscriber2.Received[4].GetType(), Is.EqualTo(typeof(TestServiceRequest1)));
}
public void DCAwareRoundRobinPolicyTestInParallel()
{
var hostList = new List<Host>
{
TestHelper.CreateHost("0.0.0.1", "dc1"),
TestHelper.CreateHost("0.0.0.2", "dc2"),
TestHelper.CreateHost("0.0.0.3", "dc1"),
TestHelper.CreateHost("0.0.0.4", "dc2"),
TestHelper.CreateHost("0.0.0.5", "dc1"),
TestHelper.CreateHost("0.0.0.6", "dc2"),
TestHelper.CreateHost("0.0.0.7", "dc1"),
TestHelper.CreateHost("0.0.0.8", "dc2"),
TestHelper.CreateHost("0.0.0.9", "dc1"),
TestHelper.CreateHost("0.0.0.10", "dc2")
};
var localHostsLength = hostList.Count(h => h.Datacenter == "dc1");
const string localDc = "dc1";
var clusterMock = new Mock<ICluster>();
clusterMock
.Setup(c => c.AllHosts())
.Returns(hostList);
//Initialize the balancing policy
var policy = new DCAwareRoundRobinPolicy(localDc, 1);
policy.Initialize(clusterMock.Object);
var allHosts = new ConcurrentBag<Host>();
var firstHosts = new ConcurrentBag<Host>();
Action action = () =>
{
var hosts = policy.NewQueryPlan(null, null).ToList();
//Check that the value is not repeated
Assert.AreEqual(0, hosts.GroupBy(x => x)
.Where(g => g.Count() > 1)
.Select(y => y.Key)
.Count());
firstHosts.Add(hosts[0]);
//Add to the general list
foreach (var h in hosts)
{
allHosts.Add(h);
}
};
var actions = new List<Action>();
const int times = 100;
for (var i = 0; i < times; i++)
{
actions.Add(action);
}
TestHelper.ParallelInvoke(actions);
//Check that the first nodes where different
foreach (var h in hostList)
{
if (h.Datacenter == localDc)
{
Assert.AreEqual(times/localHostsLength, firstHosts.Count(hc => hc == h));
}
else
{
Assert.AreEqual(0, firstHosts.Count(hc => hc == h));
}
}
clusterMock.Verify();
}
public void TestWmtsRequestInParallel()
{
// arrange
var resourceUrls = CreateResourceUrls();
var request = new WmtsRequest(resourceUrls);
var urls = new ConcurrentBag<Uri>(); // List is not thread save
var tileInfo = new TileInfo {Index = new TileIndex(8938, 5680, "14")};
// act
var requests = new List<Func<Uri>>();
for (var i = 0; i < 150; i++) requests.Add(() => request.GetUri(tileInfo));
Parallel.ForEach(requests, r => urls.Add(r()));
// assert
var count = urls.Count(u => u.ToString() == "http://maps1.wien.gv.at/wmts/lb/farbe/google3857/14/5680/8938.jpeg");
Assert.True(count == 50);
}
public void LogAlwaysShouldBeThreadSafe()
{
var bag = new ConcurrentBag<string>();
MockLogger.Setup(m => m.Error(It.IsAny<object>())).Callback<object>(msg => bag.Add(msg.ToString()));
MockLogger.Setup(m => m.Info(It.IsAny<object>())).Callback<object>(msg => bag.Add(msg.ToString()));
MockLogger.Setup(m => m.IsErrorEnabled).Returns(true);
var threads = new List<Thread>();
for (int threadNumber = 0; threadNumber < 20; threadNumber++)
{
if (threadNumber % 2 == 0)
{
threads.Add(new Thread(() =>
{
for (int i = 0; i < 10; i++)
{
Subject.LogError(() => "Foo");
}
}));
}
else
{
threads.Add(new Thread(() =>
{
for (int i = 0; i < 10; i++)
{
Subject.LogAlways(() => "Always");
}
}));
}
}
threads.ForEach(t => t.Start());
foreach (var thread in threads)
{
thread.Join();
}
Assert.AreEqual(200, bag.Count);
Assert.AreEqual(100, bag.Count(msg => msg == "Foo"));
Assert.AreEqual(100, bag.Count(msg => msg == "Always"));
}
public void DispatchesMessages()
{
const int numberOfMessages = 1000;
var settings = new MessageDispatcherSettings();
settings.InputChannel.WithDefault(new InMemoryMessageChannel());
settings.InvalidChannel.WithDefault(new InMemoryMessageChannel());
settings.MessageProcessorTypes.WithDefault(new List<Type> { typeof(FakeMessageProcessor) });
settings.DurationOfDispatchingSlice.WithDefault(new TimeSpan(0, 0, 0, 0, 200));
settings.NumberOfMessagesToDispatchPerSlice.WithDefault(30);
_dispatcher.Configure(settings);
_dispatcher.Enable();
Assert.AreEqual(MessageDispatcherState.Enabled, _dispatcher.State);
_transport.Open();
var recordIds = new ConcurrentBag<Guid>();
var start = DateTime.Now;
for (var j = 0; j < numberOfMessages; j++)
{
var record = GetRecord();
_transport.Send(record);
recordIds.Add(record.Identifier);
}
Console.WriteLine("Sent 1000 messages in {0} ms", (DateTime.Now - start).TotalMilliseconds);
Console.WriteLine("Waiting for messages to be processed");
start = DateTime.Now;
Assert.AreEqual(numberOfMessages, recordIds.Count);
var numberOfMessagesProcessed = 0;
do
{
Thread.Sleep(200);
numberOfMessagesProcessed = recordIds.Where(id => _registry.GetPublicationRecord(id).Completed).Count();
Console.WriteLine("{0} messages processed", numberOfMessagesProcessed);
}
while (numberOfMessagesProcessed < recordIds.Count());
Console.WriteLine("Completed in {0} seconds", (DateTime.Now - start).TotalSeconds);
_dispatcher.Disable();
Assert.AreEqual(MessageDispatcherState.Disabled, _dispatcher.State);
Assert.IsTrue(FakeMessageProcessor.ProcessedAnyMessages);
}
static void Main(string[] args)
{
Console.WriteLine("Crank v{0}", typeof(Program).Assembly.GetName().Version);
if (args.Length < 2)
{
Console.WriteLine("Usage: crank [url] [numclients]");
return;
}
ServicePointManager.DefaultConnectionLimit = Int32.MaxValue;
string url = args[0];
int clients = Int32.Parse(args[1]);
TaskScheduler.UnobservedTaskException += OnUnobservedTaskException;
var connections = new ConcurrentBag<Connection>();
var sw = Stopwatch.StartNew();
Task.Factory.StartNew(() =>
{
Parallel.For(0, clients, i =>
{
try
{
var connection = new Connection(url);
connection.Received += data =>
{
Console.WriteLine(data);
};
connection.Error += e =>
{
Console.WriteLine("ERROR: Client {0}, {1}", i, e.GetBaseException());
};
connection.Closed += () =>
{
Console.WriteLine("CLOSED: {0}", i);
};
connection.Start().Wait();
connections.Add(connection);
}
catch (Exception e)
{
Console.WriteLine("Failed to start client {0}. {1}", i, e);
}
});
Console.WriteLine("Started {0} connection(s).", connections.Count);
});
Console.WriteLine("Press any key to stop running...");
Console.Read();
sw.Stop();
Console.WriteLine("Total Running time: {0}s", sw.Elapsed);
Console.WriteLine("End point: {0}", url);
Console.WriteLine("Total connections: {0}", clients);
Console.WriteLine("Active connections: {0}", connections.Count(c => c.IsActive));
Console.WriteLine("Stopped connections: {0}", connections.Count(c => !c.IsActive));
}
public async Task CannotRemoveTheSameSubscriptionConcurrently(int concurrencyLevel, int nSubscriptions)
{
// arrange
var subscriptions = Enumerable.Range(1, nSubscriptions)
.Select(n => new Subscription("topic" + n, "endpoint" + n, "dummy"))
.ToArray();
// add the subscriptions so they can be removed
foreach (var subscription in subscriptions)
{
await this.store.AddSubscription(subscription);
}
// act & assert
// for each subscription, tries to add it concurrently
foreach (var subscription in subscriptions)
{
var tasks = new ConcurrentBag<Task<bool>>();
var subscriptionToAdd = subscription; //capture context
Parallel.For(0, concurrencyLevel, c =>
{
tasks.Add(this.store.RemoveSubscription(subscription));
});
await Task.WhenAll(tasks);
// ensures only one concurrent task was able to add the subscription
Assert.AreEqual(1, tasks.Count(t => t.Result == true));
}
}
public void TestForeach()
{
const int numValues = 100000;
Scheduler.WaitFor(WriteLotsOfValuesInBatch(Tangle, numValues, -1));
var keys = new List<int>();
for (int i = 0; i < numValues; i += 2)
keys.Add(i);
long startTime = Time.Ticks;
var bag = new ConcurrentBag<int>();
var fForeach = Tangle.ForEach(keys, (key, value) => bag.Add(value));
Scheduler.WaitFor(fForeach);
decimal elapsedSeconds = (decimal)(Time.Ticks - startTime) / Time.SecondInTicks;
Console.WriteLine(
"{0} values in ~{1:00.000} second(s) at ~{2:00000.00} values/sec.",
keys.Count, elapsedSeconds, keys.Count / elapsedSeconds
);
Assert.AreEqual(keys.Count, bag.Count());
Assert.AreEqual(
keys.OrderBy((k) => k).ToArray(),
bag.OrderBy((k) => k).ToArray()
);
}
private static void SampleConnections(CrankArguments arguments, ConcurrentBag<Connection> connections, TimeSpan elapsed)
{
var connecting = connections.Count(c => c.State == ConnectionState.Connecting);
var connected = connections.Count(c => c.State == ConnectionState.Connected);
var reconnecting = connections.Count(c => c.State == ConnectionState.Reconnecting);
var disconnected = connections.Count(c => c.State == ConnectionState.Disconnected);
Mark(arguments, (ulong)connecting, "Connections Connecting");
Mark(arguments, (ulong)connected, "Connections Connected");
Mark(arguments, (ulong)reconnecting, "Connections Reconnecting");
Mark(arguments, (ulong)disconnected, "Connections Disconnected");
var transportState = "";
if (connections.First().Transport.GetType() == typeof(AutoTransport))
{
transportState = String.Format(", Transport={0}ws|{1}ss|{2}lp",
connections.Count(c => c.Transport.Name.Equals("webSockets", StringComparison.InvariantCultureIgnoreCase)),
connections.Count(c => c.Transport.Name.Equals("serverSentEvents", StringComparison.InvariantCultureIgnoreCase)),
connections.Count(c => c.Transport.Name.Equals("longPolling", StringComparison.InvariantCultureIgnoreCase)));
}
Console.WriteLine(String.Format("[{0}] Connections: {1}/{2}, State={3}|{4}c|{5}r|{6}d",
elapsed,
connections.Count(),
arguments.NumClients,
connecting,
connected,
reconnecting,
disconnected)
+ transportState);
}
// button events
private void ButtonGenerateClick(object sender, EventArgs e)
{
// start logging
Program.Log.Write("\nGenerate Data: " + DateTime.Now.ToString("u"));
Program.Log.Write("Skip Missing Animations? " + this.checkBoxSkipMissingAnimations.Checked);
Program.Log.Write("Delete Old Data? " + this.checkBoxClearOldJson.Checked);
Program.Log.Write("Clear Favorites? " + this.checkBoxReseedFavorites.Checked);
// initialize variables
int validPoserCount;
int emptyPoserCount;
int poserPacksCount;
var removedPoserCount = 0;
var jsonOutputDirectory = this.skyrimDirectory + Resources.PoserHotkeysDataPath;
// ensure output directory exists
if (!Directory.Exists(jsonOutputDirectory))
{
Directory.CreateDirectory(jsonOutputDirectory);
}
// clear favorites...
if (this.checkBoxReseedFavorites.Checked)
{
var favoritesJson = Directory.GetFiles(jsonOutputDirectory, Resources.FavoritesJsonName);
if (favoritesJson.Length <= 0)
{
Program.Log.Write(Resources.FavoritesJsonName + " not found.");
}
else
{
removedPoserCount++;
Program.Log.Write("Deleting " + Resources.FavoritesJsonName);
File.Delete(favoritesJson.First());
}
}
// clear old data...
if (this.checkBoxClearOldJson.Checked)
{
foreach (var jsonFile in Directory.GetFiles(jsonOutputDirectory, "*.json")
.Where(jsonFile => !jsonFile.Contains(Resources.FavoritesJsonName)))
{
removedPoserCount++;
Program.Log.Write("Deleting " + jsonFile);
File.Delete(jsonFile);
}
}
// generate data...
using (var pleaseWaitMessage = new FormPleaseWaitMessage())
{
pleaseWaitMessage.Show(this);
pleaseWaitMessage.Update();
try
{
// get checked poser list
var checkedPosers = this.checkedListBoxPosers.CheckedItems.OfType<Poser>();
var checkedPosersList = checkedPosers as IList<Poser> ?? checkedPosers.ToList();
// extract all packs from all checked posers
var results = new ConcurrentBag<PopulatePacksResults>();
Parallel.ForEach(
checkedPosersList,
poser =>
{
results.Add(poser.PopulatePacks(this.checkBoxSkipMissingAnimations.Checked));
});
// log results
foreach (var result in results)
{
Program.Log.Write("\nPoser: " + result.PoserName);
Program.Log.Write("\tLoaded Packs: " + result.PacksAdded.Count);
Program.Log.Write("\tLoaded Pack Names: " + String.Join(", ", result.PacksAdded));
Program.Log.Write("\tLoaded Animations: " + result.AnimationsAdded.Count);
Program.Log.Write("\tMissing Animations: " + result.AnimationsMissing.Count);
Program.Log.Write("\tMissing Animations Names: " + String.Join(", ", result.AnimationsMissing));
}
// get summary counts
validPoserCount = results.Count(x => x.PacksAdded.Count > 0);
emptyPoserCount = results.Count(x => x.PacksAdded.Count <= 0);
poserPacksCount = results.Sum(x => x.PacksAdded.Count);
// write to disc
SeedFavoritesJson(jsonOutputDirectory);
WriteJson(checkedPosersList, jsonOutputDirectory);
Program.Log.Write("\nGeneration complete! " + DateTime.Now.ToString("u"));
}
catch (Exception exception)
{
Program.Log.Write("Fatal Error: " + exception.Message);
throw;
}
pleaseWaitMessage.Close();
}
// show summary
var summaryMessage =
String.Format(
$"Generated {validPoserCount} data files containing {poserPacksCount} poser packs."
+ $"\nSkipped {emptyPoserCount} posers that had 0 packs."
+ $"\nRemoved {removedPoserCount} old data files."
+ $"\n\nOutput written to:\n{jsonOutputDirectory}");
MessageBox.Show(summaryMessage, Resources.CommonString_Summary, MessageBoxButtons.OK);
Program.Log.Write(summaryMessage);
this.buttonQuit.Select();
}
static void Main(string[] args)
{
Console.WriteLine("Crank v{0}", typeof(Program).Assembly.GetName().Version);
if (args.Length < 2)
{
Console.WriteLine("Usage: crank [url] [numclients] <batchSize> <batchInterval>");
return;
}
ServicePointManager.DefaultConnectionLimit = Int32.MaxValue;
_running = true;
string url = args[0];
int clients = Int32.Parse(args[1]);
int batchSize = args.Length < 3 ? 50 : Int32.Parse(args[2]);
int batchInterval = args.Length < 4 ? 500 : Int32.Parse(args[3]);
TaskScheduler.UnobservedTaskException += OnUnobservedTaskException;
var connections = new ConcurrentBag<Connection>();
var sw = Stopwatch.StartNew();
Task.Factory.StartNew(() =>
{
Console.WriteLine("Ramping up connections. Batch size {0}.", batchSize);
var rampupSw = Stopwatch.StartNew();
ConnectBatches(url, clients, batchSize, batchInterval, connections).ContinueWith(task =>
{
Console.WriteLine("Started {0} connection(s).", connections.Count);
Console.WriteLine("Setting up event handlers");
rampupSw.Stop();
Console.WriteLine("Ramp up complete in {0}.", rampupSw.Elapsed);
});
});
Console.WriteLine("Press any key to stop running...");
Console.Read();
sw.Stop();
_running = false;
Console.WriteLine("Total Running time: {0}", sw.Elapsed);
Console.WriteLine("End point: {0}", url);
Console.WriteLine("Total connections: {0}", clients);
Console.WriteLine("Active connections: {0}", connections.Count(c => c.IsActive));
Console.WriteLine("Stopped connections: {0}", connections.Count(c => !c.IsActive));
Console.WriteLine("Closing connection(s).");
foreach (var connection in connections)
{
connection.Stop();
}
}
private void worker_DoWork(object sender, DoWorkEventArgs e)
{
BackgroundWorker self = (BackgroundWorker)sender;
Config cfg = (Config)e.Argument;
CultureInfo ci = new CultureInfo("en-US");
int totalCount = 1;
int processedCount = 0;
int progressValue = 0;
List<Uri> products = new List<Uri>();
// walk through categories
Uri category = cfg.Uri;
while(category != null)
{
string content = null;
for (content = Web.GetContent(category); content == null; content = Web.GetContent(category))
{
self.ReportProgress(progressValue, CONNECTION_ERROR);
Console.WriteLine("Failed {0}", category);
Thread.Sleep(SLEEPING_TIME);
}
Console.WriteLine("Read {0}", category);
// get products
MatchCollection mc = RE.Products.Matches(content);
foreach (Match m in mc)
{
Uri prod = new Uri(category, new Uri(m.Groups[1].Value, UriKind.Relative));
totalCount++;
products.Add(prod);
}
// check the next page
Match np = RE.Page.Match(content);
if (np.Success)
{
category = new Uri(category, new Uri(np.Groups[1].Value, UriKind.Relative));
}
else
category = null;
progressValue = (int)(100.0f / totalCount * processedCount);
self.ReportProgress(progressValue, "Reading categories");
}
// walk through products
ConcurrentBag<Photo> photoStorage = new ConcurrentBag<Photo>();
Parallel.ForEach(products, product =>
{
string content = null;
for (content = Web.GetContent(product); content == null; content = Web.GetContent(product))
{
self.ReportProgress(progressValue, CONNECTION_ERROR);
Console.WriteLine("Failed {0}", product);
Thread.Sleep(SLEEPING_TIME);
}
Console.WriteLine("Read {0}", product);
// check global suitability of the product:
// skip this product in case when it is out of stock on Zappos
if (!RE.ZapposOutOfStock.IsMatch(content))
{
try
{
string brand = "";
string name = "";
float price = 0.0f;
float oldPrice = 0.0f;
string sku = "";
List<Photo> photoList = new List<Photo>();
List<string> sizeList = new List<string>();
Match brandname = RE.ZapposBrandName.Match(content);
if (brandname.Success)
{
brand = brandname.Groups[1].Value;
name = brandname.Groups[2].Value;
price = float.Parse(RE.ZapposPrice.Match(content).Groups[2].Value, ci.NumberFormat);
sku = RE.ZapposSku.Match(content).Groups[1].Value;
Match mop = RE.ZapposOldPrice.Match(content);
if (mop.Success)
{
oldPrice = float.Parse(mop.Groups[1].Value, ci.NumberFormat);
}
// get the Size's Id
Match mSI = RE.ZapposSizeId.Match(content);
if (mSI.Success)
{
string styleId = mSI.Groups[1].Value;
Regex reZapposSizeWrapper = RE.GetZapposSizeWrapper(styleId);
Match mSizeWrap = reZapposSizeWrapper.Match(content);
if (mSizeWrap.Success)
{
MatchCollection mcSizeList = RE.ZapposSize.Matches(mSizeWrap.Groups[1].Value);
foreach (Match mcs in mcSizeList)
sizeList.Add(mcs.Groups[1].Value);
}
else
{
Regex reZapposSingleSize = RE.GetZapposSingleSize(styleId);
Match mZSS = reZapposSingleSize.Match(content);
if (mZSS.Success)
sizeList.Add(mZSS.Groups[1].Value);
else
throw new Exception("Cannot get product's size info: " + product.ToString());
}
}
// determine the style id to take an appropriate photo set
Match zsi = RE.ZapposStyleId.Match(content);
if (zsi.Success)
{
string style = zsi.Groups[1].Value;
// large images
HashSet<string> larges = new HashSet<string>();
Regex reZapposPhotosLarge = RE.GetZapposPhotosLarge(style);
MatchCollection mp = reZapposPhotosLarge.Matches(content);
foreach (Match m in mp)
{
photoList.Add(new Photo(m.Groups[2].Value, PhotoSize.Large));
larges.Add(m.Groups[1].Value);
}
// medium images
Regex reZapposPhotosMedium = RE.GetZapposPhotosMedium(style);
mp = reZapposPhotosMedium.Matches(content);
foreach (Match m in mp)
{
if (!larges.Contains(m.Groups[1].Value))
photoList.Add(new Photo(m.Groups[2].Value, PhotoSize.Medium));
}
// must be at least one picture
if (photoList.Count == 0)
throw new Exception("No photos for " + product.ToString());
}
else
throw new Exception("No style id for " + product.ToString());
}
else
{
// may be we are at couture.zappos.com?
Match cbns = RE.CoutureBrandNameSku.Match(content);
if (!cbns.Success) throw new Exception("invalid source");
brand = cbns.Groups[1].Value;
name = cbns.Groups[2].Value;
sku = cbns.Groups[3].Value;
price = float.Parse(RE.CouturePrice.Match(content).Groups[1].Value, ci.NumberFormat);
Match mop = RE.CoutureOldPrice.Match(content);
if (mop.Success)
{
oldPrice = float.Parse(mop.Groups[1].Value, ci.NumberFormat);
}
Match mCPS = RE.CouturePhotoSource.Match(content);
if (mCPS.Success)
{
string cps_content = null;
Uri cps_photo_source = null;
cps_photo_source = new Uri(product, mCPS.Groups[1].Value);
cps_content = Web.GetContent(cps_photo_source);
for (cps_content = Web.GetContent(cps_photo_source); cps_content == null; cps_content = Web.GetContent(cps_photo_source))
{
self.ReportProgress(progressValue, CONNECTION_ERROR);
Console.WriteLine("Failed {0}", cps_photo_source);
Thread.Sleep(SLEEPING_TIME);
}
Console.WriteLine("Read {0}", cps_photo_source);
MatchCollection cphotos = RE.CouturePhotos.Matches(cps_content);
foreach (Match mcp in cphotos)
photoList.Add(new Photo(mcp.Groups[1].Value, PhotoSize.Large));
}
else
throw new Exception("no couture photo source: " + product);
}
// make photo objects
int newprice = (int)(0.5f + price + price / 100.0f * cfg.Percents);
string strPrice = newprice.ToString("C", ci);
string strOldPrice = String.Empty;
if (oldPrice > 0.0f)
{
int newOldPrice = (int)(0.5f + oldPrice + oldPrice / 100.0f * cfg.Percents);
strOldPrice = newOldPrice.ToString("C", ci);
}
foreach (Photo pObj in photoList.Take(cfg.PhotosLimit))
{
pObj.Brand = brand;
pObj.Name = name;
pObj.Price = strPrice;
pObj.OldPrice = strOldPrice;
pObj.Sku = sku;
pObj.Sizes.AddRange(sizeList);
photoStorage.Add(pObj);
Interlocked.Increment(ref totalCount);
}
}
catch (Exception ex)
{
// an error happened
Console.WriteLine(ex.Message);
throw new Exception(ex.Message);
}
}
Interlocked.Increment(ref processedCount);
progressValue = (int)(100.0f / totalCount * processedCount);
self.ReportProgress(progressValue, "Reading products");
});
// in case when folder separation is needed, we create those folders
if (cfg.DirectoryLimit > 0)
{
int numberOfDirectories = (int)Math.Ceiling((double)((float)photoStorage.Count / cfg.DirectoryLimit));
for (int dn = 0; dn < numberOfDirectories; dn++)
{
string subdir = (dn + 1).ToString("D4");
string dirPath = Path.Combine(cfg.Folder, subdir);
if (!Directory.Exists(dirPath)) Directory.CreateDirectory(dirPath);
var part = photoStorage.Skip(cfg.DirectoryLimit * dn).Take(cfg.DirectoryLimit);
foreach (Photo p in part)
p.SubDirectory = subdir;
}
}
// download and update photos
var unprocessedPhotos = photoStorage.Where(p => p.Status == PhotoStatus.New || p.Status == PhotoStatus.Downloaded).ToArray();
while (unprocessedPhotos.Length > 0)
{
Parallel.ForEach(unprocessedPhotos, photo =>
{
// download or convert
string folder = cfg.Folder;
if (!String.IsNullOrEmpty(photo.SubDirectory))
folder = Path.Combine(cfg.Folder, photo.SubDirectory);
string orgname = Path.Combine(folder, "org." + photo.FileName);
string destname = Path.Combine(folder, photo.FileName);
if (photo.Status == PhotoStatus.New)
{
// try download
DownloadResult result = Web.GetFile(photo.Uri, orgname);
switch (result)
{
case DownloadResult.NotFound:
// just skip the picture
photo.Status = PhotoStatus.Rejected;
break;
case DownloadResult.Error:
// try again
self.ReportProgress(progressValue, CONNECTION_ERROR);
Console.WriteLine("Failed {0}", photo.Uri);
Thread.Sleep(SLEEPING_TIME);
break;
case DownloadResult.Ok:
photo.Status = PhotoStatus.Downloaded;
Console.WriteLine("Read {0}", photo.Uri);
break;
}
}
if (photo.Status == PhotoStatus.Downloaded)
{
// try convert
if (Painter.ApplyPhoto(orgname, photo, cfg, destname))
{
photo.Status = PhotoStatus.Processed;
progressValue = (int)(100.0f / totalCount * Interlocked.Increment(ref processedCount));
try
{
// delete the original file
File.Delete(orgname);
}
catch (Exception) { }
self.ReportProgress(progressValue, "Downloading pictures");
}
else
{
self.ReportProgress(progressValue, CONVERSION_ERROR);
}
}
});
unprocessedPhotos = photoStorage.Where(p => p.Status == PhotoStatus.New || p.Status == PhotoStatus.Downloaded).ToArray();
}
e.Result = String.Format("Downloaded {0} pictures from {1} products",
photoStorage.Count(p => p.Status == PhotoStatus.Processed),
products.Count);
}
public void TokenAwarePolicyRoundRobinsOnLocalReplicas()
{
var hostList = new List<Host>
{
//5 local nodes and 4 remote
TestHelper.CreateHost("0.0.0.1", "dc1"),
TestHelper.CreateHost("0.0.0.2", "dc1"),
TestHelper.CreateHost("0.0.0.3", "dc2"),
TestHelper.CreateHost("0.0.0.4", "dc2"),
TestHelper.CreateHost("0.0.0.5", "dc1"),
TestHelper.CreateHost("0.0.0.6", "dc1"),
TestHelper.CreateHost("0.0.0.7", "dc2"),
TestHelper.CreateHost("0.0.0.8", "dc2"),
TestHelper.CreateHost("0.0.0.9", "dc1")
};
var clusterMock = new Mock<ICluster>(MockBehavior.Strict);
clusterMock
.Setup(c => c.AllHosts())
.Returns(hostList)
.Verifiable();
clusterMock
.Setup(c => c.GetReplicas(It.IsAny<string>(), It.IsAny<byte[]>()))
.Returns<string, byte[]>((keyspace, key) =>
{
var i = key[0];
return hostList.Where(h =>
{
//The host at with address == k and the next one
var address = TestHelper.GetLastAddressByte(h);
return address == i || address == i + 1;
}).ToList();
})
.Verifiable();
var policy = new TokenAwarePolicy(new DCAwareRoundRobinPolicy("dc1", 2));
policy.Initialize(clusterMock.Object);
var firstHosts = new ConcurrentBag<Host>();
var k = new RoutingKey { RawRoutingKey = new byte[] { 1 } };
//key for host :::1 and :::2
var actions = new List<Action>();
const int times = 100;
for (var i = 0; i < times; i++)
{
actions.Add(() =>
{
var h = policy.NewQueryPlan(null, new SimpleStatement().SetRoutingKey(k)).First();
firstHosts.Add(h);
});
}
var parallelOptions = new ParallelOptions();
parallelOptions.TaskScheduler = new ThreadPerTaskScheduler();
parallelOptions.MaxDegreeOfParallelism = 1000;
Parallel.Invoke(parallelOptions, actions.ToArray());
Assert.AreEqual(times, firstHosts.Count);
//Half the times
Assert.AreEqual(times / 2, firstHosts.Count(h => TestHelper.GetLastAddressByte(h) == 1));
Assert.AreEqual(times / 2, firstHosts.Count(h => TestHelper.GetLastAddressByte(h) == 2));
clusterMock.Verify();
}
public void RemoveTest()
{
var TestObject = new ConcurrentBag<int>(new int[] { 1, 2, 3, 4, 5, 6 });
TestObject = new ConcurrentBag<int>(TestObject.Remove((x) => x % 2 == 0));
Assert.Equal(3, TestObject.Count());
foreach (int Item in TestObject)
Assert.False(Item % 2 == 0);
}