public void WhenRemoveLastMessageShouldRemoveIt()
{
var batchMessage = new Message(new Uri("http://post.com"), "payload");
batchMessage.Id = 1;
var batch = new Batch();
batch.Add(batchMessage);
batchMessage = new Message(new Uri("http://post2.com"), "payload2");
batchMessage.Id = 2;
batch.Add(batchMessage);
batchMessage = new Message(new Uri("http://post3.com"), "payload3");
batchMessage.Id = 3;
batch.Add(batchMessage);
batch.RemoveLastMessage();
Assert.Equal(2, batch.Events.Count);
var message = batch.Events[0];
Assert.Equal("http://post.com/", message.Url.AbsoluteUri);
Assert.Equal(1, message.ReynaId);
Assert.Equal("payload", message.Payload);
message = batch.Events[1];
Assert.Equal("http://post2.com/", message.Url.ToString());
Assert.Equal(2, message.ReynaId);
Assert.Equal("payload2", message.Payload);
}
public void batchtestall()
{
ServiceContext context = Initializer.InitializeServiceContextQbo();
DataService service = new DataService(context);
Customer customer1 = new Customer();
string guid = Guid.NewGuid().ToString("n");
customer1.GivenName = guid.Substring(0, 25);
string GivenNameCustomer1 = customer1.GivenName;
customer1.Title = guid.Substring(0, 15);
customer1.MiddleName = guid.Substring(0, 5);
customer1.FamilyName = guid.Substring(0, 25);
customer1.DisplayName = guid.Substring(0, 20);
Customer customer2 = new Customer();
guid = Guid.NewGuid().ToString("n");
customer2.GivenName = guid.Substring(0, 25);
string GivenNameCustomer2 = customer2.GivenName;
customer2.Title = guid.Substring(0, 15);
customer2.MiddleName = guid.Substring(0, 5);
customer2.FamilyName = guid.Substring(0, 25);
customer2.DisplayName = guid.Substring(0, 20);
try
{
Batch batch = service.CreateNewBatch();
batch.Add(customer1, "addcustomer1", OperationEnum.create);
batch.Add(customer2, "addcustomer2", OperationEnum.create);
batch.Add("select * from Customer startPosition 0 maxResults 10", "queryCustomer");
batch.Execute();
IntuitBatchResponse inuititemresponse = batch.IntuitBatchItemResponses[0];
Assert.IsTrue(inuititemresponse.ResponseType == ResponseType.Entity);
Customer resultcustomer1 = inuititemresponse.Entity as Customer;
Assert.IsFalse(string.IsNullOrEmpty(resultcustomer1.Id));
Assert.IsTrue(resultcustomer1.GivenName == GivenNameCustomer1);
inuititemresponse = batch.IntuitBatchItemResponses[1];
Assert.IsTrue(inuititemresponse.ResponseType == ResponseType.Entity);
Customer resultcustomer2 = inuititemresponse.Entity as Customer;
Assert.IsFalse(string.IsNullOrEmpty(resultcustomer2.Id));
Assert.IsTrue(resultcustomer2.GivenName == GivenNameCustomer2);
inuititemresponse = batch.IntuitBatchItemResponses[2];
Assert.IsTrue(inuititemresponse.ResponseType == ResponseType.Query);
List <Customer> customers = inuititemresponse.Entities.ToList().ConvertAll(item => item as Customer);
Assert.IsTrue(customers.Count >= 2);
}
catch (Ipp.Exception.IdsException ex)
{
Assert.Fail(ex.ToString());
}
}
public void BatchIDLengthTest()
{
ServiceContext context = Initializer.InitializeServiceContextQbo();
DataService service = new DataService(context);
Customer customer = new Customer();
string guid = Guid.NewGuid().ToString("N");
customer.GivenName = guid.Substring(0, 25);
customer.Title = guid.Substring(0, 15);
customer.MiddleName = guid.Substring(0, 5);
customer.FamilyName = guid.Substring(0, 25);
customer.DisplayName = guid.Substring(0, 20);
try
{
Batch batch = service.CreateNewBatch();
batch.Add(customer, Guid.NewGuid().ToString("N").Substring(0, 25), OperationEnum.create);
batch.Add(new CDCQuery()
{
ChangedSince = DateTime.Now.AddDays(-1), ChangedSinceSpecified = true, Entities = "Customer"
}, "cdcOpration");
batch.Execute();
}
catch (Ipp.Exception.IdsException ex)
{
Assert.Fail(ex.ToString());
}
}
internal static ReadOnlyCollection <IntuitBatchResponse> Batch <T>(ServiceContext context, Dictionary <OperationEnum, object> operationDictionary) where T : IEntity
{
DataService service = new DataService(context);
List <T> addedList = new List <T>();
List <T> newList = new List <T>();
QueryService <T> entityContext = new QueryService <T>(context);
Batch batch = service.CreateNewBatch();
foreach (KeyValuePair <OperationEnum, object> entry in operationDictionary)
{
if (entry.Value.GetType().Name.Equals(typeof(T).Name))
{
batch.Add(entry.Value as IEntity, entry.Key.ToString() + entry.Value.GetType().Name, entry.Key);
}
else
{
batch.Add(entry.Value as string, "Query" + typeof(T).Name);
}
}
batch.Execute();
return(batch.IntuitBatchItemResponses);
}
public void BatchTest()
{
string accessTokenQBO = ConfigurationManager.AppSettings["AccessTokenQBO"];
string accessTokenSecretQBO = ConfigurationManager.AppSettings["AccessTokenSecretQBO"];
string consumerKeyQBO = ConfigurationManager.AppSettings["ConsumerKeyQBO"];
string ConsumerSecretQBO = ConfigurationManager.AppSettings["ConsumerSecretQBO"];
string realmIAQBO = ConfigurationManager.AppSettings["RealmIAQBO"];
OAuthRequestValidator oAuthRequestValidator = new OAuthRequestValidator(accessTokenQBO, accessTokenSecretQBO, consumerKeyQBO, ConsumerSecretQBO);
ServiceContext context = new ServiceContext(realmIAQBO, IntuitServicesType.QBO, oAuthRequestValidator);
DataService service = new DataService(context);
context.IppConfiguration.Message.Response.CompressionFormat = Intuit.Ipp.Core.Configuration.CompressionFormat.GZip;
context.IppConfiguration.Message.Response.SerializationFormat = Intuit.Ipp.Core.Configuration.SerializationFormat.Json;
Customer customer = new Customer();
string guid = Guid.NewGuid().ToString("N");
customer.GivenName = guid.Substring(0, 25);
customer.Title = guid.Substring(0, 15);
customer.MiddleName = guid.Substring(0, 5);
customer.FamilyName = guid.Substring(0, 25);
customer.DisplayName = guid.Substring(0, 20);
try
{
Batch batch = service.CreateNewBatch();
batch.Add(customer, "addCustomer", OperationEnum.create);
batch.Add(new CDCQuery()
{
ChangedSince = DateTime.Now.AddDays(-1), ChangedSinceSpecified = true, Entities = "Customer"
}, "cdcOpration");
batch.Execute();
IntuitBatchResponse addCustomerResponse = batch.IntuitBatchItemResponses[0];
if (addCustomerResponse.ResponseType != ResponseType.Exception)
{
Customer addedcustomer = addCustomerResponse.Entity as Customer;
Assert.IsNotNull(addedcustomer);
Assert.IsFalse(string.IsNullOrEmpty(addedcustomer.Id));
}
IntuitBatchResponse CDCQueryResponse = batch.IntuitBatchItemResponses[1];
if (CDCQueryResponse.ResponseType != ResponseType.Exception)
{
Dictionary <string, List <IEntity> > cdcCustomers = CDCQueryResponse.CDCResponse.entities;
Assert.IsNotNull(cdcCustomers);
Assert.IsTrue(cdcCustomers.Count > 0);
foreach (KeyValuePair <string, List <IEntity> > entry in cdcCustomers)
{
Assert.IsTrue(entry.Value.ElementAt(0).GetType() == new Customer().GetType());
}
}
else
{
Assert.Fail();
}
}
catch (Ipp.Exception.IdsException ex)
{
Assert.Fail(ex.ToString());
}
}
public void SendTest()
{
string accessKey = pandora_csharp_sdkTests.Properties.Settings.Default.AccessKey;
string secretKey = pandora_csharp_sdkTests.Properties.Settings.Default.SecretKey;
string repoName = pandora_csharp_sdkTests.Properties.Settings.Default.NewRepoName;
Auth auth = new Auth(accessKey, secretKey);
DataSender dataSender = new DataSender(auth);
Batch batch = new Batch();
//timestamp date
//sensor string
//length long
//value float
//working boolean
//alias jsonstring
DateTime timestamp = DateTime.Now;
for (int i = 1; i <= 1000; i++)
{
string sensor = string.Format("传感器{0}", i);
int length = new Random().Next(1000);
double value = new Random().NextDouble() * 1000;
bool working = length % 2 == 0;
List <string> alias = new List <string>();
alias.Add("big");
alias.Add("fast");
Point p = new Point();
p.Append("timestamp", timestamp);
p.Append("sensor", sensor);
p.Append("length", length);
p.Append("value", value);
p.Append("working", working);
p.Append("alias", alias);
if (batch.CanAdd(p))
{
batch.Add(p);
}
else
{
// send the data first
Console.WriteLine("send part batch length: {0}", batch.GetSize());
HttpResponse response1 = dataSender.Send(repoName, batch);
//Console.WriteLine(batch.ToString());
Assert.AreEqual(System.Net.HttpStatusCode.OK, response1.StatusCode);
batch.Clear();// clear for next batch
batch.Add(p);
}
}
if (batch.GetSize() > 0)
{
//send the remained data
Console.WriteLine("send final batch length: {0}", batch.GetSize());
HttpResponse response2 = dataSender.Send(repoName, batch);
//Console.WriteLine(batch.ToString());
Assert.AreEqual(System.Net.HttpStatusCode.OK, response2.StatusCode);
}
}
public void AddEmptyQueryToBatch()
{
Batch batch = GetBatch();
string query = string.Empty;
batch.Add(query, "emptyQuery");
batch.Add("query * from Customer", "customerQuery");
}
public void AddDuplicateItemToBatch()
{
Batch batch = GetBatch();
string queryId = string.Empty;
batch.Add("query * from Invoice", "customerQuery");
batch.Add("query * from Customer", "customerQuery");
}
public void AddDuplicateItemToBatchEntity()
{
Batch batch = GetBatch();
Customer customer1 = GetCustomer();
Customer customer2 = GetCustomer();
batch.Add(customer1, "customerQuery", OperationEnum.create);
batch.Add(customer2, "customerQuery", OperationEnum.create);
}
public void AddNullEntityInBatch()
{
ServiceContext context = Initializer.InitializeServiceContextQbo();
DataService service = new DataService(context);
Customer customer = null;
Batch batch = service.CreateNewBatch();
batch.Add(customer, "Customer", OperationEnum.create);
batch.Add("query * from Customer", "Customer");
}
protected IEnumerator <object> WriteLotsOfValuesInBatch(Tangle <int> tangle, int numIterations, int direction)
{
int batchSize = 256;
Batch <int> batch = null;
if (direction > 0)
{
for (int i = 0; i < numIterations; i++)
{
if (batch == null)
{
batch = Tangle.CreateBatch(batchSize);
}
batch.Add(i, i);
if (batch.Count == batchSize)
{
yield return(batch.Execute());
batch = null;
}
}
}
else
{
for (int i = numIterations - 1; i >= 0; i--)
{
if (batch == null)
{
batch = Tangle.CreateBatch(batchSize);
}
batch.Add(i, i);
if (batch.Count == batchSize)
{
yield return(batch.Execute());
batch = null;
}
}
}
if (batch != null)
{
yield return(batch.Execute());
}
}
public void AddNullIdToBatchEntity()
{
Batch batch = GetBatch();
Customer customer = GetCustomer();
batch.Add(customer, null, OperationEnum.create);
}
public void AddEmptyQueryToBatchEntity()
{
Batch batch = GetBatch();
Customer customer = null;
batch.Add(customer, "emptyCustomer", OperationEnum.create);
}
public void AddNullIdToBatch()
{
Batch batch = GetBatch();
string queryId = string.Empty;
batch.Add("query * from Invoice", null);
}
private void HandleSold(ItemViewModel item)
{
if (item.IsSold)
{
SearchItemText = string.Empty;
if (!Batch.Contains(item))
{
Batch.Add(item);
}
}
else
{
Batch.Remove(item);
}
var seller = Sellers.Where(s => s.Id == item.Seller.Id || s is AllSellerViewModel)
.Select(s => {
s.Update();
return(s);
})
.ToList();
RaisePropertyChanged(nameof(BatchValue));
}
public void AddNullEntityInBatch()
{
string accessTokenQBO = ConfigurationManager.AppSettings["AccessTokenQBO"];
string accessTokenSecretQBO = ConfigurationManager.AppSettings["AccessTokenSecretQBO"];
string consumerKeyQBO = ConfigurationManager.AppSettings["ConsumerKeyQBO"];
string ConsumerSecretQBO = ConfigurationManager.AppSettings["ConsumerSecretQBO"];
string realmIAQBO = ConfigurationManager.AppSettings["RealmIAQBO"];
OAuthRequestValidator oAuthRequestValidator = new OAuthRequestValidator(accessTokenQBO, accessTokenSecretQBO, consumerKeyQBO, ConsumerSecretQBO);
ServiceContext context = new ServiceContext(realmIAQBO, IntuitServicesType.QBO, oAuthRequestValidator);
DataService service = new DataService(context);
Customer customer = null;
Batch batch = service.CreateNewBatch();
batch.Add(customer, "Customer", OperationEnum.create);
batch.Add("query * from Customer", "Customer");
}
/// <summary>Appends a byte array to the current batch.</summary>
/// <param name="data">The byte array containing the data to append.</param>
public void Append(Byte[] data)
{
if (data.Length > m_maxBatchBytes)
{
throw new ArgumentOutOfRangeException("data", "A single data object cannot be larger than " + m_maxBatchBytes.ToString() + " bytes.");
}
if (m_minimalLossEnabled)
{
TransferBatchAsync(this, new Batch {
data
}); return;
}
Boolean taken = false;
m_lock.Enter(ref taken);
Batch oldBatch = null;
if (m_batchBytes + data.Length > m_maxBatchBytes)
{
// Data won't fit in current batch, create a new batch so we can transfer the old batch
oldBatch = CreateNewBatch();
}
// Add new data to current (or new) batch
m_batch.Add(data);
m_batchBytes += data.Length;
m_lock.Exit();
// If batch swapped, transfer it
TransferBatchAsync(this, oldBatch);
}
public void AddEmptyIdToBatchEntity()
{
Batch batch = GetBatch();
string queryId = string.Empty;
Customer customer = GetCustomer();
batch.Add(customer, queryId, OperationEnum.create);
}
public void UseSameBatchIDTwice()
{
ServiceContext context = Initializer.InitializeServiceContextQbo();
DataService service = new DataService(context);
Customer customer = new Customer();
string guid = Guid.NewGuid().ToString("N");
customer.GivenName = guid.Substring(0, 25);
customer.Title = guid.Substring(0, 15);
customer.MiddleName = guid.Substring(0, 5);
customer.FamilyName = guid.Substring(0, 25);
customer.DisplayName = guid.Substring(0, 20);
Batch batch = service.CreateNewBatch();
batch.Add(customer, "Customer", OperationEnum.create);
batch.Add("query * from Customer", "Customer");
}
internal void AddToBatch(MySqlCommand command)
{
if (Batch == null)
{
Batch = new List <MySqlCommand>();
}
Batch.Add(command);
}
public static void Main(string[] args)
{
if (args.Length != 4)
{
Console.Error.WriteLine("Usage: SHS.PageRank <leader> <store> <d> <iters>");
}
else
{
var sw = Stopwatch.StartNew();
var store = new Service(args[0]).OpenStore(Guid.Parse(args[1]));
double d = double.Parse(args[2]);
int numIters = int.Parse(args[3]);
long n = store.NumUrls();
using (var wr = new BinaryWriter(new BufferedStream(new FileStream("pr-scores-" + 0 + ".bin", FileMode.Create, FileAccess.Write)))) {
for (long i = 0; i < n; i++)
{
wr.Write(1.0 / n);
}
}
var scores = store.AllocateUidState <double>();
var uidBatch = new Batch <long>(50000);
for (int k = 0; k < numIters; k++)
{
scores.SetAll(x => d / n);
using (var rd = new BinaryReader(new BufferedStream(new FileStream("pr-scores-" + k + ".bin", FileMode.Open, FileAccess.Read)))) {
foreach (long u in store.Uids())
{
uidBatch.Add(u);
if (uidBatch.Full || store.IsLastUid(u))
{
var linkBatch = store.BatchedGetLinks(uidBatch, Dir.Fwd);
var uniqLinks = new UidMap(linkBatch);
var scoreArr = scores.GetMany(uniqLinks);
foreach (var links in linkBatch)
{
double f = (1.0 - d) * rd.ReadDouble() / links.Length;
foreach (var link in links)
{
scoreArr[uniqLinks[link]] += f;
}
}
scores.SetMany(uniqLinks, scoreArr);
uidBatch.Reset();
}
}
}
using (var wr = new BinaryWriter(new BufferedStream(new FileStream("pr-scores-" + (k + 1) + ".bin", FileMode.Create, FileAccess.Write)))) {
foreach (var us in scores.GetAll())
{
wr.Write(us.val);
}
}
File.Delete("pr-scores-" + k + ".bin");
Console.WriteLine("Iteration {0} complete", k);
}
Console.WriteLine("Done. {0} iterations took {1} seconds.", numIters, 0.001 * sw.ElapsedMilliseconds);
}
}
public void WhenCallingToJsonShouldReturnExpected()
{
var payload1 = "{\"Version Incremental\":\"20150514.093204\",\"User\":\"user\",\"Brand\":\"os\",\"device\":\"device\",\"Network Operator Name\":\"unknown\",\"Manufacturer\":\"MM\",\"updated\":1448534970738,\"list\":[{\"name\":\"System\",\"version\":\"4.1.1\",\"flag\":true},{\"name\":\"backupcon\",\"version\":\"4.1.1\",\"flag\":false}],\"Version Release\":\"4.1.1\",\"MAC\":\"MAC\"}";
var batchMessage1 = new Message(new Uri("http://post.com"), payload1);
batchMessage1.Id = 12;
var payload2 = "{\"Brand\":\"os\",\"device\":\"device\",\"Network Operator Name\":\"unknown\",\"Manufacturer\":\"MM\",\"updated\":1448534970738,\"list\":[{\"name\":\"System\",\"version\":\"4.1.1\",\"flag\":true},{\"name\":\"backupcon\",\"version\":\"4.1.1\",\"flag\":false}],\"Version Release\":\"4.1.1\",\"MAC\":\"MAC\"}";
var batchMessage2 = new Message(new Uri("http://post2.com"), payload2);
batchMessage2.Id = 14;
var batch = new Batch();
batch.Add(batchMessage1);
batch.Add(batchMessage2);
var expected = "{\"events\":[{\"url\":\"http://post.com/\", \"reynaId\":12, \"payload\":{\"Version Incremental\":\"20150514.093204\",\"User\":\"user\",\"Brand\":\"os\",\"device\":\"device\",\"Network Operator Name\":\"unknown\",\"Manufacturer\":\"MM\",\"updated\":1448534970738,\"list\":[{\"name\":\"System\",\"version\":\"4.1.1\",\"flag\":true},{\"name\":\"backupcon\",\"version\":\"4.1.1\",\"flag\":false}],\"Version Release\":\"4.1.1\",\"MAC\":\"MAC\"}}, {\"url\":\"http://post2.com/\", \"reynaId\":14, \"payload\":{\"Brand\":\"os\",\"device\":\"device\",\"Network Operator Name\":\"unknown\",\"Manufacturer\":\"MM\",\"updated\":1448534970738,\"list\":[{\"name\":\"System\",\"version\":\"4.1.1\",\"flag\":true},{\"name\":\"backupcon\",\"version\":\"4.1.1\",\"flag\":false}],\"Version Release\":\"4.1.1\",\"MAC\":\"MAC\"}}]}";
var actual = batch.ToJson();
Assert.Equal(expected, actual);
}
public void AddTwentyFivePlusToBatch()
{
Batch batch = GetBatch();
string queryId = string.Empty;
for (int i = 0; i <= 26; i++)
{
batch.Add("query * from Customer", "customerQuery");
}
}
public void AddTwentyFivePlusToBatchEntity()
{
Batch batch = GetBatch();
string queryId = string.Empty;
for (int i = 0; i <= 26; i++)
{
Customer customer = GetCustomer();
batch.Add(customer, "Customer", OperationEnum.create);
}
}
public void RemoveWrongItem()
{
Batch batch = GetBatch();
for (int i = 0; i <= 10; i++)
{
Customer customer = GetCustomer();
batch.Add(customer, "Customer" + i.ToString(), OperationEnum.create);
}
batch.Remove("Customer20");
}
protected override bool Idle(object message)
{
if (message is ReplicationWrite)
{
Batch = Batch.Add(((ReplicationWrite)message).WithReplyToDefault(Sender));
TryWriteBatch();
Context.Become(Writing);
return(true);
}
return(false);
}
public void RemoveItem()
{
Batch batch = GetBatch();
for (int i = 0; i <= 10; i++)
{
Customer customer = GetCustomer();
batch.Add(customer, "Customer" + i.ToString(), OperationEnum.create);
}
batch.Remove("Customer0");
IntuitBatchResponse item = batch["Customer0"] as IntuitBatchResponse;
}
public void TestBatch()
{
List <long> list = new List <long>();
FlushReason lastReason = FlushReason.AfterAdd;
Batch <long> batch = new Batch <long>(TimeSpan.FromSeconds(0.2), 1000L, l => l,
(lst, counts, reason) =>
{
list.AddRange(lst);
lastReason = reason;
});
list.Clear();
batch.Add(500);
Thread.Sleep(1000);
//Should flush
Assert.Single(list);
Assert.Equal(FlushReason.Timer, lastReason);
list.Clear();
batch.Add(500);
batch.Add(700);
//Should flush
Assert.Single(list);
Assert.Equal(FlushReason.BeforeAdd, lastReason);
list.Clear();
batch.Add(300);
//Should flush
Assert.Equal(2, list.Count);
Assert.Equal(FlushReason.AfterAdd, lastReason);
list.Clear();
batch.Add(500);
//Should not flush
Assert.Empty(list);
batch.Stop();
//Should flush now
Assert.Single(list);
Assert.Equal(FlushReason.Stop, lastReason);
}
private void RunSingle(MessageDispatch dispatch, IMessageHandlerInvoker invoker)
{
if (!_isRunning)
{
return;
}
var batch = new Batch(1);
batch.Add(new Entry(dispatch, invoker));
RunBatch(batch);
}
public void WhenCallingToJsonShouldReturnExpected()
{
var payload1 = "{\"Version Incremental\":\"20150514.093204\",\"User\":\"user\",\"Brand\":\"os\",\"device\":\"device\",\"Network Operator Name\":\"unknown\",\"Manufacturer\":\"MM\",\"updated\":1448534970738,\"list\":[{\"name\":\"System\",\"version\":\"4.1.1\",\"flag\":true},{\"name\":\"backupcon\",\"version\":\"4.1.1\",\"flag\":false}],\"Version Release\":\"4.1.1\",\"MAC\":\"MAC\"}";
var batchMessage1 = new Message(new Uri("http://post.com"), payload1);
batchMessage1.Id = 12;
var payload2 = "{\"Brand\":\"os\",\"device\":\"device\",\"Network Operator Name\":\"unknown\",\"Manufacturer\":\"MM\",\"updated\":1448534970738,\"list\":[{\"name\":\"System\",\"version\":\"4.1.1\",\"flag\":true},{\"name\":\"backupcon\",\"version\":\"4.1.1\",\"flag\":false}],\"Version Release\":\"4.1.1\",\"MAC\":\"MAC\"}";
var batchMessage2 = new Message(new Uri("http://post2.com"), payload2);
batchMessage2.Id = 14;
var batch = new Batch();
batch.Add(batchMessage1);
batch.Add(batchMessage2);
var expected = "{\"events\":[{\"url\":\"http://post.com/\", \"reynaId\":12, \"payload\":{\"Version Incremental\":\"20150514.093204\",\"User\":\"user\",\"Brand\":\"os\",\"device\":\"device\",\"Network Operator Name\":\"unknown\",\"Manufacturer\":\"MM\",\"updated\":1448534970738,\"list\":[{\"name\":\"System\",\"version\":\"4.1.1\",\"flag\":true},{\"name\":\"backupcon\",\"version\":\"4.1.1\",\"flag\":false}],\"Version Release\":\"4.1.1\",\"MAC\":\"MAC\"}}, {\"url\":\"http://post2.com/\", \"reynaId\":14, \"payload\":{\"Brand\":\"os\",\"device\":\"device\",\"Network Operator Name\":\"unknown\",\"Manufacturer\":\"MM\",\"updated\":1448534970738,\"list\":[{\"name\":\"System\",\"version\":\"4.1.1\",\"flag\":true},{\"name\":\"backupcon\",\"version\":\"4.1.1\",\"flag\":false}],\"Version Release\":\"4.1.1\",\"MAC\":\"MAC\"}}]}";
var actual = batch.ToJson();
Assert.Equal(expected, actual);
}
public void WhenCallingAddShouldAddNewMessage()
{
var batchMessage = new Message(new Uri("http://post.com/1/info"), "payload");
batchMessage.Id = 1;
var batch = new Batch();
batch.Add(batchMessage);
Assert.Equal(1, batch.Events.Count);
var message = batch.Events[0];
Assert.Equal("http://post.com/1/info", message.Url.AbsoluteUri);
Assert.Equal(1, message.ReynaId);
Assert.Equal("payload", message.Payload);
}
public UserInterfaceObserver()
{
_timer.AutoReset = true;
_timer.Elapsed += (sender, args) =>
{
if (_batch.messages.Any())
{
Batch batch = null;
_lock.Write(() =>
{
batch = _batch;
_batch = new Batch();
});
var passthrough = new PassthroughMessage(batch);
EventAggregator.SendMessage(passthrough);
}
};
_timer.Enabled = true;
_timer.Start();
_readingTask = Task.Factory.StartNew(() =>
{
foreach (var message in _messages.GetConsumingEnumerable())
{
if (message is IBatchedMessage)
{
// Just want this to have a lower priority
_lock.Read(() =>
{
_batch.Add(message);
return(string.Empty);
});
}
else
{
var passthrough = new PassthroughMessage(message);
EventAggregator.SendMessage(passthrough);
// TODO -- really, really don't like this
if (message is SpecExecutionCompleted)
{
EventAggregator.SendMessage(message);
}
}
}
});
}
public void WhenRemoveLastMessageAndOnlyOneMessageAvailableShouldNotRemoveIt()
{
var batchMessage = new Message(new Uri("http://post.com/api"), "payload");
batchMessage.Id = 1;
var batch = new Batch();
batch.Add(batchMessage);
batch.RemoveLastMessage();
Assert.Equal(1, batch.Events.Count);
var message = batch.Events[0];
Assert.Equal("http://post.com/api", message.Url.ToString());
Assert.Equal(1, message.ReynaId);
Assert.Equal("payload", message.Payload);
}
public static void Main(string[] args)
{
if (args.Length != 4) {
Console.Error.WriteLine("Usage: SHS.PageRank <leader> <store> <d> <iters>");
} else {
var sw = Stopwatch.StartNew();
var store = new Service(args[0]).OpenStore(Guid.Parse(args[1]));
double d = double.Parse(args[2]);
int numIters = int.Parse(args[3]);
long n = store.NumUrls();
using (var wr = new BinaryWriter(new BufferedStream(new FileStream("pr-scores-" + 0 + ".bin", FileMode.Create, FileAccess.Write)))) {
for (long i = 0; i < n; i++) wr.Write(1.0 / n);
}
var scores = store.AllocateUidState<double>();
var uidBatch = new Batch<long>(50000);
for (int k = 0; k < numIters; k++) {
scores.SetAll(x => d / n);
using (var rd = new BinaryReader(new BufferedStream(new FileStream("pr-scores-" + k + ".bin", FileMode.Open, FileAccess.Read)))) {
foreach (long u in store.Uids()) {
uidBatch.Add(u);
if (uidBatch.Full || store.IsLastUid(u)) {
var linkBatch = store.BatchedGetLinks(uidBatch, Dir.Fwd);
var uniqLinks = new UidMap(linkBatch);
var scoreArr = scores.GetMany(uniqLinks);
foreach (var links in linkBatch) {
double f = (1.0 - d) * rd.ReadDouble() / links.Length;
foreach (var link in links) {
scoreArr[uniqLinks[link]] += f;
}
}
scores.SetMany(uniqLinks, scoreArr);
uidBatch.Reset();
}
}
}
using (var wr = new BinaryWriter(new BufferedStream(new FileStream("pr-scores-" + (k + 1) + ".bin", FileMode.Create, FileAccess.Write)))) {
foreach (var us in scores.GetAll()) wr.Write(us.val);
}
File.Delete("pr-scores-" + k + ".bin");
Console.WriteLine("Iteration {0} complete", k);
}
Console.WriteLine("Done. {0} iterations took {1} seconds.", numIters, 0.001 * sw.ElapsedMilliseconds);
}
}
public static void Main(string[] args)
{
if (args.Length != 4) {
Console.Error.WriteLine("Usage: SHS.PageRankFT <leader> <store> <d> <iters>");
} else {
var sw = Stopwatch.StartNew();
var store = new Service(args[0]).OpenStore(Guid.Parse(args[1]));
Action<Action> Checkpointed = delegate(Action checkpointedBlock) {
while (true) {
try {
checkpointedBlock();
store.Checkpoint();
break;
} catch (ServerFailure) {
Console.Error.WriteLine("Restarting from checkpoint");
// go again
}
}
};
double d = double.Parse(args[2]);
int numIters = int.Parse(args[3]);
long n = store.NumUrls();
UidState<double> oldScores = null, newScores = null;
Checkpointed(delegate() {
newScores = store.AllocateUidState<double>();
oldScores = store.AllocateUidState<double>();
oldScores.SetAll(uid => 1.0 / n);
});
for (int k = 0; k < numIters; k++) {
Checkpointed(delegate() {
var uidBatch = new Batch<long>(50000);
newScores.SetAll(x => d / n);
foreach (long u in store.Uids()) {
uidBatch.Add(u);
if (uidBatch.Full || store.IsLastUid(u)) {
var linkBatch = store.BatchedGetLinks(uidBatch, Dir.Fwd);
var newMap = new UidMap(linkBatch);
var oldSc = oldScores.GetMany(uidBatch);
var newSc = newScores.GetMany(newMap);
for (int i = 0; i < uidBatch.Count; i++) {
var links = linkBatch[i];
double f = (1.0 - d) * oldSc[i] / links.Length;
foreach (var link in links) {
newSc[newMap[link]] += f;
}
}
newScores.SetMany(newMap, newSc);
uidBatch.Reset();
}
}
});
var tmp = newScores; newScores = oldScores; oldScores = tmp;
Console.WriteLine("Done with iteration {0}", k);
}
using (var wr = new BinaryWriter(new BufferedStream(new FileStream("pr-scores.bin", FileMode.Create, FileAccess.Write)))) {
foreach (var us in newScores.GetAll()) wr.Write(us.val);
}
Console.WriteLine("Done. {0} iterations took {1} seconds.", numIters, 0.001 * sw.ElapsedMilliseconds);
}
}
public static void Main(string[] args)
{
if (args.Length != 3) {
Console.Error.WriteLine("Usage: SHS.ASP <leader> <store> [f|b|u]");
} else {
var sw = Stopwatch.StartNew();
var shs = new Service(args[0]).OpenStore(Guid.Parse(args[1]));
var dists = shs.AllocateUidState<byte>();
var seeds = shs.AllocateUidState<long>();
var cands = shs.AllocateUidState<bool>();
var batch = new Batch<long>(1000000);
long numCands = 0;
bool fwd = args[2] == "f" || args[2] == "u";
bool bwd = args[2] == "b" || args[2] == "u";
foreach (long u in shs.Uids()) {
batch.Add(u);
if (batch.Full || shs.IsLastUid(u)) {
int[] fwdDegs = shs.BatchedGetDegree(batch, Dir.Fwd);
int[] bwdDegs = shs.BatchedGetDegree(batch, Dir.Bwd);
bool[] isCands = new bool[batch.Count];
for (int i = 0; i < batch.Count; i++) {
isCands[i] = fwd && bwd ? fwdDegs[i] + bwdDegs[i] > 1 : fwdDegs[i] > 0 && bwdDegs[i] > 0;
if (isCands[i]) numCands++;
}
cands.SetMany(batch, isCands);
batch.Reset();
}
}
System.Random rand = new System.Random(12345);
int dim = 0;
batch = new Batch<long>(1000);
for (; (long)1 << dim <= numCands; dim++) {
long numSeeds = (long)1 << dim;
dists.SetAll(x => 0xff);
seeds.SetAll(x => -1);
double remainingSamps = numSeeds;
double remainingCands = numCands;
foreach (var uc in cands.GetAll()) {
if (uc.val) {
if (rand.NextDouble() < remainingSamps / remainingCands) {
batch.Add(uc.uid);
remainingSamps--;
}
remainingCands--;
}
if (batch.Full || shs.IsLastUid(uc.uid)) {
dists.SetMany(batch, ((long[])batch).Select(x => (byte)0).ToArray());
seeds.SetMany(batch, batch);
batch.Reset();
}
}
for (byte k = 0; k < 0xff; k++) {
long hits = 0;
foreach (var x in dists.GetAll()) {
if (x.val == k) {
batch.Add(x.uid);
hits++;
}
if (batch.Full || shs.IsLastUid(x.uid)) {
if (bwd) ProcessBatch(shs, dists, seeds, batch, (byte)(k + 1), Dir.Fwd);
if (fwd) ProcessBatch(shs, dists, seeds, batch, (byte)(k + 1), Dir.Bwd);
batch.Reset();
}
}
if (hits == 0) break;
}
using (var wr = new BinaryWriter(new GZipStream(new BufferedStream(new FileStream("sketchslice-" + args[2] + "-" + dim.ToString("d2") + ".bin", FileMode.Create, FileAccess.Write)), CompressionMode.Compress))) {
long rch = 0; // Number of reachable URls
foreach (var x in dists.GetAll().Zip(seeds.GetAll(), (d, s) => System.Tuple.Create(d, s))) {
if (x.Item1.val < 0xff) rch++;
wr.Write(x.Item1.val);
wr.Write(x.Item2.val);
}
}
}
using (var wr = new BinaryWriter(new GZipStream(new BufferedStream(new FileStream("sketches-" + args[2] + ".bin", FileMode.Create, FileAccess.Write)), CompressionMode.Compress))) {
wr.Write(dim);
var readers = new BinaryReader[dim];
for (int i = 0; i < dim; i++) {
readers[i] = new BinaryReader(new GZipStream(new BufferedStream(new FileStream("sketchslice-" + args[2] + "-" + i.ToString("d2") + ".bin", FileMode.Open, FileAccess.Read)), CompressionMode.Decompress));
}
while (true) {
try {
for (int i = 0; i < dim; i++) {
wr.Write(readers[i].ReadByte());
wr.Write(readers[i].ReadInt64());
}
} catch (EndOfStreamException) {
break;
}
}
for (int i = 0; i < dim; i++) {
readers[i].Close();
}
}
Console.WriteLine("Done. Job took {0} seconds.", 0.001 * sw.ElapsedMilliseconds);
}
}
public void WhenRemoveLastMessageShouldRemoveIt()
{
var batchMessage = new Message(new Uri("http://post.com"), "payload");
batchMessage.Id = 1;
var batch = new Batch();
batch.Add(batchMessage);
batchMessage = new Message(new Uri("http://post2.com"), "payload2");
batchMessage.Id = 2;
batch.Add(batchMessage);
batchMessage = new Message(new Uri("http://post3.com"), "payload3");
batchMessage.Id = 3;
batch.Add(batchMessage);
batch.RemoveLastMessage();
Assert.Equal(2, batch.Events.Count);
var message = batch.Events[0];
Assert.Equal("http://post.com/", message.Url.AbsoluteUri);
Assert.Equal(1, message.ReynaId);
Assert.Equal("payload", message.Payload);
message = batch.Events[1];
Assert.Equal("http://post2.com/", message.Url.ToString());
Assert.Equal(2, message.ReynaId);
Assert.Equal("payload2", message.Payload);
}
public static void Main(string[] args)
{
if (args.Length != 2) {
Console.Error.WriteLine("Usage: SHS.SCC2 <leader> <store>");
} else {
var sw = Stopwatch.StartNew();
var shs = new Service(args[0]).OpenStore(Guid.Parse(args[1]));
var map = shs.AllocateUidState<int>(); // Mapping from UID to local ID
int numVerts = 0; // Number of core vertices
var batch = new Batch<long>(500000);
foreach (long u in shs.Uids()) {
batch.Add(u);
if (batch.Full || shs.IsLastUid(u)) {
int[] fwdDegs = shs.BatchedGetDegree(batch, Dir.Fwd);
int[] bwdDegs = shs.BatchedGetDegree(batch, Dir.Bwd);
var mapChunk = new int[batch.Count];
for (int i = 0; i < batch.Count; i++) {
mapChunk[i] = fwdDegs[i] == 0 || bwdDegs[i] == 0 ? -1 : numVerts++;
}
map.SetMany(batch, mapChunk);
batch.Reset();
}
}
uint numEdges = 0;
foreach (var up in map.GetAll()) {
if (up.val != -1) batch.Add(up.uid);
if (batch.Full || shs.IsLastUid(up.uid)) {
long[][] nbors = shs.BatchedGetLinks(batch, Dir.Fwd);
int[] mappedNbors = map.GetMany(Flatten(nbors));
int q = 0;
for (int i = 0; i < nbors.Length; i++) {
for (int j = 0; j < nbors[i].Length; j++) {
if (mappedNbors[q++] != -1) numEdges++;
}
}
batch.Reset();
}
}
uint[] pastLast = new uint[numVerts]; // one past last link of that page
var links = new int[numEdges];
int p = 0;
uint r = 0;
foreach (var up in map.GetAll()) {
if (up.val != -1) batch.Add(up.uid);
if (batch.Full || shs.IsLastUid(up.uid)) {
long[][] nbors = shs.BatchedGetLinks(batch, Dir.Fwd);
int[] mappedNbors = map.GetMany(Flatten(nbors));
int q = 0;
for (int i = 0; i < nbors.Length; i++) {
for (int j = 0; j < nbors[i].Length; j++) {
int id = mappedNbors[q++];
if (id != -1) links[r++] = id;
}
pastLast[p++] = r;
}
batch.Reset();
}
}
var bv = new BitVector(numVerts); // All false at creation
int[] stk = new int[numVerts];
int stkPtr = stk.Length;
for (int u = 0; u < numVerts; u++) {
if (!bv[u]) {
bv[u] = true;
Frame frame = new Frame(null, u, pastLast);
while (frame != null) {
while (frame.ctr < pastLast[frame.id]) {
int v = links[frame.ctr++];
if (!bv[v]) {
bv[v] = true;
frame = new Frame(frame, v, pastLast);
}
}
stk[--stkPtr] = frame.id;
frame = frame.parent;
}
}
}
p = 0;
r = 0;
foreach (var up in map.GetAll()) {
if (up.val != -1) batch.Add(up.uid);
if (batch.Full || shs.IsLastUid(up.uid)) {
long[][] nbors = shs.BatchedGetLinks(batch, Dir.Bwd);
int[] mappedNbors = map.GetMany(Flatten(nbors));
int q = 0;
for (int i = 0; i < nbors.Length; i++) {
for (int j = 0; j < nbors[i].Length; j++) {
int id = mappedNbors[q++];
if (id != -1) links[r++] = id;
}
pastLast[p++] = r;
}
batch.Reset();
}
}
var pam = new long[numVerts];
p = 0;
foreach (var up in map.GetAll()) {
if (up.val != -1) pam[p++] = up.uid;
}
using (var sccWr = new BinaryWriter(new BufferedStream(new FileStream("scc-main.bin", FileMode.Create, FileAccess.Write)))) {
using (var idxWr = new BinaryWriter(new BufferedStream(new FileStream("scc-index.bin", FileMode.Create, FileAccess.Write)))) {
long sccPos = 0;
bv.SetAll(false);
for (int i = 0; i < stk.Length; i++) {
int u = stk[i];
if (!bv[u]) {
long sccSize = 0;
bv[u] = true;
Frame frame = new Frame(null, u, pastLast);
while (frame != null) {
while (frame.ctr < pastLast[frame.id]) {
int v = links[frame.ctr++];
if (!bv[v]) {
bv[v] = true;
frame = new Frame(frame, v, pastLast);
}
}
sccWr.Write(pam[frame.id]);
sccSize++;
frame = frame.parent;
}
idxWr.Write(sccSize);
idxWr.Write(sccPos);
sccPos += sccSize;
}
}
foreach (var up in map.GetAll()) {
if (up.val == -1) {
sccWr.Write(up.uid);
idxWr.Write(1L);
idxWr.Write(sccPos++);
}
}
}
}
var dict = new System.Collections.Generic.Dictionary<long, long>();
using (var ib = new BinaryReader(new BufferedStream(new FileStream("scc-index.bin", FileMode.Open, FileAccess.Read)))) {
while (true) {
try {
long size = ib.ReadInt64();
long pos = ib.ReadInt64();
if (!dict.ContainsKey(size)) dict[size] = 0;
dict[size]++;
} catch (EndOfStreamException) {
break;
}
}
}
long maxSize = 0;
long numSCCs = 0;
foreach (var kv in dict) {
if (kv.Key > maxSize) maxSize = kv.Key;
numSCCs += kv.Value;
}
Console.WriteLine("Done. Job took {0} seconds.", 0.001 * sw.ElapsedMilliseconds);
}
}