public void CanCreateUpdateAndDeleteUserSegments()
{
var userSegment = new UserSegment()
{
Name = "My Test User Segment",
UserType = UserType.signed_in_users
};
var res = api.HelpCenter.UserSegments.CreateUserSegment(userSegment);
Assert.Greater(res.UserSegment.Id, 0);
res.UserSegment.UserType = UserType.staff;
var update = api.HelpCenter.UserSegments.UpdateUserSegment(res.UserSegment);
Assert.That(update.UserSegment.UserType, Is.EqualTo(res.UserSegment.UserType));
Assert.That(api.HelpCenter.UserSegments.DeleteUserSegment(res.UserSegment.Id.Value), Is.True);
}
public async Task CanCreateUpdateAndDeleteUserSegmentsAsync()
{
var userSegment = new UserSegment()
{
Name = "My Test User Segment Async",
UserType = UserType.signed_in_users
};
var res = await api.HelpCenter.UserSegments.CreateUserSegmentAsync(userSegment);
Assert.True(res.UserSegment.Id > 0);
res.UserSegment.UserType = UserType.staff;
var update = await api.HelpCenter.UserSegments.UpdateUserSegmentAsync(res.UserSegment);
Assert.Equal(update.UserSegment.UserType, res.UserSegment.UserType);
Assert.True(await api.HelpCenter.UserSegments.DeleteUserSegmentAsync(res.UserSegment.Id.Value));
}
private void ProceedSegmentCounter()
{
UserSegment segment = null;
foreach (var lexeme in Lexemes)
{
if (lexeme.Error != null)
{
continue;
}
if (lexeme.Tokens.Count == 2 &&
lexeme.Tokens[1].Type == TokenType.SegmentKeyword)
{
segment = UserSegments.Find(p => p.Name == lexeme.Tokens[0].StringValue);
}
else if (lexeme.Tokens.Count == 2 && lexeme.Tokens[1].Type == TokenType.EndsKeyword)
{
segment = null;
}
else
{
// Только Equ может не иметь сегмента
if (segment == null)
{
if (lexeme.Tokens.Count == 3 && lexeme.Tokens[1].Type == TokenType.EquDirective ||
lexeme.Tokens.Count == 1 && lexeme.Tokens[0].Type == TokenType.EndKeyword)
{
continue;
}
else
{
lexeme.Error = new Error(ErrorType.UnexpectedDirective, lexeme.Tokens[0]);
continue;
}
}
lexeme.Segment = segment;
}
}
}
public void LoadUserSegment(byte[] bytes, UserSegment segment)
{
this.Bytes = bytes;
if (segment != null)
{
this.SegmentName.Text = segment.Name;
this.Offset.Text = segment.Offset.ToString("X");
this.Length.Text = segment.Length.ToString("X");
this.Address.Text = segment.Address.ToString();
interactor.SelectArchitecture(segment.Architecture.Name);
this.ReadMode.Checked = (segment.AccessMode & AccessMode.Read) != 0;
this.WriteMode.Checked = (segment.AccessMode & AccessMode.Read) != 0;
this.ExecuteMode.Checked = (segment.AccessMode & AccessMode.Read) != 0;
}
else
{
this.Offset.Text = "0";
this.Length.Text = "0";
this.Address.Text = "0";
}
}
public IResponse <NoValue> AddUserSegment(UserSegmentCreateModel createModel)
{
Logger.Message($"AddUserSegment({createModel.Name})");
var result = new Response <NoValue>();
try
{
var userSegment = new UserSegment
{
Name = createModel.Name,
ZipCode = createModel.ZipCode,
DateOfBirthFrom = createModel.DateOfBirthFrom,
DateOfBirthTo = createModel.DateOfBirthTo,
Gender = (int)createModel.Gender
};
_uow.UserSegments.Add(userSegment);
_uow.Complete();
result.Status = Status.Success;
}
catch (Exception ex)
{
if (ex.InnerException != null && ex.InnerException.InnerException != null && ex.InnerException.InnerException.Message.Contains("duplicate key"))
{
result.Message = Message.UniqueName;
}
else
{
result.Message = Message.SomethingWrongError;
}
result.Status = Status.Error;
Logger.ErrorWithException(ex);
}
return(result);
}
public void SaveProgress(UserSegment userSegment, long count, int term)
{
List <long> countList;
switch (userSegment)
{
case UserSegment.MuKakin:
countList = this.MukakinUserCounts;
break;
case UserSegment.BiKakin:
countList = this.BikakinUserCounts;
break;
case UserSegment.TyuKakin:
countList = this.TyukakinUserCounts;
break;
case UserSegment.JuKakin:
countList = this.JukakinUserCounts;
break;
case UserSegment.Sekiyuo:
countList = this.SekiyuoCounts;
break;
default:
throw new System.ArgumentException(nameof(userSegment));
}
for (var i = countList.Count; i < term + 1; i++)
{
countList.Add(0L);
}
countList[term] = count;
}
private List <UserSegment> ExtractSegmentDataFromXml(XDocument xDoc)
{
List <UserSegment> segments = new List <UserSegment>();
List <UserSegment> customSegments = new List <UserSegment>();
XNamespace nsDxp = "http://schemas.google.com/analytics/2009";
XNamespace nsAtom = "http://www.w3.org/2005/Atom";
XName entryElementName = nsAtom + "entry";
XName segmentElementName = nsDxp + "segment";
IEnumerable <XElement> segmentElements = xDoc.Root.Elements(entryElementName);
UserSegment noSegment = new UserSegment();
noSegment.SegmentName = "Default (use if uncertain)";
noSegment.SegmentId = "";
segments.Add(noSegment);
foreach (XElement temp in segmentElements)
{
XElement segmentElement = temp.Element(segmentElementName);
string segmentId = segmentElement.Attribute("id").Value;
string segmentName = segmentElement.Attribute("name").Value;
int iSegmentId;
if (int.TryParse(segmentId.Substring(segmentId.IndexOf(':') + 2), out iSegmentId) && iSegmentId > -1)
{
customSegments.Add(new UserSegment()
{
SegmentName = segmentName, SegmentId = segmentId
});
}
else
{
segments.Add(new UserSegment()
{
SegmentName = segmentName, SegmentId = segmentId
});
}
}
customSegments.Sort(delegate(UserSegment a, UserSegment b) { return(a.SegmentName.CompareTo(b.SegmentName)); });
if (customSegments.Count > 0)
{
segments.Add(new UserSegment()
{
SegmentName = "____________________________", SegmentId = ""
});
}
segments.AddRange(customSegments);
return(segments);
//XNamespace dxp = "http://schemas.google.com/analytics/2009";
//XName segmentElementName = dxp + "segment";
//IEnumerable<XElement> segmentElements = xDoc.Root.Elements(segmentElementName);
//UserSegment noSegment = new UserSegment();
//noSegment.SegmentName = "Default (use if uncertain)";
//noSegment.SegmentId = "";
//segments.Add(noSegment);
//foreach (XElement segmentElement in segmentElements)
//{
// UserSegment segment = new UserSegment();
// segment.SegmentId = segmentElement.FirstAttribute.Value;
// segment.SegmentName = segmentElement.FirstAttribute.NextAttribute.Value;
// if (!segmentElement.FirstAttribute.Value.Contains("-"))
// {
// customerSegments.Add(segment);
// }
// else
// {
// segments.Add(segment);
// }
//}
//UserSegment defCustSeparator = new UserSegment();
//defCustSeparator.SegmentName = "____________________________";
//defCustSeparator.SegmentId = "";
//segments.Add(defCustSeparator);
//List<UserSegment> allSegments = new List<UserSegment>();
//foreach (UserSegment segment in customerSegments)
//{
// segments.Add(segment);
//}
//return segments;
}
public async Task <IndividualUserSegmentResponse> CreateUserSegmentAsync(UserSegment UserSegment)
{
return(await GenericPostAsync <IndividualUserSegmentResponse>($"help_center/user_segments.json", new IndividualUserSegmentResponse { UserSegment = UserSegment }));
}
public IndividualUserSegmentResponse CreateUserSegment(UserSegment UserSegment)
{
return(GenericPost <IndividualUserSegmentResponse>($"help_center/user_segments.json", new IndividualUserSegmentResponse {
UserSegment = UserSegment
}));
}
public IndividualUserSegmentResponse UpdateUserSegment(UserSegment UserSegment)
{
return(GenericPut <IndividualUserSegmentResponse>($"help_center/user_segments/{UserSegment.Id}.json", new IndividualUserSegmentResponse {
UserSegment = UserSegment
}));
}
public async static Task Update(ResponseUpdaterSetting config)
{
var serviceProvider = InjectService(config);
var _resultCache = serviceProvider.GetService <IResultCache>();
var _crowler = serviceProvider.GetService <ICrowlerCache>();
var _hostCache = serviceProvider.GetService <IHostCache>();
var _productCache = serviceProvider.GetService <IProductCache>();
var _segmentCache = serviceProvider.GetService <ISegmentCache>();
var _redisCache = serviceProvider.GetService <IRedisCache>();
var _aeroclient = serviceProvider.GetService <IAsyncClient>();
var _errorLogger = serviceProvider.GetService <IErrorLogger>();
var requestDb = _redisCache.GetDatabase(RedisDatabases.UserSegmentsRequest);
var responseDb = _redisCache.GetDatabase(RedisDatabases.UserSegmentsZero);
//var staticClient = serviceProvider.GetService<RestClient>()
long PreparedCount = 0;
long timeOut = 0;
var batchSize = config.BatchSize;
long PreparedCountFull = 0;
var tasks = new List <Task <RestSharp.IRestResponse <List <DruidData> > > >();
var segmentId = config.SegmentId;
var hostId = config.HostId;
var query = config.Query;
var stopWatch = new Stopwatch();
if (config.Query)
{
var statType = config.StatType;
while (true)
{
try
{
List <string> batchRequestIds = new List <string>();
Dictionary <string, List <DruidData> > usersDictionary = new Dictionary <string, List <DruidData> >();
while (batchRequestIds.Count <= batchSize)
{
//var batchItem = await requestDb.ListRightPopAsync($"Segment_{segmentId}_Request");
var batchItem = await requestDb.ListRightPopAsync($"Empty_{segmentId}");
if (!batchItem.HasValue)
{
Thread.Sleep(4000);
break;
}
PreparedCount++;
if (await _crowler.IsCrowler(long.Parse(batchItem)))
{
continue;
}
if (responseDb.KeyExists($"Full_{segmentId}:" + batchItem.ToString()))
{
continue;
}
batchRequestIds.Add(batchItem);
}
if (batchRequestIds.Count == 0)
{
Thread.Sleep(4000);
continue;
}
stopWatch.Reset();
stopWatch.Start();
foreach (var item in batchRequestIds)
{
List <UserSegment> finalSegmentsForRedis = new List <UserSegment>();
Key matchKey = new Key("Sticky", "Activity", $"{ item }_{config.HostId}");
var record = _aeroclient.Get(new Policy(), matchKey);
if (record != null)
{
UserSegment productVisitSegments = new UserSegment()
{
HostId = config.HostId ?? 0,
SegmentId = config.SegmentId ?? 0
};
var r = record.GetValue(config.StatType);
List <string> productIds = new List <string>();
if (r != null)
{
PreparedCountFull++;
productIds = r.ToString().Split(",").Where(c => c != "").ToList();
}
if (productIds.Count != 0)
{
foreach (var productId in productIds.OrderBy(c => Guid.NewGuid()).Take(5))
{
var productData = await _productCache.FindProduct(config.HostId ?? 0, productId);
if (productData != null && !string.IsNullOrEmpty(productData.ProductName) && !string.IsNullOrEmpty(productData.Url) && !string.IsNullOrEmpty(productData.ImageAddress) && productData.IsAvailable && (productData.Price ?? 0) != 0)
{
productVisitSegments.Products.Add(new Models.Redis.HostProduct()
{
Id = productData.Id, ImageAddress = productData.ImageAddress, IsAvailable = productData.IsAvailable, ProductName = productData.ProductName, Url = productData.Url, Price = productData.Price, UpdateDate = productData.UpdateDate
});
}
}
finalSegmentsForRedis.Add(productVisitSegments);
}
}
await responseDb.StringSetAsync($"Full_{config.SegmentId}:" + item.ToString(), Newtonsoft.Json.JsonConvert.SerializeObject(finalSegmentsForRedis), finalSegmentsForRedis.Count() == 0?TimeSpan.FromMinutes(config.EmptyExpirationInMinutes) : TimeSpan.FromMinutes(config.FullExpirationInMinutes));
}
Console.Clear();
Console.WriteLine($"Segment {segmentId}");
Console.WriteLine(stopWatch.Elapsed.TotalSeconds + "<--AvgTime To ms");
Console.WriteLine(PreparedCount.ToString("N0") + " <---Total Count" + "\n" + PreparedCountFull.ToString("N0") + " <---Filled Count");
Console.WriteLine(timeOut);
}
catch (Exception ex)
{
Console.Write(ex.Message);
Thread.Sleep(TimeSpan.FromSeconds(4));
}
}
}
else if (config.Categories)
{
var catSegment = (await _segmentCache.GetAllActiveSegments()).Where(c => !string.IsNullOrEmpty(c.AudienceExtra)).ToList();
var categories = catSegment.Select(c => c.AudienceExtra);
var categoystring = string.Join(',', categories.Select(c => "'" + c + "'"));
Dictionary <string, List <DruidData> > usersDictionary = new Dictionary <string, List <DruidData> >();
while (true)
{
try
{
List <string> batchRequestIds = new List <string>();
if (PreparedCount % 1000 == 0)
{
catSegment = (await _segmentCache.GetAllActiveSegments()).Where(c => !string.IsNullOrEmpty(c.AudienceExtra)).ToList();
categories = catSegment.Select(c => c.AudienceExtra);
categoystring = string.Join(',', categories.Select(c => "'" + c + "'"));
}
while (batchRequestIds.Count <= batchSize)
{
//var batchItem = await requestDb.ListRightPopAsync($"Segment_{segmentId}_Request");
var batchItem = await requestDb.ListRightPopAsync($"Empty_Category");
if (!batchItem.HasValue)
{
Thread.Sleep(4000);
break;
}
if (await _crowler.IsCrowler(long.Parse(batchItem)))
{
continue;
}
if (responseDb.KeyExists($"Full_Category:" + batchItem.ToString()))
{
continue;
}
batchRequestIds.Add(batchItem);
PreparedCount++;
}
if (batchRequestIds.Count == 0)
{
Thread.Sleep(4000);
continue;
}
stopWatch.Reset();
stopWatch.Start();
var restClient = new RestSharp.RestClient(config.DruidClient)
{
Timeout = 4000
};
var request3 = new RestSharp.RestRequest("druid/v2/sql", RestSharp.Method.POST);
string currentQuery = $"select Distinct(CategoryName),UserId from ProductActions WHERE UserId in ({string.Join(',', batchRequestIds.Select(c => "'" + c + "'"))}) AND CategoryName In({categoystring}) and HostId=17 GROUP BY CategoryName,UserId limit {batchSize * catSegment.Count()}";
request3.AddJsonBody(new DruidRequest()
{
query = currentQuery
});
var responseList = await restClient.ExecuteTaskAsync <List <DruidData> >(request3);
if (responseList.StatusCode == System.Net.HttpStatusCode.OK)
{
var grouped = responseList.Data.GroupBy(c => c.UserId);
foreach (var item in grouped)
{
if (item.Count() != 0)
{
usersDictionary.TryAdd(item.Key, item.ToList());
}
}
}
foreach (var item in batchRequestIds)
{
var found = false;
List <DruidData> userList = new List <DruidData>();
found = usersDictionary.TryGetValue(item, out userList);
if (found)
{
PreparedCountFull++;
var cats = userList.Select(c => c.CategoryName);
var segmentIds = catSegment.Where(c => cats.Contains(c.AudienceExtra));
List <UserSegment> finalSegmentsForRedis = new List <UserSegment>();
if (segmentIds != null)
{
finalSegmentsForRedis.AddRange(segmentIds.Select(c => new UserSegment()
{
HostId = config.HostId ?? 0, SegmentId = c.Id
}));
}
await responseDb.StringSetAsync($"Full_Category:" + item.ToString(), Newtonsoft.Json.JsonConvert.SerializeObject(finalSegmentsForRedis), TimeSpan.FromMinutes(config.FullExpirationInMinutes));
}
}
stopWatch.Stop();
Console.Clear();
Console.WriteLine($"Category Segment Creation");
Console.WriteLine(stopWatch.Elapsed.TotalSeconds + "<--AvgTime To ms");
Console.WriteLine(PreparedCount.ToString("N0") + " <---Total Count" + "\n" + PreparedCountFull.ToString("N0") + " <---Filled Count");
}
catch
{
}
}
}
else
{
// var excludedSegmentString = await _redisCache.GetDatabase(RedisDatabases.CacheData).StringGetAsync("ExcludedSegments");
// var excludedSegments = excludedSegmentString.ToString().Split(",").Select(c => int.Parse(c)).ToList();
var activeSegments = (await _segmentCache.GetAllActiveSegments()).ToList();
var pagedSegments = activeSegments.Where(c => c.AudienceId == 1).Select(c => c.Id.ToString());
var productSegment = activeSegments.Where(c => c.AudienceId != 1 && c.AudienceExtra == null).ToList();
var activeHostIds = productSegment.Select(c => c.HostId).Distinct();
while (true)
{
if (PreparedCount >= 100 && PreparedCount % 100 == 0)
{
activeSegments = (await _segmentCache.GetAllActiveSegments()).ToList();
pagedSegments = activeSegments.Where(c => c.AudienceId == 1).Select(c => c.Id.ToString());
productSegment = activeSegments.Where(c => c.AudienceId != 1 && c.AudienceExtra == null).ToList();
activeHostIds = productSegment.Select(c => c.HostId).Distinct();
}
try
{
Dictionary <string, List <DruidData> > usersDictionary = new Dictionary <string, List <DruidData> >();
var item = await requestDb.ListRightPopAsync($"Empty_General");
// var item = await requestDb.ListGetByIndexAsync("Empty_General",0);
if (!item.HasValue)
{
Thread.Sleep(4000);
continue;
}
PreparedCount++;
if (await _crowler.IsCrowler(long.Parse(item)))
{
continue;
}
if (responseDb.KeyExists($"Full_General:" + item.ToString()))
{
continue;
}
stopWatch.Reset();
stopWatch.Start();
List <UserSegment> finalSegmentsForRedis = new List <UserSegment>();
Dictionary <int, Dictionary <string, object> > userData = new Dictionary <int, Dictionary <string, object> >();
foreach (var activeHost in activeHostIds)
{
Key matchKey = new Key("Sticky", "Activity", $"{ item }_{activeHost}");
var record = _aeroclient.Get(new Policy(), matchKey);
if (record == null)
{
continue;
}
Dictionary <string, object> dic = record.bins;
userData.TryAdd(activeHost, dic);
}
if (userData.Count == 0)
{
await responseDb.StringSetAsync($"Full_General:" + item.ToString(), Newtonsoft.Json.JsonConvert.SerializeObject(finalSegmentsForRedis), TimeSpan.FromMinutes(config.EmptyExpirationInMinutes));
continue;
}
foreach (var segment in productSegment)
{
var foundinDataDic = false;
var userDataforhost = new Dictionary <string, object>();
foundinDataDic = userData.TryGetValue(segment.HostId, out userDataforhost);
if (!foundinDataDic)
{
continue;
}
var statType = segment.AudienceId == 2 ? StatTypes.ProductView : segment.AudienceId == 3 ? StatTypes.AddToCart : segment.AudienceId == 5 ? StatTypes.ProductPurchase : segment.AudienceId == 6 ? StatTypes.Like : "";
if (statType == "")
{
continue;
}
var foundSegmentDatainUserdata = false;
var segmentData = new object();
foundSegmentDatainUserdata = userDataforhost.TryGetValue(statType, out segmentData);
if (!foundSegmentDatainUserdata)
{
continue;
}
List <string> productIds = new List <string>();
PreparedCountFull++;
UserSegment productVisitSegments = new UserSegment()
{
HostId = segment.HostId,
SegmentId = segment.Id
};
if (segment.ActionId == 2)
{
productIds = segmentData.ToString().Split(",").Where(c => c != "").Distinct().ToList();
if (productIds.Count != 0)
{
foreach (var productId in productIds.OrderBy(c => Guid.NewGuid()).Take(5))
{
var productData = await _productCache.FindProduct(segment.HostId, productId);
if (productData != null && !string.IsNullOrEmpty(productData.ProductName) && !string.IsNullOrEmpty(productData.Url) && !string.IsNullOrEmpty(productData.ImageAddress) && productData.IsAvailable && (productData.Price ?? 0) != 0)
{
productVisitSegments.Products.Add(new Models.Redis.HostProduct()
{
Id = productData.Id, ImageAddress = productData.ImageAddress, IsAvailable = productData.IsAvailable, ProductName = productData.ProductName, Url = productData.Url, Price = productData.Price, UpdateDate = productData.UpdateDate
});
}
}
}
}
finalSegmentsForRedis.Add(productVisitSegments);
}
foreach (var hosts in activeHostIds)
{
var foundinDataDic = false;
var userDataforhost = new Dictionary <string, object>();
foundinDataDic = userData.TryGetValue(hosts, out userDataforhost);
if (!foundinDataDic)
{
continue;
}
var foundSegmentDatainUserdata = false;
var segmentData = new object();
foundSegmentDatainUserdata = userDataforhost.TryGetValue("PagedMemberships", out segmentData);
if (!foundSegmentDatainUserdata)
{
continue;
}
if (segmentData == null)
{
continue;
}
var segmentMembershipList = segmentData.ToString().Split(",");
if (segmentMembershipList != null && segmentMembershipList.Count() == 0)
{
continue;
}
var productVisitSegments = new List <UserSegment>();
foreach (var item1 in segmentMembershipList)
{
if (pagedSegments.Contains(item1))
{
productVisitSegments.Add(new UserSegment()
{
HostId = hosts,
SegmentId = int.Parse(item1)
});
}
}
List <string> membership = new List <string>();
finalSegmentsForRedis.AddRange(productVisitSegments);
}
await responseDb.StringSetAsync($"Full_General:" + item.ToString(), Newtonsoft.Json.JsonConvert.SerializeObject(finalSegmentsForRedis), finalSegmentsForRedis.Count() == 0?TimeSpan.FromMinutes(config.EmptyExpirationInMinutes) : TimeSpan.FromMinutes(config.FullExpirationInMinutes));
Console.Clear();
Console.WriteLine($"All Product Segment");
Console.WriteLine(stopWatch.Elapsed.TotalSeconds + "<--AvgTime To ms");
Console.WriteLine(PreparedCount.ToString("N0") + " <---Total Count" + "\n" + PreparedCountFull.ToString("N0") + " <---Filled Count");
Console.WriteLine(timeOut);
}
catch (Exception ex)
{
Console.Write(ex.Message);
Thread.Sleep(TimeSpan.FromSeconds(4));
}
}
}
}
