protected override async Task <object> OnDataRequest(DataQueryOptions filter, CancellationTokenSource cancellationTokenSource)
{
// get data
DataResult <Task <DomainProjectForAdmin> > data = await _adminProjectService.GetAsyncSequenceAsync(filter);
// loading and shaping data
IEnumerable <Task <AdminProject> > projectTasks = data.Results.Select(t => t.ContinueWith(u => _mapper.Map <DomainProjectForAdmin, AdminProject>(u.Result), cancellationTokenSource.Token));
List <AdminProject> projects = (await Task.WhenAll(projectTasks)).ToList();
if (projects.Count == 0)
{
cancellationTokenSource.Cancel();
}
return(projects);
}
public async Task <IEnumerable <TrendingWatch> > GetWeeklyTrendsSequenceAsync(DataQueryOptions filter)
{
List <DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
long?version = null;
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName <TrendingWatch>(x => x.Version),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by product id
version = Int64.Parse(f.Value.ToString());
break;
}
}
IEnumerable <DomainMostSignaledItem> data = await _cassandraStatisticsService.GetMostViewedForLastWeekAsync(StatisticsSpaces.Projects, MemoryPageSize, version);
var pageData = new List <DomainMostSignaledItem>();
int count = 0;
int skip = filter.Skip.HasValue ? filter.Skip.Value : 0;
var uniqueIds = new HashSet <string>();
var result = new List <TrendingWatch>();
foreach (DomainMostSignaledItem mostSignaledItem in data)
{
if (result.Count >= MemoryPageSize || count > MemoryPageSize * MaxScanPages)
{
// memory limit reached
break;
}
if (filter.Take.HasValue && result.Count >= filter.Take.Value + skip)
{
// all items loaded
break;
}
// checking uniqueness
if (!uniqueIds.Contains(mostSignaledItem.ItemId))
{
// there can be duplicated ItemIds with different Count values due to race conditions
uniqueIds.Add(mostSignaledItem.ItemId);
}
else
{
// skipping duplicate
count++;
continue;
}
// adding to memory page
pageData.Add(mostSignaledItem);
// checking limits
if ((count > 0 && count % MemoryPageSize == 0) ||
(filter.Take.HasValue && pageData.Count >= filter.Take.Value))
{
// loading and filtering items
List <TrendingWatch> page = await LoadAndFilterDataAsync(pageData, filter);
result.AddRange(page);
pageData.Clear();
}
count++;
}
if (pageData.Count > 0)
{
// loading last page
List <TrendingWatch> page = await LoadAndFilterDataAsync(pageData, filter);
result.AddRange(page);
pageData.Clear();
}
// Skip
result = result.Skip(skip).ToList();
// Take
if (filter.Take.HasValue)
{
result = result.Take(filter.Take.Value).ToList();
}
return(result);
}
private async Task <List <TrendingWatch> > LoadAndFilterDataAsync(IEnumerable <DomainMostSignaledItem> pageData, DataQueryOptions filter)
{
// Preparing
int? productId = null;
WatchState? videoState = null;
List <DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName <TrendingWatch>(x => x.Generator),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by product id
productId = Int32.Parse(f.Value.ToString());
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <TrendingWatch>(x => x.State),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by video state
videoState = (WatchState)Enum.Parse(typeof(WatchState), f.Value.ToString());
}
}
// Loading
List <DomainMostSignaledItem> items = pageData.ToList();
List <Watch> watches = await _watchProjectService.GetByIdsAsync(items.Select(i => i.ItemId).ToArray(), string.Empty);
// Processing projects
var filtered = new List <TrendingWatch>();
foreach (DomainMostSignaledItem i in items)
{
DomainMostSignaledItem item = i;
Watch watch = watches.FirstOrDefault(w => w.Id == item.ItemId);
// Display only existing & public videos
if (watch == null || watch.Access != ProjectAccess.Public)
{
continue;
}
// filtering by product id
if (productId.HasValue)
{
if (watch.Generator != productId.Value)
{
continue;
}
}
// filterring by state
if (videoState.HasValue)
{
if (watch.State != videoState.Value)
{
continue;
}
}
TrendingWatch trendingWatch = _mapper.Map <Watch, TrendingWatch>(watch);
trendingWatch.Version = item.Version;
filtered.Add(trendingWatch);
}
return(filtered);
}
public DataResult <Task <DomainUserForAdmin> > GetAsyncSequence(DataQueryOptions filter)
{
var query = new List <IMongoQuery>(filter.Filters.Count);
// filtering
List <DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName <DomainUserForAdmin>(x => x.UserName),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by user name
query.Add(Query.Text(f.Value.ToString().ToLowerInvariant()));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <DomainUserForAdmin>(x => x.Email),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by email
string email = f.Value.ToString().ToLowerInvariant();
var expression = new BsonRegularExpression(string.Format("^{0}.*", Regex.Escape(email)));
query.Add(Query <UserEntity> .Matches(p => p.Email, expression));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <DomainUserForAdmin>(x => x.ProductType),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by product type
var productId = Int32.Parse(f.Value.ToString());
query.Add(Query <UserEntity> .EQ(p => p.ProductId, productId));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <DomainUserForAdmin>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
// by created date
var date = (DateTime)f.Value;
switch (f.Type)
{
case DataFilterTypes.Equal:
query.Add(Query <UserEntity> .EQ(p => p.Created, date));
break;
case DataFilterTypes.LessThan:
query.Add(Query <UserEntity> .LT(p => p.Created, date));
break;
case DataFilterTypes.LessThanOrEqual:
query.Add(Query <UserEntity> .LTE(p => p.Created, date));
break;
case DataFilterTypes.GreaterThan:
query.Add(Query <UserEntity> .GT(p => p.Created, date));
break;
case DataFilterTypes.GreaterThanOrEqual:
query.Add(Query <UserEntity> .GTE(p => p.Created, date));
break;
}
}
else
{
throw new NotSupportedException(string.Format("Filter {0} by property {1} is not supported", f.Type, f.Name));
}
}
// Filter only users
query.Add(Query <UserEntity> .EQ(p => p.Roles, DomainRoles.User));
MongoCursor <UserEntity> cursor = _userRepository.Collection.Find(query.Count > 0 ? Query.And(query) : Query.Null);
IMongoSortBy sortOrder = null;
// sorting
if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName <DomainUserForAdmin>(x => x.UserName),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by name
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy <UserEntity> .Ascending(p => p.Name)
: SortBy <UserEntity> .Descending(p => p.Name);
}
else if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName <DomainUserForAdmin>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by created
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy <UserEntity> .Ascending(p => p.Created)
: SortBy <UserEntity> .Descending(p => p.Created);
}
else if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName <DomainUserForAdmin>(x => x.ProductType),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by product type
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy <UserEntity> .Ascending(p => p.ProductId)
: SortBy <UserEntity> .Descending(p => p.ProductId);
}
if (sortOrder != null)
{
cursor.SetSortOrder(sortOrder);
}
// paging
if (filter.Skip.HasValue)
{
cursor.SetSkip(filter.Skip.Value);
}
if (filter.Take.HasValue)
{
cursor.SetLimit(filter.Take.Value);
}
// Count of results
long?count = null;
if (filter.Count)
{
count = cursor.Count();
}
// post-processing
return(new DataResult <Task <DomainUserForAdmin> >(cursor.Select(GetUserDataAsync), count));
}
protected abstract Task <object> OnDataRequest(DataQueryOptions filter, CancellationTokenSource cancellationTokenSource);
public async Task PublishAsync(DataQueryOptions filter, string fileName, CancellationTokenSource cancellationTokenSource)
{
try
{
// Get container reference
CloudBlobContainer container = _blobClient.GetContainerReference(CsvContainerName);
await container.CreateIfNotExistsAsync(cancellationTokenSource.Token);
// Delete old if exists
CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
await blob.DeleteIfExistsAsync(cancellationTokenSource.Token);
// Create temp file and serialize data to it
using (var stream = new MemoryStream())
{
// Prepare data filter
int limit = filter.Take.HasValue && filter.Take.Value > 0 ? filter.Take.Value : Int32.MaxValue;
if (filter.Take.HasValue && filter.Take.Value > 0)
{
// limit specified
if (filter.Take.Value > CsvBatchSize)
{
// splitting data into batches
filter.Take = CsvBatchSize;
}
}
else
{
filter.Take = CsvBatchSize;
}
filter.Skip = filter.Skip.HasValue && filter.Skip.Value > 0 ? filter.Skip.Value : 0;
// Adding utf-8 BOM. Details http://stackoverflow.com/questions/4414088/how-to-getbytes-in-c-sharp-with-utf8-encoding-with-bom/4414118#4414118
byte[] bom = Encoding.UTF8.GetPreamble();
await stream.WriteAsync(bom, 0, bom.Length, cancellationTokenSource.Token);
// Batch serialization
var serializationTokenSource = new CancellationTokenSource();
while (!cancellationTokenSource.IsCancellationRequested && !serializationTokenSource.IsCancellationRequested)
{
// loading batch
object data = await OnDataRequest(filter, serializationTokenSource);
// serializing batch
await _mediaTypeFormatter.WriteToStreamAsync(data.GetType(), data, stream, null, null, cancellationTokenSource.Token);
if (filter.Skip + filter.Take == limit)
{
// all loaded
break;
}
// next batch
filter.Skip += CsvBatchSize;
if (filter.Skip + CsvBatchSize >= limit)
{
filter.Take = limit - filter.Skip;
}
}
// Upload result to blob
stream.Position = 0;
await blob.UploadFromStreamAsync(stream, cancellationTokenSource.Token);
}
// Set content type
await blob.FetchAttributesAsync(cancellationTokenSource.Token);
blob.Properties.ContentType = "text/csv";
await blob.SetPropertiesAsync(cancellationTokenSource.Token);
}
catch (Exception e)
{
Trace.TraceError("Failed to publish csv file: {0}", e);
}
}
public async Task <IEnumerable <DomainTrackingStatPerDate> > GetStatsPerDateAsync(string subscriptionId, string url, DataQueryOptions filter)
{
if (string.IsNullOrEmpty(subscriptionId))
{
throw new ArgumentNullException("subscriptionId");
}
// Filter
DateTime?dateFromFilter = null;
DateTime?dateToFilter = null;
List <DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName <DomainTrackingStatPerDate>(x => x.Date),
StringComparison.OrdinalIgnoreCase) == 0)
{
// by created date
DateTime date = ((DateTime)f.Value).Date;
switch (f.Type)
{
case DataFilterTypes.Equal:
dateFromFilter = date;
dateToFilter = date.AddDays(1);
break;
case DataFilterTypes.LessThan:
dateToFilter = date;
break;
case DataFilterTypes.LessThanOrEqual:
dateToFilter = date.AddDays(1);
break;
case DataFilterTypes.GreaterThan:
dateFromFilter = date.AddDays(1);
break;
case DataFilterTypes.GreaterThanOrEqual:
dateFromFilter = date;
break;
}
}
else
{
throw new NotSupportedException(string.Format("Filter {0} by property {1} is not supported", f.Type, f.Name));
}
}
// Aggregate per date
IEnumerable <TrackingStatPerDateEntity> aggregation = await _trackingStatRepository.AggregatePerDateAsync(subscriptionId, url, dateFromFilter, dateToFilter);
// Project
IEnumerable <DomainTrackingStatPerDate> projectedResult = aggregation.Select(r => _mapper.Map <TrackingStatPerDateEntity, DomainTrackingStatPerDate>(r));
return(projectedResult);
}
public async Task <DataResult <DomainTrackingStatPerUrl> > GetStatsPerUrlAsync(string subscriptionId, DataQueryOptions filter)
{
if (string.IsNullOrEmpty(subscriptionId))
{
throw new ArgumentNullException("subscriptionId");
}
// Filter
string redirectUrlFilter = null;
DateTime?dateFromFilter = null;
DateTime?dateToFilter = null;
List <DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName <DomainTrackingStatPerUrl>(x => x.RedirectUrl),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by redirect url
redirectUrlFilter = f.Value.ToString();
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <DomainTrackingStatPerUrl>(x => x.DateFrom),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by date from
var date = (DateTime)f.Value;
dateFromFilter = date;
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <DomainTrackingStatPerUrl>(x => x.DateTo),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by date to
var date = (DateTime)f.Value;
dateToFilter = date.AddDays(1);
}
else
{
throw new NotSupportedException(string.Format("Filter {0} by property {1} is not supported", f.Type, f.Name));
}
}
// Aggregate per redirect url
DataResultEntity <TrackingStatPerUrlEntity> aggregation =
await
_trackingStatRepository.AggregatePerUrlAsync(subscriptionId, redirectUrlFilter, dateFromFilter, dateToFilter, filter.OrderBy,
filter.OrderByDirection == OrderByDirections.Desc, filter.Skip, filter.Take, filter.Count);
IEnumerable <DomainTrackingStatPerUrl> projectedResult = aggregation.Results.Select(r => _mapper.Map <TrackingStatPerUrlEntity, DomainTrackingStatPerUrl>(r));
return(new DataResult <DomainTrackingStatPerUrl>(projectedResult, aggregation.Count));
}
public async Task <IEnumerable <ExampleProject> > GetSequenceAsync(DataQueryOptions filter)
{
// Get example managers
List <DomainUser> users = await _adminUserService.GetUsersInRoleAsync(DomainRoles.ExamplesManager);
if (users.Count == 0)
{
return(new List <ExampleProject>());
}
// Get their projects
IEnumerable <DomainProject> projects = await _projectService.GetProjectListByUsersAsync(users);
// Only public videos
projects = projects.Where(p => p.Access == ProjectAccess.Public);
// Paging
if (filter.Skip.HasValue)
{
projects = projects.Skip(filter.Skip.Value);
}
if (filter.Take.HasValue)
{
projects = projects.Take(filter.Take.Value);
}
// Sorting
if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName <ExampleProject>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
projects = filter.OrderByDirection == OrderByDirections.Asc
? projects.OrderBy(p => p.Created)
: projects.OrderByDescending(p => p.Created);
}
List <DomainProject> result = projects.ToList();
// Aggregating project data
List <Task <ExampleProject> > tasks = result.Select(GetProjectAsync).ToList();
await Task.WhenAll(tasks);
List <ExampleProject> examples = tasks.Select(t => t.Result).ToList();
// Post-processing
// Get comments and total comment counts
string[] projectIds = result.Select(p => p.Id).ToArray();
Dictionary <string, int> commentsCounts = (await _commentRepository.GetCommentsCountByProjectsAsync(projectIds)).ToDictionary(c => c.ProjectId, c => c.CommentsCount);
Dictionary <string, List <CommentEntity> > recentComments = await _commentRepository.GetRecentCommentsByProjectsAsync(projectIds, RecentCommentsLimit);
// Get related users list
var userIdsSet = new HashSet <string>();
foreach (var projectComment in recentComments)
{
foreach (CommentEntity c in projectComment.Value)
{
if (userIdsSet.Contains(c.UserId))
{
continue;
}
userIdsSet.Add(c.UserId);
}
}
Dictionary <string, UserEntity> commentUsers = (await _userRepository.GetUsersByIdsAsync(userIdsSet.ToArray())).ToDictionary(u => u.Id);
// Append comments data to exmaples
foreach (ExampleProject example in examples)
{
// comments total count
example.Comments = commentsCounts.ContainsKey(example.Id) ? commentsCounts[example.Id] : 0;
// comments list
if (recentComments.ContainsKey(example.Id))
{
var exampleComments = new List <Comment>();
List <CommentEntity> projectComments = recentComments[example.Id];
foreach (CommentEntity projectComment in projectComments)
{
UserEntity commentAuthor = commentUsers.ContainsKey(projectComment.UserId) ? commentUsers[projectComment.UserId] : null;
DomainComment domainComment = _mapper.Map <Tuple <CommentEntity, UserEntity>, DomainComment>(new Tuple <CommentEntity, UserEntity>(projectComment, commentAuthor));
Comment comment = _mapper.Map <DomainComment, Comment>(domainComment);
if (commentAuthor != null)
{
comment.AvatarUrl = _userAvatarProvider.GetAvatar(commentAuthor.Email);
}
exampleComments.Add(comment);
}
exampleComments.Reverse();
example.CommentList = exampleComments;
}
else
{
example.CommentList = new List <Comment>();
}
}
return(examples);
}
public async Task <DataResult <DomainTrackingStat> > GetUrlStatsAsync(string subscriptionId, string url, DataQueryOptions filter)
{
if (string.IsNullOrEmpty(subscriptionId))
{
throw new ArgumentNullException("subscriptionId");
}
if (string.IsNullOrEmpty(url))
{
throw new ArgumentNullException("url");
}
IQueryable <TrackingStatEntity> data = _trackingStatRepository.Context.Where(s => s.SubscriptionId == subscriptionId && s.RedirectUrl == url);
// Filter
List <DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName <DomainTrackingStat>(x => x.Date),
StringComparison.OrdinalIgnoreCase) == 0)
{
// by created date
var date = ((DateTime)f.Value).Date;
switch (f.Type)
{
case DataFilterTypes.Equal:
data = data.Where(p => p.Date > date && p.Date < date.AddDays(1));
break;
case DataFilterTypes.LessThan:
data = data.Where(p => p.Date < date);
break;
case DataFilterTypes.LessThanOrEqual:
data = data.Where(p => p.Date < date.AddDays(1));
break;
case DataFilterTypes.GreaterThan:
data = data.Where(p => p.Date > date);
break;
case DataFilterTypes.GreaterThanOrEqual:
data = data.Where(p => p.Date >= date);
break;
}
}
else
{
throw new NotSupportedException(string.Format("Filter {0} by property {1} is not supported", f.Type, f.Name));
}
}
// Sort
if (!string.IsNullOrEmpty(filter.OrderBy))
{
if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName <DomainTrackingStat>(x => x.Date),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by name
data = filter.OrderByDirection == OrderByDirections.Asc
? data.OrderBy(p => p.Date)
: data.OrderByDescending(p => p.Date);
}
else
{
throw new NotSupportedException(string.Format("Ordering by {0} is not supported", filter.OrderBy));
}
}
// Count of results
long?count = null;
if (filter.Count)
{
count = data.LongCount();
}
// Paging
if (filter.Skip.HasValue)
{
data = data.Skip(filter.Skip.Value);
}
if (filter.Take.HasValue)
{
data = data.Take(filter.Take.Value);
}
// post-processing
IQueryable <DomainTrackingStat> results = data.Select(s => _mapper.Map <TrackingStatEntity, DomainTrackingStat>(s));
return(new DataResult <DomainTrackingStat>(results, count));
}
public async Task <DataResult <Task <DomainProjectForAdmin> > > GetAsyncSequenceAsync(DataQueryOptions filter)
{
var query = new List <IMongoQuery>(filter.Filters.Count);
// filtering
List <DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName <DomainProjectForAdmin>(x => x.Name),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by name
query.Add(Query.Text(f.Value.ToString().ToLowerInvariant()));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <DomainProjectForAdmin>(x => x.ProductType),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by product type
var productId = Int32.Parse(f.Value.ToString());
query.Add(Query <ProjectEntity> .EQ(p => p.ProductId, productId));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <DomainProjectForAdmin>(x => x.UserId),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by user id
query.Add(Query <ProjectEntity> .EQ(p => p.UserId, f.Value.ToString()));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <DomainProjectForAdmin>(x => x.UserName),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by user name
List <UserEntity> profiles = await _userRepository.FindByNameAsync(f.Value.ToString());
if (profiles.Count == 0)
{
// no users found
return(new DataResult <Task <DomainProjectForAdmin> >(new Task <DomainProjectForAdmin>[] { }));
}
List <string> allIds = profiles.Select(prof => prof.Id).ToList();
query.Add(Query <ProjectEntity> .Where(p => allIds.Contains(p.UserId)));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <DomainProjectForAdmin>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
// by created date
var date = (DateTime)f.Value;
switch (f.Type)
{
case DataFilterTypes.Equal:
query.Add(Query <ProjectEntity> .EQ(p => p.Created, date));
break;
case DataFilterTypes.LessThan:
query.Add(Query <ProjectEntity> .LT(p => p.Created, date));
break;
case DataFilterTypes.LessThanOrEqual:
query.Add(Query <ProjectEntity> .LTE(p => p.Created, date));
break;
case DataFilterTypes.GreaterThan:
query.Add(Query <ProjectEntity> .GT(p => p.Created, date));
break;
case DataFilterTypes.GreaterThanOrEqual:
query.Add(Query <ProjectEntity> .GTE(p => p.Created, date));
break;
}
}
else
{
throw new NotSupportedException(string.Format("Filter {0} by property {1} is not supported", f.Type, f.Name));
}
}
if (!filters.Any() && !string.IsNullOrEmpty(filter.OrderBy))
{
// MongoDb 2.6 HACK!!!
// adding fake query to hint proper index
if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName <DomainProjectForAdmin>(x => x.Name),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by name
query.Add(Query <ProjectEntity> .Exists(p => p.Name));
}
else if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName <DomainProjectForAdmin>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by created
query.Add(Query <ProjectEntity> .Exists(p => p.Created));
}
else if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName <DomainProjectForAdmin>(x => x.ProductType),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by product type
query.Add(Query <ProjectEntity> .Exists(p => p.ProductId));
}
}
MongoCursor <ProjectEntity> cursor = _projectRepository.Collection.Find(query.Count > 0 ? Query.And(query) : Query.Null);
IMongoSortBy sortOrder = null;
// sorting
if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName <DomainProjectForAdmin>(x => x.Name),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by name
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy <ProjectEntity> .Ascending(p => p.Name)
: SortBy <ProjectEntity> .Descending(p => p.Name);
}
else if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName <DomainProjectForAdmin>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by created
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy <ProjectEntity> .Ascending(p => p.Created)
: SortBy <ProjectEntity> .Descending(p => p.Created);
}
else if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName <DomainProjectForAdmin>(x => x.ProductType),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by product type
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy <ProjectEntity> .Ascending(p => p.ProductId)
: SortBy <ProjectEntity> .Descending(p => p.ProductId);
}
if (sortOrder != null)
{
cursor.SetSortOrder(sortOrder);
}
// paging
if (filter.Skip.HasValue)
{
cursor.SetSkip(filter.Skip.Value);
}
if (filter.Take.HasValue)
{
cursor.SetLimit(filter.Take.Value);
}
// Count of results
long?count = null;
if (filter.Count)
{
count = cursor.Count();
}
// post-processing
return(new DataResult <Task <DomainProjectForAdmin> >(cursor.Select(GetProjectDataAsync), count));
}
public PagedList<Account> GetAccountPagedList(DataQueryOptions<Account> searchOption)
{
var linq = GetAccounts().Where(m => !m.SystemAccount_N).AsQueryable();
linq = linq.Append(searchOption);
return linq.ToPagedList(searchOption);
}
public async Task <DataResult <Watch> > GetSequenceAsync(DataQueryOptions filter, string userId)
{
var query = new List <IMongoQuery>(filter.Filters.Count);
// Filtering
List <DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
string requestedUserId = null;
var searchList = new List <string>();
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName <Watch>(x => x.Name),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by name
searchList.Add(f.Value.ToString().ToLowerInvariant());
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <Watch>(x => x.UserId),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by user id
requestedUserId = f.Value.ToString();
query.Add(Query <ProjectEntity> .EQ(p => p.UserId, requestedUserId));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <Watch>(x => x.Generator),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by product id
int productId = Int32.Parse(f.Value.ToString());
query.Add(Query <ProjectEntity> .EQ(p => p.ProductId, productId));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <Watch>(x => x.ProjectType),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by project type
object projectType = Enum.Parse(typeof(ProjectType), f.Value.ToString());
query.Add(Query <ProjectEntity> .EQ(p => p.ProjectType, (int)projectType));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <Watch>(x => x.ProjectSubtype),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by project subtype
object projectSubtype = Enum.Parse(typeof(ProjectSubtype), f.Value.ToString());
query.Add(Query <ProjectEntity> .EQ(p => p.ProjectSubtype, (int)projectSubtype));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <Watch>(x => x.External.VideoUri),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by external video uri
query.Add(Query <ProjectEntity> .EQ(p => p.VideoSource, f.Value));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <Watch>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
// by created date
var date = (DateTime)f.Value;
switch (f.Type)
{
case DataFilterTypes.Equal:
query.Add(Query <ProjectEntity> .EQ(p => p.Created, date));
break;
case DataFilterTypes.LessThan:
query.Add(Query <ProjectEntity> .LT(p => p.Created, date));
break;
case DataFilterTypes.LessThanOrEqual:
query.Add(Query <ProjectEntity> .LTE(p => p.Created, date));
break;
case DataFilterTypes.GreaterThan:
query.Add(Query <ProjectEntity> .GT(p => p.Created, date));
break;
case DataFilterTypes.GreaterThanOrEqual:
query.Add(Query <ProjectEntity> .GTE(p => p.Created, date));
break;
}
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <Watch>(x => x.HitsCount),
StringComparison.OrdinalIgnoreCase) == 0)
{
// by hits count
int threshold = Int32.Parse(f.Value.ToString());
if (f.Type == DataFilterTypes.GreaterThan)
{
query.Add(Query <ProjectEntity> .GT(p => p.HitsCount, threshold));
}
else if (f.Type == DataFilterTypes.GreaterThanOrEqual)
{
query.Add(Query <ProjectEntity> .GTE(p => p.HitsCount, threshold));
}
else
{
query.Add(Query <ProjectEntity> .EQ(p => p.HitsCount, threshold));
}
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName <Watch>(x => x.State),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by video state
var videoState = (WatchState)Enum.Parse(typeof(WatchState), f.Value.ToString());
if (videoState == WatchState.Uploading)
{
var avsx = Query <ProjectEntity> .EQ(p => p.AvsxFileId, null);
var originalFileId = Query <ProjectEntity> .EQ(p => p.OriginalVideoFileId, null);
var externalSource = Query <ProjectEntity> .EQ(p => p.VideoSource, null);
query.Add(Query.Or(avsx, Query.And(originalFileId, externalSource)));
}
else if (videoState == WatchState.Encoding)
{
var avsx = Query <ProjectEntity> .NE(p => p.AvsxFileId, null);
var originalFileId = Query <ProjectEntity> .NE(p => p.OriginalVideoFileId, null);
var encodedVideosName = NameOfHelper.PropertyName <ProjectEntity>(x => x.EncodedVideos);
var encodedVideos = Query.NotExists(string.Format("{0}.0", encodedVideosName));
query.Add(Query.And(avsx, originalFileId, encodedVideos));
}
else if (videoState == WatchState.Ready)
{
var avsx = Query <ProjectEntity> .NE(p => p.AvsxFileId, null);
var externalSource = Query <ProjectEntity> .NE(p => p.VideoSource, null);
var originalFileId = Query <ProjectEntity> .NE(p => p.OriginalVideoFileId, null);
var encodedVideosName = NameOfHelper.PropertyName <ProjectEntity>(x => x.EncodedVideos);
var encodedVideos = Query.Exists(string.Format("{0}.0", encodedVideosName));
query.Add(Query.And(avsx, Query.Or(externalSource, Query.And(originalFileId, encodedVideos))));
}
}
else
{
throw new NotSupportedException(string.Format("Filter {0} by property {1} is not supported", f.Type, f.Name));
}
}
string searchText = String.Join(" ", searchList);
if (!String.IsNullOrEmpty(searchText))
{
query.Add(Query.Text(searchText));
}
// we should see only public videos for other users
if (requestedUserId != userId)
{
query.Add(Query <ProjectEntity> .EQ(p => p.Access, (int)ProjectAccess.Public));
}
MongoCursor <ProjectEntity> cursor = _projectRepository.Collection.Find(query.Count > 0 ? Query.And(query) : Query.Null);
// Sorting
IMongoSortBy sortOrder = null;
if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName <Watch>(x => x.Name),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by name
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy <ProjectEntity> .Ascending(p => p.Name)
: SortBy <ProjectEntity> .Descending(p => p.Name);
}
else if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName <Watch>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by created
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy <ProjectEntity> .Ascending(p => p.Created)
: SortBy <ProjectEntity> .Descending(p => p.Created);
}
else if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName <Watch>(x => x.HitsCount),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by hits count
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy <ProjectEntity> .Ascending(p => p.HitsCount)
: SortBy <ProjectEntity> .Descending(p => p.HitsCount);
}
if (sortOrder != null)
{
cursor.SetSortOrder(sortOrder);
}
// Paging
if (filter.Skip.HasValue)
{
cursor.SetSkip(filter.Skip.Value);
}
if (filter.Take.HasValue)
{
cursor.SetLimit(filter.Take.Value);
}
// Count of results
long?count = null;
if (filter.Count)
{
count = cursor.Count();
}
List <ProjectEntity> projects = cursor.ToList();
// Get comments and total comment counts
string[] projectIds = projects.Select(p => p.Id).ToArray();
Dictionary <string, int> commentsCounts = (await _commentRepository.GetCommentsCountByProjectsAsync(projectIds)).ToDictionary(c => c.ProjectId, c => c.CommentsCount);
Dictionary <string, List <CommentEntity> > recentComments = await _commentRepository.GetRecentCommentsByProjectsAsync(projectIds, RecentCommentsLimit);
// Get project user ids united with comment user ids to load entities in batch
string[] userIds = GetUserIds(projects, recentComments);
Dictionary <string, UserEntity> users = (await _userRepository.GetUsersByIdsAsync(userIds)).ToDictionary(u => u.Id);
// Post-processing
return(new DataResult <Watch>(projects.Select(p => AggregateProject(p, userId, users, commentsCounts, recentComments)), count));
}
public async Task <IEnumerable <UserInfo> > GetProjectReportersSequenceAsync(string projectId, DataQueryOptions filter)
{
var result = new List <UserInfo>();
IEnumerable <DomainItemSignal> data = await _cassandraStatisticsService.GetItemAbuseSequenceAsync(StatisticsSpaces.Projects, projectId, MemoryPageSize);
var pageData = new List <DomainItemSignal>();
int count = 0;
int skip = filter != null && filter.Skip.HasValue ? filter.Skip.Value : 0;
foreach (DomainItemSignal user in data)
{
if (result.Count >= MemoryPageSize || count > MemoryPageSize * MaxScanPages)
{
// memory limit reached
break;
}
if (filter != null && filter.Take.HasValue && result.Count >= filter.Take.Value + skip)
{
// all items loaded
break;
}
// adding to page
pageData.Add(user);
// checking limits
if ((count > 0 && count % MemoryPageSize == 0) ||
(filter != null && filter.Take.HasValue && pageData.Count >= filter.Take.Value))
{
// loading and filtering items
List <UserInfo> page = await LoadUserPageDataAsync(pageData);
result.AddRange(page);
pageData.Clear();
}
count++;
}
if (pageData.Count > 0)
{
// loading last page
List <UserInfo> page = await LoadUserPageDataAsync(pageData);
result.AddRange(page);
pageData.Clear();
}
// Skip
result = result.Skip(skip).ToList();
// Take
if (filter != null && filter.Take.HasValue)
{
result = result.Take(filter.Take.Value).ToList();
}
return(result);
}
public async Task <IEnumerable <Watch> > GetUserLikesSequenceAsync(string userId, DataQueryOptions filter)
{
var result = new List <Watch>();
IEnumerable <DomainUserSignal> data = await _cassandraStatisticsService.GetUserLikesSequenceAsync(StatisticsSpaces.Projects, userId, MemoryPageSize);
var pageData = new List <DomainUserSignal>();
int count = 0;
int skip = filter != null && filter.Skip.HasValue ? filter.Skip.Value : 0;
var uniqueIds = new HashSet <string>();
foreach (DomainUserSignal item in data)
{
if (result.Count >= MemoryPageSize || count > MemoryPageSize * MaxScanPages)
{
// memory limit reached
break;
}
if (filter != null && filter.Take.HasValue && result.Count >= filter.Take.Value + skip)
{
// all items loaded
break;
}
// checking uniqueness
if (!uniqueIds.Contains(item.ItemId))
{
// there can be duplicated ItemIds with different DateTime values due to race conditions
uniqueIds.Add(item.ItemId);
}
else
{
// skipping duplicate
count++;
continue;
}
// adding to memory page
pageData.Add(item);
// checking limits
if ((count > 0 && count % MemoryPageSize == 0) ||
(filter != null && filter.Take.HasValue && pageData.Count >= filter.Take.Value))
{
// loading and filtering items
List <Watch> page = await LoadProjectPageDataAsync(pageData, userId);
result.AddRange(page);
pageData.Clear();
}
count++;
}
if (pageData.Count > 0)
{
// loading last page
List <Watch> page = await LoadProjectPageDataAsync(pageData, userId);
result.AddRange(page);
pageData.Clear();
}
// Skip
result = result.Skip(skip).ToList();
// Take
if (filter != null && filter.Take.HasValue)
{
result = result.Take(filter.Take.Value).ToList();
}
return(result);
}
