public async Task<IEnumerable<UserInfo>> GetProjectReportersSequenceAsync(string projectId, DataQueryOptions filter)
{
var result = new List<UserInfo>();
IEnumerable<DomainItemSignal> data = await _cassandraStatisticsService.GetItemAbuseSequenceAsync(StatisticsSpaces.Projects, projectId, MemoryPageSize);
var pageData = new List<DomainItemSignal>();
int count = 0;
int skip = filter != null && filter.Skip.HasValue ? filter.Skip.Value : 0;
foreach (DomainItemSignal user in data)
{
if (result.Count >= MemoryPageSize || count > MemoryPageSize*MaxScanPages)
{
// memory limit reached
break;
}
if (filter != null && filter.Take.HasValue && result.Count >= filter.Take.Value + skip)
{
// all items loaded
break;
}
// adding to page
pageData.Add(user);
// checking limits
if ((count > 0 && count%MemoryPageSize == 0) ||
(filter != null && filter.Take.HasValue && pageData.Count >= filter.Take.Value))
{
// loading and filtering items
List<UserInfo> page = await LoadUserPageDataAsync(pageData);
result.AddRange(page);
pageData.Clear();
}
count++;
}
if (pageData.Count > 0)
{
// loading last page
List<UserInfo> page = await LoadUserPageDataAsync(pageData);
result.AddRange(page);
pageData.Clear();
}
// Skip
result = result.Skip(skip).ToList();
// Take
if (filter != null && filter.Take.HasValue)
{
result = result.Take(filter.Take.Value).ToList();
}
return result;
}
public async Task<DataResult<DomainTrackingStatPerUrl>> GetStatsPerUrlAsync(string subscriptionId, DataQueryOptions filter)
{
if (string.IsNullOrEmpty(subscriptionId))
{
throw new ArgumentNullException("subscriptionId");
}
// Filter
string redirectUrlFilter = null;
DateTime? dateFromFilter = null;
DateTime? dateToFilter = null;
List<DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName<DomainTrackingStatPerUrl>(x => x.RedirectUrl),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by redirect url
redirectUrlFilter = f.Value.ToString();
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<DomainTrackingStatPerUrl>(x => x.DateFrom),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by date from
var date = (DateTime)f.Value;
dateFromFilter = date;
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<DomainTrackingStatPerUrl>(x => x.DateTo),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by date to
var date = (DateTime)f.Value;
dateToFilter = date.AddDays(1);
}
else
{
throw new NotSupportedException(string.Format("Filter {0} by property {1} is not supported", f.Type, f.Name));
}
}
// Aggregate per redirect url
DataResultEntity<TrackingStatPerUrlEntity> aggregation =
await
_trackingStatRepository.AggregatePerUrlAsync(subscriptionId, redirectUrlFilter, dateFromFilter, dateToFilter, filter.OrderBy,
filter.OrderByDirection == OrderByDirections.Desc, filter.Skip, filter.Take, filter.Count);
IEnumerable<DomainTrackingStatPerUrl> projectedResult = aggregation.Results.Select(r => _mapper.Map<TrackingStatPerUrlEntity, DomainTrackingStatPerUrl>(r));
return new DataResult<DomainTrackingStatPerUrl>(projectedResult, aggregation.Count);
}
protected override async Task<object> OnDataRequest(DataQueryOptions filter, CancellationTokenSource cancellationTokenSource)
{
// get data
DataResult<Task<DomainProjectForAdmin>> data = await _adminProjectService.GetAsyncSequenceAsync(filter);
// loading and shaping data
IEnumerable<Task<AdminProject>> projectTasks = data.Results.Select(t => t.ContinueWith(u => _mapper.Map<DomainProjectForAdmin, AdminProject>(u.Result), cancellationTokenSource.Token));
List<AdminProject> projects = (await Task.WhenAll(projectTasks)).ToList();
if (projects.Count == 0)
{
cancellationTokenSource.Cancel();
}
return projects;
}
public async Task PublishAsync(DataQueryOptions filter, string fileName, CancellationTokenSource cancellationTokenSource)
{
try
{
// Get container reference
CloudBlobContainer container = _blobClient.GetContainerReference(CsvContainerName);
await container.CreateIfNotExistsAsync(cancellationTokenSource.Token);
// Delete old if exists
CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
await blob.DeleteIfExistsAsync(cancellationTokenSource.Token);
// Create temp file and serialize data to it
using (var stream = new MemoryStream())
{
// Prepare data filter
int limit = filter.Take.HasValue && filter.Take.Value > 0 ? filter.Take.Value : Int32.MaxValue;
if (filter.Take.HasValue && filter.Take.Value > 0)
{
// limit specified
if (filter.Take.Value > CsvBatchSize)
{
// splitting data into batches
filter.Take = CsvBatchSize;
}
}
else
{
filter.Take = CsvBatchSize;
}
filter.Skip = filter.Skip.HasValue && filter.Skip.Value > 0 ? filter.Skip.Value : 0;
// Adding utf-8 BOM. Details http://stackoverflow.com/questions/4414088/how-to-getbytes-in-c-sharp-with-utf8-encoding-with-bom/4414118#4414118
byte[] bom = Encoding.UTF8.GetPreamble();
await stream.WriteAsync(bom, 0, bom.Length, cancellationTokenSource.Token);
// Batch serialization
var serializationTokenSource = new CancellationTokenSource();
while (!cancellationTokenSource.IsCancellationRequested && !serializationTokenSource.IsCancellationRequested)
{
// loading batch
object data = await OnDataRequest(filter, serializationTokenSource);
// serializing batch
await _mediaTypeFormatter.WriteToStreamAsync(data.GetType(), data, stream, null, null, cancellationTokenSource.Token);
if (filter.Skip + filter.Take == limit)
{
// all loaded
break;
}
// next batch
filter.Skip += CsvBatchSize;
if (filter.Skip + CsvBatchSize >= limit)
{
filter.Take = limit - filter.Skip;
}
}
// Upload result to blob
stream.Position = 0;
await blob.UploadFromStreamAsync(stream, cancellationTokenSource.Token);
}
// Set content type
await blob.FetchAttributesAsync(cancellationTokenSource.Token);
blob.Properties.ContentType = "text/csv";
await blob.SetPropertiesAsync(cancellationTokenSource.Token);
}
catch (Exception e)
{
Trace.TraceError("Failed to publish csv file: {0}", e);
}
}
protected abstract Task<object> OnDataRequest(DataQueryOptions filter, CancellationTokenSource cancellationTokenSource);
public async Task<DataResult<Watch>> GetSequenceAsync(DataQueryOptions filter, string userId)
{
var query = new List<IMongoQuery>(filter.Filters.Count);
// Filtering
List<DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
string requestedUserId = null;
var searchList = new List<string>();
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName<Watch>(x => x.Name),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by name
searchList.Add(f.Value.ToString().ToLowerInvariant());
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<Watch>(x => x.UserId),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by user id
requestedUserId = f.Value.ToString();
query.Add(Query<ProjectEntity>.EQ(p => p.UserId, requestedUserId));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<Watch>(x => x.Generator),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by product id
int productId = Int32.Parse(f.Value.ToString());
query.Add(Query<ProjectEntity>.EQ(p => p.ProductId, productId));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<Watch>(x => x.ProjectType),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by project type
object projectType = Enum.Parse(typeof (ProjectType), f.Value.ToString());
query.Add(Query<ProjectEntity>.EQ(p => p.ProjectType, (int)projectType));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<Watch>(x => x.ProjectSubtype),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by project subtype
object projectSubtype = Enum.Parse(typeof (ProjectSubtype), f.Value.ToString());
query.Add(Query<ProjectEntity>.EQ(p => p.ProjectSubtype, (int)projectSubtype));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<Watch>(x => x.External.VideoUri),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by external video uri
query.Add(Query<ProjectEntity>.EQ(p => p.VideoSource, f.Value));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<Watch>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
// by created date
var date = (DateTime)f.Value;
switch (f.Type)
{
case DataFilterTypes.Equal:
query.Add(Query<ProjectEntity>.EQ(p => p.Created, date));
break;
case DataFilterTypes.LessThan:
query.Add(Query<ProjectEntity>.LT(p => p.Created, date));
break;
case DataFilterTypes.LessThanOrEqual:
query.Add(Query<ProjectEntity>.LTE(p => p.Created, date));
break;
case DataFilterTypes.GreaterThan:
query.Add(Query<ProjectEntity>.GT(p => p.Created, date));
break;
case DataFilterTypes.GreaterThanOrEqual:
query.Add(Query<ProjectEntity>.GTE(p => p.Created, date));
break;
}
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<Watch>(x => x.HitsCount),
StringComparison.OrdinalIgnoreCase) == 0)
{
// by hits count
int threshold = Int32.Parse(f.Value.ToString());
if (f.Type == DataFilterTypes.GreaterThan)
{
query.Add(Query<ProjectEntity>.GT(p => p.HitsCount, threshold));
}
else if (f.Type == DataFilterTypes.GreaterThanOrEqual)
{
query.Add(Query<ProjectEntity>.GTE(p => p.HitsCount, threshold));
}
else
{
query.Add(Query<ProjectEntity>.EQ(p => p.HitsCount, threshold));
}
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<Watch>(x => x.State),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by video state
var videoState = (WatchState)Enum.Parse(typeof (WatchState), f.Value.ToString());
if (videoState == WatchState.Uploading)
{
var avsx = Query<ProjectEntity>.EQ(p => p.AvsxFileId, null);
var originalFileId = Query<ProjectEntity>.EQ(p => p.OriginalVideoFileId, null);
var externalSource = Query<ProjectEntity>.EQ(p => p.VideoSource, null);
query.Add(Query.Or(avsx, Query.And(originalFileId, externalSource)));
}
else if (videoState == WatchState.Encoding)
{
var avsx = Query<ProjectEntity>.NE(p => p.AvsxFileId, null);
var originalFileId = Query<ProjectEntity>.NE(p => p.OriginalVideoFileId, null);
var encodedVideosName = NameOfHelper.PropertyName<ProjectEntity>(x => x.EncodedVideos);
var encodedVideos = Query.NotExists(string.Format("{0}.0", encodedVideosName));
query.Add(Query.And(avsx, originalFileId, encodedVideos));
}
else if (videoState == WatchState.Ready)
{
var avsx = Query<ProjectEntity>.NE(p => p.AvsxFileId, null);
var externalSource = Query<ProjectEntity>.NE(p => p.VideoSource, null);
var originalFileId = Query<ProjectEntity>.NE(p => p.OriginalVideoFileId, null);
var encodedVideosName = NameOfHelper.PropertyName<ProjectEntity>(x => x.EncodedVideos);
var encodedVideos = Query.Exists(string.Format("{0}.0", encodedVideosName));
query.Add(Query.And(avsx, Query.Or(externalSource, Query.And(originalFileId, encodedVideos))));
}
}
else
{
throw new NotSupportedException(string.Format("Filter {0} by property {1} is not supported", f.Type, f.Name));
}
}
string searchText = String.Join(" ", searchList);
if (!String.IsNullOrEmpty(searchText))
{
query.Add(Query.Text(searchText));
}
// we should see only public videos for other users
if (requestedUserId != userId)
{
query.Add(Query<ProjectEntity>.EQ(p => p.Access, (int)ProjectAccess.Public));
}
MongoCursor<ProjectEntity> cursor = _projectRepository.Collection.Find(query.Count > 0 ? Query.And(query) : Query.Null);
// Sorting
IMongoSortBy sortOrder = null;
if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName<Watch>(x => x.Name),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by name
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy<ProjectEntity>.Ascending(p => p.Name)
: SortBy<ProjectEntity>.Descending(p => p.Name);
}
else if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName<Watch>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by created
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy<ProjectEntity>.Ascending(p => p.Created)
: SortBy<ProjectEntity>.Descending(p => p.Created);
}
else if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName<Watch>(x => x.HitsCount),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by hits count
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy<ProjectEntity>.Ascending(p => p.HitsCount)
: SortBy<ProjectEntity>.Descending(p => p.HitsCount);
}
if (sortOrder != null)
{
cursor.SetSortOrder(sortOrder);
}
// Paging
if (filter.Skip.HasValue)
{
cursor.SetSkip(filter.Skip.Value);
}
if (filter.Take.HasValue)
{
cursor.SetLimit(filter.Take.Value);
}
// Count of results
long? count = null;
if (filter.Count)
{
count = cursor.Count();
}
List<ProjectEntity> projects = cursor.ToList();
// Get comments and total comment counts
string[] projectIds = projects.Select(p => p.Id).ToArray();
Dictionary<string, int> commentsCounts = (await _commentRepository.GetCommentsCountByProjectsAsync(projectIds)).ToDictionary(c => c.ProjectId, c => c.CommentsCount);
Dictionary<string, List<CommentEntity>> recentComments = await _commentRepository.GetRecentCommentsByProjectsAsync(projectIds, RecentCommentsLimit);
// Get project user ids united with comment user ids to load entities in batch
string[] userIds = GetUserIds(projects, recentComments);
Dictionary<string, UserEntity> users = (await _userRepository.GetUsersByIdsAsync(userIds)).ToDictionary(u => u.Id);
// Post-processing
return new DataResult<Watch>(projects.Select(p => AggregateProject(p, userId, users, commentsCounts, recentComments)), count);
}
public async Task<IEnumerable<ExampleProject>> GetSequenceAsync(DataQueryOptions filter)
{
// Get example managers
List<DomainUser> users = await _adminUserService.GetUsersInRoleAsync(DomainRoles.ExamplesManager);
if (users.Count == 0)
{
return new List<ExampleProject>();
}
// Get their projects
IEnumerable<DomainProject> projects = await _projectService.GetProjectListByUsersAsync(users);
// Only public videos
projects = projects.Where(p => p.Access == ProjectAccess.Public);
// Paging
if (filter.Skip.HasValue)
{
projects = projects.Skip(filter.Skip.Value);
}
if (filter.Take.HasValue)
{
projects = projects.Take(filter.Take.Value);
}
// Sorting
if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName<ExampleProject>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
projects = filter.OrderByDirection == OrderByDirections.Asc
? projects.OrderBy(p => p.Created)
: projects.OrderByDescending(p => p.Created);
}
List<DomainProject> result = projects.ToList();
// Aggregating project data
List<Task<ExampleProject>> tasks = result.Select(GetProjectAsync).ToList();
await Task.WhenAll(tasks);
List<ExampleProject> examples = tasks.Select(t => t.Result).ToList();
// Post-processing
// Get comments and total comment counts
string[] projectIds = result.Select(p => p.Id).ToArray();
Dictionary<string, int> commentsCounts = (await _commentRepository.GetCommentsCountByProjectsAsync(projectIds)).ToDictionary(c => c.ProjectId, c => c.CommentsCount);
Dictionary<string, List<CommentEntity>> recentComments = await _commentRepository.GetRecentCommentsByProjectsAsync(projectIds, RecentCommentsLimit);
// Get related users list
var userIdsSet = new HashSet<string>();
foreach (var projectComment in recentComments)
{
foreach (CommentEntity c in projectComment.Value)
{
if (userIdsSet.Contains(c.UserId))
{
continue;
}
userIdsSet.Add(c.UserId);
}
}
Dictionary<string, UserEntity> commentUsers = (await _userRepository.GetUsersByIdsAsync(userIdsSet.ToArray())).ToDictionary(u => u.Id);
// Append comments data to exmaples
foreach (ExampleProject example in examples)
{
// comments total count
example.Comments = commentsCounts.ContainsKey(example.Id) ? commentsCounts[example.Id] : 0;
// comments list
if (recentComments.ContainsKey(example.Id))
{
var exampleComments = new List<Comment>();
List<CommentEntity> projectComments = recentComments[example.Id];
foreach (CommentEntity projectComment in projectComments)
{
UserEntity commentAuthor = commentUsers.ContainsKey(projectComment.UserId) ? commentUsers[projectComment.UserId] : null;
DomainComment domainComment = _mapper.Map<Tuple<CommentEntity, UserEntity>, DomainComment>(new Tuple<CommentEntity, UserEntity>(projectComment, commentAuthor));
Comment comment = _mapper.Map<DomainComment, Comment>(domainComment);
if (commentAuthor != null)
{
comment.AvatarUrl = _userAvatarProvider.GetAvatar(commentAuthor.Email);
}
exampleComments.Add(comment);
}
exampleComments.Reverse();
example.CommentList = exampleComments;
}
else
{
example.CommentList = new List<Comment>();
}
}
return examples;
}
public async Task<IEnumerable<Watch>> GetUserLikesSequenceAsync(string userId, DataQueryOptions filter)
{
var result = new List<Watch>();
IEnumerable<DomainUserSignal> data = await _cassandraStatisticsService.GetUserLikesSequenceAsync(StatisticsSpaces.Projects, userId, MemoryPageSize);
var pageData = new List<DomainUserSignal>();
int count = 0;
int skip = filter != null && filter.Skip.HasValue ? filter.Skip.Value : 0;
var uniqueIds = new HashSet<string>();
foreach (DomainUserSignal item in data)
{
if (result.Count >= MemoryPageSize || count > MemoryPageSize*MaxScanPages)
{
// memory limit reached
break;
}
if (filter != null && filter.Take.HasValue && result.Count >= filter.Take.Value + skip)
{
// all items loaded
break;
}
// checking uniqueness
if (!uniqueIds.Contains(item.ItemId))
{
// there can be duplicated ItemIds with different DateTime values due to race conditions
uniqueIds.Add(item.ItemId);
}
else
{
// skipping duplicate
count++;
continue;
}
// adding to memory page
pageData.Add(item);
// checking limits
if ((count > 0 && count%MemoryPageSize == 0) ||
(filter != null && filter.Take.HasValue && pageData.Count >= filter.Take.Value))
{
// loading and filtering items
List<Watch> page = await LoadProjectPageDataAsync(pageData, userId);
result.AddRange(page);
pageData.Clear();
}
count++;
}
if (pageData.Count > 0)
{
// loading last page
List<Watch> page = await LoadProjectPageDataAsync(pageData, userId);
result.AddRange(page);
pageData.Clear();
}
// Skip
result = result.Skip(skip).ToList();
// Take
if (filter != null && filter.Take.HasValue)
{
result = result.Take(filter.Take.Value).ToList();
}
return result;
}
public DataResult<Task<DomainClientForAdmin>> GetAsyncSequence(DataQueryOptions filter)
{
var query = new List<IMongoQuery>(filter.Filters.Count);
// filtering
List<DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName<DomainClientForAdmin>(x => x.Name),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by name
query.Add(Query.Text(f.Value.ToString().ToLowerInvariant()));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<DomainUserForAdmin>(x => x.Email),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by email
string email = f.Value.ToString().ToLowerInvariant();
var expression = new BsonRegularExpression(string.Format("^{0}.*", Regex.Escape(email)));
query.Add(Query<CompanyEntity>.Matches(p => p.Email, expression));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<DomainClientForAdmin>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
// by created date
var date = (DateTime)f.Value;
switch (f.Type)
{
case DataFilterTypes.Equal:
query.Add(Query<CompanyEntity>.EQ(p => p.Created, date));
break;
case DataFilterTypes.LessThan:
query.Add(Query<CompanyEntity>.LT(p => p.Created, date));
break;
case DataFilterTypes.LessThanOrEqual:
query.Add(Query<CompanyEntity>.LTE(p => p.Created, date));
break;
case DataFilterTypes.GreaterThan:
query.Add(Query<CompanyEntity>.GT(p => p.Created, date));
break;
case DataFilterTypes.GreaterThanOrEqual:
query.Add(Query<CompanyEntity>.GTE(p => p.Created, date));
break;
}
}
else
{
throw new NotSupportedException(string.Format("Filter {0} by property {1} is not supported", f.Type, f.Name));
}
}
MongoCursor<CompanyEntity> cursor = _companyRepository.Collection.Find(query.Count > 0 ? Query.And(query) : Query.Null);
IMongoSortBy sortOrder = null;
// sorting
if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName<DomainClientForAdmin>(x => x.Name),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by name
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy<CompanyEntity>.Ascending(p => p.NameSort)
: SortBy<CompanyEntity>.Descending(p => p.NameSort);
}
else if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName<DomainClientForAdmin>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by created
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy<CompanyEntity>.Ascending(p => p.Created)
: SortBy<CompanyEntity>.Descending(p => p.Created);
}
if (sortOrder != null)
{
cursor.SetSortOrder(sortOrder);
}
// paging
if (filter.Skip.HasValue)
{
cursor.SetSkip(filter.Skip.Value);
}
if (filter.Take.HasValue)
{
cursor.SetLimit(filter.Take.Value);
}
// Count of results
long? count = null;
if (filter.Count)
{
count = cursor.Count();
}
// post-processing
return new DataResult<Task<DomainClientForAdmin>>(cursor.Select(GetClientDataAsync), count);
}
public async Task<IEnumerable<TrendingWatch>> GetWeeklyTrendsSequenceAsync(DataQueryOptions filter)
{
List<DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
long? version = null;
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName<TrendingWatch>(x => x.Version),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by product id
version = Int64.Parse(f.Value.ToString());
break;
}
}
IEnumerable<DomainMostSignaledItem> data = await _cassandraStatisticsService.GetMostViewedForLastWeekAsync(StatisticsSpaces.Projects, MemoryPageSize, version);
var pageData = new List<DomainMostSignaledItem>();
int count = 0;
int skip = filter.Skip.HasValue ? filter.Skip.Value : 0;
var uniqueIds = new HashSet<string>();
var result = new List<TrendingWatch>();
foreach (DomainMostSignaledItem mostSignaledItem in data)
{
if (result.Count >= MemoryPageSize || count > MemoryPageSize*MaxScanPages)
{
// memory limit reached
break;
}
if (filter.Take.HasValue && result.Count >= filter.Take.Value + skip)
{
// all items loaded
break;
}
// checking uniqueness
if (!uniqueIds.Contains(mostSignaledItem.ItemId))
{
// there can be duplicated ItemIds with different Count values due to race conditions
uniqueIds.Add(mostSignaledItem.ItemId);
}
else
{
// skipping duplicate
count++;
continue;
}
// adding to memory page
pageData.Add(mostSignaledItem);
// checking limits
if ((count > 0 && count%MemoryPageSize == 0) ||
(filter.Take.HasValue && pageData.Count >= filter.Take.Value))
{
// loading and filtering items
List<TrendingWatch> page = await LoadAndFilterDataAsync(pageData, filter);
result.AddRange(page);
pageData.Clear();
}
count++;
}
if (pageData.Count > 0)
{
// loading last page
List<TrendingWatch> page = await LoadAndFilterDataAsync(pageData, filter);
result.AddRange(page);
pageData.Clear();
}
// Skip
result = result.Skip(skip).ToList();
// Take
if (filter.Take.HasValue)
{
result = result.Take(filter.Take.Value).ToList();
}
return result;
}
private async Task<List<TrendingWatch>> LoadAndFilterDataAsync(IEnumerable<DomainMostSignaledItem> pageData, DataQueryOptions filter)
{
// Preparing
int? productId = null;
WatchState? videoState = null;
List<DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName<TrendingWatch>(x => x.Generator),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by product id
productId = Int32.Parse(f.Value.ToString());
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<TrendingWatch>(x => x.State),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by video state
videoState = (WatchState)Enum.Parse(typeof (WatchState), f.Value.ToString());
}
}
// Loading
List<DomainMostSignaledItem> items = pageData.ToList();
List<Watch> watches = await _watchProjectService.GetByIdsAsync(items.Select(i => i.ItemId).ToArray(), string.Empty);
// Processing projects
var filtered = new List<TrendingWatch>();
foreach (DomainMostSignaledItem i in items)
{
DomainMostSignaledItem item = i;
Watch watch = watches.FirstOrDefault(w => w.Id == item.ItemId);
// Display only existing & public videos
if (watch == null || watch.Access != ProjectAccess.Public)
{
continue;
}
// filtering by product id
if (productId.HasValue)
{
if (watch.Generator != productId.Value)
{
continue;
}
}
// filterring by state
if (videoState.HasValue)
{
if (watch.State != videoState.Value)
{
continue;
}
}
TrendingWatch trendingWatch = _mapper.Map<Watch, TrendingWatch>(watch);
trendingWatch.Version = item.Version;
filtered.Add(trendingWatch);
}
return filtered;
}
public async Task<DataResult<Task<DomainProjectForAdmin>>> GetAsyncSequenceAsync(DataQueryOptions filter)
{
var query = new List<IMongoQuery>(filter.Filters.Count);
// filtering
List<DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName<DomainProjectForAdmin>(x => x.Name),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by name
query.Add(Query.Text(f.Value.ToString().ToLowerInvariant()));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<DomainProjectForAdmin>(x => x.ProductType),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by product type
var productId = Int32.Parse(f.Value.ToString());
query.Add(Query<ProjectEntity>.EQ(p => p.ProductId, productId));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<DomainProjectForAdmin>(x => x.UserId),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by user id
query.Add(Query<ProjectEntity>.EQ(p => p.UserId, f.Value.ToString()));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<DomainProjectForAdmin>(x => x.UserName),
StringComparison.OrdinalIgnoreCase) == 0 && f.Type == DataFilterTypes.Equal)
{
// by user name
List<UserEntity> profiles = await _userRepository.FindByNameAsync(f.Value.ToString());
if (profiles.Count == 0)
{
// no users found
return new DataResult<Task<DomainProjectForAdmin>>(new Task<DomainProjectForAdmin>[] { });
}
List<string> allIds = profiles.Select(prof => prof.Id).ToList();
query.Add(Query<ProjectEntity>.Where(p => allIds.Contains(p.UserId)));
}
else if (string.Compare(f.Name, NameOfHelper.PropertyName<DomainProjectForAdmin>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
// by created date
var date = (DateTime)f.Value;
switch (f.Type)
{
case DataFilterTypes.Equal:
query.Add(Query<ProjectEntity>.EQ(p => p.Created, date));
break;
case DataFilterTypes.LessThan:
query.Add(Query<ProjectEntity>.LT(p => p.Created, date));
break;
case DataFilterTypes.LessThanOrEqual:
query.Add(Query<ProjectEntity>.LTE(p => p.Created, date));
break;
case DataFilterTypes.GreaterThan:
query.Add(Query<ProjectEntity>.GT(p => p.Created, date));
break;
case DataFilterTypes.GreaterThanOrEqual:
query.Add(Query<ProjectEntity>.GTE(p => p.Created, date));
break;
}
}
else
{
throw new NotSupportedException(string.Format("Filter {0} by property {1} is not supported", f.Type, f.Name));
}
}
if (!filters.Any() && !string.IsNullOrEmpty(filter.OrderBy))
{
// MongoDb 2.6 HACK!!!
// adding fake query to hint proper index
if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName<DomainProjectForAdmin>(x => x.Name),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by name
query.Add(Query<ProjectEntity>.Exists(p => p.Name));
}
else if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName<DomainProjectForAdmin>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by created
query.Add(Query<ProjectEntity>.Exists(p => p.Created));
}
else if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName<DomainProjectForAdmin>(x => x.ProductType),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by product type
query.Add(Query<ProjectEntity>.Exists(p => p.ProductId));
}
}
MongoCursor<ProjectEntity> cursor = _projectRepository.Collection.Find(query.Count > 0 ? Query.And(query) : Query.Null);
IMongoSortBy sortOrder = null;
// sorting
if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName<DomainProjectForAdmin>(x => x.Name),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by name
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy<ProjectEntity>.Ascending(p => p.Name)
: SortBy<ProjectEntity>.Descending(p => p.Name);
}
else if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName<DomainProjectForAdmin>(x => x.Created),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by created
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy<ProjectEntity>.Ascending(p => p.Created)
: SortBy<ProjectEntity>.Descending(p => p.Created);
}
else if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName<DomainProjectForAdmin>(x => x.ProductType),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by product type
sortOrder = filter.OrderByDirection == OrderByDirections.Asc
? SortBy<ProjectEntity>.Ascending(p => p.ProductId)
: SortBy<ProjectEntity>.Descending(p => p.ProductId);
}
if (sortOrder != null)
{
cursor.SetSortOrder(sortOrder);
}
// paging
if (filter.Skip.HasValue)
{
cursor.SetSkip(filter.Skip.Value);
}
if (filter.Take.HasValue)
{
cursor.SetLimit(filter.Take.Value);
}
// Count of results
long? count = null;
if (filter.Count)
{
count = cursor.Count();
}
// post-processing
return new DataResult<Task<DomainProjectForAdmin>>(cursor.Select(GetProjectDataAsync), count);
}
public async Task<IEnumerable<DomainTrackingStatPerDate>> GetStatsPerDateAsync(string subscriptionId, string url, DataQueryOptions filter)
{
if (string.IsNullOrEmpty(subscriptionId))
{
throw new ArgumentNullException("subscriptionId");
}
// Filter
DateTime? dateFromFilter = null;
DateTime? dateToFilter = null;
List<DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName<DomainTrackingStatPerDate>(x => x.Date),
StringComparison.OrdinalIgnoreCase) == 0)
{
// by created date
DateTime date = ((DateTime)f.Value).Date;
switch (f.Type)
{
case DataFilterTypes.Equal:
dateFromFilter = date;
dateToFilter = date.AddDays(1);
break;
case DataFilterTypes.LessThan:
dateToFilter = date;
break;
case DataFilterTypes.LessThanOrEqual:
dateToFilter = date.AddDays(1);
break;
case DataFilterTypes.GreaterThan:
dateFromFilter = date.AddDays(1);
break;
case DataFilterTypes.GreaterThanOrEqual:
dateFromFilter = date;
break;
}
}
else
{
throw new NotSupportedException(string.Format("Filter {0} by property {1} is not supported", f.Type, f.Name));
}
}
// Aggregate per date
IEnumerable<TrackingStatPerDateEntity> aggregation = await _trackingStatRepository.AggregatePerDateAsync(subscriptionId, url, dateFromFilter, dateToFilter);
// Project
IEnumerable<DomainTrackingStatPerDate> projectedResult = aggregation.Select(r => _mapper.Map<TrackingStatPerDateEntity, DomainTrackingStatPerDate>(r));
return projectedResult;
}
public async Task<DataResult<DomainTrackingStat>> GetUrlStatsAsync(string subscriptionId, string url, DataQueryOptions filter)
{
if (string.IsNullOrEmpty(subscriptionId))
{
throw new ArgumentNullException("subscriptionId");
}
if (string.IsNullOrEmpty(url))
{
throw new ArgumentNullException("url");
}
IQueryable<TrackingStatEntity> data = _trackingStatRepository.Context.Where(s => s.SubscriptionId == subscriptionId && s.RedirectUrl == url);
// Filter
List<DataFilterRule> filters = filter.Filters.Where(f => f.Value != null).ToList();
foreach (DataFilterRule dataFilterRule in filters)
{
DataFilterRule f = dataFilterRule;
if (string.Compare(f.Name, NameOfHelper.PropertyName<DomainTrackingStat>(x => x.Date),
StringComparison.OrdinalIgnoreCase) == 0)
{
// by created date
var date = ((DateTime)f.Value).Date;
switch (f.Type)
{
case DataFilterTypes.Equal:
data = data.Where(p => p.Date > date && p.Date < date.AddDays(1));
break;
case DataFilterTypes.LessThan:
data = data.Where(p => p.Date < date);
break;
case DataFilterTypes.LessThanOrEqual:
data = data.Where(p => p.Date < date.AddDays(1));
break;
case DataFilterTypes.GreaterThan:
data = data.Where(p => p.Date > date);
break;
case DataFilterTypes.GreaterThanOrEqual:
data = data.Where(p => p.Date >= date);
break;
}
}
else
{
throw new NotSupportedException(string.Format("Filter {0} by property {1} is not supported", f.Type, f.Name));
}
}
// Sort
if (!string.IsNullOrEmpty(filter.OrderBy))
{
if (string.Compare(filter.OrderBy, NameOfHelper.PropertyName<DomainTrackingStat>(x => x.Date),
StringComparison.OrdinalIgnoreCase) == 0)
{
// order by name
data = filter.OrderByDirection == OrderByDirections.Asc
? data.OrderBy(p => p.Date)
: data.OrderByDescending(p => p.Date);
}
else
{
throw new NotSupportedException(string.Format("Ordering by {0} is not supported", filter.OrderBy));
}
}
// Count of results
long? count = null;
if (filter.Count)
{
count = data.LongCount();
}
// Paging
if (filter.Skip.HasValue)
{
data = data.Skip(filter.Skip.Value);
}
if (filter.Take.HasValue)
{
data = data.Take(filter.Take.Value);
}
// post-processing
IQueryable<DomainTrackingStat> results = data.Select(s => _mapper.Map<TrackingStatEntity, DomainTrackingStat>(s));
return new DataResult<DomainTrackingStat>(results, count);
}
