private async Task <Dictionary <string, int> > GenDbStrs(List <ImageGalleryDb.Models.ImageGallery.Image> query)
{
var tags = query
.Where(x => x.MSVision.Tags != null)
.SelectMany(x => x.MSVision.Tags.Select(y => y.Name));
var descriptionTags = query
.Where(x => x.MSVision.Description != null)
.Where(x => x.MSVision.Description.Tags != null)
.SelectMany(x => x.MSVision.Description.Tags);
var colors = query
.Where(x => x.MSVision.Color != null)
.Where(x => x.MSVision.Color.DominantColors != null)
.SelectMany(x => x.MSVision.Color.DominantColors.Select(x => x.Name));
var strings = new List <string>();
strings.AddRange(tags);
strings.AddRange(descriptionTags);
strings.AddRange(colors);
var strs = strings
.Distinct()
.Select(x => new Str {
Name = x
})
.ToList();
await Insert(strs);
using var imContext = new ImContext(_im_dbContextOptions);
return((await imContext.Str.ToListAsync())
.ToDictionary(x => x.Name, x => x.Id));
}
private async Task UpdateTagTopMin()
{
using var imContext = new ImContext(_im_dbContextOptions);
var strs = await imContext.Str.ToListAsync();
foreach (var item in strs)
{
var count = await imContext.ImageStr.Where(x => x.StrId == item.Id).CountAsync();
var take = 1000;
var sc = imContext.ImageStr.Where(x => x.StrId == item.Id)
.Where(x => x.Score != null)
.OrderByDescending(x => x.Score).Take(take)
.OrderBy(x => x.Score);
var score = (await sc.FirstOrDefaultAsync())?.Score;
item.TopScoreMin = score;
if (score != null)
{
var scMin = await imContext.ImageStr.Where(x => x.StrId == item.Id)
.Where(x => x.Score != null)
.OrderBy(x => x.Score).FirstOrDefaultAsync();
item.TopScoreMinMin = scMin.Score;
}
}
await imContext.SaveChangesAsync();
}
public async Task <string> Random(int h = 220)
{
if (h < 1 && h > 2048)
{
return(null);
}
var id = _random.Next(0, _data.ImagesCount - 1);
using var imContext = new ImContext(_data.Options);
var gid = (await imContext.Image.Skip(id).FirstOrDefaultAsync()).GoogleDriveId;
return($"https://drive.google.com/thumbnail?id={gid}&sz=h{h}");
}
private async Task Insert <T>(List <T> list) where T : class
{
var val = 50000;
for (int i = 0; i < list.Count; i += val)
{
_logger.Info($"saving {typeof(T).Name}: {i}/{list.Count}");
using var imContext = new ImContext(_im_dbContextOptions);
imContext.ChangeTracker.AutoDetectChangesEnabled = false;
await imContext.AddRangeAsync(list.Skip(i).Take((val)).ToArray());
await imContext.SaveChangesAsync();
}
}
public async Task <IActionResult> Image(string strId)
{
if (string.IsNullOrEmpty(strId))
{
return(null);
}
using var imContext = new ImContext(_data.Options);
var image = await imContext.Image
.Where(x => x.SeoId == strId)
.FirstOrDefaultAsync();
if (image == null)
{
return(null);
}
var strs = await imContext.ImageStr.Include(x => x.Str).Where(x => x.ImageId == image.Id).ToListAsync();
var tags = strs
.Where(y => y.IsTag == ImageGalleryDb.Models.Im.Boolean.True).Select(y => y.Str.Name).ToList();
var topTags = strs.Where(x => x.Score > 0.9).Select(y => y.Str.Name).ToList();
var descriptionCaptions = await imContext.DescriptionCaption
.Where(x => x.ImageDescriptionCaptions.Any(y => y.ImageId == image.Id)).ToListAsync();
return(View(new ImageResponce
{
Image = new Models.Gallery.Image
{
Id = image.GoogleDriveId,
Title = image.Title,
DescriptionCaptions = descriptionCaptions
.Select(y => y.Name).ToList(),
Tags = tags,
TopTags = topTags,
DominantColors = strs
.Where(y => y.IsDominantColor == ImageGalleryDb.Models.Im.Boolean.True)
.Select(y => y.Str.Name).ToList(),
Width = image.GoogleDriveWidth,
Height = image.GoogleDriveHeight
}
}));
}
private async Task <Dictionary <string, int> > GenDbCategories(List <ImageGalleryDb.Models.ImageGallery.Image> query)
{
var items = query
.Where(x => x.MSVision.Categories != null)
.SelectMany(x => x.MSVision.Categories.Select(y => y.Name))
.Distinct()
.Select(x => new ImageGalleryDb.Models.Im.Category
{
Name = x
})
.ToList();
await Insert(items);
using var imContext = new ImContext(_im_dbContextOptions);
return((await imContext.Category.ToListAsync())
.ToDictionary(x => x.Name, x => x.Id));
}
private async Task UpdateCategoryTopMin()
{
using var imContext = new ImContext(_im_dbContextOptions);
var cats = await imContext.Category.ToListAsync();
foreach (var item in cats)
{
var count = await imContext.ImageCategory.Where(x => x.CategoryId == item.Id).CountAsync();
var take = 1000;
var sc = imContext.ImageCategory.Where(x => x.CategoryId == item.Id)
.OrderByDescending(x => x.Score).Take(take)
.OrderBy(x => x.Score);
var score = (await sc.FirstOrDefaultAsync()).Score;
item.TopScoreMin = score;
}
await imContext.SaveChangesAsync();
}
private async Task <Dictionary <string, int> > GenDbImages(List <ImageGalleryDb.Models.ImageGallery.Image> query)
{
var items = query
.Select(x => new Image
{
GoogleDriveId = x.GoogleDrive.GoogleId,
GoogleDriveWidth = x.GoogleDrive.Width,
GoogleDriveHeight = x.GoogleDrive.Height,
SeoId = x.SeoId,
Title = x.Title,
MSVisionAccentColor = x.MSVision.Color?.AccentColor,
MSVisionIsBWImg = (x.MSVision.Color != null) && (x.MSVision.Color.IsBWImg)
? ImageGalleryDb.Models.Im.Boolean.True
: ImageGalleryDb.Models.Im.Boolean.False
}).ToList();
await Insert(items);
using var imContext = new ImContext(_im_dbContextOptions);
return((await imContext.Image.ToListAsync())
.ToDictionary(x => x.GoogleDriveId, x => x.Id));
}
public static async Task Main(string[] args)
{
var settingsData = File.ReadAllText("settings.clj");
var config = SConvert.DeserializeObject <Config.Config>(settingsData);
LoggerService.Init(config.LogPath);
var logger = LogManager.GetLogger(Logger.Logger.log.ToString());
var loggerFactory = ((ILoggerFactory)(new LoggerFactory())).AddNLog();
//var xloggerFactory = ((ILoggerFactory)(LoggerFactory.Create(x=>x.AddConsole())));
logger.Info("App starting ...");
var data = new Data
{
//Options = options
};
var dataSource = $"DataSource={config.DbPath}";
var profiler = MiniProfiler.StartNew("My Pofiler Name");
using (var connection = new Microsoft.Data.Sqlite.SqliteConnection(dataSource))
{
connection.Open();
var connectionInMemory = new Microsoft.Data.Sqlite.SqliteConnection("DataSource=:memory:");
connectionInMemory.Open();
var builderInMemory = new DbContextOptionsBuilder().UseSqlite(connectionInMemory);
data.Options = builderInMemory.Options;
connection.BackupDatabase(connectionInMemory);
}
//var connection = new Microsoft.Data.Sqlite.SqliteConnection(dataSource);
//var connectionInMemoryProfile = new StackExchange.Profiling.Data.ProfiledDbConnection(connection, MiniProfiler.Current);
//var dbContextOptionsBuilder = new DbContextOptionsBuilder().UseSqlite(connectionInMemoryProfile);
//var options = dbContextOptionsBuilder.Options;
using (var imContext = new ImContext(data.Options))
{
data.TopTags = await imContext.Str
.Where(x => x.TopScoreMin > 0.5)
.OrderByDescending(x => x.TopScoreMin)
.Select(x => x.Name)
.ToListAsync();
var categories = await imContext.Category
.Select(x => x.Name)
.ToListAsync();
data.SortedCategories = categories
.Where(x => x.IndexOf('_') == x.Length - 1)
.Select(x => x.TrimEnd('_'))
.AsParallel()
.OrderBy(x => x)
.Select(x => new Category
{
Id = string.Intern(x + "_"),
Name = x,
Categories = categories
.Where(y => y.IndexOf(x) == 0 && y.Length > x.Length + 1)
.Select(y => new Category
{
Id = string.Intern(y),
Name = y.Replace("_", " ").Remove(0, x.Length + 1)
})
.OrderBy(x => x.Name)
.ToList()
})
.ToList();
data.ImagesCount = await imContext.Image.CountAsync();
data.Colors = await imContext.ImageStr
.Where(x => x.IsDominantColor == ImageGalleryDb.Models.Im.Boolean.True)
.Select(x => x.Str.Name)
.Distinct()
.ToListAsync();
//var images = await imContext.Image.ToListAsync();
//var sitemapService = new SitemapService(logger, images, config.Domain);
//// TODO: comment this code
//var path = Path.Combine(Environment.CurrentDirectory, "wwwroot", "sitemap");
//var fileNames = await sitemapService.ReGenerate(path);
//fileNames.ForEach(x => Console.WriteLine(x));
}
var startupService = new StartupService(logger, config, data);
startupService.LoggerFactory = loggerFactory;
CreateWebHostBuilder(args, startupService).Build().Run();
}
public async Task <ErrorResponce> Images([FromBody] ImageQuery imageQuery)
{
if (imageQuery == null ||
string.IsNullOrWhiteSpace(imageQuery.Query) ||
imageQuery.Query.Length > 128)
{
return(new ErrorResponce {
ErrorCode = ErrorCode.BadRequestParams
});
}
var query = imageQuery.Query;
var page = imageQuery.Page;
_logger.Info($"query: {query}; page: {page}");
query = query.ToLower();
query = Regex.Replace(query, @"[^a-zA-Z_ ]+", " ");
query = Regex.Replace(query, @"[ ]+", " ");
query = Regex.Replace(query, @"[__]+", "_");
query = query.Trim();
var arr = query.Split(' ').Distinct().ToList();
arr.Remove("and");
arr.Remove("or");
if (arr.Count > 6 || arr.Count == 0)
{
return(new ErrorResponce {
ErrorCode = ErrorCode.BadRequestParams
});
}
var possibleCategories = arr.Where(x => x.Contains('_')).ToList();
possibleCategories.ForEach(x => arr.Remove(x));
if (possibleCategories.Count > 1 ||
(possibleCategories.Count == 0 && arr.Count == 0))
{
return(new ImagesResponce());
}
using var imContext = new ImContext(_data.Options);
//IQueryable<ImageGalleryDb.Models.Im.Image> imQ = null;
var isBW = ImageGalleryDb.Models.Im.Boolean.False;
if (arr.Count > 0)
{
var colors = _data.Colors.Where(x => arr.Contains(x)).ToList();
if (colors.Count == 2 &&
colors.Contains("black") &&
colors.Contains("white"))
{
isBW = ImageGalleryDb.Models.Im.Boolean.True;
}
}
var imas = imContext.Image.AsQueryable();
if (isBW == ImageGalleryDb.Models.Im.Boolean.True)
{
imas = imContext.Image.Where(x => x.MSVisionIsBWImg == ImageGalleryDb.Models.Im.Boolean.True);
}
if (possibleCategories.Count > 0)
{
var category = (from im in imContext.Category
where im.Name == possibleCategories[0]
select im).FirstOrDefault();
if (category == null)
{
return(new ImagesResponce());
}
imas = from im in imas
join imc in imContext.ImageCategory on im.Id equals imc.ImageId
where imc.CategoryId == category.Id && imc.Score > (category.TopScoreMin - 0.001)
orderby imc.Score descending
select im;
}
if (arr.Count == 1)
{
var tag = (from im in imContext.Str
where im.Name == arr[0]
select im).FirstOrDefault();
if (tag == null)
{
return(new ImagesResponce());
}
if (tag.TopScoreMin != null)
{
imas = from im in imas
join imc in imContext.ImageStr on im.Id equals imc.ImageId
where imc.StrId == tag.Id && imc.Score > (tag.TopScoreMin - 0.001)
orderby imc.Score descending
select im;
}
else
{
imas = (from im in imas
join imc in imContext.ImageStr on im.Id equals imc.ImageId
where imc.StrId == tag.Id
select im).Take(1000);
}
}
else if (arr.Count > 1)
{
var word = string.Join(' ', arr);
var tagWord = (from im in imContext.Str
where im.Name == word
select im).FirstOrDefault();
if (tagWord != null)
{
if (tagWord.TopScoreMin != null)
{
imas = from im in imas
join imc in imContext.ImageStr on im.Id equals imc.ImageId
where imc.StrId == tagWord.Id && imc.Score > (tagWord.TopScoreMin - 0.001)
orderby imc.Score descending
select im;
}
else
{
imas = (from im in imas
join imc in imContext.ImageStr on im.Id equals imc.ImageId
where imc.StrId == tagWord.Id
select im).Take(1000);
}
}
else
{
var imags = await(from im in imContext.Str
where arr.Contains(im.Name)
orderby im.TopScoreMin descending
select im).ToListAsync();
if (imags.Count != arr.Count)
{
return(new ImagesResponce());
}
if (imags[0].TopScoreMin == null)
{
var ids = imags.Select(x => x.Id).ToList();
imas = imContext.ImageStr.Where(x => ids.Contains(x.StrId))
.Take(5000)
.GroupBy(x => x.Image)
.Where(x => x.Count() == arr.Count)
.Select(x => x.Key);
}
else
{
var ids = imags.Skip(1).Select(x => x.Id).ToList();
if (imags[1].TopScoreMinMin != null)
{
var ids2 = ids.Skip(1).ToList();
imas = imContext.ImageStr.Where(x =>
(x.StrId == imags[0].Id && x.Score > (imags[0].TopScoreMin - 0.001)) ||
(x.StrId == imags[1].Id && x.Score > (imags[1].TopScoreMinMin - 0.001)) ||
ids2.Contains(x.StrId))
.GroupBy(x => x.Image)
.Where(x => x.Count() == arr.Count)
.OrderByDescending(x => x.Sum(y => y.Score))
.Select(x => x.Key);
}
else
{
imas = imContext.ImageStr.Where(x =>
(x.StrId == imags[0].Id && x.Score > (imags[0].TopScoreMin - 0.001)) ||
ids.Contains(x.StrId))
.GroupBy(x => x.Image)
.Where(x => x.Count() == arr.Count)
.OrderByDescending(x => x.Sum(y => y.Score))
.Select(x => x.Key);
}
}
}
}
_logger.Info($"Preparing: query: {query}; page: {page}");
var result = new List <Models.Gallery.Image>();
//var sql = imas.ToSql();
// Benchmark(() => {
// var list= imas
//.Skip(page * PageCount).Take(PageCount)
//.Select(x => new Models.Gallery.Image
//{
// Url = $"https://drive.google.com/thumbnail?id={x.GoogleDriveId}&sz=w480",
// StrId = x.SeoId,
//})
//.ToList();
// }, 10);
result = await imas
.Skip(page *PageCount).Take(PageCount)
.Select(x => new Models.Gallery.Image
{
Url = $"https://drive.google.com/thumbnail?id={x.GoogleDriveId}&sz=w480",
StrId = x.SeoId,
})
.ToListAsync();
_logger.Info($"Done: query: {query}; page: {page}");
return(new ImagesResponce
{
Images = result
});
}
