public virtual IQueryable <K2Field.SmartForms.Workspace.Data.WorkspaceUser> All(Func <K2Field.SmartForms.Workspace.Data.WorkspaceUser, bool> where)
{
return(DBSet.Where(where).AsQueryable());
}
public IList <TEntity> InsertAll(IEnumerable <TEntity> entities)
{
return(DBSet.AddRange(entities).ToList());
}
public TEntity Delete(TEntity entity)
{
return(DBSet.Remove(entity));
}
public virtual List <T> Find <T>(Expression <Func <TEntity, bool> > predicate, Expression <Func <TEntity, T> > selector)
{
return(DBSet.Where(predicate).Select(selector).ToList());
}
public int Count()
{
return(DBSet.Count());
}
public int GetMaxIndex()
{
return(DBSet.Any() ? DBSet.Max(m => m.Index) : 0);
}
public virtual bool Exists(Expression <Func <TEntity, bool> > predicate)
{
return(DBSet.Any(predicate));
}
public async Task <IEnumerable <ClienteProduto> > GetByProd(long prodId)
{
return(await DBSet.Where(x => x.ProdutoId == prodId).ToListAsync());
}
public virtual Task <List <TEntity> > GetAll()
{
return(DBSet.ToListAsync());
}
public async Task <IEnumerable <ClienteProduto> > GetAllWithInclude()
{
return(await DBSet.Include(x => x.Cliente).Include(x => x.Produto).ToListAsync());
}
public async Task <IEnumerable <ClienteProduto> > GetByCilent(long clientId)
{
return(await DBSet.Where(x => x.ClienteId == clientId).ToListAsync());
}
public bool ExistRelaionship(ClienteProduto cliProd)
{
return(DBSet.AsNoTracking().Any(x => (x.ClienteId == cliProd.ClienteId && x.ProdutoId == cliProd.ProdutoId) && x.Id != cliProd.Id));
}
public bool ExistBarCode(ClienteProduto cliProd)
{
return(DBSet.AsNoTracking().Any(x => x.CodigoBarra == cliProd.CodigoBarra && x.Id != cliProd.Id));
}
public override Candidato Recuperar(Expression <Func <Candidato, bool> > predicate) => DBSet .Include(h => h.lstCandidatoDisponibilidadeHoras) .Include(p => p.lstCandidatoDisponibilidadePeriodo) .Include(l => l.lstCandidatoLinguagem) .FirstOrDefault(predicate);
/// <summary>
/// 执行sql
/// 使用方式 FormattableString sql=$"select * from xx where name ={xx} and pwd={xx1} "，
/// FromSqlInterpolated内部处理sql注入的问题，直接在{xx}写对应的值即可
/// 注意：sql必须 select * 返回所有TEntity字段，
/// </summary>
/// <param name="formattableString"></param>
/// <returns></returns>
public virtual IQueryable <TEntity> FromSqlInterpolated([NotNull] FormattableString sql)
{
//DBSet.FromSqlInterpolated(sql).Select(x => new { x,xxx}).ToList();
return(DBSet.FromSqlInterpolated(sql));
}
public virtual TEntity GetByID(params object[] keyValues)
{
return(DBSet.Find(keyValues));
}
public T GetById(int id)
{
return(DBSet.Find(id));
}
private void UpdateIamgeTagStatus(ImageTag item, string status)
{
using var db = DBSet.GetCon(DBSet.SqliteDBName.Bilibili);
item.Status = status;
db.Update(currentShowTag);
}
public void OnPost(List <IFormFile> files, string imageUrl)
{
long size = 0;
this.ViewData["error"] = -1;
if (files != null && files.Count > 0)
{
var file = files[0];
var stream = file.OpenReadStream();
size = stream.Length;
var bytes = new byte[size];
stream.Read(bytes, 0, (int)size);
var ret = faceDb.SereachUserByImage(bytes);
if ((int)ret["error_code"] != 0)
{
this.ViewData["error"] = -1;
// ErrorMessage = JsonConvert.SerializeObject(ret, Formatting.Indented);
ErrorMessage = ret["error_msg"].ToString();
return;
}
// 能找到对应的人脸
ErrorMessage = JsonConvert.SerializeObject(ret, Formatting.Indented);
}
else if (!string.IsNullOrEmpty(imageUrl))
{
var ret = faceDb.SereachUserByUrl(imageUrl);
if ((int)ret["error_code"] != 0)
{
this.ViewData["error"] = -1;
// ErrorMessage = JsonConvert.SerializeObject(ret, Formatting.Indented);
ErrorMessage = ret["error_msg"].ToString();
return;
}
// ErrorMessage = JsonConvert.SerializeObject(ret, Formatting.Indented);
List <UPModel> ups = new List <UPModel>();
var arr = ret["result"]["user_list"] as JArray;
if (arr != null && arr.Count > 0)
{
using (var db = DBSet.GetCon(DBSet.SqliteDBName.Bilibili))
{
foreach (var item in arr)
{
var id = int.Parse(item["user_id"].ToString());
var dbItem = db.SingleById <UP>(id);
if (dbItem != null)
{
ups.Add(new UPModel
{
UPId = id.ToString(),
Name = dbItem.name,
FaceUrl = dbItem.face,
Rate = ((double)item["score"]).ToString("f2")
});
}
}
}
}
FindUP = ups.ToArray();
}
else
{
ErrorMessage = "上传图片或者url，你总得设置一个吧";
}
}
/// <summary>
/// 将视频封面照，拿去百度检查
/// </summary>
private static void DetectFace2(int maxGetCount = 60 * 2 * 30) // 30分钟的数据)
{
var baiduai = new FaceDetect();
int i = 0;
DateTime nextCallTime = DateTime.Now;
using (var db = DBSet.GetCon(DBSet.SqliteDBName.Bilibili))
{
foreach (var up in db.Select <UP>(o => o.follower > 3000).OrderByDescending(o => o.follower).ToArray())
{
foreach (var av in db.Select <AV>(o => o.UpId == up.Id))
{
if (isExit)
{
break;
}
var pic = new Uri(av.pic).AbsolutePath.Replace("/", "_");
// 只按照本地文件名做验证
var detect = db.Single <ImageDetect>(o => o.LocalFile == pic);
if (detect == null)
{
byte[] bytes = null;
if (FromWeb)
{
try
{
bytes = new WebClient().DownloadData(av.pic);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
else
{
// 封面照落地，根据目前采集到数据，如果将宅舞区的封面照落地的话，估计要100多G
// 再加上三次元区，估计服务器硬盘干不动
var imagePath = SpiderConfig.GetPath($"imgs/{av.UpId}/{av.Id}");
var imageFile = Path.Combine(imagePath, pic);
if (!File.Exists(imageFile))
{
continue;
}
bytes = File.ReadAllBytes(imageFile);
}
if (bytes == null)
{
continue;
}
var wait = (int)(nextCallTime - DateTime.Now).TotalMilliseconds + 1;
if (wait > 0)
{
Console.WriteLine($"wait {wait}");
Thread.Sleep(wait);
}
var start = DateTime.Now;
var ret = baiduai.DetectFromBytes(bytes);
Console.Write($"useTime:{ (DateTime.Now - start).TotalMilliseconds} ms ");
nextCallTime = DateTime.Now.AddMilliseconds(500);
if (ret != null)
{
var dbItem = new ImageDetect
{
AVId = av.Id,
UpId = av.UpId,
LocalFile = pic,
Url = av.pic,
Detect = ret.result,
};
if (ret.error_code == 0)
{
dbItem.face_num = ret.result.face_num;
if (ret.result.face_num > 0)
{
dbItem.max_face_probability = ret.result.face_list.Max(o => o.face_probability);
dbItem.max_quality = ret.result.face_list.Max(o => GetQuality(o));
}
}
db.Insert(dbItem);
Console.WriteLine(av.title);
if (maxGetCount-- < 0)
{
return;
}
// Thread.Sleep(500); // 百度的免费接口只有 2 qps，所以在这里做一下延迟。
}
}
else
{
// Console.WriteLine("忽略 " + av.title);
}
}
}
}
}
public virtual Task <bool> ExistsAsync(Expression <Func <TEntity, bool> > predicate)
{
return(DBSet.AnyAsync(predicate));
}
public static bool GetZhaiWuNewList(int page = 0, int rid = 20)
{
var ret = new Newlist().Download(page, 50, rid);
bool hasNew = false;
using (var historyCon = DBSet.GetCon(DBSet.SqliteDBName.History))
{
using (var con = DBSet.GetCon(DBSet.SqliteDBName.Bilibili))
{
foreach (var item in ret.archives)
{
var dbItem = con.SingleById <AV>(item.aid);
if (dbItem == null)
{
dbItem = new AV()
{
Id = item.aid,
UpId = item.owner.mid,
rid = rid,
};
Console.WriteLine(item.title);
UpdateAVData(dbItem, item);
con.Insert(dbItem);
// 更新封面照片
//var imagePath = SpiderConfig.GetPath($"imgs/{dbItem.UpId}/{dbItem.Id}");
//var pic = new Uri(item.pic).AbsolutePath.Replace("/","_");
//var imageFile = Path.Combine(imagePath, pic);
//if (!File.Exists(imageFile))
//{
// Console.WriteLine(imageFile);
// var bytes = new WebClient().DownloadData(item.pic);
// File.WriteAllBytes(imageFile, bytes);
//}
hasNew = true;
}
else
{
UpdateAVData(dbItem, item);
con.Update(dbItem);
}
var upItem = con.SingleById <UP>(item.owner.mid);
if (upItem == null)
{
var upspider = new UpSpider();
int mid = item.owner.mid;
Console.WriteLine($"get up {mid}");
var info = upspider.GetUpInfo(mid);
var face = upspider.GetFaceStat(mid);
var upstat = upspider.GetUpStat(mid);
upItem = new UP
{
Id = mid,
follower = face.follower,
following = face.following,
face = info.face,
jointime = info.jointime,
level = info.jointime,
name = info.name,
rank = info.rank,
sex = info.sex,
sign = info.sign,
views = upstat.archive.view,
};
con.Insert(upItem);
}
var dbHistoryItem = historyCon.SingleById <AVHistory>(item.aid);
if (dbHistoryItem == null)
{
dbHistoryItem = new AVHistory
{
Id = item.aid,
History = new List <ArchivesItem> {
item
}
};
historyCon.Insert(dbHistoryItem);
}
else
{
dbHistoryItem.History.Add(item);
historyCon.Update(dbHistoryItem);
}
}
}
}
return(hasNew);
}
public virtual List <TEntity> FromSql(string sql, params SqlParameter[] sqlParameters)
{
return(DBSet.FromSql(sql, sqlParameters).ToList());
}
public static global::System.Xml.Schema.XmlSchemaComplexType GetTypedTableSchema(global::System.Xml.Schema.XmlSchemaSet xs)
{
global::System.Xml.Schema.XmlSchemaComplexType type = new global::System.Xml.Schema.XmlSchemaComplexType();
global::System.Xml.Schema.XmlSchemaSequence sequence = new global::System.Xml.Schema.XmlSchemaSequence();
DBSet ds = new DBSet();
global::System.Xml.Schema.XmlSchemaAny any1 = new global::System.Xml.Schema.XmlSchemaAny();
any1.Namespace = "http://www.w3.org/2001/XMLSchema";
any1.MinOccurs = new decimal(0);
any1.MaxOccurs = decimal.MaxValue;
any1.ProcessContents = global::System.Xml.Schema.XmlSchemaContentProcessing.Lax;
sequence.Items.Add(any1);
global::System.Xml.Schema.XmlSchemaAny any2 = new global::System.Xml.Schema.XmlSchemaAny();
any2.Namespace = "urn:schemas-microsoft-com:xml-diffgram-v1";
any2.MinOccurs = new decimal(1);
any2.ProcessContents = global::System.Xml.Schema.XmlSchemaContentProcessing.Lax;
sequence.Items.Add(any2);
global::System.Xml.Schema.XmlSchemaAttribute attribute1 = new global::System.Xml.Schema.XmlSchemaAttribute();
attribute1.Name = "namespace";
attribute1.FixedValue = ds.Namespace;
type.Attributes.Add(attribute1);
global::System.Xml.Schema.XmlSchemaAttribute attribute2 = new global::System.Xml.Schema.XmlSchemaAttribute();
attribute2.Name = "tableTypeName";
attribute2.FixedValue = "DataTable1DataTable";
type.Attributes.Add(attribute2);
type.Particle = sequence;
global::System.Xml.Schema.XmlSchema dsSchema = ds.GetSchemaSerializable();
if (xs.Contains(dsSchema.TargetNamespace))
{
global::System.IO.MemoryStream s1 = new global::System.IO.MemoryStream();
global::System.IO.MemoryStream s2 = new global::System.IO.MemoryStream();
try {
global::System.Xml.Schema.XmlSchema schema = null;
dsSchema.Write(s1);
for (global::System.Collections.IEnumerator schemas = xs.Schemas(dsSchema.TargetNamespace).GetEnumerator(); schemas.MoveNext();)
{
schema = ((global::System.Xml.Schema.XmlSchema)(schemas.Current));
s2.SetLength(0);
schema.Write(s2);
if ((s1.Length == s2.Length))
{
s1.Position = 0;
s2.Position = 0;
for (; ((s1.Position != s1.Length) &&
(s1.ReadByte() == s2.ReadByte()));)
{
;
}
if ((s1.Position == s1.Length))
{
return(type);
}
}
}
}
finally {
if ((s1 != null))
{
s1.Close();
}
if ((s2 != null))
{
s2.Close();
}
}
}
xs.Add(dsSchema);
return(type);
}
public TEntity Insert(TEntity entity)
{
return(DBSet.Add(entity));
}
public Guest FetchByGuestName(
string fullname)
{
return(DBSet
.Single(t => t.FullName == fullname));
}
public IList <TEntity> DeleteAll(IEnumerable <TEntity> entities)
{
return(DBSet.RemoveRange(entities).ToList());
}
void TreadTrain()
{
this.Dispatcher.Invoke(() => {
labStatus.Content = "保存训练数据中...";
});
using var db = DBSet.GetCon(DBSet.SqliteDBName.Bilibili);
var tags = db.Select <ImageTag>(o => o.Status == "completed");
var folder = Path.Combine(Utils.DefaultDataFolder, "TrainData/");
var saveResult = TagsDataSet.WriteSourceData2(tags, folder);
this.Dispatcher.Invoke(() => {
labStatus.Content = "开始加载训练数据";
});
var dataSet = TagsDataSet.ReadDataSets(folder);
var model = TFModels.GetBilibiliModelV1(dataSet.Width, dataSet.Height, dataSet.LabNames.Length);
var weightsFileName = Path.Combine(folder, "bilibili.h5");
//if (File.Exists(weightsFileName))
//{
// model.load_weights(weightsFileName);
//}
isRuning = true;
var epochs = 0;
var epochsStep = 5;
this.Dispatcher.Invoke(() => {
labStatus.Content = "开始训练";
lab_pic.Content = $"训练用图片：{saveResult.PicNum}";
lab_label.Content = string.Join("\n", saveResult.LabelCount.OrderByDescending(o => o.Value).Select(o => $"{o.Key}({o.Value}) "));
});
DateTime start = DateTime.Now;
while (isRuning)
{
model.fit(dataSet.Images, dataSet.Lables, epochs: epochsStep);
var metrices = model.metrics.ToArray();
var loss = (float)metrices[0].result();
var accuracy = (float)metrices[1].result();
epochs += epochsStep;
this.Dispatcher.Invoke(() =>
{
labStatus.Content = $"训练中: {epochs} 轮 用时:{(DateTime.Now - start)}";
labLoss.Content = $"损失率: {loss * 100}";
labAccuracy.Content = $"准确率: {accuracy * 100}";
pb.Value = accuracy * 100;
});
if (accuracy > 0.95 && loss < 0.1)
{
break;
}
}
model.save_weights(weightsFileName);
TagsDataSet.ChangeModel(model, dataSet);
isSaveWeights = true;
this.Dispatcher.Invoke(() => {
btnQuit.Content = "退出";
});
}
public IList <TResult> Select <TResult>(Expression <Func <TEntity, TResult> > selector)
{
return(DBSet.Select(selector).ToList());
}
public async Task <IReadOnlyCollection <T> > GetAllByCondition(Expression <Func <T, bool> > expression) => await DBSet.Where(expression).AsNoTracking().ToListAsync();
