private static void ThreadStart()
{
int maxIteration = 1;
int count = 1;
while (count <= maxIteration)
{
using (Domain.ApplicationContext context = new Domain.ApplicationContext())
{
Good good = context.Goods.FirstOrDefault(g => !g.IsBusy && !String.IsNullOrEmpty(g.ProductDetailUrl));
good.IsBusy = true;
context.SaveChanges();
try
{
Service.Web.Methods.PopulateGoodBrowser(context, good);
}
//catch (Exception ex) { }
finally
{
good.IsBusy = false;
context.SaveChanges();
}
}
count++;
}
}
private void simpleButton11_Click(object sender, EventArgs e)
{
using (Domain.ApplicationContext context = new Domain.ApplicationContext())
{
Mouser.Service.Web.Methods.PopulateCategory(context);
}
}
private async void simpleButton5_Click(object sender, EventArgs e)
{
using (Domain.ApplicationContext context = new Domain.ApplicationContext())
{
List <Manufacturer> manufacturers = await context.Manufacturers.Where(m => m.MouserID != 0 && m.SearchText == null).ToListAsync();
ApiRegInfo apiRegInfo = await context.ApiRegInfos.FirstOrDefaultAsync(a => a.IsActive);
Proxy proxy = await context.Proxies.FirstOrDefaultAsync(p => p.IsActive);
string[] alphabet = { ".", "1", "2", "3", "4", "5", "6", "7", "8", "9" };
foreach (Manufacturer manufacturer in manufacturers)
{
foreach (string letter in alphabet)
{
this.labelControl4.Text = "[" + letter + "] - " + manufacturer.MouserID + " - " + manufacturer.NameAPI;
this.Refresh();
Thread.Sleep(1500);
await Mouser.Service.Api.Methods.SearchByKeywordMfrRequestSetResulCountAsync(context, proxy, apiRegInfo, letter, manufacturer);
}
}
}
MessageBox.Show("Done!");
}
private async void simpleButton12_Click(object sender, EventArgs e)
{
using (Domain.ApplicationContext context = new Domain.ApplicationContext())
{
String folderPath = @"D:\MouserExport";
Directory.CreateDirectory(folderPath);
var manufacturers = await context.Manufacturers.Where(m => m.MouserID != 0).ToListAsync();
foreach (var m in manufacturers)
{
await ExportToJSONAsync(folderPath, m.Id);
}
}
}
private async Task SaveJsonFileAsync(string path, int manufacturerId, Manufacturer manufacturer, Category сategory)
{
int goodsCount = 0;
using (Domain.ApplicationContext context = new Domain.ApplicationContext())
{
context.Database.CommandTimeout = 600;
goodsCount = await context.Goods.Where(p => p.Manufacturer.Id == manufacturerId && p.Category.Id == сategory.Id && p.MouserPartNumber != "N/A").CountAsync();
}
int count = 1;
int pageCount = 50000;
int loopIteration = goodsCount / pageCount + 1;
while (count <= loopIteration)
{
using (Domain.ApplicationContext context = new Domain.ApplicationContext())
{
context.Database.CommandTimeout = 600;
List <Good> goods = await context.Goods.OrderBy(g => g.Id)
.Include(p => p.AlternatePackagings)
.Include(p => p.PriceBreaks)
.Include(p => p.ProductAttributes)
.Include(p => p.ProductCompliances)
.Where(p => p.Manufacturer.Id == manufacturerId && p.Category.Id == сategory.Id && p.MouserPartNumber != "N/A")
.Skip(pageCount * (count - 1))
.Take(pageCount)
.ToListAsync();
if (goods.Count == 0)
{
return;
}
JsonSerializer serializer = new JsonSerializer();
string currentPath = loopIteration == 1 ? path + ".json" : path + "_" + count + ".json";
using (StreamWriter sw = new StreamWriter(currentPath))
using (JsonWriter writer = new JsonTextWriter(sw))
{
serializer.Serialize(writer, goods);
}
count++;
}
}
}
private async void timer3_Tick(object sender, EventArgs e)
{
if (countTimer3Iteration != 0)
{
labelControl6.Text = "В работе:" + countTimer3Iteration;
}
if (countTimer3Iteration > 10)
{
return;
}
using (Domain.ApplicationContext context = new Domain.ApplicationContext())
{
//SystemSetting systemSetting = context.SystemSettings.FirstOrDefault();
Good good = context.Goods.OrderBy(g => g.Manufacturer.Id).FirstOrDefault(g => !g.IsBusy && !g.IsWebDownloaded);
try
{
if (good != null)
{
countTimer3Iteration++;
good.IsBusy = true;
context.SaveChanges();
//if (systemSetting == null)
//{
// context.SystemSettings.Add(new SystemSetting { ApiScrapperCountRequests = 1 });
//}
//else
//{
// systemSetting.ApiScrapperCountRequests++;
//}
await Service.Api.Methods.GetFromWebAsync(context, good);
}
}
finally
{
countTimer3Iteration--;
good.IsBusy = false;
await context.SaveChangesAsync();
}
}
}
private async Task ExportToJSONAsync(string folderPath, int manufacturerId)
{
using (Domain.ApplicationContext context = new Domain.ApplicationContext())
{
Manufacturer manufacturer = await context.Manufacturers.FindAsync(manufacturerId);
Directory.CreateDirectory(folderPath + "\\" + string.Join("_", manufacturer.Name.Split(Path.GetInvalidFileNameChars())));
List <Category> categories = await context.Categories.Where(c => c.Manufacturer.Id == manufacturerId && c.IsCategory).ToListAsync();
foreach (var category in categories.OrderBy(c => c.Name))
{
Directory.CreateDirectory(folderPath + "\\" + string.Join("_", manufacturer.Name.Split(Path.GetInvalidFileNameChars())) +
"\\" + string.Join("_", category.Name.Split(Path.GetInvalidFileNameChars())));
List <Category> subCategories = await context.Categories.Where(c => c.ParentId == category.Id && !c.IsCategory).ToListAsync();
foreach (var subCategory in subCategories)
{
await SaveJsonFileAsync(
folderPath + "\\" + string.Join("_", manufacturer.Name.Split(Path.GetInvalidFileNameChars())) +
"\\" + string.Join("_", category.Name.Split(Path.GetInvalidFileNameChars())) +
"\\" + string.Join("_", subCategory.Name.Split(Path.GetInvalidFileNameChars())),
manufacturerId,
manufacturer,
subCategory);
}
}
List <Category> lostCategories = await context.Categories
.Where(c => c.ParentId == 0 && c.Manufacturer.Id == manufacturerId && !c.IsCategory)
.ToListAsync();
foreach (var lostCategory in lostCategories.OrderBy(c => c.Name))
{
await SaveJsonFileAsync(
folderPath + "\\" + string.Join("_", manufacturer.Name.Split(Path.GetInvalidFileNameChars())) +
"\\" + string.Join("_", lostCategory.Name.Split(Path.GetInvalidFileNameChars())),
manufacturerId,
manufacturer,
lostCategory);
}
}
}
private void SaveJsonFile(string path, int manufacturerId, Manufacturer manufacturer, Category сategory)
{
using (Domain.ApplicationContext context = new Domain.ApplicationContext())
{
List <Good> goods = context.Goods
.Include(p => p.AlternatePackagings)
.Include(p => p.PriceBreaks)
.Include(p => p.ProductAttributes)
.Include(p => p.ProductCompliances)
.Where(p => p.Manufacturer.Id == manufacturerId && p.Category.Id == сategory.Id && p.MouserPartNumber != "N/A")
.ToList();
JsonSerializer serializer = new JsonSerializer();
using (StreamWriter sw = new StreamWriter(path))
using (JsonWriter writer = new JsonTextWriter(sw))
{
serializer.Serialize(writer, goods);
}
}
}
private async Task StartProcessAsync()
{
int maxIteration = 10;
using (Domain.ApplicationContext context = new Domain.ApplicationContext())
using (StreamWriter sw = new StreamWriter(writePath, true, System.Text.Encoding.UTF8))
{
sw.WriteLine(DateTime.Now.ToString() + " Start");
ApiSearchSession apiSearchSession = null;
EFApiSearchSession eFApiSearchSession = new EFApiSearchSession(context);
try
{
//Получаем PartnerID
EFApiRegInfo eFApiRegInfo = new EFApiRegInfo(context);
ApiRegInfo availableApiRegInfo = await eFApiRegInfo.GetAvailableParnterIdAsync();
sw.WriteLine(availableApiRegInfo.PartnerId);
//Получаем прокси
EFProxy eFProxy = new EFProxy(context);
Proxy availableProxy = eFProxy.GetAvailableProxyAsync().Result;
sw.WriteLine(availableProxy.IPAddress);
//Получаем свободного производителя
EFManufacturer eFManufacturer = new EFManufacturer(context);
Manufacturer availableManufacturer = eFManufacturer.GetAvailableManufacturerAsync().Result;
sw.WriteLine(availableManufacturer.MouserID);
//if (availableApiRegInfo != null && availableProxy != null && availableManufacturer != null)
//{
//Создаем очередь
apiSearchSession = new ApiSearchSession
{
ApiRegInfo = availableApiRegInfo,
Date = DateTime.Today,
IsBusy = true,
Manufacturer = availableManufacturer,
Proxy = availableProxy
};
eFApiSearchSession.AddOrUpdateAsync(apiSearchSession, -1).Wait();
//Запускаем запросы api
int count = 1;
while (count <= maxIteration)
{
count++;
}
eFApiSearchSession.SetNotBusyAsync(apiSearchSession).Wait();
System.Environment.Exit(1);
//}
}
catch (Exception ex)
{
sw.WriteLine(DateTime.Now.ToString() + " " + ex.Message + ex.InnerException.Message);
}
finally
{
//Очередь освобождаем
if (apiSearchSession != null)
{
eFApiSearchSession.SetNotBusyAsync(apiSearchSession).Wait();
}
System.Environment.Exit(1);
}
}
}
//private static string writePath = @"D:\MouserClient\MouserClientApi" + Guid.NewGuid() + ".log";
static async Task Main(string[] args)
{
ApiSearchSession apiSearchSession = null;
try
{
int maxIteration = 5;
using (Domain.ApplicationContext context = new Domain.ApplicationContext())
//using (StreamWriter sw = new StreamWriter(writePath, true, System.Text.Encoding.UTF8))
{
//sw.WriteLine(DateTime.Now.ToString() + " Start");
EFApiSearchSession eFApiSearchSession = new EFApiSearchSession(context);
//Получаем PartnerID
EFApiRegInfo eFApiRegInfo = new EFApiRegInfo(context);
ApiRegInfo availableApiRegInfo = await eFApiRegInfo.GetAvailableParnterIdAsync();
//sw.WriteLine(availableApiRegInfo?.PartnerId);
try
{
//Получаем прокси
EFProxy eFProxy = new EFProxy(context);
Proxy availableProxy = await eFProxy.GetAvailableProxyAsync();
//sw.WriteLine(availableProxy?.IPAddress);
//Получаем свободного производителя
EFManufacturer eFManufacturer = new EFManufacturer(context);
Manufacturer availableManufacturer = await eFManufacturer.GetAvailableManufacturerAsync();
//sw.WriteLine(availableManufacturer?.MouserID);
if (availableApiRegInfo != null && availableProxy != null && availableManufacturer != null &&
!context.ApiSearchSessions.Where(a => a.IsBusy && a.Date == DateTime.Today).Select(a => a.Manufacturer.Id).Contains(availableManufacturer.Id))
{
//Создаем очередь
apiSearchSession = new ApiSearchSession
{
ApiRegInfo = availableApiRegInfo,
Date = DateTime.Today,
IsBusy = true,
Manufacturer = availableManufacturer,
Proxy = availableProxy,
MachineName = Environment.MachineName,
CreateDate = DateTime.Now
};
await eFApiSearchSession.AddOrUpdateAsync(apiSearchSession, -1);
//Запускаем запросы api
int count = 1;
int countIteration = availableManufacturer.StartingRecord != 0 ? (availableManufacturer.StartingRecord - 1) / 50 + 1 : 0;
int startingRecord = 0;
while (count <= maxIteration && (startingRecord == 0 || startingRecord < availableManufacturer.NumberOfResult))
{
startingRecord = countIteration != 0 ? countIteration * 50 + 1 : 0;
await Mouser.Service.Api.Methods.SearchByKeywordMfrRequestAsync
(context, availableProxy, availableApiRegInfo, availableManufacturer.SearchText ?? ".", availableManufacturer, 50, startingRecord);
await eFApiSearchSession.AddIterationAsync(apiSearchSession);
count++;
countIteration++;
}
await eFApiSearchSession.SetNotBusyAsync(apiSearchSession);
//sw.WriteLine(DateTime.Now.ToString() + " Stop");
}
}
catch (Exception ex)
{
if (ex.Message.Contains("Error Code: 429") || ex.Message.Contains("Invalid unique identifier"))
{
availableApiRegInfo.IsActive = false;
await eFApiRegInfo.AddOrUpdateAsync(availableApiRegInfo, availableApiRegInfo.Id);
}
if (apiSearchSession != null)
{
apiSearchSession.Description = ex.Message + ex.InnerException?.Message;
await eFApiSearchSession.AddOrUpdateAsync(apiSearchSession, apiSearchSession.Id);
await eFApiSearchSession.SetNotBusyAsync(apiSearchSession);
}
//using (StreamWriter sw = new StreamWriter(writePath, true, System.Text.Encoding.UTF8))
//{
// sw.WriteLine(DateTime.Now.ToString() + " " + ex.Message + ex.InnerException?.Message);
// sw.WriteLine(DateTime.Now.ToString() + " Error Stop (2)");
//}
}
finally
{
//Очередь освобождаем
if (apiSearchSession != null)
{
await eFApiSearchSession.SetNotBusyAsync(apiSearchSession);
}
}
}
}
catch (Exception ex)
{
//using (StreamWriter sw = new StreamWriter(writePath, true, System.Text.Encoding.UTF8))
//{
// sw.WriteLine(DateTime.Now.ToString() + " " + ex.Message + ex.InnerException?.Message);
// sw.WriteLine(DateTime.Now.ToString() + " Error Stop (2)");
//}
}
}