private void AddObjects() { lock (_locker) { try { while (_current < _positions && _queue.Count < _threads * 2) { DataBacklinkRow data = _serv.GetFirstDataBacklinkRowByStatusSync(CurrentTaskStatus); if (data == null) { return; } _queue.Enqueue(data); data.Status = data.Status == Status.InAQueue ? Status.PlainHttp : Status.WebDriver; _serv.UpdateDataBacklinkRowSync(data); //task2.Start(); //task2.Wait(); _current++; Info($"{_current}"); } } catch { } } }
public async Task <int> Extract() { List <string> list = File.ReadAllLines(_settings.OpenDataString).ToList(); int i = 0; int blin = _settings.BacklinkIndex; int dlin = _settings.DomainIndex; string del = _settings.Delimeter; string dom = _settings.SingleDomname; if (dom != null) { dlin = -1; } foreach (var x in list) { try { var arr = x.Split(new string[] { del }, StringSplitOptions.None); string rw = arr[blin].Replace("\"", ""); string domname = dom == null ? arr[dlin] : dom; if (!string.IsNullOrEmpty(x) && !string.IsNullOrEmpty(rw)) { DataBacklinkRow data = new DataBacklinkRow() { Domain = domname, Url = rw, Position = i }; await _serv.CreateDataBacklinkRow(data); if (ContainsExtraInfo) { DataListedInfo info = new DataListedInfo() { DataBacklinkRowID = i, Text = Newtonsoft.Json.JsonConvert.SerializeObject(arr.Where((x, j) => j != dlin && j != blin)) }; await _infoServ.CreateDataListedInfo(info); } i++; } } catch { } } return(i); }
public async Task Perform() { List <BacklinkRow> list = new List <BacklinkRow>(); for (int i = 0; i < _settings.PlainHttpThreads; i++) { DataBacklinkRow datarow = await _dataBacklinkRowService.GetFirstDataBacklinkRowByStatus(CurrentTaskStatus); datarow.Status = Status.PlainHttp; await _dataBacklinkRowService.UpdateDataBacklinkRow(datarow); list.Add(new BacklinkRow() { DataBacklinkRow = datarow }); } _plainHttp = new PlainHttp(_settings.PlainHttpThreads, FreeThread); await _plainHttp.Perform(list); list.Clear(); CurrentTaskStatus = Status.PlainHttpNotFound; _dbQueue = new DbQueue(_settings.ConnectionString, _settings.PlainHttpThreads, _settings.Positions); for (int i = 0; i < _settings.WebDriverThreads; i++) { DataBacklinkRow datarow = await _dataBacklinkRowService.GetFirstDataBacklinkRowByStatus(CurrentTaskStatus); if (datarow == null) { _settings.WebDriverThreads = i; break; } datarow.Status = Status.WebDriver; await _dataBacklinkRowService.UpdateDataBacklinkRow(datarow); list.Add(new BacklinkRow() { DataBacklinkRow = datarow }); } _webDriver = new WebDriver(_settings.WebDriverThreads, FreeThread); await _webDriver.Perform(list); _webDriver.Quit(); }
private async Task <BacklinkRow> FreeThread(BacklinkRow row) { DataBacklinkRowService serv1 = new DataBacklinkRowService(_settings.ConnectionString); DataTagAService serv2 = new DataTagAService(_settings.ConnectionString); await serv1.UpdateDataBacklinkRow(row.DataBacklinkRow); if (row.DataTagA != null) { await serv2.CreateDataTagA(row.DataTagA); } DataBacklinkRow datarow = _dbQueue.Get(); BacklinkRow newrow = new BacklinkRow() { DataBacklinkRow = datarow }; return(newrow); }
public async Task CreateDataBacklinkRow(DataBacklinkRow data) { await _context.DataBacklinkRow.AddAsync(data); await _context.SaveChangesAsync(); }
public void UpdateDataBacklinkRowSync(DataBacklinkRow data) { _context.Update(data); _context.SaveChanges(); }
public async Task UpdateDataBacklinkRow(DataBacklinkRow data) { _context.Update(data); await _context.SaveChangesAsync(); }
public DataBacklinkRow GetFirstDataBacklinkRowByStatusSync(Status status) { DataBacklinkRow row = _context.DataBacklinkRow.FirstOrDefault(x => x.Status == status); return(row); }
public async Task <DataBacklinkRow> GetFirstDataBacklinkRowByStatus(Status status) { DataBacklinkRow row = await _context.DataBacklinkRow.FirstOrDefaultAsync(x => x.Status == status); return(row); }
public async Task <int> Extract() { int blin = _settings.BacklinkIndex; int dlin = _settings.DomainIndex; string dom = _settings.SingleDomname; if (dom != null) { dlin = -1; } SheetsService service; string sheet = "Sheet1"; string[] Scopes = { SheetsService.Scope.Spreadsheets }; string ApplicationName = "ValidateLinks"; GoogleCredential credential; using (var stream = new FileStream("client_secret.json", FileMode.Open, FileAccess.Read)) { credential = GoogleCredential.FromStream(stream) .CreateScoped(Scopes); } service = new SheetsService(new BaseClientService.Initializer() { HttpClientInitializer = credential, ApplicationName = ApplicationName, }); var range = $"{sheet}!A:G"; SpreadsheetsResource.ValuesResource.GetRequest request = service.Spreadsheets.Values.Get(_settings.OpenDataString, range); var response = request.Execute(); IList <IList <object> > values = response.Values; var dt = new DataTable(); if (values != null && values.Count > 0) { for (int i = 0; i < values.Count; i++) { IList <object> row = values[i]; List <object> f = row.AsEnumerable().ToList <object>(); for (int j = 0; j < f.Count; j++) { if (i == 0) { dt.Columns.Add(f[j].ToString()); } if (i != 0) { if (j == 0) { DataRow dr = dt.NewRow(); dr[j] = f[j]; dt.Rows.Add(dr); } else { try { DataRow dr = dt.Rows[i - 1]; dr[j] = f[j]; } catch { break; } } } } } } else { MessageBox.Show("Error..."); } dt.TableName = "links"; int len = dt.Rows.Count; int clen = dt.Columns.Count; int counter = 0; for (int i = 0; i < len; i++) { if (!string.IsNullOrEmpty(dt.Rows[i][dlin].ToString()) && !string.IsNullOrEmpty(dt.Rows[i][blin].ToString()) && dt.Rows[i][blin].ToString() != "n/a") { var arr = dt.Rows[i].ItemArray; DataBacklinkRow data = new DataBacklinkRow() { Domain = dt.Rows[i][blin].ToString(), Url = dt.Rows[i][dlin].ToString(), Position = counter }; DataListedInfo info = new DataListedInfo() { DataBacklinkRowID = counter, Text = Newtonsoft.Json.JsonConvert.SerializeObject(arr.Where((x, j) => j != dlin && j != blin)) }; await _infoServ.CreateDataListedInfo(info); await _serv.CreateDataBacklinkRow(data); counter++; } } service.Dispose(); return(counter); }