private void DownloadXml(CurriculoEntry curriculumVitae, MetadataResponse response, WebClient wc) { var link = String.Format(_urlDownloadXml, response.CodRhCript); Logger.Debug($"Currículo {curriculumVitae.NumeroCurriculo} marcado para download ({link})..."); var stream = wc.OpenRead(link); if (File.Exists(_lattesModule.GetCurriculumVitaeFileName(curriculumVitae.NumeroCurriculo))) { File.Delete(_lattesModule.GetCurriculumVitaeFileName(curriculumVitae.NumeroCurriculo)); } var zis = new ZipInputStream(stream); zis.GetNextEntry(); FileStream xml = new FileStream(_lattesModule.GetCurriculumVitaeFileName(curriculumVitae.NumeroCurriculo), FileMode.CreateNew); var buffer = new byte[4096]; int read; while ((read = zis.Read(buffer, 0, buffer.Length)) > 0) { xml.Write(buffer, 0, read); } xml.Close(); _curriculumVitaesForProcess.Send(curriculumVitae); }
private void UnzipAndCopy(ManualResetEvent doneEvent, string filename, CurriculoEntry curriculumVitae) { try { int read; byte[] buffer = new byte[4096]; using (var ms = UnzipCurriculumVitae(filename)) { using (FileStream wc = new FileStream(_lattesModule.GetCurriculumVitaeFileName(curriculumVitae.NumeroCurriculo), FileMode.CreateNew)) { while ((read = ms.Read(buffer, 0, buffer.Length)) > 0) { wc.Write(buffer, 0, read); } } } _channel.Send(curriculumVitae); } catch (ZipException exception) { Logger.Error($"Erro ao importar currículo {curriculumVitae.NumeroCurriculo}: {exception.Message}\n{exception.StackTrace}"); } finally { if (Interlocked.Decrement(ref _workItemCount) == 0) { doneEvent.Set(); } } }
public MemoryStream GetCurriculumVitaeIfUpdated(CurriculoEntry curriculumVitae) { Nullable <DateTime> dataAtualizacaoLattes = null; Nullable <DateTime> dataAtualizacaoSistema; string dataAtualizacaoString; if (curriculumVitae.NumeroCurriculo == null || curriculumVitae.NumeroCurriculo == "") { Logger.Info(String.Format("Buscando Número do Currículo do Professor {0}", curriculumVitae.NomeProfessor)); curriculumVitae.NumeroCurriculo = ws.getIdentificadorCNPq(curriculumVitae.CPF, curriculumVitae.NomeProfessor, curriculumVitae.DataNascimento); if (curriculumVitae.NumeroCurriculo == null || curriculumVitae.NumeroCurriculo == "" || curriculumVitae.NumeroCurriculo.Contains("ERRO")) { return(null); } Logger.Info(String.Format("Número do Currículo do Professor {0} encontrado: {1}", curriculumVitae.NomeProfessor, curriculumVitae.NumeroCurriculo)); } // verificar se a data de atualizacao do CV é maior que a do sistema dataAtualizacaoSistema = this.GetDataAtualizacaoProfessor(curriculumVitae.NumeroCurriculo); if (dataAtualizacaoSistema != null) { dataAtualizacaoString = ws.getDataAtualizacaoCV(curriculumVitae.NumeroCurriculo); if (dataAtualizacaoString == "") { dataAtualizacaoLattes = DateTime.Today; } else { dataAtualizacaoLattes = DateTime.ParseExact(dataAtualizacaoString, "dd/MM/yyyy %H:mm:ss", null); } if (dataAtualizacaoSistema >= dataAtualizacaoLattes) { return(null); // curriculo não precisa curriculumVitaeUnserializer atualizado } } if (dataAtualizacaoLattes != null) { curriculumVitae.DataUltimaAtualizacao = (DateTime)dataAtualizacaoLattes; } byte[] zip = ws.getCurriculoCompactado(curriculumVitae.NumeroCurriculo); if (zip == null || zip.Length == 0) { Logger.Error(String.Format("Aconteceu um erro ao tentar buscar o currículo de Número {0}, favor verificar o mesmo", curriculumVitae.NumeroCurriculo)); return(null); } return(ProcessarRetornoCurriculo(zip)); }
public static void LoadCurriculums(LattesModule lattesModule, string folder) { int read; byte[] buffer = new byte[4096]; MemoryStream ms; CurriculoEntry curriculumVitae; if (!Directory.Exists(folder)) { Logger.Info(String.Format("Pasta de trabalho não foi encontrado ({0})", folder)); return; } bool exists = false; foreach (string filename in Directory.EnumerateFiles(folder)) { exists = true; String numeroCurriculo = filename.Substring(folder.Length + 1); numeroCurriculo = numeroCurriculo.Substring(0, numeroCurriculo.Length - 4); curriculumVitae = new CurriculoEntry { NumeroCurriculo = numeroCurriculo, }; if (File.Exists(lattesModule.GetCurriculumVitaeFileName(curriculumVitae.NumeroCurriculo))) { File.Delete(lattesModule.GetCurriculumVitaeFileName(curriculumVitae.NumeroCurriculo)); } if (filename.EndsWith(".xml")) { File.Copy(filename, lattesModule.GetCurriculumVitaeFileName(curriculumVitae.NumeroCurriculo)); lattesModule.AddCurriculumVitaeForProcess(curriculumVitae); continue; } ms = UnzipCurriculumVitae(filename); FileStream wc = new FileStream(lattesModule.GetCurriculumVitaeFileName(curriculumVitae.NumeroCurriculo), FileMode.CreateNew); while ((read = ms.Read(buffer, 0, buffer.Length)) > 0) { wc.Write(buffer, 0, read); } ms.Close(); lattesModule.AddCurriculumVitaeForProcess(curriculumVitae); } if (exists == false) { throw new Exception(String.Format("Não foram encontrados currículos na pasta {0} !", folder)); } }
private void DownloadCurriculumVitae(CurriculoEntry curriculumVitae) { try { if (curriculumVitae.NumeroCurriculo == null || curriculumVitae.NumeroCurriculo.Trim().Length == 0) { Logger.Error($"O número do curríuculo Lattes do professor {curriculumVitae.NomeProfessor} não foi encontrado"); //gambi pra fazer funcionar quando usando csv sem num de curric //@TODO // Descobrir qual eh o bug aqui //return; } int read; byte[] buffer = new byte[4096]; MemoryStream ms = _dcvs.GetCurriculumVitaeIfUpdated(curriculumVitae); if (ms == null) { return; } if (File.Exists(_lattesModule.GetCurriculumVitaeFileName(curriculumVitae.NumeroCurriculo))) { File.Delete(_lattesModule.GetCurriculumVitaeFileName(curriculumVitae.NumeroCurriculo)); } FileStream wc = new FileStream(_lattesModule.GetCurriculumVitaeFileName(curriculumVitae.NumeroCurriculo), FileMode.CreateNew); while ((read = ms.Read(buffer, 0, buffer.Length)) > 0) { wc.Write(buffer, 0, read); } ms.Close(); _curriculumVitaesForProcess.Send(curriculumVitae); wc.Flush(); wc.Close(); if (curriculumVitae.NomeProfessor == null || curriculumVitae.NomeProfessor.Trim().Length == 0) { Logger.Info($"Curriculo {curriculumVitae.NumeroCurriculo} baixado"); return; } Logger.Info($"Curriculo {curriculumVitae.NumeroCurriculo} - {curriculumVitae.NomeProfessor} baixado"); } catch (Exception exception) { Logger.Error($"Erro ao buscar o currículo {curriculumVitae.NumeroCurriculo}, mensagem: {exception.Message}\n{exception.StackTrace}"); } }
private void AddToChannel(CurriculoEntry ce) { if ( (ce.NumeroCurriculo != null && ce.NumeroCurriculo.Length > 0) || ( ce.NomeProfessor != null && ce.NomeProfessor.Length > 0 && ce.DataNascimento != null && ce.DataNascimento.Length > 0 && ce.CPF != null && ce.CPF.Length > 0 ) ) { _channel.Send(ce); } }
public void LoadCurriculums(ManualResetEvent doneEvent) { try { if (!Directory.Exists(_importFolder)) { Logger.Info($"Pasta de trabalho não foi encontrado ({_importFolder})"); return; } if (Directory.GetFiles(_importFolder).Length == 0) { throw new Exception($"Não foram encontrados currículos na pasta {_importFolder} !"); } var unzipDoneEvent = new ManualResetEvent(false); foreach (string filename in Directory.EnumerateFiles(_importFolder)) { string numeroCurriculo = filename.Substring(_importFolder.Length + 1); numeroCurriculo = numeroCurriculo.Substring(0, numeroCurriculo.Length - 4); var curriculumVitae = new CurriculoEntry { NumeroCurriculo = numeroCurriculo }; if (File.Exists(_lattesModule.GetCurriculumVitaeFileName(curriculumVitae.NumeroCurriculo))) { File.Delete(_lattesModule.GetCurriculumVitaeFileName(curriculumVitae.NumeroCurriculo)); } if (filename.EndsWith(".xml")) { File.Copy(filename, _lattesModule.GetCurriculumVitaeFileName(curriculumVitae.NumeroCurriculo)); _channel.Send(curriculumVitae); continue; } Interlocked.Increment(ref _workItemCount); ThreadPool.QueueUserWorkItem(o => UnzipAndCopy(unzipDoneEvent, filename, curriculumVitae)); } if (_workItemCount > 0) { unzipDoneEvent.WaitOne(); } } finally { doneEvent.Set(); } }
private void ProcessCurriculumVitae(CurriculoEntry curriculoEntry, ManualResetEvent doneEvent) { XmlDocument curriculumVitaeXml = new XmlDocument(); try { var filename = _lattesModule.GetCurriculumVitaeFileName(curriculoEntry.NumeroCurriculo); curriculumVitaeXml.Load(filename); // nescessário para o deserialize reconhecer o Xml curriculumVitaeXml.DocumentElement.SetAttribute("xmlns", "http://tempuri.org/LMPLCurriculo"); XDocument curriculumVitaeXDocument = XDocument.Parse(curriculumVitaeXml.InnerXml); CurriculoVitaeXml curriculumVitae = _curriculumVitaeUnserializer.Deserialize(curriculumVitaeXDocument.CreateReader()) as CurriculoVitaeXml; curriculoEntry.NomeProfessor = curriculumVitae.DADOSGERAIS.NOMECOMPLETO; ProfessorDAOService professorDAOService = new ProfessorDAOService(new LattesDatabase()); Logger.Debug($"Iniciando processamento currículo {curriculoEntry.NumeroCurriculo} do Professor {curriculumVitae.DADOSGERAIS.NOMECOMPLETO}..."); if (professorDAOService.ProcessCurriculumVitaeXML(curriculumVitae, curriculoEntry)) { Logger.Info($"Currículo {curriculoEntry.NumeroCurriculo} do Professor {curriculumVitae.DADOSGERAIS.NOMECOMPLETO} processado com sucesso !"); File.Delete(filename); } } catch (Exception ex) { Logger.Error($"Erro durante a leitura do XML {curriculoEntry.NumeroCurriculo}: {ex.Message}\n{ex.StackTrace}"); int sequencia = 1; while (ex.InnerException != null) { ex = ex.InnerException; Logger.Error($"Excessão Interna [{sequencia++}]: {ex.Message}\n{ex.StackTrace}"); } } finally { if (Interlocked.Decrement(ref _workItemCount) == 0) { doneEvent.Set(); } // Allow another add to the queue WorkLimiter.Release(); } }
private bool NeedsToBeUpdated(CurriculoEntry curriculumVitae, MetadataResponse response) { if (response.Document.DataAtualizacao == null || response.Document.DataAtualizacao.Length == 0) { return(true); } var horaAtualizacao = response.Document.HoraAtualizacao; if (horaAtualizacao == null || horaAtualizacao.Length == 0) { horaAtualizacao = "000000"; } var dataAtualizacaoLattes = DateTime.ParseExact( $"{response.Document.DataAtualizacao} {horaAtualizacao}", "ddMMyyyy %Hmmss", null ); if (dataAtualizacaoLattes == null) { return(true); } var dataAtualizacaoCorrente = GetDataAtualizacaoProfessor(curriculumVitae.NumeroCurriculo); if (dataAtualizacaoCorrente == null) { return(true); } if (dataAtualizacaoCorrente < dataAtualizacaoLattes) { return(true); } return(false); }
/// <summary> /// Adiciona um curriculo para curriculumVitaeUnserializer processado /// </summary> /// <param name="curriculumVitaeNumber"></param> public void AddCurriculumVitaeForProcess(CurriculoEntry curriculumVitaeNumber) { _curriculumVitaesForProcess.Push(curriculumVitaeNumber); }
/// <summary> /// Adiciona um curriculo para curriculumVitaeUnserializer verificado para download /// </summary> /// <param name="curriculumVitaeNumber"></param> public void AddCurriculumVitaeNumberToDownload(CurriculoEntry curriculumVitaeNumber) { _curriculumVitaeNumbersToDownload.Push(curriculumVitaeNumber); }
internal static void LoadCurriculumVitaeNumbers(LattesModule lattesModule) { // ler do banco de dados do RM para pegar os profissionais // Criando adaptador que busca todos os registros da planilha DataTable dataTable = new DataTable(); using (OleDbDataAdapter adapter = new OleDbDataAdapter(lattesModule.LattesCurriculumVitaeQuery, lattesModule.LattesCurriculumVitaeODBCConnection)) { CurriculoEntry ce; adapter.Fill(dataTable); foreach (DataRow row in dataTable.Rows) { ce = null; if (dataTable.Columns.Count == 1) { ce = new CurriculoEntry() { NumeroCurriculo = row[dataTable.Columns[0]].ToString().Trim(), }; } else { if (dataTable.Columns.Count == 2) { ce = new CurriculoEntry() { NumeroCurriculo = row[dataTable.Columns[0]].ToString().Trim(), NomeProfessor = row[dataTable.Columns[1]].ToString().Trim(), }; } else { if (dataTable.Columns.Count == 3) { ce = new CurriculoEntry() { NomeProfessor = row[dataTable.Columns[0]].ToString().Trim(), DataNascimento = row[dataTable.Columns[1]].ToString().Trim(), CPF = row[dataTable.Columns[2]].ToString().Trim(), NumeroCurriculo = "", }; } else { if (dataTable.Columns.Count == 4) { ce = new CurriculoEntry() { NumeroCurriculo = row[dataTable.Columns[0]].ToString().Trim(), NomeProfessor = row[dataTable.Columns[1]].ToString().Trim(), DataNascimento = row[dataTable.Columns[2]].ToString().Trim(), CPF = row[dataTable.Columns[3]].ToString().Trim(), }; } } } } if (ce != null && ((ce.NumeroCurriculo != null && ce.NumeroCurriculo.Length > 0) || (ce.NomeProfessor != null && ce.NomeProfessor.Length > 0 && ce.DataNascimento != null && ce.DataNascimento.Length > 0) && ce.CPF != null && ce.CPF.Length > 0)) { lattesModule.AddCurriculumVitaeNumberToDownload(ce); } } } }