static void Main(string[] args)
{
// login
Console.WriteLine("Logging in...");
var client = new CookieAwareWebClient();
client.BaseAddress = @"https://www.profootballfocus.com/amember/";
var loginData = new NameValueCollection();
loginData.Add("amember_login", ConfigurationManager.AppSettings["user"]);
loginData.Add("amember_pass", ConfigurationManager.AppSettings["pass"]);
client.UploadValues("member", "POST", loginData);
// download page
Console.WriteLine("Downloading Rodgers W1 Passing Page");
string url = @"https://www.profootballfocus.com/data/gstats.php?tab=by_week&season=2014&gameid=3217&teamid=1&stats=p&playerid=";
string localPath = @"C:\Users\Ian\Documents\PFF\STL@ARI - QB.html";
client.DownloadFile(url, localPath);
// Count dropdown
// close
Console.WriteLine("\nPress enter to exit");
Console.Read();
}
public void DownLoadImage(string url, string path)
{
CookieAwareWebClient mywebclient = new CookieAwareWebClient(cookie);
mywebclient.Timeout = 20000;
mywebclient.DownloadFile(url, path);
}
public void DownloadSub(string path, string downloadUrl)
{
var subdivxClient = new CookieAwareWebClient(this.loginCookies);
subdivxClient.Headers["User-Agent"] = "Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))";
var directory = Path.GetDirectoryName(path);
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
subdivxClient.DownloadFile(downloadUrl, path);
}
public static FileInfo DownloadFile(string url, string path)
{
try
{
using (var cookieAwareWebClient = new CookieAwareWebClient())
{
cookieAwareWebClient.DownloadFile(url, path);
return(new FileInfo(path));
}
}
catch (WebException ex)
{
return(null);
}
}
void DownloadFileInternal()
{
if (!asyncDownload)
{
webClient.DownloadFile(downloadAddress, downloadPath);
DownloadFileCompletedCallback(webClient, new AsyncCompletedEventArgs(null, false, null));
}
else if (userToken == null)
{
webClient.DownloadFileAsync(downloadAddress, downloadPath);
}
else
{
webClient.DownloadFileAsync(downloadAddress, downloadPath, userToken);
}
}
public static string Download(string id, string fileName = null)
{
var downloadLink = $"https://drive.google.com/u/0/uc?id={id}&export=download";
try
{
var wc = new CookieAwareWebClient();
var response = wc.DownloadString(downloadLink);
var htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(response);
if (fileName == null)
{
var filenameNode = htmlDoc.GetElementsByClassName("uc-name-size").FirstOrDefault();
fileName = filenameNode != null ? filenameNode.FirstChild.InnerText : Path.GetFileName(Path.GetTempFileName());
}
if (!Directory.Exists(AppState.TempDirectory))
{
Directory.CreateDirectory(AppState.TempDirectory);
}
var downloadPath = Path.Combine(AppState.TempDirectory, fileName);
var cookies = wc.CookieContainer.List();
foreach (Cookie cookie in cookies)
{
if (!cookie.Name.StartsWith("download_warning"))
{
continue;
}
downloadLink += $"&confirm={cookie.Value}";
break;
}
wc.DownloadFile($"{downloadLink}", downloadPath);
return(downloadPath);
}
catch (Exception ex)
{
MessageBox.Show($"{ex.Message}\r\n{ex.InnerException}");
return(null);
}
}
private void DownloadFileInternal()
{
if (!asyncDownload)
{
webClient.DownloadFile(downloadAddress, downloadPath);
// This callback isn't triggered for synchronous downloads, manually trigger it
DownloadFileCompletedCallback(webClient, new AsyncCompletedEventArgs(null, false, null));
}
else if (userToken == null)
{
webClient.DownloadFileAsync(downloadAddress, downloadPath);
}
else
{
webClient.DownloadFileAsync(downloadAddress, downloadPath, userToken);
}
}
public override IEnumerable <IFreeDocument> Execute(IEnumerable <IFreeDocument> documents)
{
foreach (var document in documents)
{
var path = document.Query(SavePath);
var directoryInfo = new DirectoryInfo(path);
var folder = directoryInfo.Parent;
if (folder == null)
{
continue;
}
if (!folder.Exists)
{
folder.Create();
}
var url = document[Column].ToString();
if (string.IsNullOrEmpty(url))
{
continue;
}
try
{
CookieAwareWebClient webClient = new CookieAwareWebClient();
if (!IsAsync)
{
webClient.DownloadFile(url, path);
}
else
{
webClient.DownloadFileAsync(new Uri(url), path);
}
//HttpStatusCode code;
//var bytes = helper.GetFile(crawler.Http, out code, url);
//if (bytes != null)
// File.WriteAllBytes(path, bytes);
}
catch (Exception ex)
{
XLogSys.Print.Error(ex);
}
yield return(document);
}
}
public void SaveLastConsole(
string name,
string path)
{
Login();
// http://129.59.105.75:8080/job/[jobName]/lastBuild/consoleText
if (JobExists(name))
{
try
{
// TODO: HEAD it and set Timeout appropriately
using (CookieAwareWebClient webClient = new CookieAwareWebClient())
{
webClient.cookies = AuthCookies;
webClient.DownloadFile(ServerUrl + String.Format(Queries.LAST_BUILD_CONSOLE_TEXT, HttpUtility.UrlEncode(name)), path);
}
}
catch (WebException ex)
{
Console.WriteLine(ex.ToString());
}
}
}
public override IEnumerable<IFreeDocument> Execute(IEnumerable<IFreeDocument> documents)
{
foreach (var document in documents)
{
var path = document.Query(SavePath);
var directoryInfo = new DirectoryInfo(path);
var folder = directoryInfo.Parent;
if (folder == null)
continue;
if (!folder.Exists)
{
folder.Create();
}
var url = document[Column].ToString();
if (string.IsNullOrEmpty(url))
continue;
try
{
CookieAwareWebClient webClient = new CookieAwareWebClient();
if (!IsAsync)
{
webClient.DownloadFile(url,path);
}
else
{
webClient.DownloadFileAsync(new Uri( url), path);
}
//HttpStatusCode code;
//var bytes = helper.GetFile(crawler.Http, out code, url);
//if (bytes != null)
// File.WriteAllBytes(path, bytes);
}
catch (Exception ex)
{
XLogSys.Print.Error(ex);
}
yield return document;
}
}
public override IEnumerable <IFreeDocument> Execute(IEnumerable <IFreeDocument> documents)
{
foreach (var document in documents)
{
var path = document.Query(SavePath);
var directoryInfo = new DirectoryInfo(path);
var isdir = IsDir(path);
var url = document[Column]?.ToString();
if (string.IsNullOrEmpty(url))
{
yield return(document);
continue;
}
DirectoryInfo folder = null;
if (!isdir)
{
folder = directoryInfo.Parent;
}
else
{
folder = directoryInfo;
}
if (folder == null)
{
yield return(document);
continue;
}
if (!folder.Exists)
{
folder.Create();
}
if (isdir)
{
path = folder.ToString();
if (path.EndsWith("/") == false)
{
path += "/";
}
path += url;
path = getFileName(path);
//path += getFileName(url);
}
if (File.Exists(path))
{
yield return(document);
continue;
}
try
{
var webClient = new CookieAwareWebClient();
if (!IsAsync)
{
webClient.DownloadFile(url, path);
}
else
{
webClient.DownloadFileAsync(new Uri(url), path);
}
//HttpStatusCode code;
//var bytes = helper.GetFile(crawler.Http, out code, url);
//if (bytes != null)
// File.WriteAllBytes(path, bytes);
}
catch (Exception ex)
{
XLogSys.Print.Error(ex);
}
yield return(document);
}
}
static void Main(string[] args)
{
Console.WriteLine("Starting.");
string ConfigFile = "Default.xml";
if (args.Length > 0)
{
ConfigFile = args[0];
}
try
{
XmlSerializer xml = new XmlSerializer(typeof(DownloadConfig));
TextReader XMLInput = new StreamReader(ConfigFile);
Config = (DownloadConfig)xml.Deserialize(XMLInput);
XMLInput.Close();
XMLInput.Dispose();
}
catch
{
Console.WriteLine("Invalid Config file. Ending.");
return;
}
int Page = 0;
WebcastSummary[] result;
List<WebcastSummary> WebcastSummaries = new List<WebcastSummary>();
do
{
result = WebcastSearch(Page++);
if (result != null) WebcastSummaries.AddRange(result);
} while (result != null && result.Length == 100);
if (WebcastSummaries.Count == 0)
{
Console.WriteLine("Ending.");
return;
}
foreach (WebcastSummary _summary in WebcastSummaries)
{
string aPath = Config.UseProxyMode == true ? Config.FilePath : Config.FilePath + @"/" + _summary.Code.Replace("/", "-");
string aCode = _summary.Code.Split('/')[1];
if (!Config.UseProxyMode && Directory.Exists(aPath) && Config.ReplaceIfExists)
{
Directory.Delete(aPath, true);
}
if (!Directory.Exists(aPath) || Config.UseProxyMode)
{
try
{
if (!Directory.Exists(aPath)) Directory.CreateDirectory(aPath);
int WebcastGetResult;
bool WebcastGetResultSpecified;
Webcast Webcast;
ServiceClient.WebcastGet(
Config.Username,
Config.Password,
_summary.Code,
_summary.Languages[0],
out WebcastGetResult,
out WebcastGetResultSpecified,
out Webcast);
Console.WriteLine("Retrieved " + Webcast.Code + ".");
CookieAwareWebClient WebClient = new CookieAwareWebClient();
WebClient.Headers.Add("Content-Type", "application/x-www-form-urlencoded");
try
{
WebClient.UploadString(Webcast.RegisterUrl, "POST", "Username="******"&Password="******"/bb/") > 0)
bbExists = true;
}
foreach (Attachment _attachment in Webcast.Attachments)
{
if ((_attachment.Location.EndsWith(".wmv") && (Config.RetrieveWebcastStreams || Config.UseProxyMode) &&
(_attachment.Location.IndexOf("/bb/") > 0
|| _attachment.Location.IndexOf("/au/") > 0
|| (!bbExists && _attachment.Location.IndexOf("/nb/") > 0) ))
|| (!_attachment.Location.EndsWith(".wmv") && Config.RetrieveWebcastAttachments && !Config.UseProxyMode))
{
try
{
string fileName = Config.UseProxyMode ? aCode + ".wmv" : _attachment.Location.Substring(_attachment.Location.LastIndexOf("/") + 1);
Console.WriteLine("Now downloading " + fileName + ".");
WebClient.DownloadFile(_attachment.Location, aPath + @"/" + fileName);
}
catch
{
Console.WriteLine("Failed.");
}
}
}
}
if (!Config.UseProxyMode && Config.RetrieveTopicAttachments)
{
foreach (Topic _topic in Webcast.Topics)
{
foreach (Attachment _attachment in _topic.Attachments)
{
try
{
Console.WriteLine("Now downloading " + _attachment.Location.Substring(_attachment.Location.LastIndexOf("/") + 1) + ".");
WebClient.DownloadFile(_attachment.Location, aPath + @"/" + _attachment.Location.Substring(_attachment.Location.LastIndexOf("/") + 1));
}
catch
{
Console.WriteLine("Failed.");
}
}
}
}
if (!Config.UseProxyMode && Config.RetrieveSlideAttachments)
{
foreach (Slide _slide in Webcast.Slides)
{
foreach (Attachment _attachment in _slide.Attachments)
{
try
{
string fileName = _attachment.Location.Substring(_attachment.Location.LastIndexOf("/", _attachment.Location.LastIndexOf("/") - 1) + 1).Replace("/","-");
Console.WriteLine("Now downloading " + fileName + ".");
WebClient.DownloadFile(_attachment.Location, aPath + @"/" + fileName);
}
catch
{
Console.WriteLine("Failed.");
}
}
}
}
}
catch
{
Console.WriteLine("Authorization failed, skipping this Webcast.");
}
if (!Config.UseProxyMode && Config.ExportWebcastXML)
{
try
{
Console.WriteLine("Writing XML.");
XmlSerializer x = new XmlSerializer(Webcast.GetType());
TextWriter XMLDestination = new StreamWriter(aPath + @"/webcast.xml");
x.Serialize(XMLDestination, Webcast);
XMLDestination.Close();
XMLDestination.Dispose();
}
catch
{
Console.WriteLine("Failed.");
}
}
WebClient.Dispose();
}
catch
{
Console.WriteLine("Unable to retrieve " + _summary.Code + ".");
}
}
}
}
private void button1_Click(object sender, EventArgs e)
{
string autoIncrementIndexOfData = cntAutoInc.ToString();
string password = "";
if (GlobalClass.TestSecretCloud() == true)
{
password = GlobalClass.ReadSecretCloud(); //Sha256
}
string InputString = "";
//******************* INITIATE PHPSESSION ***************************
NameValueCollection formData = new NameValueCollection();
CookieAwareWebClient webClient = new CookieAwareWebClient();
webClient.Encoding = System.Text.Encoding.Default;
formData.Clear();
formData["username"] = "******";
if (GlobalClass.TestPasswordCloud() == true)
{
formData["password"] = GlobalClass.ReadPasswordCloud(); //user pwd
}
byte[] responseBytes = webClient.UploadValues("http://your.url.here/lo.php", "POST", formData);
string responseHTML = Encoding.UTF8.GetString(responseBytes);
Uri uriStr;
if (Uri.TryCreate("http://your.url.here", UriKind.RelativeOrAbsolute, out uriStr) == false)
{
System.Diagnostics.Debug.WriteLine("NO");
}
foreach (Cookie cookie in webClient.CookieContainer.GetCookies(uriStr))
{
System.Diagnostics.Debug.WriteLine(cookie.Name);
System.Diagnostics.Debug.WriteLine(cookie.Value);
}
//******************* INITIATE PHPSESSION ***************************
//******************* DOWNLOAD XML FILE LIST ***************************
string fileList = webClient.DownloadString("http://your.url.here/filelist.php");
DataSet ds = new DataSet();
byte[] byteArray = Encoding.UTF8.GetBytes(fileList);
ds.ReadXml(new MemoryStream(byteArray));
if (ds.Tables.Count > 0)
{
var result = ds.Tables[0];
}
dataGridView1.DataSource = ds.Tables[0];
//******************* DOWNLOAD XML FILE LIST ***************************
//******************* DOWNLOAD IV ***************************
byte[] ivArr = webClient.DownloadData("http://your.url.here/listiv.php?id=" + autoIncrementIndexOfData);
//******************* DOWNLOAD IV ***************************
//******************* DOWNLOAD LENGTH ***************************
byte[] length = webClient.DownloadData("http://your.url.here/getLen.php?id=" + autoIncrementIndexOfData);
//filesize from files
int fileSize = 0;
foreach (byte l in length)
{
fileSize += (byte)(l - (byte)(0x30));
fileSize *= 10;
}
fileSize /= 10;
System.Diagnostics.Debug.WriteLine("filesize: " + fileSize.ToString());
//******************* DOWNLOAD LENGTH ***************************
//******************* DOWNL DATA ***************************
InputString = webClient.DownloadString("http://your.url.here/dwnl.php?id=" + autoIncrementIndexOfData);
//******************* DOWNL DATA ***************************
//******************* DOWNLOAD AS FILE ***************************
webClient.DownloadFile("http://your.url.here/download.php?id=" + autoIncrementIndexOfData.ToString(), "C:\\your\\dir\\here\\akm2Raw.jpg");
//******************* DOWNLOAD AS FILE ***************************
////******************* DOWNLOAD DOWNLOAD ***************************
string InputStringDownload = webClient.DownloadString("http://your.url.here/download.php");
////******************* DOWNLOAD DOWNLOAD ***************************
//******************* DECRYPT ***************************
// Create sha256 hash
SHA256 mySHA256 = SHA256Managed.Create();
byte[] key = mySHA256.ComputeHash(Encoding.ASCII.GetBytes(password));
string decrypted = this.DecryptString(InputString, key, ivArr, fileSize);
byte[] decryptedArr = Convert.FromBase64String(decrypted);
//******************* DECRYPT ***************************
//******************* CHECK SIGN ***************************
Encoding encoding = Encoding.UTF8;
DataTable dt = ds.Tables[0];
BinaryFormatter bf = new BinaryFormatter();
using (var ms = new MemoryStream())
{
foreach (DataRow row in dt.Rows)
{
if (row["id"].ToString() == autoIncrementIndexOfData)
{
Debug.WriteLine(row["sign"].ToString());
bf.Serialize(ms, row["sign"]);
}
}
byte[] signObj = ms.ToArray();
}
using (HMACSHA256 hmac = new HMACSHA256(key))
{
var hash = hmac.ComputeHash(StringToStream(decrypted));
// Create a new Stringbuilder to collect the bytes and create a string.
StringBuilder sBuilder = new StringBuilder();
// Loop through each byte of the hashed data
// and format each one as a hexadecimal string.
for (int i = 0; i < hash.Length; i++)
{
sBuilder.Append(hash[i].ToString("x2"));
}
// Return the hexadecimal string.
Debug.WriteLine(sBuilder.ToString());
}
Debug.WriteLine("getUUID: " + getUUID());
//******************* CHECK SIGN ***************************
File.WriteAllBytes("C:\\your\\dir\\here\\akm2.jpg", decryptedArr); // Requires System.IO
}
static void UpdateDiagramasONS()
{
LogUpdate("#Início - Diagramas ONS", true);
LogUpdate("cdre.org.br/ conectando...", true);
IEnumerable <Row> diagramasONS = ScrapOnsDiagramasAsync().GetAwaiter().GetResult();
int totalItems = diagramasONS.Count();
int counter = 1;
FileInfo jsonInfoFile = new FileInfo(Path.Combine(DiagramasDir, "info.json"));
string jsonInfo = File.Exists(jsonInfoFile.FullName) ? File.ReadAllText(jsonInfoFile.FullName) : null;
IEnumerable <Row> localDiagramas = new List <Row>();
try //problema ao corromper gravação do json //parse arquivo json inválido
{
localDiagramas = JsonConvert.DeserializeObject <IEnumerable <Row> >(jsonInfo);
}
catch (Exception)
{
}
if (LocalBookmarks.Count == 0)
{
localDiagramas = new List <Row>(); //forçar update completo caso bookmark corrompido
}
bool bookmarkUpdate = false;
var client = new CookieAwareWebClient();
foreach (var diagrama in diagramasONS)
{
LogUpdate($"{diagrama.FileLeafRef} atualizando {counter++} de {totalItems}", true);
string diagramaLink = $"https://cdre.ons.org.br{diagrama.FileRef}";
FileInfo diagramaFile = new FileInfo(Path.Combine(DiagramasDir, diagrama.FileLeafRef));
string revisao = localDiagramas.Where(w => w.FileLeafRef == diagrama.FileLeafRef).Select(s => s.Modified).FirstOrDefault();
if (revisao == diagrama.Modified && diagramaFile.Exists)
{
LogUpdate($"{diagrama.FileLeafRef} já se encontra na revisão vigente");
continue;
}
try
{
client.DownloadFile(diagramaLink, diagramaFile.FullName);
bookmarkUpdate = true;
var listBookmarks = GetPdfBookmark(diagramaFile, true, diagrama.MpoAssunto);
if (LocalBookmarks.Any(w => w.MpoCodigo == diagrama.FileLeafRef))
{
foreach (var item in LocalBookmarks)
{
if (item.MpoCodigo == diagrama.FileLeafRef)
{
item.Bookmarks = listBookmarks;
}
}
}
else
{
LocalBookmarks.Add(new ModelSearchBookmark
{
MpoCodigo = diagrama.FileLeafRef,
Bookmarks = listBookmarks
});
}
LogUpdate($"{diagrama.FileLeafRef} atualizado da modificação {revisao} para modificação {diagrama.Modified} em {diagramaFile.FullName}");
}
catch (Exception)
{
LogUpdate($"{diagrama.FileLeafRef} não foi possível atualização pelo link {diagramaLink}");
}
}
if (bookmarkUpdate)
{
File.WriteAllText(jsonBookmarkFile.FullName, JsonConvert.SerializeObject(LocalBookmarks));
}
var diagramasVigentes = diagramasONS.Select(s => $"{jsonInfoFile.Directory.FullName}\\{s.FileLeafRef}");
var diagramasLocal = Directory.GetFiles(jsonInfoFile.Directory.FullName, "*.pdf").Except(diagramasVigentes);
foreach (var item in diagramasLocal)
{
File.Delete(item);
LogUpdate($"{Path.GetFileNameWithoutExtension(item)} não está vigente e foi apagado");
}
File.WriteAllText(jsonInfoFile.FullName, JsonConvert.SerializeObject(diagramasONS));
client.Dispose();
LogUpdate("#Término - Diagramas ONS", true);
}
