public static void DownloadDocuments(string path, List <Document> fileList, NetworkCredential auth, string logFile, string docName) { Config conf = new Config(); conf.ReadConfigFile(); string MetaDataHeader = conf.ConfigList.GetValueOrDefault("MetaDataHeader"); string DocumentListCsv = conf.ConfigList.GetValueOrDefault("DocumentListCsv"); using (WebClient client = new WebClient()) { client.Credentials = auth; foreach (var item in fileList) { foreach (var document in item.Documents) { if (!File.Exists($@"{path}{document.DownloadLink.Substring(document.DownloadLink.IndexOf('=') + 1)}.{document.FileType}")) { client.DownloadFile(document.DownloadLink, $@"{path}{document.DownloadLink.Substring(document.DownloadLink.IndexOf('=')+1)}.{document.FileType}"); Logging.LogOperation("DocumentDownload", document.DownloadLink, "successful", logFile); FileOperations.WriteDocumentMetaDataCsv(DocumentListCsv, item, document, MetaDataHeader, docName, document.DownloadLink.Substring(document.DownloadLink.IndexOf('=') + 1)); } } } } }
public static string HttpRequest(string url, NetworkCredential auth, string requestMethod, string logFile) { HttpWebRequest request = (HttpWebRequest)WebRequest.CreateHttp(url); request.Method = requestMethod; request.ContentType = "application/json"; request.Credentials = auth; string response = String.Empty; try { using (StreamReader sr = new StreamReader(request.GetResponse().GetResponseStream())) { response = sr.ReadToEnd(); } } catch (WebException e) { Logging.LogOperation("Request", url, "failed" + e.Message, logFile); Console.WriteLine($"Request: {url} failed {e.Message}"); } Logging.LogOperation("Request", url, "successful", logFile); Console.WriteLine($"Request: {url} successful"); return(response); }
static void Main(string[] args) { //Read from config & set app settings //Dictionary<string, string> config = FileOperations.ReadConfig(); Config conf = new Config(); conf.ReadConfigFile(); //set up variables from config that need to be used multiple times. string rootFolderLocation = conf.ConfigList.GetValueOrDefault("ProjectDriveLocation"); string metaDataFileLocation = conf.ConfigList.GetValueOrDefault("DocumentListCsv"); string metaDataHeader = conf.ConfigList.GetValueOrDefault("MetaDataHeader"); string logFileLocation = conf.ConfigList.GetValueOrDefault("LogFileLocation"); string ProjectCheckList = conf.ConfigList.GetValueOrDefault("ProjectCheckList"); //Create log file and start logging, based on the current config context Logging.CreateLogFile(conf.ConfigList.GetValueOrDefault("LogFileLocation")); //setup network credentials NetworkCredential auth = RequestHandler.CreateCredentials(conf.ConfigList.GetValueOrDefault("User"), conf.ConfigList.GetValueOrDefault("Pwd")); //get project list from making a HTTP call to the BIW API for all project. $Top=100000 is used to get very project. var projectList = RequestHandler.ParseProjectRequest(RequestHandler.HttpRequest("https://uk-api.myconject.com/api/101/Project/?$top=100000", auth, "GET", logFileLocation)); //Uncomment the below & change the configuration file path to read from a file for the project to download. //var projectList = FileOperations.ReadProjectsFromCSV("/Volumes/Kracken/BIW_Migration/UpdatedProjects.txt"); List <string> completedProjects = null; //Obtain a list of already completed project in order to avoid redownloading. Useful for failover and dealing with the flaky BIW API if (File.Exists(ProjectCheckList)) { completedProjects = FileOperations.ReadFile(ProjectCheckList); } //write project list to disk, un comment to write the full list of projects to a CSV. Only required in the first instance of running. //FileOperations.WriteCsv(conf.ConfigList.GetValueOrDefault("ProjectListCsv"), projectList, conf.ConfigList.GetValueOrDefault("ProjectHeader"), ""); /* * The below foreahc loop is the main part of the extraction, due to the API constraints and nesting of projects, actions need to be taken sequentally * first by getting the Document Register, which responds with the list of created folders within the project, then make a call based on the project & the DocumentRegister * to ascertain if there are any documents to download, then download the projects and log to metadata & save to the file system. */ foreach (var project in projectList) { if (!completedProjects.Contains(project.id.ToString())) { //Take the project ID and make request for folders var docRegList = RequestHandler.ParseRegisterRequest(RequestHandler.HttpRequest($"https://uk-api.myconject.com/api/101/{project.id}/DocumentRegisters", auth, "GET", logFileLocation)); //create folder based on docRegList FileOperations.CreateFolderStructure($"{rootFolderLocation}/{project.id}", docRegList); //get the project docs based on the docRegList foreach (var doc in docRegList) { var documentList = RequestHandler.ParseDocumentJsonResponse(RequestHandler.HttpRequest($"https://uk-api.myconject.com/api/101/{project.id}/{doc.Id}/Documents", auth, "GET", logFileLocation)); if (documentList.Count == 0) { continue; } try { //Download each document from the doclist for the current DocumentRegister RequestHandler.DownloadDocuments($"{rootFolderLocation}/{project.id}/{doc.Name}/", documentList, auth, logFileLocation, doc.Name); } catch (Exception e) { System.Console.WriteLine(e); Logging.LogOperation("DocumentDownload", doc.ToString(), "Unsuccessful: " + e.Message, logFileLocation); } } FileOperations.OutputObj(ProjectCheckList, project.id); } } System.Console.WriteLine("fin"); }