public void AddURL(FilesItem metadataFile) { if (Program.cfgCurrent.UseHead) { this.filesToReadSizeQueue.Enqueue(metadataFile); } }
private void wsSearch_SearcherLinkFoundEvent(object sender, EventsThreads.CollectionFound <Uri> e) { foreach (Uri url in e.Data) { try { try { string fileExtension = Path.GetExtension(url.AbsolutePath).ToLowerInvariant(); if (!String.IsNullOrWhiteSpace(fileExtension) && DocumentExtractor.IsSupportedExtension(fileExtension)) { var fi = new FilesItem { Ext = fileExtension, URL = url.ToString(), Downloaded = false, MetadataExtracted = false, Date = DateTime.MinValue, ModifiedDate = DateTime.MinValue, Path = string.Empty, Size = -1 }; Program.data.files.Items.Add(fi); Program.FormMainInstance.treeViewMetadata_UpdateDocumentsNumber(); Program.FormMainInstance.panelMetadataSearch.listViewDocuments_Update(fi); Program.FormMainInstance.panelMetadataSearch.HttpSizeDaemonInst.AddURL(fi); } } catch (Exception) { } // add the url to the files list DomainsItem domain = Program.data.GetDomain(url.Host); if (domain == null) { Program.data.AddDomain(url.Host, "Crawling", Program.cfgCurrent.MaxRecursion, Program.cfgCurrent); Program.LogThis(new Log(Log.ModuleType.Crawling, "Domain found: " + url.Host, Log.LogType.medium)); domain = Program.data.GetDomain(url.Host); } domain.map.AddUrl(url.ToString()); if (domain.techAnalysis.domain == null) { domain.techAnalysis.domain = domain.Domain; } domain.techAnalysis.eventLinkFoundDetailed(null, new EventsThreads.CollectionFound <Uri>(new List <Uri> { url })); } catch { } } }
private void wsSearch_SearcherLinkFoundEvent(object sender, EventsThreads.CollectionFound <Uri> e) { foreach (Uri url in e.Data) { try { try { var fileWithMetadata = Program.FormMainInstance.panelMetadataSearch.checkedListBoxExtensions.Items.Cast <string>() .Any(checkedListbox => url.ToString().EndsWith(checkedListbox)); if (fileWithMetadata) { var fi = new FilesItem { Ext = Path.GetExtension(url.AbsolutePath).ToLower(), URL = url.ToString(), Downloaded = false, Processed = false, Date = DateTime.MinValue, ModifiedDate = DateTime.MinValue, Path = string.Empty, Size = -1 }; Program.data.files.Items.Add(fi); Program.FormMainInstance.treeViewMetadata_UpdateDocumentsNumber(); Program.FormMainInstance.panelMetadataSearch.listViewDocuments_Update(fi); Program.FormMainInstance.panelMetadataSearch.HttpSizeDaemonInst.AddURL(fi); } } catch (Exception) { } // add the url to the files list var d = Program.data.GetDomain(url.Host); d.map.AddUrl(url.ToString()); if (d.techAnalysis.domain == null) { d.techAnalysis.domain = d.Domain; } var listaUrl = new List <Uri> { url }; d.techAnalysis.eventLinkFoundDetailed(null, new EventsThreads.CollectionFound <Uri>(listaUrl)); } catch { } } }
public static List <FilesItem> UploadDocument(string fileNameLocal) { var lst = new List <FilesItem>(); try { if (!string.IsNullOrEmpty(fileNameLocal)) { var arrDocument = fileNameLocal.Split(','); foreach (var item in arrDocument) { var folder = DateTime.Now.Year + "\\" + DateTime.Now.Month + "\\" + DateTime.Now.Day + "\\"; var folderinsert = DateTime.Now.Year + "/" + DateTime.Now.Month + "/" + DateTime.Now.Day + "/"; var urlFolder = ConfigData.DocumentFolder + folder; if (!Directory.Exists(urlFolder)) { Directory.CreateDirectory(urlFolder); } if (item.Length > 1) { var fileLocal = item.Split('.'); var fileName = fileLocal[0] + "." + fileLocal[1]; File.Copy(ConfigData.TempFolder + fileName, urlFolder + fileName); var fileItem = new FilesItem { Folder = folderinsert, FileUrl = fileName, DateCreated = DateTime.Now.TotalSeconds(), TypeFile = fileLocal[1], Status = true, Name = fileName }; lst.Add(fileItem); } } return(lst); } return(new List <FilesItem>()); } catch (Exception ex) { Log2File.LogExceptionToFile(ex); return(new List <FilesItem>()); } }
/// <summary> /// 上传文件 /// </summary> public UploadedInfo Save() { Validate(); CreatePaths(); UploadedInfo uploadedInfo = new(); foreach (var file in UploadSettings.UploadParam.UploadFiles) { var item = new FilesItem(); SaveOriginal(item, file); SaveAddtional(item, file); item.SetStatus(FileStatus.Uploaded); Uploads.Files.Add(item); uploadedInfo.Uploaded.Add(item); } uploadedInfo.Deleted.AddRange(DeleteOldFiles()); uploadedInfo.Uploads = Uploads; return(uploadedInfo); }
protected virtual void SaveOriginal(FilesItem filesItem, UploadFileInfo uploadFile) { string path = Path.Combine(SavePath, $"{Guid.NewGuid()}_{Original}{uploadFile.Extension}"); using (var newStream = new FileStream(UploadSettings.WebRootPath + path, FileMode.OpenOrCreate)) { uploadFile.FileStream.CopyTo(newStream); } filesItem.O = new() { Length = uploadFile.Length, Name = uploadFile.FileName, FullPath = $"{UploadSettings.UriPath.TrimEnd('/')}/{path.Replace("\\", "/").TrimStart('/')}", UriPath = UploadSettings.UriPath, Path = path, Suff = uploadFile.Extension, No = Uploads.Files.Count }; }
/// <summary> /// 解析FileSItem类 /// </summary> /// <param name="itemPreview"></param> /// <returns></returns> private FilesItem ParseFilesItem(JsonData itemPreview) { FilesItem fi = new FilesItem(); fi.originalName = itemPreview["originalName"].ToString(); fi.url = itemPreview["url"].ToString(); var temp = itemPreview["paths"]; if (temp.Count > 0) { for (int i = 0; i < temp.Count; i++) { fi.paths.Add(temp[i].ToString().Substring(1)); } } fi.op = itemPreview["op"].ToString(); fi.md5 = itemPreview["md5"].ToString(); fi.size = itemPreview["size"].ToString(); return(fi); }
/// <summary> ///下载/更新道具 不传version默认为本地没有 /// </summary> /// <param name="id">道具ID</param> public void DownloadUpdate(int id, string version = "0.0.0", Action <string> done = null) { //获取版本信息 //若是默认的版本号,需要先获取当前道具的最新版本号 //if (version == "1.0.0") //{ // var component = JsonClass.Instance.ComponentList.Single(t => t.id == id); // if (component!=null) // { // version = component.version; // } //} Debug.Log(PortClass.Instance.CheckUpdate + id + "/" + version); HttpBase.GET(PortClass.Instance.CheckUpdate + id + "/" + version, (request, response) => { if (response == null || !response.IsSuccess) { DebugManager.Instance.LogError("请求失败!"); return; } JsonData Item = JsonMapper.ToObject(response.DataAsText.Trim()); if (Item["msg"].ToString() == "服务器中未发现可用版本") { if (done != null) { done("HaveNoUpdate"); } return; } if (Item["codeMsg"].ToString() != "SUCCESS") { Debug.Log("请求失败!"); return; } JsonData itemm = Item["data"]; var op = itemm["op"].ToString(); //如果op是full 需要先删除本地文件 if (op == "full") { string fullpath = FindFolder(id)[0] + "full.json"; //获取本地的full文件 根据本地full文件删除原有的东西 if (File.Exists(fullpath)) { StreamReader sr = File.OpenText(fullpath); string input = sr.ReadToEnd(); sr.Close(); sr.Dispose(); var fullfilejson = JsonMapper.ToObject(input.Trim()); for (int i = 0; i < (int)fullfilejson.Count; i++) { JsonData temp = fullfilejson[i]; foreach (var item in FindFolder(id)) { Debug.Log("需要删除的文件: " + item + temp["originalName"].ToString()); if (File.Exists(item + "/" + temp["originalName"].ToString())) { File.Delete(item + "/" + temp["originalName"].ToString()); } } } } GetFullJson(id); ///执行下载 } JsonData filejson = itemm["files"]; if (filejson != null) { for (int i = 0; i < (int)filejson.Count; i++) { JsonData temp = filejson[i]; FilesItem fi = ParseFilesItem(temp); JsonClass.Instance.CurrentAppUpdateFileJson.files.Add(fi); switch (temp["op"].ToString()) { case "add": //_downloadList.Add( //new Downloader.DownloadUnit( // PublicAttribute.URL + "/" + fi.url, // PublicAttribute.LocalFilePath + fi.paths[0], fi.originalName, fi.md5)); break; case "update": foreach (var path in fi.paths) { if (File.Exists(PublicAttribute.LocalFilePath + path + "/" + fi.originalName)) { File.Delete(PublicAttribute.LocalFilePath + path + "/" + fi.originalName); } } //_downloadList.Add( //new Downloader.DownloadUnit( // PublicAttribute.URL + "/" + fi.url, // PublicAttribute.LocalFilePath + fi.paths[0], fi.originalName, fi.md5)); break; case "remove": foreach (var path in fi.paths) { if (File.Exists(PublicAttribute.LocalFilePath + path + "/" + fi.originalName)) { File.Delete(PublicAttribute.LocalFilePath + path + "/" + fi.originalName); } } break; default: break; } } BatchDownloadFile(FindFolder(id), itemm["version"].ToString(), op, id, done); } else { done(FindFolder(id)[0]); } }); }
protected abstract void SaveAddtional(FilesItem filesItem, UploadFileInfo uploadFile);