public void AddURL(FilesItem metadataFile)
{
if (Program.cfgCurrent.UseHead)
{
this.filesToReadSizeQueue.Enqueue(metadataFile);
}
}
private void wsSearch_SearcherLinkFoundEvent(object sender, EventsThreads.CollectionFound <Uri> e)
{
foreach (Uri url in e.Data)
{
try
{
try
{
string fileExtension = Path.GetExtension(url.AbsolutePath).ToLowerInvariant();
if (!String.IsNullOrWhiteSpace(fileExtension) && DocumentExtractor.IsSupportedExtension(fileExtension))
{
var fi = new FilesItem
{
Ext = fileExtension,
URL = url.ToString(),
Downloaded = false,
MetadataExtracted = false,
Date = DateTime.MinValue,
ModifiedDate = DateTime.MinValue,
Path = string.Empty,
Size = -1
};
Program.data.files.Items.Add(fi);
Program.FormMainInstance.treeViewMetadata_UpdateDocumentsNumber();
Program.FormMainInstance.panelMetadataSearch.listViewDocuments_Update(fi);
Program.FormMainInstance.panelMetadataSearch.HttpSizeDaemonInst.AddURL(fi);
}
}
catch (Exception)
{
}
// add the url to the files list
DomainsItem domain = Program.data.GetDomain(url.Host);
if (domain == null)
{
Program.data.AddDomain(url.Host, "Crawling", Program.cfgCurrent.MaxRecursion, Program.cfgCurrent);
Program.LogThis(new Log(Log.ModuleType.Crawling, "Domain found: " + url.Host, Log.LogType.medium));
domain = Program.data.GetDomain(url.Host);
}
domain.map.AddUrl(url.ToString());
if (domain.techAnalysis.domain == null)
{
domain.techAnalysis.domain = domain.Domain;
}
domain.techAnalysis.eventLinkFoundDetailed(null, new EventsThreads.CollectionFound <Uri>(new List <Uri> {
url
}));
}
catch
{
}
}
}
private void wsSearch_SearcherLinkFoundEvent(object sender, EventsThreads.CollectionFound <Uri> e)
{
foreach (Uri url in e.Data)
{
try
{
try
{
var fileWithMetadata =
Program.FormMainInstance.panelMetadataSearch.checkedListBoxExtensions.Items.Cast <string>()
.Any(checkedListbox => url.ToString().EndsWith(checkedListbox));
if (fileWithMetadata)
{
var fi = new FilesItem
{
Ext = Path.GetExtension(url.AbsolutePath).ToLower(),
URL = url.ToString(),
Downloaded = false,
Processed = false,
Date = DateTime.MinValue,
ModifiedDate = DateTime.MinValue,
Path = string.Empty,
Size = -1
};
Program.data.files.Items.Add(fi);
Program.FormMainInstance.treeViewMetadata_UpdateDocumentsNumber();
Program.FormMainInstance.panelMetadataSearch.listViewDocuments_Update(fi);
Program.FormMainInstance.panelMetadataSearch.HttpSizeDaemonInst.AddURL(fi);
}
}
catch (Exception)
{
}
// add the url to the files list
var d = Program.data.GetDomain(url.Host);
d.map.AddUrl(url.ToString());
if (d.techAnalysis.domain == null)
{
d.techAnalysis.domain = d.Domain;
}
var listaUrl = new List <Uri> {
url
};
d.techAnalysis.eventLinkFoundDetailed(null,
new EventsThreads.CollectionFound <Uri>(listaUrl));
}
catch
{
}
}
}
public static List <FilesItem> UploadDocument(string fileNameLocal)
{
var lst = new List <FilesItem>();
try
{
if (!string.IsNullOrEmpty(fileNameLocal))
{
var arrDocument = fileNameLocal.Split(',');
foreach (var item in arrDocument)
{
var folder = DateTime.Now.Year + "\\" + DateTime.Now.Month + "\\" + DateTime.Now.Day + "\\";
var folderinsert = DateTime.Now.Year + "/" + DateTime.Now.Month + "/" + DateTime.Now.Day + "/";
var urlFolder = ConfigData.DocumentFolder + folder;
if (!Directory.Exists(urlFolder))
{
Directory.CreateDirectory(urlFolder);
}
if (item.Length > 1)
{
var fileLocal = item.Split('.');
var fileName = fileLocal[0] + "." + fileLocal[1];
File.Copy(ConfigData.TempFolder + fileName, urlFolder + fileName);
var fileItem = new FilesItem
{
Folder = folderinsert,
FileUrl = fileName,
DateCreated = DateTime.Now.TotalSeconds(),
TypeFile = fileLocal[1],
Status = true,
Name = fileName
};
lst.Add(fileItem);
}
}
return(lst);
}
return(new List <FilesItem>());
}
catch (Exception ex)
{
Log2File.LogExceptionToFile(ex);
return(new List <FilesItem>());
}
}
/// <summary>
/// 上传文件
/// </summary>
public UploadedInfo Save()
{
Validate();
CreatePaths();
UploadedInfo uploadedInfo = new();
foreach (var file in UploadSettings.UploadParam.UploadFiles)
{
var item = new FilesItem();
SaveOriginal(item, file);
SaveAddtional(item, file);
item.SetStatus(FileStatus.Uploaded);
Uploads.Files.Add(item);
uploadedInfo.Uploaded.Add(item);
}
uploadedInfo.Deleted.AddRange(DeleteOldFiles());
uploadedInfo.Uploads = Uploads;
return(uploadedInfo);
}
protected virtual void SaveOriginal(FilesItem filesItem, UploadFileInfo uploadFile)
{
string path = Path.Combine(SavePath, $"{Guid.NewGuid()}_{Original}{uploadFile.Extension}");
using (var newStream = new FileStream(UploadSettings.WebRootPath + path, FileMode.OpenOrCreate))
{
uploadFile.FileStream.CopyTo(newStream);
}
filesItem.O = new()
{
Length = uploadFile.Length,
Name = uploadFile.FileName,
FullPath = $"{UploadSettings.UriPath.TrimEnd('/')}/{path.Replace("\\", "/").TrimStart('/')}",
UriPath = UploadSettings.UriPath,
Path = path,
Suff = uploadFile.Extension,
No = Uploads.Files.Count
};
}
/// <summary>
/// 解析FileSItem类
/// </summary>
/// <param name="itemPreview"></param>
/// <returns></returns>
private FilesItem ParseFilesItem(JsonData itemPreview)
{
FilesItem fi = new FilesItem();
fi.originalName = itemPreview["originalName"].ToString();
fi.url = itemPreview["url"].ToString();
var temp = itemPreview["paths"];
if (temp.Count > 0)
{
for (int i = 0; i < temp.Count; i++)
{
fi.paths.Add(temp[i].ToString().Substring(1));
}
}
fi.op = itemPreview["op"].ToString();
fi.md5 = itemPreview["md5"].ToString();
fi.size = itemPreview["size"].ToString();
return(fi);
}
/// <summary>
///下载/更新道具 不传version默认为本地没有
/// </summary>
/// <param name="id">道具ID</param>
public void DownloadUpdate(int id, string version = "0.0.0", Action <string> done = null)
{
//获取版本信息
//若是默认的版本号，需要先获取当前道具的最新版本号
//if (version == "1.0.0")
//{
// var component = JsonClass.Instance.ComponentList.Single(t => t.id == id);
// if (component!=null)
// {
// version = component.version;
// }
//}
Debug.Log(PortClass.Instance.CheckUpdate + id + "/" + version);
HttpBase.GET(PortClass.Instance.CheckUpdate + id + "/" + version, (request, response) =>
{
if (response == null || !response.IsSuccess)
{
DebugManager.Instance.LogError("请求失败！");
return;
}
JsonData Item = JsonMapper.ToObject(response.DataAsText.Trim());
if (Item["msg"].ToString() == "服务器中未发现可用版本")
{
if (done != null)
{
done("HaveNoUpdate");
}
return;
}
if (Item["codeMsg"].ToString() != "SUCCESS")
{
Debug.Log("请求失败！");
return;
}
JsonData itemm = Item["data"];
var op = itemm["op"].ToString();
//如果op是full 需要先删除本地文件
if (op == "full")
{
string fullpath = FindFolder(id)[0] + "full.json";
//获取本地的full文件 根据本地full文件删除原有的东西
if (File.Exists(fullpath))
{
StreamReader sr = File.OpenText(fullpath);
string input = sr.ReadToEnd();
sr.Close();
sr.Dispose();
var fullfilejson = JsonMapper.ToObject(input.Trim());
for (int i = 0; i < (int)fullfilejson.Count; i++)
{
JsonData temp = fullfilejson[i];
foreach (var item in FindFolder(id))
{
Debug.Log("需要删除的文件: " + item + temp["originalName"].ToString());
if (File.Exists(item + "/" + temp["originalName"].ToString()))
{
File.Delete(item + "/" + temp["originalName"].ToString());
}
}
}
}
GetFullJson(id);
///执行下载
}
JsonData filejson = itemm["files"];
if (filejson != null)
{
for (int i = 0; i < (int)filejson.Count; i++)
{
JsonData temp = filejson[i];
FilesItem fi = ParseFilesItem(temp);
JsonClass.Instance.CurrentAppUpdateFileJson.files.Add(fi);
switch (temp["op"].ToString())
{
case "add":
//_downloadList.Add(
//new Downloader.DownloadUnit(
// PublicAttribute.URL + "/" + fi.url,
// PublicAttribute.LocalFilePath + fi.paths[0], fi.originalName, fi.md5));
break;
case "update":
foreach (var path in fi.paths)
{
if (File.Exists(PublicAttribute.LocalFilePath + path + "/" + fi.originalName))
{
File.Delete(PublicAttribute.LocalFilePath + path + "/" + fi.originalName);
}
}
//_downloadList.Add(
//new Downloader.DownloadUnit(
// PublicAttribute.URL + "/" + fi.url,
// PublicAttribute.LocalFilePath + fi.paths[0], fi.originalName, fi.md5));
break;
case "remove":
foreach (var path in fi.paths)
{
if (File.Exists(PublicAttribute.LocalFilePath + path + "/" + fi.originalName))
{
File.Delete(PublicAttribute.LocalFilePath + path + "/" + fi.originalName);
}
}
break;
default:
break;
}
}
BatchDownloadFile(FindFolder(id), itemm["version"].ToString(), op, id, done);
}
else
{
done(FindFolder(id)[0]);
}
});
}
protected abstract void SaveAddtional(FilesItem filesItem, UploadFileInfo uploadFile);
