public async Task <object> getMsN(object input)
{
var para = JsonConvert.DeserializeObject <Input>(input.ToString());
foreach (var item in para.filesPath)
{
Output output = new Output();
try
{
string path = FileNameUtil.getIndexPath(item);
if (string.IsNullOrEmpty(path))
{
output.status = false;
output.content = "Can't find the whole aird file.";
return(JsonConvert.SerializeObject(output));
}
//var parser = new BaseParser(path);
var msn = getMsNByTime(path, Convert.ToInt32(para.ms), Convert.ToDouble(para.time));
output.status = true;
output.content = msn;
return(JsonConvert.SerializeObject(output));
}
catch (Exception e)
{
output.status = false;
output.content = "Something error.";
return(JsonConvert.SerializeObject(output));
}
}
return(null);
}
public JobInfo(string inputFilePath, string outputFolderPath,
string type, JobParams jobParams, ListViewItem item)
{
this.jobId = inputFilePath;
this.inputFilePath = inputFilePath;
this.outputFolderPath = outputFolderPath;
this.type = type;
// 二代压缩算法StackZDPD目前不支持COMMON和SCANNING_SWATH模式
if (type.Equals(AirdType.COMMON) || type.Equals(AirdType.SCANNING_SWATH))
{
jobParams.airdAlgorithm = 1;
}
this.jobParams = jobParams;
format = Path.GetExtension(inputFilePath).Replace(".", "").ToUpper();
airdFileName = FileNameUtil.buildOutputFileName(inputFilePath);
airdFilePath = Path.Combine(outputFolderPath, airdFileName + jobParams.suffix + ".aird");
airdJsonFilePath = Path.Combine(outputFolderPath, airdFileName + jobParams.suffix + ".json");
this.cancellationTokenSource = new CancellationTokenSource();
this.progress = new Progress <string>((progressValue) =>
{
item.SubItems[3].Text = progressValue;
});
item.SubItems[2].Text = jobParams.getAirdAlgorithmStr();
item.SubItems[4].Text = outputFolderPath;
}
public ImageLabel GetImageLabelById(Topic topic, string imageId)
{
#region validation
if (topic == null)
{
throw new ArgumentNullException(nameof(topic));
}
if (string.IsNullOrEmpty(imageId))
{
throw new ArgumentNullException(nameof(imageId));
}
#endregion
// get all imagefile paths
List <string> imageFilePaths = new List <string>(GetImagePaths(topic));
foreach (string imageFilePath in imageFilePaths)
{
if (imageId.Equals(FileNameUtil.GetFileNameFromPath(imageFilePath)))
{
//create default imagelabel and set index
return(CreateImage(topic, imageFilePaths.IndexOf(imageFilePath), imageFilePath, true));
}
}
return(null);
}
public void SaveDailyLatestKmlAndPng(string savedDirectory, string kmlFileName, DataFileType type = DataFileType.VIIRS1Day)
{
long fileLength = new FileInfo(kmlFileName.Replace(".kml", ".png")).Length;
int bIndex = FileNameUtil.GetBlockIndexFromFileName(kmlFileName, type);
string[] existedFiles = Directory.GetFiles(savedDirectory, string.Format("*_{0}.kml", bIndex.ToString("D3")));
if (existedFiles.Length > 1)
{
throw new Exception();
}
if (existedFiles.Length == 0)
{
File.Copy(kmlFileName, Path.Combine(savedDirectory, Path.GetFileNameWithoutExtension(kmlFileName) + ".kml"));
File.Copy(kmlFileName.Replace(".kml", ".png"), Path.Combine(savedDirectory, Path.GetFileNameWithoutExtension(kmlFileName) + ".png"));
}
if (existedFiles.Length == 1)
{
long length = new FileInfo(existedFiles[0].Replace(".kml", ".png")).Length;
if (fileLength > length)
{
File.Delete(existedFiles[0]);
File.Delete(existedFiles[0].Replace(".kml", ".png"));
File.Copy(kmlFileName, Path.Combine(savedDirectory, Path.GetFileNameWithoutExtension(kmlFileName) + ".kml"));
File.Copy(kmlFileName.Replace(".kml", ".png"), Path.Combine(savedDirectory, Path.GetFileNameWithoutExtension(kmlFileName) + ".png"));
}
}
}
private int compareKmlByDistrictIndex(KmlFileInfo file1, KmlFileInfo file2)
{
int dIndex1 = FileNameUtil.GetBlockIndexFromFileName(file1.ShortName);
int dIndex2 = FileNameUtil.GetBlockIndexFromFileName(file2.ShortName);
return(dIndex2.CompareTo(dIndex1));
}
private IEnumerable <ImageLabel> GetImageLabels(Topic topic)
{
#region validation
if (topic == null)
{
throw new ArgumentNullException(nameof(topic));
}
#endregion
List <ImageLabel> imageLabelList = new List <ImageLabel>();
long index = 0L;
foreach (string imagePath in GetImagePaths(topic))
{
string imageFileName = FileNameUtil.GetFileNameFromPath(imagePath);
ImageLabel imgLabel = CreateImageLabel(imagePath);
imgLabel.Index = index++;
imgLabel.SetImageUrl(HttpContextAccessor.HttpContext, topic);
imageLabelList.Add(imgLabel);
}
return(imageLabelList);
}
private int compareKmlFilesByDistrictIndex(string fileName1, string fileName2)
{
int dIndex1 = FileNameUtil.GetBlockIndexFromFileName(fileName1);
int dIndex2 = FileNameUtil.GetBlockIndexFromFileName(fileName2);
return(dIndex2.CompareTo(dIndex1));
}
public uint GetImageIndexByImageFileName(Topic topic, string imageFileName)
{
#region validation
if (topic == null)
{
throw new ArgumentNullException(nameof(topic));
}
if (string.IsNullOrEmpty(imageFileName))
{
throw new ArgumentNullException(nameof(imageFileName));
}
#endregion
// get all imagefile paths
List <string> imageFilePaths = new List <string>(GetImagePaths(topic));
// extract filenames from filepaths and search for equality
uint imageIndex = 0;
foreach (string imageFilePath in imageFilePaths)
{
string imageName = FileNameUtil.GetFileNameFromPath(imageFilePath);
if (imageFileName == imageName)
{
return(imageIndex);
}
imageIndex++;
}
return(0);
}
/// <summary>
/// Get valid image paths from list of given file paths.
/// </summary>
/// <param name="filePaths">List of file paths</param>
/// <returns>List of image paths</returns>
public static IEnumerable <string> GetValidImagePaths(IEnumerable <string> filePaths)
{
#region validation
if (filePaths == null)
{
throw new ArgumentNullException(nameof(filePaths));
}
#endregion
List <string> imageFileList = new List <string>();
if (ValidImageExtensions != null && ValidImageExtensions.Count() > 0)
{
foreach (string validImageExtension in ValidImageExtensions)
{
imageFileList.AddRange(filePaths.Where(o => FileNameUtil.GetFileExtension(o).Equals(validImageExtension, StringComparison.InvariantCultureIgnoreCase)));
}
}
else
{
// return all files if no filter is set.
return(filePaths);
}
return(imageFileList);
}
public async Task <object> getFileDetail(object input)
{
Output output = new Output();
try
{
string path = FileNameUtil.getIndexPath(input.ToString());
if (string.IsNullOrEmpty(path))
{
output.status = false;
output.content = "Can't find the whole aird file.";
return(JsonConvert.SerializeObject(output));
}
var str = File.ReadAllText(path);
output.status = true;
var info = JsonConvert.DeserializeObject <AirdInfo>(str);
info.indexList = null;
output.content = info;
return(JsonConvert.SerializeObject(output));
}
catch (Exception e)
{
output.status = false;
output.content = "Something error.";
return(JsonConvert.SerializeObject(output));
}
}
/// <summary>
/// Gets list of imagelabels with additional information about
/// - having labels
/// - having segments
/// </summary>
private void FillLabelAndSegmentInformation(Topic topic, IEnumerable <ImageLabel> allImages)
{
IEnumerable <string> labeledImageFilesWithoutExtension = LabelService.GetLabeledImageFileNames(topic);
foreach (ImageLabel imageLabel in allImages)
{
imageLabel.HasLabels = labeledImageFilesWithoutExtension.Contains(FileNameUtil.GetFileNameWithoutExtension(imageLabel.Id));
}
}
private string GetLabelFileName(string imageFileName)
{
string imgFileRaw = FileNameUtil.GetFileNameWithoutExtension(imageFileName);
// labelname for that image
string labelFileName = $"{imgFileRaw}.json";
return(labelFileName);
}
private ImageLabel CreateImageLabel(string imageFilePath)
{
string imageFileName = FileNameUtil.GetFileNameFromPath(imageFilePath);
return(new ImageLabel()
{
StoragePath = imageFilePath,
Id = imageFileName,
HasLabels = false
});
}
/// <summary>
/// Determines label file name from imagefilename.
/// </summary>
/// <param name="imageFileName">Name of imagefile</param>
/// <returns>Name of labelfile</returns>
public static string GetLabelFileName(string imageFileName)
{
#region validation
if (string.IsNullOrEmpty(imageFileName))
{
throw new ArgumentNullException(nameof(imageFileName));
}
#endregion
// get name of imagefile without extension
string imageFileNameRaw = FileNameUtil.GetFileNameWithoutExtension(imageFileName);
// labelname for that image
return($"{imageFileNameRaw}.json");
}
/**
* 使用直接的关键信息进行初始化
*
* @param airdPath Aird文件路径
* @param mzCompressor mz压缩策略
* @param intCompressor intensity压缩策略
* @param mzPrecision mz数字精度
* @param airdType aird类型
* @throws ScanException 扫描异常
*/
public BaseParser(String airdPath, Compressor mzCompressor, Compressor intCompressor, int mzPrecision, String airdType)
{
var indexFile = FileNameUtil.getIndexPathByAirdPath(airdPath);
airdInfo = AirdScanUtil.loadAirdInfo(indexFile);
try
{
airdFile = File.OpenRead(airdPath);
}
catch (FileNotFoundException e)
{
Console.WriteLine(e.ToString());
Console.Write(e.StackTrace);
throw new Exception(ResultCodeEnum.AIRD_FILE_PARSE_ERROR.Message);
}
this.mzCompressor = mzCompressor;
this.intCompressor = intCompressor;
this.mzPrecision = mzPrecision;
this.type = airdType;
}
public void TestIsValidFileName()
{
var valid = FileNameUtil.IsValidFileName("test?");
Assert.IsFalse(valid);
valid = FileNameUtil.IsValidFileName(".test");
Assert.IsTrue(valid);
valid = FileNameUtil.IsValidFileName("test.");
Assert.IsFalse(valid);
valid = FileNameUtil.IsValidFileName("/test");
Assert.IsFalse(valid);
valid = FileNameUtil.IsValidFileName("\\test");
Assert.IsFalse(valid);
valid = FileNameUtil.IsValidFileName("test/");
Assert.IsFalse(valid);
valid = FileNameUtil.IsValidFileName("test\\");
Assert.IsFalse(valid);
valid = FileNameUtil.IsValidFileName("test\\test");
Assert.IsFalse(valid);
valid = FileNameUtil.IsValidFileName("test");
Assert.IsTrue(valid);
}
public void CreateValidFileName()
{
var validFileName = FileNameUtil.MakeValidFileName("test?");
Assert.IsTrue(FileNameUtil.IsValidFileName(validFileName));
validFileName = FileNameUtil.MakeValidFileName(".test");
Assert.IsTrue(FileNameUtil.IsValidFileName(validFileName));
validFileName = FileNameUtil.MakeValidFileName("test.");
Assert.IsTrue(FileNameUtil.IsValidFileName(validFileName));
validFileName = FileNameUtil.MakeValidFileName("/test");
Assert.IsTrue(FileNameUtil.IsValidFileName(validFileName));
validFileName = FileNameUtil.MakeValidFileName("\test");
Assert.IsTrue(FileNameUtil.IsValidFileName(validFileName));
validFileName = FileNameUtil.MakeValidFileName("test/");
Assert.IsTrue(FileNameUtil.IsValidFileName(validFileName));
validFileName = FileNameUtil.MakeValidFileName("test\\");
Assert.IsTrue(FileNameUtil.IsValidFileName(validFileName));
validFileName = FileNameUtil.MakeValidFileName("test\\test");
Assert.IsTrue(FileNameUtil.IsValidFileName(validFileName));
validFileName = FileNameUtil.MakeValidFileName("test");
Assert.IsTrue(FileNameUtil.IsValidFileName(validFileName));
}
private void SetHasLabels(Topic topic, IEnumerable <ImageLabel> images)
{
#region validation
if (topic == null)
{
throw new ArgumentNullException(nameof(topic));
}
if (images == null)
{
throw new ArgumentNullException(nameof(images));
}
#endregion
IEnumerable <string> labeledImageFilesWithoutExtension = LabelRepository.GetLabelFileNames(topic);
foreach (ImageLabel image in images)
{
image.HasLabels = labeledImageFilesWithoutExtension.Contains(FileNameUtil.GetFileNameWithoutExtension(image.Id));
}
}
public BaseParser(string indexPath)
{
airdInfo = AirdScanUtil.loadAirdInfo(indexPath);
if (airdInfo == null)
{
throw new Exception(ResultCodeEnum.AIRD_INDEX_FILE_PARSE_ERROR.Message);
}
try
{
airdFile = File.OpenRead(FileNameUtil.getAirdPathByIndexPath(indexPath));
}
catch (FileNotFoundException e)
{
Console.WriteLine(e.ToString());
Console.Write(e.StackTrace);
throw new Exception(ResultCodeEnum.AIRD_FILE_PARSE_ERROR.Message);
}
mzCompressor = CompressUtil.getMzCompressor(airdInfo.compressors);
intCompressor = CompressUtil.getIntCompressor(airdInfo.compressors);
mzPrecision = mzCompressor.precision;
type = airdInfo.type;
}
/// <summary>
/// Create image structure for image from given topic.
///
/// Fullfilled image contains of:
/// <ul>
/// <li>Width and height of image</li>
/// <li>HasClassifications - Flag</li>
/// <li>HasLabels - Flag</li>
/// <li>HasSegments - Flag</li>
/// </ul>
/// </summary>
/// <param name="topic">Topic where image is stored</param>
/// <param name="index">Index of image in topic</param>
/// <param name="imageFilePath">Path to image in filesystem</param>
/// <param name="full">TRUE if fullfilled object shoul be returned</param>
/// <returns>ImageLabelstructure</returns>
private ImageLabel CreateImage(Topic topic, long index, string imageFilePath, bool full = false)
{
string imageFileName = FileNameUtil.GetFileNameFromPath(imageFilePath);
ImageLabel result = new ImageLabel()
{
StoragePath = imageFilePath,
Id = imageFileName,
Index = index
};
if (full)
{
SetHasLabels(topic, new List <ImageLabel>()
{
result
});
SetImageSize(topic, result);
}
return(result);
}
private IList <ImageLabel> GetImagesInternal(Topic topic)
{
#region validation
if (topic == null)
{
throw new ArgumentNullException(nameof(topic));
}
#endregion
List <ImageLabel> images = new List <ImageLabel>();
long index = 0L;
foreach (string imagePath in GetImagePaths(topic))
{
FileNameUtil.GetFileNameFromPath(imagePath);
ImageLabel image = CreateImage(topic, index++, imagePath);
images.Add(image);
}
return(images);
}
public string[] AddImageFile(Topic topic, string fileName, System.IO.Stream imageStream)
{
#region validation
if (topic == null)
{
throw new ArgumentNullException(nameof(topic));
}
if (string.IsNullOrEmpty(fileName))
{
throw new ArgumentNullException(nameof(fileName));
}
if (imageStream == null)
{
throw new ArgumentNullException(nameof(imageStream));
}
#endregion
string[] imageFolder = GetImageFolder(topic);
fileName = fileName.Replace(' ', '_');
// check for filenames which already exists.
// check should ignore file extension, filenames are equal even if extensions are inequal (003.jpg and 003.png)
IEnumerable <string> fileNames = FileContainer.GetFiles(imageFolder).Select(o => FileNameUtil.GetFileNameWithoutExtension(o).ToLower());
// generate file name if file name already exists
while (fileNames.Contains(FileNameUtil.GetFileNameWithoutExtension(fileName.ToLower())))
{
string fileExtension = FileNameUtil.GetFileExtension(fileName);
fileName = $"{Guid.NewGuid().ToString()}{fileExtension}";
}
return(FileContainer.CreateFile(fileName, imageStream, imageFolder));
}
public IEnumerable <string> GetLabelFileNames(Topic topic)
{
#region validation
if (topic == null)
{
throw new ArgumentNullException(nameof(topic));
}
#endregion
// get paths of all labelfiles
IEnumerable <string> labelFilePaths = FileContainer.GetFiles(GetLabelsPath(topic));
// extract labelfilename without extension
// equal to imagefilename without extension
List <string> labelFileNames = new List <string>(labelFilePaths.Count());
foreach (string labelFilePath in labelFilePaths)
{
labelFileNames.Add(FileNameUtil.GetFileNameWithoutExtension(labelFilePath));
}
return(labelFileNames);
}
public ImageLabel GetImageLabelById(Topic topic, string imageId)
{
#region validation
if (topic == null)
{
throw new ArgumentNullException(nameof(topic));
}
if (string.IsNullOrEmpty(imageId))
{
throw new ArgumentNullException(nameof(imageId));
}
#endregion
// get all imagefile paths
List <string> imageFilePaths = new List <string>(GetImagePaths(topic));
foreach (string imageFilePath in imageFilePaths)
{
if (imageId.Equals(FileNameUtil.GetFileNameFromPath(imageFilePath)))
{
//create default imagelabel and set index
ImageLabel imageLabel = CreateImageLabel(imageFilePath);
imageLabel.Index = imageFilePaths.IndexOf(imageFilePath);
imageLabel.SetImageUrl(HttpContextAccessor.HttpContext, topic);
SetImageSize(topic, imageLabel);
return(imageLabel);
}
}
return(null);
}
public async Task <object> ReadFile(object input)
{
var para = JsonConvert.DeserializeObject <Input>(input.ToString());
Output output = new Output();
try
{
foreach (var item in para.filesPath)
{
string path = FileNameUtil.getIndexPath(item);
var tic = getTIC(path);
output.status = true;
output.content = tic;
return(JsonConvert.SerializeObject(output));
}
}
catch (Exception e)
{
output.status = false;
output.content = "Something error.";
return(JsonConvert.SerializeObject(output));
}
return(null);
}
static void Main(string[] args)
{
Entity.Preprocess();
int op = 3;
if (op > 0)
{
//TimeExtractor.Preprocessing.preprocess();
}
for (int packageNo = 1; packageNo <= 6; packageNo++)
{
string folder = "docs\\Package " + packageNo;
// string folder = "docs";
string[] files = Directory.GetFiles(folder, "*.txt");
foreach (string file in files)
{
switch (op)
{
case 0:
{
//Tag with stemmed PMT tags
Regex fileregex = new Regex(@"[0-9]+\.txt");
Match match = fileregex.Match(file);
if (match == null)
{
break;
}
string concept_filename = "concepts\\" + match.Value.Replace("txt", "con");
if (!File.Exists(concept_filename))
{
Console.WriteLine("Uoh, concept file \"" + concept_filename + "\" not found.");
break;
}
string[] cons = File.ReadAllLines(concept_filename);
List <Entity> entities = new List <Entity>();
foreach (string con in cons)
{
if (con.Length <= 0)
{
continue;
}
Entity entity = EntityUtil.I2b2formToEntity(con, file);
entities.Add(entity);
}
IEnumerable <Entity> PMTEntities =
from entity in entities
where entity.type == "problem" || entity.type == "treatment" || entity.type == "test"
select entity;
Annotator.Annotate(file, ".PMTstem.con", entities, true);
break;
}
case 1:
{
TimeExtractor.tools.Init.setFilePath(file);
TimeExtractor.TimeMapping.process(false);
List <SenseGroup> sensegroups = TimeVariables.TIME_ENTITIES;
List <Entity> timeEntities = new List <Entity>();
foreach (SenseGroup sg in sensegroups)
{
Entity entity = new TimeEntity();
entity.text = sg.getWords()[0];
entity.startLoc = sg.startLoc;
entity.endLoc = sg.endLoc;
entity.setTimePoint(sg.getTimePeriod().getFirstTimePoint());
timeEntities.Add(entity);
}
Annotator.Annotate(file, ".time.con", timeEntities, false);
break;
}
case 2:
{
//Tag with PMT tags + guessed classifications
Regex fileregex = new Regex(@"[0-9]+\.txt");
Match match = fileregex.Match(file);
if (match == null)
{
break;
}
string concept_filename = "concepts\\" + match.Value.Replace("txt", "con");
if (!File.Exists(concept_filename))
{
Console.WriteLine("Uoh, concept file \"" + concept_filename + "\" not found.");
break;
}
string sectionfile = file.Replace("txt", "section");
string[] sections = File.ReadAllLines(sectionfile);
string[] cons = File.ReadAllLines(concept_filename);
List <Entity> entities = new List <Entity>();
foreach (string con in cons)
{
if (con.Length <= 0)
{
continue;
}
PMTEntity entity = (PMTEntity)EntityUtil.I2b2formToEntity(con, file);
int lineNumber = entity.startLoc.line;
int ptab = sections[lineNumber - 1].IndexOf("\t");
string no = sections[lineNumber - 1].Substring(0, ptab);
string[] wa_nos = { "1.1", "5.34", "5.34.78", "5.34.78.93", "5.34.78.93.35", "5.34.78.93.38", "5.34.78.96", "5.34.78.96.45", "5.34.79", "5.34.79.103.60", "5.35", "5.35.84", "5.35.91.108" };
string[] a_nos = { "5.15", "5.22.44" };
string[] ad_nos = { "5.37.106.125" };
entities.Add(entity);
}
TimeExtractor.tools.Init.setFilePath(file);
TimeExtractor.TimeMapping.process(false);
List <SenseGroup> sensegroups = TimeVariables.TIME_ENTITIES;
foreach (SenseGroup sg in sensegroups)
{
Entity entity = new TimeEntity();
entity.text = sg.getWords()[0];
entity.startLoc = sg.startLoc;
entity.endLoc = sg.endLoc;
entity.setTimePoint(sg.getTimePeriod().getFirstTimePoint());
entities.Add(entity);
}
IEnumerable <Entity> PMTEntities =
from entity in entities
where entity.type == "problem" || entity.type == "treatment" || entity.type == "test" || entity.type == "time"
select entity;
Annotator.Annotate(file, ".PMTrelation.con", entities, false);
break;
}
case 3:
{
//From annotated time expression (including normalized) to concept files
string annotate_file = file.Replace(".txt", ".time.con");
string con_file = file.Replace(".txt", ".time-con");
Entity[] entities = Annotator.ReadAnnotate(annotate_file);
EntityUtil.ExportConcept(con_file, entities, false);
break;
}
case 4:
{
//Update the original data
//WARNING: THINK BEFORE YOU DO THIS
string annotate_file = file.Replace(".txt", ".time.con");
string raw_file = file;
Annotator.UpdateOriginalData(annotate_file, raw_file);
break;
}
case 5:
{
//Tag with standard PMT tags + revised time taggings (including normalizations)
string time_con_file = file.Replace(".txt", ".time-con");
string pmt_con_file = "concepts\\" + FileNameUtil.FileNameNoSuffix(file) + ".con";
Entity[] TIMEentities = EntityUtil.ImportConcept(time_con_file, "*.time-con", "*.txt");
Entity[] PMTentities = EntityUtil.ImportConcept(pmt_con_file, "*.con", "*.txt");
List <Entity> entities = new List <Entity>();
foreach (Entity entity in TIMEentities)
{
entities.Add(entity);
}
foreach (Entity entity in PMTentities)
{
entities.Add(entity);
}
Annotator.Annotate(file, ".PMTrelation.con", entities, false);
break;
}
}
}
}
}
