public static IDataFileReader Create(DataFileType type)
{
IDataFileReader newReader = null;
switch (type)
{
case DataFileType.pdat:
newReader = new PDATFileReader();
break;
case DataFileType.csv:
newReader = new CSVFileReader();
break;
case DataFileType.powHQ:
break;
case DataFileType.PI:
break;
case DataFileType.OpenHistorian:
break;
default:
break;
}
return(newReader);
}
public static int GetHourFromFileName(string fileName, DataFileType type = DataFileType.ABI)
{
string name = Path.GetFileNameWithoutExtension(fileName);
string[] elements = name.Split(SplitCharacters, StringSplitOptions.RemoveEmptyEntries);
int hour = 0;
switch (type)
{
case DataFileType.ABI:
case DataFileType.AHI:
throw new Exception(string.Format("Hour value is not supported by ABI or AHI file: {0}", fileName));
case DataFileType.VIIRSABI:
case DataFileType.VIIRSAHI:
if (elements.Length != 11)
{
throw new Exception(string.Format("Unsupported VIIRSABI or VIIRSAHI file name format: {0}", fileName));
}
hour = Convert.ToInt32(elements[7]);
break;
case DataFileType.VIIRS1Day:
case DataFileType.VIIRS5Day:
if (elements.Length != 12)
{
throw new Exception(string.Format("Unsupported VIIRS1Day or VIIRS5Day file name format: {0}", fileName));
}
hour = Convert.ToInt32(elements[9]);
break;
}
return(hour);
}
public static bool CheckDataFileMatch(string file, DataFileType type)
{
var tp = "";
try
{
tp = Path.GetExtension(file).Substring(1).ToLower();
}
catch
{
}
if (type.ToString().ToLower() == tp)
{
return(true);
}
else if (type == DataFileType.powHQ && tp == "mat")
{
return(true);
}
else if ((type == DataFileType.PI || type == DataFileType.OpenHistorian || type == DataFileType.OpenPDC) && tp == "xml")
{
return(true);
}
else
{
return(false);
}
}
void CalculateScaleFactorAB(DataFileType dataFileType)
{
if (dataFileType == DataFileType.Binary ||
dataFileType == DataFileType.Binary32) //i make it same, but in theory, bin32 can be more precise
{
for (int i = 0; i < this.analogChannelInformations.Count; i++)
{
double min = this.samples.Min(x => x.analogs[i]);
double max = this.samples.Max(x => x.analogs[i]);
this.analogChannelInformations[i].b = (max + min) / 2.0;
if (max != min)
{
this.analogChannelInformations[i].a = (max - min) / 32767.0; //65536
}
this.analogChannelInformations[i].Min = -32767; //by standart 1999
this.analogChannelInformations[i].Max = 32767; //by standart 1999
}
}
else if (dataFileType == DataFileType.ASCII)
{
foreach (var analogChannelInformation in this.analogChannelInformations)
{
analogChannelInformation.Min = -32767; //by standart 1999
analogChannelInformation.Max = 32767; //by standart 1999
}
}
}
public DataFileFact(DataFileType type, string name, int dataSize, int stringLength)
{
Type = type;
Name = name;
DataSize = dataSize;
StringLength = stringLength;
}
public byte[] ToByteDAT(DataFileType dataFileType, IReadOnlyList <AnalogChannelInformation> analogInformations)
{
var result = new byte[DataFileHandler.GetByteCount(this.AnalogValues.Length, this.DigitalValues.Length, dataFileType)];
int analogOneChannelLength = dataFileType == DataFileType.Binary ? 2 : 4;
int digitalByteStart = 8 + analogOneChannelLength * this.AnalogValues.Length;
System.BitConverter.GetBytes(this.Number).CopyTo(result, 0);
System.BitConverter.GetBytes(this.Timestamp).CopyTo(result, 4);
switch (dataFileType)
{
case DataFileType.Binary:
this.AnalogsToBinaryDAT(result, analogInformations);
break;
case DataFileType.Binary32:
this.AnalogsToBinary32DAT(result, analogInformations);
break;
case DataFileType.Float32:
this.AnalogsToFloat32DAT(result);
break;
default:
throw new InvalidOperationException("Not supported file type DAT");
}
this.DigitalsToDAT(result, digitalByteStart);
return(result);
}
public ActionResult DoDownload()
{
// get the from date
DateTime fromDt = Convert.ToDateTime(Request.Form["date_from"]);
// get the to date
DateTime toDt = Convert.ToDateTime(Request.Form["date_to"]);
// get type of data
DataFileType dT = (DataFileType)Enum.Parse(typeof(DataFileType), Request.Form["data_type"]);
// create the Download model
Download dn = new Download(fromDt, toDt, dT);
//download path
string dnFile = dn.GetDownloadFilePath();
//check if the download is a valid one
if (dn.isOK == true)
{
//get the download file path content and return it to the browser
return(File(dnFile, "application/octet-stream", "data-" + Path.GetRandomFileName().Replace(".", "") + ".zip"));
}
else
{
//throw new HttpException(404, "Invalid dates or data types.");//RedirectTo NoFoundPage
ViewBag.NotFound = true;
return(View("Index"));
}
}
public static DataSourceSetting MakeInputFile(DataFileType type = DataFileType.csv)
{
DataSourceSetting testFile = new DataSourceSetting(".\\TestDir\\");
testFile.FileType = type;
return(testFile);
}
public DataFileFact(DataFileType type, string name, int dataSize, int stringLength, DataFileHeaderInformation headerOverload = null)
{
Type = type;
Name = name;
DataSize = dataSize;
StringLength = stringLength;
HeaderOverload = headerOverload;
}
private void loadDataFile(DataFileType type, string dataFilePostfix)
{
var filePath = Path.Combine(DATA_FILE_PATH, $"{type.ToString().ToLower() + dataFilePostfix}.{DATA_FILE_EXTENSION}");
var stream = new FileStream(filePath, FileMode.Open, FileSystemRights.Read, FileShare.Read, 4096, FileOptions.RandomAccess);
streams[type] = new DataFile {
Stream = stream, Length = new FileInfo(filePath).Length
};
}
private static List <T> GetDataFileConverted <T>(SaveGameFile savegame, DataFileType type)
{
var fileFacts = DataFileFacts.GetDataFileFacts().First(x => x.Type == type);
var bytes = DataFileLoaders.GetDataFileBytes(savegame, fileFacts.Type, fileFacts.DataSize);
var converter = ConverterFactory.CreateConverter <T>();
var collect = ConvertToCMObject <T>(bytes, converter).ToList();
return(collect);
}
public uint[] ReadInt24Array(DataFileType file, long offset, int size)
{
byte[] bytes = ReadBytes(file, offset, size * 3);
uint[] array = new uint[size];
for (var i = 0; i < size; i++)
{
array[i] = (uint)(bytes[i * 3] | bytes[i * 3 + 1] << 8 | bytes[i * 3 + 2] << 16);
}
return(array);
}
public Database AddDataFile(string dataFile, DataFileType dataType)
{
if (!File.Exists(dataFile))
{
Debug.LogWarning($"Database - Cannot find data file \"{dataFile}\".");
return(this);
}
dataFiles.Add(new Tuple <string, DataFileType>(dataFile, dataType));
Debug.Log($"Database - Added {dataType} data file \"{dataFile}\".");
return(this);
}
public Database AddDataFolder(string dataFolder, DataFileType dataType)
{
string dataPath = new DirectoryInfo(Application.dataPath).Parent.ToString();
DirectoryInfo directoryInfo = new DirectoryInfo(dataPath + "/" + dataFolder);
foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("*.json"))
{
AddDataFile($"{dataFolder}/{fileInfo.Name}", dataType);
}
return(this);
}
public void Show(Game game, Robot robot, DataFileType type, Func <int> getBytes)
{
_game = game;
Robot = robot;
Type = type;
_getBytes = getBytes;
_disposable = new CompositeDisposable();
Observable.EveryUpdate().Merge(Observable.Return(0L))
.Subscribe(_ => ((RectTransform)transform).sizeDelta = new Vector2(Mathf.FloorToInt(getBytes() * game.Template.MemoryIndicationScale), ((RectTransform)transform).sizeDelta.y))
.AddTo(_disposable);
}
private byte[] readBytes(DataFileType file, long offset, int count)
{
byte[] bytes = new byte[count];
streams[file].Stream.Position = offset;
int bytesRead = streams[file].Stream.Read(bytes, 0, count);
if (bytesRead < count)
{
LOGGER.Warning($"DataFileReader bytes requested: {count}, bytes read: {bytesRead}");
}
return(bytes);
}
private static bool LoadDataFile <T>(string dataFile, DataFileType dataType,
DatabaseCollection <T> existing) where T : DatabaseElement
{
// JSON formatting
string dataFileContent = JsonFormatter.Format(File.ReadAllText(dataFile));
if (dataFileContent == null)
{
Debug.LogWarning($"Database - Invalid {dataType} JSON format in \"{dataFile}\" data file.");
return(false);
}
// JSON parsing
DatabaseCollection <T> additions;
try {
additions = JsonUtility.FromJson <DatabaseCollection <T> >(dataFileContent);
} catch (ArgumentException e) {
Debug.LogError($"Database.LoadDataFile(\"{dataFile}\", {dataType}) : JSON error :\n{e.Message}");
Debug.LogError($"Formatted JSON :\n{dataFileContent}");
return(false);
}
if (additions == null)
{
Debug.LogWarning(
$"Database - Invalid {dataType} JSON in \"{dataFile}\" data file.");
return(false);
}
// Database processing
foreach (var element in additions.Collection)
{
if (existing.Collection.Any(e => element.Id == e.Id))
{
Debug.LogWarning(
$"Database - {dataType} element of ID \"{element.Id}\" already exists.");
continue;
}
if (existing.NamesAreUnique && existing.Collection.Any(e => element.Name == e.Name))
{
Debug.LogWarning(
$"Database - {dataType} element of name \"{element.Name}\" already exists (unique names activated).");
continue;
}
if (!element.IsValid())
{
Debug.LogWarning($"Database - {dataType} element of ID \"{element.Id}\" is invalid.");
continue;
}
existing.Collection.Add(element);
}
return(true);
}
public static void WriteDataFile <TModel>(DataFileType dataFileType, IEnumerable <TModel> model)
{
string jsonFilePath = HostingEnvironment.MapPath(string.Format(JsonPath, dataFileType.ToString().ToLower()));
if (string.IsNullOrWhiteSpace(jsonFilePath))
{
throw new ArgumentNullException();
}
string jsonDataToWrite = JsonConvert.SerializeObject(model, Formatting.Indented);
File.WriteAllText(jsonFilePath, jsonDataToWrite);
}
public static ITransferData GetUtil(DataFileType dataType)
{
switch (dataType)
{
case DataFileType.Csv: return(new CsvTransferData());
case DataFileType.Xls: return(new XlsTransferData());
case DataFileType.Xlsx: return(new XlsxTransferData());
default: return(new CsvTransferData());
}
}
private void loadDataFile(DataFileType type)
{
var filePath = GetDataPackFile() + type.ToString().ToLower()[0];
FileStream stream = null;
try {
stream = new FileStream(filePath, FileMode.Open, FileSystemRights.Read, FileShare.Read, 4096, FileOptions.RandomAccess);
} catch (Exception e) {
logger.Exception($"Could not load data file {filePath}", e);
}
streams[type] = new DataFile {
Stream = stream, Length = new FileInfo(filePath).Length
};
}
public static IEnumerable <TModel> GetDataFile <TModel>(DataFileType dataFileType)
{
string jsonFilePath = HostingEnvironment.MapPath(string.Format(JsonPath, dataFileType.ToString().ToLower()));
if (string.IsNullOrWhiteSpace(jsonFilePath))
{
throw new ArgumentNullException();
}
string jsonContent = File.ReadAllText(jsonFilePath);
var models = JsonConvert.DeserializeObject <IEnumerable <TModel> >(jsonContent);
return(models);
}
public Download(DateTime from, DateTime to, DataFileType dT)
{
// if from date is later than earlier
if (from > to)
{
fromDate = to; toDate = from;
}
else
{
fromDate = from;
toDate = to;
}
dType = dT;
isOK = true;
}
public static int GetBlockIndexFromFileName(string fileName, DataFileType type = DataFileType.ABI)
{
string name = Path.GetFileNameWithoutExtension(fileName);
string[] elements = name.Split(SplitCharacters, StringSplitOptions.RemoveEmptyEntries);
int index = -1;
switch (type)
{
case DataFileType.ABI:
case DataFileType.AHI:
if (elements.Length != 14)
{
throw new Exception(string.Format("Unsupported ABI or AHI file name format: {0}", fileName));
}
index = Convert.ToInt32(elements[3]);
break;
case DataFileType.VIIRSABI:
case DataFileType.VIIRSAHI:
if (elements.Length != 11)
{
throw new Exception(string.Format("Unsupported VIIRSABI or VIIRSAHI file name format: {0}", fileName));
}
index = Convert.ToInt32(elements[10]);
break;
case DataFileType.VIIRS1Day:
case DataFileType.VIIRS5Day:
if (elements.Length != 12)
{
throw new Exception(string.Format("Unsupported VIIRS1Day or VIIRS5Day file name format: {0}", fileName));
}
string dayStep = elements[10].ToLower();
if (dayStep != "001day" && dayStep != "005day")
{
throw new Exception(string.Format("Bad VIIRS1Day or VIIRS5Day file name format: {0}", fileName));
}
index = Convert.ToInt32(elements[11]);
break;
}
return(index);
}
public DataFileSample(byte[] bytes, DataFileType dataFileType, int analogCount, int digitalCount)
{
this.AnalogValues = new double[analogCount];
this.DigitalValues = new bool[digitalCount];
this.Number = System.BitConverter.ToInt32(bytes, 0);
this.Timestamp = System.BitConverter.ToInt32(bytes, 4);
int digitalByteStart;
if (dataFileType == DataFileType.Binary)
{
for (int i = 0; i < analogCount; i++)
{
this.AnalogValues[i] = System.BitConverter.ToInt16(bytes, 8 + i * 2);
}
digitalByteStart = 8 + 2 * analogCount;
}
else
{
if (dataFileType == DataFileType.Binary32)
{
for (int i = 0; i < analogCount; i++)
{//TODO add test
this.AnalogValues[i] = System.BitConverter.ToInt32(bytes, 8 + i * 4);
}
}
else if (dataFileType == DataFileType.Float32)
{
for (int i = 0; i < analogCount; i++)
{//TODO add test
this.AnalogValues[i] = System.BitConverter.ToSingle(bytes, 8 + i * 4);
}
}
digitalByteStart = 8 + 4 * analogCount;
}
int digitalByteCount = DataFileHandler.GetDigitalByteCount(digitalCount);
for (int i = 0; i < digitalCount; i++)
{
int digitalByteIterator = i / 8;
this.DigitalValues[i] = Convert.ToBoolean((bytes[digitalByteStart + digitalByteIterator] >> (i - digitalByteIterator * 8)) & 1);
}
}
private int FileTypeToNumber(DataFileType type)
{
switch (type)
{
case DataFileType.FormatConvert:
return(1);
case DataFileType.CoordinateConvert:
return(2);
case DataFileType.Projection:
return(3);
default:
return(0);
}
}
public static T[] LoadAllData <T>(DataFileType type, string nodeName) where T : Data
{
if (DataFiles.TryGetValue(type, out DataFile dataFile))
{
XmlNodeList nodeList = dataFile.GetNodes(nodeName);
T[] nodes = new T[nodeList.Count];
for (int i = 0; i < nodeList.Count; i++)
{
nodes[i] = Deserialize <T>(nodeList[i]);
}
return(nodes);
}
else
{
CrashHandler.ShowException(string.Format("Data of type {0} in node {1} does not exist.", type, nodeName));
return(null);
}
}
public static DateTime GetDateFromFileName(string fileName, DataFileType type = DataFileType.ABI)
{
string name = Path.GetFileNameWithoutExtension(fileName);
string[] elements = name.Split(SplitCharacters, StringSplitOptions.RemoveEmptyEntries);
DateTime date = new DateTime();
switch (type)
{
case DataFileType.ABI:
case DataFileType.AHI:
if (elements.Length != 14)
{
throw new Exception(string.Format("Unsupported ABI or AHI file name format: {0}", fileName));
}
date = new DateTime(Convert.ToInt32(elements[4].Substring(0, 4)), Convert.ToInt32(elements[4].Substring(4, 2)), Convert.ToInt32(elements[4].Substring(6, 2)));
break;
case DataFileType.VIIRSABI:
case DataFileType.VIIRSAHI:
if (elements.Length != 11)
{
throw new Exception(string.Format("Unsupported VIIRSABI or VIIRSAHI file name format: {0}", fileName));
}
date = new DateTime(Convert.ToInt32(elements[6].Substring(1, 4)), Convert.ToInt32(elements[6].Substring(5, 2)), Convert.ToInt32(elements[6].Substring(7, 2)));
break;
case DataFileType.VIIRS1Day:
case DataFileType.VIIRS5Day:
if (elements.Length != 12)
{
throw new Exception(string.Format("Unsupported VIIRS1Day or VIIRS5Day file name format: {0}", fileName));
}
date = new DateTime(Convert.ToInt32(elements[5].Substring(1, 4)), Convert.ToInt32(elements[5].Substring(5, 2)), Convert.ToInt32(elements[5].Substring(7, 2)));
break;
}
return(date);
}
public void GenerateSqlInsertCommands(string sourceDirectory)
{
string[] kmlFileNames = Directory.GetFiles(sourceDirectory, "*.kml");
char[] SplitCharacters = { '_', '.' };
DataFileType type = DataFileType.VIIRS1Day;
foreach (string kmlFileName in kmlFileNames)
{
string name = Path.GetFileNameWithoutExtension(kmlFileName);
string[] elements = name.Split(SplitCharacters, StringSplitOptions.RemoveEmptyEntries);
if (elements.Length == 12)
{
type = DataFileType.VIIRS1Day;
string dateString = elements[6].Substring(1, 4) + "-" + elements[6].Substring(5, 2) + "-" + elements[6].Substring(7, 2) + " " + elements[8].Substring(0, 2) + ":00:00";
string districtString = elements[11];
string sqlCommand = string.Format("INSERT INTO jpssflood.kmlmetadata (ProductId, RegionId, DistrictId, Date, FileName) VALUES(1, 1, {0}, '{1}', '{2}');", districtString, dateString, name);
Console.WriteLine(sqlCommand);
}
}
}
/// <summary>
/// Using a file's extension, will deduct the right file type
/// </summary>
/// <param name="filePath"></param>
public DataFileSource(string filePath)
{
this._filePath = filePath;
switch (Path.GetExtension(filePath).ToLower())
{
case ".txt":
case ".asc":
case ".csv":
FileType = DataFileType.TextComaDelimited;
break;
case ".mdb":
FileType = DataFileType.AccessMdb;
break;
// ReSharper disable StringLiteralTypo
case ".accdb":
FileType = DataFileType.Access2007;
break;
case ".xlsb":
FileType = DataFileType.Excel2007_Xlsb;
break;
case ".xlsm":
FileType = DataFileType.Excel2007_Xlsm;
break;
case ".xlsx":
FileType = DataFileType.Excel2007_Xlsx;
break;
case ".xls":
FileType = DataFileType.Excel;
break;
// ReSharper restore StringLiteralTypo
}
}
//public section
/// <summary>
/// LoadDataFile From File
/// </summary>
/// <param name="axMapControl">ref AxMapControl</param>
/// <param name="enumDataFileType">enumDataFileType</param>
/// <returns>status String</returns>
public SystemErrorType LoadDataFromFile(ref AxMapControl axMapControl, DataFileType enumDataFileType)
{
switch (enumDataFileType)
{
case DataFileType.enumAccessFile:
return(LoadAccessFile(ref axMapControl, UtilObject.getSingleFilePathByDlg("打开Access数据文件", "Access文件(*.mdb)|*.mdb")));
case DataFileType.enumAutoCADFile:
return(LoadCADFile(ref axMapControl, UtilObject.getSingleFilePathByDlg("打开CAD文件", "CAD数据(*.dwg)|*.dwg")));
case DataFileType.enumMXDMapFile:
return(LoadMapDoc(ref axMapControl, UtilObject.getSingleFilePathByDlg("打开Mxd地图文件", "地图文件(*.mxd;*.mxt;*.pmf)|*.mxd;*.mxt;*.pmf")));
case DataFileType.enumRasterFile:
return(LoadRasterFile(ref axMapControl, UtilObject.getSingleFilePathByDlg("打开栅格数据", "栅格数据(*.tif;*.bmp;*.jpg;*.tiff)|*.tif;*.bmp;*.jpg;*.tiff")));
case DataFileType.enumShapeFile:
return(LoadShapeFile(ref axMapControl, UtilObject.getSingleFilePathByDlg("打开shape文件", "ShapeFile(*.shp)|*.shp")));
default:
return(SystemErrorType.enumTypeNotExist);
}
}
public Data(List<TickDataModel> contractList, DataFileType fileType)
{
_contractList = contractList;
_fileType = fileType;
}
public Data(string source, DataFileType fileType)
{
_source = source;
_fileType = fileType;
}
