public static void WaitUntilFileCreated(string rootPath, FileNode fileNode, DataAdaptor<DMLibDataInfo> dataAdaptor, DMLibDataType dataType, int timeoutInSec = 300) { Func<bool> checkFileCreated = null; if (dataType == DMLibDataType.Local) { string filePath = dataAdaptor.GetAddress() + fileNode.GetLocalRelativePath(); checkFileCreated = () => { return File.Exists(filePath); }; } else if (dataType == DMLibDataType.PageBlob || dataType == DMLibDataType.AppendBlob) { CloudBlobDataAdaptor blobAdaptor = dataAdaptor as CloudBlobDataAdaptor; checkFileCreated = () => { CloudBlob cloudBlob = blobAdaptor.GetCloudBlobReference(rootPath, fileNode); return cloudBlob.Exists(options: HelperConst.DefaultBlobOptions); }; } else if (dataType == DMLibDataType.BlockBlob) { CloudBlobDataAdaptor blobAdaptor = dataAdaptor as CloudBlobDataAdaptor; checkFileCreated = () => { CloudBlockBlob blockBlob = blobAdaptor.GetCloudBlobReference(rootPath, fileNode) as CloudBlockBlob; try { return blockBlob.DownloadBlockList(BlockListingFilter.All, options: HelperConst.DefaultBlobOptions).Any(); } catch (StorageException) { return false; } }; } else if (dataType == DMLibDataType.CloudFile) { CloudFileDataAdaptor fileAdaptor = dataAdaptor as CloudFileDataAdaptor; checkFileCreated = () => { CloudFile cloudFile = fileAdaptor.GetCloudFileReference(rootPath, fileNode); return cloudFile.Exists(options: HelperConst.DefaultFileOptions); }; } else { Test.Error("Unexpected data type: {0}", DMLibTestContext.SourceType); } MultiDirectionTestHelper.WaitUntil(checkFileCreated, timeoutInSec); }
void Start() { Camera.main.transform.position = new Vector3(4.5f, 6f, -11.5f); Rawdata = DataAdaptor.ReadCSVFile("Assets/stacked_data_accident1.csv"); // reading csv file foreach (string item in Rawdata) { data.Add(float.Parse(item)); // converting data from string to float } columnCount = DataAdaptor.header.Split(',').Length; rowCount = data.Count / columnCount; findMax(); makeAxis(); instantiateBars(); }
/// <summary> /// Returns the dataset in JSON-stat format /// </summary> /// <param name="collection"></param> /// <returns></returns> private static JsonStatCollection GetJsonStatCollectionObject(List <dynamic> collection) { var theJsonStatCollection = new JsonStatCollection(); var theCollectionLink = new JsonStatCollectionLink() { Item = new List <Item>() }; theJsonStatCollection.Link = theCollectionLink; if (collection == null) { return(theJsonStatCollection); } if (collection.Count == 0) { return(theJsonStatCollection); } foreach (var element in collection) { var aItem = new Item() { Href = new Uri(string.Format("{0}/{1}/{2}", Configuration_BSO.GetCustomConfig("url.application"), Utility.GetCustomConfig("APP_COOKIELINK_TABLE"), element.MtrCode)), Class = ItemClass.Dataset, Label = element.MtrTitle, Updated = DataAdaptor.ConvertToString(element.RlsLiveDatetimeFrom), Extension = new Dictionary <string, object>() }; var Frequency = new { name = element.FrqValue, code = element.FrqCode }; aItem.Extension.Add("copyright", new { name = element.CprValue, code = element.CprCode, href = element.CprUrl }); aItem.Extension.Add("exceptional", element.ExceptionalFlag); aItem.Extension.Add("language", new { code = element.LngIsoCode, name = element.LngIsoName }); aItem.Extension.Add("matrix", element.MtrCode); aItem.Extension.Add("frequency", Frequency); theCollectionLink.Item.Add(aItem); } return(theJsonStatCollection); }
internal Workflow_DTO Populate(ADO ado, Workflow_DTO dto, string ccnUsername) { Workflow_ADO wAdo = new Workflow_ADO(); ADO_readerOutput reader = wAdo.Read(ado, dto, ccnUsername, "PUBLISH"); if (!reader.hasData) { return(dto); } return(new Workflow_DTO() { WrqDatetime = DataAdaptor.ReadDateTime(reader.data[0].WrqDatetime), WrqExceptionalFlag = DataAdaptor.ReadBool(reader.data[0].WrqExceptionalFlag), WrqReservationFlag = DataAdaptor.ReadBool(reader.data[0].WrqReservationFlag), WrqArchiveFlag = DataAdaptor.ReadBool(reader.data[0].WrqArchiveFlag), WrqExperimentalFlag = DataAdaptor.ReadBool(reader.data[0].WrqExperimentalFlag) }); }
public void UploadLargeObject_Nonseekable_PageBlob_Nagitive() { DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(DMLibDataType.Stream); DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(DMLibDataType.PageBlob); DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); sourceDataInfo.RootNode.AddFileNode(new FileNode($"{FileName}_{1}B") { SizeInByte = 1, }); sourceAdaptor.GenerateData(sourceDataInfo); destAdaptor.CreateIfNotExists(); List <TransferItem> uploadItems = new List <TransferItem>(); string fileName = $"{FileName}_{1}B"; FileNode fileNode = sourceDataInfo.RootNode.GetFileNode(fileName); uploadItems.Add(new TransferItem() { SourceObject = new DMLibTestStream(sourceAdaptor.GetTransferObject(string.Empty, fileNode) as FileStream, false, true), DestObject = destAdaptor.GetTransferObject(string.Empty, fileNode), SourceType = DMLibDataType.Stream, DestType = DMLibDataType.PageBlob, IsServiceCopy = false }); // Execution var result = this.RunTransferItems( uploadItems, new TestExecutionOptions <DMLibDataInfo>() { DisableDestinationFetch = true }); Test.Assert(result.Exceptions.Count == 1 && result.Exceptions[0].Message.Contains("must be a multiple of 512 bytes."), "Verify error is expected."); sourceAdaptor.Cleanup(); destAdaptor.Cleanup(); }
// Use this for initialization void Awake() { // Read the data from the file using our "generic" adaptor rawData = DataAdaptor.ReadCSVFile("Assets/circular_heatchart_data.csv"); // Parse the data from strings to floats foreach (string data in rawData) { energyData.Add(float.Parse(data)); } max = 0; min = energyData[0]; // Find the max and min of our array for (int i = 0; i < energyData.Count; i++) { if (energyData[i] > max) { max = energyData[i]; } if (energyData[i] < min) { min = energyData[i]; } } // Create a segment for each data point for (int i = 0; i < energyData.Count; i++) { int ringNumber = i / 12; Mesh segmentWedge = CustomMeshGenerator.Create3DToroidWedge(40, ringNumber + 1, ringNumber + 2, 30, 1); GameObject newSegment = new GameObject("Chart Segment" + (i + 1)); newSegment.AddComponent <MeshCollider>().sharedMesh = segmentWedge; newSegment.AddComponent <MeshFilter>().mesh = segmentWedge; newSegment.AddComponent <MeshRenderer>().material.color = PointToColor(energyData[i]); // localRotation cannot be a vector3, have to use Quaternion newSegment.transform.localRotation = Quaternion.Euler(0, 0, -30 * (i % 12)); } // Move the camera so we can see the whole chart transform.position = new Vector3(0f, 0f, -0.21f * energyData.Count); }
/// <summary> /// Get a JSON-stat metadata based on a database read of collections /// </summary> /// <param name="collection"></param> /// <param name="statistics"></param> /// <param name="classifications"></param> /// <returns></returns> private Item GetJsonStatRelease(List <dynamic> collection, List <dynamic> statistics, List <dynamic> classifications, List <dynamic> periods, List <Format_DTO_Read> formats) { var jsStat = new Item(); // jsStat.Class = Class.Dataset; //jsStat.Version = Version.The20; jsStat.Id = new List <string>(); jsStat.Size = new List <long>(); var thisItem = collection.FirstOrDefault(); jsStat.Extension = new Dictionary <string, object>(); var Frequency = new { name = thisItem.FrqValue, code = thisItem.FrqCode }; jsStat.Extension.Add("copyright", new { name = thisItem.CprValue, code = thisItem.CprCode, href = thisItem.CprUrl }); jsStat.Extension.Add("exceptional", thisItem.ExceptionalFlag); jsStat.Extension.Add("language", new { code = thisItem.LngIsoCode, name = thisItem.LngIsoName }); jsStat.Extension.Add("matrix", thisItem.MtrCode); Format_DTO_Read fDtoMain = formats.Where(x => x.FrmType == Constants.C_SYSTEM_JSON_STAT_NAME).FirstOrDefault();// new Format_DTO_Read() { FrmDirection = Utility.GetCustomConfig("APP_FORMAT_DOWNLOAD_NAME"), FrmType = Constants.C_SYSTEM_JSON_STAT_NAME }; jsStat.Href = new Uri(Configuration_BSO.GetCustomConfig("url.restful") + string.Format(Utility.GetCustomConfig("APP_RESTFUL_DATASET"), Utility.GetCustomConfig("APP_READ_DATASET_API"), thisItem.MtrCode, fDtoMain.FrmType, fDtoMain.FrmVersion, thisItem.LngIsoCode)); jsStat.Label = thisItem.MtrTitle; jsStat.Updated = DataAdaptor.ConvertToString(thisItem.RlsLiveDatetimeFrom); jsStat.Dimension = new Dictionary <string, Dimension>(); formats.Remove(fDtoMain); var link = new DimensionLink { Alternate = new List <Alternate>() }; foreach (var f in formats) { link.Alternate.Add(new Alternate() { Href = Configuration_BSO.GetCustomConfig("url.restful") + string.Format(Utility.GetCustomConfig("APP_RESTFUL_DATASET"), Utility.GetCustomConfig("APP_READ_DATASET_API"), thisItem.MtrCode, f.FrmType, f.FrmVersion, thisItem.LngIsoCode) }); } jsStat.Link = link; formats.Add(fDtoMain); var statDimension = new Dimension() { Label = Utility.GetCustomConfig("APP_CSV_STATISTIC"), Category = new Category() { } }; jsStat.Dimension.Add(Utility.GetCustomConfig("APP_CSV_STATISTIC"), statDimension); jsStat.Id.Add(Frequency.code); List <PeriodRecordDTO_Create> plist = new List <PeriodRecordDTO_Create>(); foreach (var per in periods) { plist.Add(new PeriodRecordDTO_Create() { Code = per.PrdCode, Value = per.PrdValue }); } var timeDimension = new Dimension() { Label = Frequency.name, Category = new Category() { Index = plist.Select(v => v.Code).ToList(), Label = plist.ToDictionary(v => v.Code, v => v.Value) } }; jsStat.Dimension.Add(Frequency.code, timeDimension); foreach (var s in classifications) { Dictionary <string, string> dict = new Dictionary <string, string>(); dict.Add(s.ClsCode, s.ClsValue); var clsDimension = new Dimension() { Label = s.ClsValue, Category = new Category() { } }; jsStat.Dimension.Add(s.ClsCode, clsDimension); } jsStat.Role = new Role(); jsStat.Role.Metric = new List <string> { Utility.GetCustomConfig("APP_CSV_STATISTIC") }; jsStat.Role.Time = new List <string> { thisItem.FrqCode }; //Values will be blank in this case, all we want is metadata //jsStat.Value = new JsonStatValue() { AnythingArray = new List<ValueElement>() }; return(jsStat); }
private void UploadFromStreamTest(DMLibDataType destType, long[] variedSourceLength, int?blockSize, bool seekable, bool fixedSized) { DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(DMLibDataType.Stream); DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(destType); DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); foreach (long fileSizeInByte in variedSourceLength) { sourceDataInfo.RootNode.AddFileNode(new FileNode($"{FileName}_{fileSizeInByte}") { SizeInByte = fileSizeInByte, }); } sourceAdaptor.GenerateData(sourceDataInfo); destAdaptor.CreateIfNotExists(); List <TransferItem> uploadItems = new List <TransferItem>(); foreach (long fileSizeInByte in variedSourceLength) { string fileName = $"{FileName}_{fileSizeInByte}"; FileNode fileNode = sourceDataInfo.RootNode.GetFileNode(fileName); uploadItems.Add(new TransferItem() { SourceObject = new DMLibTestStream(sourceAdaptor.GetTransferObject(string.Empty, fileNode) as FileStream, seekable, fixedSized), DestObject = destAdaptor.GetTransferObject(string.Empty, fileNode), SourceType = DMLibDataType.Stream, DestType = destType, IsServiceCopy = false }); } // Execution var result = this.RunTransferItems( uploadItems, new TestExecutionOptions <DMLibDataInfo>() { DisableDestinationFetch = true, BlockSize = blockSize.HasValue ? blockSize.Value : 4 * 1024 * 1024 }); if (!fixedSized && (destType == DMLibDataType.PageBlob || destType == DMLibDataType.CloudFile)) { Test.Assert(result.Exceptions.Count == 1 && result.Exceptions[0].Message.Contains("Source must be fixed size"), "Verify error is expected."); } else { // Verify all files are transfered successfully Test.Assert(result.Exceptions.Count == 0, "Verify no exception occurs."); DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty); foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes()) { FileNode sourceFileNode = sourceDataInfo.RootNode.GetFileNode(destFileNode.Name); Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result."); } } sourceAdaptor.Cleanup(); destAdaptor.Cleanup(); }
public Controller() { model = new MySqlData(); }
private void DownloadToMultipleStreams(DMLibDataType sourceType, long fileSize, int blockSize = 0) { DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(sourceType); DataAdaptor <DMLibDataInfo> destAdaptor1 = GetDestAdaptor(DMLibDataType.Stream); Test.Info("SourceType: {0}, filesize {1}, blocksize {2}", sourceType, fileSize, blockSize); DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); long fileSizeInByte = fileSize; sourceDataInfo.RootNode.AddFileNode(new FileNode(FileName) { SizeInByte = fileSizeInByte, }); sourceAdaptor.GenerateData(sourceDataInfo); destAdaptor1.CreateIfNotExists(); List <TransferItem> downloadItems = new List <TransferItem>(); FileNode fileNode = sourceDataInfo.RootNode.GetFileNode(FileName); int streamCount = random.Next(2, 6); for (int i = 0; i < streamCount; ++i) { downloadItems.Add(new TransferItem() { SourceObject = sourceAdaptor.GetTransferObject(string.Empty, fileNode), DestObject = (destAdaptor1 as LocalDataAdaptor).GetTransferObject(string.Empty, $"{FileName}_{i}"), SourceType = sourceType, DestType = DMLibDataType.Stream, IsServiceCopy = false }); } var options = new TestExecutionOptions <DMLibDataInfo>() { DisableDestinationFetch = true, }; if (0 != blockSize) { options.BlockSize = blockSize; } // Execution var result = this.RunTransferItems( downloadItems, options); // Verify all files are transfered successfully Test.Assert(result.Exceptions.Count == 0, "Verify no exception occurs."); DMLibDataInfo destDataInfo = destAdaptor1.GetTransferDataInfo(string.Empty); foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes()) { var toBeValidateFileNode = destFileNode.Clone(FileName); FileNode sourceFileNode = sourceDataInfo.RootNode.GetFileNode(FileName); Test.Assert(DMLibDataHelper.Equals(sourceFileNode, toBeValidateFileNode), "Verify transfer result."); } sourceAdaptor.Cleanup(); destAdaptor1.Cleanup(); }
public static void SetCalculatedFileMD5(DMLibDataInfo dataInfo, DataAdaptor<DMLibDataInfo> destAdaptor, bool disableMD5Check = false) { foreach (FileNode fileNode in dataInfo.EnumerateFileNodes()) { if (DMLibTestBase.IsCloudBlob(DMLibTestContext.DestType)) { CloudBlobDataAdaptor cloudBlobDataAdaptor = destAdaptor as CloudBlobDataAdaptor; CloudBlob cloudBlob = cloudBlobDataAdaptor.GetCloudBlobReference(dataInfo.RootPath, fileNode); fileNode.MD5 = CloudBlobHelper.CalculateMD5ByDownloading(cloudBlob, disableMD5Check); } else if (DMLibTestContext.DestType == DMLibDataType.CloudFile) { CloudFileDataAdaptor cloudFileDataAdaptor = destAdaptor as CloudFileDataAdaptor; CloudFile cloudFile = cloudFileDataAdaptor.GetCloudFileReference(dataInfo.RootPath, fileNode); fileNode.MD5 = CloudFileHelper.CalculateMD5ByDownloading(cloudFile, disableMD5Check); } // No need to set md5 for local destination } }