public static void WaitUntilFileCreated(string rootPath, FileNode fileNode, DataAdaptor<DMLibDataInfo> dataAdaptor, DMLibDataType dataType, int timeoutInSec = 300)
{
Func<bool> checkFileCreated = null;
if (dataType == DMLibDataType.Local)
{
string filePath = dataAdaptor.GetAddress() + fileNode.GetLocalRelativePath();
checkFileCreated = () =>
{
return File.Exists(filePath);
};
}
else if (dataType == DMLibDataType.PageBlob ||
dataType == DMLibDataType.AppendBlob)
{
CloudBlobDataAdaptor blobAdaptor = dataAdaptor as CloudBlobDataAdaptor;
checkFileCreated = () =>
{
CloudBlob cloudBlob = blobAdaptor.GetCloudBlobReference(rootPath, fileNode);
return cloudBlob.Exists(options: HelperConst.DefaultBlobOptions);
};
}
else if (dataType == DMLibDataType.BlockBlob)
{
CloudBlobDataAdaptor blobAdaptor = dataAdaptor as CloudBlobDataAdaptor;
checkFileCreated = () =>
{
CloudBlockBlob blockBlob = blobAdaptor.GetCloudBlobReference(rootPath, fileNode) as CloudBlockBlob;
try
{
return blockBlob.DownloadBlockList(BlockListingFilter.All, options: HelperConst.DefaultBlobOptions).Any();
}
catch (StorageException)
{
return false;
}
};
}
else if (dataType == DMLibDataType.CloudFile)
{
CloudFileDataAdaptor fileAdaptor = dataAdaptor as CloudFileDataAdaptor;
checkFileCreated = () =>
{
CloudFile cloudFile = fileAdaptor.GetCloudFileReference(rootPath, fileNode);
return cloudFile.Exists(options: HelperConst.DefaultFileOptions);
};
}
else
{
Test.Error("Unexpected data type: {0}", DMLibTestContext.SourceType);
}
MultiDirectionTestHelper.WaitUntil(checkFileCreated, timeoutInSec);
}
void Start()
{
Camera.main.transform.position = new Vector3(4.5f, 6f, -11.5f);
Rawdata = DataAdaptor.ReadCSVFile("Assets/stacked_data_accident1.csv"); // reading csv file
foreach (string item in Rawdata)
{
data.Add(float.Parse(item)); // converting data from string to float
}
columnCount = DataAdaptor.header.Split(',').Length;
rowCount = data.Count / columnCount;
findMax();
makeAxis();
instantiateBars();
}
/// <summary>
/// Returns the dataset in JSON-stat format
/// </summary>
/// <param name="collection"></param>
/// <returns></returns>
private static JsonStatCollection GetJsonStatCollectionObject(List <dynamic> collection)
{
var theJsonStatCollection = new JsonStatCollection();
var theCollectionLink = new JsonStatCollectionLink()
{
Item = new List <Item>()
};
theJsonStatCollection.Link = theCollectionLink;
if (collection == null)
{
return(theJsonStatCollection);
}
if (collection.Count == 0)
{
return(theJsonStatCollection);
}
foreach (var element in collection)
{
var aItem = new Item()
{
Href = new Uri(string.Format("{0}/{1}/{2}", Configuration_BSO.GetCustomConfig("url.application"), Utility.GetCustomConfig("APP_COOKIELINK_TABLE"), element.MtrCode)),
Class = ItemClass.Dataset,
Label = element.MtrTitle,
Updated = DataAdaptor.ConvertToString(element.RlsLiveDatetimeFrom),
Extension = new Dictionary <string, object>()
};
var Frequency = new { name = element.FrqValue, code = element.FrqCode };
aItem.Extension.Add("copyright", new { name = element.CprValue, code = element.CprCode, href = element.CprUrl });
aItem.Extension.Add("exceptional", element.ExceptionalFlag);
aItem.Extension.Add("language", new { code = element.LngIsoCode, name = element.LngIsoName });
aItem.Extension.Add("matrix", element.MtrCode);
aItem.Extension.Add("frequency", Frequency);
theCollectionLink.Item.Add(aItem);
}
return(theJsonStatCollection);
}
internal Workflow_DTO Populate(ADO ado, Workflow_DTO dto, string ccnUsername)
{
Workflow_ADO wAdo = new Workflow_ADO();
ADO_readerOutput reader = wAdo.Read(ado, dto, ccnUsername, "PUBLISH");
if (!reader.hasData)
{
return(dto);
}
return(new Workflow_DTO()
{
WrqDatetime = DataAdaptor.ReadDateTime(reader.data[0].WrqDatetime),
WrqExceptionalFlag = DataAdaptor.ReadBool(reader.data[0].WrqExceptionalFlag),
WrqReservationFlag = DataAdaptor.ReadBool(reader.data[0].WrqReservationFlag),
WrqArchiveFlag = DataAdaptor.ReadBool(reader.data[0].WrqArchiveFlag),
WrqExperimentalFlag = DataAdaptor.ReadBool(reader.data[0].WrqExperimentalFlag)
});
}
public void UploadLargeObject_Nonseekable_PageBlob_Nagitive()
{
DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(DMLibDataType.Stream);
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(DMLibDataType.PageBlob);
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
sourceDataInfo.RootNode.AddFileNode(new FileNode($"{FileName}_{1}B")
{
SizeInByte = 1,
});
sourceAdaptor.GenerateData(sourceDataInfo);
destAdaptor.CreateIfNotExists();
List <TransferItem> uploadItems = new List <TransferItem>();
string fileName = $"{FileName}_{1}B";
FileNode fileNode = sourceDataInfo.RootNode.GetFileNode(fileName);
uploadItems.Add(new TransferItem()
{
SourceObject = new DMLibTestStream(sourceAdaptor.GetTransferObject(string.Empty, fileNode) as FileStream, false, true),
DestObject = destAdaptor.GetTransferObject(string.Empty, fileNode),
SourceType = DMLibDataType.Stream,
DestType = DMLibDataType.PageBlob,
IsServiceCopy = false
});
// Execution
var result = this.RunTransferItems(
uploadItems,
new TestExecutionOptions <DMLibDataInfo>()
{
DisableDestinationFetch = true
});
Test.Assert(result.Exceptions.Count == 1 && result.Exceptions[0].Message.Contains("must be a multiple of 512 bytes."), "Verify error is expected.");
sourceAdaptor.Cleanup();
destAdaptor.Cleanup();
}
// Use this for initialization
void Awake()
{
// Read the data from the file using our "generic" adaptor
rawData = DataAdaptor.ReadCSVFile("Assets/circular_heatchart_data.csv");
// Parse the data from strings to floats
foreach (string data in rawData)
{
energyData.Add(float.Parse(data));
}
max = 0;
min = energyData[0];
// Find the max and min of our array
for (int i = 0; i < energyData.Count; i++)
{
if (energyData[i] > max)
{
max = energyData[i];
}
if (energyData[i] < min)
{
min = energyData[i];
}
}
// Create a segment for each data point
for (int i = 0; i < energyData.Count; i++)
{
int ringNumber = i / 12;
Mesh segmentWedge = CustomMeshGenerator.Create3DToroidWedge(40, ringNumber + 1, ringNumber + 2, 30, 1);
GameObject newSegment = new GameObject("Chart Segment" + (i + 1));
newSegment.AddComponent <MeshCollider>().sharedMesh = segmentWedge;
newSegment.AddComponent <MeshFilter>().mesh = segmentWedge;
newSegment.AddComponent <MeshRenderer>().material.color = PointToColor(energyData[i]);
// localRotation cannot be a vector3, have to use Quaternion
newSegment.transform.localRotation = Quaternion.Euler(0, 0, -30 * (i % 12));
}
// Move the camera so we can see the whole chart
transform.position = new Vector3(0f, 0f, -0.21f * energyData.Count);
}
/// <summary>
/// Get a JSON-stat metadata based on a database read of collections
/// </summary>
/// <param name="collection"></param>
/// <param name="statistics"></param>
/// <param name="classifications"></param>
/// <returns></returns>
private Item GetJsonStatRelease(List <dynamic> collection, List <dynamic> statistics, List <dynamic> classifications, List <dynamic> periods, List <Format_DTO_Read> formats)
{
var jsStat = new Item();
// jsStat.Class = Class.Dataset;
//jsStat.Version = Version.The20;
jsStat.Id = new List <string>();
jsStat.Size = new List <long>();
var thisItem = collection.FirstOrDefault();
jsStat.Extension = new Dictionary <string, object>();
var Frequency = new { name = thisItem.FrqValue, code = thisItem.FrqCode };
jsStat.Extension.Add("copyright", new { name = thisItem.CprValue, code = thisItem.CprCode, href = thisItem.CprUrl });
jsStat.Extension.Add("exceptional", thisItem.ExceptionalFlag);
jsStat.Extension.Add("language", new { code = thisItem.LngIsoCode, name = thisItem.LngIsoName });
jsStat.Extension.Add("matrix", thisItem.MtrCode);
Format_DTO_Read fDtoMain = formats.Where(x => x.FrmType == Constants.C_SYSTEM_JSON_STAT_NAME).FirstOrDefault();// new Format_DTO_Read() { FrmDirection = Utility.GetCustomConfig("APP_FORMAT_DOWNLOAD_NAME"), FrmType = Constants.C_SYSTEM_JSON_STAT_NAME };
jsStat.Href = new Uri(Configuration_BSO.GetCustomConfig("url.restful") + string.Format(Utility.GetCustomConfig("APP_RESTFUL_DATASET"), Utility.GetCustomConfig("APP_READ_DATASET_API"), thisItem.MtrCode, fDtoMain.FrmType, fDtoMain.FrmVersion, thisItem.LngIsoCode));
jsStat.Label = thisItem.MtrTitle;
jsStat.Updated = DataAdaptor.ConvertToString(thisItem.RlsLiveDatetimeFrom);
jsStat.Dimension = new Dictionary <string, Dimension>();
formats.Remove(fDtoMain);
var link = new DimensionLink
{
Alternate = new List <Alternate>()
};
foreach (var f in formats)
{
link.Alternate.Add(new Alternate()
{
Href = Configuration_BSO.GetCustomConfig("url.restful") + string.Format(Utility.GetCustomConfig("APP_RESTFUL_DATASET"), Utility.GetCustomConfig("APP_READ_DATASET_API"), thisItem.MtrCode, f.FrmType, f.FrmVersion, thisItem.LngIsoCode)
});
}
jsStat.Link = link;
formats.Add(fDtoMain);
var statDimension = new Dimension()
{
Label = Utility.GetCustomConfig("APP_CSV_STATISTIC"),
Category = new Category()
{
}
};
jsStat.Dimension.Add(Utility.GetCustomConfig("APP_CSV_STATISTIC"), statDimension);
jsStat.Id.Add(Frequency.code);
List <PeriodRecordDTO_Create> plist = new List <PeriodRecordDTO_Create>();
foreach (var per in periods)
{
plist.Add(new PeriodRecordDTO_Create()
{
Code = per.PrdCode, Value = per.PrdValue
});
}
var timeDimension = new Dimension()
{
Label = Frequency.name,
Category = new Category()
{
Index = plist.Select(v => v.Code).ToList(),
Label = plist.ToDictionary(v => v.Code, v => v.Value)
}
};
jsStat.Dimension.Add(Frequency.code, timeDimension);
foreach (var s in classifications)
{
Dictionary <string, string> dict = new Dictionary <string, string>();
dict.Add(s.ClsCode, s.ClsValue);
var clsDimension = new Dimension()
{
Label = s.ClsValue,
Category = new Category()
{
}
};
jsStat.Dimension.Add(s.ClsCode, clsDimension);
}
jsStat.Role = new Role();
jsStat.Role.Metric = new List <string>
{
Utility.GetCustomConfig("APP_CSV_STATISTIC")
};
jsStat.Role.Time = new List <string>
{
thisItem.FrqCode
};
//Values will be blank in this case, all we want is metadata
//jsStat.Value = new JsonStatValue() { AnythingArray = new List<ValueElement>() };
return(jsStat);
}
private void UploadFromStreamTest(DMLibDataType destType, long[] variedSourceLength, int?blockSize, bool seekable, bool fixedSized)
{
DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(DMLibDataType.Stream);
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(destType);
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
foreach (long fileSizeInByte in variedSourceLength)
{
sourceDataInfo.RootNode.AddFileNode(new FileNode($"{FileName}_{fileSizeInByte}")
{
SizeInByte = fileSizeInByte,
});
}
sourceAdaptor.GenerateData(sourceDataInfo);
destAdaptor.CreateIfNotExists();
List <TransferItem> uploadItems = new List <TransferItem>();
foreach (long fileSizeInByte in variedSourceLength)
{
string fileName = $"{FileName}_{fileSizeInByte}";
FileNode fileNode = sourceDataInfo.RootNode.GetFileNode(fileName);
uploadItems.Add(new TransferItem()
{
SourceObject = new DMLibTestStream(sourceAdaptor.GetTransferObject(string.Empty, fileNode) as FileStream, seekable, fixedSized),
DestObject = destAdaptor.GetTransferObject(string.Empty, fileNode),
SourceType = DMLibDataType.Stream,
DestType = destType,
IsServiceCopy = false
});
}
// Execution
var result = this.RunTransferItems(
uploadItems,
new TestExecutionOptions <DMLibDataInfo>()
{
DisableDestinationFetch = true,
BlockSize = blockSize.HasValue ? blockSize.Value : 4 * 1024 * 1024
});
if (!fixedSized && (destType == DMLibDataType.PageBlob || destType == DMLibDataType.CloudFile))
{
Test.Assert(result.Exceptions.Count == 1 && result.Exceptions[0].Message.Contains("Source must be fixed size"), "Verify error is expected.");
}
else
{
// Verify all files are transfered successfully
Test.Assert(result.Exceptions.Count == 0, "Verify no exception occurs.");
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
FileNode sourceFileNode = sourceDataInfo.RootNode.GetFileNode(destFileNode.Name);
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
}
sourceAdaptor.Cleanup();
destAdaptor.Cleanup();
}
public Controller()
{
model = new MySqlData();
}
private void DownloadToMultipleStreams(DMLibDataType sourceType, long fileSize, int blockSize = 0)
{
DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(sourceType);
DataAdaptor <DMLibDataInfo> destAdaptor1 = GetDestAdaptor(DMLibDataType.Stream);
Test.Info("SourceType: {0}, filesize {1}, blocksize {2}", sourceType, fileSize, blockSize);
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
long fileSizeInByte = fileSize;
sourceDataInfo.RootNode.AddFileNode(new FileNode(FileName)
{
SizeInByte = fileSizeInByte,
});
sourceAdaptor.GenerateData(sourceDataInfo);
destAdaptor1.CreateIfNotExists();
List <TransferItem> downloadItems = new List <TransferItem>();
FileNode fileNode = sourceDataInfo.RootNode.GetFileNode(FileName);
int streamCount = random.Next(2, 6);
for (int i = 0; i < streamCount; ++i)
{
downloadItems.Add(new TransferItem()
{
SourceObject = sourceAdaptor.GetTransferObject(string.Empty, fileNode),
DestObject = (destAdaptor1 as LocalDataAdaptor).GetTransferObject(string.Empty, $"{FileName}_{i}"),
SourceType = sourceType,
DestType = DMLibDataType.Stream,
IsServiceCopy = false
});
}
var options = new TestExecutionOptions <DMLibDataInfo>()
{
DisableDestinationFetch = true,
};
if (0 != blockSize)
{
options.BlockSize = blockSize;
}
// Execution
var result = this.RunTransferItems(
downloadItems,
options);
// Verify all files are transfered successfully
Test.Assert(result.Exceptions.Count == 0, "Verify no exception occurs.");
DMLibDataInfo destDataInfo = destAdaptor1.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
var toBeValidateFileNode = destFileNode.Clone(FileName);
FileNode sourceFileNode = sourceDataInfo.RootNode.GetFileNode(FileName);
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, toBeValidateFileNode), "Verify transfer result.");
}
sourceAdaptor.Cleanup();
destAdaptor1.Cleanup();
}
public static void SetCalculatedFileMD5(DMLibDataInfo dataInfo, DataAdaptor<DMLibDataInfo> destAdaptor, bool disableMD5Check = false)
{
foreach (FileNode fileNode in dataInfo.EnumerateFileNodes())
{
if (DMLibTestBase.IsCloudBlob(DMLibTestContext.DestType))
{
CloudBlobDataAdaptor cloudBlobDataAdaptor = destAdaptor as CloudBlobDataAdaptor;
CloudBlob cloudBlob = cloudBlobDataAdaptor.GetCloudBlobReference(dataInfo.RootPath, fileNode);
fileNode.MD5 = CloudBlobHelper.CalculateMD5ByDownloading(cloudBlob, disableMD5Check);
}
else if (DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
CloudFileDataAdaptor cloudFileDataAdaptor = destAdaptor as CloudFileDataAdaptor;
CloudFile cloudFile = cloudFileDataAdaptor.GetCloudFileReference(dataInfo.RootPath, fileNode);
fileNode.MD5 = CloudFileHelper.CalculateMD5ByDownloading(cloudFile, disableMD5Check);
}
// No need to set md5 for local destination
}
}
