public async Task <string[]> UnloadToLocalFolder(PivotRequest request, string destinationFolder)
{
var unloadResult = await Unload(request);
var files = unloadResult.Files.ToArray();
var fileNames = new string[files.Length];
var urls = new string[files.Length];
var fileStreams = new Stream[files.Length];
bool succeeded = false;
try
{
for (var i = 0; i < files.Length; i++)
{
var file = files[i];
fileStreams[i] = File.Create(Path.Combine(destinationFolder, file.Name));
fileNames[i] = file.Name;
urls[i] = file.Url;
}
var copyTasks = await Task.WhenAll(GetFilesAfterUnload(urls, fileStreams));
await Task.WhenAll(copyTasks);
succeeded = true;
}
finally
{
if (fileStreams != null)
{
for (var i = 0; i < fileStreams.Length; i++)
{
if (fileStreams[i] != null)
{
fileStreams[i].Dispose();
}
}
}
if (!succeeded && fileNames != null)
{
for (var i = 0; i < fileNames.Length; i++)
{
if (!string.IsNullOrEmpty(fileNames[i]))
{
try
{
File.Delete(destinationFolder + '\\' + fileNames[i]);
}
catch { }
}
}
fileNames = null;
}
}
return(fileNames);
}
public PivotLevel ComputePivot(OrQuery filter, IEnumerable <int> axis, IEnumerable <int> values)
{
if (filter == null)
{
throw new ArgumentNullException(nameof(filter));
}
var request = new PivotRequest(filter);
request.AxisList.AddRange(axis);
request.ValuesList.AddRange(values);
var response = Channel.SendRequest(request);
if (response is ExceptionResponse exResponse)
{
throw new CacheException("Error in ComputePivot", exResponse.Message, exResponse.CallStack);
}
if (!(response is PivotResponse pivotResponse))
{
throw new CacheException("Invalid type of response received in ComputePivot()");
}
return(pivotResponse.Root);
}
private async Task <DatasetUnloadTaskResultData> Unload(PivotRequest request, int spinDelayInSeconds = 10, int maxWaitCount = 360)
{
var unloadTaskResponse = await StartUnload(request);
var taskResult = await WaitTaskResult <DatasetUnloadTaskResult>(unloadTaskResponse, spinDelayInSeconds, maxWaitCount);
return(taskResult.Data);
}
public void GetDataBegin()
{
var client = new Client("knoema.com");
var request = new PivotRequest
{
Dataset = "fzoaozc",
Header = new List <PivotRequestItem> {
new PivotRequestItem {
DimensionId = "Time", UiMode = "AllData"
}
},
Stub = new List <PivotRequestItem> {
new PivotRequestItem {
DimensionId = "country", Members = new List <object> {
1000010
}
}
},
Filter = new List <PivotRequestItem> {
new PivotRequestItem {
DimensionId = "indicator", Members = new List <object> {
1000000
}
}
},
Frequencies = new List <string> {
"A"
},
DetailColumns = new[] { "*" }
};
var result = client.GetDataBegin(request).GetAwaiter().GetResult();
Assert.IsNotNull(result.Descriptor);
Assert.AreEqual(2, result.Descriptor.DetailColumns.Count());
Assert.AreEqual("EndPeriod", result.Descriptor.DetailColumns.ElementAt(0).Name);
var detailColumns = result.Descriptor.DetailColumns.Select(c => c.Name).OrderBy(c => c).ToArray();
var detailValues = result.Data.First().DetailValues;
var valuesColumns = detailValues.Keys.OrderBy(k => k).ToArray();
Assert.IsTrue(Enumerable.SequenceEqual(detailColumns, valuesColumns));
Assert.AreEqual(29, detailValues["EndPeriod"].Count());
Assert.AreEqual(29, detailValues["Annotation"].Count());
}
private async IAsyncEnumerable <T> GetDataStreamingResponseAsync <T>(PivotRequest pivot)
{
var response = await ApiPost <StreamingDataResponse <T> >("/api/1.0/data/raw/", pivot);
foreach (var dataItem in response.Data)
{
yield return(dataItem);
}
var token = response.ContinuationToken;
while (!string.IsNullOrEmpty(token))
{
response = await ApiGet <StreamingDataResponse <T> >("/api/1.0/data/raw/", new Dictionary <string, string> {
{ "continuationToken", token }
});
foreach (var dataItem in response.Data)
{
yield return(dataItem);
}
token = response.ContinuationToken;
}
}
private Task <TaskResponse> StartUnload(PivotRequest request)
{
return(ApiPost <TaskResponse>("/api/1.0/data/unload", request));
}
public IAsyncEnumerable <FlatTimeSeriesRawData> GetFlatDataStreamingAsync(PivotRequest pivot)
{
return(GetDataStreamingResponseAsync <FlatTimeSeriesRawData>(pivot));
}
public Task <FlatTimeSeriesRawDataResponse> GetFlatDataBegin(PivotRequest pivot)
{
return(ApiPost <FlatTimeSeriesRawDataResponse>("/api/1.0/data/raw/", pivot));
}
public Task <PivotResponse> GetData(PivotRequest pivot)
{
return(ApiPost <PivotResponse>("/api/1.0/data/pivot/", pivot));
}
public Task <RegularTimeSeriesRawDataResponse> GetDataBegin(PivotRequest pivot)
{
return(ApiPost <RegularTimeSeriesRawDataResponse>("/api/1.0/data/raw/", pivot, 600000)); // 10 minutes timeout
}
public void UnloadData()
{
var clientId = ConfigurationManager.AppSettings["ClientId"];
if (string.IsNullOrEmpty(clientId))
{
return;
}
var client = new Client("knoema.org", clientId, "");
var request = new PivotRequest()
{
Dataset = "COMTRADE2015R1",
Frequencies = new List <string> {
"A"
}
};
request.Header.Add(new PivotRequestItem
{
DimensionId = "Time",
Members = { "2010-2011" },
UiMode = "range",
});
request.Stub.Add(new PivotRequestItem
{
DimensionId = "reporter",
Members = { 1000000, 1000100 }
});
request.Stub.Add(new PivotRequestItem
{
DimensionId = "indicator",
Members = { 1000010, 1000020 }
});
request.Stub.Add(new PivotRequestItem
{
DimensionId = "partner",
Members = { 1000000, 1000100 }
});
request.Stub.Add(new PivotRequestItem
{
DimensionId = "commodity",
Members = { 1000110, 1000100 }
});
var tempFolder = Path.GetTempPath() + "KNunload_" + Guid.NewGuid().ToString();
Directory.CreateDirectory(tempFolder);
try
{
var files = client.UnloadToLocalFolder(request, tempFolder).GetAwaiter().GetResult();
if (files != null)
{
for (var i = 0; i < files.Length; i++)
{
var fileName = tempFolder + '\\' + files[i];
if (File.Exists(fileName))
{
try
{
File.Delete(fileName);
}
catch (Exception)
{
}
}
}
}
}
finally
{
Directory.Delete(tempFolder, recursive: true);
}
}
public PivotRequest GetRequest()
{
var request = new PivotRequest();
request.Dataset = _dataset.Id;
if (_frequencies != null)
{
request.Frequencies = _frequencies.ToList();
}
var timeItem = new PivotRequestItem {
DimensionId = "Time"
};
if (string.IsNullOrEmpty(TimeRange))
{
timeItem.UiMode = "allData";
timeItem.Members = null;
}
else
{
timeItem.UiMode = "range";
timeItem.Members.Add(TimeRange);
}
request.Header.Add(timeItem);
foreach (var dimPair in _dimensions)
{
var dimId = dimPair.Key;
var dimKeys = dimPair.Value;
if (dimKeys == null || dimKeys.Count == 0)
{
continue;
}
var dimMeta = _client.GetDatasetDimension(_dataset.Id, dimId).GetAwaiter().GetResult();
object idVal;
string keyVal;
foreach (var member in dimMeta.Items)
{
if (member.Fields != null && member.Fields.TryGetValue("id", out idVal) && idVal != null && dimKeys.ContainsKey(idVal.ToString()))
{
keyVal = idVal.ToString();
}
else if (dimKeys.ContainsKey(member.Name))
{
keyVal = member.Name;
}
else if (dimKeys.ContainsKey(member.Key.ToString()))
{
keyVal = member.Key.ToString();
}
else
{
keyVal = null;
}
if (!string.IsNullOrEmpty(keyVal))
{
dimKeys[keyVal] = member.Key;
}
}
PivotRequestItem dimItem = null;
foreach (var memberKey in dimKeys.Values)
{
if (memberKey != 0)
{
if (dimItem == null)
{
dimItem = new PivotRequestItem {
DimensionId = dimId
};
request.Stub.Add(dimItem);
}
dimItem.Members.Add(memberKey);
}
}
}
return(request);
}
public async Task GetDataBeginExternalDataset()
{
var appId = ConfigurationManager.AppSettings["AppId"];
if (string.IsNullOrEmpty(appId))
{
throw new ArgumentException("Please add app id to configuration");
}
var appSecret = ConfigurationManager.AppSettings["AppSecret"];
if (string.IsNullOrEmpty(appSecret))
{
throw new ArgumentException("Please add app secret to configuration");
}
var host = ConfigurationManager.AppSettings["HostWithExternalDataset"];
if (string.IsNullOrEmpty(host))
{
throw new ArgumentException("Please add HostWithExternalDataset to configuration");
}
var datasetId = ConfigurationManager.AppSettings["ExternalDatasetId"];
if (string.IsNullOrEmpty(datasetId))
{
throw new ArgumentException("Please add ExternalDatasetId to configuration");
}
var client = new Client(host, appId, appSecret);
var request = new PivotRequest
{
Dataset = datasetId,
Header = new List <PivotRequestItem> {
new PivotRequestItem {
DimensionId = "Time", UiMode = "AllData"
}
},
Stub = new List <PivotRequestItem>
{
new PivotRequestItem {
DimensionId = "INDICATOR", Members = new List <object> {
"Number Of Applications Completed"
}
},
},
Filter = new List <PivotRequestItem>
{
new PivotRequestItem {
DimensionId = "PRODUCTNAME", Members = new List <object> {
"Universal Life"
}
},
new PivotRequestItem {
DimensionId = "RESIDENCYSTATE", Members = new List <object> {
"NA"
}
},
new PivotRequestItem {
DimensionId = "AGEGROUP", Members = new List <object> {
"40-44"
}
},
new PivotRequestItem {
DimensionId = "GENDER", Members = new List <object> {
"FEMALE"
}
},
new PivotRequestItem {
DimensionId = "HOUSEHOLDINCOME", Members = new List <object> {
"Over $250,000"
}
},
new PivotRequestItem {
DimensionId = "RISKCLASS", Members = new List <object> {
"NA"
}
}
},
Frequencies = new List <string> {
"D"
},
DetailColumns = new[] { "*" }
};
var result = await client.GetDataBegin(request);
Assert.IsTrue(result.Data.Any());
}
static void Main(string[] args)
{
var client = new Knoema.Client(
ConfigurationManager.AppSettings["host"], ConfigurationManager.AppSettings["clientId"], ConfigurationManager.AppSettings["appSecret"]);
Console.WriteLine("Getting dataset metadata...");
client.GetDataset("gquvbhe").ContinueWith(d =>
{
Console.WriteLine("Dataset has {0} dimensions: {1}.", d.Result.Dimensions.Count(), string.Join(", ", d.Result.Dimensions.Select(x => x.Name).ToArray()));
var stub = new List <PivotRequestItem>();
foreach (var dim in d.Result.Dimensions)
{
Console.WriteLine(string.Format("Getting \"{0}\" dataset dimension details...", dim.Name));
client.GetDatasetDimension(d.Result.Id, dim.Id).ContinueWith(t =>
{
Console.WriteLine(string.Format("Dimension \"{0}\" has {1} members: {2}.",
t.Result.Name, t.Result.Items.Count(), string.Join(", ", t.Result.Items.Select(x => x.Name.ToString()).ToArray())));
stub.Add(new PivotRequestItem()
{
DimensionId = t.Result.Id,
Members = t.Result.Items.Where(x => x.HasData).Select(x => x.Key).Take(10).Cast <object>().ToList()
});
}).Wait();
}
var header = new PivotRequestItem()
{
DimensionId = "Time"
};
for (var i = 1990; i < 2010; i++)
{
header.Members.Add(i);
}
Console.WriteLine("Getting dataset data...");
var pivotRequest = new PivotRequest()
{
Dataset = "gquvbhe",
Frequencies = new List <string>()
{
"A"
},
Stub = stub,
Header = new List <PivotRequestItem>()
{
header
}
};
client.GetData(pivotRequest).ContinueWith(t =>
{
t.Result.Tuples.Take(10).ToList().ForEach(x =>
Console.WriteLine(string.Join(", ", x.Select(y => y.Key + ":" + y.Value).ToList()))
);
Console.WriteLine(string.Format("{0} rows of data have been received.", t.Result.Tuples.Count));
Console.WriteLine("Press any key to exit.");
Console.ReadKey();
}).Wait();
}).Wait();
}
