List <T> GetAllPagedResults <T>(SncfApi sncfApi, Func <int, int, PagedResult <T> > apiFunc, int chunckSize = DEFAULT_CHUNK_SIZE) where T : new()
{
if (chunckSize <= 0)
{
throw new ArgumentOutOfRangeException("Chunck size must be > 0.");
}
int numPage = 0;
List <T> globalList = new List <T>();
bool hasData = true;
while (hasData)
{
PagedResult <T> pagedResult = apiFunc(chunckSize, numPage);
if (pagedResult.Results == null)
{
hasData = false;
}
else
{
globalList.AddRange(pagedResult.Results);
hasData = pagedResult.HasMoreData;
numPage++;
}
}
;
return(globalList);
}
/// <summary>
/// Saves data identified as "static", ie: does not change often and can save remote Hits
/// Important : this is not authorized by SNCF api terms, and is there for tests purposes
/// </summary>
/// <param name="sncfApi"></param>
private void SaveStaticData(SncfApi sncfApi, string dataDir)
{
var lines = GetAndSaveData <Line>(sncfApi, "lines", dataDir, "lines.json");
GetAndSaveData <StopArea>(sncfApi, "stop_areas", dataDir, "stop_areas.json");
GetAndSaveData <Route>(sncfApi, "routes", dataDir, "routes.json");
GetAndSaveData <StopPoint>(sncfApi, "stop_points", dataDir, "stop_points.json");
GetAndSavedRelatedData <Line, StopArea>(sncfApi, lines, "lines/{id}/stop_areas", dataDir, "lines.stop_areas.json");
}
public SncfRepository(string dataDirectory, int chunckSize = DEFAULT_CHUNK_SIZE)
{
_dataDirectory = dataDirectory;
ChunckSize = chunckSize;
CheckDirExists(_dataDirectory);
string sncfAuthKey = ConfigurationManager.AppSettings["SNCF_API_KEY"];
_api = new SncfApi(sncfAuthKey);
}
List <T> GetAndSaveData <T>(SncfApi sncfApi, string endpoint, string dataDir, string fileName) where T : new()
{
CheckDirExists(dataDir);
List <T> allItems = GetAllPagedResults <T>(sncfApi, endpoint);
var json = JsonConvert.SerializeObject(allItems, Formatting.None);
File.WriteAllText(Path.Combine(dataDir, fileName), json);
return(allItems);
}
private void SaveLineRouteSchedules(SncfApi sncfApi, List <Line> lines, string dataDir, int chunkSize = DEFAULT_CHUNK_SIZE)
{
string schedulesDir = Path.Combine(dataDir, LINE_ROUTE_SCHEDULES_DIR);
CheckDirExists(schedulesDir);
foreach (var line in lines)
{
try
{
List <RouteSchedule> allItems = GetAllPagedResults <RouteSchedule>(sncfApi, (n, p) => sncfApi.GetLineRouteSchedules(line.Id, n, p), chunkSize);
var json = JsonConvert.SerializeObject(allItems, Formatting.Indented);
File.WriteAllText(Path.Combine(schedulesDir, $"{SafeFileId(line.Id)}.json"), json);
}
catch (Exception ex)
{
Trace.TraceError($"Error in SaveLineRouteSchedules for line {line.Id} : {ex.Message}.");
}
}
}
/// <summary>
///
/// </summary>
/// <typeparam name="TBase"></typeparam>
/// <typeparam name="TRelated"></typeparam>
/// <param name="sncfApi"></param>
/// <param name="baseResourceList"></param>
/// <param name="patternPathToRelatedResource"></param>
/// <param name="dataDirectory"></param>
/// <param name="outputFileNamePattern">Should contain {id}</param>
/// <returns></returns>
public void GetAndSavedRelatedData_Detail <TBase, TRelated>(SncfApi sncfApi,
List <TBase> baseResourceList,
string patternPathToRelatedResource,
string dataDirectory,
string outputFileNamePattern) where TBase : ApiResourceBase, new()
where TRelated : new()
{
if (!outputFileNamePattern.Contains("{id}"))
{
throw new ArgumentException("outputFileNamePattern should contain {id}.");
}
string v_directory = Path.GetDirectoryName(Path.Combine(dataDirectory, outputFileNamePattern.Replace("{id}", "")));
Directory.CreateDirectory(v_directory);
Parallel.ForEach(baseResourceList, baseItem =>
//foreach (var baseItem in baseResourceList)
{
try
{
List <TRelated> allRelatedItems = GetAllPagedResults <TRelated>(sncfApi, patternPathToRelatedResource.Replace("{id}", baseItem.Id), 100);
var json = JsonConvert.SerializeObject(allRelatedItems, Formatting.Indented);
File.WriteAllText(Path.Combine(dataDirectory, outputFileNamePattern.Replace("{id}", SafeFileId(baseItem.Id))), json);
}
catch (KeyNotFoundException)
{
Trace.TraceWarning($"GetAndSavedRelatedData. Not found for {baseItem.Id}");
}
catch (Exception v_ex)
{
Trace.TraceWarning($"GetAndSavedRelatedData. Error for {baseItem.Id} : {v_ex.Message}");
}
}
);
}
public Dictionary <string, List <TRelated> > GetAndSavedRelatedData <TBase, TRelated>(SncfApi sncfApi,
List <TBase> baseResourceList,
string patternPathToRelatedResource,
string dataDirectory,
string outputFileName) where TBase : ApiResourceBase, new()
where TRelated : new()
{
//Dictionary<string, List<TRelated>> dic = new Dictionary<string, List<TRelated>>();
ConcurrentDictionary <string, List <TRelated> > dic = new ConcurrentDictionary <string, List <TRelated> >();
Parallel.ForEach(baseResourceList, baseItem =>
{
try
{
List <TRelated> allRelatedItems = GetAllPagedResults <TRelated>(sncfApi, patternPathToRelatedResource.Replace("{id}", baseItem.Id), 100);
dic.AddOrUpdate(baseItem.Id, allRelatedItems, AddRelatedItem);
}
catch (KeyNotFoundException)
{
dic.AddOrUpdate(baseItem.Id, new List <TRelated>(), AddRelatedItem);
Trace.TraceWarning("GetAndSavedRelatedData. Not found.");
}
catch (Exception v_ex)
{
Trace.TraceWarning($"GetAndSavedRelatedData. Error : {v_ex.Message}");
}
}
);
//var json = JsonConvert.SerializeObject(dic, Formatting.None);
//File.WriteAllText(Path.Combine(dataDirectory, outputFileName), json);
Dictionary <string, List <TRelated> > outDic = new Dictionary <string, List <TRelated> >();
outDic = dic.ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
var json = JsonConvert.SerializeObject(outDic, Formatting.None);
File.WriteAllText(Path.Combine(dataDirectory, outputFileName), json);
return(outDic);
}
