public static String GetFileName(InseeTypeData typedata, InseeParametersRequest inseeParameters) { //var modalite = inseeParameters.Modalite.Replace('.', '_'); //return $"{typedata}" + "_" + inseeParameters.Croisement + "@" + inseeParameters.JeuDonnees + "_" + inseeParameters.Nivgeo + "-" + inseeParameters.Codgeo + "_" + modalite; //+ "_" + inseeParameters.Codgeo return($"{typedata}" + "_" + inseeParameters.Croisement); }
static void Main(string[] args) { var logFolder = ConfigurationManager.AppSettings.Get("LogFolder"); if (logFolder == null) { throw new Exception("Invalid LogFolder"); } var logService = new SimpleLogService(logFolder); var accessToken = ConfigurationManager.AppSettings.Get("AccessToken"); if (accessToken == null) { throw new Exception("Invalid AccessToken"); } var client = new HttpInseeConnector(accessToken); var baseUrl = ConfigurationManager.AppSettings.Get("InseeBaseUrl"); if (baseUrl == null) { throw new Exception("Invalid BaseUrl"); } var dataFolder = ConfigurationManager.AppSettings.Get("DataFolder"); if (dataFolder == null) { throw new Exception("Invalid DataFolder"); } var jeuDonnees = ConfigurationManager.AppSettings.Get("JeuDonnees"); if (jeuDonnees == null) { throw new Exception("Invalid jeuDonnees"); } var croisement = ConfigurationManager.AppSettings.Get("Croisement"); if (croisement == null) { throw new Exception("Invalid croisement"); } var modalite = ConfigurationManager.AppSettings.Get("Modalite"); if (modalite == null) { throw new Exception("Invalid modalite"); } var nivgeo = ConfigurationManager.AppSettings.Get("Nivgeo"); if (nivgeo == null) { throw new Exception("Invalid nivgeo"); } var inseeParameters = new InseeParametersRequest(jeuDonnees, croisement, modalite, nivgeo, ""); var rawFileName = InseeUtils.GetFileName(InseeTypeData.RawData, inseeParameters); Boolean smartContinue = true; List <InseeCommune> listCommunes = InseeUtils.GetListOfCommunes(dataFolder); var lastCodgeoProcessed = InseeUtils.GetLastCodgeo(dataFolder, rawFileName); if (lastCodgeoProcessed != null) { var lastCommuneProcessed = listCommunes.FirstOrDefault(x => x.CODGEO == lastCodgeoProcessed); if (lastCommuneProcessed == null) { smartContinue = false; } else { listCommunes = listCommunes.Where(x => x.Index > lastCommuneProcessed.Index).ToList(); } } else { smartContinue = false; } //rate limit 30 requests per minute var watch = new Stopwatch(); var nbSecondToWait = 2; //Writing settings var firstWrite = true; var limit = ConfigurationManager.AppSettings.Get("limitNbDataItems"); var limitNbDataItems = 50; if (limit != null) { limitNbDataItems = Convert.ToInt32(limit); } var listData = new List <InseeData>(); foreach (InseeCommune commune in listCommunes) { watch.Start(); var codgeo = commune.CODGEO; inseeParameters.Codgeo = codgeo; HttpResponseMessage response; try { response = client.Get(baseUrl, InseeUtils.GetParametersUrlFormat(inseeParameters)); } catch (Exception e) { logService.LogException(e); return; } if (response.StatusCode != System.Net.HttpStatusCode.OK) { logService.Error(response.ToString()); return; } var xmlContent = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); var json = InseeUtils.GetJsonString(xmlContent); var data = JsonConvert.DeserializeObject <InseeData>(json); listData.Add(data); if (listData.Count >= limitNbDataItems) { if (firstWrite && !smartContinue) { InseeUtils.WriteRawData(listData, dataFolder, rawFileName); InseeUtils.WriteVariables(listData[0], dataFolder, InseeUtils.GetFileName(InseeTypeData.Variables, inseeParameters)); firstWrite = false; } else { InseeUtils.WriteRawData(listData, dataFolder, rawFileName, false); } listData.Clear(); } watch.Stop(); if (watch.ElapsedMilliseconds < nbSecondToWait * 1000) { Task.Delay((int)(nbSecondToWait * 1000 - watch.ElapsedMilliseconds)).GetAwaiter().GetResult(); } watch.Reset(); } InseeUtils.WriteRawData(listData, dataFolder, rawFileName, false); }
public static String GetParametersUrlFormat(InseeParametersRequest inseeParameters) { return(inseeParameters.Croisement + "@" + inseeParameters.JeuDonnees + "/" + inseeParameters.Nivgeo + "-" + inseeParameters.Codgeo + "." + inseeParameters.Modalite); }