/// <summary> /// The btn import to database_ click. /// </summary> /// <param name="sender"> /// The sender. /// </param> /// <param name="e"> /// The e. /// </param> private void btnImportToDatabase_Click(object sender, EventArgs e) { this.ApplyChanges(); using (var selectPath = new FolderBrowserDialog()) { selectPath.SelectedPath = AppDomain.CurrentDomain.BaseDirectory; if (selectPath.ShowDialog(this) == DialogResult.OK) { var importPath = selectPath.SelectedPath; var dataOptions = new DataOptions { ArchiveDataFiles = false, ShowProgressDialog = true }; this.ResultEnvironment.ArchiveManager.ImportData(importPath, dataOptions, AppContext.ProgressCallback); } } }
/// <summary> /// The import data. /// </summary> /// <param name="path"> /// The path. /// </param> /// <param name="options"> /// The options. /// </param> /// <param name="callback"> /// The callback. /// </param> /// <returns> /// The <see cref="IList"/>. /// </returns> /// <exception cref="ArgumentException"> /// </exception> public IList<TravelRoute> ImportData(string path, DataOptions options, IProgressCallback callback) { if (string.IsNullOrEmpty(path)) { path = ".\\"; } if (!Directory.Exists(path)) { throw new ArgumentException("Target path does not exist: " + path); } var result = new List<TravelRoute>(); callback.Begin(); callback.Text = "Getting file list..."; string[] fileList = Directory.GetFiles(path, "*"); var acceptedFiles = new List<string>(); foreach (var file in fileList) { string fileExt = Path.GetExtension(file); foreach (var ext in this.AcceptedExtensions) { if (string.Equals(fileExt, ext, StringComparison.OrdinalIgnoreCase)) { acceptedFiles.Add(file); break; } } } callback.Title = string.Format(CultureInfo.InvariantCulture, "Importing {0} files...", acceptedFiles.Count); callback.SetRange(0, acceptedFiles.Count); int stackCount = 0, maxStack = this._config.ProcessBatchSize; long stackSizeBytes = 0, maxStackSizeBytes = 1024 * 1024 * 20; // If data size reaches the limit: Start processing file content var processRoutes = new List<TravelRoute>(); var formatter = new ProtoBufTransfer(this.Logger); var processFiles = new List<string>(); for (int i = 0; i < acceptedFiles.Count; i++) { if (callback.IsAborting) { return result; } string file = acceptedFiles[i]; string fileName = Path.GetFileName(file); string ext = Path.GetExtension(fileName); // Get file extension in order to determine the data type this.Logger.InfoFormat("Importing file [{0}]", fileName); callback.Text = fileName; var fileSize = new FileInfo(file).Length; stackSizeBytes += fileSize; // Handle Binary data if (string.Equals(ext, BIN_EXTENSION, StringComparison.OrdinalIgnoreCase)) { var newRoutes = formatter.FromRaw<List<TravelRoute>>(file); if (newRoutes == null) { var singleRoute = formatter.FromRaw<TravelRoute>(file); if (singleRoute != null && singleRoute.Journeys.Count > 0) { processRoutes.Add(singleRoute); stackCount += singleRoute.Journeys.Count; } } else if (newRoutes.Count > 0) { foreach (var r in newRoutes) { if (r.Journeys.Count > 0) { processRoutes.AddRange(newRoutes); stackCount += r.Journeys.Count; } } } } else { // Handle the old data using the Fare Provider var newRoute = this.FareDataProvider.ReadData(File.ReadAllText(file, Encoding.Default)); if (newRoute != null && newRoute.Journeys.Count > 0) { var existRoute = processRoutes.FirstOrDefault(r => r.IsSameRoute(newRoute)); if (existRoute == null) { processRoutes.Add(newRoute); } else { existRoute.AddJourney(newRoute.Journeys, true); // Merge the journeys into the same Route } foreach (var j in newRoute.Journeys) { foreach (var d in j.Data) { if (d.DataDate.IsUndefined()) { d.DataDate = DateTime.Now; } } } stackCount += newRoute.Journeys.Count; stackSizeBytes += 20 * fileSize; // XML file needs much more resource for processing: Add "padding" to the file size boundary } } processFiles.Add(file); if (stackCount >= maxStack || stackSizeBytes >= maxStackSizeBytes || i == acceptedFiles.Count - 1) { this.FareDatabase.AddData(processRoutes, callback); result.AddRange(processRoutes); if (options.ArchiveDataFiles) { callback.Text = string.Format("Archiving {0} data entries...", processFiles.Count); this.ExportData(processRoutes, this._config.ArchivePath, DataFormat.Binary, callback); foreach (var f in processFiles) { File.Delete(f); } } processRoutes.Clear(); stackCount = 0; stackSizeBytes = 0; } callback.Increment(1); } return result; }