/// <summary>
/// The btn import to database_ click.
/// </summary>
/// <param name="sender">
/// The sender.
/// </param>
/// <param name="e">
/// The e.
/// </param>
private void btnImportToDatabase_Click(object sender, EventArgs e)
{
this.ApplyChanges();
using (var selectPath = new FolderBrowserDialog())
{
selectPath.SelectedPath = AppDomain.CurrentDomain.BaseDirectory;
if (selectPath.ShowDialog(this) == DialogResult.OK)
{
var importPath = selectPath.SelectedPath;
var dataOptions = new DataOptions { ArchiveDataFiles = false, ShowProgressDialog = true };
this.ResultEnvironment.ArchiveManager.ImportData(importPath, dataOptions, AppContext.ProgressCallback);
}
}
}
/// <summary>
/// The import data.
/// </summary>
/// <param name="path">
/// The path.
/// </param>
/// <param name="options">
/// The options.
/// </param>
/// <param name="callback">
/// The callback.
/// </param>
/// <returns>
/// The <see cref="IList"/>.
/// </returns>
/// <exception cref="ArgumentException">
/// </exception>
public IList<TravelRoute> ImportData(string path, DataOptions options, IProgressCallback callback)
{
if (string.IsNullOrEmpty(path))
{
path = ".\\";
}
if (!Directory.Exists(path))
{
throw new ArgumentException("Target path does not exist: " + path);
}
var result = new List<TravelRoute>();
callback.Begin();
callback.Text = "Getting file list...";
string[] fileList = Directory.GetFiles(path, "*");
var acceptedFiles = new List<string>();
foreach (var file in fileList)
{
string fileExt = Path.GetExtension(file);
foreach (var ext in this.AcceptedExtensions)
{
if (string.Equals(fileExt, ext, StringComparison.OrdinalIgnoreCase))
{
acceptedFiles.Add(file);
break;
}
}
}
callback.Title = string.Format(CultureInfo.InvariantCulture, "Importing {0} files...", acceptedFiles.Count);
callback.SetRange(0, acceptedFiles.Count);
int stackCount = 0, maxStack = this._config.ProcessBatchSize;
long stackSizeBytes = 0, maxStackSizeBytes = 1024 * 1024 * 20;
// If data size reaches the limit: Start processing file content
var processRoutes = new List<TravelRoute>();
var formatter = new ProtoBufTransfer(this.Logger);
var processFiles = new List<string>();
for (int i = 0; i < acceptedFiles.Count; i++)
{
if (callback.IsAborting)
{
return result;
}
string file = acceptedFiles[i];
string fileName = Path.GetFileName(file);
string ext = Path.GetExtension(fileName); // Get file extension in order to determine the data type
this.Logger.InfoFormat("Importing file [{0}]", fileName);
callback.Text = fileName;
var fileSize = new FileInfo(file).Length;
stackSizeBytes += fileSize;
// Handle Binary data
if (string.Equals(ext, BIN_EXTENSION, StringComparison.OrdinalIgnoreCase))
{
var newRoutes = formatter.FromRaw<List<TravelRoute>>(file);
if (newRoutes == null)
{
var singleRoute = formatter.FromRaw<TravelRoute>(file);
if (singleRoute != null && singleRoute.Journeys.Count > 0)
{
processRoutes.Add(singleRoute);
stackCount += singleRoute.Journeys.Count;
}
}
else if (newRoutes.Count > 0)
{
foreach (var r in newRoutes)
{
if (r.Journeys.Count > 0)
{
processRoutes.AddRange(newRoutes);
stackCount += r.Journeys.Count;
}
}
}
}
else
{
// Handle the old data using the Fare Provider
var newRoute = this.FareDataProvider.ReadData(File.ReadAllText(file, Encoding.Default));
if (newRoute != null && newRoute.Journeys.Count > 0)
{
var existRoute = processRoutes.FirstOrDefault(r => r.IsSameRoute(newRoute));
if (existRoute == null)
{
processRoutes.Add(newRoute);
}
else
{
existRoute.AddJourney(newRoute.Journeys, true); // Merge the journeys into the same Route
}
foreach (var j in newRoute.Journeys)
{
foreach (var d in j.Data)
{
if (d.DataDate.IsUndefined())
{
d.DataDate = DateTime.Now;
}
}
}
stackCount += newRoute.Journeys.Count;
stackSizeBytes += 20 * fileSize;
// XML file needs much more resource for processing: Add "padding" to the file size boundary
}
}
processFiles.Add(file);
if (stackCount >= maxStack || stackSizeBytes >= maxStackSizeBytes || i == acceptedFiles.Count - 1)
{
this.FareDatabase.AddData(processRoutes, callback);
result.AddRange(processRoutes);
if (options.ArchiveDataFiles)
{
callback.Text = string.Format("Archiving {0} data entries...", processFiles.Count);
this.ExportData(processRoutes, this._config.ArchivePath, DataFormat.Binary, callback);
foreach (var f in processFiles)
{
File.Delete(f);
}
}
processRoutes.Clear();
stackCount = 0;
stackSizeBytes = 0;
}
callback.Increment(1);
}
return result;
}
