public void Split_data_zip()
{
var file = Path.GetFullPath("data.zip");
var loader = new ZipLoader(file);
Stopwatch watch = Stopwatch.StartNew();
var count = 0;
List <AccountStub> accounts = loader.GetAccounts().ToList();
var serializer = new JsonSerializer();
using (FileStream fs = File.OpenWrite(Path.GetFullPath("accounts_1.json")))
using (var writer = new JsonTextWriter(new StreamWriter(fs)))
{
serializer.Serialize(writer, new { accounts = accounts.Take(5000) });
}
using (FileStream fs = File.OpenWrite(Path.GetFullPath("accounts_2.json")))
using (var writer = new JsonTextWriter(new StreamWriter(fs)))
{
serializer.Serialize(writer, new { accounts = accounts.Skip(5000) });
}
watch.Stop();
Console.WriteLine($"Took {watch.ElapsedMilliseconds.ToString()}");
Assert.That(count, Is.EqualTo(10_000));
}
public void Should_unpack_all_accounts()
{
var file = Path.GetFullPath("data.zip");
var loader = new ZipLoader(file);
Stopwatch watch = Stopwatch.StartNew();
var count = loader.GetAccounts().Count();
watch.Stop();
Console.WriteLine($"Took {watch.ElapsedMilliseconds.ToString()}");
Assert.That(count, Is.EqualTo(10_000));
}
public UcdLoader(string ucdFileName, LoadOptions options = 0)
{
_fileLoader = new ZipLoader(ucdFileName);
_options = options;
if ((options & LoadOptions.AllCodes) != 0)
{
_maxCodePoint = 0x10FFFF;
}
else if ((options & LoadOptions.EuropeOnly) != 0)
{
_maxCodePoint = 0x052F;
}
else
{
_maxCodePoint = 0xFFFF;
}
}
private void ParseOneFile(string path)
{
var dataBytes = LoadDataStream(path);
var appender = new FieldDataResultsAppender
{
UtcOffset = Context.LocationUtcOffset,
Settings = Context.Settings,
};
var locationInfo = !string.IsNullOrEmpty(Context.LocationIdentifier)
? appender.CreateDummyLocationInfoByIdentifier(Context.LocationIdentifier)
: null;
appender.ForcedLocationInfo = locationInfo;
appender.AppendedResults.PluginAssemblyQualifiedTypeName = Plugin.GetType().AssemblyQualifiedName;
try
{
var result = new ZipLoader
{
Plugin = Plugin,
Logger = Logger,
Appender = appender,
LocationInfo = locationInfo
}
.ParseFile(dataBytes);
SaveAppendedResults(path, appender.AppendedResults);
SummarizeResults(path, result, appender.AppendedResults);
}
catch (ExpectedException)
{
throw;
}
catch (Exception exception)
{
Log.Error("Plugin has thrown an error", exception);
throw new ExpectedException($"Unhandled plugin exception: {exception.Message}");
}
}
private UploadContext ParseLocalFile(string path)
{
var fileBytes = LoadFileBytes(path);
var appender = new FieldDataResultsAppender
{
Client = Client,
LocationCache = LocationCache,
LocationAliases = Context.LocationAliases,
Log = Log
};
foreach (var plugin in Plugins)
{
appender.SettingsFunc = () => GetPluginSettings(plugin);
var pluginName = PluginLoader.GetPluginNameAndVersion(plugin);
try
{
var resultWithAttachments = new ZipLoader
{
Plugin = plugin,
Appender = appender,
Logger = Log,
LocationInfo = null
}
.ParseFile(fileBytes);
if (resultWithAttachments.Result.Status == ParseFileStatus.CannotParse)
{
continue;
}
if (resultWithAttachments.Result.Status != ParseFileStatus.SuccessfullyParsedAndDataValid)
{
throw new ArgumentException(
$"Error parsing '{path}' with {pluginName}: {resultWithAttachments.Result.ErrorMessage}");
}
if (!appender.AppendedResults.AppendedVisits.Any())
{
throw new ArgumentException($"{pluginName} did not parse any field visits.");
}
var attachmentCount = resultWithAttachments.Attachments?.Count ?? 0;
if (appender.AppendedResults.AppendedVisits.Count > 1 && attachmentCount > 0)
{
throw new ArgumentException($"Only single-visit data files can be uploaded with attachments.");
}
Log.Info(
$"{pluginName} parsed '{path}' with {appender.AppendedResults.AppendedVisits.Count} visits: {string.Join(", ", appender.AppendedResults.AppendedVisits.Take(10).Select(v => v.FieldVisitIdentifier))}");
appender.AppendedResults.PluginAssemblyQualifiedTypeName = plugin.GetType().AssemblyQualifiedName;
return(new UploadContext
{
Path = path,
AppendedResults = appender.AppendedResults,
Attachments = resultWithAttachments.Attachments
});
}
catch (Exception e)
{
Log.Warn($"{pluginName} skipping '{path}': {e.Message}");
}
}
throw new ArgumentException($"'{path}' was not parsed by any plugin.");
}
