/// <summary>
/// Types of source for extracting.
/// </summary>
/// <param name="value"></param>
/// <param name="op"></param>
/// <param name="notNull"></param>
/// <param name="source"></param>
/// <param name="multi"></param>
public ComboExtract(ExtractBy[] value, ExtractOp op, bool notNull, ExtractSource source, bool multi)
{
Value = value;
Op = op;
NotNull = notNull;
Source = source;
Multi = multi;
}
public Extractor(ISelector selector, ExtractSource source, bool notNull, bool multi, string exprision, long count = long.MaxValue)
{
Selector = selector;
Source = source;
NotNull = notNull;
Multi = multi;
Expresion = exprision;
Count = count;
}
private string ConverExtractSourceToTableName( ExtractSource extractSource )
{
switch ( extractSource ) {
case ExtractSource.Gen1Links: return _dsImport.tblGen1Links.TableName;
case ExtractSource.Gen1Explicit: return _dsImport.tblGen1Explicit.TableName;
case ExtractSource.Gen1Implicit: return _dsImport.tblGen1Implicit.TableName;
case ExtractSource.Gen2Links: return _dsImport.tblGen2Links.TableName;
case ExtractSource.Gen2LinksFromGen1: return _dsImport.tblGen2LinksFromGen1.TableName;
case ExtractSource.Gen2ImplicitFather: return _dsImport.tblGen2ImplicitFather.TableName;
case ExtractSource.Gen2FatherFromGen1: return _dsImport.tblGen2FatherFromGen1.TableName;
case ExtractSource.Gen1Outcomes: return _dsImport.tblGen1Outcomes.TableName;
case ExtractSource.Gen2OutcomesHeight: return _dsImport.tblGen2OutcomesHeight.TableName;
case ExtractSource.Gen2OutcomesWeight: return _dsImport.tblGen2OutcomesWeight.TableName;
case ExtractSource.Gen2OutcomesMath: return _dsImport.tblGen2OutcomesMath.TableName;
default: throw new ArgumentOutOfRangeException("extractSource", extractSource, "The Extract Source is not recognized in this function.");
}
}
public void Extract(string compressedArchivePath, string outputDirectory, IProgress <ProgressReport> progress)
{
using (var archiveStream = CreateTemporaryFileStream())
{
// decompress the LZMA stream
using (var lzmaStream = File.OpenRead(compressedArchivePath))
{
CompressionUtility.Decompress(lzmaStream, archiveStream, progress);
}
var archivePath = ((FileStream)archiveStream).Name;
// reset the uncompressed stream
archiveStream.Seek(0, SeekOrigin.Begin);
// read as a zip archive
using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read))
using (var tlArchive = new ThreadLocalZipArchive(archivePath, archive))
{
List <ExtractOperation> extractOperations = new List <ExtractOperation>();
Dictionary <string, ExtractSource> sourceCache = new Dictionary <string, ExtractSource>();
// process the index to determine all extraction operations
var indexEntry = archive.GetEntry(IndexFileName);
using (var indexReader = new StreamReader(indexEntry.Open()))
{
Dictionary <string, ZipOperation> zipOperations = new Dictionary <string, ZipOperation>(StringComparer.OrdinalIgnoreCase);
for (var line = indexReader.ReadLine(); line != null; line = indexReader.ReadLine())
{
var lineParts = line.Split(pipeSeperator);
if (lineParts.Length != 2)
{
throw new Exception("Unexpected index line format, too many '|'s.");
}
string target = lineParts[0];
string source = lineParts[1];
ExtractSource extractSource;
if (!sourceCache.TryGetValue(source, out extractSource))
{
sourceCache[source] = extractSource = new ExtractSource(source, _externalFiles, tlArchive);
}
var zipSeperatorIndex = target.IndexOf("::", StringComparison.OrdinalIgnoreCase);
if (zipSeperatorIndex != -1)
{
string zipRelativePath = target.Substring(0, zipSeperatorIndex);
string zipEntryName = target.Substring(zipSeperatorIndex + 2);
string destinationPath = Path.Combine(outputDirectory, zipRelativePath);
// operations on a zip file will be sequential
ZipOperation currentZipOperation;
if (!zipOperations.TryGetValue(destinationPath, out currentZipOperation))
{
extractOperations.Add(currentZipOperation = new ZipOperation(destinationPath));
zipOperations.Add(destinationPath, currentZipOperation);
}
currentZipOperation.AddEntry(zipEntryName, extractSource);
}
else
{
string destinationPath = Path.Combine(outputDirectory, target);
extractOperations.Add(new CopyOperation(extractSource, destinationPath));
}
}
}
int opsExecuted = 0;
// execute all operations
//foreach(var extractOperation in extractOperations)
extractOperations.AsParallel().ForAll(extractOperation =>
{
extractOperation.DoOperation();
progress.Report(LocalizableStrings.Expanding, Interlocked.Increment(ref opsExecuted), extractOperations.Count);
});
}
}
}
public void AddEntry(string entryName, ExtractSource source)
{
entries.Add(Tuple.Create(entryName, source));
}
public CopyOperation(ExtractSource source, string destinationPath) : base(destinationPath)
{
Source = source;
}
public FieldExtractor(PropertyInfo field, ISelector selector, ExtractSource source, bool notNull, bool multi, string expresion)
: base(selector, source, notNull, multi, expresion)
{
Field = field;
}
public void Extract(string compressedArchivePath, string outputDirectory, IProgress<ProgressReport> progress)
{
using (var archiveStream = CreateTemporaryFileStream())
{
// decompress the LZMA stream
using (var lzmaStream = File.OpenRead(compressedArchivePath))
{
CompressionUtility.Decompress(lzmaStream, archiveStream, progress);
}
var archivePath = ((FileStream)archiveStream).Name;
// reset the uncompressed stream
archiveStream.Seek(0, SeekOrigin.Begin);
// read as a zip archive
using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read))
using (var tlArchive = new ThreadLocalZipArchive(archivePath, archive))
{
List<ExtractOperation> extractOperations = new List<ExtractOperation>();
Dictionary<string, ExtractSource> sourceCache = new Dictionary<string, ExtractSource>();
// process the index to determine all extraction operations
var indexEntry = archive.GetEntry(IndexFileName);
using (var indexReader = new StreamReader(indexEntry.Open()))
{
Dictionary<string, ZipOperation> zipOperations = new Dictionary<string, ZipOperation>(StringComparer.OrdinalIgnoreCase);
for (var line = indexReader.ReadLine(); line != null; line = indexReader.ReadLine())
{
var lineParts = line.Split(pipeSeperator);
if (lineParts.Length != 2)
{
throw new Exception("Unexpected index line format, too many '|'s.");
}
string target = lineParts[0];
string source = lineParts[1];
ExtractSource extractSource;
if (!sourceCache.TryGetValue(source, out extractSource))
{
sourceCache[source] = extractSource = new ExtractSource(source, _externalFiles, tlArchive);
}
var zipSeperatorIndex = target.IndexOf("::", StringComparison.OrdinalIgnoreCase);
if (zipSeperatorIndex != -1)
{
string zipRelativePath = target.Substring(0, zipSeperatorIndex);
string zipEntryName = target.Substring(zipSeperatorIndex + 2);
string destinationPath = Path.Combine(outputDirectory, zipRelativePath);
// operations on a zip file will be sequential
ZipOperation currentZipOperation;
if (!zipOperations.TryGetValue(destinationPath, out currentZipOperation))
{
extractOperations.Add(currentZipOperation = new ZipOperation(destinationPath));
zipOperations.Add(destinationPath, currentZipOperation);
}
currentZipOperation.AddEntry(zipEntryName, extractSource);
}
else
{
string destinationPath = Path.Combine(outputDirectory, target);
extractOperations.Add(new CopyOperation(extractSource, destinationPath));
}
}
}
int opsExecuted = 0;
// execute all operations
//foreach(var extractOperation in extractOperations)
extractOperations.AsParallel().ForAll(extractOperation =>
{
extractOperation.DoOperation();
progress.Report("Expanding", Interlocked.Increment(ref opsExecuted), extractOperations.Count);
});
}
}
}
public void AddEntry(string entryName, ExtractSource source)
{
entries.Add(Tuple.Create(entryName, source));
}
public CopyOperation(ExtractSource source, string destinationPath) : base(destinationPath)
{
Source = source;
}
public FieldExtractor(PropertyInfo field, ISelector selector, ExtractSource source, bool notNull, bool multi,string expresion)
: base(selector, source, notNull, multi, expresion)
{
Field = field;
}
private LinksDataSet.tblVariableRow[] VariablesToTranslate( ExtractSource extractSource )
{
string select = string.Format("{0}={1} AND {2}={3}",
(byte)extractSource, _dsLinks.tblVariable.ExtractSourceColumn.ColumnName,
"TRUE", _dsLinks.tblVariable.TranslateColumn.ColumnName);
LinksDataSet.tblVariableRow[] drVariablesToTranslate = (LinksDataSet.tblVariableRow[])_dsLinks.tblVariable.Select(select);
return drVariablesToTranslate;
}
private Int32 TranslateExtractSource( ExtractSource extractSource, Generation generation, bool femalesOnly, Int32[] passoverValues, DataTable dtImport )
{
Int32 gen1ReponseRecordsAddedCount = 0;
LinksDataSet.tblVariableRow[] drsVariablesToTranslate = VariablesToTranslate(extractSource);
_dsLinks.tblResponse.BeginLoadData();
string subjectIDColumnName;
string genderColumnName;
switch ( generation ) {
case Generation.Gen1:
subjectIDColumnName = Constants.Gen1SubjectIDColumn;
genderColumnName = Constants.Gen1GenderColumn;
break;
case Generation.Gen2:
subjectIDColumnName = Constants.Gen2SubjectIDColumn;
genderColumnName = Constants.Gen2GenderColumn;
break;
default:
throw new ArgumentOutOfRangeException("generation", generation, "The generation value was not recognized.");
}
foreach ( DataRow drImport in dtImport.Rows ) {
GenderBothGenerations gender = (GenderBothGenerations)Convert.ToInt32(drImport[genderColumnName]);
if ( !femalesOnly || gender == GenderBothGenerations.Female ) {
Int32 subjectID = Convert.ToInt32(drImport[subjectIDColumnName]);
Int32 subjectTag = Retrieve.SubjectTagFromSubjectIDAndGeneration(subjectID, generation, _dsLinks);
foreach ( LinksDataSet.tblVariableRow drVariable in drsVariablesToTranslate ) {
string columnName = drVariable.VariableCode;
LinksDataSet.tblResponseRow drResponse = _dsLinks.tblResponse.NewtblResponseRow();
drResponse.Generation = (byte)generation;
drResponse.SubjectTag = subjectTag;
drResponse.ExtendedID = GetExtendedID(subjectTag);
drResponse.SurveySource = drVariable.SurveySource;
drResponse.SurveyYear = drVariable.SurveyYear;
drResponse.Item = drVariable.Item; //if ( drResponse.Item == 13 ) Trace.Assert(true);
drResponse.Value = Convert.ToInt32(drImport[columnName]);
LinksDataSet.tblItemRow drItem = drVariable.tblItemRow;
if ( !(drItem.MinValue <= drResponse.Value && drResponse.Value <= drItem.MaxValue) )
throw new InvalidOperationException(string.Format("For Item '{0}', variable '{1}', the value '{2}' exceeded the bounds of [{3}, {4}].", drVariable.Item, drVariable.ID, drResponse.Value, drItem.MinValue, drItem.MaxValue));
if ( 0 <= drResponse.Value && drResponse.Value < drItem.MinNonnegative )
throw new InvalidOperationException(string.Format("For Item '{0}', variable '{1}', the value '{2}' dipped below the minimum nonnegative value of {3}.", drVariable.Item, drVariable.ID, drResponse.Value, drItem.MinNonnegative));
if ( !passoverValues.Contains(drResponse.Value) ) {
drResponse.LoopIndex = drVariable.LoopIndex;
_dsLinks.tblResponse.AddtblResponseRow(drResponse);
gen1ReponseRecordsAddedCount += 1;
}
}
}
}
_dsLinks.tblResponse.EndLoadData();
return gen1ReponseRecordsAddedCount;
}
