public void TestSplitFilesSmall()
{
IEnumerable <string> splitFile = FileSplitter.SplitFile("test.jpg", 1024 * 15, new DirectoryInfo("Parts"), isPathAbsoloute: true);
List <byte> splitFiles = new List <byte>();
foreach (var item in splitFile)
{
splitFiles.AddRange(File.ReadAllBytes(item));
File.Delete(item);
}
byte[] oldFile = File.ReadAllBytes("test.jpg");
byte[] newFile = splitFiles.ToArray();
bool IsEqual = true;
for (int i = 0; i < oldFile.Length; i++)
{
if (oldFile[i] != newFile[i])
{
IsEqual = false;
break;
}
}
Directory.Delete("Parts");
Assert.IsTrue(IsEqual);
}
static void Main(string[] args)
{
Console.WriteLine("Please write path to file");
string path = Console.ReadLine();
FileIOUploaderProvider provider = new FileIOUploaderProvider();
try
{
IEnumerable <string> files = FileSplitter.SplitFile(path, provider.MaxChunkSize, new DirectoryInfo("Parts"), isPathAbsoloute: true).ToList();
foreach (var item in provider.Upload(files, 1000))
{
Console.WriteLine(item);
}
foreach (var item in files)
{
File.Delete(item);
}
}
catch (UploadFailedException ex)
{
Console.WriteLine("Upload failed");
}
Console.WriteLine("Done");
Console.ReadLine();
}
static void Main(string[] args)
{
Console.WriteLine("Checking path, splitting files");
Console.WriteLine(Environment.NewLine + "----------------------------------------------------------------");
Console.WriteLine("Parameters: " + args.Length);
for (int i = 0; i < args.Length; i++)
{
Console.WriteLine(i + "\t" + args[i]);
}
if (args.Length != 4)
{
Console.WriteLine("\nParameters error! You should pass 3 values!\n");
Console.ReadLine();
return;
}
string expectedModification = args[0];
string inputFilePath = args[1];
string mappingFilePath = args[2];
string outputPath = args[3] + "\\";
Console.WriteLine();
Console.WriteLine("Modification name:\t" + expectedModification);
Console.WriteLine("Path to input file:\t" + inputFilePath);
Console.WriteLine("Path to mapping file:\t" + mappingFilePath);
Console.WriteLine("Path to output folder:\t" + outputPath);
Console.WriteLine("----------------------------------------------------------------" + Environment.NewLine);
FileSplitter.SplitFile(inputFilePath);
Console.WriteLine("Fixing indentations");
IndentationChecker.CheckIndentations();
Console.WriteLine("Looking for modifications");
if (!ModificationSearchTool.FindAndSaveChanges(expectedModification))
{
Console.WriteLine("ERROR: Modofication {0} not found", expectedModification);
Console.ReadLine();
return;
}
Console.WriteLine("Preparing the modifications for saving");
ModificationCleanerTool.CleanChangeCode();
Console.WriteLine("Generating Documentation() trigger");
DocumentationTrigger.UpdateDocumentationTrigger();
Console.WriteLine("Saving objects");
SaveTool.SaveObjectsToFiles(outputPath);
Console.WriteLine("Saving changelogs");
SaveTool.SaveChangesToFiles(outputPath);
Console.WriteLine("Generating <next> documentation file");
SaveTool.SaveDocumentationToFile(outputPath, DocumentationExport.GenerateDocumentationFile(outputPath, mappingFilePath));
Console.WriteLine("Saving objects grouped");
SaveTool.SaveObjectModificationFiles(outputPath);
Console.WriteLine("Success");
Console.ReadLine();
}
public void Main(string [] args)
{
FileReader fileReader = new FileReader(args[0]);
FileSplitter fileSplitter = new FileSplitter();
IEnumerable <string> lines = fileReader.Read();
IEnumerable <PersonDTO> persons = fileSplitter.TextSplitter(lines);;
DataBaseProcessor dataProcessor = new DataBaseProcessor();
database.InsertIntoPerson(persons);
}
private static void SplitFiles(FileSplitter fs, string operation)
{
Thread t = new Thread(new ParameterizedThreadStart(SplitThread));
t.SetApartmentState(ApartmentState.STA);
using (WaitDlg dlg = new WaitDlg(operation))
{
dlg.UpdateStatus("Please wait while files are being extracted and imaged");
SplitThreadParams stp = new SplitThreadParams(fs, dlg);
t.Start(stp);
dlg.ShowDialog();
}
}
private SortDriver(
string sourceFilePath,
int runSizeHintInMiB,
int maxDegreeOfParallelism,
int mergeBufferSizePerRun,
Action <string> logger)
{
_sourceFilePath = sourceFilePath;
_runSizeHintInMiB = runSizeHintInMiB;
_mergeBufferSizePerRun = mergeBufferSizePerRun < MinMergeBufferSize ? MinMergeBufferSize : mergeBufferSizePerRun;
_maxDegreeOfParallelism = maxDegreeOfParallelism;
_logger = logger;
_fileSplitter = new FileSplitter(logger);
_repositoryProvider = new ExternalRunRepositoryProvider(Path.GetDirectoryName(sourceFilePath));
}
async Task TestNumberOfParts()
{
var blobId = new BlobId();
var numberParts = 2;
var blobStoreMock = new Mock <IBlobStore>();
blobStoreMock.Setup(store => store.Put(It.IsAny <Stream>())).Returns(blobId);
var splitter = new FileSplitter(blobStoreMock.Object);
var fileSystem = TestUtilities.CreateFilesystem();
var fileStream = fileSystem.FileStream.Create(@"c:\myfile.txt", FileMode.Open);
var blobids = await splitter.SplitData(numberParts, fileStream);
Assert.Equal(numberParts, blobids.Count());
}
public void TestSplitFilesHuge()
{
IEnumerable <string> splitFile = FileSplitter.SplitFile("testhuge.zip", 1024 * 1024 * 1024, new DirectoryInfo("TestHugeParts"), isPathAbsoloute: true).ToList();
// Todo: use an md5 hash instead and then precompute testhuge.zip
// Comparing for length. because actually comparing 2.6GB of data takes too long
bool sameLength = new FileInfo("testhuge.zip").Length == splitFile.Select(x => new FileInfo(x)).Sum(x => x.Length);
foreach (var item in splitFile)
{
File.Delete(item);
}
Directory.Delete("TestHugeParts");
Assert.IsTrue(sameLength);
}
public void FileSplitterTest(int expectedCount, int splitThreshold)
{
var inFile = "INFILE.TXT";
using (StreamWriter sw = new StreamWriter(inFile))
{
for (int i = 0; i <= 2000000; i++)
{
sw.WriteLine("123");
}
}
using (var splitter = new FileSplitter(inFile, ".", splitThreshold))
{
splitter.SplitFiles();
Assert.AreEqual(expectedCount, splitter.OutFiles.Count);
}
}
public static void SplitCommonPic(IWin32Window parent)
{
string inputFileName = MainForm.GetOpenFileName(parent, "Open CommonPic.dat", "CommonPic.dat (*.dat)|commonpic*.*|All Files (*.*)|*.*", null);
if (String.IsNullOrEmpty(inputFileName))
{
return;
}
string outputFolder = MainForm.PickFolder(parent, "Pick directory to split to...", Path.GetDirectoryName(inputFileName));
if (String.IsNullOrEmpty(outputFolder))
{
return;
}
DebugLogger.Log("Tools", "Splitting and decomping CommonPic {0} to {1}", inputFileName, outputFolder);
FileSplitter fs = () => {
GTMP.GTMPFile.ExplodeCommonPic(inputFileName, outputFolder, GTMP.GTMPFile.SplitCommonPicArgs.OutputPngPicture);
};
SplitFiles(fs, "Splitting " + Path.GetFileName(inputFileName));
}
public void UploadDownloadTest()
{
FileIOUploaderProvider uploaderProvider = new FileIOUploaderProvider();
var links = uploaderProvider.Upload(FileSplitter.SplitFile("test.jpg", 51200)).ToList();
byte[] oldFile = File.ReadAllBytes("test.jpg");
byte[] newFile = new FileIODownloadProvider().Download(links).Aggregate <IEnumerable <byte> >((x, y) => x.Concat(y)).ToArray();
bool IsEqual = true;
for (int i = 0; i < oldFile.Length; i++)
{
if (oldFile[i] != newFile[i])
{
IsEqual = false;
break;
}
}
Assert.IsTrue(IsEqual);
}
public static void SplitGTMenuDat(IWin32Window parent)
{
string inputFileName = MainForm.GetOpenFileName(parent, "Open GTMenuDat.dat", "GTMenuDat.dat (*.dat)|gtmenudat*.*|All Files (*.*)|*.*", null);
if (String.IsNullOrEmpty(inputFileName))
{
return;
}
string outputFolder = MainForm.PickFolder(parent, "Pick directory to split to...", Path.GetDirectoryName(inputFileName));
if (String.IsNullOrEmpty(outputFolder))
{
return;
}
DebugLogger.Log("Tools", "Splitting and decomping GTMenuDat {0} to {1}", inputFileName, outputFolder);
FileSplitter fs = () =>
{
const GTMP.GMFile.SplitGTMenuFlags flags = GTMP.GMFile.SplitGTMenuFlags.OutputPngPicture;
GTMP.GMFile.SplitGTMenuDat(inputFileName, outputFolder, flags);
};
SplitFiles(fs, "Splitting " + Path.GetFileName(inputFileName));
}
public void TestSplitFile()
{
IEnumerable <byte[]> newSplitFile = FileSplitter.SplitFile("test.jpg", 1024);
if (File.Exists("newTest.jpg"))
{
File.Delete("newTest.jpg");
}
File.Create("newTest.jpg").Close();
using (var stream = new FileStream("newTest.jpg", FileMode.Append))
{
foreach (var item in newSplitFile)
{
stream.Write(item, 0, item.Length);
}
}
byte[] oldFile = File.ReadAllBytes("test.jpg");
byte[] newFile = File.ReadAllBytes("newTest.jpg");
bool IsEqual = true;
for (int i = 0; i < oldFile.Length; i++)
{
if (oldFile[i] != newFile[i])
{
IsEqual = false;
break;
}
}
Assert.IsTrue(IsEqual);
}
protected void Page_Load(object sender, EventArgs e)
{
if (Debugger.IsAttached)
{
CoerceUser(Session);
}
// string sAPI = Request.Headers["apikey"] ?? "";
// string sID = GetUserIDByAPIKey(sAPI);
// double nBalance = GetUserBalance(sID);
// ToDo: Implement API
string sOriginalName = Request.Headers["OriginalName"] ?? "";
int nDuration = (int)GetDouble(Request.Headers["Duration"] ?? "");
int nDensityLevel = (int)GetDouble(Request.Headers["Density"] ?? "");
string sBlockHash = Request.Headers["BlockHash"] ?? "";
int nHeight = (int)GetDouble(Request.Headers["BlockHeight"] ?? "");
if (sOriginalName != "" && sBlockHash != "")
{
int iPartNo = (int)GetDouble(Request.Headers["PartNumber"] ?? "");
int nTotalParts = (int)GetDouble(Request.Headers["TotalParts"] ?? "");
string sLocalFileName = iPartNo.ToString() + ".dat";
string sWebPath = Request.Headers["WebPath"] ?? "";
string sCPK = Request.Headers["CPK"] ?? "";
string sNetwork = Request.Headers["NetworkID"] ?? "";
string sTempArea = "SVR" + sOriginalName.GetHashCode().ToString();
string sCompleteTempPath = Path.Combine(Common.GetFolderUnchained("Temp"), sTempArea);
string sCompleteWritePath = Path.Combine(sCompleteTempPath, sLocalFileName);
double nFee = GetDouble(Request.Headers["Fee"] ?? "");
string sTXID = Request.Headers["TXID"] ?? "";
string sXML = "";
if (!Directory.Exists(sCompleteTempPath))
{
Directory.CreateDirectory(sCompleteTempPath);
}
using (Stream output = File.OpenWrite(sCompleteWritePath))
using (Stream input = Request.InputStream)
{
input.CopyTo(output);
}
bool f64 = true;
if (f64)
{
string data = System.IO.File.ReadAllText(sCompleteWritePath);
byte[] b64 = System.Convert.FromBase64String(data);
System.IO.File.WriteAllBytes(sCompleteWritePath, b64);
}
if (nTotalParts == iPartNo)
{
string sResurrectionPath = Path.Combine(Common.GetFolderUnchained("Temp"), Guid.NewGuid().ToString() + ".dat");
FileSplitter.ResurrectFile(sCompleteTempPath, sResurrectionPath);
FileInfo fi = new FileInfo(sResurrectionPath);
long iLen = fi.Length;
FileSplitter.RelinquishSpace(sOriginalName);
// Make a temp file for the resurrected version here:
string sStoreKey = sCPK + "/" + sOriginalName;
string sURL = "";
System.IO.File.Delete(sResurrectionPath);
if (sURL != "")
{
sXML = "<status>1</status><complete>1</complete><url>" + sURL + "</url><bytes>" + iLen.ToString() + "</bytes>";
sXML += "<url0>" + sURL + "</url0>";
Response.Write(sXML);
Response.End();
return;
}
else
{
sXML = "<status>0</status>";
Log("Uplink failed ");
Response.Write(sXML);
Response.End();
return;
}
}
// Error Codes
sXML = "<status>1</status>";
Response.Write(sXML);
Response.End();
return;
}
if (true)
{
if (!gUser(this).Admin)
{
MsgBox("Restricted", "Sorry this page is for admins only.", this);
return;
}
}
if (!IsPostBack)
{
FileUpload1.Attributes["onchange"] = "UploadFile(this)";
}
}
/// <summary>
/// Call this to load each file
/// </summary>
/// <param name="filename"></param>
/// <param name="maxCount"></param>
public void ParseFile(
string filename, int maxPass,
CancellationToken cancellationToken,
IProgress <Progress> progress
)
{
var progressValue = new Progress();
var fi = new FileInfo(filename);
var fileLength = fi.Length;
long totalPassCount = 0, totalErrorCount = 0;
var provider = CultureInfo.InvariantCulture;
var done = false;
#if OLDMODE
// var lineCount = 0;
// long bytesRead = 0;
//var lines = File.ReadLines(filename);
//var fileLineAtLastErrorLine = -100L; // this will help fix lines that got split somehow
//todo - better error line merging
foreach (var line1 in lines)
{
#else
var regexes = new[] { passRegex, errorRegex }
;
int matchIndex;
var fileSplitter = new FileSplitter(filename);
var line1 = fileSplitter.Next(regexes, out matchIndex);
do
{
var lineCount = fileSplitter.LineCount;
var bytesRead = fileSplitter.BytesRead;
#endif
cancellationToken.ThrowIfCancellationRequested();
if ((lineCount & 511) == 0)
{
progressValue.Message = String.Format(
"File parsing phase: {0}/{1} kb read ({2:F2}%), {3} lines read, {4} passes read, {5} errors read, {6} error lines",
bytesRead / 1024, fileLength / 1024,
bytesRead * 100.0 / fileLength,
lineCount,
totalPassCount,
totalErrorCount,
ErrorLines.Count
);
progress.Report(progressValue);
}
++lineCount;
bytesRead += line1.Length;
var line = line1;
while (!done && !String.IsNullOrEmpty(line))
{
var passMatch = passRegex.Match(line);
var errMatch = (!passMatch.Success)
? errorRegex.Match(line)
: null;
if (passMatch.Success)
{
var pass = Int32.Parse(passMatch.Groups["pass"].Value);
var frame = Int32.Parse(passMatch.Groups["frame"].Value);
uint offset;
if (
!UInt32.TryParse(passMatch.Groups["offset"].Value, NumberStyles.HexNumber, provider,
out offset))
{
throw new Exception("Illegal format");
}
uint tick;
if (!UInt32.TryParse(passMatch.Groups["tick"].Value, NumberStyles.HexNumber, provider, out tick))
{
throw new Exception("Illegal format");
}
ulong errorCount;
if (
!UInt64.TryParse(passMatch.Groups["errors"].Value, out errorCount))
{
throw new Exception("Illegal format");
}
var passItem = new PassRecord(pass, frame, offset, tick, errorCount);
line = passRegex.Replace(line, "", 1);
// do this just before last pass to ensure we got all errors
if (Passes.Count >= maxPass)
{
done = true;
}
else
{
Passes.Add(passItem);
++totalPassCount;
}
}
else if (errMatch != null && errMatch.Success)
{
var lastpass = Passes.LastOrDefault();
if (lastpass == null)
{
continue;
}
uint offset;
if (
!UInt32.TryParse(errMatch.Groups["offset"].Value, NumberStyles.HexNumber, provider,
out offset))
{
throw new Exception("Illegal format");
}
uint read;
if (!UInt32.TryParse(errMatch.Groups["read"].Value, NumberStyles.HexNumber, provider, out read))
{
throw new Exception("Illegal format");
}
uint desired;
if (
!UInt32.TryParse(errMatch.Groups["desired"].Value, NumberStyles.HexNumber, provider,
out desired))
{
throw new Exception("Illegal format");
}
var errorType = Error.ErrorTypes.Erase;
if (errMatch.Groups["type"].Value == "E")
{
errorType = Error.ErrorTypes.Erase;
}
//else if (passMatch.Groups["type"].Value == "W")
// errorType = Error.ErrorTypes.Write;
else
{
throw new Exception("Illegal format");
}
lastpass.Errors.Add(new Error(lastpass, offset, read, desired, errorType));
totalErrorCount++;
line = errorRegex.Replace(line, "", 1);
}
#if OLDMODE
else
{
var isErrorLine = true; // assume this line will be an error
if (fileLineAtLastErrorLine == lineCount - 1 && ErrorLines.Count > 0)
{
// back to back, pull out
var tempLine = ErrorLines.Last() + line;
isErrorLine &= !passRegex.IsMatch(tempLine);
isErrorLine &= !errorRegex.IsMatch(tempLine);
if (!isErrorLine)
{
// pull last one out and test it
line = tempLine;
ErrorLines.RemoveAt(ErrorLines.Count - 1);
}
}
if (isErrorLine)
{
ErrorLines.Add(line);
line = "";
}
fileLineAtLastErrorLine = lineCount;
} // error line
} // while line not all used up
}
#else
} // while line not all used up
line1 = fileSplitter.Next(regexes, out matchIndex);
} while (!done && !String.IsNullOrEmpty(line1))
public SplitThreadParams(FileSplitter fs, WaitDlg waitDlg)
{
splitter = fs;
dlg = waitDlg;
}
public void Setup()
{
_fileSplitter = new FileSplitter(TestUtils.Logger);
}
public async Task UploadFileIsSuccessful()
{
// arrange
var fileName = "foo.txt";
string filePath = Path.Combine(Path.GetTempPath(), fileName);
File.WriteAllText(filePath, "123");
long fullFileSize = new FileInfo(filePath).Length;
var hashForFile = new MD5FileHasher().CalculateHashForFile(filePath);
int resourceId = 1;
var checkFileResult = new CheckFileResult
{
StatusCode = HttpStatusCode.NoContent,
LastModified = DateTimeOffset.UtcNow,
FileNameOnServer = fileName,
HashForFileOnServer = "MyHash"
};
mockFileService.Setup(
service => service.CheckFileAsync(resourceId))
.ReturnsAsync(checkFileResult);
var sessionId = Guid.NewGuid();
var uploadSession = new UploadSession
{
SessionId = sessionId,
FileUploadChunkSizeInBytes = 1,
FileUploadMaxFileSizeInMegabytes = 10
};
var createSessionResult = new CreateSessionResult
{
StatusCode = HttpStatusCode.OK,
Session = uploadSession
};
mockFileService.Setup(
service => service.CreateNewUploadSessionAsync(resourceId))
.ReturnsAsync(createSessionResult);
var fileSplitter = new FileSplitter();
var countOfFileParts = fileSplitter.GetCountOfFileParts(createSessionResult.Session.FileUploadChunkSizeInBytes, fullFileSize);
var uploadStreamResult = new UploadStreamResult
{
StatusCode = HttpStatusCode.OK,
PartsUploaded = 1
};
mockFileService.Setup(
service => service.UploadStreamAsync(resourceId, sessionId,
It.IsAny <Stream>(), It.IsAny <FilePart>(), fileName, fullFileSize, countOfFileParts))
.ReturnsAsync(uploadStreamResult);
var commitResult = new CommitResult
{
StatusCode = HttpStatusCode.OK,
Session = uploadSession
};
mockFileService.Setup(
service => service.CommitAsync(resourceId, sessionId, fileName, hashForFile, fullFileSize,
It.IsAny <IList <FilePart> >()))
.ReturnsAsync(commitResult);
// act
await this.classUnderTest.UploadFileAsync(resourceId, filePath, this.cancellationToken);
// assert
mockFileService.Verify(
service => service.CheckFileAsync(1),
Times.Once);
mockFileService.Verify(
service => service.CreateNewUploadSessionAsync(resourceId),
Times.Once);
mockFileService.Verify(
service => service.UploadStreamAsync(resourceId, sessionId,
It.IsAny <Stream>(), It.IsAny <FilePart>(), fileName, fullFileSize, countOfFileParts),
Times.Exactly(countOfFileParts));
mockFileService.Verify(
service => service.CommitAsync(resourceId, sessionId, fileName, hashForFile, fullFileSize,
It.IsAny <IList <FilePart> >()),
Times.Once);
}
public void UploadDownloadTestException() => Assert.ThrowsException <UploadFailedException>(() => new FileIOUploaderProvider().Upload(FileSplitter.SplitFile("test.jpg", 1024 * 5), 5).ToList());
