Esempio n. 1
0
        public void TestSplitFilesSmall()
        {
            IEnumerable <string> splitFile = FileSplitter.SplitFile("test.jpg", 1024 * 15, new DirectoryInfo("Parts"), isPathAbsoloute: true);

            List <byte> splitFiles = new List <byte>();

            foreach (var item in splitFile)
            {
                splitFiles.AddRange(File.ReadAllBytes(item));
                File.Delete(item);
            }

            byte[] oldFile = File.ReadAllBytes("test.jpg");
            byte[] newFile = splitFiles.ToArray();

            bool IsEqual = true;

            for (int i = 0; i < oldFile.Length; i++)
            {
                if (oldFile[i] != newFile[i])
                {
                    IsEqual = false;
                    break;
                }
            }


            Directory.Delete("Parts");

            Assert.IsTrue(IsEqual);
        }
Esempio n. 2
0
        static void Main(string[] args)
        {
            Console.WriteLine("Please write path to file");
            string path = Console.ReadLine();
            FileIOUploaderProvider provider = new FileIOUploaderProvider();

            try
            {
                IEnumerable <string> files = FileSplitter.SplitFile(path, provider.MaxChunkSize, new DirectoryInfo("Parts"), isPathAbsoloute: true).ToList();

                foreach (var item in provider.Upload(files, 1000))
                {
                    Console.WriteLine(item);
                }

                foreach (var item in files)
                {
                    File.Delete(item);
                }
            }
            catch (UploadFailedException ex)
            {
                Console.WriteLine("Upload failed");
            }

            Console.WriteLine("Done");
            Console.ReadLine();
        }
Esempio n. 3
0
        static void Main(string[] args)
        {
            Console.WriteLine("Checking path, splitting files");
            Console.WriteLine(Environment.NewLine + "----------------------------------------------------------------");
            Console.WriteLine("Parameters: " + args.Length);
            for (int i = 0; i < args.Length; i++)
            {
                Console.WriteLine(i + "\t" + args[i]);
            }

            if (args.Length != 4)
            {
                Console.WriteLine("\nParameters error! You should pass 3 values!\n");
                Console.ReadLine();
                return;
            }

            string expectedModification = args[0];
            string inputFilePath        = args[1];
            string mappingFilePath      = args[2];
            string outputPath           = args[3] + "\\";

            Console.WriteLine();
            Console.WriteLine("Modification name:\t" + expectedModification);
            Console.WriteLine("Path to input file:\t" + inputFilePath);
            Console.WriteLine("Path to mapping file:\t" + mappingFilePath);
            Console.WriteLine("Path to output folder:\t" + outputPath);
            Console.WriteLine("----------------------------------------------------------------" + Environment.NewLine);


            FileSplitter.SplitFile(inputFilePath);
            Console.WriteLine("Fixing indentations");
            IndentationChecker.CheckIndentations();
            Console.WriteLine("Looking for modifications");
            if (!ModificationSearchTool.FindAndSaveChanges(expectedModification))
            {
                Console.WriteLine("ERROR: Modofication {0} not found", expectedModification);
                Console.ReadLine();
                return;
            }
            Console.WriteLine("Preparing the modifications for saving");
            ModificationCleanerTool.CleanChangeCode();
            Console.WriteLine("Generating Documentation() trigger");
            DocumentationTrigger.UpdateDocumentationTrigger();

            Console.WriteLine("Saving objects");
            SaveTool.SaveObjectsToFiles(outputPath);
            Console.WriteLine("Saving changelogs");
            SaveTool.SaveChangesToFiles(outputPath);
            Console.WriteLine("Generating <next> documentation file");
            SaveTool.SaveDocumentationToFile(outputPath, DocumentationExport.GenerateDocumentationFile(outputPath, mappingFilePath));
            Console.WriteLine("Saving objects grouped");
            SaveTool.SaveObjectModificationFiles(outputPath);
            Console.WriteLine("Success");

            Console.ReadLine();
        }
Esempio n. 4
0
    public void Main(string [] args)
    {
        FileReader              fileReader    = new FileReader(args[0]);
        FileSplitter            fileSplitter  = new FileSplitter();
        IEnumerable <string>    lines         = fileReader.Read();
        IEnumerable <PersonDTO> persons       = fileSplitter.TextSplitter(lines);;
        DataBaseProcessor       dataProcessor = new DataBaseProcessor();

        database.InsertIntoPerson(persons);
    }
Esempio n. 5
0
        private static void SplitFiles(FileSplitter fs, string operation)
        {
            Thread t = new Thread(new ParameterizedThreadStart(SplitThread));

            t.SetApartmentState(ApartmentState.STA);
            using (WaitDlg dlg = new WaitDlg(operation))
            {
                dlg.UpdateStatus("Please wait while files are being extracted and imaged");
                SplitThreadParams stp = new SplitThreadParams(fs, dlg);
                t.Start(stp);
                dlg.ShowDialog();
            }
        }
Esempio n. 6
0
 private SortDriver(
     string sourceFilePath,
     int runSizeHintInMiB,
     int maxDegreeOfParallelism,
     int mergeBufferSizePerRun,
     Action <string> logger)
 {
     _sourceFilePath         = sourceFilePath;
     _runSizeHintInMiB       = runSizeHintInMiB;
     _mergeBufferSizePerRun  = mergeBufferSizePerRun < MinMergeBufferSize ? MinMergeBufferSize : mergeBufferSizePerRun;
     _maxDegreeOfParallelism = maxDegreeOfParallelism;
     _logger             = logger;
     _fileSplitter       = new FileSplitter(logger);
     _repositoryProvider = new ExternalRunRepositoryProvider(Path.GetDirectoryName(sourceFilePath));
 }
Esempio n. 7
0
        async Task TestNumberOfParts()
        {
            var blobId        = new BlobId();
            var numberParts   = 2;
            var blobStoreMock = new Mock <IBlobStore>();

            blobStoreMock.Setup(store => store.Put(It.IsAny <Stream>())).Returns(blobId);
            var splitter   = new FileSplitter(blobStoreMock.Object);
            var fileSystem = TestUtilities.CreateFilesystem();
            var fileStream = fileSystem.FileStream.Create(@"c:\myfile.txt", FileMode.Open);

            var blobids = await splitter.SplitData(numberParts, fileStream);

            Assert.Equal(numberParts, blobids.Count());
        }
Esempio n. 8
0
        public void TestSplitFilesHuge()
        {
            IEnumerable <string> splitFile = FileSplitter.SplitFile("testhuge.zip", 1024 * 1024 * 1024, new DirectoryInfo("TestHugeParts"), isPathAbsoloute: true).ToList();

            // Todo: use an md5 hash instead and then precompute testhuge.zip
            // Comparing for length. because actually comparing 2.6GB of data takes too long
            bool sameLength = new FileInfo("testhuge.zip").Length == splitFile.Select(x => new FileInfo(x)).Sum(x => x.Length);

            foreach (var item in splitFile)
            {
                File.Delete(item);
            }


            Directory.Delete("TestHugeParts");

            Assert.IsTrue(sameLength);
        }
Esempio n. 9
0
        public void FileSplitterTest(int expectedCount, int splitThreshold)
        {
            var inFile = "INFILE.TXT";

            using (StreamWriter sw = new StreamWriter(inFile))
            {
                for (int i = 0; i <= 2000000; i++)
                {
                    sw.WriteLine("123");
                }
            }

            using (var splitter = new FileSplitter(inFile, ".", splitThreshold))
            {
                splitter.SplitFiles();
                Assert.AreEqual(expectedCount, splitter.OutFiles.Count);
            }
        }
Esempio n. 10
0
        public static void SplitCommonPic(IWin32Window parent)
        {
            string inputFileName = MainForm.GetOpenFileName(parent, "Open CommonPic.dat", "CommonPic.dat (*.dat)|commonpic*.*|All Files (*.*)|*.*", null);

            if (String.IsNullOrEmpty(inputFileName))
            {
                return;
            }
            string outputFolder = MainForm.PickFolder(parent, "Pick directory to split to...", Path.GetDirectoryName(inputFileName));

            if (String.IsNullOrEmpty(outputFolder))
            {
                return;
            }
            DebugLogger.Log("Tools", "Splitting and decomping CommonPic {0} to {1}", inputFileName, outputFolder);
            FileSplitter fs = () => {
                GTMP.GTMPFile.ExplodeCommonPic(inputFileName, outputFolder, GTMP.GTMPFile.SplitCommonPicArgs.OutputPngPicture);
            };

            SplitFiles(fs, "Splitting " + Path.GetFileName(inputFileName));
        }
Esempio n. 11
0
        public void UploadDownloadTest()
        {
            FileIOUploaderProvider uploaderProvider = new FileIOUploaderProvider();

            var links = uploaderProvider.Upload(FileSplitter.SplitFile("test.jpg", 51200)).ToList();


            byte[] oldFile = File.ReadAllBytes("test.jpg");
            byte[] newFile = new FileIODownloadProvider().Download(links).Aggregate <IEnumerable <byte> >((x, y) => x.Concat(y)).ToArray();

            bool IsEqual = true;

            for (int i = 0; i < oldFile.Length; i++)
            {
                if (oldFile[i] != newFile[i])
                {
                    IsEqual = false;
                    break;
                }
            }
            Assert.IsTrue(IsEqual);
        }
Esempio n. 12
0
        public static void SplitGTMenuDat(IWin32Window parent)
        {
            string inputFileName = MainForm.GetOpenFileName(parent, "Open GTMenuDat.dat", "GTMenuDat.dat (*.dat)|gtmenudat*.*|All Files (*.*)|*.*", null);

            if (String.IsNullOrEmpty(inputFileName))
            {
                return;
            }
            string outputFolder = MainForm.PickFolder(parent, "Pick directory to split to...", Path.GetDirectoryName(inputFileName));

            if (String.IsNullOrEmpty(outputFolder))
            {
                return;
            }
            DebugLogger.Log("Tools", "Splitting and decomping GTMenuDat {0} to {1}", inputFileName, outputFolder);
            FileSplitter fs = () =>
            {
                const GTMP.GMFile.SplitGTMenuFlags flags = GTMP.GMFile.SplitGTMenuFlags.OutputPngPicture;
                GTMP.GMFile.SplitGTMenuDat(inputFileName, outputFolder, flags);
            };

            SplitFiles(fs, "Splitting " + Path.GetFileName(inputFileName));
        }
Esempio n. 13
0
        public void TestSplitFile()
        {
            IEnumerable <byte[]> newSplitFile = FileSplitter.SplitFile("test.jpg", 1024);

            if (File.Exists("newTest.jpg"))
            {
                File.Delete("newTest.jpg");
            }

            File.Create("newTest.jpg").Close();

            using (var stream = new FileStream("newTest.jpg", FileMode.Append))
            {
                foreach (var item in newSplitFile)
                {
                    stream.Write(item, 0, item.Length);
                }
            }



            byte[] oldFile = File.ReadAllBytes("test.jpg");
            byte[] newFile = File.ReadAllBytes("newTest.jpg");

            bool IsEqual = true;

            for (int i = 0; i < oldFile.Length; i++)
            {
                if (oldFile[i] != newFile[i])
                {
                    IsEqual = false;
                    break;
                }
            }

            Assert.IsTrue(IsEqual);
        }
Esempio n. 14
0
        protected void Page_Load(object sender, EventArgs e)
        {
            if (Debugger.IsAttached)
            {
                CoerceUser(Session);
            }

            // string sAPI = Request.Headers["apikey"] ?? "";
            // string sID = GetUserIDByAPIKey(sAPI);
            // double nBalance = GetUserBalance(sID);
            // ToDo: Implement API

            string sOriginalName = Request.Headers["OriginalName"] ?? "";
            int    nDuration     = (int)GetDouble(Request.Headers["Duration"] ?? "");
            int    nDensityLevel = (int)GetDouble(Request.Headers["Density"] ?? "");
            string sBlockHash    = Request.Headers["BlockHash"] ?? "";
            int    nHeight       = (int)GetDouble(Request.Headers["BlockHeight"] ?? "");

            if (sOriginalName != "" && sBlockHash != "")
            {
                int    iPartNo            = (int)GetDouble(Request.Headers["PartNumber"] ?? "");
                int    nTotalParts        = (int)GetDouble(Request.Headers["TotalParts"] ?? "");
                string sLocalFileName     = iPartNo.ToString() + ".dat";
                string sWebPath           = Request.Headers["WebPath"] ?? "";
                string sCPK               = Request.Headers["CPK"] ?? "";
                string sNetwork           = Request.Headers["NetworkID"] ?? "";
                string sTempArea          = "SVR" + sOriginalName.GetHashCode().ToString();
                string sCompleteTempPath  = Path.Combine(Common.GetFolderUnchained("Temp"), sTempArea);
                string sCompleteWritePath = Path.Combine(sCompleteTempPath, sLocalFileName);
                double nFee               = GetDouble(Request.Headers["Fee"] ?? "");
                string sTXID              = Request.Headers["TXID"] ?? "";
                string sXML               = "";
                if (!Directory.Exists(sCompleteTempPath))
                {
                    Directory.CreateDirectory(sCompleteTempPath);
                }

                using (Stream output = File.OpenWrite(sCompleteWritePath))
                    using (Stream input = Request.InputStream)
                    {
                        input.CopyTo(output);
                    }

                bool f64 = true;
                if (f64)
                {
                    string data = System.IO.File.ReadAllText(sCompleteWritePath);
                    byte[] b64  = System.Convert.FromBase64String(data);
                    System.IO.File.WriteAllBytes(sCompleteWritePath, b64);
                }

                if (nTotalParts == iPartNo)
                {
                    string sResurrectionPath = Path.Combine(Common.GetFolderUnchained("Temp"), Guid.NewGuid().ToString() + ".dat");
                    FileSplitter.ResurrectFile(sCompleteTempPath, sResurrectionPath);
                    FileInfo fi = new FileInfo(sResurrectionPath);

                    long iLen = fi.Length;
                    FileSplitter.RelinquishSpace(sOriginalName);
                    // Make a temp file for the resurrected version here:
                    string sStoreKey = sCPK + "/" + sOriginalName;
                    string sURL      = "";

                    System.IO.File.Delete(sResurrectionPath);
                    if (sURL != "")
                    {
                        sXML  = "<status>1</status><complete>1</complete><url>" + sURL + "</url><bytes>" + iLen.ToString() + "</bytes>";
                        sXML += "<url0>" + sURL + "</url0>";
                        Response.Write(sXML);
                        Response.End();
                        return;
                    }
                    else
                    {
                        sXML = "<status>0</status>";
                        Log("Uplink failed ");
                        Response.Write(sXML);
                        Response.End();
                        return;
                    }
                }
                // Error Codes
                sXML = "<status>1</status>";
                Response.Write(sXML);
                Response.End();
                return;
            }

            if (true)
            {
                if (!gUser(this).Admin)
                {
                    MsgBox("Restricted", "Sorry this page is for admins only.", this);
                    return;
                }
            }

            if (!IsPostBack)
            {
                FileUpload1.Attributes["onchange"] = "UploadFile(this)";
            }
        }
Esempio n. 15
0
        /// <summary>
        /// Call this to load each file
        /// </summary>
        /// <param name="filename"></param>
        /// <param name="maxCount"></param>
        public void ParseFile(
            string filename, int maxPass,
            CancellationToken cancellationToken,
            IProgress <Progress> progress
            )
        {
            var  progressValue = new Progress();
            var  fi = new FileInfo(filename);
            var  fileLength = fi.Length;
            long totalPassCount = 0, totalErrorCount = 0;
            var  provider = CultureInfo.InvariantCulture;
            var  done     = false;

#if OLDMODE
            // var lineCount = 0;
            // long bytesRead = 0;
            //var lines = File.ReadLines(filename);
            //var fileLineAtLastErrorLine = -100L; // this will help fix lines that got split somehow
            //todo - better error line merging
            foreach (var line1 in lines)

            {
#else
            var regexes = new[] { passRegex, errorRegex }
            ;
            int matchIndex;
            var fileSplitter = new FileSplitter(filename);
            var line1        = fileSplitter.Next(regexes, out matchIndex);
            do
            {
                var lineCount = fileSplitter.LineCount;
                var bytesRead = fileSplitter.BytesRead;
#endif

                cancellationToken.ThrowIfCancellationRequested();
                if ((lineCount & 511) == 0)
                {
                    progressValue.Message = String.Format(
                        "File parsing phase: {0}/{1} kb read ({2:F2}%), {3} lines read, {4} passes read, {5} errors read, {6} error lines",
                        bytesRead / 1024, fileLength / 1024,
                        bytesRead * 100.0 / fileLength,
                        lineCount,
                        totalPassCount,
                        totalErrorCount,
                        ErrorLines.Count
                        );
                    progress.Report(progressValue);
                }

                ++lineCount;
                bytesRead += line1.Length;
                var line = line1;

                while (!done && !String.IsNullOrEmpty(line))
                {
                    var passMatch = passRegex.Match(line);
                    var errMatch  = (!passMatch.Success)
                        ? errorRegex.Match(line)
                        : null;
                    if (passMatch.Success)
                    {
                        var pass  = Int32.Parse(passMatch.Groups["pass"].Value);
                        var frame = Int32.Parse(passMatch.Groups["frame"].Value);

                        uint offset;
                        if (
                            !UInt32.TryParse(passMatch.Groups["offset"].Value, NumberStyles.HexNumber, provider,
                                             out offset))
                        {
                            throw new Exception("Illegal format");
                        }

                        uint tick;
                        if (!UInt32.TryParse(passMatch.Groups["tick"].Value, NumberStyles.HexNumber, provider, out tick))
                        {
                            throw new Exception("Illegal format");
                        }

                        ulong errorCount;
                        if (
                            !UInt64.TryParse(passMatch.Groups["errors"].Value, out errorCount))
                        {
                            throw new Exception("Illegal format");
                        }

                        var passItem = new PassRecord(pass, frame, offset, tick, errorCount);
                        line = passRegex.Replace(line, "", 1);

                        // do this just before last pass to ensure we got all errors
                        if (Passes.Count >= maxPass)
                        {
                            done = true;
                        }
                        else
                        {
                            Passes.Add(passItem);
                            ++totalPassCount;
                        }
                    }
                    else if (errMatch != null && errMatch.Success)
                    {
                        var lastpass = Passes.LastOrDefault();
                        if (lastpass == null)
                        {
                            continue;
                        }

                        uint offset;
                        if (
                            !UInt32.TryParse(errMatch.Groups["offset"].Value, NumberStyles.HexNumber, provider,
                                             out offset))
                        {
                            throw new Exception("Illegal format");
                        }
                        uint read;
                        if (!UInt32.TryParse(errMatch.Groups["read"].Value, NumberStyles.HexNumber, provider, out read))
                        {
                            throw new Exception("Illegal format");
                        }
                        uint desired;
                        if (
                            !UInt32.TryParse(errMatch.Groups["desired"].Value, NumberStyles.HexNumber, provider,
                                             out desired))
                        {
                            throw new Exception("Illegal format");
                        }

                        var errorType = Error.ErrorTypes.Erase;
                        if (errMatch.Groups["type"].Value == "E")
                        {
                            errorType = Error.ErrorTypes.Erase;
                        }
                        //else if (passMatch.Groups["type"].Value == "W")
                        //    errorType = Error.ErrorTypes.Write;
                        else
                        {
                            throw new Exception("Illegal format");
                        }
                        lastpass.Errors.Add(new Error(lastpass, offset, read, desired, errorType));
                        totalErrorCount++;
                        line = errorRegex.Replace(line, "", 1);
                    }
#if OLDMODE
                    else
                    {
                        var isErrorLine = true; // assume this line will be an error
                        if (fileLineAtLastErrorLine == lineCount - 1 && ErrorLines.Count > 0)
                        {
                            // back to back, pull out
                            var tempLine = ErrorLines.Last() + line;
                            isErrorLine &= !passRegex.IsMatch(tempLine);
                            isErrorLine &= !errorRegex.IsMatch(tempLine);

                            if (!isErrorLine)
                            {
                                // pull last one out and test it
                                line = tempLine;
                                ErrorLines.RemoveAt(ErrorLines.Count - 1);
                            }
                        }

                        if (isErrorLine)
                        {
                            ErrorLines.Add(line);
                            line = "";
                        }
                        fileLineAtLastErrorLine = lineCount;
                    } // error line
                }     // while line not all used up
            }
#else
                } // while line not all used up
                line1 = fileSplitter.Next(regexes, out matchIndex);
            } while (!done && !String.IsNullOrEmpty(line1))
Esempio n. 16
0
 public SplitThreadParams(FileSplitter fs, WaitDlg waitDlg)
 {
     splitter = fs;
     dlg      = waitDlg;
 }
Esempio n. 17
0
 public void Setup()
 {
     _fileSplitter = new FileSplitter(TestUtils.Logger);
 }
Esempio n. 18
0
        public async Task UploadFileIsSuccessful()
        {
            // arrange
            var    fileName = "foo.txt";
            string filePath = Path.Combine(Path.GetTempPath(), fileName);

            File.WriteAllText(filePath, "123");
            long fullFileSize = new FileInfo(filePath).Length;
            var  hashForFile  = new MD5FileHasher().CalculateHashForFile(filePath);

            int resourceId      = 1;
            var checkFileResult = new CheckFileResult
            {
                StatusCode          = HttpStatusCode.NoContent,
                LastModified        = DateTimeOffset.UtcNow,
                FileNameOnServer    = fileName,
                HashForFileOnServer = "MyHash"
            };

            mockFileService.Setup(
                service => service.CheckFileAsync(resourceId))
            .ReturnsAsync(checkFileResult);

            var sessionId     = Guid.NewGuid();
            var uploadSession = new UploadSession
            {
                SessionId = sessionId,
                FileUploadChunkSizeInBytes       = 1,
                FileUploadMaxFileSizeInMegabytes = 10
            };
            var createSessionResult = new CreateSessionResult
            {
                StatusCode = HttpStatusCode.OK,
                Session    = uploadSession
            };

            mockFileService.Setup(
                service => service.CreateNewUploadSessionAsync(resourceId))
            .ReturnsAsync(createSessionResult);

            var fileSplitter     = new FileSplitter();
            var countOfFileParts = fileSplitter.GetCountOfFileParts(createSessionResult.Session.FileUploadChunkSizeInBytes, fullFileSize);

            var uploadStreamResult = new UploadStreamResult
            {
                StatusCode    = HttpStatusCode.OK,
                PartsUploaded = 1
            };

            mockFileService.Setup(
                service => service.UploadStreamAsync(resourceId, sessionId,
                                                     It.IsAny <Stream>(), It.IsAny <FilePart>(), fileName, fullFileSize, countOfFileParts))
            .ReturnsAsync(uploadStreamResult);

            var commitResult = new CommitResult
            {
                StatusCode = HttpStatusCode.OK,
                Session    = uploadSession
            };

            mockFileService.Setup(
                service => service.CommitAsync(resourceId, sessionId, fileName, hashForFile, fullFileSize,
                                               It.IsAny <IList <FilePart> >()))
            .ReturnsAsync(commitResult);

            // act
            await this.classUnderTest.UploadFileAsync(resourceId, filePath, this.cancellationToken);

            // assert
            mockFileService.Verify(
                service => service.CheckFileAsync(1),
                Times.Once);

            mockFileService.Verify(
                service => service.CreateNewUploadSessionAsync(resourceId),
                Times.Once);

            mockFileService.Verify(
                service => service.UploadStreamAsync(resourceId, sessionId,
                                                     It.IsAny <Stream>(), It.IsAny <FilePart>(), fileName, fullFileSize, countOfFileParts),
                Times.Exactly(countOfFileParts));

            mockFileService.Verify(
                service => service.CommitAsync(resourceId, sessionId, fileName, hashForFile, fullFileSize,
                                               It.IsAny <IList <FilePart> >()),
                Times.Once);
        }
Esempio n. 19
0
 public void UploadDownloadTestException() => Assert.ThrowsException <UploadFailedException>(() => new FileIOUploaderProvider().Upload(FileSplitter.SplitFile("test.jpg", 1024 * 5), 5).ToList());