コード例 #1
0
        public static DFile Download(Connection share, SyncFile file, long start, int size)
        {
            try
            {
                var s = GetServerIP(share.MachineName);
                if (s != null)
                {
                    Packet p = new Packet();
                    p.Command = "" + COMMANDS.Download;
                    p.Data    = new object[]
                    {
                        share.Name,
                        share.Token,
                        file,
                        start,
                        size
                    };

                    var ret = (ReturnPacket)send(s.IP.ToString(), p);
                    if (ret.OK)
                    {
                        return((DFile)ret.Data);
                    }

                    return(null);
                }
            }
            catch { }
            return(null);
        }
コード例 #2
0
        private static SyncFile internalcreatezip(object[] files, string sharepath)
        {
            //Console.WriteLine
            _log.Info($"creating zip for share : {sharepath} ,file count = {files.Length}");
            string fn   = Guid.NewGuid().ToString() + ".zip";
            string path = sharepath + ".ts" + _S + "Temp" + _S;

            Directory.CreateDirectory(path);

            var z = RaptorDB.Common.ZipStorer.Create(path + fn, "");

            foreach (var ff in files)
            {
                SyncFile f = (SyncFile)ff;
                try
                {
                    //Console.WriteLine
                    _log.Info("ziping : " + f.F);
                    //_log.Debug("ziping : " + f.F);
                    using (var fs = new FileStream(sharepath + f.F, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                    {
                        z.AddStream(RaptorDB.Common.ZipStorer.Compression.Deflate,
                                    f.F,
                                    fs, f.D, f.D.ToString("yyyy-MM-dd HH:mm:ss"));
                    }
                }
                catch (Exception ex) { _log.Error(ex); }
            }
            z.Close();
            return(new SyncFile {
                F = fn, D = DateTime.Now, S = new FileInfo(path + fn).Length
            });
        }
コード例 #3
0
        public static void DeleteZip(string sharename, string token, SyncFile file)
        {
            var   share = Global.Shares.Find(x => x.Name == sharename);
            DFile ret   = new DFile();

            ret.err = DownloadError.NOTFOUND;
            string path = share.Path + ".ts" + _S + "Temp" + _S;

            if (File.Exists(path + file.F))
            {
                File.Delete(path + file.F);
            }
        }
コード例 #4
0
        private void DoWork()
        {
            while (!_shutdown)
            {
                DateTime _lastSave = DateTime.MinValue;
                while (_conn.isPaused == false && Global.PauseAll == false)
                {
                    if (_que.Count == 0)
                    {
                        SaveQueue();
                        break;
                    }
                    if (_que.Count > 0 && ClientCommands.isConnected(_conn.MachineName))
                    {
                        SyncFile file = null;

                        DownloadAsZipFile();

                        try
                        {
                            _que.TryDequeue(out file);
                            if (file != null)
                            {
                                ProcessFile(file);
                            }
                        }
                        catch (Exception ex)
                        {
                            _log.Error(ex);
                            if (file != null)
                            {
                                _ErrorQue.Enqueue(file);
                            }
                        }
                        if (FastDateTime.Now.Subtract(_lastSave).TotalMinutes > Global.SaveQueueEvery_X_Minute)
                        {
                            SaveQueue();
                            _lastSave = FastDateTime.Now;
                        }
                    }
                    else
                    {
                        Thread.Sleep(3000);
                    }
                }
                Thread.Sleep(1000);
            }
        }
コード例 #5
0
        private void ProcessFile(SyncFile file)
        {
            string fn = _conn.Path + file.F;

            //_log.Info("create dir : " + LongDirectory.GetDirectoryName(fn));
            LongDirectory.CreateDirectory(LongDirectory.GetDirectoryName(fn));

            if (LongFile.Exists(fn))
            {
                // kludge : for seconds resolution
                var fi = new LongFileInfo(fn);
                var d1 = fi.LastWriteTime.Ticks / _tickfilter;
                var d2 = file.D.Ticks / _tickfilter;
                if (d1 >= d2) // skip if already copied or newer
                {
                    return;
                }
            }
            _connInfo.LastFileNameDownloaded = fn;
            fn += ".!torpedosync";
            // FEATURE : resume semi downloaded file

            // remove old semi downloaded file
            LongFile.Delete(fn);

            if (TorpedoSync.Global.isWindows == false)
            {
                fn = fn.Replace("\\", "/");
            }

            _downloading = true;

            if (downloadfile(file, fn, ClientCommands.Download))
            {
                MoveExistingFileToArchive(file.F);
                // rename downloaded file
                LongFile.Move(fn, _conn.Path + file.F);

                LongFile.SetLastWriteTime(_conn.Path + file.F, file.D);
            }
            _downloading = false;
        }
コード例 #6
0
        public static DFile Download(string sharename, string token, SyncFile file, long start, int size)
        {
            var   share = Global.Shares.Find(x => x.Name == sharename);
            DFile ret   = new DFile();

            ret.err = DownloadError.NOTFOUND;
            if (share == null)
            {
                var cs = Global.ConnectionList.Find(x => x.Name == sharename);
                if (cs != null && cs.Token == token)
                {
                    sendfile(file, start, size, cs.Path, ret);
                }
            }
            else if (share.ReadOnlyToken == token || share.ReadWriteToken == token)
            {
                sendfile(file, start, size, share.Path, ret);
            }
            return(ret);
        }
コード例 #7
0
        private static void sendfile(SyncFile file, long start, int size, string path, DFile ret)
        {
            if (LongFile.Exists(path + file.F) == false)
            {
                ret.err = DownloadError.NOTFOUND;
                return;
            }
            var fi = new LongFileInfo(path + file.F);
            // kludge : for seconds resolution
            var d1 = fi.LastWriteTime.Ticks / Global.tickfilter;
            var d2 = file.D.Ticks / Global.tickfilter;

            if (d1 == d2)
            {
                FileStream fs = null;
                try
                {
                    fs = new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
                    byte[] b = new byte[size];
                    fs.Seek(start, SeekOrigin.Begin);
                    fs.Read(b, 0, size);
                    ret.err  = DownloadError.OK;
                    ret.data = b;
                }
                catch
                {
                    ret.err = DownloadError.LOCKED;
                }
                finally
                {
                    if (fs != null)
                    {
                        fs.Close();
                    }
                }
            }
            else
            {
                ret.err = DownloadError.OLDER;
            }
        }
コード例 #8
0
        public static void DeleteZip(Connection share, SyncFile file)
        {
            try
            {
                var s = GetServerIP(share.MachineName);
                if (s != null)
                {
                    Packet p = new Packet();
                    p.Command = "" + COMMANDS.DeleteZip;
                    p.Data    = new object[]
                    {
                        share.Name,
                        share.Token,
                        file
                    };

                    var ret = (ReturnPacket)send(s.IP.ToString(), p);
                }
            }
            catch { }
        }
コード例 #9
0
        public static DFile DownloadZip(string sharename, string token, SyncFile file, long start, int size)
        {
            var   share = Global.Shares.Find(x => x.Name == sharename);
            DFile ret   = new DFile();

            ret.err = DownloadError.NOTFOUND;

            if (share == null)
            {
                var cs = Global.ConnectionList.Find(x => x.Name == sharename && x.Token == token);
                if (cs != null)
                {
                    string path = cs.Path + ".ts" + _S + "Temp" + _S;
                    sendzip(file.F, start, size, path, ret);
                }
            }
            else if (share.ReadOnlyToken == token || share.ReadWriteToken == token)
            {
                string path = share.Path + ".ts" + _S + "Temp" + _S;
                sendzip(file.F, start, size, path, ret);
            }
            return(ret);
        }
コード例 #10
0
ファイル: DeltaProcessor.cs プロジェクト: zhouzu/TorpedoSync
        public static Delta ComputeDelta(State masterstate, State clientstate)
        {
            DateTime dt    = FastDateTime.Now;
            Delta    delta = new Delta();

            SortedList <string, SyncFile> sfiles = new SortedList <string, SyncFile>(clientstate.Files.Count);
            List <string> mfiles = new List <string>(masterstate.Files.Count);

            foreach (var i in clientstate.Files)
            {
                sfiles.Add(i.F, i);
            }

            foreach (var i in masterstate.Files)
            {
                mfiles.Add(i.F);
            }

            mfiles.Sort();
            masterstate.Folders.Sort();

            // if client folder not on master -> delete
            foreach (var i in clientstate.Folders)
            {
                int k = masterstate.Folders.BinarySearch(i);
                if (k < 0)
                {
                    delta.FoldersDeleted.Add(i);
                }
            }

            // if client file not on master -> delete
            foreach (var i in clientstate.Files)
            {
                int k = mfiles.BinarySearch(i.F);
                if (k < 0)
                {
                    delta.FilesDeleted.Add(i.F);
                }
            }

            foreach (var i in masterstate.Files)
            {
                SyncFile o = null;
                if (sfiles.TryGetValue(i.F, out o) == false)
                {
                    delta.FilesAdded.Add(i);
                }
                else
                {
                    // kludge : for seconds resolution
                    var d1 = i.D.Ticks / Global.tickfilter;
                    var d2 = o.D.Ticks / Global.tickfilter;
                    if (d1 > d2)
                    {
                        delta.FilesChanged.Add(i);
                    }
                }
            }
            var s = "Sync R time secs = " + FastDateTime.Now.Subtract(dt).TotalSeconds;

            //Console.WriteLine(s);
            _log.Debug(s);
            return(delta);
        }
コード例 #11
0
ファイル: DeltaProcessor.cs プロジェクト: zhouzu/TorpedoSync
        public static Delta SynchronizeSendRecieve(QueueProcessor quep, Connection share, State clientstate, Delta clientdelta)
        {
            DateTime  dt = FastDateTime.Now;
            Stopwatch sw = new Stopwatch();

            sw.Start();
            State masterLastState = GetLastState(share);
            State masterstate     = GetCurrentState(share.Path, share);

            quep.SetInfo(masterstate);

            _log.Info("  Reading state ms : " + sw.ElapsedMilliseconds);
            sw.Reset();
            var masterdelta = ComputeDelta(masterstate, masterLastState);

            _log.Info("  Master delta ms : " + sw.ElapsedMilliseconds);
            sw.Reset();

            Delta clientsend = new Delta();
            Delta masterque  = new Delta();

            SortedList <string, SyncFile> sfiles = new SortedList <string, SyncFile>(clientstate.Files.Count);
            SortedList <string, SyncFile> mfiles = new SortedList <string, SyncFile>(masterstate.Files.Count);

            foreach (var i in clientstate.Files)
            {
                sfiles.Add(i.F.ToLower(), i);
            }

            foreach (var i in masterstate.Files)
            {
                mfiles.Add(i.F.ToLower(), i);
            }
            _log.Info("  Creating dictionaries ms : " + sw.ElapsedMilliseconds);
            sw.Reset();

            masterque.FilesDeleted   = clientdelta.FilesDeleted;
            masterque.FoldersDeleted = clientdelta.FoldersDeleted;

            if (masterLastState.Files.Count > 0 || masterLastState.Folders.Count > 0)
            {
                clientsend.FilesDeleted   = masterdelta.FilesDeleted;
                clientsend.FoldersDeleted = masterdelta.FoldersDeleted;
            }

            // check master files not exist or older on client -> que for client
            foreach (var i in masterstate.Files)
            {
                SyncFile o = null;
                if (sfiles.TryGetValue(i.F.ToLower(), out o) == false)
                {
                    clientsend.FilesAdded.Add(i);
                }
                else
                {
                    // kludge : for seconds resolution
                    var d1 = i.D.Ticks / Global.tickfilter;
                    var d2 = o.D.Ticks / Global.tickfilter;
                    if (d1 > d2)
                    {
                        clientsend.FilesChanged.Add(i);
                    }
                }
            }
            _log.Info("  Generate client delta ms : " + sw.ElapsedMilliseconds);
            sw.Reset();

            // check client files not exist or older on master -> que for master
            foreach (var i in clientstate.Files)
            {
                SyncFile o = null;
                if (mfiles.TryGetValue(i.F.ToLower(), out o) == false)
                {
                    masterque.FilesAdded.Add(i);
                }
                else
                {
                    // kludge : for seconds resolution
                    var d1 = i.D.Ticks / Global.tickfilter;
                    var d2 = o.D.Ticks / Global.tickfilter;
                    if (d1 > d2)
                    {
                        masterque.FilesChanged.Add(i);
                    }
                }
            }
            _log.Info("  Generating master que ms : " + sw.ElapsedMilliseconds);
            sw.Reset();

            var s = "Sync RW time secs = " + FastDateTime.Now.Subtract(dt).TotalSeconds;

            _log.Debug(s);
            quep.QueueDelta(masterque);

            SaveState(share, masterstate);

            return(clientsend);
        }
コード例 #12
0
        private bool downloadfile(SyncFile file, string saveto, Func <Connection, SyncFile, long, int, DFile> func)
        {
            long left  = file.S;
            int  retry = 0;
            int  mb    = Global.DownloadBlockSizeMB * Global.MB;

            //Stopwatch sw = new Stopwatch();
            //sw.Start();
            while (left > 0)
            {
                long len   = left;
                long start = file.S - left;
                if (len > mb)
                {
                    len = mb;
                }
                //sw.Reset();
                DFile df = func(_conn, file, start, (int)len);
                //_connInfo.Mbs = (len / sw.ElapsedTicks) / (1000*TimeSpan.TicksPerSecond) ;
                if (df == null)
                {
                    retry++;
                    Thread.Sleep(new Random((int)FastDateTime.Now.Ticks).Next(2000) + 500);
                    if (retry > 10)
                    {
                        _log.Info(" null ");
                        _que.Enqueue(file);
                        //if (LongFile.Exists(saveto))
                        LongFile.Delete(saveto);
                        return(false);
                    }
                }
                else if (df.err == DownloadError.OK)
                {
                    retry = 0;
                    string ifn = saveto;
                    if (TorpedoSync.Global.isWindows == false)
                    {
                        ifn = saveto.Replace("\\", "/");
                    }
                    _log.Info("len = " + len + " , " + saveto.Replace(".!torpedosync", ""));
                    left -= len;
                    LongDirectory.CreateDirectory(LongDirectory.GetDirectoryName(ifn));
                    // save to disk
                    var fs = new FileStream(ifn, FileMode.OpenOrCreate);
                    fs.Seek(0L, SeekOrigin.End);
                    fs.Write(df.data, 0, df.data.Length);
                    fs.Close();
                    if (left == 0)
                    {
                        var fi = new LongFileInfo(saveto);
                        if (fi.Length != file.S)
                        {
                            // FEATURE : file length don't match
                            _log.Info("file length don't match.");
                            return(false);
                        }
                    }
                }
                else if (df.err == DownloadError.OLDER || df.err == DownloadError.LOCKED)
                {
                    _ErrorQue.Enqueue(file);
                    _connInfo.LastFileNameDownloaded = "";
                    _log.Info("Locked/Older : " + saveto);
                    return(false);
                }
                else if (df.err == DownloadError.NOTFOUND)
                {
                    _log.Info("Not Found : " + saveto);
                    _connInfo.LastFileNameDownloaded = "";
                    _ErrorQue.Enqueue(file);
                    LongFile.Delete(saveto);
                    return(false);
                }
            }
            return(true);
        }
コード例 #13
0
        private void DownloadAsZipFile()
        {
            SyncFile file = null;

            if (Global.BatchZip)
            {
                int             size    = 0;
                int             mb      = Global.BatchZipFilesUnderMB * Global.MB;
                List <SyncFile> ziplist = new List <SyncFile>();
                while (_que.Count > 10)
                {
                    if (_que.TryPeek(out file))
                    {
                        if (file.S + size < mb &&
                            file.S < Global.SmallFileSize) // small files
                        {
                            _que.TryDequeue(out file);
                            ziplist.Add(file);
                        }
                        else
                        {
                            break;
                        }
                    }
                }
                if (ziplist.Count > 0)
                {
                    try
                    {
                        _downloading = true;
                        var sf = ClientCommands.CreateZip(_conn, ziplist);
                        if (sf != null)
                        {
                            string path = _conn.Path + ".ts" + _S + "Temp" + _S;
                            LongDirectory.CreateDirectory(path);

                            if (downloadfile(sf, path + sf.F, ClientCommands.DownloadZip))
                            {
                                string zfn = path + sf.F;
                                string fn  = sf.F;

                                if (TorpedoSync.Global.isWindows == false)
                                {
                                    zfn = zfn.Replace("\\", "/");
                                }

                                var zf = ZipStorer.Open(zfn, FileAccess.Read);
                                foreach (var z in zf.ReadCentralDir())
                                {
                                    if (TorpedoSync.Global.isWindows)
                                    {
                                        fn = z.FilenameInZip.Replace("/", "\\");
                                    }
                                    else
                                    {
                                        fn = z.FilenameInZip;
                                    }

                                    MoveExistingFileToArchive(fn);
                                    LongDirectory.CreateDirectory(LongDirectory.GetDirectoryName(_conn.Path + fn));
                                    try
                                    {
                                        _log.Info("unzip : " + fn);
                                        //_log.Debug("unzip : " + fn);
                                        using (var fs = new FileStream(_conn.Path + fn, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite))
                                        {
                                            _connInfo.LastFileNameDownloaded = fn;
                                            zf.ExtractFile(z, fs);
                                            fs.Close();
                                        }
                                        var dt = z.ModifyTime;
                                        if (z.Comment != "")
                                        {
                                            //_log.Info($"date {z.Comment}");
                                            if (DateTime.TryParse(z.Comment, out dt) == false)
                                            {
                                                dt = z.ModifyTime;
                                            }
                                        }
                                        var dtt = dt;
                                        if (TorpedoSync.Global.isWindows)
                                        {
                                            dtt = dt.ToUniversalTime();
                                        }
                                        LongFile.SetLastWriteTime(_conn.Path + fn, dtt);
                                    }
                                    catch (Exception ex) { _log.Error(ex); }
                                }
                                zf.Close();
                                ClientCommands.DeleteZip(_conn, sf);
                                _log.Info("Decompress zip done : " + sf.F);
                                LongFile.Delete(zfn);
                            }
                        }
                        _downloading = false;
                    }
                    catch (Exception ex) { _log.Error(ex); }
                }
            }
        }