Edit File: ftpmirror.py
#! /usr/bin/env python """Mirror a remote ftp subtree into a local directory tree. usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat] [-l username [-p passwd [-a account]]] hostname[:port] [remotedir [localdir]] -v: verbose -q: quiet -i: interactive mode -m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o') -n: don't log in -r: remove local files/directories no longer pertinent -l username [-p passwd [-a account]]: login info (default .netrc or anonymous) -s pat: skip files matching pattern hostname: remote host w/ optional port separated by ':' remotedir: remote directory (default initial) localdir: local directory (default current) """ import os import sys import time import getopt import ftplib import netrc from fnmatch import fnmatch # Print usage message and exit def usage(*args): sys.stdout = sys.stderr for msg in args: print msg print __doc__ sys.exit(2) verbose = 1 # 0 for -q, 2 for -v interactive = 0 mac = 0 rmok = 0 nologin = 0 skippats = ['.', '..', '.mirrorinfo'] # Main program: parse command line and start processing def main(): global verbose, interactive, mac, rmok, nologin try: opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v') except getopt.error, msg: usage(msg) login = '' passwd = '' account = '' if not args: usage('hostname missing') host = args[0] port = 0 if ':' in host: host, port = host.split(':', 1) port = int(port) try: auth = netrc.netrc().authenticators(host) if auth is not None: login, account, passwd = auth except (netrc.NetrcParseError, IOError): pass for o, a in opts: if o == '-l': login = a if o == '-p': passwd = a if o == '-a': account = a if o == '-v': verbose = verbose + 1 if o == '-q': verbose = 0 if o == '-i': interactive = 1 if o == '-m': mac = 1; nologin = 1; skippats.append('*.o') if o == '-n': nologin = 1 if o == '-r': rmok = 1 if o == '-s': skippats.append(a) remotedir = '' localdir = '' if args[1:]: remotedir = args[1] if args[2:]: localdir = args[2] if args[3:]: usage('too many arguments') # f = ftplib.FTP() if verbose: print "Connecting to '%s%s'..." % (host, (port and ":%d"%port or "")) f.connect(host,port) if not nologin: if verbose: print 'Logging in as %r...' % (login or 'anonymous') f.login(login, passwd, account) if verbose: print 'OK.' pwd = f.pwd() if verbose > 1: print 'PWD =', repr(pwd) if remotedir: if verbose > 1: print 'cwd(%s)' % repr(remotedir) f.cwd(remotedir) if verbose > 1: print 'OK.' pwd = f.pwd() if verbose > 1: print 'PWD =', repr(pwd) # mirrorsubdir(f, localdir) # Core logic: mirror one subdirectory (recursively) def mirrorsubdir(f, localdir): pwd = f.pwd() if localdir and not os.path.isdir(localdir): if verbose: print 'Creating local directory', repr(localdir) try: makedir(localdir) except os.error, msg: print "Failed to establish local directory", repr(localdir) return infofilename = os.path.join(localdir, '.mirrorinfo') try: text = open(infofilename, 'r').read() except IOError, msg: text = '{}' try: info = eval(text) except (SyntaxError, NameError): print 'Bad mirror info in', repr(infofilename) info = {} subdirs = [] listing = [] if verbose: print 'Listing remote directory %r...' % (pwd,) f.retrlines('LIST', listing.append) filesfound = [] for line in listing: if verbose > 1: print '-->', repr(line) if mac: # Mac listing has just filenames; # trailing / means subdirectory filename = line.strip() mode = '-' if filename[-1:] == '/': filename = filename[:-1] mode = 'd' infostuff = '' else: # Parse, assuming a UNIX listing words = line.split(None, 8) if len(words) < 6: if verbose > 1: print 'Skipping short line' continue filename = words[-1].lstrip() i = filename.find(" -> ") if i >= 0: # words[0] had better start with 'l'... if verbose > 1: print 'Found symbolic link %r' % (filename,) linkto = filename[i+4:] filename = filename[:i] infostuff = words[-5:-1] mode = words[0] skip = 0 for pat in skippats: if fnmatch(filename, pat): if verbose > 1: print 'Skip pattern', repr(pat), print 'matches', repr(filename) skip = 1 break if skip: continue if mode[0] == 'd': if verbose > 1: print 'Remembering subdirectory', repr(filename) subdirs.append(filename) continue filesfound.append(filename) if info.has_key(filename) and info[filename] == infostuff: if verbose > 1: print 'Already have this version of',repr(filename) continue fullname = os.path.join(localdir, filename) tempname = os.path.join(localdir, '@'+filename) if interactive: doit = askabout('file', filename, pwd) if not doit: if not info.has_key(filename): info[filename] = 'Not retrieved' continue try: os.unlink(tempname) except os.error: pass if mode[0] == 'l': if verbose: print "Creating symlink %r -> %r" % (filename, linkto) try: os.symlink(linkto, tempname) except IOError, msg: print "Can't create %r: %s" % (tempname, msg) continue else: try: fp = open(tempname, 'wb') except IOError, msg: print "Can't create %r: %s" % (tempname, msg) continue if verbose: print 'Retrieving %r from %r as %r...' % (filename, pwd, fullname) if verbose: fp1 = LoggingFile(fp, 1024, sys.stdout) else: fp1 = fp t0 = time.time() try: f.retrbinary('RETR ' + filename, fp1.write, 8*1024) except ftplib.error_perm, msg: print msg t1 = time.time() bytes = fp.tell() fp.close() if fp1 != fp: fp1.close() try: os.unlink(fullname) except os.error: pass # Ignore the error try: os.rename(tempname, fullname) except os.error, msg: print "Can't rename %r to %r: %s" % (tempname, fullname, msg) continue info[filename] = infostuff writedict(info, infofilename) if verbose and mode[0] != 'l': dt = t1 - t0 kbytes = bytes / 1024.0 print int(round(kbytes)), print 'Kbytes in', print int(round(dt)), print 'seconds', if t1 > t0: print '(~%d Kbytes/sec)' % \ int(round(kbytes/dt),) print # # Remove files from info that are no longer remote deletions = 0 for filename in info.keys(): if filename not in filesfound: if verbose: print "Removing obsolete info entry for", print repr(filename), "in", repr(localdir or ".") del info[filename] deletions = deletions + 1 if deletions: writedict(info, infofilename) # # Remove local files that are no longer in the remote directory try: if not localdir: names = os.listdir(os.curdir) else: names = os.listdir(localdir) except os.error: names = [] for name in names: if name[0] == '.' or info.has_key(name) or name in subdirs: continue skip = 0 for pat in skippats: if fnmatch(name, pat): if verbose > 1: print 'Skip pattern', repr(pat), print 'matches', repr(name) skip = 1 break if skip: continue fullname = os.path.join(localdir, name) if not rmok: if verbose: print 'Local file', repr(fullname), print 'is no longer pertinent' continue if verbose: print 'Removing local file/dir', repr(fullname) remove(fullname) # # Recursively mirror subdirectories for subdir in subdirs: if interactive: doit = askabout('subdirectory', subdir, pwd) if not doit: continue if verbose: print 'Processing subdirectory', repr(subdir) localsubdir = os.path.join(localdir, subdir) pwd = f.pwd() if verbose > 1: print 'Remote directory now:', repr(pwd) print 'Remote cwd', repr(subdir) try: f.cwd(subdir) except ftplib.error_perm, msg: print "Can't chdir to", repr(subdir), ":", repr(msg) else: if verbose: print 'Mirroring as', repr(localsubdir) mirrorsubdir(f, localsubdir) if verbose > 1: print 'Remote cwd ..' f.cwd('..') newpwd = f.pwd() if newpwd != pwd: print 'Ended up in wrong directory after cd + cd ..' print 'Giving up now.' break else: if verbose > 1: print 'OK.' # Helper to remove a file or directory tree def remove(fullname): if os.path.isdir(fullname) and not os.path.islink(fullname): try: names = os.listdir(fullname) except os.error: names = [] ok = 1 for name in names: if not remove(os.path.join(fullname, name)): ok = 0 if not ok: return 0 try: os.rmdir(fullname) except os.error, msg: print "Can't remove local directory %r: %s" % (fullname, msg) return 0 else: try: os.unlink(fullname) except os.error, msg: print "Can't remove local file %r: %s" % (fullname, msg) return 0 return 1 # Wrapper around a file for writing to write a hash sign every block. class LoggingFile: def __init__(self, fp, blocksize, outfp): self.fp = fp self.bytes = 0 self.hashes = 0 self.blocksize = blocksize self.outfp = outfp def write(self, data): self.bytes = self.bytes + len(data) hashes = int(self.bytes) / self.blocksize while hashes > self.hashes: self.outfp.write('#') self.outfp.flush() self.hashes = self.hashes + 1 self.fp.write(data) def close(self): self.outfp.write('\n') # Ask permission to download a file. def askabout(filetype, filename, pwd): prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd) while 1: reply = raw_input(prompt).strip().lower() if reply in ['y', 'ye', 'yes']: return 1 if reply in ['', 'n', 'no', 'nop', 'nope']: return 0 print 'Please answer yes or no.' # Create a directory if it doesn't exist. Recursively create the # parent directory as well if needed. def makedir(pathname): if os.path.isdir(pathname): return dirname = os.path.dirname(pathname) if dirname: makedir(dirname) os.mkdir(pathname, 0777) # Write a dictionary to a file in a way that can be read back using # rval() but is still somewhat readable (i.e. not a single long line). # Also creates a backup file. def writedict(dict, filename): dir, fname = os.path.split(filename) tempname = os.path.join(dir, '@' + fname) backup = os.path.join(dir, fname + '~') try: os.unlink(backup) except os.error: pass fp = open(tempname, 'w') fp.write('{\n') for key, value in dict.items(): fp.write('%r: %r,\n' % (key, value)) fp.write('}\n') fp.close() try: os.rename(filename, backup) except os.error: pass os.rename(tempname, filename) if __name__ == '__main__': main()