#! /usr/bin/python """ #+ # NAME: # mirror # PURPOSE: # Simple interface to Perl script 'mirror' # CATEGORY: # smei/gen/python # CALLING SEQUENCE: # From command line: # # mirror.py job= local_dir= remote_dir= # remote_site= remote_user= remote_password= # recursive= recurse_hard= get_patt= # # As module: # # sts = mirror(job, local_dir, remote_site, remote_dir, remote_user, # remote_password, recursive, recurse_hard) # OPTIONAL INPUT PARAMETERS: # default: 'job' # job name (used in the name of the temporary # mirror package file in $TUB) # default: environment variable 'SMEI' # local directory where the mirror copy is maintained # The default is the location $SMEI of the local copy # of the SMEI master tree. # default: 'ips.ucsd.edu' # remote server # default: '/loc_smei' # directory tree on remote site to be mirrored # The default is a copy of the SMEI master accessible # by anonymous ftp only. # default: 'anonymous' # default: '$USER@$HOSTNAME' # user and password used by the mirror script to # log into the ftp server. # if remote_password='netrc' then an attempt is made # to obtain the password from $HOME/.netrc # recursive = # default: true # recurse_hard= # default: true (if recursive is set) # false (if recursive is not set) # passive_ftp= # default: false # get_patt = # local_ignore = # -save if set the package file created for mirror.pl is not deleted # OUTPUTS: # sts return status of mirror module # = None : if mirror was aborted. This currently occurs # only if mirror is run on ips. # = [list] : list of files downloaded (this list may be empty!!) # # updated by mirroring . The results are appended # to the log file $TUB/.log # CALLS: # mirror # RESTRICTIONS: # The following environment variables need to be defined: # SMEI points to the local copy of the SMEI master tree # SSW points to the root of the SolarSoft tree # (the Perl script mirror is part of SolarSoft). # PROCEDURE: # Location of Perl script 'mirror' and name of # mirror package file used to mirror the remote directory. # (the package file is created on the fly and is deleted again). # MODIFICATION HISTORY: # DEC-2002, Paul Hick (UCSD/CASS) # Finalized original version # DEC-2002, Paul Hick (UCSD/CASS) # Added a check to avoid overwriting the SMEI master tree # on ips.ucsd.edu # MAR-2003, Paul Hick (UCSD/CASS) # Modified to allow mirror to run as a module. # NOV-2003, Paul Hick (UCSD/CASS) # Instead of return 0 for failure and 1 for success, the return # value is now None for failure and a list of downloaded files for # success. # DEC-2003, Paul Hick (UCSD/CASS) # Changed default of recurse_hard to false if recursive is not set. # It looks like recursive=false is ignored if recurse_hard=true (the # previous default) overrides. # JAN-2004, Paul Hick (UCSD/CASS) # Added check for existence of Perl script mirror. # Added check for presence of Perl using first line of mirror script. # OCT-2005, Paul Hick (UCSD/CASS) # Added local_ignore keyword. # NOV-2005, Paul Hick (UCSD/CASS; pphick@ucsd.edu) # Added passive_ftp keyword. #- """ import os from tiny import dict_entry, start, is_there def mirror(args): perl_mirror = os.path.join(os.environ['SSW'],'gen','mirror','mirror.pl') #mirror = os.path.join(os.environ['SMEI'],'gen','perl','mirror','mirror') # Make sure that the mirror script is where it is supposed to be, and # check that Perl is where the script thinks it is. if os.path.isfile( perl_mirror ): perl = (open( perl_mirror, 'r' )).readline() perl = perl[0:len(perl)-1] if perl.find('#!') == 0: perl = perl[2:] if not os.path.isfile( perl ): print "mirror, Perl not found:", perl return [] else: print "mirror, file not found:", perl_mirror return [] temp = os.environ['TUB'] job = dict_entry( args, 'job' , 'job' ) local_dir = dict_entry( args, 'local_dir' , os.environ['SMEI']+'/') remote_site = dict_entry( args, 'remote_site' , 'ips.ucsd.edu' ) remote_dir = dict_entry( args, 'remote_dir' , '/loc_smei/' ) remote_user = dict_entry( args, 'remote_user' , 'anonymous' ) remote_password = dict_entry( args, 'remote_password', os.environ['USER']+'@'+os.environ['HOSTNAME'] ) recursive = dict_entry( args, 'recursive' , 'true' ) recurse_hard = dict_entry( args, 'recurse_hard' , 'true' ) passive_ftp = dict_entry( args, 'passive_ftp' , 'false' ) get_patt = dict_entry( args, 'get_patt' , '' ) local_ignore = dict_entry( args, 'local_ignore' , '' ) save = dict_entry( args, 'save' , 0 ) if os.environ['HOSTNAME'] == "ips.ucsd.edu": if local_dir == os.environ['SMEI']+'/': print '********** Trying to overwrite the SMEI master tree, hey?' print '********** Absolutely no way !!!' return None package = os.path.join( temp, job+'.package' ) update_log = os.path.join( temp, job+'.log' ) if recursive == 'false': recurse_hard = 'false' if remote_password == 'netrc': hosts = (open( os.path.join(os.environ['HOME'], '.netrc'), 'r' )).read() hosts = hosts.split('\n') for host in hosts: tmp = host.split() if len(tmp) >= 6: if tmp[1] == remote_site and tmp[3] == remote_user: remote_password = tmp[5] break if remote_password == 'netrc': print 'mirror, could not determine password from .netrc' return [] lines = [ 'package=%s'%job , 'site=%s'%remote_site , 'remote_dir=%s'%remote_dir , 'local_dir=%s'%local_dir , 'remote_user=%s'%remote_user , 'remote_password=%s'%remote_password, 'recursive=%s'%recursive , 'recurse_hard=%s'%recurse_hard , 'mode_copy=true' , 'passive_ftp=%s'%passive_ftp , 'use_timelocal=false' , 'user=%s'%os.environ['USER'] , 'group=users' , 'delete_excl=.+' , 'max_delete_files=99%' , 'max_delete_dirs=99%' , 'update_log=%s'%update_log , ] if get_patt != '': lines.append( 'get_patt='+get_patt) if local_ignore != '': lines.append( 'local_ignore='+local_ignore) (open( package, 'w' )).write( '\n'.join( lines )+'\n' ) # Before running the mirror script read the log file if os.path.isfile( update_log ): log = (open( update_log, 'r' )).read() log_len = len( log ) else: log_len = 0 # Run Perl mirror script #sts = os.spawnlp( os.P_WAIT, perl_mirror, perl_mirror, package ) lines = (os.popen( perl_mirror+' '+package )).read() #print lines if not save: os.remove( package ) if os.path.isfile( update_log ): # Read the log file again. # Remove content of log file already present prior to running mirror script # Build a list of files retrieved by mirror. log = (open( update_log, 'r' )).read()[log_len:].split('\n') got = [ new.split()[1] for new in log if new.find('Got ') == 0 ] else: print "mirror, log file not found:", update_log got = [] return got if __name__ == '__main__': import sys args = dict \ ( [ \ ( 'job' , start( 'job=' , sys.argv ) ), \ ( 'local_dir' , start( 'local_dir=' , sys.argv ) ), \ ( 'remote_site' , start( 'remote_site=' , sys.argv ) ), \ ( 'remote_dir' , start( 'remote_dir=' , sys.argv ) ), \ ( 'remote_user' , start( 'remote_user=' , sys.argv ) ), \ ( 'remote_password' , start( 'remote_password=' , sys.argv ) ), \ ( 'recursive' , start( 'recursive=' , sys.argv ) ), \ ( 'recurse_hard' , start( 'recurse_hard=' , sys.argv ) ), \ ( 'passive_ftp' , start( 'passive_ftp=' , sys.argv ) ), \ ( 'get_patt' , start( 'get_patt=' , sys.argv ) ), \ ( 'local_ignore' , start( 'local_ignore=' , sys.argv ) ), \ ( 'save' , is_there( '-save' , sys.argv ) ) \ ] ) status = mirror(args) sys.exit()