#! /usr/bin/python """ #+ # NAME: # l1a # PURPOSE: # Script for processing the SMEI L1A data # CATEGORY: # smei/gen/python # CALLING SEQUENCE: # l1a.py # INPUTS: # If a remote server is specified an attempt at downloading by (s)ftp is made: # # remote server # directory on remote site with L1A files # remote user # password # OPTIONAL INPUT PARAMETERS: # -source=source default: /mnt/cdrom # source directory to look for *.buf or *.buf.gz files. # For ftp this is also the directory where downloaded # *.buf.gz are put (write access to the source directory # is needed) # If L1A files are retrieved from a local L1A mirror # (-mirror is set) then this is the directory where the # L1A files taken from the mirror are copied to. # -destination=destination # destination directory for CCD frames: # default: $SMEIDB # -badname=badname # destination directory for L1A files with suspected # corrupted file names. # # -gunzip look for gzipped *.buf.gz files and unzip them, # instead of looking for *.buf files # -maxftp=maxftp maximum number of l1a*.buf.gz files downloaded by ftp, # or copied from local L1A mirror (if -mirror is set) # # -nounpack only download and unzip L1A files. Don't unpack them. # The names of successfully downloaded and unzipped files are # WILL BE ADDED to the list of locally processed L1A files. # -detach only used for ftp downloads. # If set then an L1A file is unpacked immediately after # download is complete, in parallel with the download of the # next L1A file. If not set, all L1A files are downloaded # first, and are then unpacked in a single operation. # # -l1a=l1a name of specific L1A file to be processed # -all_l1a by default only L1A files are processed that are not yet # listed in file 'file_list' (under the assumption that # these have already been dealt with). Set this keyword to # unconditionally process L1A files. # -file_list=file_list # fully-qualified name of file with names of already processed # L1A files. The file must exist already, so if necessary # create an empty one before running this script. # -nowrite suppress writing of CCD frames (useful for testing only) # -overwrite force ovewriting of existing CCD frames. # By default frames are not written to drive if a file for the # frame already exists. This is currently the primary control # for picking up either the first or second telemetry dump # (setting -overwrite speeds up processing by a factor 2). # -nodups if set then an attempt is made to remove duplicate frames # before starting to write individual frames. # -zipdir zip CCD frames when leaving subdirectory # -remove delete *.buf files after processing # -move=move move *.buf files to directory 'move' after processing # -noclones skip clone frames (see href=smei_buf_prep=) # -unblock skips test for running instances of l1a.py in ftp mode. # -nic write .nic files instead of .fts files (OBSOLETE) # -kblob buffer size for reading remote L1A files for sftp. # default is 64 kB. If set to zero the file is downloaded # with a single read (this is very slow for some reason). # -sftp download using sftp instead of ftp. # -mirror looks for L1A files in locally maintained mirror of # remote L1A files (instead of downloading by ftp or sftp) # OUTPUTS: # Updated data in # Exit code is number of L1A files processed # CALLS: # l1a_near, l1a_ftp, l1a_idl, l1a_redo, l1a_gunzip # SEE ALSO: # smei_buf # PROCEDURE: # > Only one instance of the script is allowed to run. # > The list of L1A files found locally or (when using ftp) on the remote site # are compared against the list of l1a file names in file 'file_list' # Only files not on this local list will be downloaded. Override this # behaviour using keyword all_l1a. # MODIFICATION HISTORY: # MAR-2003, Paul Hick (UCSD/CASS) # APR-2003, Paul Hick (UCSD/CASS) # Added -overwrite, -nodups, -zipdir, -remove keywords. # OCT-2003, Paul Hick (UCSD/CASS) # Added -alive keyword. # FEB-2004, Paul Hick (UCSD/CASS) # Added -nounpack keyword. # Added -noclones keyword to l1a_idl # Added -move keyword # MAR-2004, Paul Hick (UCSD/CASS) # Added keyword 'file_list' instead of hardcoding file name $HOME/l1a_files.txt. # FEB-2005, Paul Hick (UCSD/CASS) # Removed -sparseftp option (never used it). # Added -sftp switch to use sftp instead of plain ftp (requires # packages paramiko and pycrypto). # Removed all references to the module string. # SEP-2005, Paul Hick (UCSD/CASS) # After installing Paramiko 1.4 on 64-bit AMD (running Fedora Core) # download would fail for every other file (first downloaded OK, # second failed, third OK, etc.). Modified code to open a new # ssh session for each download. Also introduced kblob keyword. # DEC-2005, Paul Hick (UCSD/CASS) # Added keyword -mirror to be able to process L1A files from a # locally-maintained mirror of the L1A data base. # FEB-2008, Paul Hick (UCSD/CASS) # Added check for existence of env var $L1A. # Added check to prevent two copies of l1a.py to run at the # same time. # DEC-2009, Paul Hick (UCSD/CASS) # Exit code now is number of L1A files processed (was always zero) # APR-2011, Paul Hick (UCSD/CASS; pphick@ucsd.edu) # Added fix to l1a_mirror to detect L1A files with corrupted # file names. These are now copied to the directory specified # in keyword -badname #- """ import os, sys, tempfile, base64 tempfile.tempdir = os.environ['TUB'] from tiny import hide_env, is_there, start, run_cmd, args, count_instances def l1a_near( buf_files ): """ #+ # NAME: # l1a_near # PURPOSE: # Function to update / retrieve list of already processed L1A files # CALLING SEQUENCE: # near_list = l1a_near( buf_files ) # INPUTS: # buf_files list of file names to be added to the list of already # processed L1A files stored in file 'file_list' # (this may be an empty list) # OUTPUTS: # near_list list of processed L1A files currently # PROCEDURE: # The file names (name and type only) of all processed files are stored # in the ascii file 'file_list # MODIFICATION HISTORY: # APR-2003, Paul Hick (UCSD/CASS; pphick@ucsd.edu) #- """ global file_list say = 'l1a_near, ' # Get list of l1a files already processed locally # The names in 'file_list' do NOT include the directory if os.path.exists(file_list): near_list = open(file_list, 'r').read() near_list = near_list.split() else: near_list = [] if len(buf_files) == 0: print say+'%d local L1A files'%len(near_list) else: # Add the names in buf_files to near_list and update 'file_list' for buf_file in buf_files: tmp_file,drop = os.path.splitext(buf_file) if drop != '.gz': tmp_file = buf_file # Drop .gz if it is present drop,tmp_file = os.path.split(tmp_file) # Drop directory near_list.append( tmp_file.lower() ) near_list.sort() # Remove duplicate entries from near_list before writing the list back to disk. new_files = [] for buf_file in near_list: if new_files.count ( buf_file ) == 0: new_files.append ( buf_file ) else: print say+'duplicate '+buf_file near_list = new_files open(file_list, 'w').write('\n'.join(near_list)+'\n') return near_list def l1a_redo( byftp, buf_files, redo_list, negate ): """ #+ # NAME: # l1a_redo # PURPOSE: # Check list of available files against list of files to be reprocessed. # CALLING SEQUENCE: # list = l1a_redo( byftp, buf_files, redo_list, negate ) # INPUTS: # byftp if set, then buf_files is a list of *.buf.gz files (obtained from ftp nlist cmd) # if not set, then buf_files is a list of *.buf files available locally. # # buf_files # byftp set: list of *.buf.gz files (file names, plus extension only) # byftp not set: list of *.buf files (fully-qualified file names) # # redo_list # list of *.buf files (only file name, plus extension) # # negate negate set: return subset of buf_files NOT present in redo_list # negate NOT set: return subset of buf_files also present in redo_list # OUTPUTS: # list list of files will be a subset of buf_files. # PROCEDURE: # Compares entries in buf_files against entries in redo_list. # MODIFICATION HISTORY: # APR-2003, Paul Hick (UCSD/CASS; pphick@ucsd.edu) #- """ say = 'l1a_redo, ' redo_files = [] for buf_file in buf_files: if byftp: # File names obtained from the ftp nlist files are gzipped and may contain # uppercase letters (in particular L1A instead of l1a). Strip the gz extension # and change to lowercase tmp_file,drop = os.path.splitext( buf_file ) if drop != '.gz': tmp_file = buf_file else: # Drop directory (if present), only retain file name and extension drop,tmp_file = os.path.split( buf_file ) tmp_file = tmp_file.lower() count = redo_list.count( tmp_file ) if negate: if count == 0: redo_files.append( buf_file ) else: if count == 0: print say+buf_file+' skipped' else: redo_files.append( buf_file ) if byftp: print say+'%d remote L1A files to process'%len(redo_files) else: print say+'%d local L1A files to process'%len(redo_files) return redo_files def l1a_gunzip(gz_list, ldir, redo_list): """ #+ # NAME: # l1a_gunzip # PURPOSE: # Unzip list of *.buf.gz files (usually files downloaded from Hanscom) # CALLING SEQUENCE: # buf_files = l1a_gunzip(gz_list, ldir, redo_list) # INPUTS: # gz_list list of *.buf.gz files (only file name, plus extension) # ldir directory where *.buf.gz files are located # redo_list only entries in gz_list also on redo_list are processed # (if redo_list is empty all files in gz_list are processed) # OUTPUTS: # buf_files list of successfully unzipped *.buf files # (fully-qualified file names, located in ldir) # PROCEDURE: # Loops over all files in far_list, spawning gunzip for each file individually. # MODIFICATION HISTORY: # APR-2003, Paul Hick (UCSD/CASS; pphick@ucsd.edu) #- """ say = 'l1a_gunzip, ' buf_files = [] # Stores *.buf files (return array) for gz_file in gz_list: buf_file,ext = os.path.splitext(gz_file) gunzip = ext == '.gz' if gunzip: buf_file,ext = os.path.splitext(buf_file) if ext == '.buf': # Check for .buf if len(redo_list) == 0 or redo_list.count( gz_file ) != 0: buf_file = os.path.join( ldir, gz_file ) print say+hide_env(buf_file) if gunzip: sts = os.spawnlp( os.P_WAIT, 'gunzip', 'gunzip', '-f', buf_file ) buf_file,ext = os.path.splitext(buf_file) if os.path.isfile( buf_file ): # Check whether unzipped file exists buf_files.append( buf_file ) return buf_files # Return list of unzipped *.buf files def l1a_summary( wait ): """ #+ # NAME: # l1a_summary # PURPOSE: # CALLING SEQUENCE: # status = l1a_summary( wait ) # INPUTS: # OUTPUTS: # (none) # PROCEDURE: # MODIFICATION HISTORY: # JUL-2003, Paul Hick (UCSD/CASS; pphick@ucsd.edu) #- """ doy_summary = os.path.join( os.environ['HOME'],'l1a_doy_summary.txt' ) status = os.path.exists(doy_summary) if status: doy = open(doy_summary,'r').read() doy = doy.split('\n') # Last day summarized doy = int(doy[0])+1 # Add one doy = '%(doy)d' % vars() # Convert to string main_pro = tempfile.mktemp('.pro') # Write the main IDL program. This is just a single line calling smei_frm_summary, # with a second line to delete the main IDL program after completion. open(main_pro,'w').write( \ 'smei_frm_summary, timeset(doy='+doy+',hr=[0,24]), dest=getenv("TUB"), /silent\n' + \ 'tmp = do_file(/delete, "'+main_pro+'", /silent)\n' + \ 'exit\n' \ ) # Run the IDL program os.spawnlp( wait, 'idl', 'idl', '-quiet', main_pro ) open(doy_summary, 'w').write(doy+'\n') # Update doy_summary file return status def l1a_mtime( f1, f2 ): p1 = (os.stat(f1))[8] p2 = (os.stat(f2))[8] if p1 > p2: sts = 1 elif p1 == p2: sts = 0 else: sts = -1 return sts def l1a_idl( sys_argv, wait, buf_files, ddir ): """ #+ # NAME: # l1a_idl # PURPOSE: # Unpack *.buf files writing separate files for each frame # CALLING SEQUENCE: # l1a_idl( sys_argv, wait, buf_files, ddir) # INPUTS: # buf_files list of *.buf files or *.buf.gz (fully-qualified names) # Must have at least one element. # ddir destination directory # OUTPUTS: # (none) # PROCEDURE: # > Creates main program for starting up the IDL procedure smei_buf # > Writes ascii file with input information for smei_buf, # > Then spawns the main IDL program # > Both the IDL main program and the ascii file will be deleted by smei_buf # MODIFICATION HISTORY: # APR-2003, Paul Hick (UCSD/CASS) # FEB-2004, Paul Hick (UCSD/CASS) # Added -noclones keyword # MAR-2005, Paul Hick (UCSD/CASS; pphick@ucsd.edu) # Added -nic keyword. By default Fits files are written to disk. #- """ say = 'l1a_idl, ' # Type of data to be extracted (default: full frames) nodups = is_there('-nodups', sys_argv) if is_there('-nowrite', sys_argv): overwrite = 2 else: overwrite = is_there('-overwrite', sys_argv) zipdir = is_there('-zipdir' , sys_argv) remove = is_there('-remove' , sys_argv) noclones= is_there('-noclones', sys_argv) silent = is_there('-silent' , sys_argv) mtime = is_there('-mtime' , sys_argv) nic = is_there('-nic' , sys.argv) status = is_there('-status' , sys.argv) moveto = start('-move=', sys.argv ) if moveto != '': if not os.path.isdir(moveto): print 'non-existent directory:', moveto moveto = '' # List of email addresses to notify of completion (default: none) email = start('-email=', sys_argv) if email == '': email = 'no_email' main_pro = tempfile.mktemp('.pro') tmp_list = tempfile.mktemp('.txt') # Write the main IDL program to do the unpacking. This program calls # smei_buf with the name of the input file tmp_list as argument open(main_pro,'w').write( \ 'smei_buf, "'+tmp_list+'"\n' + \ 'tmp = do_file(/delete, "'+tmp_list+'", /silent)\n' + \ 'tmp = do_file(/delete, "'+main_pro+'", /silent)\n' + \ 'exit\n' \ ) # Write the input file for smei_buf iu = open( tmp_list, 'w' ) iu.write( ddir +'\n' ) # Destination directory iu.write( email+'\n' ) # Email addresses to notify of completion # nodups=0: Don't remove duplicate frames # nodups=1: Remove duplicate frames iu.write( ['keep','remove'][nodups]+'\n' ) # overwrite=0: Don't overwrite existing frames # overwrite=1: Overwrite existing frames iu.write( ['skip existing','overwrite existing','do not write'][overwrite]+'\n' ) iu.write( ['raw' ,'gzip' ][zipdir ]+'\n' ) # When leaving subdirectory, zip all frame files if moveto == '': iu.write( ['keep','remove'][remove ]+'\n' )# Delete *.buf files after processing else: iu.write( moveto+'\n' ) iu.write( ['keep' ,'remove'][noclones]+'\n' ) # Skip clone frame iu.write( ['loud' ,'silent'][silent ]+'\n' ) # Level of silence (0,1) iu.write( ['.fts' ,'.nic' ][nic ]+'\n' ) # Nic of Fits files iu.write( ['blind','status'][status ]+'\n' ) # Write status files # List of *.buf files to be processed (fully-qualified) names if mtime: buf_files.sort( l1a_mtime ) else: buf_files.sort() iu.write( '\n'.join(buf_files)+'\n' ) iu.close() # Process the newly arrived files (the IDL program gets its input # from the file tmp_list. os.spawnlp( wait, 'idl', 'idl', '-quiet', main_pro ) return def l1a_ftp_modtime( site, folder, far_list ): ftp_script = tempfile.mktemp('.ftp') # Write the ftp script iu = open( ftp_script, 'w' ) iu.write('cd '+folder+'\n') for x in far_list: iu.write('modtime '+x+'\n' ) iu.write('quit\n' ) iu.close() # This depends on having an entry in .netrc lst = run_cmd( 'cat '+ftp_script+' | ftp '+site, 1 ) os.remove( ftp_script ) lst = lst.split('\n') gmt = [] for x in lst: # x should be in the form # L1A_2005_058_063623_032056.buf.gz 02/27/2005 12:20:09 GMT # Occassionally the file timestamp cannot be retrieved and a line # "550 L1A_...buf.gz: No such file or directory" # shows up. We try to eliminate those here by only accepting lines # starting with L1A_. This also elimates the trailing empty entry. if x.find('L1A_') == 0: y = x.split('\011') y.reverse() y[0] = y[0][6:10]+'/'+y[0][3:5]+'/'+y[0][0:2]+y[0][10:19] gmt.append(y) gmt.sort() lst = [] for x in gmt: lst.append(x[1]) return lst def l1a_ftp( sys_argv, ldir, redo_list, all_l1a, ddir, unblock ): """ #+ # NAME: # l1a_ftp # PURPOSE: # Download .buf.gz files # CALLING SEQUENCE: # buf_files = l1a_ftp( sys_argv, ldir, redo_list, all_l1a, ddir, unblock ) # INPUTS: # sys_argv command line list sys.argv # contains the info to log into the ftp site # ldir local directory where to put the downloaded files # redo_list list of l1a files to process # Only used if len(redo_list) != 0 # all_l1a if set, process all *.buf files # if not set, only *.buf files not processed locally yet will be # downloaded. # OUTPUTS: # buf_files list *.buf.gz to be processed further # if detach is set then the return value buf_files is an empty list. # CALLS: # l1a_near, l1a_redo, l1a_idl # PROCEDURE: # > The presence of ftp-specific keywords -detach, -sparseft and -maxftp is tested # for inside this routine. # MODIFICATION HISTORY: # APR-2003, Paul Hick (UCSD/CASS) # FEB-2004, Paul Hick (UCSD/CASS; pphick@ucsd.edu) # Local L1A file names are now in lower case (l1a* instead of L1A*) #- """ global file_list say = 'l1a_ftp, ' use_sftp = is_there('-sftp' , sys.argv) if use_sftp: kblob = start('-kblob=', sys_argv) if kblob == '': kblob = 64 # Read in blocks of 64 kbytes else: kblob = int(kblob) argv = args( sys_argv ) buf_files = [] # Check whether info is available in .netrc file. If not then all info must # be input through the command line. netrc = os.path.join( os.environ['HOME'], '.netrc' ) username = '' if os.path.exists( netrc ): machines = (open(netrc,'r')).read() machines = machines.split('\n') for m in machines: part = m.split() if part.count(argv[1]) != 0: username = part[3] password = part[5] if username == '': if len(argv) != 5: print say+'specify remote info (site, dir [, user and password])' return buf_files username = argv[3] password = argv[4] print say+'remote site: '+username+'<*****>@'+argv[1]+':'+argv[2] if not unblock: if count_instances(os.path.split(argv[0])[1]+' '+argv[1],0) >= 2: sys.exit() if use_sftp: import paramiko import time keyfile = os.path.join(os.environ['HOME'],'.ssh','known_hosts') iu = open(keyfile, 'r') for line in iu: keylist = line.split(' ') if len(keylist) != 3: continue hosts, keytype, key = keylist if hosts.find(argv[1]) == 0: break else: print say+'no key for '+argv[1] iu.close() sys.exit() iu.close() key = paramiko.RSAKey(data=base64.decodestring(key)) #passwd = '' #keyfile = os.path.join(os.environ['HOME'], '.ssh','id_dsa') #key = paramiko.DSSKey.from_private_key_file(keyfile, passwd) ftp = paramiko.Transport(argv[1]) ftp.connect(hostkey=key, username=username, password=password) #ftp.connect(username=username, pkey=key) #ftp.close() sftp = ftp.open_session() sftp.exec_command('cd '+argv[2]+'; ls *.buf') far_list = sftp.makefile().read() sftp.close() ftp.close() far_list = far_list.split('\n') far_list = far_list[0:len(far_list)-1] else: from ftplib import FTP # Connect to remote site and pull down list of l1a files # These should all be l1a*.buf.gz files ftp = FTP(argv[1],username,password)# Open connection ftp.cwd(argv[2]) # CD to data directory far_list = ftp.nlst() # Get list of files ftp.close() # Close connection # Only download *.buf.gz files (in the past sometimes VCID telemetry # files have ended up in the level1a directory. near_list = [] for far_gzip in far_list: buf,ext = os.path.splitext(far_gzip) if ext == '.gz': buf,ext = os.path.splitext(buf) ext = ext+'.gz' # The file must be a buf.gz file with name # L1A_2003_123_123456_123456.buf.gz, i.e. 33 chars # or a .buf file (30 chars) if ext == '.buf.gz' and len(far_gzip) == 33: near_list.append(far_gzip) elif ext == '.buf' and len(far_gzip) == 30: near_list.append(far_gzip) else: print 'not downloading '+far_gzip far_list = near_list if len(redo_list) != 0: far_list = l1a_redo( 1, far_list, redo_list, 0 ) if not all_l1a: near_list = l1a_near( [] ) far_list = l1a_redo( 1, far_list, near_list, 1 ) if len(far_list) == 0: print say+'no remote L1A files left to download' # Nothing to download, so lets run the summary program #if not l1a_summary( os.P_NOWAIT ): # print say+'no summary plot made' else: # Test for ftp-specific keywords detach = is_there('-detach' , sys_argv) maxftp = start ('-maxftp=' , sys_argv) if maxftp == '': maxftp = '5' maxftp = eval(maxftp) far_list.sort() if len(far_list) > maxftp: far_list = far_list[0:maxftp] mtime = is_there('-mtime' , sys_argv) if mtime: far_list = l1a_ftp_modtime( argv[1], argv[2], far_list ) # Opening an sftp connection once here should work, I think, # but it never worked very well. With Paramiko 1.0 only two L1A # files could be downloaded safely in one haul; with Paramiko 1.4 # every other file would fail to download. #if use_sftp: # #ftp = paramiko.Transport(argv[1]) # #ftp.connect(hostkey=key, username=username, password=password) # #sftp = paramiko.SFTP.from_transport(ftp) #else: # ftp = FTP(argv[1],username,password) # Open connection (again) # ftp.cwd(argv[2]) # CD to data directory if not use_sftp: ftp = FTP(argv[1],username,password) # Open connection (again) ftp.cwd(argv[2]) # CD to data directory for far_gzip in far_list: tt = time.time() near_gzip = os.path.join(ldir,far_gzip.lower()) try: if use_sftp: # Open new sftp connection for each L1A file. # Ugly, but it seems to work. ftp = paramiko.Transport(argv[1]) ftp.connect(hostkey=key, username=username, password=password) sftp = ftp.open_sftp_client() iu = sftp.open( os.path.join(argv[2],far_gzip), 'r', -1 ) sz_far = iu.stat().st_size # Remote file size print say+far_gzip+' -> '+hide_env(ldir)+' (%d bytes)'%sz_far if kblob == 0: # Works but is quite slow data = iu.read() iu.close() ftp.close() sz_near = len(data) (open(near_gzip,'wb')).write(data) else: # 64 kB appears to be about optimal blob = kblob*1024 # Read in blocks of 'blob' bytes sz_near = 0 ju = open(near_gzip,'wb') data = iu.read(blob) while len(data) > 0: # Read until there is nothing left ju.write(data) sz_near = sz_near+len(data) data = iu.read(blob) ju.close() # Close local file iu.close() # Close remote file ftp.close() # Close sftp connection # Compare local and remote file size. Reject the downloaded # file if they do not match. if sz_near == sz_far: tt = time.time()-tt sz_near = sz_near/1024.0 sts = 'sftp complete %.2f MB, %.2f hours, %.2f kB/s'% \ (sz_near/1024.0,tt/3600.0,sz_near/tt) else: sts = 'local and remote size do not match, '+far_gzip else: print say+far_gzip+' -> '+hide_env(ldir) sts = ftp.retrbinary('RETR '+far_gzip, open(near_gzip,'wb').write) except: if use_sftp: ftp.close() # Close sftp connection # Error downloading far_gzip # Variable 'sts' is not defined here print 'exception on '+far_gzip os.remove( near_gzip ) # Add file to list of locally processed files to avoid # downloading over and over again. near_list = l1a_near( [near_gzip] ) print '\n\n\n'+ \ '=================== WARNING =================\n'+ \ near_gzip+'\n'+ \ 'Download interrupted. To force another download\n'+ \ 'remove entry from '+file_list+'\n'+ \ '=============================================\n\n\n' else: print say+sts if (use_sftp and \ ( \ sts.find('sftp complete') == 0 \ ) \ ) or \ (not use_sftp and \ ( \ sts == '226 Transfer complete.' or\ sts == '226 File send OK.' \ ) \ ): if detach: near_gzip = [near_gzip] near_list = l1a_near( near_gzip ) l1a_idl( sys_argv, os.P_NOWAIT, near_gzip, ddir ) else: near_gzip = os.path.split(near_gzip)[1] buf_files.append(near_gzip) else: os.remove( near_gzip ) if not use_sftp: ftp.close() # Close connection (again) if detach: print say+'detached unpack finished' return buf_files def l1a_mirror( sys_argv, ldir, bdir, redo_list, all_l1a ): global file_list say = 'l1a_mirror, ' argv = args( sys_argv ) buf_files = [] far_list = os.listdir( argv[2] ) # Only download *.buf.gz files (in the past sometimes VCID telemetry # files have ended up in the level1a directory. near_list = [] for far_gzip in far_list: buf,ext = os.path.splitext(far_gzip) if ext == '.gz': buf,ext = os.path.splitext(buf) ext += '.gz' # The file must be a buf.gz file with name # L1A_2003_123_123456_123456.buf.gz, i.e. 33 chars # or a .buf file (30 chars) if ext == '.buf.gz' and len(far_gzip) == 33: near_list.append(far_gzip) elif ext == '.buf' and len(far_gzip) == 30: near_list.append(far_gzip) elif bdir != '' and (ext == '.buf.gz' or ext == '.buf') and buf[0:4] == 'L1A_': # File name starts with L1A_ and has the right extension, but has the # wrong length; probably the filename (and hence the header) is corrupt. # These get copied to bdir (if it is defined), where they need to be # dealt with manually (fix header and filename and move to $L1A_GET). # In all cases seen so far the DOY part of the filename gets messed up, # with the time of day and duration remaining intact. The grep checks # whether a file with the indentical time/duration part is already in # file_list (this would mean it has been processed already). # If not then copy to bdir (unless it's there already). if run_cmd('grep '+buf[-13:]+' '+file_list, 0 ) == '': if not os.path.exists( os.path.join(bdir,far_gzip) ): tmp = run_cmd( 'cp -vp '+os.path.join(argv[2],far_gzip)+' '+bdir, 1 ) else: print 'not processing '+far_gzip far_list = near_list if len(redo_list) != 0: far_list = l1a_redo( 1, far_list, redo_list, 0 ) if not all_l1a: near_list = l1a_near( [] ) far_list = l1a_redo( 1, far_list, near_list, 1 ) if len(far_list) == 0: print say+'no L1A files left to process' else: maxftp = start('-maxftp=', sys_argv) if maxftp == '': maxftp = '5' maxftp = eval(maxftp) far_list.sort() if len(far_list) > maxftp: far_list = far_list[0:maxftp] for far_gzip in far_list: near_gzip = far_gzip.lower() tmp = run_cmd( 'cp -vp '+os.path.join(argv[2],far_gzip)+' '+os.path.join(ldir,near_gzip), 0 ) buf_files.append(near_gzip) return buf_files if __name__ == '__main__': name = os.path.split(sys.argv[0])[1] say = name+', ' if not os.environ.has_key('L1A'): print say+'env var L1A not defined' sys.exit() # ========= # Here we check for problems with the Promise board. # Make sure that the SMEI drives are accessible by checking that # $SMEIDB1/2003_033 is a valid directory. if is_there('-alive', sys.argv): if os.path.isdir(os.path.join(os.environ['SMEIDB1'],'2003_033')) and \ os.path.isdir(os.path.join(os.environ['SMEIDB2'],'2003_180')): print say+'SMEI drives available; all is well' else: print say+'SMEI drives not found; cold boot required' email = start('-email=', sys.argv) if email != "": cmd = 'echo "Cold reboot required" | mail '+ \ email+' -s"SMEI drives on '+os.environ["HOSTNAME"]+ \ ' apparently off-line"' status = run_cmd( cmd, 0 ) sys.exit() file_list = start('-file_list=', sys.argv) if file_list == '': print say+'no list of already-processed l1a files available' sys.exit() if not os.path.exists(file_list): print say+'file does not exist:', hide_env(file_list) sys.exit() # Source directory for the L1A files (default: /mnt/cdrom) # This is also the directory where the L1A files are downloaded. ldir = start('-source=', sys.argv) if ldir == '': ldir = '/mnt/cdrom' # Destination directory for suspected corrupt L1A files bdir = start('-badname=', sys.argv) if bdir != '': if not os.path.isdir(bdir): print say+'ignoring non-existent folder '+bdir bdir = '' bymirror = is_there('-mirror', sys.argv) if bymirror: byftp = False unblock = is_there('-unblock', sys.argv) if not unblock: if count_instances('l1a.py -source='+os.environ['L1A_GET'],0) > 1: sys.exit() else: argv = args( sys.argv ) byftp = len( argv ) > 1 unblock = is_there('-unblock', sys.argv) if byftp: if count_instances('l1a.py '+argv[1],0) > 1: sys.exit() elif not unblock: if count_instances('l1a.py -source='+os.environ['L1A_GET'],0) > 1: sys.exit() # Destination directory for the individual frames (default: $SMEIDB) # Frames are put in a subdirectory of this directory. ddir = start('-destination=', sys.argv) if ddir == '': ddir = os.environ['SMEIDB'] if ddir != "SMEIDB?": if not os.path.isdir(ddir): print say+'non-existent directory, '+hide_env(ddir) sys.exit() # Explicit list of L1A files to be processed redo_list = start('-l1a=', sys.argv) if redo_list == '': redo_list == [] else: redo_list = redo_list.split(',') # Downloaded files are gzipped, so we need to unzip. # The first command line arg (if present, should be the remote site) gunzip = is_there('-gunzip' , sys.argv) all_l1a = is_there('-all_l1a' , sys.argv) nounpack= is_there('-nounpack', sys.argv) unblock = is_there('-unblock' , sys.argv) # Get list of l1a files already processed locally # The names in file 'file_list' do NOT include the directory if byftp: # Download *.buf.gz files to ldir and unzip them buf_files = l1a_ftp( sys.argv, ldir, redo_list, all_l1a, ddir, unblock ) buf_files = l1a_gunzip( buf_files, ldir, [] ) elif bymirror: buf_files = l1a_mirror( sys.argv, ldir, bdir, redo_list, all_l1a ) buf_files = l1a_gunzip( buf_files, ldir, [] ) elif gunzip: # Unzip all *.buf.gz files in ldir buf_files = l1a_gunzip( os.listdir(ldir), ldir, redo_list ) else: # Find all *.buf files (probably on /mnt/cdrom) buf_files = [] for buf_file in os.listdir(ldir): if os.path.splitext(buf_file)[1] == '.buf': if len(redo_list) == 0 or redo_list.count(buf_file) != 0: buf_files.append( os.path.join( ldir, buf_file) ) if not all_l1a: near_list = l1a_near( [] ) buf_files = l1a_redo( 0, buf_files, near_list, 1 ) if len(buf_files) == 0: print say+'no *.buf files in '+ldir else: near_list = l1a_near( buf_files )# Add to list of locally processed l1a files if nounpack: moveto = start( '-move=', sys.argv ) if moveto != '': if os.path.isdir(moveto): for buf_file in buf_files: os.rename( buf_file, os.path.join(moveto,(os.path.split(buf_file))[1]) ) else: print 'non-existent directory:', moveto else: l1a_idl( sys.argv, os.P_WAIT, buf_files, ddir) print say+'========== ALL DONE ==========' sys.exit( len(buf_files) )