pro ref_super, week_str, days0=days0, days1=days1, stages=stages, $ dir1=dir1, dir2=dir2, dir3=dir3, dirweek=dirweek, dir_reverse=dir_reverse, $ qstop=qstop, check_del=check_del, flare1=flare1, flare2=flare2, srsget=srsget, $ ref2_subset=ref2_subset, ref2_day=ref2_day, nodelete=nodelete, $ notstages=notstages, ftp=ftp, not_sxt_patrol=not_sxt_patrol, start_stage=start_stage, $ sfd_ist=sfd_ist, override_runaway=override_runaway, retry=retry, $ testing=testing ; ;+ ;NAME: ; ref_super ;PURPOSE: ; To copy data from SIRIUS and reformat it. There are several ; different reformatting stages and several different types of ; files that are produced. ;STAGES: ; (0) copy the data from SIRIUS in 24 hour chunks, ; (1) refomat the 24 hour chunks ; (2) break the 24 hour files into 1 orbit per file ; (3) create the PNT file ; (4) create the OBS file ; (5) send mail that data copy to on-line can start ; (6) create the SDL and SDC files (and SDW if needed) ; (7) create the SOT files ; (8) create the SSL, SEL and SRD files (TODO) ; (9) create the SFS and SDP files ; (10) to make the SFD files. ; (11) (spare) (old ADS_INTO_PNT) ; (12) (spare) (old copy weekly files to LPARL) ; (13) create the SFT files ; (14) create the SFC files (slf 5-may-1995) ; (15) (spare) ; (16) (spare) ; (17) (spare) ; (18) create the MKTAP.COM file ; (19) create the SXL and SXA files ; (20) create the SSC files ; (21) copy flares to flare disk ; (22) create the SLS and SLD files ; (23) call MK_MDM to make the MO disk map ; (24) creation of the GOES 3 sec data files (G6D, G7D) ; (25) create the SXC files ;INPUT: ; week_str- The week to be reformatted in string for YY_WW ; For example '91_49' ;OPTIONAL INPUT: ; days0 - the days for stage 0. It can be an array ; 0=sunday, 1=monday, ... ; If not passed, all 7 days are done ; days1 - the days for stage 1. It can be an array ; 0=sunday, 1=monday, ... ; If not passed, all 7 days are done ; stages - A boolean array of 10 elements long specifiying ; whether a stage should be peformed or not. If not ; passed, then all stages are run. See the above ; description for details. ; notstages - a boolean array specifying wither a stage should ; NOT be run. All other stages are run. ; dir1 - The directory to put the output for stage one (the ; copying of the raw telemetry) It is also the input ; directory for stage 2. ; If not specified, the /yd disks are searched for the ; directory "reformat_a1" and "reformat_a2" ; dir2 - The output directory for stage 2 (reformatting in 24 ; hour chunks) ; If not specified, the /yd disks are searched for the ; directory "reformat_b1" and "reformat_b2" ; dir3 - The output directory for stage 3 (breaking 24 hour ; chunks into one orbit files) ; If not specified, the /yd disks are searched for the ; directory "reformat_a1" and "reformat_a2" ; ; NOTE: If the "dir" variables are arrays, the program ; assumes that however the data had to be broken up in ; stage 1, it will be broken up that way in stage 2. And ; however it is broken up in stage 2, it will be broken ; up in stage 3 (broken up between directories/disks). ; ; NOTE: Anytime all days are selected, all existing files ; in the output directory are deleted before starting the ; next stage. This is because it assumes that it was used ; for the output two stages ago and is no longer needed. ; dirweek - The directory where the weekly files for MKTAP should be ; copied to. If not passed, it will search for "reformat_week" ; dir_reverse - Used when using the default directories and we now ; want to start with "reformat_b" because of the odd number ; of read/writes to the disks ; check_del - If set, it will check that the output directory for stage #1 ; is empty before continuing. This allows the user to start ; the SIRIUS copy before backing up the previous week and moving ; it out of the way. This is because the of the swapping back and forth ; between a pair of directories and the automatic deleting of ; files ; ftp - If set, use FTP option for GET_SIRIUS ; start_stage - The stage number to start with (if 5 is passed, do stage ; #5 and all stages after it) ; ; ref2_subset - The prefixes to perform REFORMAT2 stage on ; ['sfr', 'spr', 'bda', 'wda', 'hda', 'ada', 'cba'] ; 0 1 2 3 4 5 6 ; nodelete - do not delete files prior to entry (caller handles during ; multi-tasking operations)... ;HISTORY: ; Written Dec '91 by M.Morrison ; Mar-92 (MDM) - Modifications and refinement to work "smoother" ; 13-Mar-92 (MDM) - Changed from 4 to 5 stages (broke obs and pnt ; generation into two separate stages) ; - Made it so that the end time for the last day ; gets an additional 1.5 hours of data to put datasets ; in the proper week ; 14-Mar-92 (MDM) - Modified to delete all files in the output directory ; is the next stage is going to work on all 7 days. ; 16-Mar-92 (MDM) - Changed MK_OBS and MK_PNT to use "file_list" instead ; of "findfile" ; 17-Mar-92 (MDM) - Changed GET_SIRIUS to use FLARE2 ; 18-Mar-92 (MDM) - Adjusted the log printing ; 19-Mar-92 (MDM) - Changed so that "dir" can be an array and that the ; writing to the output disk checks to see if at least ; 200 megabytes are free before writting output file for ; the stage1 (SIRIUS copy) - partition that way from then ; on. ; 20-Mar-92 (MDM) - Added stages 6 (MKTAP) and 7 (MK_SFD) ; 21-Mar-92 (MDM) - Made the MKTAP portion generate a second MKTAP1.COM ; script file so can write to two tape drives at once. ; - Also MKTAP stage sends a mail message to whoever is ; running the program to tell him it's ready for tape ; making. ; 24-Mar-92 (MDM) - Added option of using FLARE1 instead of FLARE2 ; 15-Apr-92 (MDM) - Modified so that MKTAP is only passed the directories ; where there is data (dir3 value) ; 16-Apr-92 (MDM) - Fixed major bug where last orbit file for a given day ; was being clobbered by the partial first orbit of the ; next day. Should only effect the ADA and CBA files(?) ; 17-Apr-92 (MDM) - Added "ref2_subset" to do only a subset of the prefixes ; 18-Apr-92 (MDM) - Added "dirweek" option for MKTAP ; 23-Apr-92 (MDM) - Added "ref2_day" option (only run reformat2 on one day) ; - Moved some of the stage definitions around, added stages ; to create SSL, SEL, SRF, SDL, SDC, SOT and SFU files. ; - Added "check_del" option ; - Added linford to get all messages when MKTAP finishes ; - Made creation of log file a must ; 26-Apr-92 (MDM) - Added SDW file creation also ; 3-May-92 (MDM) - Added "srsget" parameter input (to run srsget2 on ; flare1) ; 4-May-92 (MDM) - Adjusted way that # of days is calculated (because ; of weeks 53 and 1) ; - Added MK_EVN to stage 5 (where MK_OBS is run) ; 12-May-92 (MDM) - Adjusted to switch to second disk when 250 free (old was 200 megabytes) ; 24-May-92 (MDM) - Added options "flare2" and "notstages" ; Changed the default SRSGET to use FLARE1-SRSGET2 ; 25-May-92 (MDM) - Changed the logic of NOTSTAGES ; 26-May-92 (MDM) - Changed MK_PNT and MK_OBS calls not to passed the ; output file name. It will be built in those ; routines now. ; 30-May-92 (MDM) - Changed to do a listing on the aborted output file ; when GET_SIRIUS bombs out. ; 8-Jul-92 (MDM) - Modified so that stage 5 does not do the MKTAP ; step, instead stage 11 does. ; - Also added stage 12 which does a file copy of the ; weekly files to LPARL. ; - Wrote SFD and SFW files to weekly directory directly ; instead of '/yd1/sfd' ; - Changed "rm" command to set default there first ; since the large number of files+pathname causes the ; delete command to bomb ; 18-Jul-92 (MDM) - Added FREELAND to get all mail messages ; 4-Aug-92 (MDM) - Removed "days" keyword and added "days0" and "days1" ; for stage 0 and 1 (days used to be used for both stage 0 and 1) ; - Added /AUTO_DEL to REFORMAT call so old files are automatically deleted if they already exist ; 20-Aug-92 (MDM) - Added LEMEN to get all mail messages ; 27-Aug-92 (MDM) - Moved MKTAP from stage 11 to stage 18 ; - Deleted stage 12 (copy data to LPARL) ; - Added stages ; (9) create the SFS file ; (11) ADS_INTO_PNT ; (18) create the MKTAP.COM file ; (19) create the SXL files ; (20) create the SSC files ; (21) copy flares to flare disk ; - Added keyword FTP for GET_SIRIUS ; 28-Aug-92 (MDM) - Added /NOT_SXT_PATROL ; - Added START_STAGE ; 6-Oct-92 (MDM) - Adjusted call to CHK_FLARES to only pass the ; directories which have data in them. ; (passed "dir3_act" instead of "dir3") ; 9-Oct-92 (MDM) - Removed lemen from mail list ; 14-Oct-92 (MDM) - Changed from using a "hard wired" list of user's ; to notify by mail to using the environment ; variable $MAIL_REF_SUPER ; 16-Oct-92 (MDM) - Modifed stage 20 to make SSE and SSW in addition ; to making SSC (synoptic files) ; 20-Oct-92 (MDM) - Changed /CHECK_DEL call to FILE_LIST from ; ff = file_list(dir2, '*') to ; ff = file_list(dir2, '') ; - Added /BIG switch CHK_FLARES call ; 22-Oct-92 (SLF) - Changed do_ads call to do_aip call (stage 11) ; (ads copy done external to ref_super) ; 26-Oct-92 (SLF) - Use /actual switch in adsdirs call since old ; data sets are not in standard 4 week queue ; 15-Jan-93 (MDM) - Added stage 22 (create SLS and SLD) ; 3-Feb-93 (MDM) - Removed /BIG switch CHK_FLARES call ; 24-Feb-93 (MDM) - Added SFD_IST keyword optional parameter so that ; MK_SFD stage can start at a later image. ; 29-Mar-93 (MDM) - Added a check to make sure that IDL is not ; running away and starting over when it should not. ; 29-Mar-93 (MDM) - added /all switch to ADSDIRS so that "oldads" ; directories will be used ; 9-Apr-93 (MDM) - Adjusted modification to check for "runaway" to ; only run if stage 0 is selected ; 6-May-93 (MDM) - Added /OVERRIDE_RUNAWAY to allow partial reformatting ; 14-May-93 (MDM) - Changed call to MK_OBS so that it used UNIQ_FID to be ; sure to get all fileIDs. It was not doing it properly ; up until then. ; 20-May-93 (MDM) - Added stage 23 - MK_MDM ; 11-Jul-93 (MDM) - Gutted stage 12 (copy to LPARL) ; - Added stage 24 - MK_GX (G6D, G7D) ; 10-Aug-93 (MDM) - Modified stage #0 so that it will try SRSGET again ; automatically (after a 10 minute pause) if it fails ; - Added creation of ATR files to stage #3 ; 17-Aug-93 (MDM) - Added creation of ATT and insertion of ADS information ; into the ATT file. ; - Removed stage 11 (ADS_INTO_PNT) ; 18-Aug-93 (MDM) - Modified header ; 15-Sep-93 (MDM) - Added creation of SXC files ; - Added sending a REF_SUPER completed running ; message at the end of REF_SUPER ; - Added print statement to the log file about how ; long it took for all stages run ; 5-Oct-93 (MDM) - Added call to make MO listings text files ; - Print REFORMAT and SXL and SXC log times in hours ; 13-Oct-93 (MDM) - Added creation of SXA files to stage 19 ; 16-Oct-93 (MDM) - Added RETRY option (number of retries to attempt for SRSGET) ; 20-Oct-93 (MDM) - Changed the way that it figured out when to chose the second ; disk (old way was break up the days the same way that they ; were broken up for "dir1". Changed to look at available space ; when figuring for "dir2". ; 19-Nov-93 (MDM) - Added creation of SFT files as stage 13 ; 22-Nov-93 (SLF) - Protect against incorrect week id string ; 7-Jan-94 (MDM) - Removed making PNT database files ; 10-Jan-94 (MDM) - Patch for SRSGET problems spanning a year boundary ; (added routine GET_SIRIUS_YR) ; 14-Jan-94 (MDM) - Modified mail messages sent at the end ; 3-Aug-94 (MDM) - Added MK_SDP to stage 9 ; 4-Aug-94 (MDM) - Make START_STAGE=1 if DAYS1 is passed in, and DAYS0 and ; START_STAGE are not passed in. ; 15-sep-94 (SLF) - bugfix - change 'mess' to 'message' ; 3-oct-94 (SLF) - add NODELETE keyword (multi-tasking, caller handles) ; 13-Jan-95 (MDM) - Changed header ; 16-Jan-95 (MDM) - Further patch to GET_SIRIUS_YR ; 31-Mar-95 (MDM) - Added MK_SDMI to stage 6 ; 5-may-95 (SLF) - Added STAGE 14 - mk_sfc ; 22-Aug-95 (MDM) - Added mk_sxt_synmap call to stage 20 to build the synoptic map ; 15-Nov-96 (GLS) - ADded ATR2HXA_DBASE step to stage 3 just ahead of ; call to MK_ATT. ;- ; verify input week conforms to expectations (slf) testing=keyword_set(testing) if testing then begin nodelete=1 if n_elements(stages) eq 0 then stages='undefined' if n_elements(days0) eq 0 then days0='undefined' if n_elements(days1) eq 0 then days1='undefined' if n_elements(ref2_subset) eq 0 then ref2_subset='undefined' dirrev=keyword_set(dir_reverse) testfile='/ys/site/logs/ref_super.test' pr_status,text,/idldoc file_append,testfile,/new,text file_append,testfile, ['Initial...', $ 'stages=' + arr2str(stages,/trim), $ 'days0 =' + arr2str(days0,/trim), $ 'days1 =' + arr2str(days1,/trim), $ 'ref2sb=' + arr2str(ref2_subset,/trim), $ 'dirrev=' + arr2str(dirrev,/trim), $ '--------------------------------------'] prstr,rd_tfile(testfile),/nomore endif delete=1-keyword_set(nodelete) if data_chk(week_str,/string) then begin chkstr=strlen(strcompress(week_str)) if chkstr ne 5 then begin mess='Invalid string for week id: <' + week_str + '>, aborting' endif endif else begin message,'Invalid input for weekid, aborting: ; slf, 15-sep-94 endelse if n_elements(mess) ne 0 then begin message,/info,mess mail,/self, mess, subj='go_ref_super input syntax error' return endif start_time = systime(1) run_time = 0. ; qdebug = 1 fmt = '(f8.2)' ; qflare2 = 0 ;default is to use flare1 if (keyword_set(flare1)) then qflare2 = 0 if (keyword_set(flare2)) then qflare2 = 1 if (not keyword_set(srsget)) then srsget = 2 machine = ['FLARE1', 'FLARE2'] machine = machine+'-'+strtrim(srsget,2) ; if (n_elements(notstages) gt 0) then begin stages = bytarr(30) + 1 stages(where(notstages)) = 0 ;don't do the following stages end if (n_elements(days1) ne 0) and (n_elements(days0) eq 0) and (n_elements(start_stage) eq 0) then start_stage = 1 ;MDM 4-Aug-94 if (keyword_set(start_stage)) then begin stages = bytarr(30) stages(start_stage:*) = 1 end if (n_elements(stages) eq 0) then stages = bytarr(30)+1 $ else stages = [stages, bytarr(30)] ;make sure the array is long enough stages(12) = 0 ;don't do LPARL copy stages(11) = 0 ;don't do ADS_INTO_PNT ; if (not keyword_set(dir1)) then dir1 = [finddir('reformat_a1', /makecre), finddir('reformat_a2', /makecre)] if (not keyword_set(dir2)) then dir2 = [finddir('reformat_b1', /makecre), finddir('reformat_b2', /makecre)] if (not keyword_set(dir3)) then dir3 = [finddir('reformat_a1', /makecre), finddir('reformat_a2', /makecre)] if (not keyword_set(dirweek)) then dirweek = finddir('reformat_week', /makecre) ; if (keyword_set(dir_reverse)) then begin save = dir1 dir1 = dir2 dir3 = dir2 dir2 = save end ; if (n_elements(retry) eq 0) then retry = 6 ; ;----------------- Check for run-away IDL ; if (stages(0)) then begin ff = file_list(dir1, 'ada*') if (ff(0) ne '') then begin break_file, ff(0), dsk_log, dir00, filnam, ext tarr = fid2ex(strmid(filnam, 3, 6) + ext) tim2orbit, tarr, wid=wid if (wid(0) eq week_str) then begin print, 'REF_SUPER: Believe that IDL has runaway and should be stopped' print, 'REF_SUPER: The file: ', ff(0), ' exists in directory ', dir1 print, 'REF_SUPER: And it is from the requested week of ', week_str print, 'REF_SUPER: Stopping...' if (not keyword_set(override_runaway)) then stop end end end ; ;--------------------------------- SETUP PORTION ---------------------------------- ; ; --- Check for the existance of the directories ; week = fix(strmid(week_str, 3, 2)) year = fix(strmid(week_str, 0, 2)) ; ss = where(findfile('$HOME') eq 'ref_super', count) if (count ne 0) then errfil = '~/ref_super/' + ex2fid(anytim2ex(!stime)) + '.log' $ else errfil = '~/ref_super' + ex2fid(anytim2ex(!stime)) + '.log' openw, lun, errfil, /get_lun printf, lun, 'Program started ', !stime printf, lun, 'Running week # ', week_str ; week2 = week + 1 year2 = year if (week2 gt 53) then begin & year2 = year2+1 & week2 = 1 & end rd_fem, [week,week2], [year,year2], fem_data ; ss1 = where((fem_data.week eq week) and (fem_data.day_rev eq 1)) ss2 = where((fem_data.week eq week2) and (fem_data.day_rev eq 1)) ss = [ss1, ss2] first_day_orbits = fem_data(ss) ndays = n_elements(ss1) ;usually it is 7, but for week 53 and 1, it will be less ; st_getsir = first_day_orbits(0:ndays-1) en_getsir = first_day_orbits(1:ndays) day = en_getsir(ndays-1).day time = en_getsir(ndays-1).time + 1.5*60*60L*1000L ;have the end time of the last day go into the next week to be sure ;to get all "straggling" datasets check_time, time, day en_getsir(ndays-1).day = day en_getsir(ndays-1).time = time rawfil_arr = strarr(ndays) ;raw telemetry file name for i=0,ndays-1 do begin st_tarr = anytim2ex(st_getsir(i)) en_tarr = anytim2ex(en_getsir(i)) st_ftime = string(st_tarr(5), st_tarr(4), st_tarr(0:1), format='(4i2.2)') en_ftime = string(en_tarr(5), en_tarr(4), en_tarr(0:1), format='(4i2.2)') rawfil_arr(i) = st_ftime + '.' + en_ftime print, i, ' day === ', rawfil_arr(i) end ; if (n_elements(days0) eq 0) then days0 = indgen(ndays) ;do all days if (n_elements(days1) eq 0) then days1 = indgen(ndays) ;do all days if (keyword_set(qstop)) then stop ; if testing then begin file_append,testfile, ['Second...', $ 'stages=' + arr2str(stages,/trim), $ 'days0 =' + arr2str(days0,/trim), $ 'days1 =' + arr2str(days1,/trim), $ 'ref2sb=' + arr2str(ref2_subset,/trim), $ 'dirrev=' + arr2str(dirrev,/trim), $ '--------------------------------------'] prstr,rd_tfile(testfile),/nomore print,undefxxxx ; crash to get mail messages! endif if (stages(0)) then begin ;----------------------------------- SIRIUS COPY PORTION --------------------------------- stage_tot = 0 ;mainframe is the input ;dir1 = output directory if (n_elements(days0) eq ndays) then begin ;delete all files in the output directory if doing a full weeks worth for idir=0,n_elements(dir1)-1 do begin ;cmd = 'rm -f ' + concat_dir(dir1(idir), '*') cmd = 'cd ' + dir1(idir) + ' ; rm -f *' if (qdebug) then begin & help, cmd & print, cmd & end if delete then spawn, cmd end end for i=0,n_elements(days0)-1 do begin iday = days0(i) idir = 0 while ( (diskfree(dir1(idir)) lt 250) and $ (idir lt n_elements(dir1)-1) ) do begin print,' diskfree: ' + strtrim(dir1(idir),2) + ': ' + $ strtrim(diskfree(dir1(idir)),2) + ' Kb. Switching.' idir = idir+1 endwhile error = -99 itry = 1 while (error ne 0) do begin ;if (week eq 53) then begin ;MDM 10-Jan-94 if ( (anytim2ex(st_getsir(iday)))(6) ne (anytim2ex(en_getsir(iday)))(6) ) then begin ;MDM 16-Jan-95 get_sirius_yr, st_getsir(iday), en_getsir(iday), error, runtime=runtime, dir_out=dir1(idir), $ flare2=qflare2, srsget=srsget, ftp=ftp end else begin get_sirius, st_getsir(iday), en_getsir(iday), error, runtime=runtime, dir_out=dir1(idir), $ flare2=qflare2, srsget=srsget, ftp=ftp end if (error ne 0) then begin itry = itry + 1 print, 'Error from GET_SIRIUS. ERROR = ', error, ' at ', !stime spawn, 'ls -l ' + concat_dir(dir1(idir), rawfil_arr(iday)), result print, 'Directory of that file: ', result message = ['REF_SUPER: Trouble with SRSGET for file: ' + rawfil_arr(iday), $ 'Try # ' + strtrim(itry,2) + ' after 10 minute wait'] if (itry le 5) then mail, message, user='$MAIL_REF_SUPER', /self $ else mail, message, /self ;only send to self after 5 messages if (itry eq retry) then stop wait, 10*60. end end dummy = file_info(concat_dir(dir1(idir), rawfil_arr(iday)), finfo) speed1 = runtime/(finfo.size/9216./1000) speed2 = (finfo.size/1000)/(runtime*60.) printf, lun, !stime+ ' GET_SIRIUS file: '+ rawfil_arr(iday)+ ' just finished' printf, lun, 'GET_SIRIUS run time is: ' + string(runtime, fmt) + ' for ' + $ string(finfo.size/1e+6) + ' megabytes' printf, lun, 'GET_SIRIUS speed is: '+ string(speed1, format=fmt)+ ' min/1000MF ('+ $ string(speed2, format=fmt)+ ' kb/s) running on '+ machine(qflare2) flush, lun stage_tot = stage_tot + runtime end printf, lun, !stime+ ' Stage 0 (SRSGET) Finished. It took '+ string(stage_tot/60., fmt)+ ' hours for ' + $ strtrim(n_elements(days0),2) + ' days' end ; ; if stages(1) then begin ;----------------------------- REFORMATTER PORTION ------------------------------------------- stage_tot = 0 ;dir1 = input directory ;dir2 = output directory if (keyword_set(check_del)) then begin ff = file_list(dir2, '') if (ff(0) ne '') then stop, 'CHECK_DEL is set and there are files in dir: ' +dir2 end if (n_elements(days1) eq ndays) then begin ;delete all files in the output directory if doing a full weeks worth for idir=0,n_elements(dir2)-1 do begin ;cmd = 'rm -f ' + concat_dir(dir2(idir), '*') cmd = 'cd ' + dir2(idir) + ' ; rm -f *' if (qdebug) then begin & help, cmd & print, cmd & end if delete then spawn, cmd end end for i=0,n_elements(days1)-1 do begin iday = days1(i) idir = -1 repeat begin idir = idir + 1 ff = findfile( concat_dir(dir1(idir), rawfil_arr(iday))) infil = ff(0) qexist = (ff(0) ne '') end until (qexist or (idir eq n_elements(dir1)-1)) while ( (diskfree(dir2(idir)) lt 200) and (idir lt n_elements(dir2)-1) ) do idir = idir+1 ;MDM added 20-Oct-93 exist_arr = ['NOT EXIST', 'Exists'] if (qdebug) then print, rawfil_arr(iday), ' ', exist_arr(qexist) if (qexist) then begin printf, lun, !stime+ ' REFORMAT: Starting file: '+ infil reformat, infil, /all, dir_out=dir2(idir), fileid=fileid, runtime=runtime, nrec=nrec, /autodel, $ not_sxt_patrol=not_sxt_patrol printf, lun, !stime+ ' REFORMAT took '+ string(runtime/60, fmt)+ ' hours for '+ string(nrec)+ ' records (' + $ string(runtime/(nrec/1000.), format=fmt)+ ' min/1000MF)' flush, lun stage_tot = stage_tot + runtime end end printf, lun, !stime+ ' Stage 1 (REF) Finished. It took '+ string(stage_tot/60., fmt)+ ' hours for ' + $ strtrim(n_elements(days1),2) + ' days' end if stages(2) then begin ;----------------------------- REFORMAT2 PORTION ------------------------------------------- ;dir2 = input directory ;dir3 = output directory if ((not keyword_set(ref2_subset)) and (not keyword_set(ref2_day))) then begin ;delete all files in the output directory if doing a full weeks worth for idir=0,n_elements(dir3)-1 do begin ;cmd = 'rm -f ' + concat_dir(dir3(idir), '*') cmd = 'cd ' + dir3(idir) + ' ; rm -f *' if (qdebug) then begin & help, cmd & print, cmd & end if (1-keyword_set(ref2_subset)) and delete then spawn, cmd end end reformat2, dir2, fileid, dir3, runtime=runtime, errlog=lun, subset=ref2_subset, day=ref2_day printf, lun, !stime+ ' Stage 2 (REF2) Finished. It took '+ string(runtime/60., fmt)+ ' hours for ' + $ '7 ? days' flush, lun reformat, '', progverno ffff = file_list(dir2, 'ada*') for i=0,n_elements(ffff)-1 do begin rd_fheader, ffff(i), fheader tarr = anytim2ex([fheader.first_time, fheader.first_day]) ref_super_log, tarr, progverno/1000. end ;; ref_super_log, fid2ex(fileid), progverno/1000. end ; if (stages(3)) then begin ;------------------------------ PNT FILE GENERATION PORTION ------------------------------------ ;;mk_pnt, file_list(dir3,'ada??????.????'), outfil_pnt, run_time=run_time ;;printf, lun, !stime, ' Stage 3 (PNT) Finished. It took ', string(run_time, fmt), ' minutes' mk_week_file, 'atr', file_list(dir3,'ada??????.????'), run_time=run_time printf, lun, !stime, ' Stage 3 (ATR) Finished. It took ', string(run_time, fmt), ' minutes' atr2hxa_dbase, week, year printf, lun, !stime, ' Stage 3: HXA_DBASE updated.' mk_att, week_str, run_time=run_time printf, lun, !stime, ' Stage 3 (ATT) Finished. It took ', string(run_time, fmt), ' minutes' ads2att, week=week_str, run_time=run_time printf, lun, !stime, ' Stage 3 (ADS) Finished. It took ', string(run_time, fmt), ' minutes' flush, lun end ; ; if (stages(4)) then begin ;------------------------------- OBSERVING LOG PORTION ------------------------------------ ;mk_obs, file_list(dir3,'ada??????.????'), outfil_obs, run_time=run_time mk_obs, fullweek=week_str, indir=dir3, run_time=run_time ;MDM 14-May-93 printf, lun, !stime, ' Stage 4 (OBS) Finished. It took ', string(run_time, fmt), ' minutes' mk_evn, outfil_evn, week=week_str, run_time=run_time printf, lun, !stime, ' Stage 4 (EVN) Finished. It took ', string(run_time, fmt), ' minutes' flush, lun end ; if (stages(5)) then begin ;-------------------------------------- Mail message that on-line copy can start ----------- message = ['Message from REF_SUPER', $ 'Reformatting for week ' + week_str + ' is finished. You can copy the data to an on-line disk', $ 'but do not start the tapes yet'] mail, message, subj='Stages 0-4 of REF_SUPER done', user='$MAIL_REF_SUPER', /self end ; ; if (stages(6)) then begin ;------------------------------- SDL and SDC PORTION ------------------------------------ mk_sdl, file_list(dir3,'sfr??????.????'), outfil_sdl, run_time=run_time printf, lun, !stime, ' Stage 6 (SDL) Finished. It took ', string(run_time, fmt), ' minutes' mk_sdc, file_list(dir3,'sfr??????.????'), outfil_sdc, run_time=run_time printf, lun, !stime, ' Stage 6 (SDC) Finished. It took ', string(run_time, fmt), ' minutes' mk_sdw, file_list(dir3,'sfr??????.????'), outfil_sdw, run_time=run_time printf, lun, !stime, ' Stage 6 (SDW) Finished. It took ', string(run_time, fmt), ' minutes' mk_sdmi, file_list(dir3,'sfr??????.????'), run_time=run_time printf, lun, !stime, ' Stage 6 (SDMI)Finished. It took ', string(run_time, fmt), ' minutes' flush, lun end ; if (stages(7)) then begin ;------------------------------- SOT PORTION ------------------------------------ mk_sot, file_list(dir3,'sfr??????.????'), outfil_sot, run_time=run_time printf, lun, !stime, ' Stage 7 (SOT) Finished. It took ', string(run_time, fmt), ' minutes' flush, lun end ; if (stages(8)) then begin ;------------------------------- SSL, SEL, and SRD PORTION ------------------------------------ printf, lun, !stime, ' Stage 8 (SEL) NOT AVAILABLE RIGHT NOW mk_ssl, dir3, outfil_ssl, run_time=run_time printf, lun, !stime, ' Stage 8 (SSL) Finished. It took ', string(run_time, fmt), ' minutes' flush, lun end ; if (stages(9)) then begin ;------------------------------- SFS PORTION ------------------------------------ mk_sfs, file_list(dir3,'sfr??????.????'), run_time=run_time1 mk_sdp, file_list(dir3,'spr??????.????'), run_time=run_time2 run_time = run_time1 + run_time2 printf, lun, !stime, ' Stage 9 (SFS) Finished. It took ', string(run_time, fmt), ' minutes' flush, lun end ; if (stages(10)) then begin ;-------------------------- Make Desaturated composite images ------------------------------------ files = file_list(dir3,'sfr??????.????') mk_sfd, files, dirweek, run_time=run_time, ist=sfd_ist printf, lun, !stime, ' Stage 10 (SFD) Finished. It took ', string(run_time/60., fmt), ' hours' mk_sfw, files, dirweek, run_time=run_time printf, lun, !stime, ' Stage 10 (SFW) Finished. It took ', string(run_time/60., fmt), ' hours' flush, lun end ; if (stages(11)) then begin ;-------------------------------------- ADS info into PNT files ---------------------------------- ;not used end ; if (stages(12)) then begin ;-------------------------------------- Copy weekly files to LPARL ------------------------------- ;not used end ; if (stages(13)) then begin ;-------------------------------------- SFT portion ------------------------------- files = file_list(dir3,'sfr??????.????') mk_sft, files, run_time=run_time printf, lun, !stime, ' Stage 13 (SFT) Finished. It took ', string(run_time/60., fmt), ' hours' flush, lun end ; slf 5-may-1995 add SFC generation if (stages(14)) then begin ;-------------------------------------- SFC portion ------------------------------- mk_sfc, week_str, /online, /noconfirm printf, lun, !stime, ' Stage 14 (SFC) Finished. It did not take long' flush, lun endif ; ss = 0 for i=1,n_elements(dir3)-1 do begin ff = findfile(concat_dir(dir3(i), 'ada*')) if (ff(0) ne '') then ss = [ss, i] end dir3_act = dir3(ss) ; if (stages(18)) then begin ;-------------------------------------- Make Tape Script ------------------------------------ spawn, 'pwd', save_cd mktap, datadir = dir3_act, /alldata, tapename=tapename, weekdir=dirweek cd, save_cd(0) file1 = concat_dir(dirweek, 'mktap.com') file2 = concat_dir(dirweek, 'mktap1.com') cmd = "sed 's/nrmt0h/nrmt1h/g' " + file1 + " > " + file2 ;make a second file spawn, cmd ; message = ['Message from REF_SUPER', $ 'Reformatting for week ' + week_str + ' is finished. You can start the tapes', $ 'The tape name is ' + tapename] mail, message, subj='REF_SUPER done', user='$MAIL_REF_SUPER', /self ;MDM and GAL want copies of all of this even if not running it themselves end ; if (stages(19)) then begin ;-------------------------------------- Make SXL files ------------------------------------ ff = file_list(dirweek, 'sfd'+week_str+'*') if (ff(0) ne '') then begin mk_sl, ff(n_elements(ff)-1), run_time=run_time, prefix='SXA' printf, lun, !stime, ' Stage 19 (SXA) Finished. It took ', string(run_time, fmt), ' minutes' mk_sxl, ff(n_elements(ff)-1), run_time=run_time printf, lun, !stime, ' Stage 19 (SXL) Finished. It took ', string(run_time/60, fmt), ' hours' flush, lun end else begin printf, lun, !stime, ' Stage 19 (SXL) Finished. Could not find SFD file' flush, lun end end if (stages(20)) then begin ;-------------------------------------- Make SSC files ------------------------------------ ff = file_list(dirweek, 'sfd'+week_str+'*') if (ff(0) ne '') then begin run_time = 0 for ityp=0,2 do begin mk_ssx, ff(n_elements(ff)-1), type=ityp, run_time=run_time0 junk = mk_sxt_synmap(week=week_str, /write_fil, type=ityp) ;MDM added 22-Aug-95 run_time = run_time + run_time0 end printf, lun, !stime, ' Stage 20 (SSX) Finished. It took ', string(run_time, fmt), ' minutes' flush, lun end else begin printf, lun, !stime, ' Stage 20 (SSX) Finished. Could not find SFD file' flush, lun end end if (stages(21)) then begin ;-------------------------------------- Copy to Flares Directory --------------------------------- chk_flares, dir3_act, flare_files, /copy, /delete if (n_elements(flare_files) eq 0) then flare_files = 'NO FILES COPIED" message = ['Message from REF_SUPER', $ 'CHK_FLARES - files copied', $ flare_files] mail, message, subj='REF_SUPER chk_flares files copied', user='$MAIL_REF_SUPER', /self end ; if (stages(22)) then begin ;------------------------------- SLD and SLS PORTION ------------------------------------ mk_sl, file_list(dir3,'sfr??????.????'), run_time=run_time, prefix='SLD' printf, lun, !stime, ' Stage 22 (SLD) Finished. It took ', string(run_time, fmt), ' minutes' flush, lun mk_sl, file_list(dir3,'sfr??????.????'), run_time=run_time, prefix='SLS' printf, lun, !stime, ' Stage 22 (SLS) Finished. It took ', string(run_time, fmt), ' minutes' flush, lun end ; if (stages(23)) then begin ;------------------------------- MK_MDM PORTION ------------------------------------ outfil1 = getenv('DIR_GEN_MO') + '/mo_list.txt' outfil2 = getenv('DIR_GEN_MO') + '/mos_made.log' mk_mdm, /write spawn, 'rm -f ' + outfil1 spawn, 'rm -f ' + outfil2 mk_mo_list, outfil=outfil1 list_mo_log, outfil=outfil2 printf, lun, !stime, ' Stage 23 (MDM) Finished. end ; if (stages(24)) then begin ;------------------------------- MK_GXD PORTION ------------------------------------ mk_week_file, 'g6d', week=week_str mk_week_file, 'g7d', week=week_str printf, lun, !stime, ' Stage 24 (GXD) Finished. end ; if (stages(25)) then begin ;------------------------------- MK_SXC PORTION ------------------------------------ ff = file_list(dirweek, 'sfd'+week_str+'*') if (ff(0) ne '') then begin mk_sxc, ff(n_elements(ff)-1), run_time=run_time printf, lun, !stime, ' Stage 25 (SXC) Finished. It took ', string(run_time/60, fmt), ' hours' flush, lun end else begin printf, lun, !stime, ' Stage 25 (SXC) Finished. Could not find SFD file' flush, lun end end ; ;-------------------- Finish up messages ------------------------------ ; end_time = systime(1) run_time = (end_time-start_time)/60./60./24. ;days printf, lun, !stime, ' All stages above took ', string(run_time, fmt), ' days' free_lun, lun ; ;-------------------- ; if (stages(29)) then begin message = ['All 25 stages of REF_SUPER are finished for ' + week_str, $ 'You can move the SFD file over to /yd1/sfd now', $ '(if the master tapes have been made)'] end else begin junk = where(stages, count) message = ['REF_SUPER is finished with the parital stages requested for '+week_str, $ 'The stages that were run were stages # ' + arr2str(junk)] end mail, message, user='$MAIL_REF_SUPER', /self ; end