pro mk_week_file, prefix0, infil0, outfil, week=week0, run_time=run_time, $ append=append, usefid=usefid, indir=indir, outdir=outdir, $ qdebug=qdebug, mo_append=mo_append, $ same_week=same_week, qstop=qstop ;+ ;NAME: ; mk_week_file ;PURPOSE: ; Generate a weekly file. Files that can be generated are GXT, G6D, ; G7D (GOES light curve), GOL, GBL, OSP/OSF/OBD/OWH files. ;SAMPLE CALLING SEQUENCE: ; mk_week_file, 'gxt', infil ; mk_week_file, 'g6d', week=week ; mk_week_file, 'g7d', week=week, /append, /usefid ;INPUT: ; prefix - The prefix is required. It must be one of the following: ; ; GXT, G6D, G7D, G6DF, G7DF, G71,G75,G61,G75 ; GOL ; GBL ; OBD, OSF, OSP, OWH ; NTS ; ATR, ATT ; GUF ;OPTIONAL INPUT: ; infil0 - A string array with a list of the input file names. ; It should include the directory path. ; The inputfiles can be derived if the "week" option is used. ; outfil - The output filename to create. The output ; file can be derived if the "week" option is used ;OPTIONAL KEYWORD INPUT: ; week - a string with the year/week if the format ; 91_49 where 91 is the year, 49 is the week ; append - If set, try to just append the files in "infil" ; list which are not already in the output file. ; usefid - If set, use the FID time in the orbit pointer ; (so that it can be recognized when used with ; the /append switch) ; indir - The input directory ; outdir - The output directory ; same_week - If set, then only select the files with the same ; week ; mo_append-If set, then the input file list is a new separate ; list which have no names in common with the previous ; list and all files in the list should be appended ; without checking for duplicates. ;RESTRICTION: ; Cannot use /USEFID or /APPEND option with 'GBL' datatype ;HISTORY: ; Written 13-Apr-93 by M.Morrison ; 14-Apr-93 (MDM) - Expanded to handle GBL ; 20-Apr-93 (MDM) - Expanded to handle GOL ; 21-Apr-93 (MDM) - Expanded to handle OBD, OSF, OSP, OWH ; - Added keyword option /SAME_WEEK ; - Renamed from MK_GX to MK_WEEK_FILE ; 7-May-93 (MDM) - Allowed 300 orbit entries for GBL files ; 13-May-93 (MDM) - Expanded to handle NTS ; 14-May-93 (MDM) - Made NTS files not have fixed record sizes ; 18-May-93 (MDM) - Modifed the raw GBL input file name ; 19-May-93 (MDM) - Modified to remove the copy of the data file ; which was moved to the scratch directory for the ; cases when /APPEND is used, but there are no ; new datasets to append. ; 16-Jul-93 (MDM) - Expanded to handle ATR files ;V1.01 17-Jul-93 (MDM) - Added /MO_APPEND option ;V1.02 17-Jul-93 (MDM) - Corrected selection of the week number during ; automatic generation of the output file name ;V1.03 17-Jul-93 (MDM) - Added logging of the reformatter version for ATR and OBS ; as well as logging ADA2ATR version number ;V1.04 4-Oct-93 (MDM) - Corrected error which was causing NTS generation not ; to work ;V1.05 8-Oct-93 (MDM) - Corrected for case where there are no Roadmap datasets ; in the input files (makes it hard to define the output file) ;V1.06 13-Oct-93 (MDM) - Added capability of generating the G6D and G7D files ; from the GOES FITS files (G6DF, G7DF) ; - Added /QSTOP option ; 8-Nov-93 (MDM) - Added /CD to FILE_LIST call for NTS making ; 15-Nov-93 (MDM) - Modification because of fluke error with no datasets in ; the middle file (used for determining the week ID) ; 3-Jan-94 (SLF) - allow weeks with 1 day (94_01) ; 8-Mar-94 (SLF) - expanded to handle GUF (Ulyssses Ephemeris) ; 12-Jul-94 (MDM) - Modified to use FREE_LUN instead of CLOSE ;V1.07 8-Dec-94 (MDM) - Modified to use a larger NEW_ORBP structure which ; allowed the FILEID to be saved separate from the ; .TIME and .DAY to figure out when to do an append. ; Perviously, the pointers were being messed up ; when there was overlap of time periods between ; data files. ;V1.08 8-Dec-94 (MDM) - Further modification. Flag when a file is fully ; contained within ; 17-Jan-95 (MDM) - Removed redundant code ; - Added debug print statement ; - Patched case for ATR creation with /append option ; ("last_tim" was not being defined) ;V1.081 17-Jan-95 (MDM) - Add "last_tim" logic to MK_OBS_STR too ;V1.09 25-Jan-95 (MDM) - Corrected anomaly which was causing a problem when ; /APPEND is used, but the files are out of order so ; it must remake the whole file, but "last_tim" was ; remaining set and not including all of the previous ; files ;V1.091 26-Jan-95 (MDM) - Modified G6DF and G6DF input file calculated name ;V1.092 29-aug-95 (SLF) - Add GOES averages (g61/g71/g65/g75) ;V1.093 11-sep-95 (SLF) - Add SXG (sxt/goes xray log) ;V1.094 27-jan-96 (SLF) - eliminate hard coded path ; (/yd8/goes3sec_raw -> concat_dir('$ydb_raw','goes3sec_raw') ) ;V1.095 1-apr-96 (SLF) - GOES 8 and 9, G8D/G9D/G81/G85/G91/G95 ;- ; progVerNo = 1.091*1000 progName = 'MK_WEEK_FILE' ; start_time = systime(1) run_time = 0. if not data_chk(prefix0,/string) then begin tbeep message,/info,"Must supply a file prefix..." return endif goesdig=is_member(prefix0,['G61','G71','G65','G75','G81','G85','G91','G95'],/ignore_case) if goesdig then begin gsat=long(strmid(prefix0,1,1)) binsize=long(strmid(prefix0,2,1)) if keyword_set(week0) then infil0='g' + strmid(prefix0,1,1) + 'd' + week0 endif if (n_elements(infil0) ne 0) then begin if (infil0(0) eq '') then return infil = infil0 end prefix = strmid(prefix0, 0, 3) ;prefix0 is the original, prefix is the first three letters if (n_elements(week0) ne 0) then week = week0 if (n_elements(outdir) eq 0) then outdir = '$DIR_GEN_'+strupcase(prefix) ; if (n_elements(outfil) ne 0) then outfil_final = outfil if (keyword_set(week)) then begin yearn = fix(strmid(week, 0, 2)) weekn = fix(strmid(week, 3, 2)) tim1 = anytim2ints(week2ex(yearn, weekn)) ; week2 = weekn + 1 year2 = yearn if (week2 gt 53) then begin & week2=1 & year2=year2+1 & end tim2 = anytim2ints(week2ex(year2, week2)) ;start day of the next week ndays = tim2.day - tim1.day fem_data0 = replicate(tim1, ndays) dayarr=indgen(ndays) + tim1.day if ndays eq 1 then dayarr=dayarr(0) ;scaler fem_data0.day = dayarr ;---- MDM removed stuff below because FEM files do not exist before Sep-91 ;tim2 = anytim2ints(tim1, off=8*24.*60.*60) ;rd_fem, tim1, tim2, fem_data, /full_weeks ;ss = where((fem_data.week eq weekn) and (fem_data.day_rev eq 1)) ;fem_data0 = fem_data(ss) ; files = '' case strupcase(prefix) of 'ATR': begin indir = finddir(week+'a') infil = file_list(indir, 'ada*') end 'GOL': begin indir = finddir('gbo'+week+'a') infil = file_list(indir, '*') end 'GXT': begin if (n_elements(indir) eq 0) then indir = '$DIR_GEN_GOES' days = strlowcase(gt_day(fem_data0, /str, /leadzero)) days = str_replace(days, '-', '') plotseg=['a','b','c','d'] ; four 6hour plots/day for i=0,n_elements(days)-1 do files=[files, days(i) + plotseg + '_xr.plot'] files = files(1:*) infil = file_list(indir, files) end 'GUF': infil=(file_list('$DIR_GEN_GUF','*'))(0) 'GBL': begin if (n_elements(indir) eq 0) then indir = '$ydb_raw/batse_raw' tim1 = gt_day(fem_data0(0), /str) tim2 = gt_day( anytim2ints(fem_data0(n_elements(fem_data0)-1), off=24.*60.*60), /str) rd_week_file, tim1, tim2, 'GBE', gbe, status=status if keyword_set(qdebug) then print, 'GBL searching from ', tim1, ' to ', tim2, ' found: ', n_elements(gbe) if (status eq 0) then begin ;filnam = 'gbl' + ex2fid(anytim2ex(gbe)) + '_f' + string(gbe.event, format='(i4.4)') + '_sav' filnam = 'flare_' + strtrim(gbe.event,2) + '.sav' infil = file_list(indir, filnam) end else begin infil = '' end end 'NTS': begin if (n_elements(indir) eq 0) then indir = '$ydb_raw/nobeyama_raw' ff = file_list(indir, 'bf*.fits', /cd) break_file, ff, dsk_log, dir, filnam, ext tim1 = fid2ex(strmid(filnam, 2, 6)) wid = ex2week(tim1, /wid) ss = where(wid eq week) if (ss(0) eq -1) then infil = '' else infil = ff(ss) end 'SXG': begin pointer_cad= 6 ; 6 HOURs/pointer pr_week2tim,week,t0,t1 times=timegrid(t0,timegrid(t1,/day,/string), $ /string, hours=pointer_cad) infil='sxx' + ex2fid(anytim2ex(times)) infil=infil(0:n_elements(infil)-2) endcase else: begin if (strmid(strupcase(prefix),0,1) eq 'G') then begin ;G6D and G7D case 1 of strlen(prefix0) gt 3: begin if (n_elements(indir) eq 0) then indir = ['/usr5/com/morrison'] n = n_elements(fem_data0) * 8 ;8 3 hour files per day sttim_arr = anytim2ints(fem_data0(0), off = findgen(n)*3.*60.*60) entim_arr = anytim2ints(sttim_arr, off = 3.*60.*60) gspacecraft = fix(strmid(prefix, 1, 1)) infil = 'File for ' + fmt_tim(sttim_arr) endcase str2number(prefix0) lt 10: begin if (n_elements(indir) eq 0) then indir = concat_dir('$ydb_raw','goes3sec_raw') days = strmid( ex2fid( anytim2ex(fem_data0) ), 0, 6) plotseg = '.' + string(indgen(8)*300, format='(i4.4)') sc_num = 'go0' + strmid(prefix, 1, 1) for i=0,n_elements(days)-1 do files = [files, sc_num + days(i) + plotseg] files = files(1:*) infil = file_list(indir, files) endcase else: begin ; goes digitize pointer_cad= 6 ; HOUR pr_week2tim,week,t0,t1 times=timegrid(t0,timegrid(t1,/day,/string), $ /string, hours=pointer_cad) infil='gxx' + ex2fid(anytim2ex(times)) infil=infil(0:n_elements(infil)-2) endcase endcase end else begin ;OBS if (n_elements(indir) eq 0) then indir = [finddir(week+'a'), finddir(week+'b')] ;;fileid_arr = uniq_fid(indir) fileid_arr = '*' case strupcase(prefix) of 'OBD': infil = file_list(indir, 'bda' + fileid_arr) 'OSF': infil = file_list(indir, 'sfr' + fileid_arr) 'OSP': infil = file_list(indir, 'spr' + fileid_arr) 'OWH': infil = file_list(indir, 'wda' + fileid_arr) ;assumes wda exists for each hda!!! bad? end end end endcase if (infil(0) eq '') then begin print, 'MK_WEEK_FILE: No files found for week', week, ' - returning...' return end ; weekid = string(week, fix(progverno/1000), format="(a, 'a.', i2.2)") outfil = concat_dir('$DIR_REFORMAT_SC', strlowcase(prefix)+weekid) outfil_final = concat_dir(outdir, strlowcase(prefix)+weekid) end ; if (n_elements(outfil) eq 0) then begin if ((strmid(strupcase(prefix), 0, 1) eq 'O') or (strmid(strupcase(prefix), 0, 1) eq 'A')) then begin rd_fheader, infil, fheader00, ndsets ss = where(ndsets ne 0, count) ;;if (ndset eq 0) then begin ;MDM added 8-Oct-93 if (count eq 0) then begin ;MDM modified 15-Nov-93 print, 'MK_WEEK_FILE: No datasets in the input files passed:', infil print, 'MK_WEEK_FILE: Returning' return end rd_roadmap, infil(ss(count/2)), roadmap, ndset tarr = anytim2ex(roadmap(0)) week = ex2week(tarr) weekid = string(tarr(6), week, fix(progverno/1000), format="(i2.2, '_', i2.2, 'a.', i2.2)") outfil = concat_dir('$DIR_REFORMAT_SC', strlowcase(prefix)+weekid) outfil_final = concat_dir(outdir, strlowcase(prefix)+weekid) end else begin outfil = concat_dir('$DIR_REFORMAT_SC', strlowcase(prefix)+'_temporary') outfil_final = concat_dir(outdir, strlowcase(prefix)+'_temporary') end end ; ;----- Call subroutines that define the structures ; obs_struct att_struct gbo_struct, gbo_version=gbo_version ref_struct, ref ; ; ;----- Diagnostic variables ; qprint = 1 qdebug = 1 ; ;---------------------------------------- General Stuff ; refverno = 0 ref2verno = 0 case strupcase(prefix) of 'ATR': begin & data_version = 00 & spacecraft = 'YOH' & end 'SXG': begin & data_version = 00 & spacecraft = 'YOH' & end 'GXT': begin & data_version = gbo_version.gxt & spacecraft = 'GO ' & end 'G6D': begin & data_version = gbo_version.gxd & spacecraft = 'GO ' & end 'G7D': begin & data_version = gbo_version.gxd & spacecraft = 'GO ' & end 'G8D': begin & data_version = gbo_version.gxd & spacecraft = 'GO ' & end 'G9D': begin & data_version = gbo_version.gxd & spacecraft = 'GO ' & end 'GOL': begin & data_version = gbo_version.gol & spacecraft = 'GBO' & end 'GBL': begin & data_version = gbo_version.gbl & spacecraft = 'GRO' & end 'NTS': begin & data_version = gbo_version.nts & spacecraft = 'NOB' & end 'GUF': begin & data_version = gbo_version.guf & spacecraft = 'ULY' & end else: begin if goesdig then begin ; GOES averages data_version = gbo_version.gxd spacecraft = 'GO ' endif else data_version = 0 endcase endcase lunout = 2 ;output lun = 3 ;for reading the data file ; gen_struct, file_header=fhead ; norbitmax = 10*15 ;10 days max if (strupcase(prefix) eq 'GBL') then norbitmax = 300 orbit_header = {obs2_neworb_hd_rec} obs_struct, obs2_neworbit=orbit0 ;18-Dec-94 MDM modified to used "obs2" orbit record for all weely files orbit_header.neworbit_ver = 1 orbit_arr = replicate(orbit0, norbitmax) ; qfirst_time = 1 first_time = 0 first_day = 0 last_time = 0 last_day = 0 ; rsiz = 16 if (strupcase(prefix) eq 'NTS') then rsiz = 0 ;don't do any checking ; if (keyword_set(same_week)) then begin ;get all files for the same week break_file, infil, dsk_log, dir_in, filnam, ext fids = strmid(filnam, 3, 6)+ext tim2orbit, fid2ex(fids), wid=wid_arr ss = where(wid_arr eq wid_arr(n_elements(wid_arr)-1)) infil = infil(ss) end ; qnew_outfil = 1 if ((keyword_set(append) or keyword_set(mo_append)) and file_exist(outfil_final)) then begin qnew_outfil = 0 cmd = 'rm -f ' + outfil ;remove any copy of the file that might already exist spawn, cmd cmd = 'cp ' + outfil_final + ' ' + outfil spawn, cmd openu, lunout, outfil, /block, /get_lun rd_pointer, lunout, fpoint rd_fheader, lunout, fheader rd_neworb_p, outfil, orbit_arr0, orbit_header0 ; ;---- Temporary patch for append cases where file is using the old format ss_temp = where(tag_names(orbit_arr0) eq 'ST$FILEID') if (ss_temp(0) eq -1) then begin orbit_arr0 = str_copy_tags(orbit_arr(0), orbit_arr0) ;convert the structure type fid_temp = byte( ex2fid( anytim2ex(orbit_arr0) ) ) orbit_arr0.st$fileid(0:10,*) = fid_temp ;^^^^ above stuff is useless - must make new file since orbit_arr section overwrites the data section qnew_outfil = 1 print, 'MK_WEEK_FILE: Trying to append to an older version weekly file' print, 'MK_WEEK_FILE: Making a complete new file. end else begin norbit = orbit_header0.nOrbitRec orbit_arr(0:norbit-1) = orbit_arr0 bytout = fpoint.totbytes ;point at end of file bytout0 = fpoint.data_section ndatasets = fheader.ndatasets qfirst_time = 0 first_time = orbit_arr(0).time first_day = orbit_arr(0).day int2ex, fheader.last_time, fheader.last_day, tarr_last ;MDM added 17-Jan-95 last_tim = anytim2ints(tarr_last) end if (not keyword_set(mo_append) and (not qnew_outfil)) then begin ;check for overlap files - only do the new ones break_file, infil, dsk_log, dir_in, filnam, ext case strupcase(prefix) of 'GXT': fids = selsis2fid(filnam) 'G6D': fids = strmid(filnam, 4, 6)+ext 'G7D': fids = strmid(filnam, 4, 6)+ext 'G8D': fids = strmid(filnam, 4, 6)+ext 'G9D': fids = strmid(filnam, 4, 6)+ext 'NTS': fids = strmid(filnam, 2, 6) else: fids = strmid(filnam, 3, 6)+ext endcase fids_already = string(orbit_arr0.st$fileid) ;MDM modified 8-Dec-94 ss = where_arr(fids, fids_already) nn = n_elements(ss) if (max(ss) ne nn-1) then begin qnew_outfil = 1 ;something was done out of order ;want ss to start at 0 and increase by 1 ;0,1,2,3,4 is ok, 0,1,5,6,7 is not ok - want files 2,3,4 in order print, 'MK_WEEK_FILE: Cannot simply append since there are files out of order' print, 'MK_WEEK_FILE: Making a complete new file. last_tim = 0b ;MDM 25-Jan-95 if (keyword_set(qstop)) then stop end else begin if (n_elements(ss) eq n_elements(infil)) then begin print, 'MK_WEEK_FILE: No new files found. Returning' close, 2 if (outfil_final ne outfil) then spawn, 'rm -f ' + outfil ;remove the copy which was moved to the scratch directory return end infil = infil(nn:*) ;only do the new files print, 'MK_WEEK_FILE: Skipping the first ', nn, ' files' end end end if (qnew_outfil) then begin print, 'Opening a new output file: ', outfil open_da_file, lunout, '', '', outfil, rsiz, bytout, fpoint, /nolower fpoint.opt_section = bytout rdwrt, 'W', lunout, bytout, rsiz, Orbit_header, 1 rdwrt, 'W', lunout, bytout, rsiz, Orbit_arr, 1 fpoint.data_section = bytout bytout0 = bytout ;first entry record number qfirst_time = 1 ndatasets = 0L norbit = 0 ;moved here 11-Nov-92 end ; if (keyword_set(qstop)) then stop ; nfil = n_elements(infil) if (strupcase(prefix) eq 'GOL') then nfil = 1 ;do it in one step for ifil=0,nfil-1 do begin if (qdebug) then print, 'MK_WEEK_FILE working on file: ', infil(ifil) case strupcase(prefix0) of 'ATR': ada2atr, infil(ifil), data, last_tim, refverno, ref2verno, progverno2, progname2 'GOL': mk_gol_str, infil, data 'GXT': rd_goestek, infil(ifil), data 'GBL': rd_gbl_raw, infil(ifil), data 'G6D': rd_goes3sec, infil(ifil), outrec=data, /rm_bad 'G7D': rd_goes3sec, infil(ifil), outrec=data, /rm_bad 'G8D': rd_goes3sec, infil(ifil), outrec=data, /rm_bad 'G9D': rd_goes3sec, infil(ifil), outrec=data, /rm_bad 'G6DF': data = rd_goes_fits(dummy, sttim_arr(ifil), entim_arr(ifil), spacecraft=gspacecraft, indir=indir, /rm_bad) 'G7DF': data = rd_goes_fits(dummy, sttim_arr(ifil), entim_arr(ifil), spacecraft=gspacecraft, indir=indir, /rm_bad) 'NTS': rd_nts_raw, infil(ifil), data 'GUF': rd_guf_ascii, infil, data, week=week 'SXG': rd_sxtgoes,times(ifil),timegrid(times(ifil+1),sec=(-1),/string), $ data, /five_minute else: begin if goesdig then begin rd_gxd,times(ifil),timegrid(times(ifil+1),sec=(-1),/string), $ data, goes7=(gsat eq 7), goes6=(gsat eq 6), $ goes8=(gsat eq 8), goes9=(gsat eq 9) if data_chk(data,/struct) then begin adata=timeavg(data,bin=(binsize*60.)) odata=replicate(data(0),n_elements(adata)) data=str_copy_tags(odata,adata) endif endif else $ mk_obs_str, prefix, infil(ifil), data, last_tim, serial_num, refverno, ref2verno, progverno2, progname2 endcase endcase nn = n_elements(data) ;--- Get the file ID information break_file, infil(ifil), dsk_log, dir_in, filnam, ext case strupcase(prefix) of 'GXT': fid = selsis2fid(filnam) 'G6D': fid = strmid(filnam, 4, 6)+ext 'G7D': fid = strmid(filnam, 4, 6)+ext 'G8D': fid = strmid(filnam, 4, 6)+ext 'G9D': fid = strmid(filnam, 4, 6)+ext 'NTS': fid = strmid(filnam, 2, 6) else: fid = strmid(filnam, 3, 6)+ext endcase orbit_arr(norbit).st$fileid = byte(fid) if (get_nbytes(data(0)) gt 4) then begin ;see if actually read something orbit_arr(norbit).time = data(0).time orbit_arr(norbit).day = data(0).day if (strupcase(prefix) eq 'OSP') then orbit_arr(norbit).sxt_pfi = serial_num if (strupcase(prefix) eq 'OSF') then orbit_arr(norbit).sxt_ffi = serial_num orbit_arr(norbit).stEntry = ndatasets + 1 ;dataset # for which this orbit starts (start counting at 1) norbit = norbit + 1 rdwrt, 'W', lunout, bytout, rsiz, data, 1 ndatasets = ndatasets + nn if (qfirst_time) then begin first_day = data(0).day first_time = data(0).time qfirst_time = 0 end last_day = data(nn-1).day last_time = data(nn-1).time end else begin ; --- MDM added 8-Dec-94 (Ver 1.08) so that the fileID is recorded even if no data is saved orbit_arr(norbit).time = orbit_arr((norbit-1)>0).time ;previous orbit orbit_arr(norbit).day = orbit_arr((norbit-1)>0).day orbit_arr(norbit).stentry = orbit_arr((norbit-1)>0).stentry norbit = norbit + 1 end if (keyword_set(qdebug)) then print, 'Orbit ', norbit-1, ' fid= ', fid, ' pointer = ', fmt_tim(orbit_arr(norbit-1)) end ; maxsamps = get_nbytes(data(0)) totbytes = bytout orbit_header.nOrbitRec = norbit bytout = fpoint.opt_section rdwrt, 'W', lunout, bytout, rsiz, orbit_header, 1 rdwrt, 'W', lunout, bytout, rsiz, orbit_arr, 1 ;-------------------- Write out pointer and file header ; fpoint.TotBytes = totbytes fhead.first_Time = first_time fhead.first_Day = first_day fhead.last_Time = last_time fhead.last_Day = last_day fhead.nDataSets = nDataSets fhead.maxSamps = maxSamps fpoint.data_version = data_version fhead.refverno = refverno fhead.ref2verno = ref2verno file_id = ' ' ;TODO - make it something else? wrt1p_fh, lunout, fpoint, fhead, rsiz, progName, progVerNo, file_id, strupcase(prefix), 'All', $ progname2=progname2, progverno2=progverno2, spacecraft=spacecraft ; free_lun, lunout ; if (n_elements(outfil_final) ne 0) then if (outfil_final ne outfil) then begin cmd = 'rm -f ' + outfil_final ;remove any copy of the file that might already exist spawn, cmd cmd = 'mv ' + outfil + ' ' + outfil_final spawn, cmd cmd = 'chmod +w ' + outfil_final spawn, cmd outfil = outfil_final end ; end_time = systime(1) run_time = (end_time-start_time)/60. print, 'MK_WEEK_FILE took', run_time, ' minutes to run' ; if (keyword_set(qstop)) then stop end