pro reformat2, indir, fileid_in, dir_out, runtime=runtime, errlog=errlog, overwrite=overwrite, subset=subset, day=day
;
;
;
;HISTORY:
;	Written Dec '91 by M.Morrison
;	14-Mar-92 (MDM) - Changed to look at ALL orbits within the week to be
;			  extracted and append to previous files where necessary.
;			  NOTE: This reformatter stage needs to be run in 
;			  time order in order to append the files properly
;	16-Apr-92 (MDM) - Re-vamped the whole thing to work from ALL 7 24-hour
;			  files as the input, not one by one where it would be
;			  necessary to check for appending,...  Updated WRT1ORBIT
;			  to check for SXT FFI images being identical
;	17-Apr-92 (MDM) - changed how the output directory for "REFORMAT2" is determined.
;			  On 16-Apr it was made to be the first dir until only 20 megabytes were free.
;			  This resulted in files with the same fileID split between disks (all SFR are done first, then SPR, ...)
;			  The change on 17-Apr was to maintain the "splitting" that was done during stage one (SRSGET)
;			  which splits by days.
;			- Added "subset" option
;	18-Apr-92 (MDM) - Changed location of the HXA scan read to be inside REFORMAT2 instead of WRT1ORBIT
;			  Only read it when the input file has changed (like done with BCS DP_SYNC)
;	23-Apr-92 (MDM) - Adjusted for special case where S/C actual day time is in one day and the FILEID is in another day.
;			- Added "day" parameter option
;	17-Jun-92 (MDM) - Adjusted for case where there are no datasets but a file exists (BCS - DPSync)
;V2.00	 1-Jul-92 (MDM) - Corrected for HXA scan extraction.  Previously, it was not being written into the orbit files
;			  Started the "ProgVerNo" logging with ver 2.0
;V2.01	27-Jul-92 (MDM) - Modified to calculate the week # from the mid point day since SRSGET does not give data time 
;			  ordered or even what was asked for.  Caused a problem for reformatting 92_29 since there was some
;			  garbage data from 11-Jul in the 12-Jul data file, which made REFORMAT2 think we were reformatting
;			  week 92_28.
;V2.02   3-Sep-92 (MDM) - Modification to the way that the BDA file is selected for DP_SYNC reading when there are no BDA
;			  datasets to be extracted for the current orbit.  The old method assumed that there were no missing
;			  files between days.  91_37 has missing days.
;V2.03   3-Sep-92 (MDM) - Hardwired the "margin" variable to zero.  It used to be the 5 minute difference between the fileID
;			  value and the true orbit start, but that was change to match identically in Apr-92
;V2.04	22-Dec-92 (MDM) - Had to insert sort command in a UNIQ call for BCS DP_SYNC file name generation
;	 9-Apr-93 (MDM) - Modified the call to RD_FEM to use /FULL_WEEKS switch
;	14-May-93 (MDM) - Stopped making the *temp files which had the "left overs"
;
progVerNo = 2.04*1000
progName  = 'REFORMAT2.PRO'
;
start_time = systime(1)
;
qdebug = 1
;
prefix_arr = ['sfr', 'spr', 'bda', 'wda', 'hda', 'ada', 'cba']
instr_arr  = ['SXT', 'SXT', 'BCS', 'WBS', 'HXT', 'ATT', 'CBA']
;
;; subset = [2,3,4,5,6] & prefix_arr=prefix_arr(subset) & instr_arr=instr_arr(subset)	;debugging - skip SXT
;; subset = [4,5,6] & prefix_arr=prefix_arr(subset) & instr_arr=instr_arr(subset)	;debugging - skip SXT, bcs,wbs
if (keyword_set(subset)) then begin
    prefix_arr=prefix_arr(subset)
    instr_arr=instr_arr(subset)
end
;
nprefix = n_elements(prefix_arr)
;
;infil = concat_dir(indir, 'ada' + fileid_in)
infil = file_list(indir, 'ada*')
nfil = n_elements(infil)
dir_out_map = strarr(nfil)
for i=0,nfil-1 do begin
    for idir=0,n_elements(indir)-1 do begin
	if (strpos(infil(i), indir(idir)) ne -1) then dir_out_map(i) = dir_out(idir)
    end
end
rd_fheader, infil, fheader
sttim = fmt_tim( anytim2ex([0,fheader(0).first_day-1]) )
entim = fmt_tim( anytim2ex([0,fheader(nfil-1).last_day+2]) )
rd_fem, sttim, entim, fem_data, /full_weeks
;
fileid_out  = strtrim( fem_data(0).st$fileid, 2 )
fileid_ex = fid2ex(fileid_out)
margin = int2secarr(fileid_ex, fem_data(0))			;should be negative

week = ex2week( anytim2ex([0,fheader(nfil/2).first_day]) )	;MDM 27-Jul-92 - Calculate the week # from the mid point day
								;since SRSGET does not give data time ordered or what was asked for
if (keyword_set(day)) then begin
    ss = where((fem_data.week eq week) and (fem_data.iday eq day), count)
end else begin
    ss = where(fem_data.week eq week, count)
end
ist = ss(0)				;changed to look at all orbits within the week and append where necessary
ien = ss(count-1) < (n_elements(fem_data)-2)	;stop one record before the end
if (fem_data(ien+1).time/1000. lt abs(margin)) then ien = ien + 1	;special case where the actual start time of S/C is in the
								;first "margin" (usually set to 5) minutes.  This results in
								;a FILEID in the PREVIOUS day.
if (86400-fem_data(ist).time/1000. lt abs(margin)) then ist = ist + 1
								;The opposite case of above (where the FILEID is in the 
								;prior day)
;
if (qdebug) then begin
    n = ien-ist+1
    print, 'Input file (', infil, ') is to be broken into ', strtrim(n,2), ' files
    for i=ist,ien do print, fmt_tim(fem_data(i)), '   ', string(fem_data(i).st$fileid)
    print, ' '
end
;
;----- Inititialization of variables
;
orbit_times ={orbit_times_rec, orb_st_time: long(0), $
                               orb_st_day: fix(0), $
                               orb_en_time: long(0), $
                               orb_en_day: fix(0)}
;------------------ Break the files up
;
idir = 0
for iprefix=0,nprefix-1 do begin
    start_time2 = systime(1)

    prefix = prefix_arr(iprefix)
    instr  = instr_arr(iprefix)
    ;infil = concat_dir(indir, prefix + fileid_in)
    infil = file_list(indir, prefix+'*')
    if (infil(0) eq '') then goto, skip			;sorry - afterthought

    break_file, infil, dsk_log, dir_in, filnam, ext
    file_fids = strmid(filnam, 3, 6) + ext
    file_times = anytim2ints( fid2ex(file_fids) )

    rd_roadmap, infil, roadmap, ndset
    ndset = total(ndset)
    ;; if (ndset eq 0) then goto, skip			;sorry - afterthought
    ndset_out = 0
    ndp_sync_out = 0
    if (ndset gt 0) then dset_out_arr = bytarr(ndset)	;added 17-Jun
    if (ndset eq 0) then roadmap = anytim2ints([0,0])	;added 17-Jun - make a structure with .TIME and .DAY
    last_ifil = -1
    dp_sync = 0
    ndp_sync = 0
    hxa_scan = 0
    ;
    for iorbit=ist,ien do begin
	fileid_out  = strtrim( fem_data(iorbit).st$fileid, 2 )
	fileid_ex = fid2ex(fileid_out)

	orbit_times.orb_st_time = fem_data(iorbit).time
	orbit_times.orb_st_day  = fem_data(iorbit).day
	orbit_times.orb_en_time = fem_data(iorbit+1).time
	orbit_times.orb_en_day  = fem_data(iorbit+1).day
	;
	dorbit = (orbit_times.orb_en_day - orbit_times.orb_st_day)*86400 + $
				(orbit_times.orb_en_time - orbit_times.orb_st_time)/1000.
	;;margin = int2secarr(fileid_ex, [orbit_times.orb_st_time, orbit_times.orb_st_day])	;should be negative
												;value since fileid_ex is
												;moved back "margin"
	margin = 0	;3-Sep-92 (MDM) Removed since it was causing problems with roundoffs losing seconds
	if ((prefix eq 'cba') or (prefix eq 'ada')) then margin = 0	;since there are major frames to be extracted in
									;night for CBA and ADA, do not use the margin.  WHen
									;use the margin, then there is time overlap between
									;orbit datafiles and there are problems.
									;ADDED 16-Apr-92 (MDM)
	;
	tvec = int2secarr(roadmap, [orbit_times.orb_st_time, orbit_times.orb_st_day])
	dset_arr = where((tvec ge margin) and (tvec lt dorbit+margin))
							;17-Apr-92 (MDM) moved the end time that is checked to an earlier time
							;by "margin" for daytime instruments because of problems with
							;orbit prediction being bad and getting data overlap
							;There also seems to be problems when S/C is commanded into day type
							;modes when it is in night.
	if (dset_arr(0) ne -1) then begin
	    sss = sort(tvec(dset_arr))
	    dset_arr = dset_arr(sss)		;time order the data
	    dset_out_arr(dset_arr) = 1		;flag that that dataset was extracted
	    dset_str = mk_dset_str(infil, dset_arr)	;convert to the dset structure
	end else begin
	    dset_str = mk_dset_str(infil, 0)		;define the data structure
	    dset_str.dset = -99				;set flag to say don't do any extraction
;;	    dset_str.ifil = ex2dow(anytim2ex(fem_data(iorbit)))		;--- Removed 3-Sep-92
;;	    if (ex2week(anytim2ex(fem_data(iorbit))) eq 1) then begin
;;		sss = where((fem_data.week eq week) and (fem_data.day_rev eq 1))	;get number of days in week #1
;;		dset_str.ifil = dset_str.ifil - (7-n_elements(sss))			;1992 case, ifil=6 above, nsss=4
;;	    end

	    sss = where( int2secarr(file_times, fem_data(iorbit)) le 0, count)	;--- Added 3-Sep-92
	    if (count eq 0) then dset_str.ifil = 0 else dset_str.ifil = sss(count-1)
	end
	;
	ifil = dset_str.ifil
	;ifil = ifil(uniq(ifil))
	if (n_elements(ifil) ge 2) then ifil = ifil(uniq(ifil, sort(ifil))) 	;added sort command 22-Dec-92
	if (strupcase(prefix) eq 'BDA') then begin
	    if (total(ifil-last_ifil) ne 0) then begin
		print, 'Reading DP_SYNC for ', infil(ifil)
		rd_bda_dp, infil(ifil), dp_sync		;do the read here, so only do it once per input file
							;cannot read all 7 files at once so only read one file
		ndp_sync = n_elements(dp_sync)
		last_ifil = ifil
	    end

            tvec = int2secarr(dp_sync, [orbit_times.orb_st_time, orbit_times.orb_st_day])
            dp_arr = where((tvec ge 0) and (tvec lt dorbit))
	end else begin
	    dp_arr = -1	;here is no "valid" dp_sync data for non-BCS instruments
	end
	if (strupcase(prefix) eq 'ADA') then begin
	    if (total(ifil-last_ifil) ne 0) then begin
		print, 'Reading HXA Scan for ', infil(ifil)
		rd_hxa, infil(ifil), hxa_scan
		last_ifil = ifil
	    end
	end


	if (qdebug) then begin
	    n = 0
	    if (dset_arr(0) ne -1) then n=n_elements(dset_arr)
	    if (n ne 0) then print, 'There are ', n, ' ', prefix,' datasets to go into orbit  ', fmt_tim(fem_data(iorbit))
	end
	;
	wrt1orbit, infil, prefix, instr, dir_out_map, fileid_out, dset_str, dp_arr, dp_sync, $
				progname, progverno, roadmap, dset_arr, hxa_scan, $
				orbit_times=orbit_times
	;
	if (dset_arr(0) ne -1) then ndset_out = ndset_out + n_elements(dset_arr)
	if (dp_arr(0) ne -1) then ndp_sync_out = ndp_sync_out + n_elements(dp_arr)
    end
    ;
    if ((ndset_out ne ndset) and (not keyword_set(day))) then begin
	print, 'Some datasets were not saved for ', infil
	print, 'There were: ', ndset, ' datasets.  # Saved = ', ndset_out
	if (keyword_set(errlog)) then printf, errlog, !stime, ' Only ', ndset_out, ' of ', ndset, '  saved'
	dset_arr = where(dset_out_arr eq 0)
;;	if (dset_arr(0) ne -1) then begin
;;	    dset_str = mk_dset_str(infil, dset_arr)
;;	    fileid_out0 = fileid_out + '_temp'
;;	    dp_arr = -1
;;          wrt1orbit, infil, prefix, instr, dir_out_map, fileid_out0, dset_str, dp_arr, dp_sync, $
;;				progname, progverno, roadmap, dset_arr, hxa_scan, $
;;                                orbit_times=orbit_times
;;	end
    end

    skip:
    end_time2 = systime(1)
    runtime2 = (end_time2-start_time2)/60.
    if (keyword_set(errlog)) then printf, errlog, !stime, ' REFORMAT2 ', prefix, ' files took ', $
									string(runtime2,format='(f7.2)'), ' min'
    if (keyword_set(errlog)) then flush, errlog
end
;
end_time = systime(1)
runtime = (end_time-start_time)/60.

end