;+
;NAME:
;	go_proc_dps
;PURPOSE:
;	To process the DPS incoming data
;METHOD:
;	a) *.SFDU are moved to the "~dps/to_be_deleted" directory
;	b) QL HK data is slotted into HK files and moved to "to_be_deleted"
;	c) LZ HK data is copied to HK online dirs and index created and
;	   moved to "to_be_deleted"
;	d) VC2 data is moved to "~dps/raw_vc2", has an inventory created, 
;	   is renamed has the reformatter(s) run
;HISTORY:
;	Written 2-Dec-97 by M.Morrison
;	 3-Dec-97 (MDM) - Added slotting of HK QL
;	 3-Dec-97 (MDM) - More stuff
;	 4-Dec-97 (MDM) - Added offset_time
;	 8-Dec-97 (MDM) - Corrected error of LZ handling of mix of HK and VC2
;	10-Feb-98 (MDM) - Removed exclusion of APID 72
;			- Made the default answer to running the old reformatter "No"
;	19-Feb-98 (RWN) - Added DOUBLE to time offsets in last PRINT
;	12-Mar-98 (MDM) - Changed I1_DIR to I1_BASE_DIR for base_outdir
;	31-Mar-98 (MDM) - Made it run non-interactively
;			- Removed old reformatter question
;			- Made QL HK data put a flag file in for web stuff
;			- Added log file output
;	14-Apr-98 (MDM) - Put in protection to not do data older than
;			  3 days ago for VC2 data
;	14-Apr-98 (MDM) - Changed to be last 1 day only
;	15-Apr-98 (NEH) - Changed to test on difference in time instead
;	16-Apr-98 (MDM) - Delete IDL log file if no new files
;	23-Apr-98 (MDM) - Added call to dps_proc_mem_dump
;			- Made "prob_apid" be only apid3, and handles LZ and QL
;	28-Apr-98 (MDM) - Added APID 14 check for LZ data
;	29-Apr-98 (MDM) - Corrected double use of "apid" variable which
;			  did not have memory dump apid 14 files being
;			  processed
;	10-Jun-98 (MDM) - Added handling of apid 73 (dhc memory dumps)
;			- Added handling of QL HK files after recipt of LZ file
;	23-Jul-98 (MDM) - Removed APID 3 as "problem apid" for LZ data
;	28-Jul-98 (MDM) - Removed the elimination of APID 3 and 5 as "problem apid"
;			  (QL only, not LZ is a problem)
;-
;
offset_time = 0
;
if (n_elements(qmv) eq 0) then qmv = 1
;
if (keyword_set(qstop3)) then stop
;
indir = '/disk6/people/dps'
deldir = concat_dir(indir, 'to_be_deleted')
deldir3 = concat_dir(indir, 'to_be_deleted3')
logfil = '$HOME/proc_dps.log'
;
if (keyword_set(qcleanup)) then begin
    ff = '*** CLEANUP.  Finding all TRACE_G* files ***'
end else begin
    ff = file_list(indir, 'DAN*.done')		;any "done-transfering" files there?
    if (ff(0) eq '') then begin
	logmsg = !stime + '  No DAN files, so no data to process
        print, logmsg
	file_delete, getenv('IDL_BATCH_LOGFIL')
	file_append, logfil, logmsg
        stop
    end
end
;
logmsg = !stime + '  New DAN files found: '
file_append, logfil, logmsg
file_append, logfil, '        ' + ff
;
nfil = n_elements(ff)
for ifil=nfil-1,0,-1 do begin
    print, '------- Now Processing ' + ff(ifil) + ' -----------------------'
    if (keyword_set(qcleanup)) then begin
	junk = file_list(indir, 'TRACE_G*', file=ff2)
	if (junk(0) eq '') then stop, 'No data to process
    end else begin
	cmd = ['grep', 'FILE_ID', ff(ifil)]
	spawn, cmd, ff2, /noshell
	ff2 = str_replace(ff2, 'FILE_ID=', '')
	ff2 = str_replace(ff2, ';', '')
	ff2 = strtrim(ff2, 2)
    end
    ;
    infils = concat_dir(indir, ff2)
    break_file, infils, dsk_log, dir, filnam, ext
    ;
    ;---- Get rid of the .SFDU files
    ss = where(ext eq '.SFDU', nss)
    for i=0,nss-1 do spawn, ['mv', infils(ss(i)), deldir], /noshell
    ;
    ;---- Reduce the file list down to only .DAT1
    ;
    ss = where(ext eq '.DAT1', nss)
    if (nss eq 0) then stop
    ff2 = ff2(ss)	;only .DAT1 left
    ;
    nff2 = n_elements(ff2)
    ;
    infils = concat_dir(indir, ff2)
    break_file, infils, dsk_log, dir, filnam, ext
    apid = fix(strmid(filnam, 8, 2))
    qlz  = strpos(filnam, '_LZ_') ne -1		;is it level zero
    qvc2 = (apid ge 60) and (apid le 72)	;is it VC2 data
    ;
    if (keyword_set(qstop)) then stop
    ;
    ;------------------------- QL HK handling (Junk?)
    ;
    ;TODO - what to do with APID 3 data
    ;
    ;;prob_apid = [3, 14]		;72
    prob_apid = [3,5]	;don't slot APID 3 or 5
    flag = where_arr(apid, prob_apid, nss, /map_ss)
    ;ss = where((1-qlz) and (1-qvc2) and (flag ge 0), nss)
    ;ss = where((1-qvc2) and (flag ge 0), nss)
    ss = where((1-qlz) and (1-qvc2) and (flag ge 0), nss)
    if (qmv) then for i=0,nss-1 do spawn, ['mv', infils(ss(i)), deldir], /noshell
    ;
    ;------------------------- QL HK handling (slotting)
    ;
    ss = where((1-qlz) and (1-qvc2) and (apid ne 14) and (apid ne 73) and (flag eq -1), nss)
    if (nss ne 0) then file_append, logfil, [!stime + '  Files to get QL HK slotting handling: ', $
							'        ' + infils(ss)]
    for i=0,nss-1 do begin
	check_ql_after_lz, infils(ss(i)), rep_out, status, init_out=(i eq 0)
	if (status ge 0) then begin
            timarr = dps_fname2time (infils(ss(i)), junk, fid, dps, apid00)
	    cmd = ['/disk6/people/trace/egse/tm_mgr', '-t', 'gsfc_trace', $
					'-s', strmid(fid, 2, 6), $
					'-i', deldir, $
					infils(ss(i))]
	    ;/disk6/people/trace/egse/tm_mgr -t gsfc_trace -s 980407 TRACE_G078_QL_1998-04-07T10-10-33Z_V01.DAT1
	    print, cmd
	    spawn, cmd, r, /noshell
	    prstr, r
            if (keyword_set(qstop)) then stop
	end else begin
	    if (status eq -100) then begin
		mail, rep_out, users='morrison@lmsal.com,ngale@lmsal.com', subj='New LZ file needed'
		spawn, ['mv', '-f', infils(ss(i)), deldir3], /noshell	;get rid of it
	    end else begin
		spawn, ['mv', '-f', infils(ss(i)), deldir], /noshell	;get rid of it
	    end
	end
	;
	;-- Do web work flag file
	;
	if (i eq 0) then begin
	    web_flag = '$GSE_GENTM_LOG_DIR/trig_web'
	    file_append, web_flag, infils(ss(i))
	end
    end
    ;
    ;------------------------- QL/LZ Memory load handling 
    ;
    ss = where(apid eq 14, nss)
    if (nss ne 0) then file_append, logfil, [!stime + '  Files to get Memory Load Processing: ', $
							'        ' + infils(ss)]
    if (nss ne 0) then begin
	dps_proc_mem_dump, infils(ss)
        if (keyword_set(qstop)) then stop
    end
    ;
    ;------------------------- LZ HK handling
    ;
    prob_apid = [9999]
    flag = where_arr(apid, prob_apid, nss, /map_ss)
    ss = where((qlz) and (1 - qvc2) and (apid ne 14) and (flag eq -1), nss)
    if (nss ne 0) then file_append, logfil, [!stime + '  Files to get LZ HK Copy: ', $
							'        ' + infils(ss)]
    if (nss ne 0) then begin
	outdir = getenv('GSE_GENTM_LOG_DIR')
	dps_proc_lev0_hk, infils(ss), outdir
	if (qmv) then for i=0,nss-1 do spawn, ['mv', infils(ss(i)), deldir], /noshell
        if (keyword_set(qstop)) then stop
    end
    ;
    ;ff=file_list('.','*G003*LZ*.DAT1')
    ;outdir = getenv('GSE_GENTM_LOG_DIR')
    ;dps_proc_lev0_hk, ff, outdir
    ;
    ;------------------------- VC2 handling
    ;
    ss = where(qvc2 and (flag eq -1), nss)
    if (nss ne 0) then file_append, logfil, [!stime + '  Files to get VC2 processing: ', $
							'        ' + infils(ss)]
    if (nss ne 0) then begin
        timarr = dps_fname2time (infils(ss(0)), junk, fid, dps, apid00)
	outdir = getenv('TRACE_RAW_VC2')
	if (outdir eq '') then outdir = concat_dir(indir, 'raw_vc2')
	if (not file_exist(outdir)) then spawn, ['mkdir', outdir], /noshell
	;
	base_outdir = getenv('TRACE_I1_BASE_DIR')
	do_tr_inventory, infils(ss), sttim, entim, outdir=outdir, rename=outdir, outfil=outfil
	;
	print,'Reformatted data spans from: ' + fmt_tim(anytim2ints('24-May-68',off=double(sttim))) $
	       + '  to ' + fmt_tim(anytim2ints('24-May-68', off=double(entim)))
	;
	dt = (entim-sttim)/86400.	;number of days spanned
	if (dt gt 2) then begin		; see if the time difference is more than two days
	    mail, users='trace', ['GO_PROC_DPS  VC2 data spans a long period (' + strtrim(dt) + ' days', $
		'Reformatted data spans from: ' + fmt_tim(anytim2ints('24-May-68',off=double(sttim))) $
	       + '  to ' + fmt_tim(anytim2ints('24-May-68', off=double(entim))), $
		'Going to truncate and only two days from the start time']
	    entim = sttim + 86400*2
	end
	;
	do_tr_reformat, sttim, entim, base_outdir, offset_time=offset_time
	if (keyword_set(qstop2)) then stop
        if (keyword_set(qstop)) then stop
    end
    ;
    ;------------------------- DHC Memory dump handling 
    ;
    ss = where(apid eq 73, nss)
    if (nss ne 0) then file_append, logfil, [!stime + '  Files to get DHC Memory Processing: ', $
							'        ' + infils(ss)]
    if (nss ne 0) then begin
	outdir_dhc = concat_dir(getenv('GSE_DATA_DIR1'), 'log/dhc_dump')
	for i=0,nss-1 do begin
            timarr = dps_fname2time (infils(ss(i)), junk, fid, dps, apid00)
	    cmd = ['mv', infils(ss(i)), concat_dir(outdir_dhc, fid+'.sc49')]
	    print, cmd
	    spawn, cmd, /noshell
	end
        if (keyword_set(qstop)) then stop
    end
    ;
    ;------------------------- Move the DAN file out
    ;
    if (not keyword_set(qcleanup)) then begin
	if (qmv) then spawn, ['mv', ff(ifil), deldir], /noshell
    end
end
;
end