pro trace_do_data_requests, test=test, debug=debug
;+
;   Name: trace_do_data_requests
;
;   Purpose: background driver to handle TRACE WWW data request queue
;
;   History:
;     21-May-1998 - S.L.Freeland - LMSAL - driver for 'trace_order_data'
;      2-Jun-1998 - S.L.Freeland - use subdirectory orders/requested
;                                                         /current
;                                                         /finished  
;     12-Jun-1998 - S.L.Freeland - call trace_trace_wwworders_index
;     18-Aug-1998 - S.L.Freeland - call index generator every loop
;  
;   Method:
;     ----------------------------------------------------------------------
;     1) check current job area - if any active jobs, then exit
;     2) check incoming job area (via trace_submit_request)
;        If new jobs exist, then 
;            a) move jobs from ...orders -> ...orders_current
;            b) for each new job, process [via trace_order_data,/background]
;        Else (no new jobs) exit   
;     ----------------------------------------------------------------------
;-
debug=keyword_set(debug)
notify=get_logenv('TRACE_DATA_NOTIFY')
if notify eq '' then notify='freeland@sxt1.space.lockheed.com'

configure_http,/trace                     ; set WWW defaults to TRACE

; ---------- define the job queue area --------------------
topord=concat_dir(concat_dir('path_http','Data'),'orders') 

neword_dir=concat_dir(topord,'requested') ; new WWW orders
curord_dir=concat_dir(topord,'current')   ; running orders
finord_dir=concat_dir(topord,'finished')  ; done orders
; -----------------------------------------------------------

order_pat='trace_idlcat_??????_??????.success'  ; file pattern for data request
curjobs=file_list(curord_dir,order_pat,/cd)     ; check for running jobs

if debug then stop,'curjobs'

logfiles=file_list('$SSW_SITE_LOGS','go_trace_do_data_requests.log*',/cd)
logfiles=logfiles(sort(file2time(logfiles,out='tai')))
nlogs=n_elements(logfiles)
if nlogs gt 10 then begin
   logfiles=logfiles(0:nlogs-9)          ; dont clobber recent/current log
   logsizes=file_size(logfiles)
   small=where(logsizes lt 4000,scnt)
   if scnt gt 0 then begin
      box_message,'Purging '+ strtrim(scnt,2) + ' log files...'
      file_delete,logfiles(small)
   endif else box_message,'no log files to purge..'
endif else box_message,'Number of log files lt cutoff'

if curjobs(0) ne '' gt 0 then begin             ; any running jobs?
   box_message,'Jobs in progress, exiting'      ; yes, so exit
   return
endif  

newjobs=file_list(neword_dir,order_pat,/cd)
if newjobs(0) eq '' then begin
   box_message,'No new WWW data requests, exiting...;
   return
endif  

njcnt=n_elements(newjobs)
break_file,newjobs,ll,pp,ff,ee,vv
box_message,['Number of new jobs: ' + strtrim(njcnt,2), $
	     'New Job Names:','   '+ff+ee+vv]

; ------------- move new jobs -> current area ----------
jobmark=';;;; trace_do_data_requests:'
newline=jobmark + ' run at '+systime()                     ; job marker
newnames=str_replace(newjobs, neword_dir, curord_dir)      ; new names
finnames=str_replace(newjobs, neword_dir, finord_dir)      ; finished names

for i=0,njcnt-1 do begin
   jobcont=rd_tfile(newjobs(i))                         ; job contents 
   newcont=[jobcont,newline]                            ; append marker
   file_append,newnames(i),newcont,/new                 ; rewrite in cur area

   if file_size(newnames(i)) gt file_size(newjobs(i)) then begin ; CONFIRM!
      file_delete,newjobs(i)                            
    endif else begin
      mess='Problem moving job '+newjobs(i) + ' from new->current'
      mail,mess,subj='TRACE data request problem',/no_defsubj
      box_message, mess
    endelse      
endfor

; Now process the requests (use the versions in current area queue)
for i=0,njcnt-1 do begin
   box_message,'Working on request: ' + newnames(i)         ; where we are
   trace_order_data, newnames(i), /background,$             ; run the data job
      email=email, outfiles=outfiles
   finmark=jobmark + ' completed at ' + systime()           ; complete marker
   file_append,finnames(i),[rd_tfile(newnames(i)),finmark]      
   trace_request_summary, finnames(i), url, outfiles=outfiles,/nimg
   url=str_replace(url,'/www/www','http://vestige.lmsal.com')
   url=str_replace(url,'www.space.lockheed.com','vestige.lmsal.com')

   mess=['Information on your jobs is avaliable at:','  ' +url]
   mail, mess, user=email, subj='Your TRACE Data Request has Completed', /no_def
   trace_wwworders_index         ; update the index.html
   if file_size(finnames(i)) gt file_size(newnames(i)) then begin
      file_delete,newnames(i)
   endif else begin
      mess='Problem moving job '+newjobs(i) + ' from current->finished'
      box_message,mess
      mail,mess,subj='TRACE data request problem',/no_defsubj
   endelse     
endfor


if debug then stop

return
end