pro sort_summary,yyyymmdd,tel,logdiff=logdiff, FILES=files, FIRST=first, USEFULLROW=usefullrow
;+
; $Id: sort_summary.pro,v 1.8 2011/08/31 19:00:20 secchia Exp $
;
; Project     : STEREO SECCHI
;                   
; Name        : 
;               
; Purpose     : sort summary file(s) by filenames and remove duplicates, saving the lowest one after sort
;               
; Explanation : 
;               
; Use         : IDL> sort_summary, yyyymmdd, tel
;    
; Inputs      : yyyymmdd = day used in secchi_reduce only yyyymm of string is used
;               tel telescope name used in secchi_reduce  use only first character since cor's and hi's are prosecced together.
;
; Outputs     : 
;
; Keywords :	LOGDIFF = log difference in unsorted snd sort summary files
;   	FILES=	name of summary file(s) to sort and over-write
;   	/USEFULLROW 	Use full row for row uniqueness, not just filename
;   	/FIRST	Passed to uniq(); use ONLY IF summary file has already been sorted
;  
; Calls from LASCO : 
;
; Common      : 
;               
; Restrictions: Requires pipeline processing environment (user secchia or secchib)
;               
; Side effects: overwrites files in $SECCHI/summary
;               
; Category    : Pipeline 
;               
; Prev. Hist. : None.
;
; Written     : Lynn McNutt, NRL/I2, Feb 2008
;               
; Modified    :
;
; $Log: sort_summary.pro,v $
; Revision 1.8  2011/08/31 19:00:20  secchia
; fix sfx so c does not return cal
;
; Revision 1.7  2011/08/31 17:57:29  nathan
; undo last change and instead implement /USEFULLROW and /FIRST
;
; Revision 1.6  2011/08/31 16:51:06  nathan
; add /FNAME_ONLY, use uniq(/first), add some info msgs
;
; Revision 1.5  2011/08/11 18:35:18  secchib
; nr - keyword FILES=
;
; Revision 1.4  2009/04/03 22:31:00  secchib
; nr - exit gracefully if file does not exist
;
; Revision 1.3  2008/07/01 21:53:07  secchia
; nr - put outfile in /tmp
;
; Revision 1.2  2008/03/18 13:02:36  secchia
; remove idff file if size is 0, no difference found
;
; Revision 1.1  2008/02/28 18:39:56  mcnutt
; to be call in secchi_reduce at the end of a processing month
;

IF keyword_set(FILES) THEN sumlist=files ELSE BEGIN

    sfx='*.'+strlowcase(strmid(tel,0,1))+'?'

    sumdir=getenv('SECCHI')+'/summary'
    yyyymm=strmid(yyyymmdd,0,6)

    sumfiles=findfile(sumdir+'/*'+yyyymm+sfx)
    sumlist=sumfiles
    IF strmid(yyyymmdd,6,2) EQ '01' and sumfiles[0] EQ '' THEN BEGIN
	help,yyyymmdd
	message,'No file to sort.'
	return
    ENDIF
    
ENDELSE

for n=0,n_Elements(sumlist)-1 do begin
    sumfile=sumlist(n)
    print,sumfile
    sumlines=strarr(6000*31l) & d=0l & line=''
    openr,2,sumfile
    while (not eof(2)) do begin
    	readf,2,line
      	sumlines(d)=line
      	d=d+1
    endwhile   
    close,2
 
    header=sumlines[0:1]
    sumlines=sumlines[2:d]
    sumlines=sumlines(where(sumlines ne ''))
    IF keyword_set(USEFULLROW) THEN rows=sumlines ELSE rows=strmid(sumlines,0,27) 
    order=sort(rows)
    unq=uniq(rows[order], FIRST=first)
    newsumlines=[header,sumlines[order[unq]]]
    message,'Removing '+trim(d-2-n_elements(unq))+' out of '+trim(d-2)+' rows.',/info
    break_file,sumfile,di,pa,ro,sf
    fname=ro+sf

    ;write the sorted file
    outfile='/tmp/'+fname+'.new'

    openw,3,outfile,width=200
    for np=0l,long(n_elements(newsumlines)-1) do printf,3,newsumlines(np)
    close,3
    if keyword_set(logdiff) then begin
    	srtfile='/tmp/'+fname+'.sort'
	openw,3,srtfile
	oldsumlines=[header,sumlines[order]]
	FOR np=0l,n_elements(oldsumlines)-1 DO printf,3,oldsumlines[np]
	close,3
      	difffile=getenv('seb_img')+'/log/'+fname+'.diff'
      	syscmd='diff '+outfile+' '+srtfile+' >>'+difffile
      	print,syscmd
      	wait,1
      	spawn,syscmd,/sh
      	diffinfo=file_info(difffile)    
      	if (diffinfo.size eq 0) then begin
            syscmd='/bin/rm -f '+difffile
            spawn,syscmd,/sh
      	endif         
      	spawn,'rm '+srtfile,/sh
    endif

    syscmd='mv '+outfile+' '+sumfile
    print,syscmd

    wait,2
    spawn,syscmd,/sh

endfor

end