;+
; NAME:
;	FF_SUMMARY
;
;
; PURPOSE:
;	Get a 1-line summary of a FITS file (SMEI)
;
;
; CATEGORY:
;	UTILS
;
;
; CALLING SEQUENCE:
;	desc = ff_summary(file)
;
;
; INPUTS:
;	file	string	The file to summarize (if the path key is set,
;			then it can be a wildcard).
;
;
; KEYWORD_PARAMETERS:
;	/template	Return a title bar for a summary
;	path	string	A directory to search for the files. (If this
;			is given then file is optional and defaults to
;			the current directory.
;	filelist str	If this is a named variable, then it will
;			return the list of filenames (only really
;			useful when path is given).
;	startlist dbl	If this is a named variable then it will
;			contain the JD's of the image start times.
;	count	long	If this is a named variable, on return it will
;			contain the number of files found.
;	/no_file	If set then actually open each file rather
;			than using the img_hdr.txt file to get the info.
;	/norecurse	If set, then do not look in subdirectories.
;	projections str	A named variable to hold the list of
;			projections of the images.
;	/no_progress	If set, then do not create a progress bar to
;			show progress through files and directories
;			(mainly for running over a slow network).
;	/update		If set, then force a reprocessing just as if
;			the directory has been changed
;	callid	long	The widget ID of a calling widget.
;
;
; MODIFICATION HISTORY:
;	Original: 13/12/02; SJT
;	Modify for degenerate Aitoff files: 7/3/03; SJT
;	Modify to allow use of a tree to store files: 1/7/03; SJT
;	Add fold_case keyword to file_search calls: 23/7/03; SJT
;	Fix crash in single plane images: 2/10/03; SJT
;	Deal with possibility of files called .fts: 20/11/03; SJT
;	Add projection information, and projections keyword, and save
;	list: 16/2/04; SJT
;	Do recursion explicitly to allow symlinked subdirs to work:
;	12/8/04; SJT
;	Major efficiency changes: 11/11/04; SJT
;	Add list of mal-indexed directories: 31/5/05; SJT
;	Support compressed fits files: 24/5/05; SJT
;	Add no_progress keyword: 25/8/05; SJT
;	Several major speedups (removed an N^2 operation) & handle
;	Pipe 2 processing flags: Oct-Nov 07; SJT
;	Don't convert headers to structures (fails randomly &
;	intermittently): 4/8/09; SJT
;	Major rewrite, mainly to only scan modified directories on
;	call with /UPDATE: 20/4/10; SJT
;	Add callid key: 20/7/10; SJT
;-

function ff_summary, file, template = template, path = path, $
                     filelist = filelist, startlist = startlist, count $
                     = count, no_file = no_file, projections = $
                     projections, norecurse = norecurse, no_progress = $
                     no_progress, update = update, callid = callid

common ff_summary_lastpath, lastpath, laststr, lastdesc, lastepoch

if keyword_set(template) then return, $
  ' Year Day Start Length Cols Rows Flags'

now = systime(1)                ; Unix epoch

if n_elements(lastpath) eq 0 or keyword_set(no_file) then lastpath = ''
if ~keyword_set(path) then path = '.'
chdir = lastpath eq '' || ~file_same(lastpath, path)
iupdate = chdir || keyword_set(update) || now - lastepoch gt 5.4d3
if chdir then begin
    if n_elements(laststr) ne 0 then begin
        junk = temporary(laststr)
        junk = 0
    endif
    lastepoch = 0.d0
    iupdate = 1b
endif

sep = '/'

if widget_info(/active) eq 0 or keyword_set(no_progress) then $
  isprog = 0b $
else begin
    isprog = 1b
    ismodal = widget_info(callid, /valid); && widget_info(callid, $
;                                                         /modal)
endelse
  
if n_params() eq 0 then file = '*.f*t*' ; Covers .fit .FIT
                                ; .fts etc. with the fold_case option.


; Scan for directories changed since the last update (all dirs if the
; root has changed)

if iupdate then begin
    if keyword_set(norecurse) then begin
        dlist = path 
        ndir = 0
    endif else begin
        dlist = [path, file_search(path, '*', /test_directory, $
                                   count = ndir)]
        if ndir eq 0 then dlist = path
    endelse

    if isprog then $
      pm = cw_progress(title = 'Scanning for headers and files', $
                       value = 0., $
                       /column, $
                       xsize = 200, $
                       ysize = 20, $
                       max = ndir+1, $
                       group = callid, $
                       modal = ismodal) $
    else pm = 0l

    ndsc = 0l
    mdflags = bytarr(ndir+1)
    csflag = 1b
    for j = 0, ndir do begin
        dinfo = file_info(dlist[j])
        if widget_info(/valid, pm) then $
          widget_control, pm, set_value = j+1
        if dinfo.mtime lt lastepoch then continue ; This may rescan
                                ; directories that have had other
                                ; directories or non-matching files
                                ; added.
        mdflags[j] = 1b
        if csflag then begin
            cfile = file_search(dlist[j]+sep+file, /fold_case, count = $
                                countd)
            hfile = file_search(dlist[j]+sep+'img_hdr.txt', $
                                /fold_case, count = hcountd)
            count = countd
            hcount = hcountd
            csflag = 0b
        endif else begin
            cfile = [cfile, file_search(dlist[j]+sep+file, /fold_case, $
                                        count = countd)]
            hfile = [hfile, file_search(dlist[j]+sep+'img_hdr.txt', $
                                        /fold_case, count = hcountd)]

            count += countd
            hcount += hcountd
        endelse
        ndsc++
    endfor
    if n_elements(count) eq 0 || count eq 0 then begin
        if widget_info(/valid, pm) then $
          widget_control, pm, /destroy
        count = 0l
        return, ''
    endif

; Remove any null strings from the lists (added when no matches are
; found in a directory.

    if ndsc gt 0 then begin
        locs = where(cfile ne '', nc)
        if nc ne count then begin
            smei_msg, /warn, ["File count does not match", $
                              "Scan: "+string(count, format = "(I0)")+ $
                              "Where: "+string(nc, format = "(I0)")]
            count = nc
        endif
        cfile = cfile[locs]
        
        locs = where(hfile ne '', nh)
        if nh ne hcount then begin
            smei_msg, /warn, ["File header count does not match", $
                              "Scan: "+string(hcount, format = "(I0)")+ $
                              "Where: "+string(nh, format = "(I0)")]
            hcount = nh
        endif
        hfile = hfile[locs]
    endif
        
    if widget_info(/valid, pm) then $
      widget_control, pm, /destroy

endif else ndsc = 0l

; If no changes to be made just decode & return

if ~iupdate || ndsc eq 0 then begin
    desc = lastdesc
    filelist = transpose([[laststr.dir], [laststr.fname]])
    startlist = laststr.stime
    projections = laststr.project
    count = n_elements(desc)
    return, desc
endif

locs = where(mdflags)           ; Should not be able to be zero.
mdlist = dlist[locs]

; Read the image header files from the changed directories.

if not keyword_set(no_file) then begin
    if hcount eq 0 then goto, nohdr
    on_ioerror, nohdr
    lines = file_lines(hfile)
    nfe = total(lines, /int)
    hfiles = strarr(nfe)
    hdesc = strarr(nfe)

    inln = strarr(2)
    if isprog then $
      pm = cw_progress(title = 'Scanning directories', $
                       value = 0., $
                       /column, $
                       xsize = 200, $
                       ysize = 20, $
                       max = hcount, $
                       group = callid, $
                       modal = ismodal) $
    else pm = 0l

    jf = 0l
    for j = 0, hcount-1 do begin
        openr, ilu, /get, hfile[j]
        for k = 0, lines[j]-1 do begin
            readf, ilu, inln, format = "(A35, A)"

            hfiles[jf] = strtrim(inln[0])
            hdesc[jf] = inln[1]
            jf++
        endfor

        free_lun, ilu
        if widget_info(/valid, pm) then $
          widget_control, pm, set_value = j+1
    endfor

    if widget_info(/valid, pm) then $
      widget_control, pm, /destroy

    nhfiles = nfe
    goto, hdrok

    nohdr: 
    smei_msg, /warn, "No img_hdr.txt found, this will make things " + $
      "slow"
    hfiles = ''

    hdrok:
    on_ioerror, null
endif else hfiles = ''

iflag = 0b
desc = strarr(count > 1)
filelist = strarr(2, count > 1)
startlist = dblarr(count > 1)
projections = strarr(count > 1)

cfile_base = get_comp_name(file_basename(cfile), /return_base)
cfile_dir = file_dirname(cfile, /mark)
hlocs = match_lists(hfiles, cfile_base)

if count eq 0 then begin
    if n_elements(laststr) eq 0 then return, ''
    desc = lastdesc
    filelist = transpose([[laststr.dir], [laststr.fname]])
    startlist = laststr.stime
    projections = laststr.project
    count = n_elements(desc)
    return, desc
endif

if isprog then begin
    pm = cw_progress(title = 'Checking files', $
                     value = 0., $
                     /column, $
                     xsize = 200, $
                     ysize = 20, $
                     max = count, $
                     group = callid, $
                     modal = ismodal)
    ispid = 1b
endif else ispid = 0b

appflag = n_elements(laststr) gt 0 
nflags = bytarr(count)
new_str = replicate({ff_data}, count)

for j = 0l, count-1 do begin
    sf = [cfile_dir[j], cfile_base[j]]

    if (j+1) mod 20 eq 0  or j eq count - 1 then begin  
        if ispid then widget_control, pm, set_value = $
          j+1 $
        else print, string(13b), j+1, count, format = $
          "($,A,I0,' of ',I0)"
    endif

; Check if the file is already in the list
    
    if appflag then begin
        locs = where(laststr.fname eq cfile_base[j] and $
                     laststr.dir eq cfile_dir[j], nmat)
        if nmat gt 0 then continue
    endif
    nflags[j] = 1b

    if hlocs[j] eq -1 then begin ; This file wasn't listed in any
                                ; header list

        if not keyword_set(no_file) then begin
            iflag = 1b
            if n_elements(baddir) eq 0 then baddir = cfile_dir[j] $
            else baddir = [baddir, cfile_dir[j]]
        endif
        if ~file_test(cfile[j]) then begin
            cfile[j] = cfile[j]+'.gz'
            if ~file_test(cfile[j]) then begin
                printf, -2, "File "+cfile[j]+" not found"
                continue
            endif
        endif
        new_str[j] = ff_extract(cfile[j])

; CHUNK 1

    endif else if hcount ne 0 then begin
        new_str[j] = ff_tostr(hdesc[hlocs[j]], file = sf)
; CHUNK 2
    endif

endfor

if ispid then widget_control, pm, /destroy $ $
else print

if iflag then begin
    idx = uniq(baddir, sort(baddir))
    smei_msg, /warn, ["Not all files found are in " + $
                      "img_hdr.txt", $
                      "you should re-run mk_imghdrtxt on " + $
                      "this directory:", $
                      baddir[idx]]
endif

; Add any
locs = where(nflags, nnf)
if nnf gt 0 then begin
    if appflag then begin
        laststr = [laststr, new_str[locs]]
        lastdesc = [lastdesc, ff_todesc(new_str[locs])]
    endif else begin
        laststr = new_str
        lastdesc = ff_todesc(laststr)
    endelse
    idx = sort(laststr.stime)
    laststr = laststr[idx]
    lastdesc = lastdesc[idx]
endif

if chdir then lastpath = path
lastepoch = now

desc = lastdesc
filelist = transpose([[laststr.dir], [laststr.fname]])
startlist = laststr.stime
projections = laststr.project
count = n_elements(desc)

return, desc

end