;+
; NAME:
;	FF_SUMMARY
;
;
; PURPOSE:
;	Get a 1-line summary of a FITS file (SMEI)
;
;
; CATEGORY:
;	UTILS
;
;
; CALLING SEQUENCE:
;	desc = ff_summary(file)
;
;
; INPUTS:
;	file	string	The file to summarize (if the path key is set,
;			then it can be a wildcard).
;
;
; KEYWORD_PARAMETERS:
;	/template	Return a title bar for a summary
;	path	string	A directory to search for the files. (If this
;			is given then file is optional and defaults to
;			the current directory.
;	filelist str	If this is a named variable, then it will
;			return the list of filenames (only really
;			useful when path is given).
;	startlist dbl	If this is a named variable then it will
;			contain the JD's of the image start times.
;	count	long	If this is a named variable, on return it will
;			contain the number of files found.
;	/no_file	If set then actually open each file rather
;			than using the img_hdr.txt file to get the info.
;	/norecurse	If set, then do not look in subdirectories.
;	projections str	A named variable to hold the list of
;			projections of the images.
;	/no_progress	If set, then do not create a progress bar to
;			show progress through files and directories
;			(mainly for running over a slow network).
;	/update		If set, then force a reprocessing just as if
;			the directory has been changed
;	callid	long	The widget ID of a calling widget.
;
;
; MODIFICATION HISTORY:
;	Original: 13/12/02; SJT
;	Modify for degenerate Aitoff files: 7/3/03; SJT
;	Modify to allow use of a tree to store files: 1/7/03; SJT
;	Add fold_case keyword to file_search calls: 23/7/03; SJT
;	Fix crash in single plane images: 2/10/03; SJT
;	Deal with possibility of files called .fts: 20/11/03; SJT
;	Add projection information, and projections keyword, and save
;	list: 16/2/04; SJT
;	Do recursion explicitly to allow symlinked subdirs to work:
;	12/8/04; SJT
;	Major efficiency changes: 11/11/04; SJT
;	Add list of mal-indexed directories: 31/5/05; SJT
;	Support compressed fits files: 24/5/05; SJT
;	Add no_progress keyword: 25/8/05; SJT
;	Several major speedups (removed an N^2 operation) & handle
;	Pipe 2 processing flags: Oct-Nov 07; SJT
;	Don't convert headers to structures (fails randomly &
;	intermittently): 4/8/09; SJT
;	Major rewrite, mainly to only scan modified directories on
;	call with /UPDATE: 20/4/10; SJT
;	Add callid key: 20/7/10; SJT
;-

function ff_summary, file, template = template, path = path, $
                     filelist = filelist, startlist = startlist, count $
                     = count, no_file = no_file, projections = $
                     projections, norecurse = norecurse, $
                     no_progress = no_progress, update = update, $
                     callid = callid, noverify = noverify

  common ff_summary_lastpath, lastpath, laststr, lastdesc, lastepoch

  if keyword_set(template) then return, $
     ' Year Day Start Length Cols Rows Flags'

  if keyword_set(noverify) and keyword_set(no_file) then message, $
     /continue, "NOVERIFY & NO_FILE should not both be set, ignoring " + $
     "NOVERIFY"

  now = systime(1)              ; Unix epoch

  if n_elements(lastepoch) eq 0 then lastepoch = 0.d0
  if n_elements(lastpath) eq 0 or keyword_set(no_file) then lastpath = $
     ''
  if ~keyword_set(path) then path = '.'

  chdir = lastpath eq '' || ~file_same(lastpath, path)
  iupdate = chdir || keyword_set(update) || now - lastepoch gt 5.4d3

; If we are up-to-date, then just return the stored lists.
  
  if (~iupdate) then begin
     desc = lastdesc
     filelist = transpose([[laststr.dir], [laststr.fname]])
     startlist = laststr.stime
     projections = laststr.project
     count = n_elements(desc)
     return, desc
  endif

  if chdir then begin
     if n_elements(laststr) ne 0 then begin
        junk = temporary(laststr)
        junk = 0
     endif
     lastepoch = 0.d0
     iupdate = 1b
  endif

  sep = '/'

  if widget_info(/active) eq 0 or keyword_set(no_progress) then $
     isprog = 0b $
  else begin
     isprog = 1b
     ismodal = widget_info(callid, /valid) ; && widget_info(callid, $
;                                                         /modal)
  endelse
  
  if n_params() eq 0 then file = '*.f*t*' ; Covers .fit .FIT
                                ; .fts etc. with the fold_case option.


; Scan for directories changed since the last update (all dirs if the
; root has changed)

  t0 = systime(1)
  if keyword_set(norecurse) then begin
     dlist = path 
     ndir = 0
  endif else begin
     dlist = [path, file_search(path, '*', /test_directory, $
                                count = ndir, /nosort)]
     if ndir eq 0 then dlist = path
  endelse
  t1 =  systime(1)
  print, "Dir find:", t1-t0, t1-now

  if isprog then $
     pm = cw_progress(title = 'Scanning for headers', $
                      value = 0., $
                      /column, $
                      xsize = 200, $
                      ysize = 20, $
                      max = ndir+1, $
                      group = callid, $
                      modal = ismodal) $
  else pm = 0l

  ndsc = 0l
  mdflags = dlist.mtime ge lastepoch
  l_str = ptrarr(ndir+1)

  for j = 0, ndir do begin
     dinfo = file_info(dlist[j])
     if widget_info(/valid, pm) then $
        widget_control, pm, set_value = j+1
     if dinfo.mtime lt lastepoch then continue ; This may rescan
                                ; directories that have had other
                                ; directories or non-matching files
                                ; added.
     mdflags[j] = 1b
     if csflag then begin $
        cfile = file_search(dlist[j]+sep+file, /fold_case, count = $
                            countd)
        hfile = file_search(dlist[j]+sep+'img_hdr.txt', $
                            /fold_case, count = hcountd)
        count = countd
        hcount = hcountd
        csflag = 0b
     endif else begin $
        cfile = [cfile, file_search(dlist[j]+sep+file, /fold_case, $
                                    count = countd)]
        hfile = [hfile, file_search(dlist[j]+sep+'img_hdr.txt', $
                                    /fold_case, count = hcountd)]

        count += countd
        hcount += hcountd
     endelse
     ndsc++
  endfor
  t2 = systime(1)
  print, "Dir scan:", t2-t1, t2-now

  if n_elements(count) eq 0 || count eq 0 then begin
     if widget_info(/valid, pm) then $
        widget_control, pm, /destroy
     count = 0l
     return, ""
  endif

; Remove any null strings from the lists (added when no matches are
; found in a directory.

  if ndsc gt 0 then begin
     locs = where(cfile ne '', nc)
     if nc ne count then begin
        smei_msg, /warn, ["File count does not match", $
                          "Scan: "+string(count, format = "(I0)")+ $
                          "Where: "+string(nc, format = "(I0)")]
        count = nc
     endif
     cfile = cfile[locs]
     
     locs = where(hfile ne '', nh)
     if nh ne hcount then begin
        smei_msg, /warn, ["File header count does not match", $
                          "Scan: "+string(hcount, format = "(I0)")+ $
                          "Where: "+string(nh, format = "(I0)")]
        hcount = nh
     endif
     hfile = hfile[locs]
  endif
  
  if widget_info(/valid, pm) then $
     widget_control, pm, /destroy


; If no changes to be made just decode & return

  if ~iupdate || ndsc eq 0 then begin
     desc = lastdesc
     filelist = transpose([[laststr.dir], [laststr.fname]])
     startlist = laststr.stime
     projections = laststr.project
     count = n_elements(desc)
     return, desc
  endif

  locs = where(mdflags)         ; Should not be able to be zero.
  mdlist = dlist[locs]

; Read the image header files from the changed directories.

  if not keyword_set(no_file) then begin
     t3 = systime(1)
     print, "Misc:", t3-t2, t3-now
     if hcount eq 0 then goto, nohdr
     on_ioerror, nohdr
     lines = file_lines(hfile)
     nfe = total(lines, /int)
     hfiles = strarr(nfe)
     hdesc = strarr(nfe)
     t4 = systime(1)
     print, "Count hdr lines:", t4-t3, t4-now

     inln = strarr(2)
     if isprog then $
        pm = cw_progress(title = 'Scanning directories', $
                         value = 0., $
                         /column, $
                         xsize = 200, $
                         ysize = 20, $
                         max = hcount, $
                         group = callid, $
                         modal = ismodal) $
     else pm = 0l

     jf = 0l
     for j = 0, hcount-1 do begin
        openr, ilu, /get, hfile[j]
        for k = 0, lines[j]-1 do begin
           readf, ilu, inln, format = "(A35, A)"

           hfiles[jf] = strtrim(inln[0])
           hdesc[jf] = inln[1]
           jf++
        endfor

        free_lun, ilu
        if widget_info(/valid, pm) then $
           widget_control, pm, set_value = j+1
     endfor

     if widget_info(/valid, pm) then $
        widget_control, pm, /destroy

     nhfiles = nfe
     goto, hdrok

     nohdr: 
     smei_msg, /warn, "No img_hdr.txt found, this will make things " + $
               "slow"
     hfiles = ''

     hdrok:
     on_ioerror, null
     t5 = systime(1)
     print, "Read hdrs:", t5-t4, t5-now
  endif else hfiles = ''

  iflag = 0b
  desc = strarr(count > 1)
  filelist = strarr(2, count > 1)
  startlist = dblarr(count > 1)
  projections = strarr(count > 1)

  cfile_base = get_comp_name(file_basename(cfile), /return_base)
  cfile_dir = file_dirname(cfile, /mark)
  hlocs = match_lists(hfiles, cfile_base)

  if count eq 0 then begin
     if n_elements(laststr) eq 0 then return, ''
     desc = lastdesc
     filelist = transpose([[laststr.dir], [laststr.fname]])
     startlist = laststr.stime
     projections = laststr.project
     count = n_elements(desc)
     return, desc
  endif

  if isprog then begin
     pm = cw_progress(title = 'Checking files', $
                      value = 0., $
                      /column, $
                      xsize = 200, $
                      ysize = 20, $
                      max = count, $
                      group = callid, $
                      modal = ismodal)
     ispid = 1b
  endif else ispid = 0b

  appflag = n_elements(laststr) gt 0 
  nflags = bytarr(count)
  new_str = replicate({ff_data}, count)

  t6 = systime(1)
  for j = 0l, count-1 do begin
     sf = [cfile_dir[j], cfile_base[j]]

     if (j+1) mod 20 eq 0  or j eq count - 1 then begin  
        if ispid then widget_control, pm, set_value = $
                                      j+1 $
        else print, string(13b), j+1, count, format = $
                    "($,A,I0,' of ',I0)"
     endif

; Check if the file is already in the list
     
     if appflag then begin
        locs = where(laststr.fname eq cfile_base[j] and $
                     laststr.dir eq cfile_dir[j], nmat)
        if nmat gt 0 then continue
     endif
     nflags[j] = 1b

     if hlocs[j] eq -1 then begin ; This file wasn't listed in any
                                ; header list

        if not keyword_set(no_file) then begin
           iflag = 1b
           if n_elements(baddir) eq 0 then baddir = cfile_dir[j] $
           else baddir = [baddir, cfile_dir[j]]
        endif
        if ~file_test(cfile[j]) then begin
           cfile[j] = cfile[j]+'.gz'
           if ~file_test(cfile[j]) then begin
              printf, -2, "File "+cfile[j]+" not found"
              continue
           endif
        endif
        new_str[j] = ff_extract(cfile[j])

; CHUNK 1

     endif else if hcount ne 0 then begin
        new_str[j] = ff_tostr(hdesc[hlocs[j]], file = sf)
; CHUNK 2
     endif

  endfor
  t7 = systime(1)
  print, "Cross check:", t7-t6, t7-now

  if ispid then widget_control, pm, /destroy $ $
  else print

  if iflag then begin
     idx = uniq(baddir, sort(baddir))
     smei_msg, /warn, ["Not all files found are in " + $
                       "img_hdr.txt", $
                       "you should re-run mk_imghdrtxt on " + $
                       "this directory:", $
                       baddir[idx]]
  endif

; Add any
  locs = where(nflags, nnf)
  if nnf gt 0 then begin
     if appflag then begin
        laststr = [laststr, new_str[locs]]
        lastdesc = [lastdesc, ff_todesc(new_str[locs])]
     endif else begin
        laststr = new_str
        lastdesc = ff_todesc(laststr)
     endelse
     idx = sort(laststr.stime)
     laststr = laststr[idx]
     lastdesc = lastdesc[idx]
  endif
  t8 = systime(1)
  print, "Correct:", t8-t7, t8-now

  if chdir then lastpath = path
  lastepoch = now

  desc = lastdesc
  filelist = transpose([[laststr.dir], [laststr.fname]])
  startlist = laststr.stime
  projections = laststr.project
  count = n_elements(desc)

  return, desc

end