pro read_monexp_data,tel,dtea,dteb,nosort=nosort,QL=ql, DIR=dir
;+
; NAME:
;	READ_MONEXP_DATA
;
; PURPOSE:
;	This procedure reads in the MONEXP data for the given telescope and date.
;
; CATEGORY:
;	LASCO data analysis
;
; CALLING SEQUENCE:
;	READ_MONEXP_DATA,Tel,Dtea,Dteb
;
; INPUTS:
;	Tel:	String giving the telescope: C1, C2, C3
;	Dtea:	First date to be read in, either YYMMDD, or MJD or UTC
;	Dteb:	Last date to be read in, either YYMMDD, or MJD or UTC
;
; KEYWORD PARAMETERS:
;	NOSORT:	If set then the MONEXP data are not sorted by time.  The default is
;		to sort the data
;   	/QL 	Only use QL data
;   	DIR=	Use this location instead of $MON_EXP
;
; OUTPUTS:
;	Stores data into the common array
;
; COMMON BLOCKS:
;	MONEXP_DATA
;
; SIDE EFFECTS:
;	Reads in the data files of the form, $MON_EXP/cx_monexp_yymmdd.dat
;		where cx is c1, c2, c3, or c4
;		and   yymmdd is the year, month, day
;
; RESTRICTIONS:
;
; PROCEDURE:
;
; MODIFICATION HISTORY:
; 	Written by:	RAHoward, NRL, June 2 1997
;	Modified:	11/11/97  RAH	Check to see if MON_EXP file exists
;	         	06/06/97  RAH	Added nosort option, check for nrec=0
;	         	12/18/98  RAH	Added check for compressed files
;	         	10/20/05  RAH	Added swap if little endian to OPEN
;   	    	    	 1/10/11, nbr 	Eliminate Quicklook duplicates
;   	    	    	 1/25/11, nbr 	Add /QL option
;   	    	    	 3/ 8/13, nbr 	Accomodate change in header defn in Dec. 2012
;   	    	    	    	    	when types changed from DOUBLE to FLOAT
;
;	@(#)read_monexp_data.pro	1.12 03/08/13 LASCO IDL LIBRARY
;-

;
common monexp_data,date,time,fname,expt,exp0,expcmd,exp1,exp2,exp3, $
       filter,polar,nlebsum,nccdsum, $
       avg_whole,sig_whole,med_whole,min_whole,max_whole,nz_whole,mod_whole, $
       avgval,sigma,medval,minval,maxval,nzero,mode
;
;  convert dates to mjd
;  increment date from dtea to dteb
;  survey the files to determine the size of the arrays
;
mjda = YYMMDD2UTC(dtea)
mjdb = YYMMDD2UTC(dteb)
GET_LUN,lu
t = strlowcase(tel)
nrec = 0
IF keyword_set(DIR) THEN monexpdir=dir ELSE monexpdir=getenv('MON_EXP')
FOR i=mjda.mjd,mjdb.mjd DO BEGIN
    mjd = mjda
    mjd.mjd = i
    dd = UTC2YYMMDD(mjd)
    fn = monexpdir+'/'+t+'_monexp_'+dd+'.dat*'
    ff = file_search(fn)
    IF (ff(0) NE '') THEN BEGIN
       IF (STRPOS(ff(0),'.Z') NE -1)  THEN SPAWN,'uncompress '+fn
       OPENR,lu,fn,/swap_if_little_endian
       fs = FSTAT(lu)
    	recsize = 28810
        IF (fs.size MOD recsize) GT 0 THEN recsize=28782
        ; Values are type FLOAT not DOUBLE
            
       nrec = nrec+fs.size/recsize
       CLOSE,lu
    ENDIF
ENDFOR
help,nrec
IF (NREC EQ 0)    THEN BEGIN
   time    = -1
   date    = -1
   fname   = ''
   FREE_LUN,lu
   RETURN
ENDIF
lz  	=bytarr(nrec)
date    = LONARR(nrec)
time    = FLTARR(nrec)
fname   = STRARR(nrec)
expt    = FLTARR(nrec)
exp0    = FLTARR(nrec)
expcmd  = FLTARR(nrec)
exp1    = FLTARR(nrec)
exp2    = FLTARR(nrec)
exp3    = FLTARR(nrec)
filter  = STRARR(nrec)
polar   = STRARR(nrec)
nlebsum = INTARR(nrec)
nccdsum = INTARR(nrec)
avg_whole  = FLTARR(nrec)
sig_whole  = FLTARR(nrec)
med_whole  = LONARR(nrec)
min_whole  = LONARR(nrec)
max_whole  = LONARR(nrec)
nz_whole   = LONARR(nrec)
mod_whole  = LONARR(nrec)
avgval  = FLTARR(1024,nrec)
sigma   = FLTARR(1024,nrec)
medval  = LONARR(1024,nrec)
minval  = LONARR(1024,nrec)
maxval  = LONARR(1024,nrec)
nzero   = LONARR(1024,nrec)
mode    = LONARR(1024,nrec)
nr = 0
dt=0L
nm=STRPAD('',12)
fil=STRPAD('',15)
pol=STRPAD('',15)
nleb=0L
nccd=0L
avgar=fltarr(1025)
sd=fltarr(1025)
med=lonarr(1025)
mn=lonarr(1025)
mx=lonarr(1025)
nz=lonarr(1025)
md=lonarr(1025)
FOR i=mjda.mjd,mjdb.mjd DO BEGIN
    mjd = mjda
    mjd.mjd = i
    dd = UTC2YYMMDD(mjd)
    fn = GETENV('MON_EXP')+'/'+t+'_monexp_'+dd+'.dat'
    ff = file_search(fn)
    IF (ff(0) NE '') THEN BEGIN
       OPENR,lu,fn ,/swap_if_little_endian
       fs = FSTAT(lu)
    	recsize = 28810
        IF (fs.size MOD recsize) GT 0 THEN BEGIN
        ; Values are type FLOAT not DOUBLE
            recsize=28782
	    ti=0.
	    et=0.
	    e0=0.
	    ec=0.
	    e1=0.
	    e2=0.
	    e3=0.
    	ENDIF ELSE BEGIN
	    ti=0.D
	    et=0.D
	    e0=0.D
	    ec=0.D
	    e1=0.D
	    e2=0.D
	    e3=0.D
    	ENDELSE
       nrec = fs.size/recsize
       FOR j=0,nrec-1 DO BEGIN
           READU,lu,dt,ti,nm,et,e0,ec,e1,e2,e3,fil,pol, $
                    nleb,nccd,avgar,sd,med,mn,mx,nz,md
           date(nr)    = dt
           time(nr)    = ti
           fname(nr)   = nm
	   IF strmid(nm,1,1) EQ '2' THEN lz[nr]=1 
           expt(nr)    = et
           exp0(nr)    = e0
           expcmd(nr)  = ec
           exp1(nr)    = e1
           exp2(nr)    = e2
           exp3(nr)    = e3
           filter(nr)  = fil
           polar(nr)   = pol
           nlebsum(nr) = nleb
           nccdsum(nr) = nccd
           avg_whole(nr)  = avgar(0)
           sig_whole(nr)  = sd(0)
           med_whole(nr)  = med(0)
           min_whole(nr)  = mn(0)
           max_whole(nr)  = mx(0)
           nz_whole(nr)   = nz(0)
           mod_whole(nr)  = md(0)
           avgval(0,nr)  = avgar(1:*)
           sigma(0,nr)   = sd(1:*)
           medval(0,nr)  = med(1:*)
           minval(0,nr)  = mn(1:*)
           maxval(0,nr)  = mx(1:*)
           nzero(0,nr)   = nz(1:*)
           mode(0,nr)    = md(1:*)
           nr = nr+1
       ENDFOR
       CLOSE,lu
    ENDIF
ENDFOR
avgval  = REFORM(avgval,32,32,nr)
sigma   = REFORM(sigma,32,32,nr)
medval  = REFORM(medval,32,32,nr)
minval  = REFORM(minval,32,32,nr)
maxval  = REFORM(maxval,32,32,nr)
nzero   = REFORM(nzero,32,32,nr)
mode    = REFORM(mode,32,32,nr)


FREE_LUN,lu
IF KEYWORD_SET (nosort) THEN RETURN
tt = date+time/86400.d00
ts = SORT(tt)
days=tt[ts]

;--Omit quicklook duplicates or omit LZ
nout=nr
keep=bytarr(nout)
keep[*]=1
lz = lz[ts]
IF keyword_set(QL) THEN BEGIN
    wlz=where(lz,complement=wql,ncomplem=nql)
    keep[*]=0
    uq=uniq(days[wql])
    keep[wql[uq]]=1
    ; omit non-uniq days
    help,lz,wlz,wql,uq
ENDIF ELSE BEGIN
    uq=uniq(days)
    ;--for each non-unique row, pick out an LZ one, if any
    indx=bytarr(nout)
    indx[*]=1
    indx[uq]=0
    dups=where(indx,ndups)
    help,dups
    IF ndups GE 1 THEN BEGIN
	FOR i=0,ndups-1 DO BEGIN
	    wd=where(days EQ days[dups[i]],nwd)
	    IF nwd GT 0 THEN stop
	    print,trim(nwd) ,' duplicates for days=', days[dups[i]]
	    wlz=where(lz[wd],nwlz, COMPLEMENT=wql, NCOMPL=nwql)
	    ; keep the last LZ value or the last QL value if no LZ; omit the rest
	    IF nwlz GE 1 THEN ikeep=wlz[nwlz-1] ELSE ikeep=wql[nwql-1]
	    omit=bytarr(nwd)
	    omit[*]=1
	    omit[ikeep]=0

	    IF i EQ 0 THEN qldups=wd[where(omit)] ELSE qldups=[qldups,wd[where(omit)]]
	ENDFOR
	keep[qldups]=0
    	print,'Omitting ',fname[ts[qldups]]
    ENDIF
    help,wlz,wql
ENDELSE
wait,2
;stop
;--now get final arrays

date    = date(ts[where(keep)])
time    = time(ts[where(keep)])
fname   = fname(ts[where(keep)])
expt    = expt(ts[where(keep)])
exp0    = exp0(ts[where(keep)])
expcmd  = expcmd(ts[where(keep)])
exp1    = exp1(ts[where(keep)])
exp2    = exp2(ts[where(keep)])
exp3    = exp3(ts[where(keep)])
filter  = filter(ts[where(keep)])
polar   = polar(ts[where(keep)])
nlebsum = nlebsum(ts[where(keep)])
nccdsum = nccdsum(ts[where(keep)])
avg_whole  = avg_whole(ts[where(keep)])
sig_whole  = sig_whole(ts[where(keep)])
med_whole  = med_whole(ts[where(keep)])
min_whole  = min_whole(ts[where(keep)])
max_whole  = max_whole(ts[where(keep)])
nz_whole   = nz_whole(ts[where(keep)])
mod_whole  = mod_whole(ts[where(keep)])
avgval  = avgval(*,*,ts[where(keep)])
sigma   = sigma(*,*,ts[where(keep)])
medval  = medval(*,*,ts[where(keep)])
minval  = minval(*,*,ts[where(keep)])
maxval  = maxval(*,*,ts[where(keep)])
nzero   = nzero(*,*,ts[where(keep)])
mode    = mode(*,*,ts[where(keep)])
RETURN
END