; $Id: getsccarchiveinfo.pro,v 1.21 2021/07/30 13:57:27 nathan Exp $
;
; Name:     getsccarchiveinfo
;
; Purpose:  Read in data volume stats and display summaries.
;
; Inputs: 	See keywords
;
; Keywords:	Following are required:
;			/VOLUME (diskspace) or /NUMBER (number of images)
;			/FITS (level 0.5) or /SCI (raw telemetry) or /IMG (img and cal directories) or /SEQ (seq directory)
;
;		Following are optional:
;			/AHEAD or /BEHIND (default is to do both side-by-side)
;   	    	    	SPACECRAFT= A or B
;			TELESCOPE = tel where tel = cor1 or cor2 or euvi or hi1 or hi2
;			One or more of /SSR1, /SSR2, /SSRT, /SPWX (downlink channel)
;			/NOPLOT	  Do not open plot window
;			/OVERPLOT	Plot on top of previous plot instead of redrawing
;			STA_DATE=date to start stats from
;   	    	    	END_DATE=last date for stats
;			PREDICT_DATE=date for which to print estimated total volume (from sta_date);
;				defaults to 30 days from last day
;			NDAYS=number of days from sta_date to get stats for (default=to last day)
;			LOGFILE=name of file to append summary info to 
;   	    	    	USE_DIR=name of directory to get data from other than $SCC_DATA/statistics
;   USE_DIR=	Use this directory to find data files
;			
; Outputs:	crnttotal	Float	Cumulative total of subject files (MB or number)
;		crntdate	UTC structure	Last date in crnttotal
;		bm			FLTARR	Fit of crnttotal for last 45 days. If y=mx+b, bm=[b,m], x=mjd
;   	    	VEC=	Vector of values per day (MB or number)
;   	    	UDATES= Dates corresponding to VEC
;
; Side effects:	Window with plots of cumulative total for last 90 days, 
;				line representing bm, and daily totals.
;
; Caveats:  Cal image files are included in the IMG result.
;			Currently need to exit IDL to update numbers.
;			Not (yet) designed to handle errors gracefully.
;
; Category:	archive, utility, pipeline, data management, statistics
;
; $Log: getsccarchiveinfo.pro,v $
; Revision 1.21  2021/07/30 13:57:27  nathan
; fixes
;
; Revision 1.20  2021/07/30 13:21:14  nathan
; filter out extra col labels
;
; Revision 1.19  2009/12/14 18:49:22  nathan
; euvi-a to uranus/u1
;
; Revision 1.18  2008/07/14 18:38:55  nathan
; added new input keyword SPACECRAFT and output keywords VEC and UDATES
;
; Revision 1.17  2008/02/04 19:29:47  nathan
; Allow multiple telescopes; fix ahead and behind
;
; Revision 1.16  2008/01/25 17:49:37  nathan
; IF j le 1 THEN runtota=runtot
;
; Revision 1.15  2007/12/27 18:57:20  nathan
; update output msg
;
; Revision 1.14  2007/12/13 16:33:26  nathan
; add date-range error msg
;
; Revision 1.13  2007/08/31 21:23:55  nathan
; fix /noplot keyword
;
; Revision 1.12  2007/08/28 21:05:52  nathan
; add kw /overplot, predict_date, logfile
;
; Revision 1.11  2007/07/19 16:16:17  nathan
; change plot maxmin
;
; Revision 1.10  2007/07/12 17:18:27  nathan
; fix udates; plot row major; enhance psym
;
; Revision 1.9  2007/07/12 16:50:21  nathan
; add dest info to plot; constant yrange
;
; Revision 1.8  2007/07/11 20:38:37  nathan
; Do both Ahead and Behind on same plot window
;
; Revision 1.7  2007/06/11 19:22:39  nathan
; fix ndays and p.color
;
; Revision 1.6  2007/06/01 17:14:46  nathan
; return to starting dir; use white bkg in plot; check for required keywords
;
; Revision 1.5  2007/05/31 18:56:13  nathan
; use SCC_DATA
;
; Revision 1.4  2007/05/21 20:49:57  nathan
; add STA_DATE= and NDAYS=
;
; Revision 1.3  2007/05/18 17:50:29  nathan
; add /NOPLOT
;
; Revision 1.2  2007/05/16 16:12:35  nathan
; Add outputs; change input dir; add documentation, improve plotting,
; reconfigure readvolfil
;
; Revision 1.1  2007/05/10 22:24:19  nathan
; for plotting archive info and projecting future totals
;

pro readvolfile, fname, nelem, array, times

common sccdata, opcolor, opsym, datadir
; one array for each file
; dates is a list of UTC dates since 2006/11/01 (inclusive)
cd,datadir,cur=cdir
spawn,'pwd'
get_utc,utcn
print,'Reading in ',fname
inp0=readlist(fname,nrows)
w=where(strmid(inp0,0,1) eq '2',nw)
inp=inp0[w]
nrows=nw
help,nrows
array=fltarr(nelem,nrows)
IF n_elements(times) LT nrows THEN times=replicate(utcn,nrows)
FOR i=0,nrows-1 DO BEGIN
    parts=strsplit(inp[i],',',/extract)
    times[i]=str2utc(parts[0])
    array[*,i]=float(parts[1:nelem])
ENDFOR
cd,cdir

end

pro getsccarchiveinfo, crnttotal, crntdate, bm, VEC=vec, UDATES=udatesout, STA_DATE=sta_date, NDAYS=ndays, $
			AHEAD=ahead, BEHIND=behind, TELESCOPE=telescope, PREDICT_DATE=predict_date, $
			VOLUME=volume, NUMBER=number, SSR1=ssr1, SSR2=ssr2, SSRT=ssrt, SPWX=spwx, $
    	    	    	FITS=fits, SCI=sci, IMG=img, SEQ=seq, NOPLOT=noplot, OVERPLOT=overplot, $
			LOGFILE=logfile, SPACECRAFT=spacecraft, END_DATE=end_date, USE_DIR=use_dir
			
common sccdata, opcolor, opsym, datadir

IF keyword_set(USE_DIR) THEN datadir=use_dir 
IF datatype(datadir) EQ 'UND' THEN datadir=concat_dir(getenv('SCC_DATA'),'statistics')

pause=2
isahead=0
isbehind=0
IF keyword_set(SPACECRAFT) THEN BEGIN
    sc=parse_stereo_name(spacecraft,['a','b'])
    Case sc of 
    	'a': isahead=1
	'b': isbehind=1
    endcase
ENDIF
IF keyword_set(AHEAD) THEN isahead=1  
IF keyword_set(BEHIND) THEN isbehind=1  

IF (~isahead and ~isbehind) or (isahead and isbehind) THEN BEGIN
	npltcol=2
	pltxsz=756
	isahead=1
	isbehind=0
ENDIF ELSE BEGIN
	npltcol=1
	pltxsz=512
ENDELSE
IF keyword_set(OVERPLOT) THEN BEGIN
	!p.multi=[npltcol*2-1,npltcol,2,0,1]
	opcolor=opcolor+1
	opsym=opsym+1
ENDIF ELSE BEGIN
	!p.multi=[0,npltcol,2,0,1]
	opcolor=1
	opsym=4
		; will add 1 before using
ENDELSE
IF ~keyword_set(VOLUME) and ~keyword_set(NUMBER) THEN BEGIN
	message,'Must use /Volume or /Number; returning.',/info
	return
ENDIF
IF ~keyword_set(FITS) and ~keyword_set(SCI) and ~keyword_set(IMG) and ~keyword_set(SEQ) THEN BEGIN
	message,'Must use /FITS, /SCI, /IMG, or /SEQ; returning.',/info
	return
ENDIF

;tel=['cor1','cor2','euvi','hi_1','hi_2']
;src=['lz','pb','rt']
;sc=['a','b']
;sdr=['img','seq','cal']

;++++ Loop for two s/c
FOR j=1,npltcol DO BEGIN
	if j GT 1 THEN BEGIN
		isahead=0
		isbehind=1
	endif
;      SSRT   SSR1   SSR2   SPWX

; suffixes for raw image files
IF isahead THEN BEGIN
    message,'Processing data for SECCHI-A...',/info
    wait,2
    aorb='A'
    ab='a'
    sfxa=['302', '402', '502', '702', $ ; COR1-A
	  '311', '411', '511', '711', $ ; COR2-A
	  '343', '443', '543', '743', $ ; EUVI-A
	  '325', '425', '525', '725', $ ; HI1-A
	  '334', '434', '534', '734']   ; HI2-A

    ; equivalent ATTS for FITS filenames

    atsa=['s3c1A','s4c1A','s5c1A','s7c1A', $ 
	  'n3c1A','n4c1A','n5c1A','n7c1A', $
	  's3c2A','s4c2A','s5c2A','s7c2A', $
	  'd3c2A','d4c2A','d5c2A','d7c2A', $
	  '3euA','4euA','5euA','7euA', $
	  '3h1A','4h1A','5h1A','7h1A', $
	  '3h2A','4h2A','5h2A','7h2A']

ENDIF ELSE BEGIN
    message,'Processing data for SECCHI-B...',/info
    wait,2
    aorb='B'
    ab='b'
    sfxa=['307', '407', '507', '707', $ ; COR1-B
	  '316', '416', '516', '716', $ ; COR2-B
	  '348', '448', '548', '748', $ ; EUVI-B
	  '320', '420', '520', '720', $ ; HI1-B
	  '339', '439', '539', '739']   ; HI2-B

    atsa=['s3c1B','s4c1B','s5c1B','s7c1B', $ 
	  'n3c1B','n4c1B','n5c1B','n7c1B', $
	  's3c2B','s4c2B','s5c2B','s7c2B', $
	  'd3c2B','d4c2B','d5c2B','d7c2B', $
	  '3euB','4euB','5euB','7euB', $
	  '3h1B','4h1B','5h1B','7h1B', $
	  '3h2B','4h2B','5h2B','7h2B'] 
ENDELSE

; Read appropriate file  for one of above blocks
; number of files or total size of files in MB
; Mission day zero is 2006-10-25 = 2006-298

common sccarchive, volfa, volsa, nffa, nfsa, volfb, volsb, nffb, nfsb, udatesa, udatesb
; one array for each file
; dates is a list of UTC dates since 2006/11/01 (inclusive)

IF keyword_set(IMG) or keyword_set(SEQ) THEN fits=1
unt=' '
IF keyword_set(FITS) and keyword_set(VOLUME) and isahead THEN BEGIN
    IF datatype(volfa) EQ 'UND' THEN readvolfile,'sccAfitsizes.txt',28,volfa,udatesa
    numbers=volfa
    subject="-A FITS Volume"
    yax='MB'
    unt='MB'
ENDIF
IF keyword_set(FITS) and keyword_set(VOLUME) and isbehind THEN BEGIN
    IF datatype(volfb) EQ 'UND' THEN readvolfile,'sccBfitsizes.txt',28,volfb,udatesb
    numbers=volfb
    subject="-B FITS Volume"
    yax='MB'
    unt='MB'
ENDIF
IF keyword_set(SCI) and keyword_set(VOLUME) and isahead   THEN BEGIN
    IF datatype(volsa) EQ 'UND' THEN readvolfile,'sccAimgsizes.txt',20,volsa,udatesa
    numbers=volsa
    subject="-A Downlinked Image Volume"
    yax='MB'
    unt='MB'
ENDIF
IF keyword_set(SCI) and keyword_set(VOLUME) and isbehind  THEN BEGIN
    IF datatype(volsb) EQ 'UND' THEN readvolfile,'sccBimgsizes.txt',20,volsb,udatesb
    numbers=volsb
    subject="-B Downlinked Image Volume"
    yax='MB'
    unt='MB'
ENDIF

IF keyword_set(FITS) and keyword_set(NUMBER) and isahead  THEN BEGIN
    IF datatype(nffa) EQ 'UND' THEN readvolfile,'sccAfitnumbers.txt',28,nffa,udatesa
    numbers=nffa
    subject="-A FITS N-images"
    yax='N'
ENDIF
IF keyword_set(FITS) and keyword_set(NUMBER) and isbehind THEN BEGIN
    IF datatype(nffb) EQ 'UND' THEN readvolfile,'sccBfitnumbers.txt',28,nffb,udatesb
    numbers=nffb
    subject="-B FITS N-images"
    yax='N'
ENDIF
IF keyword_set(SCI) and keyword_set(NUMBER) and isahead   THEN BEGIN
    IF datatype(nfsa) EQ 'UND' THEN readvolfile,'sccAimgnumbers.txt',20,nfsa,udatesa
    numbers=nfsa
    subject="-A Downlinked N-Files"
    yax='N'
ENDIF
IF keyword_set(SCI) and keyword_set(NUMBER) and isbehind  THEN BEGIN
    IF datatype(nfsb) EQ 'UND' THEN readvolfile,'sccBimgnumbers.txt',20,nfsb,udatesb
    numbers=nfsb
    subject="-B Downlinked N-Files"
    yax='N'
ENDIF
; now we have arrays with numbers and need to combine numbers to get desired info

tssubs=indgen(20)
tfsubs=indgen(28)
telarr='SECCHI'

IF keyword_set(TELESCOPE) THEN BEGIN
    telarr=strlowcase(telescope)
    IF (where(telarr EQ 'cor1'))[0] GE 0 THEN BEGIN
		tssubs=[tssubs,0,1,2,3]
		tfsubs=[tfsubs,0,1,2,3,4,5,6,7]
		END
    IF (where(telarr EQ 'cor2'))[0] GE 0 THEN BEGIN
    		tssubs=[tssubs,4,5,6,7]
		tfsubs=[tfsubs,8,9,10,11,12,13,14,15]
		END
    IF (where(telarr EQ 'euvi'))[0] GE 0 THEN BEGIN
    		tssubs=[tssubs,8,9,10,11]
		tfsubs=[tfsubs,16,17,18,19]
		END
    wtel=where(telarr EQ ['hi1','hi_1'],nwt)
    IF nwt GT 0 THEN BEGIN
    		tssubs=[tssubs,12,13,14,15]
		tfsubs=[tfsubs,20,21,22,23]
		telarr[wtel]='hi_1'
		END
    wtel=where(telarr EQ ['hi2','hi_2'],nwt)
    IF nwt GT 0 THEN BEGIN
    		tssubs=[tssubs,16,17,18,19]
		tfsubs=[tfsubs,24,25,26,27]
		telarr[wtel]='hi_2'
		END
ENDIF  
ntssubs=n_elements(tssubs)
ntfsubs=n_elements(tfsubs)
IF ntfsubs GT 28 THEN BEGIN
    tfsubs=tfsubs[28:ntfsubs-1]
    tssubs=tssubs[20:ntssubs-1]
ENDIF
; make sure tel is a scalar for output messages
tel=strupcase(arr2str(telarr))

dsubs=indgen(28)
dest='ALL'
c=''
IF keyword_set(SSRT) THEN BEGIN
	dsubs=[dsubs,0,4,8,12,16,20,24]
	dest='SSRT'
ENDIF
IF keyword_set(SSR1) THEN BEGIN
	dsubs=[dsubs,1,5,9,13,17,21,25]
	if dest NE 'ALL' then dest=dest+'+SSR1' else dest='SSR1'
ENDIF
IF keyword_set(SSR2) THEN BEGIN
	dsubs=[dsubs,2,6,10,14,18,22,26]
	if dest NE 'ALL' then dest=dest+'+SSR2' else dest='SSR2'
ENDIF
IF keyword_set(SPWX) THEN BEGIN
	dsubs=[dsubs,3,7,11,15,19,23,27]
	if dest NE 'ALL' then dest=dest+'+SPWX' else dest='SPWX'
ENDIF

ndsubs=n_elements(dsubs)
IF ndsubs GT 28 THEN dsubs=dsubs[28:ndsubs-1]

isubs=indgen(28)
IF keyword_set(IMG) THEN isubs=[4,5,6,7,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27]
IF keyword_set(SEQ) THEN isubs=[0,1,2,3,8,9,10,11]

IF keyword_set(FITS) THEN tsubs=tfsubs ELSE tsubs=tssubs
subs=intersection(intersection(tsubs,dsubs),isubs)
;subs is column subscripts to be added together

IF (isahead) THEN udates=udatesa ELSE udates=udatesb

sz=size(numbers)
utc0=udates[0]
firstutc=utc0
utc1=udates[sz[2]-1]
lastutc=utc1
nud=n_elements(udates)
IF keyword_set(STA_DATE) THEN utc0=anytim2utc(sta_date)
IF utc0.mjd GT lastutc.mjd or utc0.mjd LT firstutc.mjd THEN BEGIN
    message,utc2str(utc0,/date)+' is out of range. Please try again.',/info
    wait,2
    return
ENDIF
IF keyword_set(END_DATE) THEN utc1=anytim2utc(end_date)
IF keyword_set(NDAYS) THEN utc1.mjd=utc0.mjd+ndays-1
ndays=utc1.mjd-utc0.mjd+1

w=where(udates.mjd EQ utc0.mjd)
sday1=w[0]
ndays=(ndays)<(sz[2]-sday1)
yymmdd0=utc2str(utc0,/date)

vec=fltarr(ndays)
runtot=fltarr(ndays)
udatesout=replicate(utc0,ndays)

FOR i=0,ndays-1 DO BEGIN
    vec[i]=total(numbers[subs,i+sday1])
    ; now have a single vector of per-day amounts
    udatesout[i]=udates[i+sday1]
    ; compute cumulative total
    IF i EQ 0 THEN runtot[i]=vec[i] ELSE runtot[i]=runtot[i-1]+vec[i]
ENDFOR

utc0p=utc0
IF ndays GT 90 THEN utc0p.mjd=utc1.mjd-90
ti1='Cumulative Total '+tel+subject
;--find predict date, and predicted trend using lesser of last 45 days or ndays
IF keyword_set(PREDICT_DATE) THEN putc=anytim2utc(predict_date) ELSE BEGIN
	putc=udates[sday1+ndays-1]
	putc.mjd=putc.mjd+30
ENDELSE 
pndays=ndays<45
psday1=sday1+ndays-pndays
bm=linfit(udates[psday1:sday1+ndays-1].mjd, runtot[(ndays-pndays)>0:ndays-1])
predtot=bm[1]*putc.mjd + bm[0]
pred0  =bm[1]*udates[0].mjd + bm[0]
dailyavg=bm[1]
dailystdev=stdev(vec)
help,predtot,dailyavg,dailystdev,vec

IF ~keyword_set(NOPLOT) THEN BEGIN
	;--Display vector and running total of vector

	IF j LT 2 and ~keyword_set(OVERPLOT) THEN BEGIN
		!y.style=3
		!x.style=3
		device,decomposed=0
		tek_color
		!p.background=255
		!p.color=0
		;IF keyword_set(PREDICTDATE) THEN BEGIN
		window,xsiz=pltxsz,ysize=756
		tmax=1.1*max(runtot)
		dmin=min(vec,max=dmax)
		dminmax=[dmin,dmax]
	ENDIF 
	IF npltcol EQ 1 THEN dminmax=0
	;--plot last 90 days + predict date
	;print,!p.multi
	IF keyword_set(OVERPLOT) THEN BEGIN
		outplot,udates[sday1:sday1+ndays-1],runtot,psym=-opsym,color=opcolor
		outplot,[udates[0],putc],[pred0,predtot],color=opcolor
		;stop
		xyouts,105,695-15*opcolor,tel+' '+dest,/device,size=1.5,color=opcolor
		!p.multi[0]=!p.multi[0]-1
		outplot,udates[sday1:sday1+ndays-1],vec,color=opcolor,psym=-opsym
	ENDIF ELSE BEGIN
		utplot,udates[sday1:sday1+ndays-1],runtot,title=ti1,ytitle=yax,yrang=[0,tmax],psym=-4,timerange=[utc0p,utc1]
		;print,!p.multi
		outplot,[udates[0],putc],[pred0,predtot]
		;print,!p.multi
		;stop
		xyouts,105,695,'Starting '+yymmdd0,/device,size=1.5
		xyouts,105,680,dest,/device,size=1.5
		;ENDIF
		ti2='Daily Total '+tel+subject
		utplot,udates[sday1:sday1+ndays-1],vec,title=ti2,ytitle=yax,yrang=dminmax,timerange=[utc0p,utc1],thick=2,psym=-6
	
	ENDELSE
	;print,!p.multi
ENDIF
 
gtz=where(vec GT 0)
mgtz=max(gtz)
crnttotal=total(vec)
crntdate =utc1
;help,crnttotal

print,''
msg='Cum Tot '+tel+subject+' '+dest+' '+trim(crnttotal)+' '+unt+' betw '+utc2str(utc0,/date)+' & '+utc2str(utc1,/date)+'; '+utc2str(putc,/date)+' will be '+trim(predtot)
print,msg
wait,1
IF keyword_set(LOGFILE) THEN BEGIN
	openw,2,logfile,/append
	printf,2,msg
	close,2
ENDIF
IF j le 1 THEN runtota=runtot
ENDFOR ;++++ END Loop for two s/c

!p.multi=0

end