;+
; NAME:
;	skyd_cat
; PURPOSE:
;	Creates the initial orbit catalogues needed by the
;	SMEI skyd daemon.
; CALLING SEQUENCE:
	PRO skyd_cat, range, $
		camera		= camera	, $
		mode		= mode		, $
		source		= source	, $
		dig 		= dig		, $
		destination = destination,$
		to_smeidb	= to_smeidb	, $
		full_count	= full_count, $
		keepskip	= keepskip	, $
		smoothdark	= smoothdark, $
		reverse_order= reverse_order
; INPUTS:
;	range			array[2]; type: time structure
;						time range for which to produce a catalogue.
;						If omitted the entire mission time is used.
; OPTIONAL INPUTS:
;	camera=camera		array or scalar; type: integer; default: [1,2,3]
;							SMEI cameras to be processed
;	mode=mode			array or scaler; type: integer; default: [0,1,2]
;							modes to be processed
;	source=source		scalar or array; type: string; default: SMEISKY?
;							list of directories with skymaps.
;							The default SMEISKY? is translated to all
;							directories assigned to env vars $SMEISKYn
;	/dig				If /dig is set then the skymaps should be located
;							in subdirectories c1,c2,c3.
;							IF source=SMEISKY? then always /dig is assumed
;	destination=destination
;						scalar; type: string; default: $TUB
;							destination directory for the skyd catalogues
;	/to_smeidb 			forces destination=$SSWDB_SMEI/cat/list
;							(see PROCEDURE)
;	/full_count			by default the sky catalogues are based on the frame
;						counts for frames satisfying the default conditions
;						used by smeidb_sky for frame selection (not a "just-bad"
;						frame, good quaternion, shutter open and base OK;
;						this is the second set of 4 columns in
;						the smei_frm_orb_m<mode>.txt files).
;						If full_count is set then the unrestricted counts for
;						all frames per orbit (the first set of 4 columns in the
;						smei_frm_orb_m<mode>.txt files) is used.
;	/keepskip			by default, orbits marked 'skip' are not written
;						into the catalogue files. Set /keepskip to override.
;	/smoothdark			create catalogues for the skymaps based on smoothed
;						dark currents (cam 3 only).
;						(By default, skyd_cat looks in the c1,c2,c3 directories
;						of $SMEISKY0/sky. Setting this keyword forces it to
;						look in the c3s directory).
;	/reverse_order		by default, catalogues are written in reverse
;						chronological order (i.e. the most recent data get
;						processed first). To reverse this, and process data
;						chronologically set this keyword.
; INCLUDE:
	@compile_opt.pro
; CALLS:
;	InitVar, TimeUnit, CheckDir, TimeUnit, boost, txt_read
;	TimeSet, TimeGet, flt_string, timeposn, FindAllFiles
;	GetFileSpec
; PROCEDURE:
;	The skyd catalogues are derived from the files
;	smei_frm_orb_m0.txt, smei_frm_orb_m1.txt and smei_frm_orb_m2.txt
;	in directory $SSWDB_SMEI/cat/list. These files list the numbers
;	of frames for each camera in each of three modes. They are
;	created and maintained by the IDL procedure
;	href=smei_orbit_stats= (through a daily cronjob).
;
;	The skyd_catalogues will have names skyd_c<cam>m<mode>.txt
;	where <cam> is the camera id (1,2,3) and <mode> the mode id
;	<0,1,2>. The files are written to directory 'destination'.
;	If /to_smeidb is set the destination directory is the same
;	as for the smei_frm_orb_m*.txt files ($SSWDB_SMEI/cat/list).
;	These catalogues are accessed by the Python script
;	href=skyd_orbit= (which in turn is used by the SMEI
;	skyd daemon href=skyd_wait=).
; MODIFICATION HISTORY:
;	DEC-2005, Paul Hick (UCSD/CASS)
;	MAR-2007, Paul Hick (UCSD/CASS)
;		Added keyword /full_count
;	APR-2009, Paul Hick (UCSD/CASS; pphick@ucsd.edu)
;		Added keyword /reverse_order
;-

; Location of files with frame counts per orbit in $SSWDB_SMEI

listdir = smei_filepath(mode='list')

InitVar, smoothdark   , /key
InitVar, reverse_order, /key

uday = TimeUnit(/day)
umin = TimeUnit(/min)

IF IsType(range, /defined) THEN BEGIN
	IF NOT IsTime(range) THEN range = TimeSet(range)
	IF n_elements(range) EQ 1 THEN BEGIN
		InitVar, ndoy, -14
		range = [TimeOp(/add,range,TimeSet(/diff,ndoy,uday)),range]
	ENDIF
	range = [TimeGet(range[0],/bot,umin), TimeGet(range[1],/eot,umin)]
	message, /info, 'catalogues for time range ['+strjoin(TimeGet(/_ydoy,range,upto=umin),' - ')+']'
ENDIF

CASE smoothdark OF
0: BEGIN
	InitVar, camera , [1,2,3]
	InitVar, mode	, [0,1,2]
END
1: BEGIN
	camera	= 3
	mode	= 1
END
ENDCASE

InitVar, source , 'SMEISKY?'
InitVar, full_count , /key
InitVar, keepskip	, /key

CASE source EQ 'SMEISKY?' OF
0: InitVar, dig, /key
1: dig = 1
ENDCASE

InitVar, to_smeidb, /key

CASE to_smeidb OF
0: InitVar, destination, getenv('TUB')
1: destination = listdir
ENDCASE

IF NOT CheckDir(destination,/silent) THEN BEGIN
	message, /info, 'destination does not exist, '+hide_env(destination)
	RETURN
ENDIF

IF source EQ 'SMEISKY?' THEN BEGIN
	destroyvar, source

	ndir = 0
	tmp  = getenv('SMEISKY'+strcompress(ndir,/rem))
	WHILE tmp NE '' DO BEGIN
		boost, source, filepath(root=tmp,'sky')
		ndir = ndir+1
		tmp = getenv('SMEISKY'+strcompress(ndir,/rem))
	ENDWHILE
ENDIF

nsource = n_elements(source)

IF nsource GT 0 THEN BEGIN

	tmp = source
	destroyvar, source

	FOR i=0,nsource-1 DO BEGIN
		CASE CheckDir(tmp[i],/silent) OF
		0: message, /info, 'source directory does not exist, '+hide_env(tmp[i])
		1: BEGIN
			message, /info, 'include source directory '+hide_env(tmp[i])
			boost, source, tmp[i]
		END
		ENDCASE
	ENDFOR

	nsource = n_elements(source)

ENDIF

IF nsource EQ 0 THEN BEGIN
	message, /info, 'no source directory specified'
	RETURN
ENDIF

usec = TimeUnit(/sec)

ncam = n_elements(camera)

camcount		= lonarr(ncam)				; Number or skymaps for each camera
camskyfile		= ptrarr(ncam,/allocate)	; File names of skymaps
camskymode 		= ptrarr(ncam,/allocate)	; Mode of skymaps
camskytime		= ptrarr(ncam,/allocate)	; Creation time of skymaps
camskyversion	= ptrarr(ncam,/allocate)	; Version number of skymap

FOR icam=0,ncam-1 DO BEGIN

	cam_id = camera[icam]

	; If /dig is set attach subdirectory c<cam_id> to each entry of source

	camsource = source
	IF dig THEN $
		FOR i=0,nsource-1 DO	$
			camsource[i] = filepath(root=source[i]	, $
				'c'+strcompress(cam_id,/rem)+(['','s'])[smoothdark])

	prefix = 'c'+strcompress(cam_id,/rem)+'sky_'
	tmp = FindAllFiles(prefix+'*.fts.gz',paths=camsource+['','m0'], /nodir, count=count)

	; Locate all the skymaps
	; The directories may contain lots of files so we pick up all files,
	; then sort things out.

	;tmp = FindAllFiles(paths=camsource+['','m0'], /nodir, count=count)

	; Sort out the skymaps for the proper camera

	;i = where( strpos(GetFileSpec(tmp,part='name'),prefix) EQ 0, count )
	;CASE count EQ 0 OF
	;0: tmp = tmp[i]
	;1: tmp = ''
	;ENDCASE

	IF IsTime(range) THEN BEGIN

		time = smei_filename(tmp)

		i = where(	TimeOp(/subtract,time,range[0],umin) GE 0 AND	$
					TimeOp(/subtract,time,range[1],umin) LE 0	, count )

		CASE count NE 0 OF
		0: tmp = ''
		1: tmp = tmp[i]
		ENDCASE

	ENDIF

	*camskyfile[icam] = tmp
	camcount   [icam] = count

	IF count GT 0 THEN BEGIN			; Skymaps found; extract mode and creation time

		cammode    = intarr(count)
		camtime    = strarr(count)
		camversion = fltarr(count)

		tmp = *camskyfile[icam]

		FOR i=0L,count-1 DO BEGIN
			h = headfits(tmp[i])		; Get Fits header
			camtime   [i] = fxpar(h,'DATE' )
			IF !err LT 0 THEN camtime[i] = fxpar(h,'CREATED') ELSE camtime[i] = TimeGet(/_ydoy,upto=usec,TimeSet(camtime[i],format='YYYY-MN-DD hh:mm:ss'))
			cammode   [i] = fxpar(h,'MODE'	  )	; Mode may not be present in old skymaps
			IF !err LT 0 THEN cammode[i] = ([-1,2,2,1])[cam_id]
			camversion[i] = fxpar(h,'SMEI_SKY')
			IF !err LT 0 THEN camversion[i] = fxpar(h,'SMEI_HTM')
		ENDFOR

		*camskymode   [icam] = cammode
		*camskytime   [icam] = camtime
		*camskyversion[icam] = camversion

	ENDIF

ENDFOR


;=======================================================
; The time format must match the format used by smei_orbit_stats.pro
; (which creates the files smei_frm_orb<mode>.txt files).

format  = 'YEAR/MN/DD (DOY) hh:mm:ss.mss'
nformat = strlen(format)

FOR imode=0,n_elements(mode)-1 DO BEGIN

	mode_id = mode[imode]

	; First read list of orbits with start times and number of frames
	; per orbit from smei_frames_per_orbit.txt file
	; These files are created and maintained by smei_orbits.pro

	filename = filepath(root=listdir,'smei_frm_orb_m'+strcompress(mode_id,/rem)+'.txt')

	IF NOT txt_read(filename, orbits) THEN BEGIN
		message, /info, 'read error, '+hide_env(filename)
		break
	ENDIF

	; The header ends with the second line of underscores

	tmp = (where(strpos(orbits,'==========') EQ 0))[1]+1
	orbits = orbits[tmp:*] 					; Drop the header records
	norbit = n_elements(orbits) 			; Number of orbits

	; Use the first record to locate several entries
	; The frame counts are the last 3 numbers on each record

	tmp = flt_string(orbits[0],fmt=fmt)
	tmp = flt_string(fmt)					; fmt should have 8I10 at end
	lencount = round(tmp[n_elements(tmp)-1]); Should be 10

	; Extract frame counts from 'orbits' for all three cameras
											; Position where frame counts begin
	tmp = strlen(orbits[0])-4*(1+full_count)*lencount
	counts = strmid(orbits,tmp,3*lencount)	; Extract frame counts for three cams
	tmp = strlen(orbits[0])-4*2*lencount
	orbits = strmid(orbits,0,tmp)			; Drop frame counts

	; The orbit start times are now the last nformat chars
	; Extract the start times and round orbit to the nearest second
	; (this is what is used in file name construction).

	tmp = strlen(orbits[0])-nformat 		; Position where start times begin
	start_time = TimeSet(strmid(orbits, tmp, nformat), format=format)
	orbits = long(strmid(orbits,0,tmp))		; Drop start times; only orbit numbers remain

	IF IsTime(range) THEN BEGIN

		i = where(	TimeOp(/subtract,start_time,range[0],umin) GE 0 AND	$
					TimeOp(/subtract,start_time,range[1],umin) LE 0, norbit )
		IF norbit EQ 0 THEN BEGIN
			message, /info, 'no orbits in ['+strjoin(TimeGet(/_ydoy,range,upto=umin),' - ')+'] for mode '+strcompress(imode,/rem)
			continue
		ENDIF

		orbits		= orbits[i]
		start_time	= start_time[i]		
		counts		= counts[i]

	ENDIF

	start_time = TimeGet(start_time,/_ydoy,upto=usec,/roundt)

	;=============
	; At this point:
	; orbits is list of orbit numbers (long integer array and reverse chronological)
	;			This array is used as starting point for the skyd catalogues.
	; start_time is list of orbit start times (time structure array)
	; counts  is frames per orbit for all cameras (string array).
	;=============

	FOR icam=0,ncam-1 DO BEGIN

		cam_id = camera[icam]

		; Now build an orbit catalogue

		; The format of the skyd catalogue is
		;	orbit_nr orbit_nr+1 start_time status'

		cat = string(orbits,format='(I6)')+' '+string(orbits+1,format='(I6)')

		; Extract frame counts for camera

		count = long(strmid(counts,(cam_id-1)*lencount,lencount))

		; Mark orbits that have frames with status 'make'

		tmp = where(count NE 0)
		IF tmp[0] NE -1 THEN cat[tmp] += ' '+start_time[tmp]+' make 0'

		; Mark orbits that have no frames with status 'skip'

		tmp = where(count EQ 0)
		IF tmp[0] NE -1 THEN cat[tmp] += ' '+start_time[tmp]+' skip  '

		pos = max([strpos(cat[0],'make'),strpos(cat[0],'skip')])

		; Mark orbits for which skymaps exist as 'done'

		IF camcount[icam] GT 0 THEN BEGIN

 			; Extract time from filename. This should match start_time

			time = timeposn(*camskyfile[icam],part='name',/extract)
			time = TimeGet(time,/_ydoy,upto=usec)

			FOR i=0L,camcount[icam]-1 DO BEGIN

				; Location in catalogue of file with time[i]

				tmp = (where(start_time EQ time[i]))[0]

				CASE tmp EQ -1 OF
				0: IF mode_id EQ (*camskymode[icam])[i] THEN		$
						 cat[tmp] = strmid(cat[tmp],0,pos)+'done '	+ $
									(*camskytime   [icam])[i]		+ $
									string(format='(F5.2)',(*camskyversion[icam])[i])			
				1: BEGIN
					message, /info, 'unmatched skymap: '+hide_env((*camskyfile[icam])[i])
					tmp =  where( strmid(start_time,0,8) EQ strmid(time[i],0,8) )
					CASE tmp[0] NE -1 OF
					0: message, /info, 'no orbit start times for this day found'
					1: message, /info, 'orbit start times on this day are:'+strjoin(start_time[tmp],' ')
					ENDCASE
				END
				ENDCASE

			ENDFOR

		ENDIF

		IF NOT keepskip THEN BEGIN
			pos = where(strpos(cat,'skip') EQ -1, count)
			IF count EQ 0 THEN cat = '' ELSE cat = cat[pos]
		ENDIF

		filename = 	'skyd_c'+strcompress(cam_id,/rem)+ $
					'm'+strcompress(mode_id,/rem)	 + $
					(['','s'])[smoothdark]+'.txt'

		CASE count EQ 0 OF

		0: BEGIN

			; Write the catalogue.

			filename = filepath(root=destination,filename)

			message, /info, 'write '+hide_env(filename)

			openw, /get_lun, iu, filename
			CASE reverse_order OF
			0: FOR i=0L,n_elements(cat)-1    DO printf, iu, cat[i]	; Reverse chronological
			1: FOR i=n_elements(cat)-1,0L,-1 DO printf, iu, cat[i]	; Chronological
			ENDCASE
			free_lun, iu

		END

		1: message, /info, 'no orbits left for '+filename

		ENDCASE

	ENDFOR
	
ENDFOR

ptr_free, camskyfile, camskymode, camskytime, camskyversion

RETURN  &  END