;+
; NAME:
;	smei_buf_read
; PURPOSE:
;	Read Level 1A file in 'buf' format
; CATEGORY:
;	camera/idl/buf
; CALLING SEQUENCE:
	FUNCTION smei_buf_read, l1a_file, pointer,	$
		get_file_header = get_file_header,	$
		file_header 	= file_header	 ,	$
		frame_headers	= frame_headers	 ,	$
		frame_nr 		= frame_nr   	 ,	$
		frame_data		= frame_data	 ,	$
		get_next_frame	= get_next_frame ,	$
		silent			= silent		 ,	$
		force_close 	= force_close	 ,	$
		trunc_time		= trunc_time	 ,	$
		start_pointer	= start_pointer	 ,	$
		email			= email
; INPUTS:
;	l1a_file			scalar; type: string
;							fully-qualified file name of SMEI l1a*.buf file.
;							To close an open file, set l1a_file = ''
;	pointer				scalar; type: integer
;							file pointer in l1a_file where the requested frame starts.
;							This argument is optional, but if known, speeds up
;							the process of finding a frame in the L1A file.
; OPTIONAL INPUT PARAMETERS:
;	/get_file_header	only fill file_header structure
;	/get_next_frame 	get next frame from file
;						If this keyword is set then the data file is kept open
;						with the file pointer after the current frame.
;	frame_nr=frame_nr
;						get frame with number 'frame_nr'
;						The frame number specified here should be taken
;						from the header array returned in a previous call
;						(this is a 1-based frame number).
;	trunc_time			array[1]; type: time structure
;						used only when collecting all frame headers (i.e.
;						keywords frame_nr_defined and get_next_frame NOT set,
;						and argument have_pointer NOT set). The collection of
;						frames stops at the indicated time instead of
;						continuing until the end-of-file is reached.
;	start_pointer		scalar; type: integer
;							when new l1a_file is opened move to file pointer
;							to start_pointer and start processing from there
;							instead of from the beginning of the file.
;							'start_pointer' is destroyed to avoid using
;							in another l1a_file.
;
;	email=email			list of email addresses to be notified of errors
;						(used in smei_buf_read and smei_frm_cvhdr).
; OUTPUTS:
;	status				scalar; type: integer
;							0: failure; 1: success
; OPTIONAL OUTPUT PARAMETERS:
;	file_header=file_header
;				 		array[1]; type: file header structure
;	frame_headers=frame_headers
;						array[n]; type: frame header structure array
;							an array with with headers for all frames is returned
;						array[1]; type: frame header structure
;							if 'frame_nr' is specified than the
;							header for the matching frame is returned
;	frame_data=frame_data
;						array[n], array[n,m]; type: unsigned int, long or float
;							if 'frame_nr' is set then the matching frame
;							data are returned here.
; INCLUDE:
	@compile_opt.pro		; On error, return to caller
	@smei_roi_mask.pro		; ROI mask common block
; COMMON BLOCKS:
	common smei_read_buf_file, iu, open_file, open_hdr, pntr, last_frm, frm_count
; CALLS:
;	InitVar, IsType, destroyvar, TimeGet, smei_buf_splitfile
;	smei_frm_cvhdr, smei_property, TimeOp, boost, hide_env
;	smei_filename, TimeUnit, IsTime, TimeLimits
; PROCEDURE:
;	The content of SMEI .buf files is described in Don Mizuno's memo:
;
;	The data area is padded to a 2D frame using the mask stored in
;	the ASCII file smei_buf_roi.txt. This file contains a 318 x 64 array with values 0,1,2 and 3.
;	0: pixels outside the region of interest
;	1: pixels in the smei fov
;	2: pixels in the covered columns left and right
;	3; pixels in the square open areas outside the smei fov
; MODIFICATION HISTORY:
;	MAR-2003, Paul Hick (UCSD/CASS)
;	APR-2003, Paul Hick (UCSD/CASS)
;	    Fixed bug in frame header structure. The naxes values in the raw
;	    header were not updated when the ROI was applied (so the nic files
;	    prior to today (Apr 20) have naxes=[0,0] in the trailer).
;	MAR-2003, Paul Hick (UCSD/CASS)
;		Entry frame_header.n_satured is filled here using the
;		data extracted from the L1A file.
;	MAR-2009, Paul Hick (UCSD/CASS; pphick@ucsd.edu)
;		Added checks for corrupted frames (as identified by
;		smei_frm_cvhdr).
;-

InitVar, get_file_header, /key
InitVar, get_next_frame , /key
InitVar, silent 		, 0
InitVar, force_close	, /key

usec = TimeUnit(/sec)

have_pointer = IsType(pointer, /defined)
frame_nr_defined = IsType(frame_nr, /defined)
get_all_frame_headers = 1-(frame_nr_defined OR get_next_frame OR have_pointer)

IF force_close THEN BEGIN

	IF IsType(iu, /defined) THEN BEGIN
		free_lun, iu
		IF NOT get_file_header then message, /info,	$
			hide_env(open_file)+' ('+strcompress(frm_count,/rem)+' frames)'
		destroyvar, iu, open_file, open_hdr, pntr, last_frm, frm_count
	ENDIF

	RETURN, 1

ENDIF

IF IsType(iu, /defined) THEN BEGIN	; There is an open file

	close_file = get_file_header OR get_all_frame_headers OR l1a_file NE open_file
	IF NOT close_file THEN				$
		IF frame_nr_defined THEN	$
			close_file = frame_nr LE last_frm

	IF close_file THEN BEGIN
		free_lun, iu
		IF NOT get_file_header THEN message, /info,	$
			hide_env(open_file)+' ('+strcompress(frm_count,/rem)+' frames)'
		destroyvar, iu, open_file, open_hdr, pntr, last_frm, frm_count
	ENDIF

ENDIF

nhdr = 256L
bhdr = bytarr(nhdr, /nozero)

IF IsType(iu, /undefined) THEN BEGIN

	openr, /get_lun, iu, l1a_file, error=error

	IF error NE 0 THEN BEGIN
		destroyvar, iu
		message, /info, !error_state.msg
		RETURN, 0
	ENDIF

	point_lun, -iu, pntr					; Initialize file pointer (to zero)

	readu, iu, bhdr
	pntr += nhdr

	open_file = l1a_file
	open_hdr  = smei_buf_splitfile(bhdr)

	IF NOT IsType(open_hdr, /structure) THEN BEGIN
		message, /info, hide_env(open_file)+', funny file name in header?, '+open_hdr
		free_lun, iu
		destroyvar, iu, open_file, open_hdr, pntr, last_frm, frm_count
		message, 'aborting'
	ENDIF

	last_frm  = 0L
	frm_count = 0L

	file_header = open_hdr

	IF IsType(start_pointer,/defined) THEN BEGIN
		pntr = start_pointer
		destroyvar, start_pointer
		point_lun, iu, pntr
	ENDIF

ENDIF

CASE get_file_header OF
0: BEGIN

	destroyvar, frame_headers
	IF get_next_frame THEN				$
		IF NOT frame_nr_defined THEN	$
			frame_nr = last_frm+1

	IF have_pointer THEN BEGIN
		IF pointer NE pntr THEN BEGIN
			pntr = pointer
			point_lun, iu, pntr 			; Set pointer
		ENDIF
	ENDIF

	; Set up I/O error henadler

	status = 0
	on_ioerror, no_more_frames

	readu, iu, bhdr
	pntr += nhdr						; Update file pointer

	corrupt_frame = 0

	next_frame:

		frame_header = smei_frm_cvhdr(	$
			from_byte		= bhdr				, $
			/to_hdr								, $
			/swap								, $
			tlm_file		= open_hdr.tlm_file	, $
			l1a_file		= open_file			, $
			l1a_pntr		= pntr-nhdr			, $
			corrupt_frame	= corrupt_frame		, $
			error_message	= error_message		)

		; This looks pretty insane, and could turn into a near-infinite loop.
		; The idea apparently is to step forward 1-byte at a time until a 256-byte
		; stretch is found that returns a valid frame header. Good luck with that.

		CASE IsType(frame_header,/struct) OF
		0: BEGIN
			IF corrupt_frame EQ 0 THEN BEGIN
				msg = 'ptr='+strcompress(pntr,/rem)+', corrupt frame header found in '+hide_env(open_file)+'('+error_message+'); attempting recovery ..'
				IF IsType(email,/defined) THEN $
					spawn, 'echo "'+msg+'" | mail -s"emergency recovery started" '+email
				message, /info, msg
			ENDIF
			corrupt_frame = 1
			;message, /info, 'skip past pntr ='+strcompress(pntr)
			point_lun, iu, ++pntr 			; Set file pointer
			readu, iu, bhdr
			GOTO, next_frame
		END
		1: BEGIN
			IF corrupt_frame EQ 1 THEN BEGIN
				msg = 'ptr='+strcompress(pntr,/rem)+', recovery succeeded; next frame in '+hide_env(open_file)+' is '+smei_property(frame_header,/name)
				IF IsType(email,/defined) THEN $
					spawn, 'echo "'+msg+'" | mail -s"emergency recovery succeeded" '+email
				message, /info, msg
				pntr += nhdr
				frame_nr = frame_header.frame_nr
				corrupt_frame = 0
			ENDIF
		END
		ENDCASE

		IF have_pointer THEN				$
			IF NOT frame_nr_defined THEN	$
				frame_nr = frame_header.frame_nr

		IF silent LE -1 THEN message, /info, smei_property(frame_header,/name)

		; Length of data area in bytes

		len_data = frame_header.n_data*abs(frame_header.bitpix)/8

		IF get_all_frame_headers THEN BEGIN

			; Skip past data area by moving file pointer

			pntr += len_data
			point_lun, iu, pntr 			; Set file pointer

			; Collect all the frame headers.

			IF frm_count EQ 0 THEN BEGIN

			;	IF open_hdr.total_frames LE 0 THEN	$
			;		message, 'Invalid # frames reported in file header:'+	$
			;			strcompress(open_hdr.total_frames)
				frame_headers = replicate(frame_header, open_hdr.total_frames)
				time_headers  = replicate(!TheTime    , open_hdr.total_frames)

			ENDIF ;ELSE IF frm_count GT open_hdr.total_frames-1 THEN BEGIN

			;	message, 'Frame index out of range:'+	$
			;		strcompress(frm_count+1)+'/'+		$
			;		strcompress(open_hdr.total_frames,/rem)
			
			;ENDIF

			frame_time = smei_property(frame_header,/time)

			IF IsTime(trunc_time) THEN BEGIN
				IF TimeOp(/subtract,frame_time,trunc_time,TimeUnit(/sec)) GT 0 THEN BEGIN
					frame_headers = frame_headers[0:frm_count-1]
					time_headers  = time_headers [0:frm_count-1]
					!error_state.msg = 'truncated headers at '+TimeGet(trunc_time,/_ydoy,upto=usec)
					goto, no_more_frames
				ENDIF
			ENDIF

			frame_headers[frm_count] = frame_header
			time_headers [frm_count] = frame_time
			frm_count++

			readu, iu, bhdr
			pntr += nhdr				; Update file pointer

			GOTO, next_frame

		ENDIF

		IF frame_nr NE frame_header.frame_nr THEN BEGIN

			; Skip past data area by moving file pointer

			pntr += len_data
			point_lun, iu, pntr 			; Set file pointer

			readu, iu, bhdr
			pntr += nhdr

			GOTO, next_frame

		ENDIF

		IF arg_present(frame_data) THEN BEGIN

			; Read data of type indicated by frame_header.bitpix

			bitpix = frame_header.bitpix

			CASE bitpix OF
			 16: data_area = uintarr( frame_header.n_data, /nozero )
			 32: data_area = long   ( frame_header.n_data, /nozero )
			-32: data_area = float  ( frame_header.n_data, /nozero )
			ENDCASE

			readu, iu, data_area
			pntr += len_data			; Update file pointer

			byteorder, data_area, sswap=bitpix EQ 16,	$
				lswap=bitpix EQ 32, xdrtof=bitpix EQ -32, /swap_if_little_endian

			mode = where( data_area GE ([65535,49147])[frame_header.flat_enabled], n_saturated  )
			frame_header.n_saturated = n_saturated

			mode = frame_header.mode

			frame_dim = *roi_siz[mode]

			CASE frame_header.full_frame OF
			0: BEGIN
				frame_data = make_array( type=IsType(data_area), dim=frame_dim )
				frame_data[ *roi_frm[mode] ] = data_area

				frame_header.full_frame = 1
				frame_header.naxes = frame_dim
				frame_header.n_data = frame_dim[0]*frame_dim[1]
			END
			1: BEGIN
				frame_data = reform(data_area, frame_header.naxes)

				IF frame_header.naxes[0] LT frame_dim[0] THEN BEGIN

					plus = frame_dim[0]-frame_header.naxes[0]
					boost, frame_data, /lead, plus=plus
					frame_header.naxes[0] = frame_dim[0]
					frame_header.n_data = frame_dim[0]*frame_header.naxes[1]

					message, /info, smei_filename(smei_property(frame_header,/time),	$
						camera=frame_header.camera,mode=mode,type='')+' boosted by'+strcompress(plus)

				ENDIF
			END
			ENDCASE

		ENDIF ELSE BEGIN

			; Skip past data area by moving file pointer

			pntr += len_data
			point_lun, iu, pntr 		; Set file pointer

		ENDELSE

		frm_count++
		frame_headers = frame_header
		time_headers  = smei_property(frame_header, /time)
		last_frm      = frame_nr
		status = 1

	no_more_frames:

	on_ioerror, null

	; Print error message (other than EOF)

	eof_reached = status EQ 0

	IF eof_reached THEN BEGIN
		eof_reached = strpos(!error_state.msg,'End of file') NE -1
		IF NOT eof_reached then message, /info, !error_state.msg
	ENDIF

	; Always close file if status=0. Return status=1
	; only when reading all frame headers (error was EOF).

	; status=1: happens when succesfully locating a single frame. Keep the
	; file open only if /get_next_frame was set, and the current frame was
	; located.

	close_file = status EQ 0 OR NOT get_next_frame

	status = status OR get_all_frame_headers

	; We assume that all frames in a file fall inside the time range indicated
	; in the L1A file name. If this is not true we may run into trouble. 

	IF get_all_frame_headers AND frm_count GT 0 THEN BEGIN
		nut = where(TimeOp(/subtract,time_headers, open_hdr.ut_start,usec) LT 0 OR	$
					TimeOp(/subtract,time_headers, open_hdr.ut_stop ,usec) GT 0)
		IF nut[0] NE -1 THEN BEGIN
			message, /info, hide_env(open_file)+' '+strjoin(TimeGet([open_hdr.ut_start,open_hdr.ut_stop],/ydoy,upto=usec),' - ')
			message, /info, strcompress(n_elements(nut),/rem)+' frames outside file time range'
			message, /info, 'frame range '+strjoin(TimeGet(TimeLimits(time_headers, /bounds),/ydoy,upto=usec),' - ')
		ENDIF
	ENDIF

END

1: BEGIN
	close_file = 1
	status = 1
END

ENDCASE

IF close_file THEN BEGIN
	free_lun, iu
	IF silent LE 0 THEN BEGIN
		IF l1a_file NE open_file THEN message, 'oops 13'
		CASE 1 OF
		get_file_header 	 : message, /info, hide_env(open_file)+' ('+strcompress(open_hdr.total_frames, /rem)+' frames claimed)'
		get_all_frame_headers: message, /info, hide_env(open_file)+' ('+strcompress(frm_count, /rem)+'/'+strcompress(open_hdr.total_frames,/rem)+' hdrs)'
		get_next_frame		 : if eof_reached then message, /info, hide_env(open_file)+' ('+strcompress(frm_count,/rem)+'/'+strcompress(open_hdr.total_frames,/rem)+' frames)'
		ELSE				 : message, /info, hide_env(open_file)+' (frame@'+TimeGet(time_headers,/ydoy,upto=usec)+')'
		ENDCASE
	ENDIF
	destroyvar, iu, open_file, open_hdr, pntr, last_frm, frm_count
ENDIF

RETURN, status  &  END