;+
; NAME:
;	smei_buf
; PURPOSE:
;	Main procedure for processing l1a files
; CATEGORY:
;	camera/idl/buf
; CALLING SEQUENCE:
	PRO smei_buf, tmp_list
; INPUTS:
;	tmp_list		scalar; type: string; default: $TUB/l1a_list.txt
;						name of ascii file created by Python script l1a.py
;						(the file will be deleted after it has been read)
; OUTPUTS:
;	.nic files to $dat/smei/nic
; INCLUDE:
	@compile_opt.pro		; On error, return to caller
; CALLS:
;	InitVar, txt_read, SetFileSpec, GetFileSpec, TimeOp, TimeGet
;	TimeUnit, smei_buf_getframe, smei_buf_get, smei_buf_mget, do_file
;	TimeSystem, hide_env
; PROCEDURE:
; >	This procedure is accessed primarily through the Python script
;	l1a.py, which creates the main IDL main program and the input
;	file 'tmp_list'.
; >	All frames are extracted from the specified list of *.buf files
;	and written to disk as *.nic files.
; MODIFICATION HISTORY:
;	MAR-2003, Paul Hick (UCSD/CASS)
;	FEB-2004, Paul Hick (UCSD/CASS)
;		Added option for 'noclones' check.
;		Added option to move processed L1A files to other directory
;	MAR-2005, Paul Hick (UCSD/CASS; pphick@ucsd.edu)
;		Added option to allow writing of Fits files and Nic files.
;-

InitVar, tmp_list, filepath(root=getenv('TUB'), 'l1a_list.txt' )
umin = TimeUnit(/minute)

sts = txt_read( tmp_list, files);, /silent )	; Read input file written by l1a.py

loc_dest    	=  0 			; destination
loc_email   	=  1 			; emails to be notified of errors
loc_nodups	 	=  2
loc_overwrite	=  3 			; skip exiting/overwrite existing/do not write individual nic frames
loc_gzip		=  4 			; gzip subdirectories
loc_remove		=  5 			; remove *.buf files
loc_noclones	=  6
loc_silent		=  7 			; level of silence
loc_nic 		=  8
loc_status		=  9
loc_file	 	= 10 			; start of *.buf file names

nfile = n_elements(files)

sts = sts AND nfile GT loc_file

IF sts THEN BEGIN

	destination = files[loc_dest]
	email = files[loc_email]

	SetFileSpec, files[loc_file]
	message, /info, 'reading '+hide_env(GetFileSpec(upto='directory'))
	message, /info, 'writing '+hide_env(destination)

	notify = email NE 'no_email'
	CASE notify OF
	0: destroyvar, email
	1: message, /info, 'email to '+email
	ENDCASE

	nodups = files[loc_nodups] EQ 'remove'
	message, /info, files[loc_nodups]+' duplicate frames'

	overwrite = (where(files[loc_overwrite] EQ ['skip existing','overwrite existing','do not write']))[0]
	message, /info, files[loc_overwrite]+' frames'
	nowrite = overwrite EQ 2

	gzip = files[loc_gzip] EQ 'gzip'
	IF gzip THEN message, /info, 'gzip directories'

	remove = files[loc_remove] EQ 'remove'
	IF remove THEN message, /info, 'delete *.buf files after processing'

	IF strmid(files[loc_remove],0,1) EQ '/' THEN	$
		moveto = CheckDir(files[loc_remove]) ELSE moveto = 0

	noclones = files[loc_noclones] EQ 'remove'
	IF noclones then message, /info, 'skipping clone frames'

	silent = files[loc_remove] EQ 'silent'
	IF silent THEN message, /info, 'silent running'

	nic = files[loc_nic] EQ '.nic'
	IF silent THEN message, /info, 'writing '+(['Fits','Nic'])[nic]+' files'

	status = files[loc_status] EQ 'status'
	IF silent THEN message, /info, 'writing status files'

	tt = TimeSystem(/silent)

	; Unzip .buf.gz files

	FOR i=loc_file,nfile-1 DO BEGIN
		SetFileSpec, files[i]
		IF GetFileSpec(part='type') EQ '.gz' THEN BEGIN
			message, /info, 'gunzip '+hide_env(files[i])
			spawn, 'gunzip '+files[i]
			files[i] = GetFileSpec(upto='name')
		ENDIF
	ENDFOR

	CASE 1 OF
	
	nowrite: smei_buf_mget, files[loc_file:*], $
		fits		= 1-nic			, $
		nic			= nic			, $
		destination	= destination	, $
		/split						, $
		/nowrite					, $
		noclones	= noclones		, $
		status		= status		, $
		silent		= silent		, $
		email		= email

	1-nodups:	$

		; All L1A files are processed individually processing frames in the
		; sequence they are stored in the files. I.e. no attempt is made to
		; remove double frames (from the second telemetry dump). If /overwrite
		; is set then presumably the 2nd dump is used predominantly.
		; If /overwrite is NOT set then presumably the 1st dump is used
		; predominantly.

		smei_buf_mget, files[loc_file:*]	, $
			fits		= 1-nic				, $
			nic			=nic				, $
			destination	= destination		, $
			/split							, $
			overwrite	= overwrite			, $
			gzip		= gzip				, $
			noclones	= noclones			, $
			status		= status			, $
			silent		= silent			, $
			email		= email

	ELSE: BEGIN

		; All L1A files are preprocessed to compile a list of available frames.
		; Duplicate frames are removed from the list. The remaining frames are
		; processed in the same order as they are stored on the L1A files.
		; The elimation of duplicate frames is done using the IDL uniq function,
		; which presumably retains the second copy, so this method favors the
		; 2nd dump.

		; Note that smei_buf_getframe has a /dupstoo keyword to suppress the
		; removal of duplicate frames. If this is used then the setting of
		; /overwrite has the same effect as smei_buf_mget on favoring the 1st
		; or 2nd dump. Since smei_buf_mget is much more memory friendly (no
		; list of available frames is compiled) it is the preferred way
		; in that case.

		hdr = smei_buf_getframe(source=files[loc_file:*], count=count)

		tt = [tt[0],TimeSystem(/silent)]

		message, /info, strjoin(TimeGet(tt,/ymd,upto=umin),'-')+	$
				strcompress( round(TimeOp(/subtract,tt[1],tt[0],umin)) )+' minutes'

		IF count GT 0 then	$
			smei_buf_get, hdr				, $
				fits		= 1-nic			, $
				nic			= nic			, $
				destination	= destination	, $
				/split						, $
				overwrite	= overwrite		, $
				gzip		= gzip			, $
				noclones	= noclones		, $
				status		= status		, $
				silent		= silent
	END
	ENDCASE

	; Deleting *.buf files

	IF moveto THEN FOR i=loc_file,nfile-1 DO	$
		tmp = do_file(/move, files[i], files[loc_remove], silent=0)

	IF remove THEN FOR i=loc_file,nfile-1 DO	$
		tmp = do_file(/delete, files[i], silent=0)

	tt = [tt[0],TimeSystem(/silent)]
	tt = strjoin(TimeGet(tt,/ymd,upto=umin),'-')+	$
		strcompress(round(TimeOp(/subtract,tt[1],tt[0], umin)))+' minutes'
	message, /info, tt

	;IF notify THEN spawn,	$
	;	'echo "'+tt+' last file '+hide_env(files[nfile-1])+'" | '+		$
	;	'mail '+email+' -s"l1a '+getenv('HOSTNAME')+' finished"'

ENDIF

RETURN  &  END