PRO LOAD_CDROM, DRIVE=K_DRIVE, DESTINATION=DESTINATION,	$
		REGISTER_ONLY=REGISTER_ONLY, DATE=DATE_SEL, ERRMSG=ERRMSG, $
		SEQUENCE=SEQ_NO, VOLUME=VOL_NO, NOREGISTER=NOREGISTER
;+
; Project     :	SOHO - CDS
;
; Name        :	LOAD_CDROM
;
; Purpose     :	Loads telemetry data files from CDROM onto hard disk.
;
; Category    :	Class3, I/O
;
; Explanation :	Takes CDS telemetry data from the final distribution CDROMs and
;		copies the data files to a hard disk.  The associated SFDU
;		files are read to extract the long version of the filename.
;
;		Data from the low, medium, and high rate telemetry directories
;		on the CDROM are merged together into a single directory on the
;		hard disk.  The format is changed to match that of realtime
;		"tm." files.
;
;		The complementary routine UNLOAD_CDROM can be used to
;		selectively delete the files loaded by LOAD_CDROM.
;
; Syntax      :	LOAD_CDROM
;
; Examples    :	LOAD_CDROM
;		LOAD_CDROM, /REGISTER_ONLY
;		LOAD_CDROM, DATE='1996-01-19'
;		LOAD_CDROM, DRIVE='SYS$CDROM:[000000]'
;		LOAD_CDROM, DESTINATION='~/cds_tm_data'
;
; Inputs      :	None.
;
; Opt. Inputs :	None.
;
; Outputs     :	None.
;
; Opt. Outputs:	None.
;
; Keywords    :	DRIVE	    = A character string giving the same of the mount
;			      point for the CDROM.  The default is "/cdrom" on
;			      Unix systems, and "CDROM:[000000]" on VMS
;			      systems.
;
;		DESTINATION = The directory to write the CDROM files.  The
;			      default is given by the environment variable
;			      CDS_CDROM_DATA_W.
;
;		REGISTER_ONLY = If set, then the CDROM is registered in the
;			      database, but no "tm." files are written out.
;
;		DATE	    = A specific date to extract from the CDROM.  Can
;			      be in any standard CDS time format.  If not set,
;			      then the entire CDROM is processed.
;
;		SEQUENCE    = Returns the sequence number of the CDROM.
;
;		VOLUME	    = Returns the volume number of the CDROM.
;
;		ERRMSG = If defined and passed, then any error messages will be
;			 returned to the user in this parameter rather than
;			 depending on the MESSAGE routine in IDL.  If no errors
;			 are encountered, then a null string is returned.  In
;			 order to use this feature, ERRMSG must be defined
;			 first, e.g.
;
;				ERRMSG = ''
;				LOAD_CDROM, ERRMSG=ERRMSG, ...
;				IF ERRMSG NE '' THEN ...
;
;		NOREGISTER = If defined, don't register the CD
;
; Calls       :	CONCAT_DIR
;
; Common      :	None.
;
; Restrictions:	When used with VMS, the format of the DRIVE parameter should be
;		like "CDROM:[000000]" rather than simply "CDROM:".
;
; Side effects:	The "tm." files created do not contain any housekeeping
;		packets.
;
;		The first six bytes in each packet of the "tm." files are not
;		in the telemetry from the spacecraft.  Apparently, the Spacetec
;		software prepends these bytes when it stores the telemetry.
;		This procedure tries to mimic what the Spacetec software does
;		to sufficient accuracy for the IDL routines which use these
;		data.
;
; Prev. Hist. :	None.
;
; History     :	Version 1, 22-Jan-1996, William Thompson, GSFC
;		Version 2, 29-Jan-1996, William Thompson, GSFC
;			Removed part of code where filename was changed to
;			disguise rate--turned out not to help.
;		Version 3, 30-Jan-1996, William Thompson, GSFC
;			Rewrote to reprocess into "tm." format.
;			Use environment variable CDS_CDROM_DATA_W to avoid
;			overwriting realtime data.
;		Version 4, 04-Apr-1996, William Thompson, GSFC
;			Correct problem where some CDROMs start one second
;			before they're supposed to.
;			Register CDROM in database
;		Version 5, 10-Apr-1996, William Thompson, GSFC
;			Modified for version 2 of the CDROM database.
;		Version 6, 22-Apr-1996, William Thompson, GSFC
;			Fixed a typo introduced in version 5.
;			Added keywords REGISTER_ONLY and DATE.
;		Version 7, 24-Apr-1996, William Thompson, GSFC
;			Added keywords VOLUME, SEQUENCE, ERRMSG.
;			Check that !PRIV is set correctly.
;		Version 8, 20-May-1996, William Thompson, GSFC
;			Test for multiple versions of the same file.
;               Version 8.0001 27-June-1996, Dominic Zarro, GSFC
;                       Allowed for bug in SORT which requires vector input.
;		Version 9, 12 September 1996, Eddie Breeveld, MSSL/UCL
;			NOREGISTER keyword to stop registering the CD
;		Version 10, 18-Aug-1997, William Thompson, GSFC
;			Correct problem where some CDROMs end one second after
;			they're supposed to.
;		Version 11, 17-Sep-1997, William Thompson, GSFC
;			Correct problem on days with leap seconds.
;		Version 12, 25-Aug-2000, William Thompson, GSFC
;			Filter out packets with wild timestamps.
;		Version 13, 08-Mar-2002, William Thompson, GSFC
;			Made sure that input DRIVE keyword not modified.
;               Version 14, 08-May-2012, William Thompson, GSFC
;                       Increase valid time limits to beyond 2012
;
; Contact     :	WTHOMPSON
;-
;
@fitsgen.cmn
	ON_ERROR, 2
;
;  Make sure that !PRIV is set correctly.
;
	IF (!PRIV LT 2) and (not KEYWORD_SET(NOREGISTER)) THEN BEGIN	
	    MESSAGE = '!PRIV must be at least 2 to write into the database'
	    GOTO, HANDLE_ERROR
	ENDIF
;
;  Determine the value of the DRIVE parameter.
;
	IF N_ELEMENTS(K_DRIVE) EQ 1 THEN DRIVE = K_DRIVE ELSE BEGIN
	    CASE OS_FAMILY() OF
		'unix':	DRIVE = '/cdrom'
		'vms':	DRIVE = 'CDROM:[000000]'
		ELSE:	BEGIN
		    MESSAGE = 'Only supported in VMS or Unix'
		    GOTO, HANDLE_ERROR
		END
	    ENDCASE
	ENDELSE
;
;  Determine the destination directory.
;
	IF N_ELEMENTS(DESTINATION) EQ 0 THEN BEGIN
	    DESTINATION = GETENV('CDS_CDROM_DATA_W')
	    IF DESTINATION EQ '' THEN BEGIN
		MESSAGE = 'Environment variable CDS_CDROM_DATA_W not defined'
		GOTO, HANDLE_ERROR
	    ENDIF
	ENDIF
;
;  Decide whether or not upper- or lowercase filenames are being used.
;
	VOLDESC = FINDFILE(CONCAT_DIR(DRIVE,'voldesc.sfd*'), COUNT=COUNT)
	LOWER = COUNT NE 0
	IF NOT LOWER THEN BEGIN
	    VOLDESC = FINDFILE(CONCAT_DIR(DRIVE,'VOLDESC.SFD*'),	$
		    COUNT=COUNT)
	    UPPER = COUNT NE 0
	    IF NOT UPPER THEN BEGIN
		MESSAGE = 'Incorrect CDROM mounted'
		GOTO, HANDLE_ERROR
	    ENDIF
	ENDIF
;
;  Open up the voldesc.sfd file, and read in the sequence number and volume
;  number.
;
	OPENR, UNIT, VOLDESC(0), /GET_LUN
	SEQ_NO = -1
	VOL_NO = -1
	LINE = ''
	WHILE (NOT EOF(UNIT)) AND ((SEQ_NO LT 0) OR (VOL_NO LT 0)) DO BEGIN
	    READF, UNIT, LINE
	    IF STRMID(LINE,0,16) EQ 'SEQUENCE NUMBER:' THEN	$
		    SEQ_NO = LONG(STRMID(LINE,16,STRLEN(LINE)-16))
	    IF STRMID(LINE,0,14) EQ 'VOLUME NUMBER:' THEN	$
		    VOL_NO = LONG(STRMID(LINE,14,STRLEN(LINE)-14))
	ENDWHILE
	FREE_LUN, UNIT
;
;  Set up the name of the base directory for CDS telemetry data.
;
	DIRS = ['data', 'so']
	IF NOT LOWER THEN DIRS = STRUPCASE(DIRS)
	BASE = CONCAT_DIR(DRIVE, /DIR, DIRS(0))
	BASE = CONCAT_DIR(BASE,  /DIR, DIRS(1))
;
;  Step through the three CDS telemetry rates, and collect all the filenames.
;
	RATES = ['g026', 'g027', 'g028']
	LZ = 'lz'
	DAT_FILES = '*.d*'
	SFD_FILES = '*.s*'
	IF NOT LOWER THEN BEGIN
	    RATES = STRUPCASE(RATES)
	    LZ = STRUPCASE(LZ)
	    DAT_FILES = STRUPCASE(DAT_FILES)
	    SFD_FILES = STRUPCASE(SFD_FILES)
	ENDIF
	FILES = ''
	SFDUS = ''
	FOR I_RATE = 0, 2 DO BEGIN
;
;  Set up the name of the directory containing the telemetry data.
;
	    PATH = CONCAT_DIR(BASE, /DIR, RATES(I_RATE))
	    PATH = CONCAT_DIR(PATH, /DIR, LZ)
;
;  Find all the SFDU and data files in this directory.  Make sure that only the
;  highest version number is used.
;
	    TEMP = FINDFILE(CONCAT_DIR(PATH, SFD_FILES), COUNT=COUNT)
	    IF COUNT GT 0 THEN BEGIN
		BREAK_FILE, TEMP, DISK, DIR, NAME
		S = REVERSE(SORT([NAME]))
		TEMP = TEMP(S)
		NAME = NAME(S)
		LAST = ''
		FOR I = 0, COUNT-1 DO BEGIN
		    TEST = STRMID(NAME(I),0,4)
		    IF TEST NE LAST THEN SFDUS = [SFDUS, TEMP(I)]
		    LAST = TEST
		ENDFOR
	    ENDIF
;
	    TEMP = FINDFILE(CONCAT_DIR(PATH, DAT_FILES), COUNT=COUNT)
	    IF COUNT GT 0 THEN BEGIN
		BREAK_FILE, TEMP, DISK, DIR, NAME
		S = REVERSE(SORT([NAME]))
		TEMP = TEMP(S)
		NAME = NAME(S)
		LAST = ''
		FOR I = 0, COUNT-1 DO BEGIN
		    TEST = STRMID(NAME(I),0,4)
		    IF TEST NE LAST THEN FILES = [FILES, TEMP(I)]
		    LAST = TEST
		ENDFOR
	    ENDIF
	ENDFOR
;
;  Remove the dummy first filename.
;
	N_FILES = N_ELEMENTS(FILES) - 1
	IF N_FILES EQ 0 THEN BEGIN
	    MESSAGE = 'No data files found on the CDROM'
	    GOTO, HANDLE_ERROR
	ENDIF
	FILES = FILES(1:*)
	SFDUS = SFDUS(1:*)
;
;  Open up all the SFDU files, and get the start and stop times.
;
	GET_LUN, UNIT
	LINE = ''
	FOR I_FILE = 0, N_FILES-1 DO BEGIN
	    OPENR, UNIT, SFDUS(I_FILE)
	    WHILE NOT EOF(UNIT) DO BEGIN
		READF, UNIT, LINE
		IF STRMID(LINE,0,11) EQ 'START_DATE=' THEN	$
			START_DATE = STRMID(LINE,11,STRPOS(LINE,';')-11)
		IF STRMID(LINE,0,10) EQ 'STOP_DATE=' THEN	$
			STOP_DATE = STRMID(LINE,10,STRPOS(LINE,';')-10)
	    ENDWHILE
	    CLOSE, UNIT
;
	    START_DATE = STR2UTC(START_DATE)
	    STOP_DATE  = STR2UTC(STOP_DATE)
	    IF I_FILE EQ 0 THEN BEGIN
		STARTS = START_DATE
		STOPS  = STOP_DATE
	    END ELSE BEGIN
		STARTS = [STARTS, START_DATE]
		STOPS  = [STOPS,  STOP_DATE ]
	    ENDELSE
	ENDFOR
	DATES  = STARTS.MJD
	FREE_LUN, UNIT
;
;  Register the CDROM.  If the REGISTER_ONLY keyword was set, then return at
;  this point.
;
	IF NOT KEYWORD_SET(NOREGISTER) THEN BEGIN
	    DATES = DATES(REM_DUP(DATES))
	    FOR I = 0, N_ELEMENTS(DATES)-1 DO	$
		    TEST = REG_CDROM( SEQ_NO, VOL_NO, DATES(I) )
	ENDIF
	IF KEYWORD_SET(REGISTER_ONLY) THEN RETURN
;
;  If the DATE keyword was set, then filter out all but the files pertaining to
;  the selected date.
;
	IF N_ELEMENTS(DATE_SEL) EQ 1 THEN BEGIN
	    TEMP = ANYTIM2UTC(DATE_SEL)
	    W = WHERE(STARTS.MJD EQ TEMP.MJD, N_FILES)
	    IF N_FILES EQ 0 THEN BEGIN
		MESSAGE = 'Selected date not found on CDROM'
		GOTO, HANDLE_ERROR
	    ENDIF
	    STARTS = STARTS(W)
	    STOPS = STOPS(W)
	    FILES = FILES(W)
	ENDIF
;
;  Define the arrays for the telemetry files.
;
	STATUS = LONARR(N_FILES)
	UNIT = LONARR(N_FILES)
	IP = LONARR(N_FILES)
	NP = LONARR(N_FILES)
	TIME = DBLARR(N_FILES)
	PACKET = BYTARR(300, N_FILES)
	P = BYTARR(300)
	MIN_START = MIN(UTC2TAI(STARTS))
	MAX_STOP  = MAX(UTC2TAI(STOPS))
;
;  Get a unit number for the output file.
;
	GET_LUN, OUT
	OUTFILE = ''
;
;  Open up all the data files simultaneously.  Read in the first packet, and
;  extract the time.
;
	FOR I_FILE = 0, N_FILES-1 DO BEGIN
	    GET_LUN, U
	    UNIT(I_FILE) = U
	    FILE = FILES(I_FILE)
	    STATUS(I_FILE) = OPEN_TM_FILE(U, FILE, MODE)
	    IF MODE NE 4 THEN BEGIN
		MESSAGE = 'File ' + FILE + ' not opened with correct mode'
		GOTO, HANDLE_ERROR
	    ENDIF
	    IP(I_FILE) = I_PACK
	    NP(I_FILE) = NUM_PACK
	    REPEAT BEGIN
		STATUS(I_FILE) = IP(I_FILE) LT NP(I_FILE)
		IF STATUS(I_FILE) THEN BEGIN
		    READU, UNIT(I_FILE), P
		    PACKET(0,I_FILE) = P
		    IP(I_FILE) = IP(I_FILE) + 1
		    TIME(I_FILE) = OBT2TAI(P(6:11))
		ENDIF
	    ENDREP UNTIL (TIME(I_FILE) GT 1.196D9) AND (TIME(I_FILE) LT 2D9)
	ENDFOR
;
;  Keep reading packets until all the files are exhausted.
;
	J_FILE = -1
	TAI0 = 0.0D0
	LASTFILE = ''
	WHILE TOTAL(STATUS NE 0) NE 0 DO BEGIN
;
;  Select which file the packet should be taken from.
;
	    I_FILE = WHERE(STATUS NE 0)
	    T = TIME(I_FILE)
	    W = WHERE(T EQ MIN(T), COUNT)
	    T = T(W(0))
	    IF COUNT GT 1 THEN MESSAGE, /CONTINUE,	$
		    'Duplicate packets with time ' + TAI2UTC(T, /ECS)
	    I_FILE = I_FILE(W(0))
	    IF I_FILE NE J_FILE THEN PRINT,	$
		    'Selecting packets from file ' + FILES(I_FILE)
	    J_FILE = I_FILE
;
;  Write out the packet to the appropriate file.  Take into account the fact
;  that the DDF don't always get the TAI-UTC conversion correctly, so don't try
;  to open any files before the start time listed in the headers, or after the
;  end date.
;
	    IF ((T - TAI0) GE 3600.D0) AND (T LT MAX_STOP) THEN BEGIN
		DATE = TAI2UTC(T > MIN_START, /EXTERNAL)
		YEAR = STRMID(STRING(DATE.YEAR,FORMAT='(I4)'),2,2)
		DOY = STRMID(STRING(UTC2DOY(DATE)+1000,FORMAT='(I4)'),1,3)
		HOUR = STRMID(STRING(DATE.HOUR+100,FORMAT='(I3)'),1,2)
		OUTFILE = 'tm.' + YEAR + '_' + DOY + '_' + HOUR
		IF OUTFILE NE LASTFILE THEN BEGIN
		    CLOSE, OUT
		    OPENW, OUT, CONCAT_DIR(DESTINATION, OUTFILE), /BLOCK
		    PRINT, 'Writing to output file ' + OUTFILE
		    LASTFILE = OUTFILE
;
;  Calculate the TAI time for the beginning of the hour.
;
		    DATE.MINUTE = 0
		    DATE.SECOND = 0
		    DATE.MILLISECOND = 0
		    TAI0 = UTC2TAI(DATE)
		ENDIF
	    ENDIF
;
;  Insert the proper 6 extra bytes before each packet.
;
	    EXTRA = [516, 302, 0]
	    REV_SWAP, EXTRA
	    WRITEU, OUT, EXTRA
	    WRITEU, OUT, PACKET(*,I_FILE)
;
;  Read in the next packet for the selected file.
;
	    REPEAT BEGIN
		STATUS(I_FILE) = IP(I_FILE) LT NP(I_FILE)
		IF STATUS(I_FILE) THEN BEGIN
		    READU, UNIT(I_FILE), P
		    PACKET(0,I_FILE) = P
		    IP(I_FILE) = IP(I_FILE) + 1
		    TIME(I_FILE) = OBT2TAI(P(6:11))
		ENDIF
	    ENDREP UNTIL (TIME(I_FILE) GT 1.196D9) AND (TIME(I_FILE) LT 2D9)
	ENDWHILE
;
	FOR I = 0,N_FILES-1 DO FREE_LUN, UNIT(I)
	FREE_LUN, OUT
	RETURN
;
;  Error handling point.
;
HANDLE_ERROR:
	IF N_ELEMENTS(ERRMSG) NE 0 THEN ERRMSG = 'LOAD_CDROM: ' + MESSAGE $
		ELSE MESSAGE, MESSAGE, /CONTINUE
;
	END