pro rd_xda, infil, dset_arr, index, data, roadmap, extra, $
		nodata=nodata, filidx=filidx, dset_str=dset_arr0, $
		qstop=qstop, qdebug=qdebug, q2debug=q2debug, $
		start_end=start_end
;+
;NAME:
;	RD_XDA
;PURPOSE:
;	Read and extract data from any of the Yohkoh data files 
;	(SDA, BDA, WDA, HDA, ADA, or CBA).  
;CALLING SEQUENCE:
;	Rd_XDA, infil, dset_arr, index, data, roadmap
;	Rd_XDA, infil, dset_arr, index, data
;	rd_xda, infil, dset_arr, index, data, roadmap, dset_str=dset_arr
;	Rd_XDA, infil, dset_arr, index, /nodata
;INPUT:
;	infil	- input file specification - it can be a vector
;		  (WARNING - do not mix and match FFI and PFI)
;	dset_arr: vector of dataset numbers to extract (indices vector)
;               * "dset_arr" can be
;                     1. An indicie vector ("SS") from a search of the roadmap
;			 of ALL of the files in "infil".
;                     2. a structure with fields ".dset" and ".ifil".
;                        ".dset" is the dataset number WITHIN THE FILE, and
;                        ".ifil" is the index of the filename within the
;                        infil array.  The structure is "N" elements long
;                        where "N" is the total number of datasets to extract.
;		* The order that the images are placed in the output 
;		  variable "data" is the same order that they 
;		  appear in "dset_arr"
;               * If "-1" is passed, the whole file is read
;                 and the dset_arr returned is the "indgen(ndset)"
;OPTIONAL KEYWORD INPUT:
;	nodata  : If present, only the index is read
;	start_end: Optionally read only a portion of the image.
;		  For SXT:	START_END is a 2-element array specifying 
;				the absolute FRE IDL coordinates of the
;				start and the number of lines to read.
;				It reads full strips (all columns)
;OUTPUT:
;	index	: data-index logical record (one for each 
;		  image requested)
;	data	: 2D or 3D image array
;	extra	: Only applicable for BCS and ATT files.  If the call
;		  includes a variable name here, then:
;			For BCS, the DP_SYNC records are returned
;			For ATT, the HXA_SCAN records are returned
;OPTIONAL OUTPUT:
;	roadmap	: summary of data-index logical record, note
;		  all roadmaps records in the file are returned.
;		  In the case where "infil" is an array, roadmap
;		  is the concatenated roadmap for all files that
;		  are being read for the particular extraction.
;OPTIONAL KEYWORD OUTPUT:
;       filidx  : a vector the same length as the roadmap
;                 with the index of the file associated with that entry
;                 (ie: 0,0,0,0,1,1,1,1,2,2,2,2,...)
;	dset_str: the structure form of the requested datasets 
;		  passed in with "dset_arr".  If user passes in
;		  a simple vector ("SS"), "dset_str" is the structure
;		  form of the output.
;EXAMPLES:
;	rd_xda, filename, indgen(10), index, data, roadmap
;	returns 1st 10 images in file=filename
;RESTRICION:
;	CAUTION: If the filename list is long and RD_XDA is going to
;	be called many times getting only a small number of data sets,
;	then it is recommended to use the structure option for "dset_arr"
;	(call MK_DSET_STR externally) because RD_FHEADER is called for
;	all files each time RD_XDA is called when not using the 
;	structure option for "dset_arr".  
;
;	Also, if RD_XDA is going to be called many times with the same
;	"infil" is is good to use the roadmap parameter.  Since the
;	roadmap will match the previous read made by RD_ROADMAP, it
;	will not be read again and this will save time
;HISTORY:
;	written by Mons Morrison, Fall 90.
;	11-Nov-91 MDM - Added option to not read the data
;	12-Nov-91 MDM - added the option to read the whole
;			file by passing a -1
;	 7-Dec-91 MDM - added option to pass a vector of filenames
;			as well as the "filidx" parameter
;	19-Dec-91 MDM - Changed "roadmap" reading to only read the
;			files that are being extracted from.
;	10-Jan-92 MDM - Moved "idset" increment command outside of
;			the read data option so that the /nodata
;			option would work properly (fixed bug)
;	10-Jan-91 MDM - Allowed user to pass "-1" for datasets with
;			an array of file names.
;	28-Feb-92 MDM - Changed calling sequence to allow "dset_arr"
;			to be a simple integer array ("SS") or a structure
;			with .DSET and .IFIL.  "SS" option can be used 
;			even if infil is an array (previously restricted
;			to turn it into a structure in external routine)
;			Uses MK_DSET_STR internally.
;			Also added "dset_str" optional output capability
;	 6-Mar-92 MDM - Adjusted the documentation header some
;       19-Mar-92 MDM - Changed not to use the "roadmap2" method of getting
;                       the byte offset.  Instead, uses the full roadmap.
;                       This is to speed up the reading if reading the same
;                       file multiple times.
;                       Also, changed call to RD_INDEX to pass the byte offset
;                       so it does not have to read RD_ROADMAP
;	20-Mar-92 MDM - Changed so output structure version is not updated if
;			the different structure is switching to an OLDER version
;	21-Mar-92 (MDM) -------------------------------------------------------
;			* Changed from several routines (RD_SDA, RD_BDA, RD_WDA, 
;			  RD_HDA, RD_ADA, and RD_CBA) to be one main driver and
;			  several specialized data extraction routines.
;			* Also allowed the data to be extracted in any order
;			  but to do this, the original implementation where 
;			  the routines only called RD_ROADMAP with the files
;			  that were to have data extract was removed.  This 
;			  should be ok since RD_ROADMAP now avoids reading the
;			  roadmap if the same files are specified two times in
;			  a row.
;	27-Jul-92 (MDM) - Added HXI data extraction capability
;       23-Oct-92 (MDM) - Use STR_COPY_TAGS instead of STR_COPY
;	13-Nov-92 (MDM) - Added BSC data extraction capability
;	19-Apr-93 (MDM) - Added SXL data extraction capability
;	12-Jul-93 (MDM) - Expanded to be able to extract SXT secondary database
;			  files: (SDL, SLD, SLS, SOT, SXC, SXL)
;	11-Oct-93 (MDM) - Expanded to handle SXA
;	 4-Jan-94 (MDM) - Big changes with the index reading (because of 
;			  SXT .HIS records)
;			- Modified to set /NODATA automatically if the data
;			  variable is not included in the RD_XDA call
;			- Added /QSTOP
;	28-Jul-94 (MDM) - Made the loop variable INTEGER*4 so that over
;			  32,000 datasets can be read.
;	30-Nov-94 (MDM) - Modified to add START_END option (to read a portion
;			  of the image)
;	29-Mar-95 (MDM) - Put the 30-Nov-94 modifications online
;			- SXT compression data portion added to RD_SDA_DATA
;-
;
if (n_params(0) lt 4) then nodata=1
;
get_lun, lun
nfiles = n_elements(infil)
;
rd_fheader, infil(0), fheader, ndset
instr = strupcase(fheader(0).st$instrument)
if (instr eq 'HXT') then if (strupcase(fheader(0).st$file_type) eq 'HXI') then instr = 'HXI'
if (instr eq 'BCS') then if (strupcase(fheader(0).st$file_type) eq 'BSC') then instr = 'BSC'
;if (instr eq 'SXT') then if (strupcase(fheader(0).st$file_type) eq 'SXL') then instr = 'SXL'
if (instr eq 'SXT') then begin
    ok_list = ['SDL', 'SLS', 'SLD', 'SOT', 'SXC', 'SXL', 'SXA']
    ss = where(ok_list eq strupcase(fheader(0).st$file_type))
    if (ss(0) ne -1) then begin
	instr = 'STR'
	instr2 = strupcase(fheader(0).st$file_type)
    end
end
;
dset_arr0 = mk_dset_str(infil, dset_arr)	;turn the input into a structure
;TODO - check that the requested files are in fact available - mark with a -1 if not available
ss = where(dset_arr0.ifil ne -1, count)
if (count eq 0) then begin
    message, "All datasets to be extracted access files that don't exist or have no data", /info
    return
end
dset_arr0 = dset_arr0(ss)
nout = n_elements(dset_arr0)
;
rd_roadmap, infil, roadmap, ndset, recsize, filidx=filidx               ;get roadmap for all files that are to be read
if (total(ndset) eq 0) then begin
    print, 'RD_XDA: # of datasets in the file(s) is zero'
    print, 'Returning
    return
end
;
offset = lonarr(nfiles)
for i=1,nfiles-1 do offset(i) = total(ndset(0:i-1))
dset_arr3 = dset_arr0.dset + offset(dset_arr0.ifil)	;indicie in roadmap 

rd_index, infil(dset_arr0(0).ifil), roadmap(dset_arr3(0)).byteSkip, index0      ;;calling seq changed 19-Mar-92

index_ref = index0
for itag=0,n_tags(index_ref)-1 do index_ref.(itag).index_version = 0	;zero out the index version numbers
nindexbyte = index0.gen.nindexbyte
index_version = get_index_ver(index0)
index = replicate(index_ref, nout)
data = 0	;free up previously defined memory
qsimple_copy = 1
;
last_ifil = -1
for idset=0L,nout-1 do begin
    ifil = dset_arr0(idset).ifil
    dset = dset_arr3(idset)

    if (ifil ne last_ifil) then begin
	close, lun
	openr, lun, infil(ifil), /block
	rd_pointer, lun, pointer
	last_ifil = ifil
    end
    ;
    ;---------------------------------------- Read the index portion ------------------------------------
    ibyt = roadmap(dset).byteSkip
    rdwrt, 'R', lun, ibyt, 0, index0, 1
    ;
    ;read the index assuming that the index version was the same.  If it is different, then read it 
    ;again using "rd_index" which makes the structure the proper new structure.  Replicate the new structure
    ;and then copy all of the old fields into the new structure.

    chk_index_version = get_index_ver(index0)
    qnew = 0
    if (total(chk_index_version-index_version) gt 0) then qnew = 1	;only change the output structure type if the change
									;was from a old to a new.  New to old does not
									;require a new output index structure type

    if (keyword_set(q2debug)) then print, chk_index_version, index_version, qnew

    del = index0.gen.nindexbyte - nindexbyte	;MDM added 4-Jan-94
    case 1 of
	del gt 0: qnew = 1	;have come across a .HIS, where started without one ==> new output structure to be defined
	del lt 0: begin		;first dset had .HIS, now have one without .HIS
		    if (keyword_set(qdebug)) then print, 'Simple copy disabled'
		    qsimple_copy = 0
		    rd_index, lun, roadmap(dset).byteSkip, index0
		    nindexbyte = index0.gen.nindexbyte
		  end
	else: 
    endcase

    if (qnew) then begin
	rd_index, lun, roadmap(dset).byteSkip, index0
	index_ref = index0
	for itag=0,n_tags(index_ref)-1 do index_ref.(itag).index_version = 0	;zero out the index version numbers
	index_version = get_index_ver(index0)
	nindexbyte = index0.gen.nindexbyte
	index_tmp = replicate(index_ref, nout)		;make output structure the data type of the latest structure
	for itag=0,n_tags(index_tmp)-1 do index_tmp.(itag).index_version = 0	;zero out the index version numbers
	;for i=0,idset-1 do index_tmp(i) = str_copy_tags(index0, index(i), /vercopy)	;copy old structure into new
	index_tmp(0:idset-1) = str_copy_tags(index_ref, index(0:idset-1), /vercopy)	;copy old structure into new
	if (keyword_set(qdebug)) then print, 'Simple copy Enabled'
	if (keyword_set(qstop)) then stop
	index = index_tmp
	qsimple_copy = 1
    end

    ;---------------------------------------- Update the index array ------------------------------------

    if (qsimple_copy) then index(idset) = index0 $
	  else index(idset) = str_copy_tags(index_ref, index0, /vercopy)	;inserting non-.HIS into .HIS structure

    ;
    ;---------------------------------------- Read the data portion ------------------------------------

    ibyt = roadmap(dset).byteSkip + index(idset).gen.nIndexByte

    if (n_elements(start_end) eq 2) and (instr eq 'SXT') then begin
	nx = gt_shape(index0, /x)
	ny = gt_shape(index0, /y)
	y0 = gt_corner(index0, /y)
	n_fr_skip = long(start_end(0) - y0) > 0
	nskip = long(n_fr_skip / (2^gt_res(index0)))
	dtyp = index0.gen.data_word_type mod 16		;1=byte, 2=integer=2bytes, 4=float=4bytes
	ibyt = ibyt + (nskip*nx*dtyp)
	if (idset eq 0) then new_y_corner = fltarr(nout)
	new_y_corner(idset) = y0 + n_fr_skip
	;
	nlin_out  = start_end(1)
	nlin_read = nlin_out < (ny-nskip)
    end


    qread_data = 1
    if (keyword_set(nodata)) then qread_data = 0
    if (qread_data) then begin
	case instr of
	    'BCS': rd_bda_data, lun, ibyt, idset, nout, index, data, roadmap, dset_arr3
	    'BSC': rd_bsc_data, lun, ibyt, idset, nout, index, data, roadmap, dset_arr3
	    'HXT': rd_hda_data, lun, ibyt, idset, nout, index, data, roadmap, dset_arr3
	    'HXI': rd_hxi_data, lun, ibyt, idset, nout, index, data, roadmap, dset_arr3
	    'SXT': rd_sda_data, lun, ibyt, idset, nout, index, data, roadmap, dset_arr3, nlin_out, nlin_read
	    'WBS': rd_wda_data, lun, ibyt, idset, nout, index, data, roadmap, dset_arr3

	    'CBA': rd_cba_data, lun, ibyt, idset, nout, index, data, roadmap, dset_arr3
	    'ATT': rd_ada_data, lun, ibyt, idset, nout, index, data, roadmap, dset_arr3, pointer

	    'STR': rd_str_data, lun, ibyt, idset, nout, index, data, roadmap, dset_arr3, instr2
	endcase
    end		;if read data

end
;
if (instr eq 'WBS') then wbs_swap, index, data, nodata=nodata
;
if (n_params(0) eq 6) then begin
	case instr of
	    'BCS': rd_bda_dp, infil, extra
	    'ATT': rd_hxa, infil, extra
	    else: message, 'Cannot use the "extra" parameter for this type of file: ' + instr, /info
	endcase
end
;
if (his_exist(index)) then begin	;MDM added 4-Jan-94
    his_index, /enable
    if (keyword_set(qdebug)) then print, 'Checking for non .HIS records'
    ss = where(index.his.index_version eq 0, n)
    if (n ne 0) then begin
	if (keyword_set(qdebug)) then print, 'Found non .HIS records'
	index_tmp = index(ss)
	his_index, index_tmp, /init
	index(ss) = index_tmp
    end
    if (keyword_set(qstop)) then stop
end
;
if (keyword_set(new_y_corner)) then begin
    his_index, /enable
    for i=0,nout-1 do begin
	corner = [gt_corner(index(i), /x), new_y_corner(i)]
	shape  = [gt_shape(index(i), /x), nlin_out]
	his_index, index, i, 'corner_ext', corner
	his_index, index, i, 'corner_sav', corner
	index(i).sxt.shape_sav = shape
    end
end
;
;--- Since we might save this index with SAV_SDA or SAV_BDA, the index versions must be updated to be the
;    index version of the structure which it is stored in
;
if (keyword_set(qstop)) then stop
for itag=0,n_tags(index)-1 do index.(itag).index_version = index_version(itag)	;MDM added 4-Jan-94 since have /VERCOPY in
										;STR_COPY_TAGS above

free_lun, lun
end