;--------------------------------------------------------------------------- ; Document name: RAW2BIN.PRO ; Created by: Dietmar Germerott, MPAE Lindau, November 11, 1998 ; ; Last Modified: Fri Nov 2 23:27:00 2007 (GERMEROTT@sumop1.nascom.nasa.gov) ;--------------------------------------------------------------------------- ; PRO raw2bin, filenames,no_image=no_image,no_hk255=no_hk255,no_hk248=nohk248,$ no_hk249=no_hk249, no_hk250=no_hk250, no_hk251=no_hk251, $ no_hk252=no_hk252, file_spec=file_spec,qklfile=qklfile, $ debug_mode=debug_mode, pcklevel=pcklevel, saverst=saverst, $ out_path=out_path, nohkall=nohkall, noimage=noimage, $ nolog_=nolog_, pcegse_=pcegse_ ;+ ; PROJECT: ; SOHO - SUMER ; ; NAME: ; RAW2BIN ; ; PURPOSE: ; Raw Telemetry processing from SUMOP2 ; ; CATEGORY: ; SUMER Data Processing ; ; EXPLANATION: ; This procedure processes the SC TM-Rawfiles of received SUMER TM ; stored on SUMOP2 ; ; SYNTAX: ; raw2bin, filenames or raw2bin,file='sumer_raw:sc*.*' ; ; EXAMPLES: ; ; INPUTS: ; filenames - a list of files to process ; ; OPTIONAL INPUTS: ; None. ; ; OUTPUTS: ; None. ; ; OPTIONAL OUTPUTS: ; filenames - returns the filelist found if the file_spec keyword was set ; ; KEYWORDS: ; file_spec - file specification to look for building own filelist ; this keyword overwrites the filenames parameter ; no_images - don't write images to files ; no_hk248 - don't write HK248 records to file ; no_hk249 - don't write HK249 records to file ; no_hk250 - don't write HK250 records to file ; no_hk251 - don't write HK251 records to file ; no_hk252 - don't write HK252 records to file ; no_hk255 - don't write HK255 records to file ; nohkall - don't write any Scienc HK ; noimage - don't write images (for HK selection) ; ; file_spec - Filespecification for Raw Data ; qklfile - indicate its a QKL file ; debug_mode- PRINT some more informatio during execution ; pcklevel ; saverst - produced files are restore files ; out_path - directory where to put files ; nolog_ - don't log the last processed file ; pcegse_ - source file is extracted from SUMER PC EGSE Recorder ; ; COMMON: ; None. ; ; RESTRICTIONS: ; None. ; ; SIDE EFFECTS: ; None. ; ; HISTORY: ; Version 1, November 11, 1998, Dietmar Germerott, MPAE Lindau. Written ; Version 1.1, April 13, 1999, Dietmar Germerott, MPAE Lindau ; corrected Byte1 Images (swap) ; Version 1.2, April 12, 2002, Dietmar Germerott, MPAE Lindau ; added more features (save Restore files) and keywords ; Version 1.3, January 22, 2003, Dietmar Germerott, MPAE Lindau ; added support for PC EGSE Recorder extracted raw files ; Version 1.4, April 19, 2006, Dietmar Germerott, MPAE Lindau ; corrected write of hk records for 251 and 248 ; ; CONTACT: ; Dietmar Germerott, MPAE Lindau (germerott@linmpi.mpg.de) ;- ; ; ON_ERROR, 2 IF ((N_ELEMENTS(filenames) EQ 0) AND (NOT KEYWORD_SET(file_spec))) THEN BEGIN MESSAGE,"SYNTAX: RAW2BIN,filelist or RAW2BIN,file_spec='sc*.*'" return ENDIF IF KEYWORD_SET(file_spec) THEN BEGIN filenames = FINDFILE(file_spec, count=fil_cnt) IF fil_cnt EQ 0 THEN BEGIN msginpro = ' No files found for file_spec '+file_spec MESSAGE,msginpro,/cont RETURN ENDIF ENDIF IF KEYWORD_SET(debug_mode) THEN debug = 1 ELSE debug = 0 IF KEYWORD_SET(interval_t) THEN _t_int = interval_t ELSE _t_int = 30. _s_tim = SYSTIME(1) IF KEYWORD_SET(pcklevel) THEN pcklev = LONG(pcklevel) ELSE pcklev = 1000l IF KEYWORD_SET(nohkall) THEN BEGIN no_hk248 = 1; don't write HK248 records to file no_hk249 = 1; don't write HK249 records to file no_hk250 = 1; don't write HK250 records to file no_hk251 = 1; don't write HK251 records to file no_hk252 = 1; don't write HK252 records to file no_hk255 = 1; don't write HK255 records to file ENDIF ;--------------------------------------------------------------------------- ; Setup Variables ;--------------------------------------------------------------------------- ff = filenames s_flag = 0 iflag = 0 hk255s = 26 hk248s = 72 hk249s = 72 hk250s = 38 hk251s = 26 hk252s = 56 hk200s = 404 i_heads = 92 hkmax = 10000 hk255a = BYTARR(hk255s,hkmax) hk255cnt = 0l hk248a = BYTARR(hk248s,hkmax) hk248cnt = 0l hk249a = BYTARR(hk249s,hkmax) hk249cnt = 0l hk250a = BYTARR(hk250s,hkmax) hk250cnt = 0l hk251a = BYTARR(hk251s,hkmax) hk251cnt = 0l hk252a = BYTARR(hk252s,hkmax) hk252cnt = 0l all_img_cnt = 0l img_cnt = LONARR(50) img_cnt(*) = 0l badimg_cnt = LONARR(50) badimg_cnt(*) = 0l qkl_f = 0 qkl_split = 0 qkl_pos = 0l ;--------------------------------------------------------------------------- ; Creating last processed file ;--------------------------------------------------------------------------- last_file = concat_dir('cd_image_loc','lastprocfile.txt') flastfile = FINDFILE(last_file, count=flnum) IF flnum LE 0 THEN BEGIN OPENW, uout, last_file, /GET_LUN PRINTF, uout, 'NOFILE' FREE_LUN, uout ENDIF ;--------------------------------------------------------------------------- ; Processing files ;--------------------------------------------------------------------------- FOR n=0, N_ELEMENTS(ff)-1 DO BEGIN ;loop over files PRINT,'processing file nr',n,' ',ff(n),' => ',FIX(100./N_ELEMENTS(ff)*n),' %' break_file,ff(n),d_log,d_dir,d_fnam,d_ext IF STRPOS(STRUPCASE(d_ext),'QKL') GE 0 THEN qkl_f = 1 ELSE qkl_f = 0 qkl_file_label: CASE 1 OF (KEYWORD_SET(qklfile) OR qkl_f):BEGIN pcksize = 416l IF debug THEN PRINT,ff(n) OPENR, uin, ff(n), /stream, /GET_LUN IF qkl_split THEN BEGIN IF rempcknum GT pcklev THEN BEGIN pcknum = pcklev rempcknum = rempcknum-pcklev qkl_split = 1 ;still 1 IF debug THEN PRINT,' Remaining Packkets: ',rempcknum ENDIF ELSE BEGIN pcknum = rempcknum qkl_split = 0 ENDELSE ENDIF ELSE BEGIN tmp = STRARR(9) READF, uin, tmp IF STRPOS(tmp(4), 'NUM_PACK') LT 0 THEN BEGIN FREE_LUN, uin GOTO, file_error ENDIF IF debug THEN PRINT, tmp(4) yy = EXECUTE(tmp(4)) totpcknum = LONG(num_pack) rempcknum = totpcknum IF debug THEN pRINT, totpcknum POINT_LUN, -uin, qkl_pos IF debug THEN PRINT, qkl_pos IF rempcknum GT pcklev THEN BEGIN pcknum = pcklev rempcknum = rempcknum-pcklev qkl_split = 1 ;set flag ENDIF ELSE BEGIN pcknum = rempcknum ENDELSE ENDELSE IF debug THEN PRINT,'Offset = ',qkl_pos a = ASSOC(uin, BYTARR(pcksize, pcknum), qkl_pos) r_pck = a(0) ;IF debug THEN PRINT,r_pck(0:19,0) qkl_pos = qkl_pos + N_ELEMENTS(r_pck); cannot be done by point_lun IF debug THEN PRINT,' Next Pos to read: ',qkl_pos IF debug THEN PRINT, ' loading done ...' FREE_LUN, uin nn = WHERE(r_pck(0, *) NE '88'xb, ncnt) IF ncnt NE 0 THEN BEGIN PRINT,' Packet Error - at least',ncnt,' packets bad' GOTO,file_error ENDIF x_pck = r_pck(12:415,*) & r_pck=0 IF debug THEN PRINT,' extracting done ...' END (KEYWORD_SET(pcegse_)):BEGIN pcksize = 416l r_pck = BYTARR(pcksize,1) r_pck(*) = 0b ON_IOERROR, skip_this_2 OPENR, uin, ff(n), /stream, /GET_LUN ss = FSTAT(uin) pcknum = ss.size/pcksize a = ASSOC(uin, BYTARR(pcksize, pcknum)) r_pck = a(0) IF NOT KEYWORD_SET(nolog_) THEN BEGIN OPENu, uout, last_file, /GET_LUN break_file,ff(n),dvnam,dxnam,fxnam,exnam PRINTF, uout, fxnam+exnam FREE_LUN, uout ENDIF FREE_LUN, uin skip_this_2: nn = WHERE (r_pck(0,*) NE '88'xb,ncnt) IF ncnt NE 0 THEN BEGIN PRINT,' Packet Error - at least',ncnt,' packets bad' GOTO,file_error ENDIF x_pck = r_pck(12:415, *) & r_pck=0 END ELSE:BEGIN ;normal raw tm file (sumop2) pcksize = 424l r_pck = BYTARR(pcksize,1) r_pck(*) = 0b IF debug THEN PRINT,' Packetsize = ',pcksize ON_IOERROR, skip_this OPENR, uin, ff(n), /stream, /GET_LUN ss = FSTAT(uin) pcknum = ss.size/pcksize a = ASSOC(uin, BYTARR(pcksize, pcknum)) r_pck = a(0) IF NOT KEYWORD_SET(nolog_) THEN BEGIN OPENu, uout, last_file, /GET_LUN PRINTF, uout, ff(n) FREE_LUN, uout ENDIF FREE_LUN, uin skip_this: nn = WHERE (r_pck(2,*) NE '88'xb,ncnt) IF ncnt NE 0 THEN BEGIN PRINT,' Packet Error - at least',ncnt,' packets bad' GOTO,file_error ENDIF x_pck = r_pck(14:417, *) & r_pck=0 END ENDCASE x_pck = REFORM(x_pck, 404l*pcknum) ;make one dimension ;IF debug THEN PRINT,'x_pck = ',x_pck(0:20) IF s_flag EQ 0 THEN BEGIN s_flag = 1 w_pck = x_pck ENDIF ELSE w_pck = [w_pck, x_pck] x_pck = 0 ;save mem esync = WHERE(w_pck EQ 'eb'xb) xa_ptr = 0 a_ptr = esync(xa_ptr) w_pcklen = N_ELEMENTS(w_pck) enough = 1 IF a_ptr LT 0 THEN a_ptr = w_pcklen-4 ;set to end WHILE enough DO BEGIN ;IF debug THEN PRINT,'at begin ',w_pck(a_ptr:a_ptr+4) IF debug THEN BEGIN _p_tim = SYSTIME(1) -_s_tim IF _p_tim GE _t_int THEN BEGIN PRINT,' A-PTR at ',SYSTIME(),' ->',a_ptr _s_tim = SYSTIME(1) ENDIF ENDIF IF w_pck(a_ptr) NE 'eb'xb THEN GOTO ,async_label ;maybe wrong data CASE w_pck(a_ptr+1) OF ;SYNC check (EBh90h) '90'xb:BEGIN CASE w_pck(a_ptr+2) OF '81'xb:BEGIN ;HK processing CASE w_pck(a_ptr+3) OF 255b:BEGIN ;HK255 enough = check_enough(a_ptr, w_pcklen, hk255s) IF enough THEN BEGIN hk255a(*,hk255cnt) = w_pck(a_ptr:a_ptr+hk255s-1) hk255cnt = hk255cnt+1 a_ptr = a_ptr +hk255s ;write array if filled, and reset IF hk255cnt GE hkmax THEN BEGIN PRINT, 'counter ',hk255cnt IF NOT KEYWORD_SET(no_hk255) THEN $ write_hkbydate, hk255a(*,0:hk255cnt-1) hk255cnt = 0l ;reset counter hk255a(*,*) = 0b ;reset array to 0 ENDIF ENDIF END 248b:BEGIN ;HK248 PRINT,'HK 248' enough = check_enough(a_ptr, w_pcklen, hk248s) IF enough THEN BEGIN hk248a(*,hk248cnt) = w_pck(a_ptr:a_ptr+hk248s-1) hk248cnt = hk248cnt+1 a_ptr = a_ptr +hk248s ;write array if filled, and reset IF hk248cnt GE hkmax THEN BEGIN IF NOT KEYWORD_SET(no_hk248) THEN $ write_hkbydate, hk248a(*,0:hk248cnt-1) hk248cnt = 0l ;reset counter hk248a(*,*) = 0b ;reset array to 0 ENDIF ENDIF END 249b:BEGIN ;HK249 PRINT,'HK 249' enough = check_enough(a_ptr, w_pcklen, hk249s) IF enough THEN BEGIN hk249a(*,hk249cnt) = w_pck(a_ptr:a_ptr+hk249s-1) hk249cnt = hk249cnt+1 a_ptr = a_ptr +hk249s ;write array if filled, and reset IF hk249cnt GE hkmax THEN BEGIN IF NOT KEYWORD_SET(no_hk249) THEN $ write_hkbydate, hk249a(*,0:hk249cnt-1) hk249cnt = 0l ;reset counter hk249a(*,*) = 0b ;reset array to 0 ENDIF ENDIF END 250b:BEGIN ;HK250 PRINT,'HK 250' enough = check_enough(a_ptr, w_pcklen, hk250s) IF enough THEN BEGIN hk250a(*,hk250cnt) = w_pck(a_ptr:a_ptr+hk250s-1) hk250cnt = hk250cnt+1 a_ptr = a_ptr +hk250s ;write array if filled, and reset IF hk250cnt GE hkmax THEN BEGIN IF NOT KEYWORD_SET(no_hk250) THEN $ write_hkbydate, hk250a(*,0:hk250cnt-1) hk250cnt = 0l ;reset counter hk250a(*,*) = 0b ;reset array to 0 ENDIF ENDIF END 251b:BEGIN ;HK251 PRINT,'HK 251' enough = check_enough(a_ptr, w_pcklen, hk251s) IF enough THEN BEGIN hk251a(*,hk251cnt) = w_pck(a_ptr:a_ptr+hk251s-1) hk251cnt = hk251cnt+1 a_ptr = a_ptr +hk251s ;write array if filled, and reset IF hk251cnt GE hkmax THEN BEGIN IF NOT KEYWORD_SET(no_hk251) THEN $ write_hkbydate, hk251a(*,0:hk251cnt-1) hk251cnt = 0l ;reset counter hk251a(*,*) = 0b ;reset array to 0 ENDIF ENDIF END 252b:BEGIN ;HK252 PRINT,'HK 252' enough = check_enough(a_ptr, w_pcklen, hk252s) IF enough THEN BEGIN hk252a(*,hk252cnt) = w_pck(a_ptr:a_ptr+hk252s-1) hk252cnt = hk252cnt+1 a_ptr = a_ptr +hk252s ;write array if filled, and reset IF hk252cnt GE hkmax THEN BEGIN IF NOT KEYWORD_SET(no_hk252) THEN $ write_hkbydate, hk252a(*,0:hk252cnt-1) hk252cnt = 0l ;reset counter hk252a(*,*) = 0b ;reset array to 0 ENDIF ENDIF END 200b:BEGIN ;idle packets enough = check_enough(a_ptr, w_pcklen, hk200s) IF enough THEN BEGIN ;print,' hk200b',a_ptr ;print,w_pck(a_ptr:a_ptr+hk200s) a_ptr = a_ptr +hk200s ;print,' hk200e',a_ptr,w_pck(a_ptr:a_ptr+3) ENDIF END ELSE:a_ptr = a_ptr+1;make async END END '80'xb:BEGIN ;Image Header IF iflag THEN BEGIN PRINT, 'saving incomplete image' ;image header GOTO, save_label ENDIF enough = check_enough(a_ptr, w_pcklen, i_heads) IF enough THEN BEGIN all_img_cnt = all_img_cnt + 1 iflag = 1 header_data = BYTARR(100) header_data(92) = [255b, 255b] ;fid=-1 header_data(0) = w_pck(a_ptr:a_ptr+i_heads-1) IF debug THEN print,'Image typ:',header_data(3) img_info = tki_image_form(header_data(3)) image_data = BYTARR(img_info.num, img_info.len) ii_cnt = BYTARR(img_info.num) & ii_cnt(*)=0b a_ptr = a_ptr +i_heads ENDIF END ELSE:BEGIN ;image rows if in Image mode (iflag=1) IF iflag THEN BEGIN enough = check_enough(a_ptr, w_pcklen, a_ptr+4+img_info.len-1) IF enough THEN BEGIN row_num = w_pck(a_ptr+2)*256+w_pck(a_ptr+3) ; PRINT, 'image row ', row_num IF ((row_num LT img_info.num) AND (row_num GE 0)) THEN BEGIN ii_cnt(row_num) = 1b tmp_ptr = a_ptr+4+img_info.len-1 ; PRINT,w_pck(a_ptr:tmp_ptr) image_data(row_num,*) = w_pck(a_ptr+4:tmp_ptr) a_ptr = tmp_ptr+1 IF row_num EQ img_info.num-1 THEN BEGIN save_label: ;don't save if keyword no_image is set IF NOT(KEYWORD_SET(no_image)) THEN BEGIN IF img_info.type EQ 'B1' THEN BEGIN ;swap B1 images tmp_ev = INDGEN(img_info.len/2)*2 tmp_od = tmp_ev+1 tmp_data = image_data tmp_data(*, tmp_ev) = image_data(*, tmp_od) ;corrected tmp_ev to tmp_od 19990413 tmp_data(*, tmp_od) = image_data(*, tmp_ev) image_data = tmp_data & tmp_data=0 ENDIF ;save image here img_cnt(header_data(3)) = img_cnt(header_data(3))+1 tmp = WHERE(ii_cnt NE 1, tcnt) IF tcnt GT 0 THEN BEGIN ;reporting some informations badimg_cnt(header_data(3)) = badimg_cnt(header_data(3))+1 PRINT, 'missing rows' header_data(98) = 1b ;QAC = 1 header_data(99) = 1b ;MDU = 1 ENDIF IF NOT KEYWORD_SET(noimage) THEN BEGIN IF KEYWORD_SET(saverst) THEN BEGIN OPENW, uouttmp, 'sys$login:tmp_123456xxx.tmp', /delete, /GET_LUN WRITEU, uouttmp, TRANSPOSE(image_data) POINT_LUN, uouttmp, 0 ;--------------------------------------------------------------------------- ; Build Image format dependend Array ;--------------------------------------------------------------------------- CASE img_info.type OF 'B1': data1 = BYTARR(img_info.z, img_info.y) 'B2': data1 = INTARR(img_info.z, img_info.y) 'I2': data1 = INTARR(img_info.z, img_info.y) 'R4': data1 = FLTARR(img_info.z, img_info.y) 'I4': data1 = LONARR(img_info.z, img_info.y) ELSE:BEGIN msg_infunc = ' Image Error - Wrong Imageformat: '+STRTRIM(STRING(FIX(iheader(3))), 2) FREE_LUN, uouttmp GOTO, handle_tmp_error END ENDCASE READU, uouttmp, data1 FREE_LUN, uouttmp IF img_info.type EQ 'I4' THEN BEGIN ;special action for history mem BYTEORDER, /lswap, data1 BYTEORDER, /sswap, data1 ENDIF ;--------------------------------------------------------------------------- ; Convert Data to machine specific format ; and rotate (in binaryfile the data is stored column by column) ;--------------------------------------------------------------------------- ieee_to_host, data1 CASE img_info.type OF 'B2': data1 = fix2long(data1) 'I2': data1 = fix2long(data1) ELSE: ENDCASE data1 = TRANSPOSE(data1) sum_save_image, header_data, data1, saverst=saverst, out_path=out_path handle_tmp_error: ENDIF ELSE BEGIN image_data = TRANSPOSE(image_data) dt = utc_head(header_data, /ext) o_filename = STRING(format=$ '("cd_image_loc:sum_",i4.4,"-",i2.2,"-",i2.2,"_xx.img")', $ dt.year, dt.month, dt.day) ;xx - indicates from raw TM of SUMOP2 OPENW, uout, o_filename, /append, /stream, /GET_LUN WRITEU, uout, header_data, image_data FREE_LUN, uout ENDELSE ENDIF ENDIF iflag = 0 ENDIF ENDIF ELSE a_ptr = a_ptr + 1 ;make async ENDIF ENDIF ELSE a_ptr = a_ptr + 1 ;make async END END END ELSE:BEGIN ;not a sync async_label: PRINT, 'async',a_ptr,' ', w_pck(a_ptr:a_ptr+3) a_ptr = a_ptr +1 xa_ptr = WHERE(esync GE a_ptr, tcnt) xa_ptr = xa_ptr(0) IF tcnt GT 0 THEN BEGIN a_ptr = esync(xa_ptr(0)) ENDIF ELSE BEGIN enough = 0 ENDELSE ;stop END END IF enough THEN enough = check_enough(a_ptr, w_pcklen, 4) ;check for first 4 bytes of sync ENDWHILE w_pck = w_pck(a_ptr:*) IF qkl_split THEN GOTO, qkl_file_label PRINT,all_img_cnt file_error: ENDFOR tmp_cnt = WHERE(img_cnt GT 0,cnt) FOR n=0,cnt-1 DO BEGIN PRINT,' Format ',tmp_cnt(n),' = ',img_cnt(tmp_cnt(n)),' Images ',badimg_cnt(tmp_cnt(n)),' bad' ENDFOR ;write cached HK data IF ((hk255cnt GT 0) AND NOT(KEYWORD_SET(no_hk255))) THEN BEGIN PRINT,' writing',hk255cnt,' HK255 packets' hk255a = hk255a(*, 0:hk255cnt-1) write_hkbydate,hk255a ENDIF IF ((hk248cnt GT 0) AND (NOT (KEYWORD_SET(no_hk248)))) THEN BEGIN PRINT,' writing',hk248cnt,' HK248 packets' hk248a = hk248a(*, 0:hk248cnt-1) write_hkbydate,hk248a ENDIF IF ((hk249cnt GT 0) AND NOT(KEYWORD_SET(no_hk249))) THEN BEGIN PRINT,' writing',hk249cnt,' HK249 packets' hk249a = hk249a(*, 0:hk249cnt-1) write_hkbydate,hk249a ENDIF IF ((hk250cnt GT 0) AND NOT KEYWORD_SET(no_hk250)) THEN BEGIN PRINT,' writing',hk250cnt,' HK250 packets' hk250a = hk250a(*, 0:hk250cnt-1) write_hkbydate,hk250a ENDIF IF ((hk251cnt GT 0) AND NOT KEYWORD_SET(no_hk251)) THEN BEGIN PRINT,' writing',hk251cnt,' HK251 packets' hk251a = hk251a(*, 0:hk251cnt-1) write_hkbydate,hk251a ENDIF IF ((hk252cnt GT 0) AND NOT KEYWORD_SET(no_hk252)) THEN BEGIN PRINT,' writing',hk252cnt,' HK252 packets' hk252a = hk252a(*, 0:hk252cnt-1) write_hkbydate,hk252a ENDIF RETURN END ;--------------------------------------------------------------------------- ; End of 'RAW2BIN.PRO'. ;---------------------------------------------------------------------------