Pro Hsi_contact2fits, infile1, infile2, $
Outfiles = outfiles, Out_plot_files = out_plot_files, $
orbit_directory = orbit_directory, $
state_vector_dir = state_vector_dir, $
quiet = quiet, Output_dir = output_dir, $
Filedb_dir = filedb_dir, init_filedb = init_filedb, $
No_filedb = no_filedb, Itos = itos, Xxx = xxx, $
max_packets = max_packets, $
min_packets = min_packets, $
All_flare = all_flare, File_type = file_type, $
Ext_itos = ext_itos, $
alt_smex = alt_smex, $
One_file = one_file, Smex = smex, $
Time_test_binsize = time_test_binsize, $
Time_test_minrate = time_test_minrate, $
Clk_delta_dir = clk_delta_dir, $
do_fits = do_fits, archive = archive, $
do_catalog = do_catalog, $
no_bad_pak = no_bad_pak, $
ignore_orbit = ignore_orbit, $
time_intv = time_intv, $
eph_time_intv = eph_time_intv, $
as_time_intv = as_time_intv, $
ref_time = ref_time, $
dont_copy_old_files = dont_copy_old_files, $
old_level0_dir = old_level0_dir, $
no_daily_catalog = no_daily_catalog, $
catalog_dir = catalog_dir, $
yes_qlook_aspect = yes_qlook_aspect, $
reprocess_smex = reprocess_smex, $
no_check_newpak = no_check_newpak, $
only_this_file = only_this_file, $
flare_position_lvl0 = flare_position_lvl0, $
arch_start_time = arch_start_time, $
arch_stop_time = arch_stop_time, $
no_files_done = no_files_done, $
do_img_spc_too = do_img_spc_too, $
long_archive = long_archive, $
mult_files = mult_files, $ ;set this if this is not the last of multiple files processed
_extra = _extra
;all packets and collect times for the contact files
Common all_packets, pakf, timf, badf
Common hsi_clock_drift_block, drift_times, drift, drift_zero
set_plot, 'Z' ;set this for batch mode
message, /info, 'Version 2018-05-29'
;handle files here,
infiles = infile1
IF(datatype(infile2) EQ 'STR') THEN infiles = [infiles, infile2]
If(n_elements(infiles) Gt 1) Then Begin ;this should only happen for BGS contacts
nn = n_elements(infiles)
For j = 0, nn-1 Do Begin
If(j Lt nn-1) Then not_last_file = 1b Else not_last_file = 0b
Hsi_contact2fits, infiles(j), $
Outfiles = outfiles, $
Out_plot_files = out_plot_files, $
orbit_directory = orbit_directory, $
state_vector_dir = state_vector_dir, $
quiet = quiet, Output_dir = output_dir, $
Filedb_dir = filedb_dir, init_filedb = init_filedb, $
No_filedb = no_filedb, Itos = itos, Xxx = xxx, $
max_packets = max_packets, $
min_packets = min_packets, $
All_flare = all_flare, File_type = file_type, $
Ext_itos = ext_itos, $
alt_smex = alt_smex, $
One_file = one_file, Smex = smex, $
Time_test_binsize = time_test_binsize, $
Time_test_minrate = time_test_minrate, $
Clk_delta_dir = clk_delta_dir, $
do_fits = do_fits, archive = archive, $
do_catalog = do_catalog, $
no_bad_pak = no_bad_pak, $
ignore_orbit = ignore_orbit, $
time_intv = time_intv, $
eph_time_intv = eph_time_intv, $
as_time_intv = as_time_intv, $
ref_time = ref_time, $
dont_copy_old_files = dont_copy_old_files, $
old_level0_dir = old_level0_dir, $
no_daily_catalog = no_daily_catalog, $
catalog_dir = catalog_dir, $
yes_qlook_aspect = yes_qlook_aspect, $
reprocess_smex = reprocess_smex, $
no_check_newpak = no_check_newpak, $
only_this_file = only_this_file, $
flare_position_lvl0 = flare_position_lvl0, $
arch_start_time = arch_start_time, $
arch_stop_time = arch_stop_time, $
no_files_done = no_files_done, $
do_img_spc_too = do_img_spc_too, $
long_archive = long_archive, $
mult_files = not_last_file, $
_extra = _extra
Endfor
Return
Endif
;hardwire all env variables
setenv, 'SKIP_NETWORK_SEARCH=1' ;never look for files
set_logenv, 'HSI_ARCHIVE_MOUNTED', 'true' ;never look for files except where they belong
IF(NOT keyword_set(xxx)) THEN BEGIN
If(keyword_set(archive) Or Keyword_set(do_catalog)) Then Begin
set_logenv, 'HSI_DATA_DIR', '/disks/hessidata'
set_logenv, 'HSI_DATA_USER', '/disks/hessidata'
set_logenv, 'HSI_USER_DATA', '/disks/hessidata'
set_logenv, 'HSI_FILEDB_OLD', $
'/disks/sunny/raid2/dbase_archive'
set_logenv, 'HSI_CATALOG_DIR', $
'/disks/hessidata/metadata/catalog'
set_logenv, 'HSI_CATALOG_ARCHIVE', $
'/disks/hessidata/metadata/catalog'
set_logenv, 'HSI_OLD_LEVEL0_DIR', $
'/disks/hessidata/old_level0'
set_logenv, 'HSI_QLOOK_PLOT_DIR', $
'/disks/hessidata/metadata/latest'
set_logenv, 'HSI_FILEDB_PUB', $
'/disks/hessidata/dbase'
set_logenv, 'HSI_CLK_DELTA_DIR', $
'/disks/sunny/home/dbase'
set_logenv, 'DIR_GEN_GEV', '/sswdb/gev'
set_logenv, 'DIR_GEN_NAR', '/sswdb/nar'
Endif Else Begin
set_logenv, 'HSI_DATA_DIR', $
'/disks/sunny/home/raid_queue'
set_logenv, 'HSI_DATA_USER', $
'/disks/sunny/home/raid_queue'
set_logenv, 'HSI_USER_DATA', $
'/disks/sunny/home/raid_queue'
set_logenv, 'HSI_FILEDB_OLD', $
'/disks/sunny/raid2/dbase_archive'
set_logenv, 'HSI_CLK_DELTA_DIR', $
'/disks/sunny/home/dbase'
set_logenv, 'HSI_CATALOG_DIR', $
'/disks/hessidata/metadata/catalog'
set_logenv, 'HSI_CATALOG_ARCHIVE', $
'/disks/hessidata/metadata/catalog'
set_logenv, 'HSI_OLD_LEVEL0_DIR', $
'/disks/hessidata/old_level0'
set_logenv, 'HSI_QLOOK_PLOT_DIR', $
'/disks/hessidata/metadata/latest'
set_logenv, 'HSI_FILEDB_PUB', $
'/disks/hessidata/dbase'
Endelse
set_logenv, 'HSI_FILEDB_ARCHIVE', $
'/disks/sunny/home/dbase'
set_logenv, 'HSI_DATA_ARCHIVE', $
'/disks/hessidata'
set_logenv, 'HSI_FLARE_LIST_ARCHIVE', $
'/disks/sunny/home/dbase'
set_logenv, 'HSI_FILEDB_DIR', $
'/disks/sunny/home/dbase'
set_logenv, 'HSI_FDF_DIR', $
'/disks/sunny/home/workdir/SatTrack'
set_logenv, 'HSI_PREDICTED_DIR', $
'/disks/sunny/home/workdir/SatTrack'
set_logenv, 'HSI_TRACK_ARCHIVE', $
'/disks/sunny/raid2/bfds/Products/HESSI'
set_logenv, 'HSI_VC0_INPUT_DIR', $
'/disks/sunstruck/raid2/temporary_files/timetest'
ENDIF
no_files_done = 0b
lvl0 = 0 ;do all of the qlooks
IF(KEYWORD_SET(do_fits)) THEN BEGIN
fits_only = 1b
catalog_only = 0b
IF(keyword_set(no_check_newpak)) THEN check_newpak = 0 $
ELSE check_newpak = 1b
ENDIF ELSE IF(keyword_set(do_catalog)) THEN BEGIN
fits_only = 0b
catalog_only = 1b
check_newpak = 1b
ENDIF ELSE IF(keyword_set(archive)) THEN BEGIN
fits_only = 0b
catalog_only = 1b
check_newpak = 1b
ENDIF ELSE IF(keyword_set(reprocess_smex)) THEN BEGIN
fits_only = 1b
catalog_only = 0b
check_newpak = 0b
ENDIF ELSE BEGIN
fits_only = 1b ;21-Apr-2003, jmm
catalog_only = 0b
IF(keyword_set(no_check_newpak)) THEN check_newpak = 0 $
ELSE check_newpak = 1b
ENDELSE
;default check_newpak to 0 for now, 10-apr-2002, jmm
check_newpak = 0b
;default lvl0 to 1 for now, 10-apr-2002, jmm
; lvl0 = 1
lvl0 = 0 ;21-apr-2003, jmm
;add do flare position temporarily for sims, 15-apr-2002
IF(n_elements(flare_position_lvl0) GT 0) THEN BEGIN
flr_lvl0 = flare_position_lvl0
ENDIF ELSE flr_lvl0 = 0 ;default, 19-sep-2002, new default, 21-apr-2003
;default, 16-feb-2002
If(keyword_set(archive) Or keyword_set(do_catalog)) Then Begin
dont_copy_old_files = 1
Endif Else dont_copy_old_files = 0
;You need a minimum file time, the longest interval with respect to
;either the rates, ephemeris, or aspect (usually eph), scale
;everything to time_intv (which represents the spin period)
IF(NOT keyword_set(time_intv)) THEN time_intv = 4.0d0
IF(NOT keyword_set(as_time_intv)) THEN as_time_intv = 20.0d0
IF(NOT keyword_set(eph_time_intv)) THEN eph_time_intv = 20.0d0
IF(eph_time_intv GE time_intv) THEN BEGIN ;scale
n1 = round(eph_time_intv/time_intv)
eph_time_intv = n1*time_intv
ENDIF ELSE BEGIN
n1 = round(time_intv/eph_time_intv)
eph_time_intv = time_intv/n1
ENDELSE
IF(as_time_intv GE time_intv) THEN BEGIN ;scale
n1 = round(as_time_intv/time_intv)
as_time_intv = n1*time_intv
ENDIF ELSE BEGIN
n1 = round(time_intv/as_time_intv)
as_time_intv = time_intv/n1
ENDELSE
file_time_intv = max([time_intv, eph_time_intv, as_time_intv])
IF(NOT keyword_set(ref_time)) THEN ref_time = anytim('4-jul-2000 00:00')
;Close all files!
free_all_lun
qquiet = KEYWORD_SET(quiet)
outfiles = -1
IF(KEYWORD_SET(max_packets)) THEN maxpak = long(max_packets) $
ELSE maxpak = 100000l
IF(KEYWORD_SET(min_packets)) THEN minpak = long(min_packets) $
ELSE minpak = 0l
IF(KEYWORD_SET(filedb_dir)) THEN fdbdir = filedb_dir $
ELSE fdbdir = '$HSI_FILEDB_DIR'
fdbdir_pub = '$HSI_FILEDB_PUB'
IF(KEYWORD_SET(output_dir)) THEN outdir = output_dir $
ELSE outdir = '$HSI_DATA_DIR'
IF(NOT KEYWORD_SET(one_file)) THEN one_file = 0b
IF(keyword_set(old_level0_dir)) THEN dir_old = old_level0_dir $
ELSE dir_old = '$HSI_OLD_LEVEL0_DIR'
;Input the filedb structure, for checking for files with the same
;orbit, and the catalog stuff
IF(KEYWORD_SET(no_filedb)) THEN check_filedb = 0b ELSE BEGIN
;If arch_start_time and arch_stop time are set -- only input filedbs
; in that time range
IF(KEYWORD_SET(archive)) THEN BEGIN ;set start and end times here for normal processing
IF(NOT keyword_set(arch_start_time)) THEN BEGIN
arch_start_time = anytim(!stime)-14.0*24.0*3600.0 ;only do the last 14 days, jmm, 2017-05-17
ENDIF ELSE arch_start_time = anytim(arch_start_time)
IF(NOT keyword_set(arch_stop_time)) THEN BEGIN
arch_stop_time = anytim(!stime)+8.0*3600.0 ;Be sure to test in UT
ENDIF ELSE arch_stop_time = anytim(arch_stop_time)
ENDIF
;Now input the filedbs
IF(KEYWORD_SET(arch_start_time) AND KEYWORD_SET(arch_stop_time)) THEN BEGIN
filedb = hsi_mult_filedb_inp(filedb_dir = fdbdir, $
file_type = 'LEVEL0', quiet = quiet, $
dont_copy_old_files = dont_copy_old_files, $
qfiledb_trange = qfiledb_trange, $
filedb_trange = [arch_start_time, arch_stop_time], $
/months_only)
ENDIF ELSE BEGIN
filedb = hsi_mult_filedb_inp(filedb_dir = fdbdir, $
file_type = 'LEVEL0', quiet = quiet, $
dont_copy_old_files = dont_copy_old_files, $
qfiledb_trange = qfiledb_trange)
ENDELSE
IF(datatype(filedb) EQ 'STC') THEN check_filedb = 1 $
ELSE check_filedb = 0
filedb_old = filedb
ENDELSE
;For the catalog only option
IF(KEYWORD_SET(archive) OR KEYWORD_SET(do_catalog)) THEN BEGIN
IF(datatype(infiles) EQ 'STR') THEN outfiles = infiles
GOTO, catalog_skip
ENDIF
;zero clock drift, reset later
hsi_clock_drift, /zero_init_drift
;Here run the clock_drift update, which checks for input files, and
;updates the full-mission clock_drift file. Moved here 17-jul-2006,
;because a fix to hsi_clock_drift_soc for empty files needs a
;free_all_lun
hsi_clock_drift_soc
;Ok, there will be N files input.
n_infiles = N_ELEMENTS(infiles)
;check for file types
IF(KEYWORD_SET(file_type)) THEN BEGIN
ftyp = file_type
IF(N_ELEMENTS(ftyp) NE n_infiles) THEN $
ftyp = replicate(ftyp[0], n_infiles)
ENDIF ELSE BEGIN
IF(KEYWORD_SET(itos)) THEN ftyp = replicate('itos', n_infiles) $
ELSE IF(KEYWORD_SET(smex)) THEN ftyp = replicate('smex', n_infiles) $
ELSE ftyp = replicate('raw', n_infiles)
ENDELSE
;set up database text files:
bad_pak_file0 = concat_dir(fdbdir, 'bad_packet_dbase.txt')
bad_pak_file = loc_file(bad_pak_file0, count = bad_pak_file_count)
IF(bad_pak_file_count EQ 0) THEN BEGIN
IF(NOT qquiet) THEN message, /info, 'Creating file:'+bad_pak_file0
openw, bdunit, bad_pak_file0, /get_lun
printf, bdunit, 'FILE: ', bad_pak_file0
printf, bdunit, 'BIT ERRORS IN VC1, VC3 FILES: '
printf, bdunit, 'FILE', 'N_PACKETS', 'N_BAD_RS', $
'N_BAD_OTHER', 'N_BAD_TOTAL', format = '(a4, a34, 3a14)'
ENDIF ELSE openw, bdunit, bad_pak_file0, /get_lun, /append
vc_tfile0 = concat_dir(fdbdir, 'vc_filetimes.txt')
vc_tfile = loc_file(vc_tfile0, count = vc_tfile_count)
IF(vc_tfile_count EQ 0) THEN BEGIN
IF(NOT qquiet) THEN message, /info, 'Creating file:'+vc_tfile0
openw, vcunit, vc_tfile0, /get_lun
printf, vcunit, 'FILE: ', vc_tfile0
printf, vcunit, 'START AND END TIMES FOR VC1, VC3 FILES: '
printf, vcunit, 'FILE', 'PACKET START_TIME', 'PACKET END_TIME', $
format = '(a4, a48, a28)'
ENDIF ELSE openw, vcunit, vc_tfile0, /get_lun, /append
;Ok, that's it, read the packets into memory from the files
file_count = 0
;Strip directories for ascii output
break_file, infiles, a1, a2, filnam, ext
infiles_0 = filnam+ext
;Get time info from infiles_0 for bad packet tests, 2016-06-15, jmm
If(keyword_set(smex) Or keyword_set(reprocess_smex)) Then Begin
; files are vc?_YYYYMMDDhhmmss.0
ifile_tmp = strmid(infiles_0, 4, 8)+'_'+strmid(infiles_0, 12, 6)
ifile_time = anytim(file2time(ifile_tmp))
Endif Else If(keyword_set(alt_smex)) Then Begin
;files are BGS.HESSI.TLM_VC?_X.YYYY_DOY_hhmmss.dat
ll = strlen('BGS.HESSI.TLM_VC0_X.')
yr = strmid(infiles_0, ll, 4)
doy = strmid(infiles_0, ll+5, 3)
tt = strmid(infiles_0, ll+9, 6)
ut0 = YYDOY_2_UT(yr+doy)
ut0 = strmid(time2file(ut0), 0, 8)+'_'+tt
ifile_time = anytim(file2time(ut0))
Endif Else Begin
message, /info, 'No file times input, can''t use file time for bad packets'
ifile_time = bytarr(n_infiles)
Endelse
FOR j = 0, n_infiles-1 DO BEGIN
smex_in = keyword_set(smex) OR keyword_set(reprocess_smex)
temp_read_all_packets, infiles[j], pak, tim, itos = itos, $
ext_itos = ext_itos, smex_sync_frame = smex_in, $
alt_smex = alt_smex, header_out = header_out
IF(datatype(pak) EQ 'STC') THEN BEGIN
npak = N_ELEMENTS(pak)
IF(NOT qquiet) THEN BEGIN
print, 'FILE:', infiles[j]
print, 'Time Range = ', anytim(minmax(tim), /ccsds)
ENDIF
;check for bad packets
IF(NOT keyword_set(no_bad_pak)) THEN BEGIN
hsi_bad_pak, pak, tim, bad_flag, $
time_test_binsize = time_test_binsize, $
time_test_min_rate = time_test_min_rate, $
quiet = quiet, header = header_out, $
ifile_time = ifile_time[j]
ENDIF ELSE bad_flag = bytarr(npak)
;Write out bad statistics
bad_rs = where(bad_flag GE 100, nbad_rs)
bad_here = where(bad_flag GT 0 AND bad_flag LT 100, nbad_here)
break_file, infiles[j], a1, a2, a3, a4
printf, bdunit, infiles_0[j], npak, nbad_rs, nbad_here, $
nbad_rs+nbad_here, format = '(a24, 4i14)'
not_bad = where(bad_flag EQ 0)
;dump bad packets, by default, jmm, 2016-06-16, keeping bad packets is stupid
IF(NOT qquiet) THEN message, /info, 'Dropping bad packets'
IF(not_bad[0] NE -1) THEN BEGIN
pak = pak[not_bad]
tim = tim[not_bad]
bad_flag = bad_flag[not_bad]
ENDIF ELSE BEGIN
message, /info, 'NO GODD PACKETS IN: '+infiles[j]
pak = -1
tim = -1
bad_flag = -1
ENDELSE
ENDIF
;write out file time, if there are any packets left
IF(datatype(pak) EQ 'STC') THEN BEGIN
t00 = min(tim, max = t01)
printf, vcunit, infiles_0[j], anytim(/ccsds, t00), $
anytim(/ccsds, t01), format = '(a24, 2a28)'
IF(NOT qquiet) THEN help, pak
IF(file_count EQ 0) THEN BEGIN
pakf = temporary(pak)
timf = temporary(tim)
badf = temporary(bad_flag)
file_count = file_count+1
ENDIF ELSE BEGIN
pakf = [temporary(pakf), temporary(pak)]
timf = [temporary(timf), temporary(tim)]
badf = [temporary(badf), temporary(bad_flag)]
file_count = file_count+1
ENDELSE
ENDIF
;Next file
ENDFOR
free_lun, bdunit
free_lun, vcunit
;If no files, , return
IF(file_count EQ 0) THEN BEGIN
message, /info, 'No Good Packets in any files:'
FOR j = 0, n_infiles-1 DO print, infiles[j]
no_files_done = 1b
return
ENDIF
;Now you have packets, sort with time, adp packets will be resorted by
;sequence count in packet2fits.pro
ss = sort(timf)
timf = timf[ss]
pakf = pakf[ss]
badf = badf[ss]
n_packets = N_ELEMENTS(pakf)
;Now the packet array has to be split up into times with respect to orbits
;You need the orbit data
notbad = where(badf EQ 0, nnotbad)
IF(nnotbad EQ 0) THEN BEGIN
message, /info, 'No Good Packets in files: '
FOR j = 0, n_infiles-1 DO print, infiles[j]
no_files_done = 1b
return
ENDIF
;Here you need to set the clock drift
hsi_clock_drift, /init
;The Good times are needed for the orbit files:
orb_trange = [min(timf[notbad]), max(timf[notbad])]
;Reset the packet times, for good packets including clock drift
timf[notbad] = hsi_sctime2any(pakf[notbad].collect_time)
;Reset orb_range, to include changes in timf
orb_trange = [min(timf[notbad]), max(timf[notbad])]
;Here, take the packet with the latest good time, and write out the
;decimation settings, you need the app_ids, only use monitor rates or
;SOH packets
app_id = fix(pakf[notbad].app_id, 0, nnotbad)
byteorder, app_id, /ntohs
app_id = mask(app_id, 0, 11)
x15 = where(app_id Eq 1 Or app_id Eq 102 Or app_id Eq 152, nx15)
If(nx15 Gt 0) Then Begin
ttest = timf[notbad[x15]]
ptest = pakf[notbad[x15]]
maxtimf = max(ttest, maxtimpt)
hsi_write_decimation_settings, ptest[maxtimpt], filedb_dir = fdbdir, $
filedb_pub = fdbdir_pub, _extra = _extra
delvarx, ttest, ptest ;free up memory?
Endif
delvarx, app_id
;put all packets with timf in the file time range
below = where(timf lt orb_trange[0])
above = where(timf GT orb_trange[1])
IF(below[0] NE -1) THEN BEGIN ;append packets with bad times
timf[below] = orb_trange[0]
ENDIF
IF(above[0] NE -1) THEN BEGIN ;append packets with bad times
timf[above] = max(orb_trange[1])
ENDIF
;Get orbit data
hsi_rd_orbit_files, orb_trange, orbit_table, $
eclipse_table, saa_table, orbit_directory = orbit_directory
;If there is no orbit data, then all of the packets go into one file
IF(KEYWORD_SET(ignore_orbit)) THEN BEGIN
no_file_table: ignore_orbit = 1b
IF(NOT qquiet) THEN message, /info, 'No Orbit Data, 1 file'
orbit_table = -1
check_filedb = 0b
ignore_orbit = 1b
file_table = {start_time:0.0d0, orbit_start:0l, end_time:0.0d0, $
orbit_end:0l, duration:0.0d0}
;Round times to time_intv
file_times = hsi_set_file_time(orb_trange, $
ref_time = ref_time, $
time_intv = file_time_intv)
file_table.start_time = file_times[0]
file_table.end_time = file_times[1]+file_time_intv
file_table.duration = file_table.end_time - file_table.start_time
ENDIF ELSE BEGIN
ignore_orbit = 0b
IF(datatype(orbit_table) NE 'STC') THEN BEGIN
file_table = hsi_filedb_2_file_table(filedb)
ENDIF ELSE BEGIN
;Find file times from orbit data
hsi_eclipse2filetimes, orbit_table, eclipse_table, file_table, $
time_intv = file_time_intv, ref_time = ref_time, $
filedb = filedb
;only use the elements of the File_table that are spanned by the
;packet times
ENDELSE
start_file = max(where(file_table.start_time LE orb_trange[0]))
end_file = min(where(file_table.end_time GE orb_trange[1]))
;Some error checking
IF(start_file[0] EQ -1) THEN BEGIN
message, /info, 'Bad file start times'
no_files_done = 1b
RETURN
ENDIF
IF(end_file[0] EQ -1) THEN BEGIN
message, /info, 'Bad file end times'
no_files_done = 1b
RETURN
ENDIF
IF(end_file[0] LT start_file[0]) THEN BEGIN
message, /info, 'Start_file greater than end_file??'
no_files_done = 1b
RETURN
ENDIF
;Must be ok, dude
file_table = file_table[start_file[0]:end_file[0]]
ENDELSE
n_files = N_ELEMENTS(file_table)
IF(NOT qquiet) THEN print, 'There are ', n_files, ' Orbits'
;Break up the filename, don't put directories in the data_source
IF(n_infiles EQ 1) THEN data_source = infiles_0 $
ELSE IF(n_infiles EQ 2) THEN BEGIN
data_source = infiles_0[0]+','+infiles_0[1]
ENDIF ELSE data_source = infiles_0[0]+', etc...'
;Split up the packets into the orbits and write them out
file_count = 0
file_c_other = 0
del_ftmp_all = -1
FOR j = 0l, n_files-1 DO BEGIN
;Get *all* the packets for this orbit
Hsi_1orbit_allpak, file_table[j], filedb, check_filedb, $
outdir, orbit_table, ptmp, ttmp, badftmp, $
quiet = quiet, version = version_new, files_del = del_ftmp, $
check_newpak = check_newpak, filedb_other_files = filedb_other_j
IF(datatype(del_ftmp) EQ 'STR') THEN BEGIN ;oops, 2-oct-2002, jmm
IF(datatype(del_ftmp_all) EQ 'STR') THEN BEGIN
del_ftmp_all = [del_ftmp_all, del_ftmp]
ENDIF ELSE del_ftmp_all = del_ftmp
ENDIF
;Keep track of other files
IF(is_struct(filedb_other_j)) THEN BEGIN
IF(is_struct(filedb_other_allj)) THEN BEGIN
filedb_other_allj = [filedb_other_allj, filedb_other_j]
ENDIF ELSE filedb_other_allj = filedb_other_j
ENDIF
IF(datatype(ptmp) EQ 'STC') THEN BEGIN
;Get file times
IF(one_file OR ignore_orbit) THEN maxpak = 1.0e20
file_times = hsi_pak2filetimes(ptmp, maxpak = maxpak, $
minpak = minpak, $
collect_time = ttmp, $
time_intv = file_time_intv, $
quiet = quiet, $
ref_time = ref_time, $
file_start_time = $
file_table[j].start_time, $
file_end_time = $
file_table[j].end_time)
;and write out the packets
Hsi_write_level0, ptmp, filex, filedb_j, collect_time = ttmp, $
data_dir = outdir, quiet = quiet, file_times = file_times, $
version = version_new, orbit_table = orbit_table, $
file_table = file_table[j], $
time_intv = time_intv, ref_time = ref_time, $
bad_flag = badftmp, data_source = data_source, $
reprocess_smex = reprocess_smex
;filedb_j has the filedb structures for the new files
IF(datatype(filedb_j) EQ 'STC') THEN BEGIN
n_files_j = N_ELEMENTS(filedb_j)
;add to outfiles array and filedb structure
;Set up clock drift in filedb here
;get a time array, file start times, and the last file end time
tim_arr = [filedb_j.start_time, filedb_j[n_files_j-1].end_time]
hsi_clock_drift, tim_arr, tim_arr_out, drift_out
; add drifts to the filedb structure
filedb_j.clock_drift_start = drift_out[0:n_files_j-1]
filedb_j.clock_drift_end = drift_out[1:*]
IF(file_count EQ 0) THEN BEGIN
file_count = n_files_j
outfiles = filex
filedb_allj = filedb_j
ENDIF ELSE BEGIN
file_count = file_count+n_files_j
outfiles = [outfiles, filex]
filedb_allj = [filedb_allj, filedb_j]
ENDELSE
ENDIF ELSE BEGIN
IF(NOT qquiet) THEN message, /info, 'No file in range:'+$
anytim(file_table[j].start_time, /ccsds)+' TO '+$
anytim(file_table[j].end_time, /ccsds)
ENDELSE
ENDIF ELSE BEGIN
IF(NOT qquiet) THEN message, /info, 'No packets in range:'+$
anytim(file_table[j].start_time, /ccsds)+' TO '+$
anytim(file_table[j].end_time, /ccsds)
ENDELSE
ENDFOR
; If you have read in the filedb, append the new filedb to the old one
IF(file_count GT 0) THEN BEGIN
IF(KEYWORD_SET(init_filedb)) THEN filedb = filedb_allj $
ELSE BEGIN
IF(datatype(filedb) EQ 'STC') THEN filedb = [filedb, filedb_allj] $
ELSE filedb = filedb_allj
ENDELSE
hsi_filedb_sort, filedb
ENDIF ELSE BEGIN
message, /info, 'No files'
no_files_done = 1b
RETURN
ENDELSE
;Output the filedb,
If(total(qfiledb_trange) Gt 0.0) Then Begin
otp_trange = [min([file_table.start_time, qfiledb_trange]), $
max([file_table.end_time, qfiledb_trange])]
Endif Else Begin
otp_trange = [min(file_table.start_time), $
max(file_table.end_time)]
Endelse
;write out the filedb
If(file_count Gt 0) And (Not keyword_set(no_filedb)) Then Begin
hsi_mult_filedb_otp, filedb, otp_trange, $
reprocess_smex = reprocess_smex, quiet = quiet, $
file_type = 'LEVEL0', filedb_dir = fdbdir, filedb_pub = fdbdir_pub
Endif
;Now do the catalog stuff, if asked for
;But first get the qlook and inslog filedbs
catalog_skip: IF(fits_only EQ 0) THEN BEGIN
message, /info, 'Qlook Processing'
qlook_filedb = hsi_mult_filedb_inp(filedb_dir = fdbdir, $
file_type = 'OBS_SUMMARY', $
quiet = quiet, $
dont_copy_old_files = $
dont_copy_old_files, $
qfiledb_trange = qlook_qfiledb_trange)
inslog_filedb = hsi_mult_filedb_inp(filedb_dir = fdbdir, $
file_type = 'INSTRUMENT_LOG', $
quiet = quiet, $
dont_copy_old_files = $
dont_copy_old_files, $
qfiledb_trange = inslog_qfiledb_trange)
fullrate_filedb = hsi_mult_filedb_inp(filedb_dir = fdbdir, $
file_type = 'FULL_RATE', $
quiet = quiet, $
dont_copy_old_files = $
dont_copy_old_files, $
qfiledb_trange = fullrate_qfiledb_trange)
;Copy old flare list files
IF(NOT qquiet) THEN message, /info, 'Doing Catalog'
IF(dont_copy_old_files EQ 0) THEN BEGIN
old_file0 = concat_dir(fdbdir, 'hessi_flare_list*fits')
old_file = loc_file(old_file0, count = bb)
IF(bb GT 0) THEN BEGIN
f01 = get_logenv('$HSI_FILEDB_OLD')
cmd = '/bin/cp '+old_file0+' '+f01
IF(NOT qquiet) THEN message, /info, 'Spawning: '+cmd
spawn, cmd
ENDIF
ENDIF
;Archive option simply works on the first file in the filedb that has
;status_flag eq 0, and a start time after
;arch_start_time, and a start time before arch_stop_time
;Both archive and do_catalog will do the whole orbit unless the
;/only_this_file keyword is set, Note that this also requires a
;filedb entry for the file, and that you can only do one file at a
;time. Note that files are no longer copied, as of 12-jul-2002
IF(keyword_set(archive) OR keyword_set(do_catalog)) THEN BEGIN
IF(NOT keyword_set(only_this_file)) THEN BEGIN
;here get all of the files for this orbit time period
IF(keyword_set(archive)) THEN BEGIN
todo0 = where(filedb.status_flag EQ 0 AND $
filedb.start_time GE arch_start_time AND $
filedb.start_time LE arch_stop_time)
ENDIF ELSE BEGIN
break_file, outfiles[0], a, b, c, d
file_id = c+d
todo0 = where(filedb.file_id EQ file_id)
ENDELSE
IF(todo0[0] EQ -1) THEN BEGIN
message, /info, 'No Files to Process'
no_files_done = 1b
message, /info, '****FINISHED SUCCESSFULLY****'
return
ENDIF
todo0 = todo0[0]
fdb0 = filedb[todo0]
todo = where(filedb.orbit_start EQ fdb0.orbit_start AND $
filedb.orbit_end EQ fdb0.orbit_end AND $
filedb.status_flag NE -1)
ntodo = n_elements(todo)
ostart = strtrim(string(fdb0.orbit_start), 2)
oend = strtrim(string(fdb0.orbit_end), 2)
;do this for each file
bbc = 0
FOR j = 0, ntodo-1 DO BEGIN
outfilex = hsi_find_in_archive(filedb[todo[j]].file_id, $
data_dir = outdir, $
/nodialog, count = bb)
IF(bb GT 0) THEN BEGIN
outfilex = outfilex[0] ;1 file at a time
IF(bbc EQ 0) THEN BEGIN
outfilex_all = outfilex
ENDIF ELSE BEGIN
outfilex_all = [outfilex_all, outfilex]
ENDELSE
bbc = bbc+1
ENDIF
ENDFOR
IF(bbc Ne ntodo) THEN BEGIN
message, /info, 'All Files for the given orbit are not present'
message, /info, 'No processing done'
For j = 0, ntodo-1 Do print, filedb[todo[j]].file_id
message, /info, '****FINISHED SUCCESSFULLY****'
no_files_done = 1b
RETURN
ENDIF
outfiles = outfilex_all
file_count = 1
ENDIF ELSE BEGIN
file_count = 1
outfiles = outfilex[0]
ENDELSE
ENDIF ELSE file_count = n_elements(outfiles)
;Do all of the files for a given orbit
done_flag = bytarr(file_count)
break_file, outfiles, a, b, c, d
file_id = c+d
flare_count = 0
;keep track of the files processed to do full_rates
files_proc = ''
FOR j = 0, file_count-1 DO BEGIN
IF(keyword_set(archive) OR keyword_set(DO_catalog)) THEN BEGIN
ofilesj = outfiles
ENDIF ELSE BEGIN
IF(datatype(filedb_allj) EQ 'STC') THEN BEGIN
IF(done_flag[j] EQ 0) THEN BEGIN
fj = where(filedb_allj.file_id EQ file_id[j])
fdbj = filedb_allj[fj]
ostart = strtrim(string(fdbj.orbit_start), 2)
oend = strtrim(string(fdbj.orbit_end), 2)
;changed to check only the end orbit, jmm, 2013-11-06
all_fj = where(filedb_allj.orbit_end EQ fdbj.orbit_end)
; all_fj = where(filedb_allj.orbit_start EQ $
; fdbj.orbit_start AND $
; filedb_allj.orbit_end EQ fdbj.orbit_end)
ofilesj = outfiles[all_fj]
done_flag[all_fj] = 1
ENDIF ELSE ofilesj = 'NEXT'
ENDIF ELSE BEGIN ;one at a time
ofilesj = outfiles[j]
ENDELSE
ENDELSE
IF(ofilesj[0] NE 'NEXT') THEN BEGIN
nfiles_here = n_elements(ofilesj)
IF(NOT qquiet) THEN BEGIN
IF(datatype(ostart) EQ 'UND') THEN ostart = '0'
IF(datatype(oend) EQ 'UND') THEN oend = '0'
message, /info, 'ORBIT #s: '+ ostart+' -- '+oend
print, nfiles_here, ' FILES'
FOR l = 0, nfiles_here-1 DO print, ofilesj[l]
ENDIF
hsi_do_catalog, ofilesj, filedb, $
oobs, opktr, orpd, oqmn, oflr, oqpg, $ ; ofrt, $ ;added ofrt, 16-sep-2009
filedb_dir = fdbdir, orbit_directory = orbit_directory, $
data_dir = outdir, state_vector_dir = state_vector_dir, $
all_flare = all_flare, time_intv = time_intv, $
eph_time_intv = eph_time_intv, $
filedb_fname = filedb_fname, old_file_label = old_file_label, $
ref_time = ref_time, as_time_intv = as_time_intv, $
file_time_intv = file_time_intv, ignore_orbit = ignore_orbit, $
no_qlook_pointing = lvl0, no_qlook_roll_period = lvl0, $
no_qlook_roll_angle = lvl0, flare_list_object = oflr, $
no_flare_position = flr_lvl0, archive = archive, $
_extra = _extra ;can't do full_rate because of need for decimation correction
dflr = oflr -> get(/data)
If(is_struct(dflr)) Then Begin
If(flare_count Eq 0) Then oflr_all = hsi_new_qlook_obj(oflr) $
Else oflr_all = hsi_qlook_concat(oflr_all, oflr)
flare_count = flare_count+n_elements(dflr)
Endif
;update bck9 model, new as of 2011-04-15, jmm
;Not used anymore, will replace 2016-01-15, jmm
; If(obj_valid(oobs)) Then Begin
; troobs = oobs -> data_time_range()
; hsi_night_bck_update_oti, troobs, /update, _extra = _extra
; Endif
;Update obssumm files
hsi_update_daily_catalog, [oobs, oflr, oqpg], qlook_filedb, $
catalog_dir = catalog_dir, qtime_range = qtime_range, $
file_type = 'OBS_SUMMARY', del_files = dqfiles, $
_extra = _extra
;Files to delete,
If(datatype(dqfiles) Eq 'STR') Then Begin
If(datatype(dqfiles0) NE 'STR') Then dqfiles0 = dqfiles $
Else dqfiles0 = [dqfiles0, dqfiles]
Endif
;Track the whole time_range
If(qtime_range[0] ne -1) Then Begin
If(datatype(tq) Eq 'UND') Then tq = qtime_range $
Else tq = [min([tq, qtime_range]), max([tq, qtime_range])]
Endif
;Update inslog file
hsi_update_daily_catalog, [opktr, orpd, oqmn], inslog_filedb, $
catalog_dir = catalog_dir, qtime_range = qtime_range, $
file_type = 'INSTRUMENT_LOG', del_files = dqfiles, $
_extra = _extra
;Files to delete, filedb_updates
If(datatype(dqfiles) Eq 'STR') Then Begin
If(datatype(dqfiles0) NE 'STR') Then dqfiles0 = dqfiles $
Else dqfiles0 = [dqfiles0, dqfiles]
Endif
;Track the whole time_range
If(qtime_range[0] ne -1) Then Begin
If(datatype(tq) Eq 'UND') Then tq = qtime_range $
Else tq = [min([tq, qtime_range]), max([tq, qtime_range])]
Endif
;Do the qlook plot files
hsi_do_qlook_plots, oobs, oimg, ospc, $
orpd, oqmn, quiet = quiet, _extra = _extra
;Set status_flag for files that are done to +1, build a filedb
;array for the files that are done
break_file, ofilesj, a, b, c, d
ofilesj_id = c+d
For l = 0, nfiles_here-1 Do Begin
done_file = where(filedb.file_id EQ ofilesj_id[l])
If(done_file[0] Ne -1) Then filedb[done_file].status_flag = 1
Endfor
files_proc = [files_proc, ofilesj]
Endif
ENDFOR
;write out the filedb files
IF(file_count GT 0) AND (NOT keyword_set(no_filedb)) THEN BEGIN
If(datatype(otp_trange) NE 'UND') Then Begin
If(Datatype(tq) NE 'UND') Then Begin
tq = [min([tq, otp_trange]), max([tq, otp_trange])]
Endif Else tq = otp_trange
Endif Else Begin
If(Datatype(tq) Eq 'UND') Then Begin
message, /info, 'No relevant Time Range, this should not happen'
tq = [anytim('5-feb-2002 0:00'), anytim(!stime)]
Endif
Endelse
If(fits_only eq 0) Then Begin ;you did this seconds ago for fits_only=1
hsi_mult_filedb_otp, filedb, tq, do_catalog = do_catalog, $
archive = archive, reprocess_smex = reprocess_smex, quiet = quiet, $
file_type = 'LEVEL0', filedb_dir = fdbdir, filedb_pub = fdbdir_pub, $
long_archive = long_archive
Endif
;The inslog and obssumm file should be written out always, too,even
;for fits_only=1, because this process may be handling all of the
;filedb updates
If(total(qlook_qfiledb_trange Gt 0)) Then Begin
qtq = [min([tq, qlook_qfiledb_trange]), $
max([tq, qlook_qfiledb_trange])]
Endif Else qtq = tq
If(total(inslog_qfiledb_trange Gt 0)) Then Begin
itq = [min([tq, inslog_qfiledb_trange]), $
max([tq, inslog_qfiledb_trange])]
Endif Else itq = tq
hsi_mult_filedb_otp, qlook_filedb, qtq, reprocess_smex = reprocess_smex, $
quiet = quiet, file_type = 'OBS_SUMMARY', filedb_dir = fdbdir, $
filedb_pub = fdbdir_pub, do_catalog = do_catalog, archive = archive
hsi_mult_filedb_otp, inslog_filedb, itq, reprocess_smex = reprocess_smex, $
quiet = quiet, file_type = 'INSTRUMENT_LOG', filedb_dir = fdbdir, $
filedb_pub = fdbdir_pub, do_catalog = do_catalog, archive = archive
ENDIF
;Here you can do the full_rate
If(n_elements(files_proc) Gt 1) Then Begin
files_proc = files_proc[1:*]
hsi_do_fullrate, files_proc, ofrt, $
data_dir = outdir, quiet = quiet, _extra = _extra
If(obj_valid(ofrt)) Then Begin
;Update fullrate file
hsi_update_daily_catalog, ofrt, fullrate_filedb, $
catalog_dir = catalog_dir, qtime_range = qtime_range, $
file_type = 'FULL_RATE', del_files = dqfiles, $
_extra = _extra
;Plot full-rates, front detectors summed
hsi_do_full_rate_plots, oobs, ofrt, quiet = quiet, _extra = _extra
;Files to delete, filedb_updates
If(datatype(dqfiles) Eq 'STR') Then Begin
If(datatype(dqfiles0) NE 'STR') Then dqfiles0 = dqfiles $
Else dqfiles0 = [dqfiles0, dqfiles]
Endif
;Track the whole time_range
If(qtime_range[0] ne -1) Then Begin
If(datatype(tq) Eq 'UND') Then tq = qtime_range $
Else tq = [min([tq, qtime_range]), max([tq, qtime_range])]
Endif
If(total(fullrate_qfiledb_trange Gt 0)) Then Begin
ftq = [min([tq, fullrate_qfiledb_trange]), $
max([tq, fullrate_qfiledb_trange])]
Endif Else ftq = tq
hsi_mult_filedb_otp, fullrate_filedb, ftq, reprocess_smex = reprocess_smex, $
quiet = quiet, file_type = 'FULL_RATE', filedb_dir = fdbdir, $
filedb_pub = fdbdir_pub, do_catalog = do_catalog, archive = archive
Endif
Endif
;Here do the qlook images, if asked for
oobs_trange = oobs -> data_time_range()
If(keyword_set(do_img_spc_too)) Then Begin
hsi_do_qlook_image_alt, oflr_all, filedb_dir = fdbdir, quiet = quiet, $
time_range = oobs_trange, xxx = xxx, _extra = _extra
Endif
;Update whole mission flare list,
hsi_whole_flare_list, oflr_all, oobs_trange, archive = archive, long_archive = long_archive, $
do_catalog = do_catalog, filedb_dir = fdbdir, filedb_pub = fdbdir_pub
ENDIF ELSE oflr_all = -1
;Now move old files away, if not reprocessing
IF(datatype(del_ftmp_all) EQ 'STR') THEN BEGIN
hsi_filedb_sort, del_ftmp_all ;no non-unique filenames, jmm, 9-May-2005
ss_del_ftmp = uniq(del_ftmp_all)
del_ftmp_all = del_ftmp_all[ss_del_ftmp]
del_file_count = n_elements(del_ftmp_all)
FOR k = 0, del_file_count-1 DO BEGIN
;this file may have moved
del_ftmpk = hsi_find_in_archive(del_ftmp_all[k], data_dir = outdir, $
/nodialog, count = bb)
IF(bb GT 0) THEN BEGIN
IF(keyword_set(reprocess_smex)) THEN BEGIN
IF(NOT qquiet) THEN $
message, /info, 'Not Moving: '+del_ftmpk[0]+$
' to '+dir_old
ENDIF ELSE BEGIN
IF(NOT qquiet) THEN $
message, /info, 'Moving: '+del_ftmpk[0]+' to '+dir_old
spawn, '/bin/mv '+del_ftmpk[0]+' '+dir_old
ENDELSE
ENDIF
ENDFOR
ENDIF
;Move old qlook files away here
IF(datatype(dqfiles0) EQ 'STR') THEN BEGIN
FOR j = 0, n_elements(dqfiles0)-1 DO BEGIN
cmd = '/bin/mv '+dqfiles0[j]+' '+dir_old
If(Keyword_set(archive)) Then Begin
IF(NOT keyword_set(quiet)) THEN $
message, /info, 'Spawning: '+cmd
spawn, cmd
Endif Else Begin
IF(NOT keyword_set(quiet)) THEN $
message, /info, 'Not Spawning: '+cmd
Endelse
ENDFOR
ENDIF
;Finished
If(~keyword_set(mult_files)) Then $
message, /info, '****FINISHED SUCCESSFULLY****'
free_all_lun
heap_gc
delvarx, pakf, timf, badf
delvarx, ptmp, ttmp, badftmp
delvarx, pak, tim, bad_flag
help, /memory
END