;+
; Project     : SOHO - CDS     
;                   
; Name        : MK_HEAD_CAT
;               
; Purpose     : Create/update CDS FITS header catalogue.
;               
; Explanation : Reads CDS FITS headers and adds entries to catalogue.
;               
; Use         : IDL> mk_head_cat [, update=update, all=all, year=year]
;    
; Inputs      : None
;               
; Opt. Inputs : upate - specifies the number of previous days the update
;                       should cover  ie "update=5" means update for any
;                       files created in the last 5 days.
;               
; Outputs     : Catalogue is written to cds_header_cat and into save files
;               subdivided by year ie cds_header_cat.96  etc
;               
; Opt. Outputs: None
;               
; Keywords    : ALL  - if set then process all existing files.
;               YEAR - only deal with that year's data.
;               NOENV - do not redefine env variables
;
; Calls       : CDSHEADFITS
;
; Common      : None
;               
; Restrictions: None
;               
; Side effects: None
;               
; Category    : Catalogues
;               
; Prev. Hist. : None
;
; Written     : C D Pike, RAL, 9-Apr-97
;               
; Modified    : Add year keyword and only keep yearly catalogues.  CDP, 23-Jul-97
;               Added NOENV
;               Adjust for year 2000
;               Adjust for year 2001 and beyond
;
; Version     : Version 7 05-Jan-04
;-            


pro mk_head_cat, update=update, all=all, year=year, noenv=noenv

if n_elements(year) eq 0 then begin
   print,'Must specify year keyword.'
   return
endif

if year ge 2000 then year = year - 2000
if year gt 1900 then year = year - 1900

;
;  maintained by RAL so can use local routines
;
if not keyword_set(noenv) then begin
   use_eof_cat,/q
   use_eof_data,/q
endif

;
;  set up the basic structure record
;
cat_rec = {ANGLE:0.0,$
           CATEGORY:'',$
           CMP_NAME:'',$
           CMP_NO:0,$
           COMMENTS:strarr(15),$                                  
           COMP_ERR:0,$
           COMP_ID:0,$
           COMP_OPT:0,$
           DATASRC:'',$
           DATE:'',$
           DATE_END:'',$
           DATE_OBS:'',$
           DETECTOR:'',$
           DEXSIZE:'',$
           DW_ID:0,$
           EXPCOUNT:0,$
           EXPTIME:0.0,$
           FILENAME:'',$
           GSET_ID:0,$
           HEL_LAT:0,$
           HEL_LONG:0,$
           INS_ROLL:0.0,$
           INS_X0:0.0,$
           INS_Y0:0.0,$
           IXWIDTH:0.0,$
           IYWIDTH:0.0,$
           LL_ID:0,$
           MIR_POS:0,$
           NWINDOWS:0,$
           NX:0,$
           NY:0,$
           OBJECT:'',$
           OBJ_ID:'',$
           OBS_PROG:'',$
           OBT_END:0.0d0,$
           OBT_TIME:0.0d0,$
           OPS_L:0,$
           OPS_R:0,$
           PROG_NUM:0,$
           RADIUS:0,$
           RAS_ID:0,$
           RAS_VAR:0,$
           SCI_OBJ:'',$
           SCI_SPEC:'',$
           SEQ_IND:0,$
           SER_ID:0,$
           SLIT_NUM:0,$
           SLIT_POS:0,$
           STUDYVAR:0,$
           STUDY_ID:0,$
           TITLE:'',$
           TITLE_ID:0,$
           TRACKING:0,$
           VALID:0,$
           VDS_ACC:0,$
           VDS_MODE:0,$
           VDS_PMCP:0,$
           VDS_ORI:0,$
           WAVE:'',$
           XCEN:0.0,$
           XSTEP:0,$
           YCEN:0.0,$
           YSTEP:0}

;
;  get a list of all existing files if not updating
;
if n_elements(update) eq 0 then begin
   f = find_files('s*','$CDS_FITS_DATA')
   n = sort(f)
   f = f(n)
   rfl = f
   save,rfl,/xdr,file=concat_dir('$CDS_HEAD_CAT','ral_file_loc')
   nc = 0
   cat = cat_rec

;
;  don't forget need to add oddballs for 1996 (ie file num > 13000)
;  (not this way now... USE env variable instead to pick em out)
;
;   if year eq 96 then begin
;      ystart = 0
;      yend = 6382
;   endif

;   if year eq 97 then begin
;      ystart = 6383
;      yend = 10086
;   endif

   if year eq 98 then begin
      ystart = 10087
      yend = 99999
   endif

   fnum = fix(str_pick(f,'/s','r'))
   nk = where(fnum ge ystart and fnum le yend,count)
   if count gt 0 then begin
      f = f(nk)
   endif else begin
      print,'No valid files for year ',year
      return
   endelse
 
   print,'Making new catalogue with '+trim(n_elements(f))+' entries.'
   nf = n_elements(f)
   cat = replicate(cat,nf)
endif else begin
   rfl = 0
   restore,concat_dir('$CDS_HEAD_CAT','ral_file_loc')
   if update gt 0 then hc_update, update
   f = rd_ascii(concat_dir('$CDS_HEAD_CAT','newfits'))
   if n_elements(f) eq 1 and f(0) eq '' then return
   rfl = [rfl,f]
   rfl = rfl(rem_dup(rfl))
   save,rfl,/xdr,file=concat_dir('$CDS_HEAD_CAT','ral_file_loc')
   headcat,cat,year=year
   print,year
   if datatype(cat,1) eq 'Integer' then cat = cat_rec
   nc = n_elements(cat)
   print,'Updating existing catalogue ('+trim(nc)+') with '+$
            trim(n_elements(f))+' new entries.'
   nf = n_elements(f)
   new = replicate(cat(nc-1),nf)
   cat = [cat,new]
endelse

ntot = long(n_elements(cat))
nstart = long(nc)
if nc gt 0 then f = [strarr(nc),f]
print,'Starting at entry ',nstart

;
;  read each file's header and create catalogue entry
;
for i=nstart,ntot-1 do begin
   print,fmt_vect([i+1,ntot],del='/',/no)+'  Reading '+f(i)
   break_file,f(i),disk,dir,fff,ext
   h = cdsheadfits(fff)
   if n_elements(h) gt 1 then begin
      cat(i).ANGLE = fxpar(h,'ANGLE')
      cat(i).CATEGORY = strtrim(fxpar(h,'CATEGORY'),2)
      cat(i).CMP_NAME = strtrim(fxpar(h,'CMP_NAME'),2)
      cat(i).CMP_NO = fxpar(h,'CMP_NO')
      cat(i).COMMENTS = ''
      cat(i).COMP_ERR = fxpar(h,'COMP_ERR')
      cat(i).COMP_ID = fxpar(h,'COMP_ID')
      cat(i).COMP_OPT = fxpar(h,'COMP_OPT')
      cat(i).DATASRC = strtrim(fxpar(h,'DATASRC'),2)
      cat(i).DATE = strtrim(fxpar(h,'DATE'),2)
      cat(i).DATE_END = strtrim(fxpar(h,'DATE_END'),2)
      cat(i).DATE_OBS = strtrim(fxpar(h,'DATE_OBS'),2)
      cat(i).DETECTOR = strtrim(fxpar(h,'DETECTOR'),2)
      cat(i).DW_ID = fxpar(h,'DW_ID')
      cat(i).EXPCOUNT = fxpar(h,'EXPCOUNT')
      cat(i).EXPTIME = fxpar(h,'EXPTIME')
      cat(i).FILENAME = strtrim(fxpar(h,'FILENAME'),2)
      cat(i).GSET_ID = fxpar(h,'GSET_ID')
      cat(i).INS_ROLL = fxpar(h,'INS_ROLL')
      cat(i).INS_X0 = fxpar(h,'INS_X0')
      cat(i).INS_Y0 = fxpar(h,'INS_Y0')
      cat(i).IXWIDTH = fxpar(h,'IXWIDTH')
      cat(i).IYWIDTH = fxpar(h,'IYWIDTH')
      cat(i).LL_ID = fxpar(h,'LL_ID')
      cat(i).MIR_POS = fxpar(h,'MIR_POS')
      cat(i).NWINDOWS = fxpar(h,'NWINDOWS')
      cat(i).NX = fxpar(h,'NX')
      cat(i).NY = fxpar(h,'NY')
      cat(i).OBJECT = strtrim(fxpar(h,'OBJECT'),2)
      cat(i).OBJ_ID = strtrim(fxpar(h,'OBJ_ID'),2)
      cat(i).OBS_PROG = strtrim(fxpar(h,'OBS_PROG'),2)
      cat(i).OBT_END = fxpar(h,'OBT_END')
      cat(i).OBT_TIME = fxpar(h,'OBT_TIME')
      cat(i).OPS_L = fxpar(h,'OPS_L')
      cat(i).OPS_R = fxpar(h,'OPS_R')
      cat(i).PROG_NUM = fxpar(h,'PROG_NUM')
      cat(i).RAS_ID = fxpar(h,'RAS_ID')
      cat(i).RAS_VAR = fxpar(h,'RAS_VAR')
      cat(i).SCI_OBJ = strtrim(fxpar(h,'SCI_OBJ'),2)
      cat(i).SCI_SPEC = strtrim(fxpar(h,'SCI_SPEC'),2)
      cat(i).SEQ_IND = fxpar(h,'SEQ_IND')
      cat(i).SER_ID = fxpar(h,'SER_ID')
      cat(i).SLIT_NUM = fxpar(h,'SLIT_NUM')
      cat(i).SLIT_POS = fxpar(h,'SLIT_POS')
      cat(i).STUDYVAR = fxpar(h,'STUDYVAR')
      cat(i).STUDY_ID = fxpar(h,'STUDY_ID')
      cat(i).TITLE = strtrim(fxpar(h,'TITLE'),2)
      cat(i).TITLE_ID = fxpar(h,'TITLE_ID')
      cat(i).TRACKING = fxpar(h,'TRACKING')
      cat(i).VALID = fxpar(h,'SEQVALID')
      cat(i).VDS_ACC = fxpar(h,'VDS_ACC')
      cat(i).VDS_MODE = fxpar(h,'VDS_MODE')
      cat(i).VDS_PMCP = fxpar(h,'VDS_PMCP')
      cat(i).VDS_ORI = fxpar(h,'VDS_ORI')
      get_datawin,cat(i).dw_id,dw
      if dw.dw_id ge 0 then begin
         cat(i).dexsize = fmt_vect([dw.w_width,dw.w_height])
         n = where(strpos(dw.wins.win_name,'Back') lt 0)
         w = ' '
         for j=0,n_elements(n)-1 do w = w + '/'+strtrim(dw.wins(n(j)).win_name,2)
         cat(i).WAVE = strtrim(w,2)
      endif else begin
         cat(i).WAVE = ''
      endelse
      cat(i).XCEN = fxpar(h,'XCEN')
      cat(i).XSTEP = fxpar(h,'XSTEP')
      cat(i).YCEN = fxpar(h,'YCEN')
      cat(i).YSTEP = fxpar(h,'YSTEP')
      bl = arcmin2hel(cat(i).xcen/60.,cat(i).ycen/60.,date=cat(i).date_obs)
      cat(i).HEL_LAT = fix(round(bl(0)))
      cat(i).HEL_LONG = fix(round(bl(1)))
      cat(i).RADIUS = fix(sqrt(cat(i).xcen*cat(i).xcen+cat(i).ycen*cat(i).ycen))
      if n_elements(update) eq 0 then begin
         if i mod 100  eq 0 then save,cat,/xdr,file='cds_header_cat'
      endif

;
;  check correct year
;
      if fix(strmid(cat(i).date_obs,2,2)) ne year then begin
         print,'NOT year '+trim(year)+'  '+cat(i).filename 
         cat(i).filename = ''
      endif
   endif else cat(i).filename = ''
endfor

;
;  FINISHED so
;
;  kill blank entries
;
n = where(cat.filename ne '',count)
if count gt 0 then cat = cat(n)

;
;  remove any duplicates
;
ff = cat.filename
nd = rem_dup(ff,indgen(n_elements(ff)))
cat = cat(nd)

;
;  sort by time
;
t = cat.obt_time
n = sort(t)
cat = cat(n)

;
;  add the comments
;
print,'Adding hc comments'
hc_add_comment,cat,year=year,/quiet


;
;  Info at end
;
nc2 = n_elements(cat)
print,'# Files added = ',nc2-nc
print,'# Total catalogue entries = ',nc2


;
;  save 
;
if year eq 96 then begin
   cat96 = temporary(cat)
   save,cat96,/xdr,file=concat_dir('$CDS_HEAD_CAT','cds_header_cat.'+trim(year))
endif
if year eq 97 then begin
   cat97 = temporary(cat)
   save,cat97,/xdr,file=concat_dir('$CDS_HEAD_CAT','cds_header_cat.'+trim(year))
endif
if year eq 98 then begin
   cat98 = temporary(cat)
   save,cat98,/xdr,file=concat_dir('$CDS_HEAD_CAT','cds_header_cat.'+trim(year))
endif
if year eq 99 then begin
   cat99 = temporary(cat)
   save,cat99,/xdr,file=concat_dir('$CDS_HEAD_CAT','cds_header_cat.'+trim(year))
endif
if year eq 0 then begin
   cat00 = temporary(cat)
   save,cat00,/xdr,file=concat_dir('$CDS_HEAD_CAT','cds_header_cat.00')
endif
if year eq 1 then begin
   cat01 = temporary(cat)
   save,cat01,/xdr,file=concat_dir('$CDS_HEAD_CAT','cds_header_cat.01')
endif
if year eq 2 then begin
   cat02 = temporary(cat)
   save,cat02,/xdr,file=concat_dir('$CDS_HEAD_CAT','cds_header_cat.02')
endif
if year eq 3 then begin
   cat03 = temporary(cat)
   save,cat03,/xdr,file=concat_dir('$CDS_HEAD_CAT','cds_header_cat.03')
endif
if year eq 4 then begin
   cat04 = temporary(cat)
   save,cat04,/xdr,file=concat_dir('$CDS_HEAD_CAT','cds_header_cat.04')
endif
if year eq 5 then begin
   cat05 = temporary(cat)
   save,cat05,/xdr,file=concat_dir('$CDS_HEAD_CAT','cds_header_cat.05')
endif
if year eq 6 then begin
   cat06 = temporary(cat)
   save,cat06,/xdr,file=concat_dir('$CDS_HEAD_CAT','cds_header_cat.06')
endif
if year eq 7 then begin
   cat07 = temporary(cat)
   save,cat07,/xdr,file=concat_dir('$CDS_HEAD_CAT','cds_header_cat.07')
endif


;
;  save the weekly version
;
get_utc,utc
mjd2date,utc.mjd,y,m,d
headcat,cat,year=y
now = utc2tai(utc)
now_7 = now - 86400.*7.
n = where(cat.obt_time ge now_7,count)
if count gt 0 then begin
   print,'Creating the weekly catalogue...'
   cat = cat(n)
   save,cat,/xdr,file=concat_dir('$CDS_HEAD_CAT','cds_header_cat.weekly')
endif


end