diff --git a/docs/conf.py b/docs/conf.py index 7f0c9ac536..d245f1ca92 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -90,6 +90,8 @@ ("py:class", "t.Type"), ("py:class", "Config"), ("py:class", "Unicode"), + ("py:class", "StrDict"), + ("py:class", "ClassesType") ] # The suffix(es) of source filenames. diff --git a/lstchain/calib/camera/calibration_calculator.py b/lstchain/calib/camera/calibration_calculator.py index d57a6ae082..95d3c9fa27 100644 --- a/lstchain/calib/camera/calibration_calculator.py +++ b/lstchain/calib/camera/calibration_calculator.py @@ -76,7 +76,8 @@ class CalibrationCalculator(Component): hg_lg_ratio = traits.Float( 1., - help='HG/LG ratio applied if use_scaled_low_gain is True. In case of calibrated data the ratio should be 1.' + help='HG/LG ratio applied if use_scaled_low_gain is True. The ratio is ~1 for calibrated data, ~17.4 for uncalibrated data.' + ).tag(config=True) classes = ( @@ -214,8 +215,8 @@ def calculate_calibration_coefficients(self, event): # define unusables on number of estimated pe npe_deviation = calib_data.n_pe - npe_median[:,np.newaxis] - # cut on the base of pe statistical uncertainty (adding a 7% spread due to different detection QE among PMs) - tot_std = np.sqrt(npe_median + (self.relative_qe_dispersion * npe_median)**2) + # cut on the base of the pe statistical uncertainty over the camera + tot_std = self.expected_npe_std(npe_median, ff_data.n_events) npe_outliers = ( np.logical_or(npe_deviation < self.npe_median_cut_outliers[0] * tot_std[:,np.newaxis], @@ -296,3 +297,41 @@ def output_interleaved_results(self, event): new_ff = True return new_ped, new_ff + + def expected_npe_std(self, npe_median, n_events): + + """ + The expected standard deviation of the estimated npe over the camera is given in principle by + + std_pe_mean=std_npe/sqrt((n_events)+ (relative_qe_dispersion*npe)**2) + + where the relative_qe_dispersion is mainly due to different detection QE among PMs. + + However, due to the systematics correction associated to the B term, a linear and quadratic + noise component must be added, these components depend on the sample statistics per pixel (n_events). + The parameters in this function (linear_noise_params and quadratic_noise_params) have been obtained with + a fit of the std of filter scan taken in date 2023/05/10 and considering + n_events = [1000,2500,5000,7500,10000,30000] + + """ + basic_variance = npe_median/n_events + (self.relative_qe_dispersion * npe_median)**2 + + # [par0_HG, par0_LG],[par1_HG, par1_LG] + linear_noise_params=np.array(([1.79717813 ,1.72458305e+00],[0.0231544, -1.62036639e-03])) + + # [par0_HG, par0_LG],[par1_HG, par1_LG] + quadratic_noise_params=np.array(([4.99670969e-04, 0.00142218],[ 2.49034290e-05,0.0001207])) + + # function to estimate the added noise components as function of the sample statistcs + def noise_term(n_events, par): + return par[0]/(np.sqrt(n_events))+par[1] + + linear_term = noise_term(n_events, linear_noise_params) + quadratic_term = noise_term(n_events, quadratic_noise_params) + + added_variance = (linear_term * npe_median)**2 + (quadratic_term * npe_median**2)**2 + + std = np.sqrt(basic_variance + added_variance) + + return std + diff --git a/lstchain/calib/camera/pedestals.py b/lstchain/calib/camera/pedestals.py index 87434cf014..0acd75bf85 100644 --- a/lstchain/calib/camera/pedestals.py +++ b/lstchain/calib/camera/pedestals.py @@ -106,6 +106,7 @@ def __init__(self, subarray, **kwargs): self.extractor = ImageExtractor.from_name( self.charge_product, parent=self, subarray=subarray ) + def _extract_charge(self, event): """ @@ -117,7 +118,7 @@ def _extract_charge(self, event): event : general event container """ - + # copy the waveform be cause we do not want to change it for the moment waveforms = np.copy(event.r1.tel[self.tel_id].waveform) @@ -155,9 +156,10 @@ def calculate_pedestals(self, event): Returns: True if the mon.tel[tel_id].pedestal is updated, False otherwise """ + # initialize the np array at each cycle waveform = event.r1.tel[self.tel_id].waveform - + # re-initialize counter if self.num_events_seen == self.sample_size: self.num_events_seen = 0 @@ -165,7 +167,7 @@ def calculate_pedestals(self, event): pixel_mask = event.mon.tel[self.tel_id].pixel_status.hardware_failing_pixels self.trigger_time = event.trigger.time - + if self.num_events_seen == 0: self.time_start = self.trigger_time self.setup_sample_buffers(waveform, self.sample_size) @@ -231,7 +233,7 @@ def store_results(self, event): def setup_sample_buffers(self, waveform, sample_size): """Initialize sample buffers""" - + n_channels = waveform.shape[0] n_pix = waveform.shape[1] shape = (sample_size, n_channels, n_pix) @@ -257,7 +259,7 @@ def calculate_pedestal_results(self, trace_integral, masked_pixels_of_sample): trace_integral, mask=masked_pixels_of_sample ) - + # mean and std over the sample per pixel max_sigma = self.sigma_clipping_max_sigma pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( diff --git a/lstchain/conftest.py b/lstchain/conftest.py index 6ff6b5bba0..9ad24ab969 100644 --- a/lstchain/conftest.py +++ b/lstchain/conftest.py @@ -106,11 +106,13 @@ def observed_dl1_files(temp_dir_observed_files, run_summary_path): datacheck_file1 = temp_dir_observed_files / "datacheck_dl1_LST-1.Run02008.0000.h5" dvr_file1 = temp_dir_observed_files / "DVR_settings_LST-1.Run02008.h5" pixmasks_file1 = temp_dir_observed_files / "Pixel_selection_LST-1.Run02008.0000.h5" - + interleaved_file1 = temp_dir_observed_files / "interleaved/interleaved_LST-1.Run02008.0000.h5" + # Second set of files dl1_output_path2 = temp_dir_observed_files / "dl1_LST-1.Run02008.0100.h5" muons_file2 = temp_dir_observed_files / "muons_LST-1.Run02008.0100.fits" datacheck_file2 = temp_dir_observed_files / "datacheck_dl1_LST-1.Run02008.0100.h5" + interleaved_file2 = temp_dir_observed_files / "interleaved/interleaved_LST-1.Run02008.0100.h5" run_program( "lstchain_data_r0_to_dl1", @@ -198,12 +200,42 @@ def observed_dl1_files(temp_dir_observed_files, run_summary_path): 'datacheck1': datacheck_file1, 'dvr_file1': dvr_file1, 'pixmasks_file1': pixmasks_file1, + 'interleaved_file1': interleaved_file1, 'dl1_file2': dl1_output_path2, 'muons2': muons_file2, - 'datacheck2': datacheck_file2 + 'datacheck2': datacheck_file2, + 'interleaved_file2': interleaved_file2 } + +@pytest.mark.private_data +@pytest.fixture(scope="session") +def interleaved_r1_file(temp_dir_observed_files, run_summary_path): + test_pedcal_run = test_data / 'real/R0/20200218/LST-1.1.Run02006.0000_first50.fits.fz' + + run_program( + "lstchain_data_r0_to_dl1", + "-f", + test_pedcal_run, + "-o", + temp_dir_observed_files, + "--pedestal-file", + test_drs4_pedestal_path, + "--calibration-file", + test_calib_path, + "--time-calibration-file", + test_time_calib_path, + "--pointing-file", + test_drive_report, + '--run-summary-path', + run_summary_path, + "--default-trigger-type=tib" + ) + + return temp_dir_observed_files / "interleaved/interleaved_LST-1.Run02006.0000.h5" + + @pytest.fixture(scope="session") def simulated_dl2_file(temp_dir_simulated_files, simulated_dl1_file, rf_models): """ diff --git a/lstchain/io/io.py b/lstchain/io/io.py index 0d990fe822..98e672baa3 100644 --- a/lstchain/io/io.py +++ b/lstchain/io/io.py @@ -82,6 +82,10 @@ dl1_params_tel_mon_ped_key = "/dl1/event/telescope/monitoring/pedestal" dl1_params_tel_mon_cal_key = "/dl1/event/telescope/monitoring/calibration" dl1_params_tel_mon_flat_key = "/dl1/event/telescope/monitoring/flatfield" +dl1_mon_tel_CatB_ped_key = "/dl1/monitoring/telescope/catB/pedestal" +dl1_mon_tel_CatB_cal_key = "/dl1/monitoring/telescope/catB/calibration" +dl1_mon_tel_CatB_flat_key = "/dl1/monitoring/telescope/catB/flatfield" + dl1_params_lstcam_key = "/dl1/event/telescope/parameters/LST_LSTCam" dl1_images_lstcam_key = "/dl1/event/telescope/image/LST_LSTCam" dl2_params_lstcam_key = "/dl2/event/telescope/parameters/LST_LSTCam" @@ -858,6 +862,7 @@ def write_calibration_data(writer, mon_index, mon_event, new_ped=False, new_ff=F mon_event.flatfield.prefix = '' mon_event.calibration.prefix = '' mon_index.prefix = '' + monitoring_table='telescope/monitoring' # update index if new_ped: @@ -870,20 +875,20 @@ def write_calibration_data(writer, mon_index, mon_event, new_ped=False, new_ff=F if new_ped: # write ped container writer.write( - table_name="telescope/monitoring/pedestal", + table_name=f"{monitoring_table}/pedestal", containers=[mon_index, mon_event.pedestal], ) if new_ff: # write calibration container writer.write( - table_name="telescope/monitoring/flatfield", + table_name=f"{monitoring_table}/flatfield", containers=[mon_index, mon_event.flatfield], ) # write ff container writer.write( - table_name="telescope/monitoring/calibration", + table_name=f"{monitoring_table}/calibration", containers=[mon_index, mon_event.calibration], ) diff --git a/lstchain/onsite.py b/lstchain/onsite.py index 728d5010ae..12fc99fe31 100644 --- a/lstchain/onsite.py +++ b/lstchain/onsite.py @@ -1,20 +1,40 @@ from glob import glob +import logging from pathlib import Path +from enum import Enum, auto from pkg_resources import resource_filename from datetime import datetime import tempfile +from astropy.time import Time +import pymongo from .paths import parse_calibration_name +log = logging.getLogger(__name__) + DEFAULT_BASE_PATH = Path('/fefs/aswg/data/real') DEFAULT_R0_PATH = DEFAULT_BASE_PATH / 'R0' -LEVEL_A_PIXEL_DIR = 'monitoring/PixelCalibration/Cat-A' +DEFAULT_DL1_PATH = DEFAULT_BASE_PATH / 'DL1' +CAT_A_PIXEL_DIR = 'monitoring/PixelCalibration/Cat-A' +CAT_B_PIXEL_DIR = 'monitoring/PixelCalibration/Cat-B' + DEFAULT_CONFIG = Path(resource_filename( 'lstchain', "data/onsite_camera_calibration_param.json", )) +DEFAULT_CONFIG_CAT_B_CALIB = Path(resource_filename( + 'lstchain', + "data/catB_camera_calibration_param.json", +)) +class DataCategory(Enum): + #: Real-Time data processing + A = auto() + #: Onsite processing + B = auto() + #: Offsite processing + C = auto() def is_date(s): try: @@ -72,13 +92,14 @@ def find_r0_subrun(run, sub_run, r0_dir=DEFAULT_R0_PATH): ''' Find the given subrun R0 file (i.e. globbing for the date part) ''' + file_list = rglob_symlinks(r0_dir, f'LST-1.1.Run{run:05d}.{sub_run:04d}*.fits.fz') # ignore directories that are not a date, e.g. "Trash" file_list = [p for p in file_list if is_date(p.parent.name)] if len(file_list) == 0: - raise IOError(f"Run {run} not found\n") + raise IOError(f"Run {run} not found in r0_dir {r0_dir} \n") if len(file_list) > 1: raise IOError(f"Found more than one file for run {run}.{sub_run}: {file_list}") @@ -88,7 +109,7 @@ def find_r0_subrun(run, sub_run, r0_dir=DEFAULT_R0_PATH): def find_pedestal_file(pro, pedestal_run=None, date=None, base_dir=DEFAULT_BASE_PATH): # pedestal base dir - ped_dir = Path(base_dir) / LEVEL_A_PIXEL_DIR / "drs4_baseline" + ped_dir = Path(base_dir) / CAT_A_PIXEL_DIR / "drs4_baseline" if pedestal_run is None and date is None: raise ValueError('Must give at least `date` or `run`') @@ -123,7 +144,7 @@ def find_run_summary(date, base_dir=DEFAULT_BASE_PATH): def find_time_calibration_file(pro, run, time_run=None, base_dir=DEFAULT_BASE_PATH): '''Find a time calibration file for given run ''' - time_dir = Path(base_dir) / LEVEL_A_PIXEL_DIR / "drs4_time_sampling_from_FF" + time_dir = Path(base_dir) / CAT_A_PIXEL_DIR / "drs4_time_sampling_from_FF" # search the last time run before or equal to the calibration run @@ -155,7 +176,7 @@ def find_time_calibration_file(pro, run, time_run=None, base_dir=DEFAULT_BASE_PA def find_systematics_correction_file(pro, date, sys_date=None, base_dir=DEFAULT_BASE_PATH): - sys_dir = Path(base_dir) / LEVEL_A_PIXEL_DIR / "ffactor_systematics" + sys_dir = Path(base_dir) / CAT_A_PIXEL_DIR / "ffactor_systematics" if sys_date is not None: path = (sys_dir / sys_date / pro / f"ffactor_systematics_{sys_date}.h5").resolve() @@ -171,3 +192,111 @@ def find_systematics_correction_file(pro, date, sys_date=None, base_dir=DEFAULT_ selected_date = next((day for day in sys_date_list if day <= date), sys_date_list[-1]) return (sys_dir / selected_date / pro / f"ffactor_systematics_{selected_date}.h5").resolve() + +def find_calibration_file(pro, calibration_run=None, date=None, category=DataCategory.A, base_dir=DEFAULT_BASE_PATH): + + if category == DataCategory.A: + cal_dir = Path(base_dir) / CAT_A_PIXEL_DIR / "calibration" + elif category == DataCategory.B: + cal_dir = Path(base_dir) / CAT_B_PIXEL_DIR / "calibration" + else: + raise ValueError(f'Argument \'category\' can be only \'DataCategory.A\' or \'DataCategory.B\', not {category}') + + if calibration_run is None and date is None: + raise ValueError('Must give at least `date` or `run`') + + if calibration_run is not None: + # search a specific calibration run + file_list = sorted(cal_dir.rglob(f'{pro}/calibration*.Run{calibration_run:05d}.0000.h5')) + + if len(file_list) == 0: + raise IOError(f"Calibration file from run {calibration_run} not found\n") + + return file_list[0].resolve() + + # search for a unique calibration file from the same date + file_list = sorted((cal_dir / date / pro).glob('calibration*.0000.h5')) + if len(file_list) == 0: + raise IOError(f"No calibration file found for date {date}") + + if len(file_list) > 1: + raise IOError(f"Too many calibration files found for date {date}: {file_list}, choose one run\n") + + return file_list[0].resolve() + +def find_DL1_subrun(run, sub_run, dl1_dir=DEFAULT_DL1_PATH): + ''' + Find the given subrun DL1 file (i.e. globbing for the date part) + ''' + file_list = rglob_symlinks(dl1_dir, f'dl1_LST-1.Run{run:05d}.{sub_run:04d}*.h5') + + # ignore directories that are not a date, e.g. "Trash" + file_list = [p for p in file_list if is_date(p.parent.name)] + + if len(file_list) == 0: + raise IOError(f"Run {run} not found\n") + + if len(file_list) > 1: + raise IOError(f"Found more than one file for run {run}.{sub_run}: {file_list}") + + return file_list[0] + +def find_interleaved_subruns(run, r0_dir=DEFAULT_R0_PATH, dl1_dir=DEFAULT_DL1_PATH): + ''' + Find the given subrun of interleaved file in onsite tree (dir [dl1_dir]/interleaved) + ''' + + # look in R0 to find the date + r0_list = find_r0_subrun(run,0,r0_dir) + date = r0_list.parent.name + + # search the files + interleaved_dir = Path(f"{dl1_dir}/{date}/interleaved") + + file_list = sorted(interleaved_dir.rglob(f'interleaved_LST-1.Run{run:05d}.*.h5')) + + if len(file_list) == 0: + raise IOError(f"Run {run} not found in interleaved dir {interleaved_dir}\n") + + return file_list + +def find_filter_wheels(run, database_url): + """read the employed filters from mongodb""" + + # there was a change of Mongo DB data names on 5/12/2022 + NEW_DB_NAMES_DATE = Time("2022-12-04T00:00:00") + + filters = None + try: + + myclient = pymongo.MongoClient(database_url) + + mydb = myclient["CACO"] + mycol = mydb["RUN_INFORMATION"] + mydoc = mycol.find({"run_number": {"$eq": run}}) + for x in mydoc: + date = Time(x["start_time"]) + if date < NEW_DB_NAMES_DATE: + w1 = int(x["cbox"]["wheel1 position"]) + w2 = int(x["cbox"]["wheel2 position"]) + else: + w1 = int(x["cbox"]["CBOX_WheelPosition1"]) + w2 = int(x["cbox"]["CBOX_WheelPosition2"]) + + filters = f"{w1:1d}{w2:1d}" + + except Exception as e: # In the case the entry says 'No available' + log.exception(f"\n >>> Exception: {e}") + raise IOError( + "--> No mongo DB filter information." + " You must pass the filters by argument: -f [filters]" + ) + + if filters is None: # In the case the entry is missing + raise IOError( + "--> No filter information in mongo DB." + " You must pass the filters by argument: -f [filters]" + ) + + return filters + \ No newline at end of file diff --git a/lstchain/reco/dl1_to_dl2.py b/lstchain/reco/dl1_to_dl2.py index fec9342004..7a7da9ad0c 100644 --- a/lstchain/reco/dl1_to_dl2.py +++ b/lstchain/reco/dl1_to_dl2.py @@ -622,7 +622,11 @@ def apply_models(dl1, + config['particle_classification_features'] + config['disp_classification_features'], ) - + # if no events in dl2, e.g. for bad time interval from Cat-B calibration + if len(dl2) == 0: + logger.warning("No events in dl2.") + return dl2 + # Update parameters related to target direction on camera frame for MC data # taking into account of the abrration effect using effective focal length is_simu = 'disp_norm' in dl2.columns diff --git a/lstchain/reco/r0_to_dl1.py b/lstchain/reco/r0_to_dl1.py index 059358b73e..f77d3d36e8 100644 --- a/lstchain/reco/r0_to_dl1.py +++ b/lstchain/reco/r0_to_dl1.py @@ -448,9 +448,13 @@ def r0_to_dl1( # initialize the writer of the interleaved events interleaved_writer = None - if 'write_interleaved_events' in config: + if 'write_interleaved_events' in config and not is_simu: interleaved_writer_config = Config(config['write_interleaved_events']) dir, name = os.path.split(output_filename) + + # create output dir in the data-tree if necessary + dir = f"{dir}/interleaved" + os.makedirs(dir, exist_ok=True) if 'dl1' in name: name = name.replace('dl1', 'interleaved').replace('LST-1.1', 'LST-1') else: @@ -507,6 +511,7 @@ def r0_to_dl1( add_config_metadata(container, config) # write the first calibration event (initialized from calibration h5 file) + # TODO: these data are supposed to change table_path with "dl1/monitoring/telescope/CatA" in short future write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], @@ -520,6 +525,7 @@ def r0_to_dl1( new_ped_event, new_ff_event = calibration_calculator.process_interleaved(event) # write monitoring containers if updated + # these data a supposed to be replaced by the Cat_B data in a short future if new_ped_event or new_ff_event: write_calibration_data(writer, calibration_index, @@ -761,6 +767,7 @@ def r0_to_dl1( # at the end of event loop ask calculation of remaining interleaved statistics new_ped, new_ff = calibration_calculator.output_interleaved_results(event) # write monitoring events + # these data a supposed to be replaced by the Cat_B data in a short future write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], diff --git a/lstchain/reco/utils.py b/lstchain/reco/utils.py index adb46e4c95..d419e8c814 100644 --- a/lstchain/reco/utils.py +++ b/lstchain/reco/utils.py @@ -46,7 +46,8 @@ "rotate", "sky_to_camera", "source_dx_dy", - "source_side" + "source_side", + "get_events_in_GTI" ] # position of the LST1 @@ -829,3 +830,25 @@ def apply_src_r_cut(events, src_r_min, src_r_max): ] return events + +def get_events_in_GTI(events, CatB_cal_table): + """ + Select events in good time intervals (GTI) on the base + of the GTI defined the catB calibration table (dl1_mon_tel_CatB_cal_key) + + Parameters + ---------- + events : pandas DataFrame or astropy.table.QTable + Data frame or table of DL1 or DL2 events. + CatB_cal_table: table of CatB calibration applied to the events (dl1_mon_tel_CatB_cal_key) + + Returns + ------- + sel_events: selected events + """ + + gti = CatB_cal_table['gti'] + + gti_mask = gti[events['calibration_id']] + + return events[gti_mask] \ No newline at end of file diff --git a/lstchain/scripts/lstchain_data_r0_to_dl1.py b/lstchain/scripts/lstchain_data_r0_to_dl1.py index 758123ac21..38adf8e72f 100644 --- a/lstchain/scripts/lstchain_data_r0_to_dl1.py +++ b/lstchain/scripts/lstchain_data_r0_to_dl1.py @@ -155,6 +155,9 @@ def main(): output_dir = args.output_dir.absolute() output_dir.mkdir(exist_ok=True, parents=True) + interleaved_dir = output_dir/"interleaved" + interleaved_dir.mkdir(exist_ok=True, parents=True) + if not args.input_file.is_file(): log.error('Input file does not exist or is not a file') sys.exit(1) diff --git a/lstchain/scripts/lstchain_dl1_to_dl2.py b/lstchain/scripts/lstchain_dl1_to_dl2.py index ac94ebc356..94714b7de2 100644 --- a/lstchain/scripts/lstchain_dl1_to_dl2.py +++ b/lstchain/scripts/lstchain_dl1_to_dl2.py @@ -9,7 +9,7 @@ import argparse from pathlib import Path import joblib - +import logging import numpy as np import pandas as pd import astropy.units as u @@ -40,7 +40,7 @@ from lstchain.reco import dl1_to_dl2 from lstchain.reco.utils import filter_events, impute_pointing, add_delta_t_key - +logger = logging.getLogger(__name__) parser = argparse.ArgumentParser(description=__doc__) # Required arguments @@ -76,6 +76,7 @@ def apply_to_file(filename, models_dict, output_dir, config): + data = pd.read_hdf(filename, key=dl1_params_lstcam_key) if 'lh_fit_config' in config.keys(): @@ -169,14 +170,14 @@ def apply_to_file(filename, models_dict, output_dir, config): ) if config['disp_method'] == 'disp_vector': - dl2_df = dl1_to_dl2.apply_models(data_with_srcdep_param, + dl2 = dl1_to_dl2.apply_models(data_with_srcdep_param, models_dict['cls_gh'], models_dict['reg_energy'], reg_disp_vector=models_dict['disp_vector'], effective_focal_length=effective_focal_length, custom_config=config) elif config['disp_method'] == 'disp_norm_sign': - dl2_df = dl1_to_dl2.apply_models(data_with_srcdep_param, + dl2 = dl1_to_dl2.apply_models(data_with_srcdep_param, models_dict['cls_gh'], models_dict['reg_energy'], reg_disp_norm=models_dict['disp_norm'], @@ -184,11 +185,16 @@ def apply_to_file(filename, models_dict, output_dir, config): effective_focal_length=effective_focal_length, custom_config=config) - dl2_srcdep = dl2_df.drop(srcindep_keys, axis=1) + dl2_srcdep = dl2.drop(srcindep_keys, axis=1) dl2_srcdep_dict[k] = dl2_srcdep if i == 0: - dl2_srcindep = dl2_df[srcindep_keys] + dl2_srcindep = dl2[srcindep_keys] + + # do not write file if empty + if len(dl2) == 0: + logger.warning("No dl2 output file written.") + return output_dir.mkdir(exist_ok=True) output_file = output_dir.joinpath(filename.name.replace('dl1', 'dl2', 1)) diff --git a/lstchain/scripts/lstchain_dl1ab.py b/lstchain/scripts/lstchain_dl1ab.py index 549e787c29..94ce67634f 100644 --- a/lstchain/scripts/lstchain_dl1ab.py +++ b/lstchain/scripts/lstchain_dl1ab.py @@ -44,6 +44,9 @@ from lstchain.io.io import ( dl1_images_lstcam_key, dl1_params_lstcam_key, + dl1_mon_tel_CatB_cal_key, + dl1_mon_tel_CatB_ped_key, + dl1_mon_tel_CatB_flat_key, global_metadata, write_metadata, ) @@ -76,6 +79,13 @@ help='path to the Cat-B calibration file ', ) +parser.add_argument( + '--max-unusable-pixels', + type=int, + default=70, + help='Maximum accepted number of unusable pixels. Default: 70 (= 10 modules)', +) + parser.add_argument( '-c', '--config', dest='config_file', @@ -132,6 +142,10 @@ def main(): catB_time_correction = np.array(catB_calib["time_correction"]) catB_unusable_pixels = np.array(catB_calib["unusable_pixels"]) + + # add good time interval column (gti) + catB_calib['gti'] = np.max(np.sum(catB_unusable_pixels, axis=2),axis=1) < args.max_unusable_pixels + pixel_index = np.arange(constants.N_PIXELS) @@ -235,7 +249,7 @@ def main(): } if catB_calib: - parameters_to_update["calibration_id"] = np.int32 + parameters_to_update["calibration_id"] = np.int32 nodes_keys = get_dataset_keys(args.input_file) if args.no_image: @@ -324,7 +338,7 @@ def main(): image[unusable_pixels] = 0 # time flafielding - peak_time = peak_time - time_correction + peak_time = peak_time + time_correction # store it to save it later image_table['image'][ii] = image @@ -440,9 +454,9 @@ def main(): # write a cat-B calibrations in DL1b if catB_calib: - write_table(catB_calib, outfile, "/dl1/event/telescope/monitoring/catB/calibration") - write_table(catB_pedestal, outfile, "/dl1/event/telescope/monitoring/catB/pedestal") - write_table(catB_flatfield, outfile, "/dl1/event/telescope/monitoring/catB/flatfield") + write_table(catB_calib, outfile, dl1_mon_tel_CatB_cal_key) + write_table(catB_pedestal, outfile, dl1_mon_tel_CatB_ped_key) + write_table(catB_flatfield, outfile, dl1_mon_tel_CatB_flat_key) write_metadata(metadata, args.output_file) diff --git a/lstchain/scripts/onsite/onsite_create_calibration_file.py b/lstchain/scripts/onsite/onsite_create_calibration_file.py index fd553152f5..aeb417edc1 100755 --- a/lstchain/scripts/onsite/onsite_create_calibration_file.py +++ b/lstchain/scripts/onsite/onsite_create_calibration_file.py @@ -13,8 +13,6 @@ import subprocess import sys from pathlib import Path -from astropy.time import Time -import pymongo import lstchain import lstchain.visualization.plot_calib as calib @@ -23,13 +21,14 @@ from lstchain.onsite import ( DEFAULT_BASE_PATH, DEFAULT_CONFIG, - LEVEL_A_PIXEL_DIR, + CAT_A_PIXEL_DIR, create_pro_symlink, find_r0_subrun, find_pedestal_file, find_run_summary, find_systematics_correction_file, find_time_calibration_file, + find_filter_wheels, ) # parse arguments @@ -80,7 +79,7 @@ optional.add_argument('--config', help="Config file", default=DEFAULT_CONFIG, type=Path) -optional.add_argument('--mongodb', help="Mongo data-base connection", default="mongodb://10.200.10.101:27017/") +optional.add_argument('--mongodb', help="Mongo data-base (CACO DB) connection.", default="mongodb://10.200.10.161:27018/") optional.add_argument('-y', '--yes', action="store_true", help='Do not ask interactively for permissions, assume true') optional.add_argument('--no_pro_symlink', action="store_true", help='Do not update the pro dir symbolic link, assume true') @@ -118,7 +117,7 @@ def main(): # looks for the filter values in the database if not given if args.filters is None: - filters = search_filter(run, args.mongodb) + filters = find_filter_wheels(run, args.mongodb) else: filters = args.filters @@ -146,7 +145,7 @@ def main(): print(f"\n--> Input file: {input_file}") # verify output dir - calib_dir = args.base_dir / LEVEL_A_PIXEL_DIR + calib_dir = args.base_dir / CAT_A_PIXEL_DIR output_dir = calib_dir / "calibration" / date / prod_id if not output_dir.exists(): print(f"--> Create directory {output_dir}") @@ -232,8 +231,8 @@ def main(): f"--LSTEventSource.LSTR0Corrections.drs4_time_calibration_path={time_file}", f"--LSTEventSource.LSTR0Corrections.drs4_pedestal_path={pedestal_file}", f"--LSTEventSource.use_flatfield_heuristic={args.use_flatfield_heuristic}", - f"--FlatFieldCalculator.sample_size={stat_events}", - f"--PedestalCalculator.sample_size={stat_events}", + f"--FlasherFlatFieldCalculator.sample_size={stat_events}", + f"--PedestalIntegrator.sample_size={stat_events}", f"--config={config_file}", f"--log-file={log_file}", "--log-file-level=INFO", @@ -248,46 +247,11 @@ def main(): print(f"\n--> PRODUCING PLOTS in {plot_file} ...") mon = read_calibration_file(output_file, tel_id) - calib.plot_calibration_results(mon.pedestal, mon.flatfield, mon.calibration, run, plot_file) + calib.plot_calibration_results(mon.pedestal, mon.flatfield, mon.calibration, run, plot_file,"Cat-A") print("\n--> END") -def search_filter(run, database_url): - """read the employed filters form mongodb""" - - # there was a change of Mongo DB data names on 5/12/2022 - NEW_DB_NAMES_DATE = Time("2022-12-04T00:00:00") - - filters = None - try: - - myclient = pymongo.MongoClient(database_url) - - mydb = myclient["CACO"] - mycol = mydb["RUN_INFORMATION"] - mydoc = mycol.find({"run_number": {"$eq": run}}) - for x in mydoc: - date = Time(x["start_time"]) - if date < NEW_DB_NAMES_DATE: - w1 = int(x["cbox"]["wheel1 position"]) - w2 = int(x["cbox"]["wheel2 position"]) - else: - w1 = int(x["cbox"]["CBOX_WheelPosition1"]) - w2 = int(x["cbox"]["CBOX_WheelPosition2"]) - - filters = f"{w1:1d}{w2:1d}" - - except Exception as e: - print(f"\n >>> Exception: {e}") - raise IOError( - "--> No mongo DB filter information." - " You must pass the filters by argument: -f [filters]" - ) - - return filters - - def define_FF_selection_range(filters): """ return the range of charges to select the FF events """ diff --git a/lstchain/scripts/onsite/onsite_create_calibration_files_with_batch.py b/lstchain/scripts/onsite/onsite_create_calibration_files_with_batch.py index 2b1be44229..2ca74567c6 100755 --- a/lstchain/scripts/onsite/onsite_create_calibration_files_with_batch.py +++ b/lstchain/scripts/onsite/onsite_create_calibration_files_with_batch.py @@ -18,7 +18,7 @@ import lstchain from lstchain.onsite import ( DEFAULT_BASE_PATH, - LEVEL_A_PIXEL_DIR, + CAT_A_PIXEL_DIR, find_r0_subrun, DEFAULT_CONFIG, ) @@ -111,7 +111,7 @@ def main(): output_base_name = args.output_base_name - calib_dir = base_dir / LEVEL_A_PIXEL_DIR + calib_dir = base_dir / CAT_A_PIXEL_DIR if shutil.which('srun') is None: sys.exit(">>> This script needs a slurm batch system. Stop") @@ -178,7 +178,6 @@ def main(): f"--sub_run={sub_run}", f"-b {base_dir}", f"-s {stat_events}", - f"--r0-dir={r0_dir}", f"--output_base_name={output_base_name}", f"--config={config_file}", ] @@ -207,6 +206,9 @@ def main(): if args.use_flatfield_heuristic is False: cmd.append("--no-flatfield-heuristic") + if args.no_pro_symlink is True: + cmd.append("--no_pro_symlink") + cmd.extend(remaining_args) # join command together with newline, line continuation and indentation diff --git a/lstchain/scripts/onsite/onsite_create_cat_B_calibration_file.py b/lstchain/scripts/onsite/onsite_create_cat_B_calibration_file.py new file mode 100755 index 0000000000..db996ea60a --- /dev/null +++ b/lstchain/scripts/onsite/onsite_create_cat_B_calibration_file.py @@ -0,0 +1,223 @@ +#!/usr//bin/env python + +""" + + Onsite script for creating a Cat-B flat-field calibration file to be run as a command line: + + --> onsite_create_cat_B_calibration_file + +""" + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +import lstchain +import lstchain.visualization.plot_calib as calib +from lstchain.io.data_management import query_yes_no +from lstchain.io import read_calibration_file +from lstchain.onsite import ( + DEFAULT_BASE_PATH, + DEFAULT_CONFIG_CAT_B_CALIB, + CAT_B_PIXEL_DIR, + create_pro_symlink, + find_interleaved_subruns, + find_systematics_correction_file, + find_calibration_file, + find_filter_wheels, +) + +MAX_SUBRUNS = 100000 + +# parse arguments +parser = argparse.ArgumentParser(description='Create flat-field calibration Cat-B files', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +required = parser.add_argument_group('required arguments') +optional = parser.add_argument_group('optional arguments') + +required.add_argument('-r', '--run_number', help="Run number of interleaved data", + type=int, required=True) + +version = lstchain.__version__ + +optional.add_argument('-c', '--catA_calibration_run', + help="Cat-A calibration run to be used. If None, it looks for the calibration run of the date of the interleaved data.", + type=int) +optional.add_argument('-v', '--prod_version', help="Version of the production", + default=f"v{version}") +optional.add_argument('-s', '--statistics', help="Number of events for the flat-field and pedestal statistics", + type=int, default=2500) +optional.add_argument('-b', '--base_dir', help="Root dir for the output directory tree", type=Path, + default=DEFAULT_BASE_PATH) +optional.add_argument('--dl1-dir', help="Root dir for the input DL1 tree. By default, /DL1 will be used", + type=Path) +optional.add_argument('--r0-dir', help="Root dir for the input r0 tree. By default, /R0 will be used", + type=Path) + +optional.add_argument( + '--sys_date', + help=( + "Date of systematic correction file (format YYYYMMDD). \n" + "Default: automatically search the best date \n" + ), +) +optional.add_argument('--no_sys_correction', + help="Systematic corrections are not applied. \n", + action='store_true', + default=False) +optional.add_argument('--output_base_name', help="Base of output file name (change only for debugging)", + default="calibration") + +optional.add_argument('--n_subruns', help="Number of subruns to be processed", + type=int, default=MAX_SUBRUNS) + +optional.add_argument('-f', '--filters', help="Calibox filters") +optional.add_argument('--tel_id', help="telescope id. Default = 1", type=int, default=1) + +optional.add_argument('--config', help="Config file", default=DEFAULT_CONFIG_CAT_B_CALIB, type=Path) +optional.add_argument('--mongodb', help="Mongo data-base (CACO DB) connection.", default="mongodb://10.200.10.161:27018/") + +optional.add_argument('-y', '--yes', action="store_true", help='Do not ask interactively for permissions, assume true') +optional.add_argument('--no_pro_symlink', action="store_true", + help='Do not update the pro dir symbolic link, assume true') + + +def main(): + args, remaining_args = parser.parse_known_args() + run = args.run_number + n_subruns = args.n_subruns + prod_id = args.prod_version + stat_events = args.statistics + + sys_date = args.sys_date + no_sys_correction = args.no_sys_correction + tel_id = args.tel_id + config_file = args.config + yes = args.yes + pro_symlink = not args.no_pro_symlink + r0_dir = args.r0_dir or args.base_dir / 'R0' + dl1_dir = args.dl1_dir or args.base_dir / 'DL1' + + + # looks for the filter values in the database if not given + if args.filters is None: + filters = find_filter_wheels(run, args.mongodb) + else: + filters = args.filters + + if filters is None: + sys.exit(f"Missing filter value for run {run}. \n") + + print(f"\n--> Start calculating Cat-B calibration from run {run}, filters {filters}") + + # verify config file + if not config_file.exists(): + raise IOError(f"Config file {config_file} does not exists. \n") + + print(f"\n--> Config file {config_file}") + + # verify input file + input_files = find_interleaved_subruns(run, r0_dir, dl1_dir) + input_path = input_files[0].parent + print(f"\n--> Found {len(input_files)} interleaved subruns in {input_path}") + if n_subruns < MAX_SUBRUNS: + print(f"--> Process {n_subruns} subruns") + + # verify output dir + date = input_files[0].parents[1].name + calib_dir = args.base_dir / CAT_B_PIXEL_DIR + output_dir = calib_dir / "calibration" / date / prod_id + if not output_dir.exists(): + print(f"\n--> Create directory {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + + if pro_symlink: + pro = "pro" + create_pro_symlink(output_dir) + else: + pro = prod_id + + # make log dir + log_dir = output_dir / "log" + if not log_dir.exists(): + print(f"--> Create directory {log_dir}") + log_dir.mkdir(parents=True, exist_ok=True) + + cat_A_calib_file = find_calibration_file(pro, args.catA_calibration_run, date=date, base_dir=args.base_dir) + print(f"\n--> Cat-A calibration file: {cat_A_calib_file}") + + # define systematic correction file + if no_sys_correction: + systematics_file = None + else: + systematics_file = find_systematics_correction_file(pro, date, sys_date, args.base_dir) + + print(f"\n--> F-factor systematics correction file: {systematics_file}") + + # define charge file names + print("\n***** PRODUCE CAT_B CALIBRATION FILE ***** ") + + if filters is not None: + filter_info = f"_filters_{filters}" + else: + filter_info = "" + + input_file_pattern=f"interleaved_LST-1.Run{run:05d}.*.h5" + output_name = f"cat_B_calibration{filter_info}.Run{run:05d}" + + output_file = output_dir / f'{output_name}.h5' + print(f"\n--> Output file {output_file}") + + log_file = log_dir / f"{output_name}.log" + print(f"\n--> Log file {log_file}") + + if output_file.exists(): + remove = False + + if not yes and os.getenv('SLURM_JOB_ID') is None: + remove = query_yes_no(">>> Output file exists already. Do you want to remove it?") + + if yes or remove: + os.remove(output_file) + os.remove(log_file) + else: + print("\n--> Output file exists already. Stop") + exit(1) + + # + # produce ff calibration file + # + + cmd = [ + "lstchain_create_cat_B_calibration_file", + f"--input_path={input_path}", + f"--output_file={output_file}", + f"--input_file_pattern={input_file_pattern}", + f"--n_subruns={n_subruns}", + f"--cat_A_calibration_file={cat_A_calib_file}", + f"--LSTCalibrationCalculator.systematic_correction_path={systematics_file}", + f"--FlasherFlatFieldCalculator.sample_size={stat_events}", + f"--PedestalIntegrator.sample_size={stat_events}", + f"--config={config_file}", + f"--log-file={log_file}", + "--log-file-level=INFO", + *remaining_args, + ] + + print("\n--> RUNNING...") + subprocess.run(cmd, check=True) + + # plot and save some results + plot_file = f"{output_dir}/log/{output_name}.pdf" + + print(f"\n--> PRODUCING PLOTS in {plot_file} ...") + mon = read_calibration_file(output_file, tel_id) + calib.plot_calibration_results(mon.pedestal, mon.flatfield, mon.calibration, run, plot_file,"Cat-B") + + print("\n--> END") + +if __name__ == '__main__': + main() + diff --git a/lstchain/scripts/onsite/onsite_create_cat_B_calibration_files_with_batch.py b/lstchain/scripts/onsite/onsite_create_cat_B_calibration_files_with_batch.py new file mode 100755 index 0000000000..be6b61f50e --- /dev/null +++ b/lstchain/scripts/onsite/onsite_create_cat_B_calibration_files_with_batch.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python + +""" + + Onsite script to process (in batch) the interleaved events of several runs + and to create the Cat-B calibration files + + --> onsite_create_catB_calibration_files_with_batch -r xxx yyy zzz + +""" + +import argparse +import shutil +import subprocess +import sys +from datetime import datetime +from pathlib import Path + +import lstchain +from lstchain.onsite import ( + DEFAULT_BASE_PATH, + CAT_B_PIXEL_DIR, + find_interleaved_subruns, + DEFAULT_CONFIG_CAT_B_CALIB, +) + +# parse arguments +parser = argparse.ArgumentParser( + description='Reconstruct filter scan, this must be run after the night calibration scripts', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +required = parser.add_argument_group('required arguments') +optional = parser.add_argument_group('optional arguments') + +required.add_argument('-r', '--run_list', help="Run numbers of intereleaved data", + type=int, nargs="+") +optional.add_argument('-f', '--filters_list', help="Filter list (same order as run list)", + type=int, nargs="+") +version = lstchain.__version__ +optional.add_argument('-v', '--prod_version', + help="Version of the production", + default=f"v{version}") +optional.add_argument('-s', '--statistics', + help="Number of events for the flat-field and pedestal statistics", + type=int, + default=2500) +optional.add_argument('-b', '--base_dir', help="Root dir for the output directory tree", type=Path, + default=DEFAULT_BASE_PATH) +optional.add_argument('--dl1-dir', help="Root dir for the input r tree. By default, /DL1 will be used", + type=Path) +optional.add_argument('--r0-dir', help="Root dir for the input r0 tree. By default, /R0 will be used", + type=Path) + +optional.add_argument('--n_subruns', help="Number of subruns to be processed", + type=int) + +optional.add_argument('--sys_date', + help="Date of systematic corrections (format YYYYMMDD). \n" + "Default: automatically search the best date \n") +optional.add_argument('--no_sys_correction', + help="Systematic corrections are not applied. \n", + action='store_true', + default=False) + +optional.add_argument('-y', '--yes', action="store_true", help='Do not ask interactively for permissions, assume true') +optional.add_argument('--no_pro_symlink', action="store_true", + help='Do not update the pro dir symbolic link, assume true') + +optional.add_argument('--config', help="Config file", default=DEFAULT_CONFIG_CAT_B_CALIB, type=Path) + + +optional.add_argument('--queue', + help="Slurm queue. Deafault: short ", + default="short") + + +def main(): + args, remaining_args = parser.parse_known_args() + run_list = args.run_list + n_subruns = args.n_subruns + + filters_list = args.filters_list + + prod_id = args.prod_version + stat_events = args.statistics + base_dir = args.base_dir + + config_file = args.config + sys_date = args.sys_date + no_sys_correction = args.no_sys_correction + yes = args.yes + queue = args.queue + r0_dir = args.r0_dir or args.base_dir / 'R0' + dl1_dir = args.dl1_dir or Path(args.base_dir) / 'DL1' + + calib_dir = base_dir / CAT_B_PIXEL_DIR + + if shutil.which('srun') is None: + sys.exit(">>> This script needs a slurm batch system. Stop") + + print(f"\n--> Start to reconstruct runs {run_list}") + + # verify config file + if not config_file.exists(): + sys.exit(f"Config file {config_file} does not exists. \n") + + print(f"\n--> Config file {config_file}") + + # for old runs or if the data-base is not available + # it is possible to give the filter list + if filters_list is not None and len(filters_list) != len(run_list): + sys.exit("Filter list length must be equal to run list length. Verify \n") + + # loops over runs and send jobs + filters = None + for i, run in enumerate(run_list): + print(f"\n--> Run {run} ") + if filters_list is not None: + filters = filters_list[i] + + input_files = find_interleaved_subruns(run, r0_dir, dl1_dir) + + date = input_files[0].parent.name + input_path = input_files[0].parent + print(f"--> Found {len(input_files)} interleaved subruns in {input_path}") + if n_subruns: + print(f"--> Process {n_subruns} subruns") + + # verify output dir + date = input_files[0].parents[1].name + calib_dir = args.base_dir / CAT_B_PIXEL_DIR + output_dir = calib_dir / "calibration" / date / prod_id + if not output_dir.exists(): + print(f"--> Create directory {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + + # make log dir + log_dir = output_dir / "log" + if not log_dir.exists(): + print(f"--> Create directory {log_dir}") + log_dir.mkdir(parents=True, exist_ok=True) + + # job file + now = datetime.now().replace(microsecond=0).isoformat(sep='T') + job_file = log_dir / f"run_{run}_date_{now}.job" + + with job_file.open(mode="w") as fh: + fh.write("#!/bin/bash\n") + fh.write("#SBATCH --job-name=%s.job\n" % run) + fh.write("#SBATCH --output=log/run_%s_date_%s.out\n" % (run, now)) + fh.write("#SBATCH --error=log/run_%s_date_%s.err\n" % (run, now)) + fh.write("#SBATCH -p %s\n" % queue) + fh.write("#SBATCH --cpus-per-task=1\n") + fh.write("#SBATCH --mem-per-cpu=10G\n") + fh.write("#SBATCH -D %s \n" % output_dir) + + cmd = [ + "srun", + "onsite_create_cat_B_calibration_file", + f"-r {run}", + f"-v {prod_id}", + f"--dl1-dir {dl1_dir}", + f"--r0-dir {r0_dir}", + f"-b {base_dir}", + f"-s {stat_events}", + f"--config={config_file}", + ] + + + if filters is not None: + cmd.append(f"--filters={filters}") + + if sys_date is not None: + cmd.append(f"--sys_date={sys_date}") + + if yes: + cmd.append("--yes") + + if no_sys_correction: + cmd.append("--no_sys_correction") + + if n_subruns: + cmd.append(f"--n_subruns={n_subruns}") + + if args.no_pro_symlink is True: + cmd.append("--no_pro_symlink") + + cmd.extend(remaining_args) + + # join command together with newline, line continuation and indentation + fh.write(" \\\n ".join(cmd)) + fh.write('\n') + + subprocess.run(["sbatch", job_file], check=True) + + +if __name__ == '__main__': + main() diff --git a/lstchain/scripts/onsite/onsite_create_drs4_pedestal_file.py b/lstchain/scripts/onsite/onsite_create_drs4_pedestal_file.py index 1a1e10a31f..08a98bdcf1 100755 --- a/lstchain/scripts/onsite/onsite_create_drs4_pedestal_file.py +++ b/lstchain/scripts/onsite/onsite_create_drs4_pedestal_file.py @@ -17,7 +17,7 @@ from lstchain.io.data_management import query_yes_no from lstchain.onsite import ( DEFAULT_BASE_PATH, - LEVEL_A_PIXEL_DIR, + CAT_A_PIXEL_DIR, create_pro_symlink, find_r0_subrun, ) @@ -70,7 +70,7 @@ def main(): date = input_file.parent.name # verify and make output dir - calib_dir = base_dir / LEVEL_A_PIXEL_DIR + calib_dir = base_dir / CAT_A_PIXEL_DIR output_dir = calib_dir / "drs4_baseline" / date / prod_id if not output_dir.exists(): print(f"--> Create directory {output_dir}") diff --git a/lstchain/scripts/onsite/onsite_create_drs4_time_file.py b/lstchain/scripts/onsite/onsite_create_drs4_time_file.py index e2d7270ca1..0c5d898601 100755 --- a/lstchain/scripts/onsite/onsite_create_drs4_time_file.py +++ b/lstchain/scripts/onsite/onsite_create_drs4_time_file.py @@ -16,7 +16,7 @@ from lstchain.onsite import ( DEFAULT_BASE_PATH, DEFAULT_CONFIG, - LEVEL_A_PIXEL_DIR, + CAT_A_PIXEL_DIR, create_pro_symlink, find_r0_subrun, find_pedestal_file, @@ -98,7 +98,7 @@ def main(): print(f"\n--> Input file: {input_file}") # verify output dir - calib_dir = base_dir / LEVEL_A_PIXEL_DIR + calib_dir = base_dir / CAT_A_PIXEL_DIR output_dir = calib_dir / "drs4_time_sampling_from_FF" / date / prod_id if not output_dir.exists(): diff --git a/lstchain/scripts/onsite/onsite_create_ffactor_systematics_file.py b/lstchain/scripts/onsite/onsite_create_ffactor_systematics_file.py index f4ad9d5b23..a0b3c5c2e4 100755 --- a/lstchain/scripts/onsite/onsite_create_ffactor_systematics_file.py +++ b/lstchain/scripts/onsite/onsite_create_ffactor_systematics_file.py @@ -12,7 +12,7 @@ import lstchain from lstchain.io.data_management import query_yes_no -from lstchain.onsite import LEVEL_A_PIXEL_DIR, create_pro_symlink, DEFAULT_BASE_PATH +from lstchain.onsite import CAT_A_PIXEL_DIR, create_pro_symlink, DEFAULT_BASE_PATH def none_or_str(value): if value == "None": @@ -54,7 +54,7 @@ def main(): prefix = args.input_prefix yes = args.yes pro_symlink = not args.no_pro_symlink - calib_dir = base_dir / LEVEL_A_PIXEL_DIR + calib_dir = base_dir / CAT_A_PIXEL_DIR # verify config file if not config_file.exists(): diff --git a/lstchain/scripts/tests/test_lstchain_scripts.py b/lstchain/scripts/tests/test_lstchain_scripts.py index c425abde7b..29176f0cd9 100644 --- a/lstchain/scripts/tests/test_lstchain_scripts.py +++ b/lstchain/scripts/tests/test_lstchain_scripts.py @@ -93,9 +93,11 @@ def test_lstchain_data_r0_to_dl1(observed_dl1_files): assert observed_dl1_files["dl1_file1"].is_file() assert observed_dl1_files["muons1"].is_file() assert observed_dl1_files["datacheck1"].is_file() + assert observed_dl1_files["interleaved_file1"].is_file() assert observed_dl1_files["dl1_file2"].is_file() assert observed_dl1_files["muons2"].is_file() assert observed_dl1_files["datacheck2"].is_file() + assert observed_dl1_files["interleaved_file2"].is_file() @pytest.mark.private_data diff --git a/lstchain/tests/test_lstchain.py b/lstchain/tests/test_lstchain.py index ffd09aac02..61764ff8cf 100644 --- a/lstchain/tests/test_lstchain.py +++ b/lstchain/tests/test_lstchain.py @@ -13,7 +13,7 @@ from lstchain.reco.utils import filter_events from lstchain.reco.dl1_to_dl2 import build_models -test_data = Path(os.getenv('LSTCHAIN_TEST_DATA', 'test_data')) +test_data = Path(os.getenv('LSTCHAIN_TEST_DATA', 'test_data')).absolute() test_r0_path = test_data / 'real/R0/20200218/LST-1.1.Run02008.0000_first50.fits.fz' test_r0_path2 = test_data / 'real/R0/20200218/LST-1.1.Run02008.0100_first50.fits.fz' test_drs4_r0_path = test_data / 'real/R0/20200218/LST-1.1.Run02005.0000_first50.fits.fz' @@ -24,6 +24,8 @@ test_drs4_pedestal_path = calib_path / f'drs4_baseline/20200218/{calib_version}/drs4_pedestal.Run02005.0000.h5' test_time_calib_path = calib_path / f'drs4_time_sampling_from_FF/20191124/{calib_version}/time_calibration.Run01625.0000.h5' test_drive_report = test_data / 'real/monitoring/DrivePositioning/DrivePosition_log_20200218.txt' +test_run_summary_path = test_data / 'real/monitoring/RunSummary/RunSummary_20200218.ecsv' +test_systematics_path = test_data / 'real/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/ctapipe-v0.17/ffactor_systematics_20200725.h5' def test_r0_to_dl1(tmp_path, mc_gamma_testfile): @@ -236,6 +238,10 @@ def test_get_source_dependent_parameters_observed(observed_dl1_files): src_dep_df_wobble['off_180']['expected_src_y'], 0., atol=1e-2 ) +@pytest.mark.private_data +def test_get_interleaved_r1_file(interleaved_r1_file): + assert interleaved_r1_file.is_file() + def test_build_models(simulated_dl1_file, rf_models): infile = simulated_dl1_file diff --git a/lstchain/tests/test_onsite.py b/lstchain/tests/test_onsite.py index 8623616b99..4bb1362c65 100644 --- a/lstchain/tests/test_onsite.py +++ b/lstchain/tests/test_onsite.py @@ -176,7 +176,6 @@ def test_find_systematics_correction_file(): path = find_systematics_correction_file(pro=PRO, date='20200218', sys_date='20190101', base_dir=BASE_DIR) - def test_rglob_symlinks(tmp_path): from lstchain.onsite import rglob_symlinks @@ -218,3 +217,27 @@ def test_rglob_symlinks(tmp_path): matches = rglob_symlinks(r0, "run*.dat") # check we get an iterator and not a list assert len(list(matches)) == 6 + +@pytest.mark.private_data +def test_find_calibration_file(): + from lstchain.onsite import find_calibration_file + + # find by run_id + path = find_calibration_file(pro=PRO, calibration_run=9506, base_dir=BASE_DIR) + assert path.name == 'calibration_filters_52.Run09506.0000.h5' + + # find by night + path = find_calibration_file(pro=PRO, date='20200218', base_dir=BASE_DIR) + assert path.name == 'calibration_filters_52.Run02006.0000.h5' + + # if both are given, run takes precedence + path = find_calibration_file(pro=PRO, calibration_run=2006, date='20191124', base_dir=BASE_DIR) + assert path.name == 'calibration_filters_52.Run02006.0000.h5' + + with pytest.raises(IOError): + # if many calibration runs in one date + find_calibration_file(pro=PRO, date='20221001', base_dir=BASE_DIR) + + with pytest.raises(IOError): + # wrong run + find_calibration_file(pro=PRO, calibration_run=2010, base_dir=BASE_DIR) diff --git a/lstchain/tools/lstchain_create_cat_B_calibration_file.py b/lstchain/tools/lstchain_create_cat_B_calibration_file.py index 27235cbcef..7fe7524bbb 100644 --- a/lstchain/tools/lstchain_create_cat_B_calibration_file.py +++ b/lstchain/tools/lstchain_create_cat_B_calibration_file.py @@ -4,7 +4,7 @@ import numpy as np from pathlib import Path from copy import deepcopy -from traitlets import Unicode, Bool +from traitlets import Unicode, Bool, Integer from tqdm.auto import tqdm from ctapipe.instrument.subarray import SubarrayDescription from ctapipe.core import Provenance, traits @@ -57,6 +57,11 @@ class CatBCalibrationHDF5Writer(Tool): help='Pattern for searching the input files with interleaved events to be processed' ).tag(config=True) + n_subruns = Integer( + 1000000, + help='Number of subruns to be processed' + ).tag(config=True) + cat_A_calibration_file = Unicode( 'catA_calibration.hdf5', help='Name of category A calibration file' @@ -75,6 +80,7 @@ class CatBCalibrationHDF5Writer(Tool): ("s", "systematics_file"): "LSTCalibrationCalculator.systematic_correction_path", ("input_file_pattern"): 'CatBCalibrationHDF5Writer.input_file_pattern', ("input_path"): 'CatBCalibrationHDF5Writer.input_path', + ("n_subruns"): 'CatBCalibrationHDF5Writer.n_subruns', } @@ -109,9 +115,22 @@ def __init__(self, **kwargs): def setup(self): - self.log.info("Opening file") - self.input_paths = sorted(Path(f"{self.input_path}").rglob(f"{self.input_file_pattern}")) + + tot_subruns = len(self.input_paths) + if tot_subruns == 0: + self.log.critical(f"= No interleaved files found to be processed in " + f"{self.input_path} with pattern {self.input_file_pattern}") + self.exit(1) + + if self.n_subruns > tot_subruns: + self.n_subruns = tot_subruns + + # keep only the requested subruns + self.input_paths = self.input_paths[:self.n_subruns] + + self.log.info(f"Process {self.n_subruns} subruns ") + self.subarray = SubarrayDescription.from_hdf(self.input_paths[0]) self.processor = CalibrationCalculator.from_name( @@ -121,7 +140,6 @@ def setup(self): ) tel_id = self.processor.tel_id - group_name = 'tel_' + str(tel_id) self.log.info(f"Open output file {self.output_file}") @@ -150,7 +168,7 @@ def start(self): monitoring_data = deepcopy(self.cat_A_monitoring_data) for path in self.input_paths: - self.log.info(f"read {path}") + self.log.debug(f"read {path}") with EventSource(path,parent=self) as eventsource: @@ -180,7 +198,7 @@ def start(self): # if pedestal event if self._is_pedestal(event, tel_id): - + if self.processor.pedestal.calculate_pedestals(event): new_ped = True count_ped = count+1 @@ -237,7 +255,6 @@ def start(self): if stop: break - def finish(self): self.log.info(f"Written {self.n_calib} calibration events") diff --git a/lstchain/tools/tests/test_create_calibration_files.py b/lstchain/tools/tests/test_create_calibration_files.py new file mode 100644 index 0000000000..6e03ebcb4e --- /dev/null +++ b/lstchain/tools/tests/test_create_calibration_files.py @@ -0,0 +1,57 @@ +import numpy as np + +from ctapipe.core import run_tool +from ctapipe_io_lst.constants import N_GAINS, N_PIXELS +from ctapipe.io import read_table +from lstchain.onsite import DEFAULT_CONFIG + +from lstchain.tests.test_lstchain import ( + test_data, + test_drs4_pedestal_path, + test_time_calib_path, + test_run_summary_path, + test_systematics_path +) + + +def test_create_calibration_file(tmp_path): + '''Test the lstchain_create_calibration_file tool''' + from lstchain.tools.lstchain_create_calibration_file import CalibrationHDF5Writer + + input_path = test_data / "real/R0/20200218/LST-1.1.Run02006.0000_first50.fits.fz" + output_path = tmp_path / "calibration_02006.h5" + stat_events = 90 + + ret = run_tool( + CalibrationHDF5Writer(), + argv=[ + f"--input_file={input_path}", + f"--output_file={output_path}", + f"--LSTCalibrationCalculator.systematic_correction_path={test_systematics_path}", + f"--LSTEventSource.EventTimeCalculator.run_summary_path={test_run_summary_path}", + f"--LSTEventSource.LSTR0Corrections.drs4_time_calibration_path={test_time_calib_path}", + f"--LSTEventSource.LSTR0Corrections.drs4_pedestal_path={test_drs4_pedestal_path}", + "--LSTEventSource.default_trigger_type=tib", + f"--FlasherFlatFieldCalculator.sample_size={stat_events}", + f"--PedestalIntegrator.sample_size={stat_events}", + "--events_to_skip=0", + f"--config={DEFAULT_CONFIG}", + "--overwrite", + ], + cwd=tmp_path, + ) + + assert ret == 0, 'Running CalibrationHDF5Writer tool failed' + assert output_path.is_file(), 'Output file not written' + + cal_data = read_table(output_path, '/tel_1/calibration')[0] + + n_pe = cal_data['n_pe'] + unusable_pixels = cal_data['unusable_pixels'] + dc_to_pe = cal_data["dc_to_pe"] + + assert n_pe.shape == (N_GAINS, N_PIXELS) + assert np.sum(unusable_pixels) == 16 + assert np.isclose(np.median(n_pe[~unusable_pixels]), 86.45, rtol=0.1) + assert np.isclose(np.median(dc_to_pe[~unusable_pixels], axis=0), 0.0135, rtol=0.01) + diff --git a/lstchain/tools/tests/test_create_cat_B_calibration_files.py b/lstchain/tools/tests/test_create_cat_B_calibration_files.py new file mode 100644 index 0000000000..18c37ffbf0 --- /dev/null +++ b/lstchain/tools/tests/test_create_cat_B_calibration_files.py @@ -0,0 +1,50 @@ +import numpy as np + +from ctapipe.core import run_tool +from ctapipe_io_lst.constants import N_GAINS, N_PIXELS +from ctapipe.io import read_table +from lstchain.onsite import DEFAULT_CONFIG_CAT_B_CALIB + +from lstchain.tests.test_lstchain import ( + test_systematics_path, + test_calib_path +) + +def test_create_catB_calibration_file(tmp_path,interleaved_r1_file): + '''Test the lstchain_create_cat_B_calibration_file tool''' + from lstchain.tools.lstchain_create_cat_B_calibration_file import CatBCalibrationHDF5Writer + + input_path = interleaved_r1_file.parent + output_path = tmp_path / "calibration_cat_B_02006.h5" + stat_events = 90 + + ret = run_tool( + CatBCalibrationHDF5Writer(), + argv=[ + f"--input_path={input_path}", + f"--output_file={output_path}", + f"--cat_A_calibration_file={test_calib_path}", + f"--LSTCalibrationCalculator.systematic_correction_path={test_systematics_path}", + f"--FlasherFlatFieldCalculator.sample_size={stat_events}", + f"--PedestalIntegrator.sample_size={stat_events}", + f"--config={DEFAULT_CONFIG_CAT_B_CALIB}", + "--overwrite", + ], + cwd=tmp_path, + ) + + assert ret == 0, 'Running CalibrationHDF5Writer tool failed' + assert output_path.is_file(), 'Output file not written' + + cal_data = read_table(output_path, '/tel_1/calibration')[0] + + n_pe = cal_data['n_pe'] + unusable_pixels = cal_data['unusable_pixels'] + dc_to_pe = cal_data["dc_to_pe"] + + assert n_pe.shape == (N_GAINS, N_PIXELS) + + assert np.sum(unusable_pixels) == 4 + assert np.isclose(np.median(n_pe[~unusable_pixels]), 86.34, rtol=0.1) + assert np.isclose(np.median(dc_to_pe[~unusable_pixels], axis=0), 1.07, rtol=0.01) + diff --git a/lstchain/visualization/plot_calib.py b/lstchain/visualization/plot_calib.py index 59cdd73e6b..eacd7a8d77 100644 --- a/lstchain/visualization/plot_calib.py +++ b/lstchain/visualization/plot_calib.py @@ -1,11 +1,15 @@ import numpy as np from astropy import units as u +import logging +import sys from ctapipe.coordinates import EngineeringCameraFrame from ctapipe.visualization import CameraDisplay from ctapipe_io_lst import load_camera_geometry from matplotlib import pyplot as plt from matplotlib.backends.backend_pdf import PdfPages +log = logging.getLogger(__name__) + __all__ = [ "plot_calibration_results", ] @@ -15,7 +19,7 @@ plot_dir = "none" -def plot_calibration_results(ped_data, ff_data, calib_data, run=0, plot_file=None): +def plot_calibration_results(ped_data, ff_data, calib_data, run=0, plot_file=None, calib_type="Cat-A"): """ plot camera calibration quantities @@ -32,6 +36,17 @@ def plot_calibration_results(ped_data, ff_data, calib_data, run=0, plot_file=Non plot_file: name of the output PDF file. No file is produced if name is not provided """ + + if calib_type == "Cat-A": + charge_unit = "[ADC]" + gain_unit = "[ADC/pe]" + elif calib_type == "Cat-B": + charge_unit = "[pe]" + gain_unit = "[cat-B/cat-A]" + else: + log.critical(f'Wrong calib_type: {calib_type}. It must be Cat-A or Cat-B') + sys.exit(1) + # read geometry camera = load_camera_geometry() camera = camera.transform_to(EngineeringCameraFrame()) @@ -45,7 +60,7 @@ def plot_calibration_results(ped_data, ff_data, calib_data, run=0, plot_file=Non # first figure fig = plt.figure(1, figsize=(12, 24)) plt.tight_layout() - fig.suptitle(f"Run {run}", fontsize=25) + fig.suptitle(f"Run {run}, {calib_type} calibration", fontsize=25) pad = 420 image = ff_data.charge_median mask = ff_data.charge_median_outliers @@ -57,13 +72,13 @@ def plot_calibration_results(ped_data, ff_data, calib_data, run=0, plot_file=Non select = np.logical_not(mask[chan]) disp = CameraDisplay(camera) mymin = np.median(image[chan][select]) - 2 * np.std(image[chan][select]) - mymax = np.median(image[chan][select]) + 2 * np.std(image[chan][select]) - disp.set_limits_minmax(mymin, mymax) + mymax = np.median(image[chan][select]) + 2 * np.std(image[chan][select]) disp.highlight_pixels(mask[chan], linewidth=2) disp.image = image[chan] disp.cmap = plt.cm.coolwarm - # disp.axes.text(lposx, 0, f'{channel[chan]} signal charge (ADC)', rotation=90) - plt.title(f"{channel[chan]} signal charge [ADC]") + disp.set_limits_minmax(mymin, mymax) + + plt.title(f"{channel[chan]} signal charge {charge_unit}") disp.add_colorbar() image = ff_data.charge_std @@ -76,12 +91,12 @@ def plot_calibration_results(ped_data, ff_data, calib_data, run=0, plot_file=Non disp = CameraDisplay(camera) mymin = np.median(image[chan][select]) - 2 * np.std(image[chan][select]) mymax = np.median(image[chan][select]) + 2 * np.std(image[chan][select]) - disp.set_limits_minmax(mymin, mymax) disp.highlight_pixels(mask[chan], linewidth=2) disp.image = image[chan] + disp.set_limits_minmax(mymin, mymax) disp.cmap = plt.cm.coolwarm # disp.axes.text(lposx, 0, f'{channel[chan]} signal std [ADC]', rotation=90) - plt.title(f"{channel[chan]} signal std [ADC]") + plt.title(f"{channel[chan]} signal std {charge_unit}") disp.add_colorbar() image = ped_data.charge_median @@ -94,12 +109,12 @@ def plot_calibration_results(ped_data, ff_data, calib_data, run=0, plot_file=Non disp = CameraDisplay(camera) mymin = np.median(image[chan][select]) - 2 * np.std(image[chan][select]) mymax = np.median(image[chan][select]) + 2 * np.std(image[chan][select]) - disp.set_limits_minmax(mymin, mymax) disp.highlight_pixels(mask[chan], linewidth=2) disp.image = image[chan] + disp.set_limits_minmax(mymin, mymax) disp.cmap = plt.cm.coolwarm # disp.axes.text(lposx, 0, f'{channel[chan]} pedestal [ADC]', rotation=90) - plt.title(f"{channel[chan]} pedestal [ADC]") + plt.title(f"{channel[chan]} pedestal {charge_unit}") disp.add_colorbar() image = ped_data.charge_std @@ -112,12 +127,13 @@ def plot_calibration_results(ped_data, ff_data, calib_data, run=0, plot_file=Non disp = CameraDisplay(camera) mymin = np.median(image[chan][select]) - 2 * np.std(image[chan][select]) mymax = np.median(image[chan][select]) + 2 * np.std(image[chan][select]) - disp.set_limits_minmax(mymin, mymax) + disp.highlight_pixels(mask[chan], linewidth=2) disp.image = image[chan] + disp.set_limits_minmax(mymin, mymax) disp.cmap = plt.cm.coolwarm # disp.axes.text(lposx, 0, f'{channel[chan]} pedestal std [ADC]', rotation=90) - plt.title(f"{channel[chan]} pedestal std [ADC]") + plt.title(f"{channel[chan]} pedestal std {charge_unit}") disp.add_colorbar() plt.subplots_adjust(top=0.92) @@ -156,8 +172,8 @@ def plot_calibration_results(ped_data, ff_data, calib_data, run=0, plot_file=Non disp.highlight_pixels(mask[chan], linewidth=2) mymin = np.median(image[chan]) - 2 * np.std(image[chan]) mymax = np.median(image[chan]) + 2 * np.std(image[chan]) - disp.set_limits_minmax(mymin, mymax) disp.image = image[chan] + disp.set_limits_minmax(mymin, mymax) disp.cmap = plt.cm.coolwarm disp.set_limits_minmax(0.7, 1.3) plt.title(f"{channel[chan]} relative signal") @@ -179,7 +195,7 @@ def plot_calibration_results(ped_data, ff_data, calib_data, run=0, plot_file=Non mymax = np.median(image[chan]) + 2 * np.std(image[chan]) disp.set_limits_minmax(mymin, mymax) disp.cmap = plt.cm.coolwarm - plt.title(f"{channel[chan]} photon-electrons") + plt.title(f"{channel[chan]} pe, {np.sum(mask[chan])} unusable pixels") # disp.axes.text(lposx, 0, f'{channel[chan]} photon-electrons', rotation=90) disp.add_colorbar() @@ -285,7 +301,7 @@ def plot_calibration_results(ped_data, ff_data, calib_data, run=0, plot_file=Non plt.tight_layout() plt.title(f"pedestal sample of {n_ped} events") plt.ylabel("pixels", fontsize=20) - plt.xlabel("pedestal", fontsize=20) + plt.xlabel(f"pedestal {charge_unit}", fontsize=20) median = np.median(median_ped[select]) rms = np.std(median_ped[select]) label = f"Median {median:3.2f}, std {rms:3.2f}" @@ -295,7 +311,7 @@ def plot_calibration_results(ped_data, ff_data, calib_data, run=0, plot_file=Non # pedestal std plt.subplot(424) plt.ylabel("pixels", fontsize=20) - plt.xlabel("pedestal std", fontsize=20) + plt.xlabel(f"pedestal std {charge_unit}", fontsize=20) median = np.median(ped_std[select]) rms = np.std(ped_std[select]) label = f"Median {median:3.2f}, std {rms:3.2f}" @@ -356,14 +372,14 @@ def plot_calibration_results(ped_data, ff_data, calib_data, run=0, plot_file=Non disp.image = gain disp.cmap = plt.cm.coolwarm - plt.title("flat-fielded gain [ADC/pe]") + plt.title(f"flat-fielded gain {gain_unit}") disp.add_colorbar() plt.subplots_adjust(top=0.92) # gain plt.subplot(428) plt.tight_layout() plt.ylabel("pixels", fontsize=20) - plt.xlabel("flat-fielded gain [ADC/pe]", fontsize=20) + plt.xlabel(f"flat-fielded gain {gain_unit}", fontsize=20) median = np.median(gain) rms = np.std(gain) label = f"Median {median:3.2f}, std {rms:3.2f}"